python爬虫的初级入门

1.如何获取一个网站的所有路径？

#coding:utf-8
import urllib2
import re
import itertools
import threading
import urlparse
import robotparser
#download是用于下载网页的，并附加服务器名，添加重新访问网站的次数
def download(url,user_agent='wswp',num_retries=2):
print 'DownLoading:',url
headers = {'User-agent':user_agent}
request = urllib2.Request(url,headers=headers)
try:
html=urllib2.urlopen(request).read()
except urllib2.URLError as e:
print 'DownLoad error',e.reason
html=None
if num_retries>0:
if hasattr(e,'code') and 500<=e.code<600:
return download(url,user_agent,num_retries=-1)
return html
#crawl_sitemap是用于进行网站地图的爬取，将所有<a href="(site)">中site爬取出来
def crawl_sitemap(url):
sitemap = download(url)
links = re.findall('<a href="(\S*)">',sitemap)
for link in links:
html = download(link)
#crawl_sitemap('http://sitemap.163.com/')

#link_crawler方法是用于将种子地址的所有连接都爬取下来
def link_crawler(seed_url,link_regex):
crawl_queue = [seed_url]
seen=set(crawl_queue)
while crawl_queue:
url=crawl_queue.pop()
html=download(url)
for link in get_links(html):
if link in get_links(html):
if re.match(link_regex,link):
link=urlparse.urljoin(seed_url,link)
if link not in seen:
seen.add(link)
crawl_queue.append(link)

def get_links(html):
webpage_regex=re.compile('\S*')
return webpage_regex.findall(html)

在最后运行 link_crawler('http://map.163.com'')就可以完成爬取所有网址的操作了！

二。如何进行网址抓取？

1可以使用正则表达式

import re
import urllib2
def download(url,user_agent='wswp',num_retries=2):
print 'DownLoading:',url
headers = {'User-agent':user_agent}
request = urllib2.Request(url,headers=headers)
try:
html=urllib2.urlopen(request).read()
except urllib2.URLError as e:
print 'DownLoad error',e.reason
html=None
if num_retries>0:
if hasattr(e,'code') and 500<=e.code<600:
return download(url,user_agent,num_retries=-1)
return html
url='http://sitemap.163.com/'
html=download(url)
print re.findall('<a href="(\S*)">',html)

2.可以使用美味汤beautifulsoup

from bs4 import BeautifulSoup
f=open("index.html")
file_object = open('index.html')
try:
f = file_object.read( )
finally:
file_object.close( )
soup = BeautifulSoup(f)
for link in soup.find_all('a'):
print(link.get('href'))

3.可以使用lxml进行提取