Python爬虫简单实战：抓取小猪短租西安市前五页民房数据

from bs4 import BeautifulSoup
import requests

urls=[]            #url列表
for page in range(1,6,1):        #前五页
page_url='http://xa.xiaozhu.com/search-duanzufang-p{}-0/'.format(page)
home_page=requests.get(page_url)
soup_home=BeautifulSoup(home_page.text,'lxml')
detail_urls=soup_home.select('a.resule_img_a')                 #取得每个民房的详情页url
for detail_url in detail_urls:
detail_url=detail_url.get('href')
urls.append(detail_url)

for url in urls:
web=requests.get(url)
soup=BeautifulSoup(web.text,'lxml')
#select方法获得的是列表，列表中的第一个元素(唯一一个元素)即是要找的信息 用索引取出后在使用方法
#因为 beautifulsoup 的方法并不能列表类型的元素使用
titles=soup.select('div.pho_info > h4 > em')[0].text    #the difference between text() and get_text()
adds=soup.select('div.pho_info > p')[0].get('title')
prices=soup.select('div.day_l > span')[0].text
pics=soup.select('div.pho_show_big > div:nth-of-type(2) > img')[0].get('src')
member_pics=soup.select('div.member_pic > a > img')[0].get('src')
member_names=soup.select('div.w_240 > h6 > a')[0].text
gender_ico=soup.select('div.member_pic > div')[0].get('class')[0]
if gender_ico=='member_ico1':              #由图标名称来判断房东性别
gender='female'
else:gender='male'

   data={
'title':titles,
'add':adds,
'prices':prices,
'pic':pics,
'member_pic':member_pics,
'member_name':member_names,
}
print(data)