selenium + phantomjs+python 外网动态爬虫

1.工具：

pycharm

selenium

phantomjs

python

2.代码：

# -*- coding: utf-8 -*-
from selenium import webdriver

driver = webdriver.PhantomJS(executable_path='/Users/test/Downloads/phantomjs-2.1.1-macosx/bin/phantomjs')
#定义URL数组，方便后续循环访问
loveurl_real = []

#定义保存爬取内容的文件
image_file = open('/Users/test/Documents/imageurls.txt','w+')
love_file = open('/Users/test/Documents/lovecounts.txt','w+')

def init():
driver.get("https://www.instagram.com/explore/tags/selfie/")

def crawler():
imglist = driver.find_elements_by_class_name("_icyx7")
for imgurl in imglist:
image_file.write(imgurl.get_attribute("src")+'\n')

lovelist = driver.find_elements_by_css_selector("._8mlbc._vbtk2._t5r8b")

for loveurl in lovelist:
loveurl_real.append(loveurl.get_attribute("href"))

#依次循环访问页面
def skip():
for url in loveurl_real:
driver.get(url)
if isElementExist(driver,"_tf9x3"):
love = driver.find_element_by_class_name("_tf9x3").text
love_file.write(filter(str.isdigit, love.encode("utf-8")) + '\n')
elif isElementExist(driver,"_9jphp"):
love = driver.find_element_by_class_name(("_9jphp")).text
love_file.write(filter(str.isdigit, love.encode("utf-8"))+'\n')
else:
love_file.write("0" + '\n')

#定义一个函数来判断页面是否有某个元素！！！
def isElementExist(driver, element):
flag = True
driver = driver
try:
driver.find_element_by_class_name(element)
return flag
except:
flag = False
return flag

#记得关闭文件和窗口
def drop():
driver.quit()
image_file.close()
love_file.close()

#刚进入页面时点击“更多”按钮，加载了更多的内容
def click_more():
more_button = driver.find_element_by_css_selector("._8imhp._glz1g")
more_button.click()
js1 = 'return document.body.scrollHeight'
js2 = 'window.scrollTo(0, document.body.scrollHeight)'
old_scroll_height = 0
while (driver.execute_script(js1) > old_scroll_height):
old_scroll_height = driver.execute_script(js1)
driver.execute_script(js2)
time.sleep(2)

if __name__ == '__main__':
init()
click_more()
crawler()
skip()
drop()

3.总结

一定要先看看跳转的页面，如果是‘< a >’标签的话可以不用click，直接获取它的href属性值再driver.get(url)即可。

判断页面是否有某个元素时，如果直接判断会报错然后停止运行，所以应该是写一个函数，try-catch来捕捉异常，然后根据返回的值进行判断。

实现了模拟浏览器点击，和鼠标滑动的效果。

4.爬取的网页及内容

https://www.instagram.com/explore/tags/selfie/ 上面的图片以及点赞数～