Python爬虫的一些总结

常用库:

如何异步爬取？

import time
from datetime import timedelta
from tornado import httpclient, gen, ioloop, queues

class AsySpider(object):

def __init__(self, urls, concurrency):
self.urls = urls
self.concurrency = concurrency
self._q = queues.Queue()
self._fetching = set()
self._fetched = set()

def handle_page(self, url, html):
print(html)

@gen.coroutine
def get_page(self, url):
try:
response = yield httpclient.AsyncHTTPClient().fetch(url)
print('######fetched %s' % url)
except Exception as e:
print('Exception: %s %s' % (e, url))
raise gen.Return('')
raise gen.Return(response.body)

@gen.coroutine
def _run(self):

@gen.coroutine
def fetch_url():
current_url = yield self._q.get()
try:
if current_url in self._fetching:
return

print('fetching****** %s' % current_url)
self._fetching.add(current_url)
html = yield self.get_page(current_url)
self._fetched.add(current_url)

self.handle_page(current_url, html)

for i in range(self.concurrency):
if self.urls:
yield self._q.put(self.urls.pop())

finally:
self._q.task_done()

@gen.coroutine
def worker():
while True:
yield fetch_url()

self._q.put(self.urls.pop())

# Start workers, then wait for the work queue to be empty.
for _ in range(self.concurrency):
worker()
yield self._q.join(timeout=timedelta(seconds=300))
assert self._fetching == self._fetched

def run(self):
io_loop = ioloop.IOLoop.current()
io_loop.run_sync(self._run)

def main():
urls = ['http://www.baidu.com'] * 100
s = AsySpider(urls, 10)
s.run()

if __name__ == '__main__':
main()

如何模拟成浏览器?

如何提交表单？模拟登录

如何模拟成搜索引擎？

如何抓取手机app的内容？

碰到验证码怎么办?

遇到动态生成的内容怎么办？

Ref