第一次用python 写的简单爬虫 记录在自己的博客

#python.py
from bs4 import BeautifulSoup
import urllib.request
from MySqlite import MySqlite

global g_intid
g_intid=0
def GetBlogTileAndName(url):
res = urllib.request.urlopen(url)
html = res.read()
res.close()
str(html, 'utf-8')
soup=BeautifulSoup(html)
divs=soup.find_all(attrs={"class":"postTitle"})

for divname in divs:
print("title:=",divname.a.string,"href:=",divname.a["href"])
global  g_intid
g_intid+=1
x=MySqlite()
x.InsertDate(g_intid,divname.a["href"],divname.a.string)
def GetBlogPage(url):
res = urllib.request.urlopen(url)
html = res.read()
res.close()
str(html, 'utf-8')
soup=BeautifulSoup(html)
divPager=soup.find(attrs={"class":"pager"})
print(divPager.string)

for i in range(1,8) :
url=r"http://www.cnblogs.com/FCoding/default.html?page="+str(i)
GetBlogTileAndName(url)

　　

#MySqlite.py

class MySqlite(object):
"""description of class"""
def __init__(self, *args):
return super().__init__(*args)
def callstr(self,str):
print(str)

def InsertDate(self,id,url,title):
conn = sqlite3.connect(r"d:\123.db")
c=conn.cursor()
#try:
#    c.execute('create table blog (ID intergeer,url text,title text , PRIMARY KEY(ID))')
#except ValueError:
#    print("error My")
strExe="insert into blog values ({0}, \"{1}\",\"{2}\")".format(id,url,title)
print(id)
#c.execute('insert into blog values (last_insert_rowid(),url,title)')
c.execute(strExe)
conn.commit()
c.close()
conn.close()

def GetDate(self):
import sqlite3
conn = sqlite3.connect(r"d:\123.db")
c=conn.cursor()
res=c.execute("select count(*) from blog")
res=c.fetchone()
print(res[0])
data=c.execute("select * from blog")
for item in data:
for ite in item:
print(ite)
conn.commit()
c.close()
conn.close()

　简述一下功能：

通过urllib 下载网页 使用BeautifulSoup 解析

调用find_all(attrs={"class":"postTitle"})

找到HTML 中所有class=posttitle 的tag

然后遍历 取出title 和href 保存到数据库中

此程序 无容错。新手无笑！