利用python脚本抓取AC的代码[爬虫+HTMLParser+handle_entityref+正则表达式+模拟登陆+文件操作]

#-*- coding=utf-8 -*-
import time,urllib2,urllib,re,HTMLParser,os
from htmlentitydefs import entitydefs

class PageParser(HTMLParser.HTMLParser):#翻译实体
def __init__(self):
self.data=""
self.readcode=0
HTMLParser.HTMLParser.__init__(self)
def handle_starttag(self,tag,attrs):
if tag=='textarea':
self.readcode=1
def handle_data(self,data):
if self.readcode:
self.data+=data
def handle_endtag(self,tag):
if tag=='textarea':
self.readcode=0
def handle_entityref(self,name):
if entitydefs.has_key(name):
self.handle_data(entitydefs[name])
else:
self.handle_data('&'+name+';')
def getdata(self):
return self.data

global res
def getACUrl():
step=1
r=re.compile(unicode("下一页","utf8"))
#.{500}?\"
r2=re.compile("<input type=\"hidden\" name=\"__VIEWSTATE\" id=\"__VIEWSTATE\" value=\".{1,50000}?\" />")
url = "http://algorithm.fzu.edu.cn/OnlineJudgeUserStatus.aspx"
parms = {
'__EVENTTARGET':'ctl00$MainRightHolder$UserStatusGridView',
'ctl00$MainRightHolder$UserIdTextBox':'*****',
}
login1=urllib2.urlopen(url)
pagedata=login1.read()
s1=r2.findall(pagedata)
if len(s1)==0:
return
parms['__VIEWSTATE']=s1[0][64:-4]#form表单的提取
while True:
try:
if step==1:
parms['__EVENTARGUMENT']='Page$First'
else:
parms['__EVENTARGUMENT']='Page$Next'
step+=1
login = urllib2.urlopen(url,urllib.urlencode(parms))
data=(unicode(login.read(),"utf8"))
#            fout=open("c:\\1.html","wb+")
#            fout.write(data.encode("GBK"))
#            fout.close()
findurl(data)
if len(r.findall(data))==0:
break
s1=r2.findall(data)
if len(s1)==0:
return
parms['__VIEWSTATE']=s1[0][64:-4]
data=""
except Exception,e:
print(e)
break
def Login(username,password):#模拟登陆
try:
cookies = urllib2.HTTPCookieProcessor()
opener = urllib2.build_opener(cookies)
urllib2.install_opener(opener)
parms = {
'__VIEWSTATE':r'/wEPDwULLTE2ODk5MTAyOTUPZBYCAgMPZBYCAgUPEA8WAh4LXyFEYXRhQm91bmRnZBAVBgzmnIDmlrDkv6Hmga8kMTLmnIg15pel566X5rOV6K++5YGc5LiK5LiA5qyh77yM6K++EuS8mOengOS9nOS4muWAmemAiSgg5YWz5LqO6aKY55uu55qE5pe26Ze056m66Ze06ZmQ5Yi255qE6ZeuJOWFs+S6jueZu+mZhuezu+e7n+eUqOaIt+WQjeS4juWvhueggQg+PuabtOWkmhUGABdTaG93QnVsbGV0aW4uYXNweD9iaWQ9NRdTaG93QnVsbGV0aW4uYXNweD9iaWQ9NBdTaG93QnVsbGV0aW4uYXNweD9iaWQ9MxdTaG93QnVsbGV0aW4uYXNweD9iaWQ9MhFCdWxsZXRpbkxpc3QuYXNweBQrAwZnZ2dnZ2cWAGQYAgUeX19Db250cm9sc1JlcXVpcmVQb3N0QmFja0tleV9fFgEFEkJhbm5lciRMb2dpbkJ1dHRvbgUgQmFubmVyJFVzZXJDb250cm9sUGFuZWxNdWx0aVZpZXcPD2RmZOAvQzwaH/EzyqdrNO7IO2UefuMIHdnWhg02m4yXus4K',
'Banner$LoginButton.x':'17',
'Banner$LoginButton.y':'5'
}
parms[r"Banner$UserNameText"]=username
parms[r"Banner$Password"]=password

loginUrl = "http://algorithm.fzu.edu.cn/Default.aspx"
login = urllib2.urlopen(loginUrl,urllib.urlencode(parms))
h=(unicode(login.read(),"utf8"))
# loginer = urllib2.urlopen("http://poj.org/")#登录主页
#  print(loginer.read().decode("utf8"))
except Exception,e:
print(e)
def findurl(data):
r=re.compile("<a class=\"underline\" href=\".{1,500}?\" target=\"_blank\">.{1,500}?</a></td><td><a class=\"hover-underline\" href=\".{1,500}?\" target=\"_blank\">AC\|AC\|AC\|AC\|AC\|AC\|AC\|AC\|AC\|AC\|</a>")
h=r.findall(data)
import string
global res
for s in h:
y=string.find(s,"cpp")
if y==-1:
continue
x=y
while(s[x]!='>'):
x-=1
name=s[x+1:y+3]
if name in res.keys():
continue
x=string.find(s,"href=")
if x==-1:
continue
y=string.find(s,"\" target=")
if y!=-1:
ss=s[x+6:y]
x=string.find(ss,"&")
if x!=-1:
ss="http://algorithm.fzu.edu.cn/"+ss[:x+1]+ss[x+5:]
res[name]=ss
def getACUrl2():
r=re.compile("<a target='_blank' href='.{1,500}?'>.{1,500}?</a></td><td>AC\|AC\|AC\|AC\|AC\|AC\|AC\|AC\|AC\|AC\|</td>")
url = "http://algorithm.fzu.edu.cn/StuMain.aspx"
try:
login = urllib2.urlopen(url)
data=(unicode(login.read(),"utf8"))
h=r.findall(data)
import string
global res
for s in h:
x=string.find(s,"cpp")
if x==-1:
continue
y=x
while x>0 and s[x]!='>':
x-=1
if x<0:
continue
name=s[x+1:y+3]
if name in res.keys():
continue
t=y
y=x
x=t
while not( x>4 and s[x]=='=' and s[x-1]=='f' and s[x-2]=='e' and s[x-3]=='r' and s[x-4]=='h'):
x-=1
if x==-1:
continue
while y>=0 and s[y]!='\'':
y-=1
if y<0:
continue
res[name]="http://algorithm.fzu.edu.cn/"+s[x+2:y]
except Exception,e:
print(e)
if __name__ == '__main__':
username=raw_input("请输入用户名:")
password=raw_input("请输入密码:")
global res
res={}
Login(username,password)#模拟登陆
getACUrl2()#获取管理后台中的AC代码
getACUrl()#获取AC代码对应的url
if not os.path.exists("ACCode"):
os.mkdir("ACCode")
for key in res:
print key
u=urllib2.urlopen(res[key])#根据获取的url，逐个访问，并将代码保存到本地
pp=PageParser()
pp.feed(u.read())
data=pp.getdata()
fout=open('ACCode'+'\\'+key,"wb+")
fout.write(data)
fout.close()
print "共%d份AC代码" % len(res)