【Python编程】网页URL提取实例

import urllib.request

import re

def ssubcatagory(urllink,j):

    fp = urllib.request.urlopen(urllink)

    filecontent = fp.read()

    content = str(filecontent)

    if(content.find("UTF-8",0,1000) != -1 or content.find("utf-8",0,1000) != -1):

        mystr = filecontent.decode('UTF-8')

    elif(content.find("gbk",0,1000) != -1 or content.find("GBK",0,1000) != -1):

        mystr = filecontent.decode('GBK')

    else:

        mystr = filecontent.decode('GB2312')

           

    if(j <= 10) :

        ssubcata = re.findall(r'<div class="\kag sclearfix\">(.*?)</div></div></div>',mystr)

        for i in ssubcata:

            #print(i)

            sssubcata = re.findall(r'blank\">(.*?)</a>',i)

            print("三级：",sssubcata[0])

            #urladdress =re.findall(r'href=\"http\://(.*?)\" class=\"kaj\">(.*?)</a>',i)

            urladdress =re.findall(r'href=\"http\://(.*?)\" class=\"kaj\">(.*?)</a>',i)

            for k in urladdress:

                print(k[0],k[1])

               

                               

           

        #print(ssubcata[0])

   

    #print(i,subcata)

 

def subcatagory(mystr,j):

    k = 1

    if(j == 1) :

        subcata = re.findall(r'生活服务</b></h3><ul class=\"list clearfix\">(.*?)</ul></div>',mystr)

        #href="http://gouwu.hao123.com/" class="link"><b>购物</b>

        print("一级：生活服务--")

        ssubcata = re.findall(r'href=\"(.*?)\" class=\"link\"><b>(.*?)</b>',str(subcata))

        for i in ssubcata:

            print("二级：",i[0],i[1])

            #gouwu caipiao and so on

            ssubcatagory(i[0],k)

            k = k + 1

        #print(ssubcata[0])

   

    #print(i,subcata)

def mainfun():

    url = 'http://www.hao123.com/sitemap'

    fp = urllib.request.urlopen(url)

    content = fp.read()

    mystr = content.decode('GBK')

    #print(mystr)

    file = open('results.txt','w')

    file.write(mystr)

    file.close

    #<div class="section" id="生活服务">

    catapattern = re.findall(r'<div class=\"section\" id=\"(.*?)\">',mystr)

    #print(catapattern[0])

    j = 0

    for i in catapattern:

        j = j + 1

        subcatagory(mystr,j)

       

    #print(catapattern)

    fp.close()

mainfun()