C#实现web信息自动抓取

背景
随着Internet的普及，网络信息正以极高的速度增长，在这么多数据中找到自己需要的信息是一件很繁琐的事情，找到需要的信息后如何获取也是件麻烦的事。这就需要Internet信息抓取程序来代替人工的操作。
所谓Internet信息抓取程序，就是程序会按照用户的关键词或关键网站来收集相应的信息，并提供给用户想要的信息格式。
信息量的增加会带来信息网站发布人员工作量的剧增，为实现信息发布系统实现信息自
动发布、减少工作人员工作量、即时跟踪最新信息，就需要自动信息提供程序，因此Internet信息抓取程序应运而生。

目标
实现自定义网站信息分类抓取，存入本地数据库、生成静态页面或其它用户定义的信息结构，并下载与信息相关的多媒体文件。


开发




l 目标站点结构分析




本步骤是准确抓取信息个关键。




首先要选择更新频率高的页面做为抓取地址，然后分析要抓取内容页面url特点。




然后分析要抓取信息页面的元素特性，比如标题位置，内容位置 等，得到定位标记点。




将以上信息写成自己的配置文件或存到数据库中。




每个网站都需要分析，写出单独的配置文件，供抓取程序使用。




l 信息提取




根据配置文件取得要抓取页面url，使用HttpWebRequest类获取内容：




//获取http页面函数




public string Get_Http(string a_strUrl,int timeout)








{




string strResult ;








try








{




HttpWebRequest myReq = (HttpWebRequest)HttpWebRequest.Create(a_strUrl) ;




myReq.Timeout = timeout;




HttpWebResponse HttpWResp = (HttpWebResponse)myReq.GetResponse();








Stream myStream = HttpWResp.GetResponseStream () ;








StreamReader sr = new StreamReader(myStream , Encoding.Default);




StringBuilder strBuilder = new StringBuilder();




while (-1 != sr.Peek())








{




strBuilder.Append(sr.ReadLine()+"/r/n");




}








strResult = strBuilder.ToString();




}




catch(Exception exp)








{




strResult = "错误：" + exp.Message ;




}








return strResult ;








}




获取页面内容后，分析页面中连接地址取到要抓取的url：




//处理页面标题和链接




public string SniffWebUrl( string urlStr,string blockB,string blockE )








{




string urlch1 = "";




string urlch2 = "";




int end_n1 = 0;




int end_nums = 0;




int end_nums1 = 0;




int end_nums2 = 0;




int end_nums3 = 0;




string reUTStr = "";




string reTitle = "";




string ret = "";




try








{




int pos01 = urlStr.IndexOf( "." );




int pos02 = urlStr.LastIndexOf( "/" );




if( pos01 < 0 )








{




return "";




}




if( pos02 < 0 )








{




return "";




}




int pos03 = urlStr.IndexOf( "/",pos01 );




if ( pos03 < 0 )








{




urlch1 = urlStr;




urlch2 = urlStr;




}




else








{




urlch1 = urlStr.Substring( 0,pos03 );




urlch2 = urlStr.Substring( 0,pos02 );




}








string tmpAllStr = new PublicFun().Get_Http( urlStr ,time1);








int pos1 = tmpAllStr.IndexOf( blockB );




int pos2 = tmpAllStr.IndexOf( blockE,pos1 + blockB.Length );




if ( pos1>0 && pos2>0 && pos2>pos1 )








{




ret = tmpAllStr.Substring( pos1 + blockB.Length,pos2 - pos1 - blockB.Length );




ret = ret.Substring( ret.IndexOf( "<" ));




while( ret.IndexOf( "<A" ) >= 0 )








{




ret = ret.Substring( 0,ret.IndexOf( "<A" ) ) + "<a" + ret.Substring( ret.IndexOf( "<A" ) + 2 );




}




while( ret.IndexOf( "</A" ) >=0 )








{




ret = ret.Substring( 0,ret.IndexOf( "</A" ) ) + "</a" + ret.Substring( ret.IndexOf( "</A" ) + 3 );




}




while( ret.IndexOf( "Href=" ) >=0 )








{




ret = ret.Substring( 0,ret.IndexOf( "Href=" )) + "href=" + ret.Substring( ret.IndexOf( "Href=" ) + 5 );




}




while( ret.IndexOf( "HREF=" ) >=0 )








{




ret = ret.Substring( 0,ret.IndexOf( "HREF=" )) + "href=" + ret.Substring( ret.IndexOf( "HREF=" ) + 5 );




}




while( ret.IndexOf( "href= " ) >=0 )








{




ret = ret.Substring( 0,ret.IndexOf( "href= " )) + "href=/"" + ret.Substring( ret.IndexOf( "href= " ) + 6 );




}




}




tmpAllStr = ret;




int begin_nums = tmpAllStr.IndexOf( "href=" );








while ( begin_nums >= 0 )








{




string tmpStrA = "";




string tmpStrB = tmpAllStr.Substring( begin_nums + 5,1 );




if ( tmpStrB == "/"" )








{




end_n1 = begin_nums + 6;




if ( ( end_n1 + 1 ) > tmpAllStr.Length )








{




return "";




}




tmpStrA = tmpAllStr.Substring( begin_nums+6,1 );




}




else








{




end_n1 = begin_nums + 5;




tmpStrA = tmpStrB;




}








if ( tmpStrA == "#" )








{




tmpAllStr = tmpAllStr.Substring( end_n1 );




begin_nums = tmpAllStr.IndexOf( "href=" );




}




else








{




end_nums1 = tmpAllStr.IndexOf( " ",end_n1 );




end_nums2 = tmpAllStr.IndexOf( ">",end_n1 );




end_nums3 = tmpAllStr.IndexOf( "</a",end_nums2 );








if ( ( end_nums3 >= 0 ) && ( end_nums2 >= 0 ) )








{




reTitle = tmpAllStr.Substring( end_nums2 + 1,end_nums3 - end_nums2 - 1 );








if ( end_nums1 > end_nums2 )








{




end_nums = end_nums2;




}




else








{




if ( end_nums1 < 0 )








{




end_nums = end_nums2;




}




else








{




end_nums = end_nums1;




}




}




string str4 = tmpAllStr.Substring( end_nums-1, end_nums - end_nums + 1 );








if ( str4 =="/"" || str4 == " " )








{




end_nums = end_nums - 1;




}




string sTotalOne = tmpAllStr.Substring( end_n1,end_nums - end_n1 );








if ( sTotalOne.IndexOf( "http://" ) <0 )








{




if ( sTotalOne.IndexOf( "/" ) == 0 )








{




sTotalOne = urlch1 + sTotalOne;




}




else








{




int linshiIntNum = 0;




int flags = 0;




string urlChange = urlStr;;




while( sTotalOne.IndexOf( "../" ) >= 0 )








{




sTotalOne = sTotalOne.Substring( sTotalOne.IndexOf( "../" ) + 3 );




linshiIntNum = linshiIntNum + 1;




flags = flags +1;




}




while( ( urlChange.LastIndexOf( "/" ) >= 0 ) && ( linshiIntNum >= 0 ) )








{




urlChange = urlChange.Substring( 0,urlChange.LastIndexOf( "/" ) );




linshiIntNum = linshiIntNum - 1;




}




if ( flags == 0 )








{




sTotalOne = urlch2 + "/" + sTotalOne;




}




else








{




sTotalOne = urlChange + "/" + sTotalOne;




}




}




}




reUTStr = reUTStr + new PublicFun().RemoveHtmlCode( reTitle ) + sTotalOne;








tmpAllStr = tmpAllStr.Substring( end_nums3 + 4 );




begin_nums = tmpAllStr.IndexOf( "href=" );




}




else








{




begin_nums = -1;




}




}




}




return reUTStr;




}




catch( Exception e)








{




return "";




}




}








得到要抓取内容的url后，处理该页面：




//获取链接内容并分类处理




public string GetWebContent( string gatherUrl,string subUrl,string subTitle,string b_Content,string e_Content,string b_Filter,string e_Filter,string root )








{




string tmpAllStr = "";




string dfStrB = "";




string dfStrE = "";




string rePicStr = "";//图片返回路径




string reContentStr = "";




string picHtml = "images"; //本地图片路径








string urlch1 ="";




string urlch2 ="";




int pos1 = gatherUrl.IndexOf( "." );




int pos2 = gatherUrl.LastIndexOf( "/" );




if( pos1 < 0 )








{




return "";




}




if( pos2 < 0 )








{




return "";




}




int pos3 = gatherUrl.IndexOf( "/",pos1 );




if ( pos3 < 0 )








{




urlch1 = gatherUrl;




urlch2 = gatherUrl;




}




else








{




urlch1 = gatherUrl.Substring( 0,pos3 );




urlch2 = gatherUrl.Substring( 0,pos2 );




}








tmpAllStr = new PublicFun().Get_Http( subUrl,time1 );




//取稿源




string docFromStr = "";




if ( dfStrB != "" && dfStrE != "" )








{




if ( tmpAllStr != "" )








{




int b_docF = tmpAllStr.IndexOf( dfStrB );




if ( b_docF > 0 )








{




int e_docF = tmpAllStr.IndexOf( dfStrE,b_docF + dfStrB.Length );




if ( e_docF > 0 && e_docF > b_docF && e_docF - b_docF < 20 )








{




docFromStr = tmpAllStr.Substring( b_docF + dfStrB.Length, e_docF - b_docF - dfStrB.Length );




}




}




}




}




//取内容




if ( tmpAllStr != "" )








{




int begin_strnum = tmpAllStr.IndexOf( b_Content );




if ( begin_strnum < 0 )








{




return "";




}




int end_strnum = tmpAllStr.IndexOf( e_Content,begin_strnum + b_Content.Length );




if ( end_strnum < 0 )








{




return "";




}




string sTotalSubM = "";




if ( end_strnum > begin_strnum )








{




sTotalSubM = tmpAllStr.Substring ( begin_strnum,end_strnum - begin_strnum );




}








if ( sTotalSubM == "" )








{




return "";




}




//过滤无用信息




int bfnum = sTotalSubM.IndexOf( b_Filter );




if ( bfnum > -1 )








{




int efnum = sTotalSubM.IndexOf( e_Filter,bfnum );




if ( efnum > -1 )








{




if ( efnum > bfnum )








{




sTotalSubM = sTotalSubM.Substring( 0,bfnum ) + sTotalSubM.Substring( efnum + e_Filter.Length );




}




}




}




//格式化图片标记








while( sTotalSubM.IndexOf( "Src=" ) >= 0 )








{




sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "Src=" ) ) + "src=" + sTotalSubM.Substring( sTotalSubM.IndexOf( "Src=" ) + 4 );




}




while( sTotalSubM.IndexOf( "SRC=" ) >= 0 )








{




sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "SRC=" ) ) + "src=" + sTotalSubM.Substring( sTotalSubM.IndexOf( "SRC=" ) + 4 );




}




while( sTotalSubM.IndexOf( "src= " ) >= 0 )








{




sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "src= " ) ) + "src=/"" + sTotalSubM.Substring( sTotalSubM.IndexOf( "src= " ) + 5 );




}








//取图片地址




int end_n12 = 0;




int end_nums2 = 0;




int begin_nums2 = sTotalSubM.IndexOf( "src=" );




while( begin_nums2 >= 0 )








{




String tmpStr = sTotalSubM.Substring( begin_nums2 + 4,1 );




if ( tmpStr == "/"" )








{




end_n12 = begin_nums2 + 5;




}




else








{




end_n12 = begin_nums2 + 4;




}




int end_nums2a = sTotalSubM.IndexOf( " ",end_n12 );




int end_nums2b = sTotalSubM.IndexOf( ">",end_n12 );




if ( end_nums2b < 0 )








{




break;




}




if ( end_nums2a > end_nums2b )








{




end_nums2 = end_nums2b;




}




else








{




if (end_nums2a<0)








{




end_nums2 = end_nums2b;




}




else








{




end_nums2 = end_nums2a;




}




}




tmpStr = sTotalSubM.Substring( end_nums2-1,1 );




if ( tmpStr == "/"" || tmpStr == " " )








{




end_nums2 = end_nums2 - 1;




}




string tmpPicStr = sTotalSubM.Substring( end_n12,end_nums2 - end_n12 );








if ( tmpPicStr.IndexOf( "http://" ) < 0 )








{




if ( tmpPicStr.IndexOf( "/" ) == 0 )








{




tmpPicStr = urlch1 + tmpPicStr;




}




else








{




int linshiIntNum = 0;




int flags = 0;




string urlChange = subUrl;




while( tmpPicStr.IndexOf( "../" ) >= 0 )








{




tmpPicStr = tmpPicStr.Substring( tmpPicStr.IndexOf("../") + 3 );




linshiIntNum = linshiIntNum + 1;




flags = flags + 1;




}




while( ( urlChange.LastIndexOf( "/" ) >= 0 ) && ( linshiIntNum >= 0 ) )








{




urlChange = urlChange.Substring( 0,urlChange.LastIndexOf( "/" ) );




linshiIntNum = linshiIntNum - 1;




}




if ( flags == 0 )








{




tmpPicStr = urlch2 + "/" + tmpPicStr;




}




else








{




tmpPicStr = urlChange + "/" + tmpPicStr;




}




}




}




//tmpPicStr = tmpPicStr.ToLower();




string tmpPicStrTmp = tmpPicStr.ToLower();




//if ( tmpPicStr.IndexOf( ".jpg" ) > 0 || tmpPicStr.IndexOf( ".gif" ) > 0 || tmpPicStr.IndexOf( ".bmp" ) > 0 )




if ( tmpPicStrTmp.IndexOf( ".jpg" ) > 0 || tmpPicStrTmp.IndexOf( ".gif" ) > 0 || tmpPicStrTmp.IndexOf( ".bmp" ) > 0 )








{




rePicStr = rePicStr + "||" + tmpPicStr ;








int flagN2 = tmpPicStr.LastIndexOf( "/" );




string fileN2 = picHtml + tmpPicStr.Substring( flagN2 );




sTotalSubM = sTotalSubM.Substring( 0,end_nums2 ) + ">******" + fileN2 + "******<" + sTotalSubM.Substring( end_nums2 );








begin_nums2 = sTotalSubM.IndexOf( "src=", end_nums2 + fileN2.Length + 22 );




}




else








{




begin_nums2 = sTotalSubM.IndexOf( "src=", end_nums2 + 4 );




}




}




if ( rePicStr.Length > 2 )




rePicStr = rePicStr.Substring(2);








//内容处理 格式化关键标记




while( sTotalSubM.IndexOf( "<P" ) >= 0 )








{




sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<P" ) ) + "|****|<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<P" ) + 2 );




}




while( sTotalSubM.IndexOf( "<p" ) >= 0 )








{




sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<p" ) ) + "|****|<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<p" ) + 2 );




}




while( sTotalSubM.IndexOf( "</P" ) >= 0 )








{




sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "</P" ) ) + "|****|<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "</P" ) + 3 );




}




while( sTotalSubM.IndexOf( "</p" ) >= 0 )








{




sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "</p" ) ) + "|****|<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "</p" ) + 3 );




}




while( sTotalSubM.IndexOf( "<br" ) >=0 )








{




sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<br" ) ) + "+****+<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<br" ) + 3 );




}




while( sTotalSubM.IndexOf( "<BR" ) >= 0 )








{




sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<BR" ) ) + "+****+<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<BR" ) + 3 );




}




while( sTotalSubM.IndexOf( "<Br" ) >= 0 )








{




sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<Br" ) ) + "+****+<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<Br" ) + 3 );




}




while( sTotalSubM.IndexOf( "<bR" ) >= 0 )








{




sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<bR" ) ) + "+****+<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<bR" ) + 3 );




}








//去除html标记




int linshiInt1 = sTotalSubM.IndexOf( "<" );




int linshiInt2 = sTotalSubM.IndexOf( ">" );








if ( linshiInt2 < linshiInt1 )








{




sTotalSubM = sTotalSubM.Substring( linshiInt2 + 1 );




}




int linshiInt11 = sTotalSubM.LastIndexOf( "<" );




int linshiInt12 = sTotalSubM.LastIndexOf( ">" );




if ( linshiInt12 < linshiInt11 )








{




sTotalSubM = sTotalSubM.Substring( 0,linshiInt12 + 1 );




}




linshiInt1 = sTotalSubM.IndexOf( "<" );




while ( linshiInt1 >= 0 )








{




linshiInt2 = sTotalSubM.IndexOf( ">",linshiInt1 );




if ( linshiInt2 >= 0 )








{




sTotalSubM = sTotalSubM.Substring( 0,linshiInt1 ) + sTotalSubM.Substring( linshiInt2 + 1 );




}




else








{




sTotalSubM = sTotalSubM.Substring( 0,linshiInt1 );




}




linshiInt1 = sTotalSubM.IndexOf("<");




}








//还原关键标记




int linshiInt3 = 0;




int linshiInt4 = 0;








while( sTotalSubM.IndexOf( "+****+" ) >= 0 )








{




sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "+****+" ) ) + "<br>/n" + sTotalSubM.Substring( sTotalSubM.IndexOf( "+****+" ) + 9 );




}




while( sTotalSubM.IndexOf( "|****|" ) >= 0 )








{




sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "|****|" ) ) + "<br>/n" + sTotalSubM.Substring( sTotalSubM.IndexOf( "|****|" ) + 9 );




}




while( sTotalSubM.IndexOf( "******" ) >= 0 )








{




linshiInt3 = sTotalSubM.IndexOf( "******" ) + 9;




linshiInt4 = sTotalSubM.IndexOf( "******",linshiInt3 );




if ( linshiInt4 >= 0 )








{




int tmpPos = sTotalSubM.IndexOf( "******" );




string tmpStr1 = sTotalSubM.Substring( 0,tmpPos );




string tmpStr2 = sTotalSubM.Substring( linshiInt3,linshiInt4 - linshiInt3 );




string tmpStr3 = sTotalSubM.Substring( linshiInt4 + 9 );




sTotalSubM = tmpStr1 + "<img src=" + tmpStr2 + ">" + tmpStr3;




}




else








{




break;




}




}




//去除内容中的标题




if ( sTotalSubM.IndexOf( subTitle ) >= 0 )








{




sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( subTitle ) ) + sTotalSubM.Substring( sTotalSubM.IndexOf( subTitle ) + subTitle.Length );




}




reContentStr = sTotalSubM;




//调用下载图片功能




//下载图片到指定目录




string[] img_Url = new PublicFun().split( rePicStr,"||" );




for ( int i=0;i<img_Url.Length;i++ )








{




if ( img_Url[i] != "" )








{




new PublicFun().Get_Img( img_Url[i],10000,root + "//images//" + img_Url[i].Substring( img_Url[i].LastIndexOf("/")+1 ) );




}




}




}




return reContentStr;








}




以上方法返回要取得的信息，包括标题内容，图片地址等。








下载页面中图片：




//下载图片




public void Get_Img(string a_strUrl,int timeout,string filepath)








{




try








{




HttpWebRequest myReq = (HttpWebRequest)HttpWebRequest.Create(a_strUrl) ;




myReq.Timeout = timeout;




HttpWebResponse HttpWResp = (HttpWebResponse)myReq.GetResponse();




Stream myStream = HttpWResp.GetResponseStream () ;




Bitmap map = new Bitmap( myStream );




PictureBox picB = new PictureBox();




picB.Image = (Image)map;




string path = filepath.Substring( 0,filepath.LastIndexOf( "//" ) );




if (!Directory.Exists(path))








{




CreateDir( path );




}




picB.Image.Save(filepath);




}




catch(Exception exp)








{




string ss = exp.Message;




WriteLog( filepath.Substring(0,filepath.LastIndexOf("//")) + "//error.log",a_strUrl + "--" + ss + "/r/n");




}




}




l 保存文件或入库



上面取得的信息可以按自己的要求保存。

****设计的时候没有使用url按层次循环抓取，这样定义抓取url效率更高，速度更快。
测试程序下载：http://bjfile.focus.cn/file/15379/NetBugV102.rar

如有建议请发送EMail：xiancai@qianlong.com 或msn yutao728@hotmail.com

注：此版本只提供静态文件存储功能，不提供数据库接口，不提供自定义网站功能。

本程序运行需要先安装.net 框架1.1。