C#获取网页数据的两个函数

string getPageText(string url)
{
string retVal = "";
dtUpdate = DateTime.Now;
labMessage.Text = "";
int pageNo = 1;
string strUrl = "";
do
{
try
{
using (var wc = new WebClient())
{
wc.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");
wc.Encoding = Encoding.UTF8;
strUrl = url + "&page=" + pageNo;
log("打开页：" + strUrl + "<br/>");
string str = wc.DownloadString(strUrl);
str = str.Replace("<script", "");
pageNo = procLine(str);                  //处理获取到的文本
}
}
catch (Exception ee)
{
log("获取网页时错误：" + ee.Message);
}
} while (pageNo > 0);
using (SqlConnection conn = new SqlConnection(strConn))
{
try
{
conn.Open();
string strSql = "delete from 线路 where 更新时间<>@更新时间";
SqlCommand cmd = new SqlCommand(strSql, conn);
cmd.Parameters.Add("@更新时间", System.Data.SqlDbType.DateTime, 4).Value = dtUpdate;
int sl = cmd.ExecuteNonQuery();
if (sl > 0) log("删除了未更新的线路：" + sl + "条。");
}
catch (Exception ee)
{
log("错误：" + ee.Message);
}
}
return retVal;
}

int procLine(string strHtml)
{
int retVal = 0;
HtmlDocument htmlDoc = new HtmlDocument();      // 获取html元素（htmlContext为html页面字符串）
htmlDoc.LoadHtml(strHtml);                      // 加载html页面

HtmlNode node = htmlDoc.DocumentNode;
//HtmlNodeCollection has = node.SelectNodes("//div[contains(@onclick, '/RealtimeQuery?')]");  //获取各条线路的 div
HtmlNodeCollection has = node.SelectNodes("//a");  //获取各条线路的 <a href="/LineDetailQuery?lineId=1&direction=2&">1路(老福山花园站→博览城市场中路站)</a>
if (has != null)
foreach (HtmlNode hn in has)
if (hn.Attributes.Contains("href"))
{   //<a href="/LineDetailQuery?lineId=1&direction=2&">1路(老福山花园站→博览城市场中路站)</a>
string href = hn.Attributes["href"].Value;
if (href.StartsWith("/LineDetailQuery?lineId="))
{
string[] ss = href.Split(new char[] { '?', '=', '&', ';' }, StringSplitOptions.RemoveEmptyEntries);
if (ss.Length > 6)
{
int lineId = 0;
int dir = 0;
int.TryParse(ss[2], out lineId);
int.TryParse(ss[5], out dir);
string[] lineInfo = hn.InnerText.Split(new char[] { '(', '→', ')' }, StringSplitOptions.RemoveEmptyEntries);
string lineName = lineInfo[0];
string from = lineInfo[1];
string to = lineInfo[2];
int bh = lineId + dir * 100000;
string str = "lineId={0}, dir={1}, lineName={2}, from={3}, to={4} <br/>";
log(string.Format(str, lineId, dir, lineName, from, to));
//将线路更新到数据库
using (SqlConnection conn = new SqlConnection(strConn))
{
try
{
conn.Open();
string strSql = "update 线路 set 线路号=@线路号, 行向=@行向, 线路名称=@线路名称, 始发站点=@始发站点, 开往方向=@开往方向, 更新时间=@更新时间 where 编号=@编号";
SqlCommand cmd = new SqlCommand(strSql, conn);
cmd.Parameters.Add("@线路号", System.Data.SqlDbType.Int, 4).Value = lineId;
cmd.Parameters.Add("@行向", System.Data.SqlDbType.Int, 4).Value = dir;
cmd.Parameters.Add("@线路名称", System.Data.SqlDbType.NVarChar, 50).Value = lineName;
cmd.Parameters.Add("@始发站点", System.Data.SqlDbType.NVarChar, 50).Value = from;
cmd.Parameters.Add("@开往方向", System.Data.SqlDbType.NVarChar, 50).Value = to;
cmd.Parameters.Add("@更新时间", System.Data.SqlDbType.DateTime, 4).Value = dtUpdate;
cmd.Parameters.Add("@编号", System.Data.SqlDbType.Int, 4).Value = bh;
if (cmd.ExecuteNonQuery() == 0)
{
cmd.CommandText = "insert into 线路(编号, 线路号, 行向, 线路名称, 始发站点, 开往方向, 更新时间)values(@编号, @线路号,@行向,@线路名称,@始发站点,@开往方向,@更新时间)";
cmd.ExecuteNonQuery();
}
}
catch (Exception ee)
{
log("更新线路到数据库时错误：" + ee.Message);
}
}
}
}
}
//<font style="font-size:13px;">[2/26]</font>
HtmlNodeCollection hasPage = node.SelectNodes("//font");  //获取各条线路的 <a href="/LineDetailQuery?lineId=1&direction=2&">1路(老福山花园站→博览城市场中路站)</a>
if (hasPage != null)
foreach (HtmlNode hn in hasPage)
if (hn.Attributes.Contains("style"))
if (hn.Attributes["style"].Value == "font-size:13px;")
{
string[] ss = hn.InnerText.Split(new char[] { '[', '/', ']' }, StringSplitOptions.RemoveEmptyEntries);
if (ss.Length > 1)
{
int pageNo = 0;
int pageTotal = 0;
int.TryParse(ss[0], out pageNo);
int.TryParse(ss[1], out pageTotal);
if (pageTotal > 0)
if (pageNo < pageTotal)
{
log("=================第" + pageNo + "页/共" + pageTotal + "页===========<br/><br/>");
retVal = pageNo + 1;
}
else
log("==================最后一页===============");
}
}
return retVal;
}