在C#中提取网页的超链接的方法
2015-10-28 16:47
381 查看
using System;
using System.Xml;
using System.Text;
using System.Net;
using System.IO;
using System.Collections;
using System.Text.RegularExpressions;
public class App
{
public static void Main()
{
string strCode;
ArrayList alLinks;
Console.Write("请输入一个网页地址:");
string strURL = Console.ReadLine();
if (strURL.Substring(0, 7) != @"http://")
{
strURL = @"http://" + strURL;
}
Console.WriteLine("正在获取页面代码,请稍侯...");
strCode = GetPageSource(strURL);
Console.WriteLine("正在提取超链接,请稍侯...");
alLinks = GetHyperLinks(strCode);
Console.WriteLine("正在写入文件,请稍侯...");
WriteToXml(strURL, alLinks);
}
// 获取指定网页的HTML代码
static string GetPageSource(string URL)
{
Uri uri = new Uri(URL);
HttpWebRequest hwReq = (HttpWebRequest)WebRequest.Create(uri);
HttpWebResponse hwRes = (HttpWebResponse)hwReq.GetResponse();
hwReq.Method = "Get";
hwReq.KeepAlive = false;
StreamReader reader = new StreamReader(hwRes.GetResponseStream(), System.Text.Encoding.GetEncoding("GB2312"));
return reader.ReadToEnd();
}
// 提取HTML代码中的网址
static ArrayList GetHyperLinks(string htmlCode)
{
ArrayList al = new ArrayList();
string strRegex = @"+/.)+[/w-]+(/[/w]http://([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?";
Regex r = new Regex(strRegex, RegexOptions.IgnoreCase);
MatchCollection m = r.Matches(htmlCode);
for (int i = 0; i <= m.Count - 1; i++)
{
bool rep = false;
string strNew = m[i].ToString();
// 过滤重复的URL
foreach (string str in al)
{
if (strNew == str)
{
rep = true;
break;
}
}
if (!rep) al.Add(strNew);
}
al.Sort();
return al;
}
// 把网址写入xml文件
static void WriteToXml(string strURL, ArrayList alHyperLinks)
{
XmlTextWriter writer = new XmlTextWriter("HyperLinks.xml", Encoding.UTF8);
writer.Formatting = Formatting.Indented;
writer.WriteStartDocument(false);
writer.WriteDocType("HyperLinks", null, "urls.dtd", null);
writer.WriteComment("提取自" + strURL + "的超链接");
writer.WriteStartElement("HyperLinks");
writer.WriteStartElement("HyperLinks", null);
writer.WriteAttributeString("DateTime", DateTime.Now.ToString());
foreach (string str in alHyperLinks)
{
string title = GetDomain(str);
string body = str;
writer.WriteElementString(title, null, body);
}
writer.WriteEndElement();
writer.WriteEndElement();
writer.Flush();
writer.Close();
}
// 获取网址的域名后缀
static string GetDomain(string strURL)
{
string retVal;
string strRegex = @"(\.com/|\.net/|\.cn/|\.org/|\.gov/)";
Regex r = new Regex(strRegex, RegexOptions.IgnoreCase);
Match m = r.Match(strURL);
retVal = m.ToString();
strRegex = @"\.|/$";
retVal = Regex.Replace(retVal, strRegex, "").ToString();
if (retVal == "")
retVal = "other";
return retVal;
}
}
using System.Xml;
using System.Text;
using System.Net;
using System.IO;
using System.Collections;
using System.Text.RegularExpressions;
public class App
{
public static void Main()
{
string strCode;
ArrayList alLinks;
Console.Write("请输入一个网页地址:");
string strURL = Console.ReadLine();
if (strURL.Substring(0, 7) != @"http://")
{
strURL = @"http://" + strURL;
}
Console.WriteLine("正在获取页面代码,请稍侯...");
strCode = GetPageSource(strURL);
Console.WriteLine("正在提取超链接,请稍侯...");
alLinks = GetHyperLinks(strCode);
Console.WriteLine("正在写入文件,请稍侯...");
WriteToXml(strURL, alLinks);
}
// 获取指定网页的HTML代码
static string GetPageSource(string URL)
{
Uri uri = new Uri(URL);
HttpWebRequest hwReq = (HttpWebRequest)WebRequest.Create(uri);
HttpWebResponse hwRes = (HttpWebResponse)hwReq.GetResponse();
hwReq.Method = "Get";
hwReq.KeepAlive = false;
StreamReader reader = new StreamReader(hwRes.GetResponseStream(), System.Text.Encoding.GetEncoding("GB2312"));
return reader.ReadToEnd();
}
// 提取HTML代码中的网址
static ArrayList GetHyperLinks(string htmlCode)
{
ArrayList al = new ArrayList();
string strRegex = @"+/.)+[/w-]+(/[/w]http://([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?";
Regex r = new Regex(strRegex, RegexOptions.IgnoreCase);
MatchCollection m = r.Matches(htmlCode);
for (int i = 0; i <= m.Count - 1; i++)
{
bool rep = false;
string strNew = m[i].ToString();
// 过滤重复的URL
foreach (string str in al)
{
if (strNew == str)
{
rep = true;
break;
}
}
if (!rep) al.Add(strNew);
}
al.Sort();
return al;
}
// 把网址写入xml文件
static void WriteToXml(string strURL, ArrayList alHyperLinks)
{
XmlTextWriter writer = new XmlTextWriter("HyperLinks.xml", Encoding.UTF8);
writer.Formatting = Formatting.Indented;
writer.WriteStartDocument(false);
writer.WriteDocType("HyperLinks", null, "urls.dtd", null);
writer.WriteComment("提取自" + strURL + "的超链接");
writer.WriteStartElement("HyperLinks");
writer.WriteStartElement("HyperLinks", null);
writer.WriteAttributeString("DateTime", DateTime.Now.ToString());
foreach (string str in alHyperLinks)
{
string title = GetDomain(str);
string body = str;
writer.WriteElementString(title, null, body);
}
writer.WriteEndElement();
writer.WriteEndElement();
writer.Flush();
writer.Close();
}
// 获取网址的域名后缀
static string GetDomain(string strURL)
{
string retVal;
string strRegex = @"(\.com/|\.net/|\.cn/|\.org/|\.gov/)";
Regex r = new Regex(strRegex, RegexOptions.IgnoreCase);
Match m = r.Match(strURL);
retVal = m.ToString();
strRegex = @"\.|/$";
retVal = Regex.Replace(retVal, strRegex, "").ToString();
if (retVal == "")
retVal = "other";
return retVal;
}
}
相关文章推荐
- 未能创建Microsoft Visual C# 2008 编译器(解决方案)
- C#语言基础
- C#委托实现工厂规则注入
- C#3.0新特性之扩展方法介绍
- C#开源资源大汇总
- C#窗体——四则运算
- 使用VS2010 C#开发ActiveX控件
- 关于Foreach是如何实现的问题
- C#中关闭第一个Form窗口,打开另一个新窗口方法
- c#网页方式获取新浪微博的微博数据
- C#.NET 封装自定义组件(控件)Dll
- C# 摄像头
- C#Windows服务程序安装常见问题解决方法
- c# int Int32 Int64 的区别
- C#虚函数
- C#水晶报表教程
- c#基础数据操作之遍历DataTable并输出
- C# 异常处理(Catch Throw)IL分析
- C# 异常处理(Catch Throw)IL分析
- WebBrowser脚本错误的完美解决方案