获取搜狗引擎 添加任意关键词 后的html源码（c#）

class Program
{
private static ManageKeywordBll mkBll = new ManageKeywordBll();
private static WeixinquanMessageBll wqBll = new WeixinquanMessageBll();
private static CookieHelp ch = new CookieHelp();
private static string userAgentLiShi = "";
private static string result = "";//返回html结果
private static string[] kw;//关键词
private static string id = "";//获取 验证码返回的 id用作SUNID的cookie值

static void Main(string[] args)
{
#region 开始
ManageKeyword model = new ManageKeyword();
List<ManageKeyword> list = new List<ManageKeyword>();
string cookieAll = "";//防止cookie信息丢失，URL链接失效问题
model.Module = "6,";
list = mkBll.GetList(model);
foreach (ManageKeyword data in list)
{
List<WeixinquanMessage> listWeixin = new List<WeixinquanMessage>();
string url = "";
string keyword = data.KeyWord;
if (keyword.IndexOf("*") != -1)
{
keyword = keyword.Replace("*", " ");
}
//long datetime = ConvertDateTimeToInt(true, DateTime.Now);//13位时间戳
//long datetime16 = ConvertDateTimeToInt(false, DateTime.Now);//16位时间戳
url = "http://weixin.sogou.com/weixin?"
+ "query=" + keyword + "&"
+ "_sug_type_=1&"
+ "sut=0&"
+ "sourceid=inttime_all&"//当天的内容inttime_day 全部时间的inttime_all
+ "ri=0&"
+ "_sug_=n&"
+ "type=2&"//type=2 微信号 type=1 公众号
+ "ie=utf-8&"
//+ "sst0=" + datetime + "&"
+ "interation=&"
//+ "interV=kKIOkrELjboJmLkElbYTkKIKmbELjbkRmLkElbk%3D_1893302304&"
+ "tsn=0&"//0：全部时间，1：一天内，2：一星期内
+ "page=(*)";
//+ "dp=1";
//url = HttpUtility.UrlEncode(url);

cookieAll = GetHtmlByYzm(url, keyword);
//由于读取一次URL之后有10条新闻需要处理 在这处理10新闻的间隙中
//cookiecontianer或许保存了过期的SUID,与之前读出URL是产生的SUID不一致 导致 访问内页新闻详情时链接失效
listWeixin = GetTitleContent(result, keyword, cookieAll);
foreach (WeixinquanMessage wm in listWeixin)
{
wm.KeyWord = data.KeyWord;
wqBll.Add(wm);
}

cookieAll = "";//清空一下
url = "";//清空一下
}
#endregion
Console.WriteLine("OK");
Console.ReadKey();
}

/// <summary>
/// 将日期转化为Unix时间戳
/// </summary>
/// <param name="time">时间日期格式</param>
/// <returns>长整型数据</returns>
private static long ConvertDateTimeToInt(bool flag, System.DateTime time)
{
long t = 0;
System.DateTime startTime = TimeZone.CurrentTimeZone.ToLocalTime(new System.DateTime(1970, 1, 1, 0, 0, 0, 0));
if (flag)
{
t = (time.Ticks - startTime.Ticks) / 10000;   //除10000调整为13位
}
else
{
t = time.Ticks - startTime.Ticks;//16位
}
return t;
}

/// <summary>
/// 将时间戳转化为时间日期格式
/// </summary>
/// <param name="timeStamp"></param>
/// <returns></returns>
private static DateTime ConvertStringToDateTime(string timeStamp)
{
DateTime dtStart = TimeZone.CurrentTimeZone.ToLocalTime(new DateTime(1970, 1, 1));
long lTime = long.Parse(timeStamp + "0000000");
TimeSpan toNow = new TimeSpan(lTime);

return dtStart.Add(toNow);
}

/// <summary>
/// 随机生成一个useragent
/// </summary>
/// <returns></returns>
private static void RandomUserAgent()
{
Random r = new Random();
int random = r.Next(0, UserAgentList.userAgentShuZu.Length);
userAgentLiShi = UserAgentList.userAgentShuZu[random];
}

/// <summary>
/// 获取验证码返回的id值
/// </summary>
/// <param name="keywords">关键词</param>
/// <returns>id值</returns>
private static string GetSNUID(string keywords)
{
string id = "";
string referer = "";
for (int j = 0; j < 10; j++)
{
ch.MyCookieContainer = new CookieContainer();//清空掉原有的cookie
ch.Cookie = "";//清空掉原有的cookie
long tc = ConvertDateTimeToInt(true, DateTime.Now) / 1000;
string imageUrl = "http://weixin.sogou.com/antispider/util/seccode.php?tc=" + tc.ToString() + "";//图片的src地址
Image image = ch.getImage(imageUrl, "UTF-8");
CodeFactory cf = new CodeFactory(CodeType.YDM);
CodeModel cm = cf.GetCode(image);
string url = "http://weixin.sogou.com/antispider/thank.php";
string postdata = "c=" + cm.Code + "&r=/weixin?type=2&query=" + keywords + "&ie=utf8&_sug_=n&_sug_type_=&v=5";
referer = HttpUtility.UrlEncode("http://weixin.sogou.com/antispider/?from=/weixin?type=2&query=" + keywords + "&ie=utf8&_sug_=n&_sug_type_=");
id = ch.PostAndGetResult(url, "UTF-8", postdata, "", "", referer);

if (id.IndexOf("解封成功，正在为您跳转来源地址...") > -1)
{
break;
}
}

id = id.Substring(id.IndexOf("id")).Replace("\"", "").Replace(":", "").Replace(" ", "");
id = id.Substring(2, id.Length - 3);

return id;
}

/// <summary>
/// 通过破解验证码的方式获取网页源码
/// </summary>
/// <param name="urls">要访问的URL</param>
/// <param name="keywords">要访问的关键词</param>
/// <returns>cookie值</returns>
private static string GetHtmlByYzm(string urls,string keywords)
{
bool isOk = false;
string cookies = "";
result = ch.getHtml(urls, "UTF-8", "", "", "", "	Mozilla / 5.0(Windows NT 10.0; WOW64; rv: 45.0) Gecko / 20100101 Firefox / 45.0");
cookies = ch.Cookie;

for ( int i = 0; i < 10; i++)
{
isOk = true;
if (result.IndexOf("请输入验证码") > -1)
{
isOk = false;
if (cookies.IndexOf("SNUID") > -1)
{
cookies = cookies.Substring(0, cookies.IndexOf("SNUID") - 1);
}
//经测试如果出现验证码 seccodeRight=success;refresh=1,必须得用 successCount =1|Thu, 31 Mar 2016 07:58:56 GMT; 可以不用
cookies += ";SUV=00677DCD6F11A86256FCD05375656513;seccodeRight=success;refresh=1;SNUID=" + id;
result = ch.getHtml(urls, "UTF-8", cookies, "", "", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0");//Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0
}

//解决出现验证码 获取一次id 可执行4-5次
if (result.IndexOf("请输入验证码") > -1)
{
id = GetSNUID(keywords);
}

if (isOk)
{
break;
}
}

return cookies;
}

/// <summary>
/// 把中文的标点符号转换回来
/// </summary>
/// <param name="parm"></param>
/// <returns></returns>
private static string ReplaceZhongWenBiaoDian(string parm)
{
string result = "";
string[] a = { " ", "&", """, "'", "“", "”", "—", "<", ">", "·", "…" };
string[] b = { " ", "&", "\"", "'", "“", "”", "—", "<", ">", "·", "…" };
for (int i = 0; i < a.Length; i++)
{
if (parm.Contains(a[i]))
{
parm = parm.Replace(a[i], b[i]);
}
}

result = parm;
return result;
}

/// <summary>
/// 把html 标签里的class style 全部去掉
/// </summary>
/// <param name="htmltag"></param>
/// <returns></returns>
private static string RegexReplaceHtmlTag(string htmltag)
{
string result = "";
List<string> list = new List<string>();
Regex regex = new Regex(@"<([a-z|A-Z|0-9]+?) [^>]*?>", RegexOptions.IgnoreCase);
MatchCollection mtc = regex.Matches(htmltag);
foreach (Match m in mtc)
{
if (!list.Contains(m.Groups[1].Value))
{
list.Add(m.Groups[1].Value);
}

}
string[] h = { "h1", "h2", "h3", "h4", "h5", "h6" };//把h标签换成p标签
for (int i = 0; i < list.Count; i++)
{
htmltag = Regex.Replace(htmltag, @"<" + list[i] + " [^>]*?>", "<" + list[i] + ">", RegexOptions.IgnoreCase);
if (h.Contains(list[i]))
{
htmltag = htmltag.Replace("<" + list[i] + ">", "<p>").Replace("</" + list[i] + ">", "</p>");
}
}
result = htmltag.Replace("<img>", "").Replace("<iframe>", "").Replace("</iframe>", "");
return result;
}

/// <summary>
/// 正则匹配
/// </summary>
/// <param name="source">html源码</param>
/// <param name="format">匹配表达式</param>
/// <param name="isFrist">是否取值group[0]与group[1]</param>
/// <returns>匹配的值</returns>
private static string RegexPP(string source, string format, bool isFrist)
{
Regex regexRepostsCount = new Regex(format, RegexOptions.IgnoreCase);
Match m = regexRepostsCount.Match(source);
string rc = "";
rc = m.Groups[0].Value;
if (!isFrist)
{
rc = m.Groups[1].Value;
}

return rc;
}

/// <summary>
/// 获取文章的相关信息
/// </summary>
/// <param name="str">html源码</param>
/// <param name="keywords">关键词</param>
/// <param name="cookies">用到的cookie值</param>
/// <returns>List<WeixinquanMessage>的集合</returns>
private static List<WeixinquanMessage> GetTitleContent(string str,string keywords,string cookies)
{
str = str.Replace("\\r", "").Replace("\\t", "").Replace("\\n", "");
List<WeixinquanMessage> list = new List<WeixinquanMessage>();

kw = new string[1];
kw[0] = keywords;
if (keywords.Contains(" "))
{
kw = new string[2];
kw = keywords.Split(' ');//如何关键词中包含* 之拆分为 两个词
}

//获取整个文章
Regex regexSource = new Regex(@"<div class=""wx-rb wx-rb3"" [^>]*?>(\s|\S)+?(</div>\s*?){3}", RegexOptions.IgnoreCase);//(\s|\S)+?
MatchCollection mtcSource = regexSource.Matches(str);

string[] source = new string[mtcSource.Count];
int i = 0;
foreach (Match m in mtcSource)
{
source[i++] = m.Groups[0].Value;
}

for (int j = 0; j< source.Length; j++)
{
source[j] = source[j].Replace("\n", "");
WeixinquanMessage wm = new WeixinquanMessage();
string a = "";
string divTxtBox = "";
bool isKeyWord = false;//用来判断关键词 是否 存在于标题与内容之间

//匹配标题 与 URL
a = RegexPP(source[j], @"<div class=""txt-box""><h4>([\s|\S]*?)</h4>", false);
//url
wm.Url = "http://weixin.sogou.com" + RegexPP(a, @"<a [^>]*? href=""([\s\S]*?)""[^>]*?>", false).Replace("amp;", "");
//标题
wm.WeixinTitle = ReplaceZhongWenBiaoDian(Regex.Replace(a, @"<[\s\S]*?>", "", RegexOptions.IgnoreCase));

//判断该关键字是否存在于标题当中
if ((wm.WeixinTitle.IndexOf(kw[0]) != -1))
{
isKeyWord = true;
}
if (kw.Length > 1)
{
isKeyWord = false;
if ((wm.WeixinTitle.IndexOf(kw[1]) != -1))
{
isKeyWord = true;
}
}

//时间与作者
//时间
divTxtBox = RegexPP(source[j], @"<div class=""s-p"" [^>]*?>([\s|\S]*?)</a>", true);
string publishTime = RegexPP(divTxtBox, @"<div class=""s-p"" t=""([\d]*?)""[^>]*?>", false);
wm.PublishTime = ConvertStringToDateTime(publishTime);
//微信的用户名
string UserName = RegexPP(source[j], @"<div class=""s-p"" [^>]*?>([\s|\S]*?)</a>", false);
wm.UserName = RegexPP(UserName, @"<a [^>]*? title=""([\s|\S]*?)""[^>]*?>", false);

//通过URL获取文章内容、作者的信息等
string allContent = ch.getHtml(wm.Url, "utf-8", cookies, "", "", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:45.0) Gecko/20100101 Firefox/45.0");//ch.Cookie,http://weixin.sogou.com

//获取有效的URL地址
string biz = RegexPP(allContent, @"var biz = (\s|\S)+?;", true).Replace("||", "").Replace("\"", "").Replace(" ", "").Replace("var", "").Replace(";", "");
string sn = RegexPP(allContent, @"var sn = (\s|\S)+?;", true).Replace("||", "").Replace("\"", "").Replace(" ", "").Replace("var", "").Replace(";", "");
string mid = RegexPP(allContent, @"var mid = (\s|\S)+?;", true).Replace("||", "").Replace("\"", "").Replace(" ", "").Replace("var", "").Replace(";", "");
string idx = RegexPP(allContent, @"var idx = (\s|\S)+?;", true).Replace("||", "").Replace("\"", "").Replace(" ", "").Replace("var", "").Replace(";", "");
wm.Url = "http://mp.weixin.qq.com/s?__" + biz + "&" + mid + "&" + idx + "&" + sn + "&3rd=MzA3MDU4NTYzMw==&scene=6#rd";

string headTitle = RegexPP(allContent, @"<div class=""rich_media_meta_list"">(\s|\S)+?(</div>\s*?){2}", true);
//作者
wm.Author = RegexPP(headTitle, @"<em class=""rich_media_meta rich_media_meta_text"">([\s|\S]*?)<em>", false);
if (String.IsNullOrEmpty(wm.Author))
{
wm.Author = "空";
}
//微信号
wm.UserId = RegexPP(headTitle, @"<span class=""profile_meta_value"">([\s|\S]*?)</span>", false);
Random r = new Random();
if (String.IsNullOrEmpty(wm.UserId))//没有微信号 给一个默认的
{
int random = r.Next(0, 1000);
wm.UserId = wm.UserName + (Convert.ToInt64(DateTime.Now.ToString("yyyyMMddHHmmssms")) + random).ToString();
}
//文章内容不包含<p> <br/>标签
wm.WeixinViewContent = "";
string txt = "";
txt = RegexPP(allContent, @"<div class=""rich_media_content ""[^>]*?>([\s|\S]*?)</div>", false);
wm.WeixinViewContent = Regex.Replace(txt, @"<[\s\S]*?>", "", RegexOptions.IgnoreCase);
wm.WeixinViewContent = ReplaceZhongWenBiaoDian(wm.WeixinViewContent.Replace("\\r", "").Replace("\\n", "").Replace("\\t", "").Replace(" ", ""));

//文章内容包含 <p> <br/>标签
string section = "";
section= RegexPP(allContent, @"<div class=""rich_media_content ""[^>]*?>([\s|\S]*?)</div>", true);
wm.WeixinAllContent = RegexReplaceHtmlTag(section);
wm.WeixinAllContent = ReplaceZhongWenBiaoDian(wm.WeixinAllContent.Replace("\\r", "").Replace("\\n", "").Replace("\\t", "").Replace(" ", ""));

if (String.IsNullOrEmpty(wm.WeixinViewContent))
{
wm.WeixinViewContent = "文章内容为空或为图片格式。";
}

//如果该关键字不存在于标题当中，再进行匹配文章内容
if (!isKeyWord)
{
if ((wm.WeixinViewContent.IndexOf(kw[0]) != -1))
{
isKeyWord = true;
}
if (kw.Length > 1)
{
isKeyWord = false;
if ((wm.WeixinViewContent.IndexOf(kw[1]) != -1))
{
isKeyWord = true;
}
}
}

//如果该关键词既不存在于标题中，又不存在与内容中则直接结束本次循环
if (!isKeyWord)
{
continue;
}

list.Add(wm);
}

return list;
}
}