MongoDBcrud操作，采集部分代码

using System;
using System.Collections.Generic;
using System.ComponentModel.Design;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using CDPWIB.DAL;
using CDPWIB.Data;
using CommonUtility;
using HtmlAgilityPack;
using MongoDB.Driver;
using MongoDB.Driver.Builders;
using MongoDB.Driver.Linq;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using WebKit;

namespace CDPWIB.WebCollection
{
internal class QiDianCol : INovalCollect
{
private int Source = Convert.ToInt32(NovalSource.QiDian);

private readonly MongoCollection<NovalTempBase> Novalcol =
MongoConnectionFactory.GetMongoCollction<NovalTempBase>("Noval", typeof (NovalTempBase));

public void GetNovalTypeTemp()
{
try
{
var typecol = MongoConnectionFactory.GetMongoCollction<NovalTypeTemp>("Noval", typeof (NovalTypeTemp));
var subcol = MongoConnectionFactory.GetMongoCollction<NovalSubType>("Noval", typeof (NovalSubType));
// 大类 http://www.qidian.com/Javascript/qidian.bookstore.js?t=20130917 string typeshtml =
NetHelper.HttpGet("http://www.qidian.com/Javascript/qidian.bookstore.js?t=20130917")
.Replace("/", "")
.Replace(" ", "")
.Replace("\r", "")
.Replace("\n", "")
.Replace("\t", "")
.Replace("|", "")
.Replace(" ", "");
;
string subtypes =
NetHelper.HttpGet("http://script.cmfu.com/script/BookStore.js ")
.Replace(" ", "")
.Replace("\r", "")
.Replace("\n", "")
.Replace("\t", "")
.Replace("|", "")
.Replace(" ", "");
;

Match mtype = Regex.Match(typeshtml, "CategoryArr:(.*?)]]",
RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline);

string typesstring = mtype.Groups[1].Value + "]]";
JArray typearr = (JArray) JsonConvert.DeserializeObject(typesstring);
//JsonTextWriter

Match msubtype = Regex.Match(subtypes, "SubCategoryArr=(.*?);",
RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline);

string subtypesstring = msubtype.Groups[1].Value;

JArray subarr = (JArray) JsonConvert.DeserializeObject(subtypesstring);

List<NovalTypeTemp> lstypes = new List<NovalTypeTemp>(10);
//CategoryArr: [["全部", "-1"], ["玄幻", "21"], ["奇幻", "1"], ["武侠", "2"], ["仙侠", "22"], ["都市", "4"], ["历史", "5"], ["军事", "6"], ["游戏", "7"]
for (int i = 0; i < typearr.Count; i++)
{
if (typearr[i][1].ToString() != "-1")
{
NovalTypeTemp type = new NovalTypeTemp()
{
WebNum = typearr[i][1].ToString().ToInt(),
Name = typearr[i][0].ToString(),
Source = Source
};
lstypes.Add(type);
}
}
IMongoQuery query = Query<NovalTypeTemp>.EQ(p => p.Source, Source);

typecol.Remove(query);

typecol.InsertBatch(lstypes);
List<NovalSubType> subtypels = new List<NovalSubType>(300);

foreach (var NovalTypeTemp in lstypes)
{

for (int i = 0; i < subarr.Count; i++)
{
var obj = subarr[i];
if (obj[0].ToString() == NovalTypeTemp.WebNum.ToString())
{
NovalSubType subtype = new NovalSubType()
{
Name = obj[2].ToString(),
ParentWebNum = NovalTypeTemp.WebNum,
WebNum = obj[1].ToString().ToInt(),
Source = Source
};
subtypels.Add(subtype);
}
}

}
query = Query<NovalSubType>.EQ(p => p.Source, Source);
subcol.Remove(query);
subcol.InsertBatch(subtypels);
}
catch (Exception ex)
{
throw;
}
}

/// <summary>
/// 根据点击数页面查小说
/// </summary>
public void GetNovals()
{
//取1到10页
//得到月点击排行小说。
string sourcehtml = string.Empty;
HtmlDocument htmldocc = new HtmlDocument();
List<NovalTempBase> qdls = new List<NovalTempBase>(500);
for (int j = 1; j < 11; j++)
{
sourcehtml =
NetHelper.HttpGet("http://top.qidian.com/Book/TopDetail.aspx?TopType&Time=2&PageIndex=" + j);
;
htmldocc.LoadHtml(sourcehtml);
var doc = htmldocc.GetElementbyId("textlist");
//string tablehtml = "<table>" + doc.InnerHtml + "</table>";
//     htmldocc.LoadHtml(tablehtml);
//一页50列
for (int i = 2; i < 52; i++)
{
var trdoc = doc.SelectSingleNode("tr[" + i + "]");
//这里的下标，从1算起
var tdtype = trdoc.SelectSingleNode("td[2]/a");
var tdbook = trdoc.SelectSingleNode("td[3]/a[1]");
var tdclick = trdoc.SelectSingleNode("td[4]");
var tdauth = trdoc.SelectSingleNode("td[5]/a");
Match typematch = Regex.Match(tdtype.OuterHtml, "ChannelId=(\\d*?)&SubCategoryId=(\\d*?)'");
Match bookmatck = Regex.Match(tdbook.OuterHtml, "Book/(\\d*?).aspx");
Match authmatch = Regex.Match(tdauth.OuterHtml, "id=(\\d*?)\"");
int authid = authmatch.Groups[1].Value.ToInt();
int type = typematch.Groups[1].Value.ToInt();
int subtype = typematch.Groups[2].Value.ToInt();
int booknum = bookmatck.Groups[1].Value.ToInt();
string bookname = tdbook.InnerText.Trim();
//http://image.cmfu.com/books/3127618/3127618.jpg
string titleimg = "http://image.cmfu.com/books/" + booknum + "/" + booknum + ".jpg";

bool exist= qdls.Exists(p => p.SourceWebNum == booknum);
if (!exist)
{
NovalTempBase qidian = new NovalTempBase()
{
AuthName = tdauth.InnerText.Trim(),
AuthId = authid,
SubType = subtype,
TitleImg = titleimg,
Title = bookname,
TotalClick = tdclick.InnerText.ToInt(),
TotalComment = 0,
Type = type,
SourceWebNum = booknum,
Source = Source
};
qdls.Add(qidian);
}

}
}

PublicMethod.InsertAndUpdateNovalTmp(qdls,Source);
}

//public void GetNovalsByType()
//{
//}
/// <summary>
/// 得到小说章节 ，个别来源，带分卷。
/// </summary>
public void GetNovalChapers()
{

//http://sight.qq.com/book/chapterpage?uin=0&g_tk=5381&callback=_Callback&pagesize=100&pageno=2&bid=16043&_r=0.6934567329008132
var novalcol = MongoConnectionFactory.GetMongoCollction<NovalTempBase>("Noval", typeof (NovalTempBase));

var books = novalcol.AsQueryable().Where(p=>p.Source==Source).ToList();
foreach (var infoQidian in books)
{
GetSingleNovalChapers(infoQidian.SourceWebNum);
}
}

public void GetSingleNovalChapers(int novalwebnum)
{
IMongoQuery q2 = Query<NovalVolumeTemp>.EQ(p => p.Source, Source);
IMongoQuery q1 = Query<NovalVolumeTemp>.EQ(p => p.NovalWebNum, novalwebnum);
IMongoQuery[] qarray = { q1, q2 };

IMongoQuery query = Query.And(qarray);

var chaptercol = MongoConnectionFactory.GetMongoCollction<NovalChapterTemp>("Noval", typeof(NovalChapterTemp));
var volumecol = MongoConnectionFactory.GetMongoCollction<NovalVolumeTemp>("Noval", typeof (NovalVolumeTemp));
List<NovalChapterTemp> lschapters = new List<NovalChapterTemp>(1000);
List<NovalVolumeTemp> lsvolumes = new List<NovalVolumeTemp>(10);
int chapterorder = 1;
int volumeorder = 1;
HtmlDocument htmldocc = new HtmlDocument();
//http://read.qidian.com/BookReader/3127618.aspx

string sourcehtml = string.Empty;
string url = "http://read.qidian.com/BookReader/" + novalwebnum + ".aspx";
try
{
sourcehtml = NetHelper.HttpGet(url);
//目录主页
htmldocc.LoadHtml(sourcehtml);
var doc = htmldocc.GetElementbyId("content");
int i = 1;

var topdoc = doc.SelectSingleNode("div[" + i + "]");
while (topdoc != null)
{
var topa = topdoc.SelectSingleNode("div/a");
//如果是vip章节，没有这个A标签。
int topnum;
//分卷信息
if (topa != null)
{
string topahtml = topa.OuterHtml;
//href="http://www.qidian.com/BookReader/vol,107580,486625.aspx"
Match m = Regex.Match(topahtml, ",(\\d*?).aspx");
topnum = m.Groups[1].Value.ToInt();
var topaname = topdoc.SelectSingleNode("div/b");
string topname = topaname.InnerText.Trim();

topname = topname.Replace(" ", "").Split(';')[1];
//if(topname=="作品相关")
NovalVolumeTemp volume = new NovalVolumeTemp()
{
Sort = volumeorder,
WebNum = topnum,
Name = topname,
NovalWebNum = novalwebnum,
Source = Source
};
lsvolumes.Add(volume);
volumeorder++;
}
else
{
topnum = 0;
}

var contextdoc = doc.SelectSingleNode("div[" + (i + 1) + "]");
var chaperas = contextdoc.SelectNodes("div/ul/li/a");
//<a itemprop='url' href="http://read.qidian.com/BookReader/107580,20901221.aspx" title='凡人修仙传
字数：84  更新时间：2008-08-01 07:54:48'><span itemprop='headline'>呵呵！终于上架了！</span></a>
//,(\d*?).aspx
string chaptername = string.Empty;
//章节信息
int chapterwebnum = 0;
for (int x = 0; x < chaperas.Count; x++)
{
var chapera = chaperas[x];
chaptername = chapera.InnerText.Trim();
Match chapmatchwebnum = Regex.Match(chapera.OuterHtml, ",(\\d*?).aspx");
chapterwebnum = chapmatchwebnum.Groups[1].Value.ToInt();
NovalChapterTemp chapter = new NovalChapterTemp()
{
Name = chaptername,
Sort = chapterorder,
WebNum = chapterwebnum,
VolumeId = topnum
,
NovalWebNum = novalwebnum,
Source = Source
};
lschapters.Add(chapter);
chapterorder++;
}
i += 2;
topdoc = doc.SelectSingleNode("div[" + i + "]");
}
volumecol.Remove(query);
volumecol.InsertBatch(lsvolumes);
PublicMethod.InsertChapterTempToSQL(lschapters, Source, novalwebnum);

}
catch (Exception ex)
{
return;
}

}

public void GetNovalCilckComment()
{
var novalcol = MongoConnectionFactory.GetMongoCollction<NovalTempBase>("Noval", typeof(NovalTempBase));

var books = novalcol.AsQueryable().Where(p => p.Source == Source).ToList();
string sourcehtml = string.Empty;
string url = string.Empty;
HtmlDocument htmldocc = new HtmlDocument();
foreach (var novalTempBase in books)
{
//http://www.qidian.com/Book/3106580.aspx
url = "http://www.qidian.com/Book/" + novalTempBase.SourceWebNum + ".aspx";
sourcehtml = NetHelper.HttpGet(url);
htmldocc.LoadHtml(sourcehtml);
var cliclickdiv = htmldocc.GetElementbyId("contentdiv");
// /div/div/div[1]/table/tbody/tr/td[1]

var clickcount =
cliclickdiv.SelectSingleNode("div/div[1]/table/tr/td[1]")
.InnerText.Replace("总点击", "")
.Replace("：", "").Trim();

int click = Convert.ToInt32(clickcount);

//    string urlcom = "http://forum.qidian.com/NewForum/List.aspx?BookId=3106580";
////http://forum.qidian.com/NewForum/List.aspx?BookId=3106580

// //   http://c.pingba.qidian.com/BookComment.aspx?BookId=3106580 //    url = "http://c.pingba.qidian.com/BookComment.aspx?" + novalTempBase.SourceWebNum;
//    sourcehtml = NetHelper.HttpGet(url);
//    htmldocc.LoadHtml(sourcehtml);
novalTempBase.TotalClick = click;
novalcol.Save(novalTempBase);
}

//目录主页

}

}
}