c# 使用正则表达式 提取章节小说正文全本篇

这一节主要内容是使用正则表达式提取网站的正文，主要面向于小说章节网站。其中涉及到一些其他知识点，比如异步读取、异步流写入等，代码中都会有详细的注解。现在流行的网络文学都是每日一更或几更，没有一个统一的下载入口。以下我将实现一个简单的章节小说下载器的功能，将章节小说以整本的形式下载保存，保守估计能下载网络上70%以上小说。

先看看小说网站的网页源码,天蚕土豆的大主宰第一章。

http://www.biquge.com/4_4606/991334.html 笔趣网

private async Task downLoadNovel(byte[] bytes, string url)
{
title = string.Empty;
nextPageUrl = string.Empty;
content = string.Empty;
novelInfo = string.Empty;

try
{
byte[] response = bytes;
if (bytes == null)
{
response = await httpClient.GetByteArrayAsync(url);
}
content = Encoding.Default.GetString(response, 0, response.Length - 1);
//获取网页字符编码描述信息
var charSetMatch = Regex.Match(content, "<meta([^<]*)charset=([^<]*)\"", RegexOptions.IgnoreCase | RegexOptions.Multiline);

string webCharSet = charSetMatch.Groups[2].Value;
if (chartSet == null || chartSet == "")
chartSet = webCharSet;

if (chartSet != null && chartSet != "" && Encoding.GetEncoding(chartSet) != Encoding.Default)
content = Encoding.GetEncoding(chartSet).GetString(response, 0, response.Length - 1);
}
catch (Exception ex)
{
throw ex;
}
//小说主域名
if (webSiteDomain.Length == 0)
{
var websiteDomainMath = Regex.Match(url, "(http).*(/)", RegexOptions.IgnoreCase);
webSiteDomain = websiteDomainMath.Groups[0].Value;
}

//标题信息
var titleInfoMath = Regex.Match(content, "(<title>)([^>]*)(</title>)", RegexOptions.IgnoreCase | RegexOptions.Multiline);
title = titleInfoMath.Groups[2].Value;

content = content.Replace("'", "\"").Replace("\r\n", "");

for (int i = 0; i < contextPatterns.Length; i++)
{
var cpattern = contextPatterns[i];
if (novelInfo.Length == 0)
{
//正文信息
var webInfoMath = Regex.Matches(content, cpattern, RegexOptions.IgnoreCase | RegexOptions.Multiline);

for (int j = 0; j < webInfoMath.Count; j++)
{
foreach (Group g in webInfoMath[j].Groups)
{
var value = Regex.Split(g.Value, contextNewLine, RegexOptions.IgnoreCase);
if (value.Length > 5)
{
novelInfo = g.Value;
foreach (var pattern in filterPatterns)
novelInfo = Regex.Replace(novelInfo, pattern, new MatchEvaluator(p => null));

novelInfo = Regex.Replace(novelInfo, contextNewLine, new MatchEvaluator(p => "\r\n"));
break;
}
}
}

}
else
break;
}

bytes = null;

for (int i = 0; i < nextPagePatterns.Length; i++)
{
if (nextPageUrl.Length == 0)
{
//下一页信息
var webNextPageMath = Regex.Match(content, nextPagePatterns[i], RegexOptions.IgnoreCase | RegexOptions.Multiline);
if (webNextPageMath.Groups.Count > 0)
{
foreach (Group g in webNextPageMath.Groups)
{
if (!g.Value.EndsWith("\""))
nextPageUrl = g.Value;
if (nextPageUrl.StartsWith("/"))
nextPageUrl = nextPageUrl.Substring(1);
if (!nextPageUrl.StartsWith("http", true, null) && (Regex.IsMatch(nextPageUrl, "[a-z]") || Regex.IsMatch(nextPageUrl, "[0-9]")) && !url.EndsWith(nextPageUrl))
{
nextPageUrl = webSiteDomain + nextPageUrl;
}
try
{
bytes = await httpClient.GetByteArrayAsync(nextPageUrl);
break;
}
catch
{
continue;
}
}

}
}
else
break;
}
bool isAdd = false;
cacheNovel.ForEach(p =>
{
if (p == (title + novelInfo))
{
isAdd = true;
}
});

if (!isAdd)
{
if (title.Length > 0)
{
writeNovelLog("正在下载章节:" + title);
}

writeNovelLog("章节长度:" + novelInfo.Length);

cacheNovel.Add(title + novelInfo);

if (nextPageUrl.Length > 0)
{
writeNovelLog("下一页:" + nextPageUrl);

await downLoadNovel(bytes, nextPageUrl);
}
else
{
downloadFinish();
}
}
else
{
writeNovelLog("存在重复的章节,章节名称:" + title + " 地址:" + url);
downloadFinish();
}
}

异步下载网页流、解析数据

最后效果