TPL使用异步编程模型(APM)实现简单网页抓取

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;

namespace fofo.NetSpider
{
public class SimpleNetSpider
{
public void Start()
{
string[] urls = InitUrls();

Task<WebResponse>[] tasks = urls.Select(url => GetResponseAsync(url)).ToArray();

Task.WaitAll(tasks);
}

private string[] InitUrls()
{
string[] urls =
{
"http://www.baidu.com/",
"http://www.cnblogs.com",
"http://www.csdn.com"
};

return urls;
}

// 启动任务
private Task<WebResponse> GetResponseAsync(string strUrl)
{
WebRequest webRequest = WebRequest.Create(strUrl);
Task<WebResponse> task = Task<WebResponse>.Factory.FromAsync(webRequest.BeginGetResponse, GetResponseAsyncCompleted, webRequest);

return task;
}

// 抓取结束回调函数
private WebResponse GetResponseAsyncCompleted(IAsyncResult result)
{
WebRequest webRequest = (WebRequest)result.AsyncState;
WebResponse response = webRequest.EndGetResponse(result);

using (System.IO.StreamReader reader = new System.IO.StreamReader(response.GetResponseStream()))
{
// 具体处理
}

return response;
}
}
}