支持Cookie并开放了一些特殊设置项的HttpWebClient

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using System.Collections.Specialized;
using System.Web;

namespace Common.Helpers
{
/// <summary>
/// 网络访问辅助类
/// </summary>
public class HttpWebClient : WebClient
{
#region 公共属性
/// <summary>
/// 浏览器用户标识,默认采用Chrome的标识
/// </summary>
public string UserAgent { get; set; }
/// <summary>
/// Cookie容器
/// </summary>
public CookieContainer CookieContainer { get; set; }
/// <summary>
/// 如果 POST 请求需要 100-Continue 响应，则为 true；否则为 false。
/// </summary>
public bool Expect100Continue { get; set; }

private WebResponse m_LastWebResponse = null;
/// <summary>
/// 最后一次的响应对象
/// </summary>
public WebResponse LastWebResponse { get { return this.m_LastWebResponse; } }

private int m_Timeout = 120000;
/// <summary>
/// 超时时间，默认120000毫秒(120秒)
/// </summary>
public int Timeout
{
get { return m_Timeout; }
set { m_Timeout = value; }
}

private HttpWebClientSetting m_HttpWebClientSetting = null;
/// <summary>
/// WebClient设置项,该属性始终不会为null
/// </summary>
public HttpWebClientSetting HttpWebClientSetting
{
get
{
if (m_HttpWebClientSetting == null)
{
m_HttpWebClientSetting = new HttpWebClientSetting();
}
return m_HttpWebClientSetting;
}
set
{
m_HttpWebClientSetting = value ?? new HttpWebClientSetting();
}
}

/// <summary>
/// 预处理Web请求对象的委托方法(会在每次获取WebRequest对象后调用)，默认值为null
/// </summary>
public Action<HttpWebRequest> PrepareProcessWebRequest { get; set; }
#endregion

#region 构造方法
public HttpWebClient()
: this(new CookieContainer())
{
}

public HttpWebClient(CookieContainer cookieContainer)
{
this.CookieContainer = cookieContainer;
this.UserAgent = UserAgentValues.FireFox;
this.Expect100Continue = false;
}
#endregion

#region 重写方法，增加对CookieContainer的支持
protected override WebRequest GetWebRequest(Uri address)
{
if (!string.IsNullOrEmpty(this.UserAgent))
{
this.Headers.Add(HttpRequestHeader.UserAgent, this.UserAgent);
}

WebRequest request = base.GetWebRequest(address);
request.Timeout = this.Timeout;

if (request is HttpWebRequest)
{
HttpWebRequest httpRequest = request as HttpWebRequest;
httpRequest.CookieContainer = this.CookieContainer;
httpRequest.ServicePoint.Expect100Continue = this.Expect100Continue; // 取消100-continue

//读取自定义设置项
if (this.HttpWebClientSetting != null)
{
httpRequest.AllowAutoRedirect = this.HttpWebClientSetting.AllowAutoRedirect;
}

//使用外部委托属性处理Request对象
if (this.PrepareProcessWebRequest != null)
{
this.PrepareProcessWebRequest(httpRequest);
}
}

return request;
}
#endregion

#region 重写方法，增加对响应对象的访问
protected override WebResponse GetWebResponse(WebRequest request)
{
WebResponse response = base.GetWebResponse(request);
this.m_LastWebResponse = response;
return response;
}
#endregion

#region (public) 向一个URL用POST提交数据，并返回其响应内容 PostData
/// <summary>
/// 向一个URL用POST提交数据，并返回其响应内容
/// ZhangQingFeng    2014-12-14    Add
///    EditLog：
///        ZhangQingFeng    2015-05-12    Edit        因WebClient的UpdateValues方法中固定为UTF-8格式进行UrlEncode，因此此处需用UploadString方式来间接实现    --见微软WebClient类源码UploadValuesInternal方法中
/// </summary>
/// <param name="url">请求的URL</param>
/// <param name="data">要提交的数据</param>
/// <param name="encoding">请求所使用的编码</param>
/// <param name="responseEncoding">响应内容所使用的编码，为null时使用请求的编码</param>
/// <returns>响应的内容</returns>
public string PostData(string url, NameValueCollection data, Encoding encoding, Encoding responseEncoding)
{
WebClient client = this;

/*
client.Encoding = encoding ?? Encoding.UTF8;

byte[] response = client.UploadValues(url, "POST", data ?? new NameValueCollection());

string html = string.Empty;

if (responseEncoding == null)
{
html = client.Encoding.GetString(response);
}
else
{
html = responseEncoding.GetString(response);
}
*/

client.Encoding = encoding ?? Encoding.UTF8;
client.Headers.Add(HttpRequestHeader.ContentType, "application/x-www-form-urlencoded");

string delimiter = String.Empty;
StringBuilder values = new StringBuilder();
foreach (string name in data.AllKeys)
{
values.Append(delimiter);
values.Append(HttpUtility.UrlEncode(name, encoding));
values.Append("=");
values.Append(HttpUtility.UrlEncode(data[name], encoding));
delimiter = "&";
}

byte[] arrData = client.UploadData(url, "POST", Encoding.ASCII.GetBytes(values.ToString()));
string html = (responseEncoding ?? client.Encoding).GetString(arrData);

return html;
}

/// <summary>
/// 向一个URL用POST提交数据，并返回其响应内容
/// ZhangQingFeng    2014-12-14    Add
/// </summary>
/// <param name="url">请求的URL</param>
/// <param name="data">要提交的数据</param>
/// <param name="encoding">请求和响应所使用的编码</param>
/// <returns>响应的内容</returns>
public string PostData(string url, NameValueCollection data, Encoding encoding)
{
return PostData(url, data, encoding, null);
}

/// <summary>
/// 向一个URL用POST提交数据，并返回其响应内容(使用this.Encoding来作请求编码和响应编码)
/// ZhangQingFeng    2014-12-14    Add
/// </summary>
/// <param name="url">请求的URL</param>
/// <param name="data">要提交的数据</param>
/// <returns>响应的内容</returns>
public string PostData(string url, NameValueCollection data)
{
return PostData(url, data, this.Encoding);
}
#endregion

#region (public) 向一个URL用POST提交数据，并返回其响应内容 PostData
/// <summary>
/// 向一个URL用POST提交数据，并返回其响应内容
/// ZhangQingFeng    2014-12-14    Add
/// </summary>
/// <param name="url">请求的URL</param>
/// <param name="data">要提交的数据</param>
/// <param name="encoding">请求和响应内容所使用的编码</param>
/// <returns>响应的内容</returns>
public string PostData(string url, Dictionary<string, string> data, Encoding encoding, Encoding responseEncoding)
{
NameValueCollection postData = new NameValueCollection();
if (data != null)
{
foreach (var item in data)
{
postData.Add(item.Key, item.Value);
}
}
return PostData(url, postData, encoding, responseEncoding);
}

/// <summary>
/// 向一个URL用POST提交数据，并返回其响应内容
/// ZhangQingFeng    2014-12-14    Add
/// </summary>
/// <param name="url">请求的URL</param>
/// <param name="data">要提交的数据</param>
/// <param name="encoding">请求和响应所使用的编码</param>
/// <returns>响应的内容</returns>
public string PostData(string url, Dictionary<string, string> data, Encoding encoding)
{
return PostData(url, data, encoding, null);
}

/// <summary>
/// 向一个URL用POST提交数据，并返回其响应内容(使用this.Encoding来作请求编码和响应编码)
/// ZhangQingFeng    2014-12-14    Add
/// </summary>
/// <param name="url">请求的URL</param>
/// <param name="data">要提交的数据</param>
/// <returns>响应的内容</returns>
public string PostData(string url, Dictionary<string, string> data)
{
return PostData(url, data, this.Encoding);
}
#endregion

#region 辅助类
/// <summary>
/// 浏览器用户标识类
/// </summary>
public class UserAgentValues
{
public static readonly string FireFox = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:37.0) Gecko/20100101 Firefox/37.0";
public static readonly string Chrome = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36";
public static readonly string IE8 = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2;)";
}
#endregion
}

/// <summary>
/// HttpWebClient对象设置类
/// </summary>
public class HttpWebClientSetting
{
private bool m_AllowAutoRedirect = true;
/// <summary>
/// 当响应内容为重定向时客户端是否自动重定向(如果该属性为true，则取到的响应则为重定向后的内容，否则则为响应原文)，默认值为true
/// </summary>
public bool AllowAutoRedirect
{
get { return m_AllowAutoRedirect; }
set { m_AllowAutoRedirect = value; }
}
}
}

HttpWebClient
在做页面抓取的过程中，发现自带的WebClient不够灵活，因此做了一个实现。

关于在PostData方法中不使用UploadValues()方法的原因：

1.查看微软的源代码实现时发现，无论设置请求时的Encoding是否为GB2312，在使用WebClient的UploadValues()上传内容时，其内在都是使用UTF-8编码进行UrlEncode，因此传到服务端中的数据中若包含有中文时则一定会乱码，因此重写PostData以规避此问题。

关于HttpWebClientSetting中的AllowAutoRedirect属性：

在WebClient发起请求时，若响应内容为重定向，则WebClient会自动做重定向，因此该类提供此设置项以控制在访问时是否自动做重定向(第二次访问Refer后的网站时会将请求中的Refer头置空，将该AllowAutoRedirect设置为false，然后手动从Response.Header中取出Location对象地址，设置Refer后再访问，则可真实模拟浏览器访问，从而避开一些网站的防抓取设置)

关于HttpWebClient中的LastWebResponse属性：

当存在多次重定向时，系统记录了最后一次返回的内容，从此内容的Header中取出ResponseUri，则可以取到最后返回响应的页面真实地址，从而为下一次的设置请求Refer头作准备。

大约就是如此，后期如有Bug会继续更新。