HttpWebRequest提高效率之连接数,代理，自动跳转,gzip请求等设置问题

先设置4个：

[csharp]
webrequest.ServicePoint.Expect100Continue = false;
//是否使用 Nagle 不使用 提高效率
webrequest.ServicePoint.UseNagleAlgorithm = false;
//最大连接数
webrequest.ServicePoint.ConnectionLimit = 65500;
//数据是否缓冲 false 提高效率
webrequest.AllowWriteStreamBuffering = false;

代理的使用：我用的代理都是不要账号和密码的，因为要不是免费的，要是是我们自己的，对来访IP 有限制

[csharp]
if (proxy!=null)
{
model.proxy = proxy;
WebProxy myProxy = new WebProxy(proxy.ProxyInfo, false);
myProxy.Credentials = new NetworkCredential("", "");
webrequest.Proxy = myProxy;
}
else
{
webrequest.Proxy = GlobalProxySelection.GetEmptyWebProxy();
}

webrequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");

带上gzip，如果对方支持gzip，流量上可以节省五分之四，原来250KB，用gzip压缩有 大概在50KB，减少带宽压力，增加速度

值得注意的是，如果是gzip，返回值必须要gzip解压的。

[csharp]
if (response.ContentEncoding.ToLower().Contains("gzip"))
{
using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
{
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{

model.html = reader.ReadToEnd();
}
}
}

还有一点：gzip解压的时候回耗费CPU，所以这个需要注意下

附完成方法

[csharp]
/// <summary>
/// Get 方式 获取数据
/// </summary>
/// <param name="webUrl"></param>
/// <returns></returns>
public Model.WebRequestReturnModel WebRequestGetHtmlByGet(string webUrl, CookieContainer cookieContainer, Encoding encoding, string refer,Model.Proxy proxy)
{
Model.WebRequestReturnModel model = new Model.WebRequestReturnModel();
HttpWebRequest webrequest = null;
try
{
webrequest = WebRequest.Create(webUrl) as HttpWebRequest;

webrequest.ServicePoint.Expect100Continue = false;
webrequest.ServicePoint.UseNagleAlgorithm = false;
webrequest.ServicePoint.ConnectionLimit = 65500;
webrequest.AllowWriteStreamBuffering = false;
if (proxy!=null)
{
model.proxy = proxy;
WebProxy myProxy = new WebProxy(proxy.ProxyInfo, false);
myProxy.Credentials = new NetworkCredential("", "");
webrequest.Proxy = myProxy;
}
else
{
webrequest.Proxy = GlobalProxySelection.GetEmptyWebProxy();
}
//设置其他的herader
webrequest =WebRequestSetHeaders(webrequest);
if (!String.IsNullOrEmpty(refer))
{
webrequest.Referer = refer;
}
else
{
webrequest.Referer = webUrl;
}
webrequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
webrequest.CookieContainer = cookieContainer;
webrequest.Timeout = 5000;
webrequest.UserAgent = "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)";
webrequest.Accept = "*/*";
webrequest.KeepAlive = true;
webrequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");

using (HttpWebResponse response = (HttpWebResponse)webrequest.GetResponse())
{

if (response.ContentEncoding.ToLower().Contains("gzip"))
{
using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
{
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{

model.html = reader.ReadToEnd();
}
}
}
else if (response.ContentEncoding.ToLower().Contains("deflate"))
{
using (DeflateStream stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress))
{
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{

model.html = reader.ReadToEnd();
}

}
}
else
{
using (Stream stream = response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{

model.html = reader.ReadToEnd();
}
}
}
}
}
catch (Exception ex)
{
model.exception = ex;
model.html = string.Empty;

}
model.cookieContainer = cookieContainer;
return model;
}