httpanalyzer 结合 HttpWebRequest Post的运用

下载地址： http://files.cnblogs.com/Fooo/HttpAnalyzerFullV3和汉化补丁.rar

如上图： 安装httpanalyzerstdv3及汉化补丁后，利用GreenBrowser.exe绿色浏览器打开目标采集网址，接着点击httpanalyzerstdv的执行按钮，下一步在GreenBrowser中进行登陆及其它操作完毕后，便可以在httpanalyzerstdv中查看所需要到的Http 参数或Cookie集合。

(注：标签发送里的所有参数，正常情况下在HttpWebRequest的post时，必须填写完整)

public string LoginPorcess()
{
remotesessionid = string.Empty;
this.LoadBase.ClearMyCookieContainer();
string responseString = this.LoadBase.GetData(HomeUrl, Encoding.UTF8);
CheckResonseString(responseString, HomeUrl);

List<string> _postdatalist = HTMLParser.GetPostData(responseString, "input", "__VIEWSTATE", "MemberServer", "SatelliteServer", "__PREVIOUSPAGE"); // en-US
string tpldata = "__EVENTTARGET=&__EVENTARGUMENT=&__LASTFOCUS=&__VIEWSTATE={0}&ctl00%24MCPH%24LF%24UserName={1}&ctl00%24MCPH%24LF%24Password={2}&ctl00%24MCPH%24LF%24LB=Login&ctl00%24MCPH%24LF%24Culture=zh-cn&ctl00%24MCPH%24LF%24PriceStyle=decimal&ctl00%24MCPH%24LF%24MemberServer={3}&ctl00%24MCPH%24LF%24SatelliteServer={4}&ctl00%24MCPH%24UPBF%24LDDL=en-US&ctl00%24MCPH%24UPBF%24PSDDL=decimal&__PREVIOUSPAGE={5}";
string indata = string.Format(tpldata, HttpUtility.UrlEncode(_postdatalist[0]), HttpUtility.UrlEncode(Username), HttpUtility.UrlEncode(Password), HttpUtility.UrlEncode(_postdatalist[1]), HttpUtility.UrlEncode(_postdatalist[2]), HttpUtility.UrlEncode(_postdatalist[3]));
responseString = this.LoadBase.PostData(LoginUrl, RefererUrl, indata);
CheckResonseString(responseString, LoginUrl + indata);

if (responseString.Contains("customerId"))
{
int _end = responseString.IndexOf("高分</a>");
int _start = responseString.LastIndexOf("<a href=", _end);
if (_end > _start) {
string _href = responseString.Substring(_start + 9, _end - _start);
string[] _array = _href.Split('/');
remotesessionid = Username;
HomeUrl_ = "http://" + _array[2];
}
}

return remotesessionid;
}

public string GetData(string url)
{
return GetData(url, string.Empty, encode);
}
public string GetData(string url, Encoding ec)
{
return GetData(url, string.Empty, ec);
}
public string GetData(string url, string refererurl)
{
return GetData(url, refererurl, encode);
}
public string GetData(string url, string refererurl, Encoding ec)
{
encode = ec;
string receiveData = string.Empty;
try
{
HttpWebRequest myHttpWebRequest = WebRequest.Create(url) as HttpWebRequest;

myHttpWebRequest.Timeout = timeout;
myHttpWebRequest.Method = "GET";
SetRequestHeaders(myHttpWebRequest, refererurl, url.Substring(0, 5));

HttpWebResponse myHttpWebResponse = myHttpWebRequest.GetResponse() as HttpWebResponse;
myHttpWebResponse.Cookies = myCookieContainer.GetCookies(myHttpWebRequest.RequestUri);

//************
Stream myResponseStream = myHttpWebResponse.GetResponseStream();
StreamReader myStreamReader;
if (encode != null) {
myStreamReader = new StreamReader(myResponseStream, encode);
} else {
myStreamReader = new StreamReader(myResponseStream, Encoding.Default);
}
receiveData = myStreamReader.ReadToEnd();

myStreamReader.Close();
myResponseStream.Close();
} catch (Exception ex) {
return "操作超时";
}
return receiveData;
}

public string PostData(string url, string indata)
{
return PostData(url, string.Empty, indata);
}
public string PostData(string url, string refererurl, string indata)
{
string receiveData = string.Empty;
try
{
HttpWebRequest myHttpWebRequest = WebRequest.Create(url) as HttpWebRequest;

myHttpWebRequest.Timeout = timeout;
myHttpWebRequest.Method = "POST";
SetRequestHeaders(myHttpWebRequest, refererurl, url.Substring(0, 5));
myHttpWebRequest.ContentLength = indata.Length;

Stream myRequestStream = myHttpWebRequest.GetRequestStream();
StreamWriter myStreamWriter = new StreamWriter(myRequestStream, Encoding.Default);
myStreamWriter.Write(indata);
myStreamWriter.Close();
myRequestStream.Close();

HttpWebResponse myHttpWebResponse = myHttpWebRequest.GetResponse() as HttpWebResponse;
myHttpWebResponse.Cookies = myCookieContainer.GetCookies(myHttpWebRequest.RequestUri);
Stream myResponseStream = myHttpWebResponse.GetResponseStream();
StreamReader myStreamReader;
if (encode != null) {
myStreamReader = new StreamReader(myResponseStream, encode);
} else {
myStreamReader = new StreamReader(myResponseStream, Encoding.Default);
}
receiveData = myStreamReader.ReadToEnd();

myStreamReader.Close();
myResponseStream.Close();
} catch (Exception ex) {
System.Diagnostics.Debug.WriteLine(ex.Message.ToString());
return "操作超时";
}
return receiveData;
}

private void SetRequestHeaders(HttpWebRequest myRequest, string refer, string protocol)
{
myRequest.ContentType = "application/x-www-form-urlencoded";
myRequest.KeepAlive = true;
myRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322)";
myRequest.Headers.Add("Cache-Control", "no-cache");
myRequest.Headers.Add("Pragma", "no-cache");
myRequest.Accept = "*/*";
myRequest.AutomaticDecompression = DecompressionMethods.GZip;
myRequest.Credentials = CredentialCache.DefaultCredentials;
myRequest.UnsafeAuthenticatedConnectionSharing = true;
myRequest.UseDefaultCredentials = true;
myRequest.AllowWriteStreamBuffering = false;

myRequest.CookieContainer = myCookieContainer;
if (refer != string.Empty)
myRequest.Referer = refer;

if (protocol.ToLower() == "https")
{
try
{
ServicePointManager.CertificatePolicy = new CertPolicy();
if (securityFile != string.Empty)
{
X509Certificate Cert = X509Certificate.CreateFromCertFile(securityFile);
myRequest.ClientCertificates.Add(Cert);
}
}
catch
{
}
}
}

public string EncodeConvert(string ecContent, Encoding encode, int codepage)
{
byte[] bytes = System.Text.Encoding.Default.GetBytes(ecContent);
bytes = System.Text.Encoding.Convert(encode, System.Text.Encoding.GetEncoding(codepage), bytes);
return System.Text.Encoding.GetEncoding(codepage).GetString(bytes);

}

using System;
using System.Collections.Generic;
using System.Text;

namespace Match
{
public class HTMLParser {
public static List<string> GetPostData(string source, string tags, params string[] namelist)
{
List<string> m_postdatalist = new List<string>();

List<string> m_list = new List<string>();
for (int i = 0; i < namelist.Length; i++) {
m_list.Add(namelist[i]);
}
ParseHTML parse = new ParseHTML();
parse.Source = source;
while (!parse.Eof()) {
char ch = parse.Parse();
if (ch == 0) {
AttributeList tag = parse.GetTag();
if (tag.Name.ToLower() == tags.ToLower()) {
foreach (string _name in m_list) {
if ((tag["name"] != null && tag["name"].Value == _name) || (tag["id"] != null && tag["id"].Value == _name)) {
m_postdatalist.Add(tag["value"].Value);
m_list.Remove(_name);
break;
}
}
}
}
}

return m_postdatalist;
}
}

public class ParseHTML : Parse
{

public AttributeList GetTag()
{
AttributeList tag = new AttributeList();
tag.Name = m_tag;

foreach (Attribute x in List)
{
tag.Add((Attribute)x.Clone());
}

return tag;
}

public String BuildTag()
{
String buffer = "<";
buffer += m_tag;
int i = 0;
while (this[i] != null) {// has attributes
buffer += " ";
if (this[i].Value == null) {
if (this[i].Delim != 0)
buffer += this[i].Delim;
buffer += this[i].Name;
if (this[i].Delim != 0)
buffer += this[i].Delim;
} else {
buffer += this[i].Name;
if (this[i].Value != null) {
buffer += "=";
if (this[i].Delim != 0)
buffer += this[i].Delim;
buffer += this[i].Value;
if (this[i].Delim != 0)
buffer += this[i].Delim;
}
}
i++;
}
buffer += ">";
return buffer;
}

protected void ParseTag()
{
m_tag = "";
Clear();

// Is it a comment?
if ((GetCurrentChar() == '!') &&
(GetCurrentChar(1) == '-') &&
(GetCurrentChar(2) == '-')) {
while (!Eof()) {
if ((GetCurrentChar() == '-') &&
(GetCurrentChar(1) == '-') &&
(GetCurrentChar(2) == '>'))
break;
if (GetCurrentChar() != '\r')
m_tag += GetCurrentChar();
Advance();
}
m_tag += "--";
Advance();
Advance();
Advance();
ParseDelim = (char)0;
return;
}

// Find the tag name
while (!Eof()) {
if (IsWhiteSpace(GetCurrentChar()) || (GetCurrentChar() == '>'))
break;
m_tag += GetCurrentChar();
Advance();
}

EatWhiteSpace();

// Get the attributes
while (GetCurrentChar() != '>') {
ParseName = "";
ParseValue = "";
ParseDelim = (char)0;

ParseAttributeName();

if (GetCurrentChar() == '>') {
AddAttribute();
break;
}

// Get the value(if any)
ParseAttributeValue();
AddAttribute();
}
Advance();
}

public char Parse()
{
if (GetCurrentChar() == '<') {
Advance();

char ch = char.ToUpper(GetCurrentChar());
if ((ch >= 'A') && (ch <= 'Z') || (ch == '!') || (ch == '/')) {
ParseTag();
return (char)0;
} else return (AdvanceCurrentChar());
} else return (AdvanceCurrentChar());
}
}

public class Parse : AttributeList
{
/// <summary>
/// The source text that is being parsed.
/// </summary>
private string m_source;

/// <summary>
/// The current position inside of the text that
/// is being parsed.
/// </summary>
private int m_idx;

/// <summary>
/// The most reciently parsed attribute delimiter.
/// </summary>
private char m_parseDelim;

/// <summary>
/// This most receintly parsed attribute name.
/// </summary>
private string m_parseName;

/// <summary>
/// The most reciently parsed attribute value.
/// </summary>
private string m_parseValue;

/// <summary>
/// The most reciently parsed tag.
/// </summary>
public string m_tag;

/// <summary>
/// Determine if the specified character is whitespace or not.
/// </summary>
/// <param name="ch">A character to check</param>
/// <returns>true if the character is whitespace</returns>
public static bool IsWhiteSpace(char ch)
{
return ("\t\n\r ".IndexOf(ch) != -1);
}

/// <summary>
/// Advance the index until past any whitespace.
/// </summary>
public void EatWhiteSpace()
{
while (!Eof()) {
if (!IsWhiteSpace(GetCurrentChar()))
return;
m_idx++;
}
}

/// <summary>
/// Determine if the end of the source text has been
/// reached.
/// </summary>
/// <returns>True if the end of the source text has been
/// reached.</returns>
public bool Eof()
{
return (m_idx >= m_source.Length);
}

/// <summary>
/// Parse the attribute name.
/// </summary>
public void ParseAttributeName()
{
EatWhiteSpace();
// get attribute name
while (!Eof()) {
if (IsWhiteSpace(GetCurrentChar()) ||
(GetCurrentChar() == '=') ||
(GetCurrentChar() == '>'))
break;
m_parseName += GetCurrentChar();
m_idx++;
}

EatWhiteSpace();
}

/// <summary>
/// Parse the attribute value
/// </summary>
public void ParseAttributeValue()
{
if (m_parseDelim != 0)
return;

if (GetCurrentChar() == '=') {
m_idx++;
EatWhiteSpace();
if ((GetCurrentChar() == '\'') ||
(GetCurrentChar() == '\"')) {
m_parseDelim = GetCurrentChar();
m_idx++;
while (GetCurrentChar() != m_parseDelim) {
m_parseValue += GetCurrentChar();
m_idx++;
}
m_idx++;
} else {
while (!Eof() &&
!IsWhiteSpace(GetCurrentChar()) &&
(GetCurrentChar() != '>')) {
m_parseValue += GetCurrentChar();
m_idx++;
}
}
EatWhiteSpace();
}
}

/// <summary>
/// Add a parsed attribute to the collection.
/// </summary>
public void AddAttribute()
{
Attribute a = new Attribute(m_parseName,
m_parseValue, m_parseDelim);
Add(a);
}

/// <summary>
/// Get the current character that is being parsed.
/// </summary>
/// <returns></returns>
public char GetCurrentChar()
{
return GetCurrentChar(0);
}

/// <summary>
/// Get a few characters ahead of the current character.
/// </summary>
/// <param name="peek">How many characters to peek ahead for.</param>
/// <returns>The character that was retrieved.</returns>
public char GetCurrentChar(int peek)
{
if ((m_idx + peek) < m_source.Length)
return m_source[m_idx + peek];
else
return (char)0;
}

/// <summary>
/// Obtain the next character and advance the index by one.
/// </summary>
/// <returns>The next character</returns>
public char AdvanceCurrentChar()
{
return m_source[m_idx++];
}

/// <summary>
/// Move the index forward by one.
/// </summary>
public void Advance()
{
m_idx++;
}

/// <summary>
/// The last attribute name that was encountered.
/// </summary>
public string ParseName
{
get {
return m_parseName;
}

set {
m_parseName = value;
}
}

/// <summary>
/// The last attribute value that was encountered.
/// </summary>
public string ParseValue
{
get {
return m_parseValue;
}

set {
m_parseValue = value;
}
}

/// <summary>
/// The last attribute delimeter that was encountered.
/// </summary>
public char ParseDelim
{
get {
return m_parseDelim;
}

set {
m_parseDelim = value;
}
}

/// <summary>
/// The text that is to be parsed.
/// </summary>
public string Source
{
get {
return m_source;
}

set {
m_source = value;
}
}
}

public class AttributeList : Attribute
{

/// <summary>
/// An internally used Vector. This vector contains
/// the entire list of attributes.
/// </summary>
protected List<Attribute> m_list;

/// <summary>
/// Make an exact copy of this object using the cloneable interface.
/// </summary>
/// <returns>A new object that is a clone of the specified object.</returns>
public override Object Clone()
{
AttributeList rtn = new AttributeList();

for (int i = 0; i < m_list.Count; i++)
{
rtn.Add((Attribute)this[i].Clone());
}

return rtn;
}

/// <summary>
/// Create a new, empty, attribute list.
/// </summary>
public AttributeList()
: base("", "") {
m_list = new List<Attribute>();
}

/// <summary>
/// Add the specified attribute to the list of attributes.
/// </summary>
/// <param name="a">An attribute to add to this AttributeList.</param>
public void Add(Attribute a)
{
m_list.Add(a);
}

/// <summary>
/// Clear all attributes from this AttributeList and return it
/// to a empty state.
/// </summary>
public void Clear()
{
m_list.Clear();
}

/// <summary>
/// Returns true of this AttributeList is empty, with no attributes.
/// </summary>
/// <returns>True if this AttributeList is empty, false otherwise.</returns>
public bool IsEmpty()
{
return (m_list.Count <= 0);
}

/// <summary>
/// If there is already an attribute with the specified name,
/// then it will have its value changed to match the specified value.
/// If there is no Attribute with the specified name, then one will
/// be created. This method is case-insensitive.
/// </summary>
/// <param name="name">The name of the Attribute to edit or create. Case-insensitive.</param>
/// <param name="value">The value to be held in this attribute.</param>
public void Set(string name, string value)
{
if (name == null)
return;
if (value == null)
value = "";

Attribute a = this[name];

if (a == null) {
a = new Attribute(name, value);
Add(a);
} else
a.Value = value;
}

/// <summary>
/// How many attributes are in this AttributeList
/// </summary>
public int Count
{
get {
return m_list.Count;
}
}

/// <summary>
/// A list of the attributes in this AttributeList
/// </summary>
public List<Attribute> List
{
get {
return m_list;
}
}

/// <summary>
/// Access the individual attributes
/// </summary>
public Attribute this[int index]
{
get {
if (index < m_list.Count)
return (Attribute)m_list[index];
else
return null;
}
}

/// <summary>
/// Access the individual attributes by name.
/// </summary>
public Attribute this[string index]
{
get {
int i = 0;

while (this[i] != null) {
if (this[i].Name.ToLower().Equals((index.ToLower())))
return this[i];
i++;
}
return null;
}
}
}

/// <summary>
/// Attribute holds one attribute, as is normally stored in
/// an HTML or XML file. This includes a name, value and delimiter.
///
/// This spider is copyright 2003 by Jeff Heaton. However, it is
/// released under a Limited GNU Public License (LGPL). You may
/// use it freely in your own programs. For the latest version visit
/// http://www.jeffheaton.com. ///
/// </summary>
public class Attribute : ICloneable
{
/// <summary>
/// The name of this attribute
/// </summary>
private string m_name;

/// <summary>
/// The value of this attribute
/// </summary>
private string m_value;

/// <summary>
/// The delimiter for the value of this
/// attribute(i.e. " or ').
/// </summary>
private char m_delim;

/// <summary>
/// Construct a new Attribute. The name, delim and value
/// properties can be specified here.
/// </summary>
/// <param name="name">The name of this attribute.</param>
/// <param name="value">The value of this attribute.</param>
/// <param name="delim">The delimiter character for the value.</param>
public Attribute(string name, string value, char delim)
{
m_name = name;
m_value = value;
m_delim = delim;
}

/// <summary>
/// The default constructor. Construct a blank attribute.
/// </summary>
public Attribute()
: this("", "", (char)0) {
}

/// <summary>
/// Construct an attribute without a delimiter.
/// </summary>
/// <param name="name">The name of this attribute.</param>
/// <param name="value">The value of this attribute.</param>
public Attribute(String name, String value)
: this(name, value, (char)0) {
}

/// <summary>
/// The delimiter for this attribute.
/// </summary>
public char Delim
{
get
{
return m_delim;
}

set
{
m_delim = value;
}
}

/// <summary>
/// The name for this attribute.
/// </summary>
public string Name
{
get
{
return m_name;
}

set
{
m_name = value;
}
}

/// <summary>
/// The value for this attribute.
/// </summary>
public string Value
{
get
{
return m_value;
}

set
{
m_value = value;
}
}

#region ICloneable Members
public virtual object Clone() {
return new Attribute(m_name, m_value, m_delim);
}
#endregion
}
}