网页爬虫WebCrawler（1）-Http网页内容抓取

在windows下的C++通过Http协议实现对网页的内容抓取：

    首先介绍下两个重要的包（一般是在linux下的开源数据包，在windows下则调用其动态链接库dll）：curl包和pthreads_dll,其中curl包解释为命令行浏览器，通过调用内置的curl_easy_setopt等函数即可实现特定的网页内容获取（正确的编译导入的curl链接库，还需要另外一个包C-ares）。pthreads是多线程控制包，当中包含了互斥变量加锁和解锁。程序进程分配等函数。

下载地址：点击打开链接。其中要正确的导入外接动态链接库，需要步骤：1，项目->属性->配置属性->C/C++->常规->附加包含目录（添加include的路径）,2，项目->属性->配置属性->连接器->常规->附加库目录（添加lib包含的路径）；3，在链接器->输入->附加依赖项（libcurld.lib ；pthreadVC2.lib；ws2_32.lib；winmm.lib；wldap32.lib；areslib.lib添加）4，在c/c++->预处理器->预处理器定义（_CONSOLE；BUILDING_LIBCURL；HTTP_ONLY）

    具体实现过程介绍：

1：自定义hashTable结构，用以存储获取的string字符。以hashTable类的形式实现，包含hash表set类型，以及add、find和几种常见的string哈希方式函数

Code:

///HashTable.h
#ifndef HashTable_H
#define HashTable_H

#include <set>
#include <string>
#include <vector>

class HashTable
{
public:
HashTable(void);
~HashTable(void);
unsigned int ForceAdd(const std::string& str);
unsigned int Find(const std::string& str);

/*string的常见的hash方式*/
unsigned int RSHash(const std::string& str);
unsigned int JSHash  (const std::string& str);
unsigned int PJWHash (const std::string& str);
unsigned int ELFHash (const std::string& str);
unsigned int BKDRHash(const std::string& str);
unsigned int SDBMHash(const std::string& str);
unsigned int DJBHash (const std::string& str);
unsigned int DEKHash (const std::string& str);
unsigned int BPHash  (const std::string& str);
unsigned int FNVHash (const std::string& str);
unsigned int APHash  (const std::string& str);

private:
std::set<unsigned int> HashFunctionResultSet;
std::vector<unsigned int> hhh;
};
#endif

/////HashTable.cpp
#include "HashTable.h"

HashTable::HashTable(void)
{
}

HashTable::~HashTable(void)
{
}

unsigned int HashTable::ForceAdd(const std::string& str)
{
unsigned int i=ELFHash(str);
HashFunctionResultSet.insert(i);
return i;
}

unsigned int HashTable::Find(const std::string& str)
{
int ff=hhh.size();
const unsigned int i=ELFHash(str);
std::set<unsigned int>::const_iterator it;
if(HashFunctionResultSet.size()>0)
{
it=HashFunctionResultSet.find(i);
if(it==HashFunctionResultSet.end())
return -1;
}
else
{
return -1;
}
return i;
}

/*几种常见的字符串hash方式实现函数*/
unsigned int HashTable::APHash(const std::string& str)
{
unsigned int hash=0xAAAAAAAA;
for(std::size_t i=0;i<str.length();i++)
{
hash^=((i & 1) == 0) ? (  (hash <<  7) ^ str[i] * (hash >> 3)) :
(~((hash << 11) + str[i] ^ (hash >> 5)));
}
return hash;
}

unsigned int HashTable::BKDRHash(const std::string& str)
{
unsigned int seed=131;   //31 131 1313 13131 131313 etc
unsigned int hash=0;
for(std::size_t i=0;i<str.length();i++)
{
hash=(hash*seed)+str[i];
}
return hash;
}

unsigned int HashTable::BPHash(const std::string& str)
{
unsigned int hash = 0;
for(std::size_t i = 0; i < str.length(); i++)
{
hash = hash << 7 ^ str[i];
}
return hash;
}

unsigned int HashTable::DEKHash(const std::string& str)
{
unsigned int hash = static_cast<unsigned int>(str.length());
for(std::size_t i = 0; i < str.length(); i++)
{
hash = ((hash << 5) ^ (hash >> 27)) ^ str[i];
}
return hash;
}

unsigned int HashTable::DJBHash(const std::string& str)
{
unsigned int hash = 5381;

for(std::size_t i = 0; i < str.length(); i++)
{
hash = ((hash << 5) + hash) + str[i];
}
return hash;
}

unsigned int HashTable::ELFHash(const std::string& str)
{
unsigned int hash=0;
unsigned int x=0;
for(std::size_t i = 0; i < str.length(); i++)
{
hash=(hash<<4)+str[i];
if((x = hash & 0xF0000000L) != 0)
hash^=(x>>24);
hash&=~x;
}
return hash;
}

unsigned int HashTable::FNVHash(const std::string& str)
{
const unsigned int fnv_prime = 0x811C9DC5;
unsigned int hash = 0;
for(std::size_t i = 0; i < str.length(); i++)
{
hash *= fnv_prime;
hash ^= str[i];
}
return hash;
}

unsigned int HashTable::JSHash(const std::string& str)
{
unsigned int hash = 1315423911;
for(std::size_t i = 0; i < str.length(); i++)
{
hash ^= ((hash << 5) + str[i] + (hash >> 2));
}
return hash;
}

unsigned int HashTable::PJWHash(const std::string& str)
{
unsigned int BitsInUnsignedInt = (unsigned int)(sizeof(unsigned int) * 8);
unsigned int ThreeQuarters     = (unsigned int)((BitsInUnsignedInt  * 3) / 4);
unsigned int OneEighth         = (unsigned int)(BitsInUnsignedInt / 8);
unsigned int HighBits          = (unsigned int)(0xFFFFFFFF) << (BitsInUnsignedInt - OneEighth);
unsigned int hash              = 0;
unsigned int test              = 0;

for(std::size_t i = 0; i < str.length(); i++)
{
hash = (hash << OneEighth) + str[i];
if((test = hash & HighBits)  != 0)
hash = (( hash ^ (test >> ThreeQuarters)) & (~HighBits));
}
return hash;
}

unsigned int HashTable::RSHash(const std::string& str)
{
unsigned int b    = 378551;
unsigned int a    = 63689;
unsigned int hash = 0;

for(std::size_t i = 0; i < str.length(); i++)
{
hash = hash * a + str[i];
a    = a * b;
}
return hash;
}

unsigned int HashTable::SDBMHash(const std::string& str)
{
unsigned int hash = 0;
for(std::size_t i = 0; i < str.length(); i++)
{
hash = str[i] + (hash << 6) + (hash << 16) - hash;
}
return hash;
}

2：实现进程间的互斥处理函数(另外提供进行当前操作的进程ID，以便加锁机制)。以SingleTone类实现。该类只能有静态函数Instance建立一个唯一的类对象。以互斥的方式实现对hashTable的基本操作，当中的变量加锁和解锁有mutex类来实现，具体参见代码：

////mutex.h
#ifndef mutex_H
#define mutex_H

#pragma once

#include "pthread.h"

class mutex
{
pthread_mutex_t& m_mutex;
public:
mutex(pthread_mutex_t& m):m_mutex(m)
{
pthread_mutex_lock(&m_mutex);
}

~mutex(void)
{
pthread_mutex_unlock(&m_mutex);
}
};
#endif

 

////SingleTone.h
#ifndef SingleTone_H
#define SingleTone_H

#include <string>
#include <list>
#include <map>
#include "Constants.h"
#include "HashTable.h"
#include "pthread.h"
#include "curl/curl.h"

class SingleTone{

public:
static SingleTone* Instance();
void push_back(std::string s);
void pop_back();
int size();
std::list<std::string>::reference back();
std::list<std::string>::iterator begin();
std::list<std::string>::iterator end();
void push_front(std::string s);
bool empty();

unsigned int Get_m_UniqueMap_ForceAdd(const std::string& key,const std::string& url);
unsigned int Get_m_Unique
4000
Map_Find(const std::string& key,const std::string& url);
HashTable Get_m_UniqueMap(const std::string& key);
void Set_m_UniqueMap(const std::string& key,HashTable& hash);
CURL* GetpCurl();

protected:
SingleTone();
~SingleTone();
pthread_mutex_t m_singleton_mutex;

private:
static SingleTone* m_pSingleTone;
std::list<std::string> m_LinkStack;
std::map<std::string,HashTable> m_UniqueMap;
CURL *m_pcurl;
};
#endif

#include "SingleTone.h"
#include "mutex.h"

SingleTone* SingleTone::m_pSingleTone=NULL;

SingleTone::SingleTone()
{
pthread_mutex_init(&m_singleton_mutex,NULL);
m_pcurl=curl_easy_init();
}

SingleTone::~SingleTone()
{
pthread_mutex_destroy(&m_singleton_mutex);
}

SingleTone* SingleTone::Instance()
{
if(m_pSingleTone==NULL){
m_pSingleTone=new SingleTone();
}
return (m_pSingleTone);
}

void SingleTone::push_back(std::string s)
{
mutex m(m_singleton_mutex);
return m_LinkStack.push_back(s);
}

void SingleTone::pop_back()
{
mutex m(m_singleton_mutex);
return m_LinkStack.pop_back();
}

int SingleTone::size()
{
return m_LinkStack.size();
}

std::list<std::string>::iterator SingleTone::begin()
{
return m_LinkStack.begin();
}

std::list<std::string>::reference SingleTone::back()
{
mutex m(m_singleton_mutex);
return m_LinkStack.back();
}

std::list<std::string>::iterator SingleTone::end()
{
return m_LinkStack.end();
}

void SingleTone::push_front(std::string s)
{
mutex  m(m_singleton_mutex);
return m_LinkStack.push_front(s);
}

bool SingleTone::empty()
{
return m_LinkStack.empty();
}

unsigned int SingleTone::Get_m_UniqueMap_ForceAdd(const std::string& key,const std::string& url)
{
mutex  m(m_singleton_mutex);
return m_UniqueMap[key].ForceAdd(url);
}

unsigned int SingleTone::Get_m_UniqueMap_Find(const std::string& key,const std::string& url)
{

HashTable hss = m_UniqueMap[key];
unsigned int uiRet =hss.Find(url);
//unsigned int uiRet = m_UniqueMap[key]->Find(url);
return uiRet;
}

HashTable SingleTone::Get_m_UniqueMap(const std::string& key)
{
return m_UniqueMap[key];
}

void SingleTone::Set_m_UniqueMap(const std::string& key,HashTable& hash)
{
m_UniqueMap[key] = hash;

}

CURL* SingleTone::GetpCurl()
{
return m_pcurl;
}

3:实现HTTP对网页内容的获取：功能包含初始网页内容的获取，和URL设置等函数。这个过程要求是互斥的，所以引入SingleTone类的内容。

Code:

/////Http.h
#ifndef Http_H
#define Http_H

#include "curl/curl.h"
#include "pthread.h"
#include <string>

using namespace std;

class Http
{
public:
Http(void);
~Http(void);
bool InitCurl(void);
bool InitCurl(const std::string& url, std::string& szbuffer);
bool DeInitCurl();
void setUrl(const std::string& url);
string setUrl();
const string getBuffer();

private:
static void writer(void* buffer,size_t size,size_t nmemb,void* f);
int setBuffer(char* buffer,size_t size,size_t nmemb);
CURL *m_pcurl;
char m_errorBuffer[CURL_ERROR_SIZE];
string m_szbuffer;
string m_szUrl;
pthread_mutex_t m_http_mutex;
};
#endif

#include "Http.h"
#include "SingleTone.h"
#include "mutex.h"

Http::Http(void)
{
m_pcurl=SingleTone::Instance()->GetpCurl();
}

Http::~Http(void)
{
}

bool Http::InitCurl(void)
{
return false;
}

int Http::setBuffer(char *buffer, size_t size, size_t nmemb)
{
int result = 0;
if (buffer!=NULL)
{
m_szbuffer.append(buffer, size * nmemb);
result = size * nmemb;
}
buffer = NULL ;
return result;
}

void Http::writer(void *buffer, size_t size, size_t nmemb,void* f)
{
static_cast<Http*>(f)->setBuffer((char*)buffer,size,nmemb);
}

bool Http::InitCurl(const std::string& url, std::string& szbuffer)
{
pthread_mutex_init(&m_http_mutex,NULL);
Http::m_szUrl=url;
CURLcode result;
if(m_pcurl)
{
curl_easy_setopt(m_pcurl, CURLOPT_ERRORBUFFER, Http::m_errorBuffer);
curl_easy_setopt(m_pcurl, CURLOPT_URL,m_szUrl.c_str());
curl_easy_setopt(m_pcurl, CURLOPT_HEADER, 0);
curl_easy_setopt(m_pcurl, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(m_pcurl, CURLOPT_WRITEFUNCTION,Http::writer);
curl_easy_setopt(m_pcurl, CURLOPT_WRITEDATA,this);

result = curl_easy_perform(m_pcurl);
}
if(result!=CURLE_OK)
return false;
szbuffer=m_szbuffer;
m_szbuffer.clear();
m_szUrl.clear();
pthread_mutex_destroy(&m_http_mutex);
return true;
}

bool Http::DeInitCurl()
{
curl_easy_cleanup(m_pcurl);
curl_global_cleanup();
m_pcurl = NULL;

return true;
}

const string Http::getBuffer()
{
return m_szbuffer;
}

string Http::setUrl()
{
return Http::m_szUrl;
}

void Http::setUrl(const std::string& url)
{
Http::m_szUrl = url;
}

其中 m_szbuffer存放网页的内容。初始网页的内容存放在Init函数的形参。