hadoop的c++版wordcount例子(streaming方式)

0.数据文件

我爱你
首都
北京
我爱你
北京
我爱你
伟大首都
北京
首都
我爱java
come
go

1.map
#include <iostream>
using namespace std;
void map(){
string line;
getline(cin,line);
while(!cin.eof()){
cout << line <<
"\t" << "1" <<endl;
getline(cin,line);
}

}
int main(int argc,char** argv){
map();
}

2.reduce
#include <iostream>
#include <vector>
using namespace std;
//自个写的string分割split方法，无奈cpp官方没提供
vector<string> split(const string& src, const string& separator)
{
vector<string> dest;
string str = src;
string substring;
string::size_type start = 0, index;
do
{
index = str.find_first_of(separator,start);
if (index != string::npos)
{
substring = str.substr(start,index-start);
dest.push_back(substring);
start = str.find_first_not_of(separator,index);
if (start == string::npos) return dest;
}
}while(index != string::npos);
substring = str.substr(start);
dest.push_back(substring);
return dest;
}

void reduce(){
string last_word = ""; //用作reduce的sort后的分界
string line;
getline(cin,line);
int word_num = 0;
while(!cin.eof()){
try{
vector<string> all = split(line,"\t");
string word = all[0];
//first time last_word is ""
if("" == last_word){
last_word = word;
word_num = 0;
}
//repeat word occurs
if(word == last_word){
word_num++;
}
//not equal current word,next word
else{
cout << last_word << "\t" << word_num <<endl;
word_num = 1;
last_word = word;
}
getline(cin,line);
}
catch(const exception& e){
cerr << e.what() <<endl;
}
}
cout << last_word << "\t" << word_num <<endl;
}

int main(int argc,char** argv){
reduce();
}

3.启动命令
hadoop fs -rmr /output-data

hadoop jar /home/machen/hadoop/hadoop-1.0.3/contrib/streaming/hadoop-streaming-1.0.3.jar -file /home/machen/hadoop/hadoop-1.0.3/WordCount/python/stream_map.out -file /home/machen/hadoop/hadoop-1.0.3/WordCount/python/stream_reduce.out -mapper /home/machen/hadoop/hadoop-1.0.3/WordCount/python/stream_map.out -reducer /home/machen/hadoop/hadoop-1.0.3/WordCount/python/stream_reduce.out -input /input-data -output /output-data