爬取京东评论、分词+词频统计、词云图展示

一、爬取京东评论

京东评论竟然全部对外开放

public class CommentCrawler {
final static PoolingHttpClientConnectionManager httpClientConnectionManager = new PoolingHttpClientConnectionManager();
final static int MAX_PAGE = 50;

static HttpClient getClient() {
return HttpClients.custom().setConnectionManager(httpClientConnectionManager).build();
}

static String getUrl(String productId, int page) {
return String.format(
"http://sclub.jd.com/comment/productPageComments.action?productId=%s&score=0&sortType=3&page=%d&pageSize=10",
productId, page);
}

static Comment commentFromJson(JSONObject json, String productId) {
return new Comment(json.getLongValue("id"), productId, json.getString("score"), json.getString("content"));
}

public static boolean crawlComments(String productId) {
try {
int maxPage = 1;
int nowPage = 0;
HttpClient client = getClient();
while (nowPage < maxPage) {
String url = getUrl(productId, nowPage);
HttpGet get = new HttpGet(url);
HttpResponse resp = client.execute(get);
JSONObject json = JSON.parseObject(EntityUtils.toString(resp.getEntity()));
JSONArray comments = json.getJSONArray("comments");
if (comments.size() == 0)
return false;
CommentService ser = new CommentService();
for (int i = 0; i < comments.size(); i++) {
Comment comment = commentFromJson(comments.getJSONObject(i), productId);
ser.insertComment(comment);
}
if (nowPage == 0) {
maxPage = json.getInteger("maxPage");
ser.insertProduct(new Product(productId, comments.getJSONObject(0).getString("referenceName")));
}
nowPage++;
}
ProductJudger.judge(productId);
return true;
} catch (Exception e) {
e.printStackTrace();
}
return false;
}
}

二、结巴分词

jieba分词原本是Python版的，有人把它改成了Java版，名字也改成了jieba-analysis

JiebaSegmenter segmenter = new JiebaSegmenter();
List<Word> list = segmenter.sentenceProcess(str);
for (Word i : list) {
String token = i.getToken();
if (stopWords.contains(token)) {
continue;
}
Integer cnt = map.get(token);
if (cnt == null) {
cnt = 0;
}
map.put(token, new Integer(cnt + 1));
}

三、词云图

用到d3.js，d3-cloud.js这两个库，d3.js是“Data Driven Document”，d3-cloud这个库还是比较难用的，主要是官方实例代码太少了。

这里给出一个例子：每一个词云图都对应一个字典，这个字典就是“词语：频率”这样的键值对。给定多个字典，每一个字典都要渲染成一个词云图。

<div id="word-clouds" style="text-align: center"></div>
<script>
var wordClouds = <%=request.getAttribute("wordClouds")%>;
$(document).ready(function () {
for (var i in wordClouds) {
var it = wordClouds[i];
var divId = "product_wordcloud" + it['productId'];
$("#word-clouds").append("<h3 align='center'>商品" + it['productId'] + "词云图</h3>")
.append("<div id='" + divId + "'></div>");
createWordCloud(transformWordFraquency(it['words']), "#" + divId)
}
})
</script>

还需要编写如下JS代码

var fill = d3.scale.category20();// 20种颜色
var wordCloudWidth = 800, wordCloudHeight = 400;
var font_name = "楷体", font_weight = "bold", max_font_size = 50;
var word_count = 50;// 显示词汇个数
var word_max_size = 60;// 显示词汇字体最大字号
var word_min_size = 10;// 显示词汇字体最小字号
/*
* 函数名称：transformWordFrequency
* 参数words：字典类型，形如“{word1:cnt1,word2:cnt2,word3:cnt3}” 返回值：{text，size}数组
*/

function transformWordFraquency(words) {
var ar = []
for ( var i in words) {
ar.push({
"text" : i,
"size" : words[i]
})
}
// 按照字体的大小从大到小进行排序，只取出现次数较多的前几名
ar.sort(function(x, y) {
return y['size'] - x['size'];
})
ar = ar.slice(0, Math.min(word_count, ar.length));
for (var i = 0; i < ar.length; i++) {
ar[i]['size'] = word_max_size - (word_max_size - word_min_size)
/ ar.length * i;
}
return ar;
}
/*
* wordMap是[{text:"",size:""}]形式的数组 selector是即将渲染到的目标位置
*/
function createWordCloud(wordMap, selector) {
d3.layout.cloud().size(
[ wordCloudWidth * 2 - 100, wordCloudHeight * 2 - 100 ]).words(
wordMap).font(font_name).fontWeight(font_weight).fontSize(
function(d) {
return d.size;
}).rotate(function() {
return 0;
}).on("end", function(words) {
renderWordCloud(words, selector)
}).start();
}
/*
* 像这种风格的函数调用，这些函数的调用顺序不能变
*/
function renderWordCloud(words, selector) {
d3.select(selector).append("svg").attr("width", wordCloudWidth).attr(
"height", wordCloudHeight).append("g")
.attr(
"transform",
"translate(" + wordCloudWidth / 2 + "," + wordCloudHeight
/ 2 + ")").selectAll("text").data(words).enter()// 进入words，相当于for循环
.append("text").style("font-family", font_name).style(
"font-weight", font_weight)
// .attr("text-anchor", "middle")
.style("font-size", function(d) {// 字体大小
return d.size + "px";
}).style("fill", function(d, i) {// 字体颜色
return fill(i);
}).attr(
"transform",
function(d) {
return "translate(" + [ d.x, d.y ] + ") rotate("
+ d.rotate + ")";
}).text(function(d) {
return d.text;
});
}