基于字的文本相似度算法——Jacard算法

一、算法原理

基于字的文本相似度Jacard 算法的原理是：

（1）计算两个文本中字的交集

（2）计算两个文本中字的并集

（3）交集内的字的个数除以并集内的字的个数即为文本相似度值

（4)根据设置的阈值判断是否相似

二、算法的C++实现

这里引用的StringUtil.hpp文件引自：
https://github.com/yanyiwu/cppjieba/blob/master/deps/limonp/StringUtil.hpp
/*
* JaccardSimilarity.hpp
*
* Created: 2016年10月2日
* Author: tang
*/

#ifndef SRC_JACCARD_SIMILARITY_HPP_
#define SRC_JACCARD_SIMILARITY_HPP_
#include <algorithm>
#include <iostream>
#include <vector>
#include <set>
#include "StringUtil.hpp"

using namespace std;

class JaccardSimilarity
{
public:

JaccardSimilarity()
{
}

double CalculateTextSimilarity(string &str1,string &str2)
{
vector<uint16_t> words_for_str1;
vector<uint16_t> words_for_str2;
vector<uint16_t>::iterator it;

if(!utf8ToUnicode< vector<uint16_t> >(str1,words_for_str1) ||
!utf8ToUnicode< vector<uint16_t> >(str2,words_for_str2 ) )
{
cout<<"TransCode Error"<<endl;
return 0.;
}

for(it=words_for_str1.begin();it!=words_for_str1.end();)
{
if(codeFilter(*it))
{
++it;
}
else
{
it=words_for_str1.erase(it);
}
}

for(it=words_for_str2.begin();it!=words_for_str2.end();)
{
if(codeFilter(*it))
{
++it;
}
else
{
it=words_for_str2.erase(it);
}
}

if(words_for_str1.size()+words_for_str2.size()<1)
return 1.;

vector<uint16_t> words_intersection;
vector<uint16_t> words_union;
std::sort(words_for_str1.begin(),words_for_str1.end());
std::sort(words_for_str2.begin(),words_for_str2.end());
std::set_intersection(words_for_str1.begin(),words_for_str1.end(),
words_for_str2.begin(),words_for_str2.end(),
std::inserter(words_intersection,words_intersection.begin()));

std::set_union(words_for_str1.begin(),words_for_str1.end(),
words_for_str2.begin(),words_for_str2.end(),
std::inserter(words_union,words_union.begin()));

double inter=words_intersection.size();
double wunion=words_union.size();

return inter/wunion;
}

bool codeFilter(int code)
{
if ((code < 0x4e00 || code > 0x9fa5) &&
!(code >= '0' && code <= '9') &&
!(code >= 'a' && code <= 'z') &&
!(code >= 'A' && code <= 'Z'))
return false;

return true;
}

};

#endif /* SRC_JACCARD_SIMILARITY_HPP_ */

三、算法的java实现
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

public class JaccardSimilarity{

public JaccardSimilarity() {
}

public boolean codeFilter(int code) {
if ((code < 19968 || code > 40869)
&& !(code >= '0' && code <= '9')
&& !(code >= 'a' && code <= 'z')
&& !(code >= 'A' && code <= 'Z')) {
return false;
}
return true;
}

public double CalculateTextSim(String content, String compareContent) {
if(null == content || null == compareContent)
return 0.0;
Map<String, Integer> cntMap = new HashMap<String, Integer>();
Set<String> cntSet = new HashSet<String>();
Map<String, Integer> cmpCntMap = new HashMap<String, Integer>();
Set<String> cmpCntSet = new HashSet<String>();

for (int i = 0; i != content.length(); i++) {
int k = 0;
if (codeFilter(content.codePointAt(i))) {
if (cntMap.containsKey("" + content.charAt(i))) {
Integer count = cntMap.get("" + content.charAt(i));
count = count + 1;
cntMap.put("" + content.charAt(i), count);
k = count;
} else {
cntMap.put("" + content.charAt(i), new Integer(1));
k = 1;
}
String tmpString = content.charAt(i) + "" + k;
cntSet.add(tmpString);
}
}

for (int i = 0; i != compareContent.length(); i++) {
int k = 0;
if (codeFilter(compareContent.codePointAt(i))) {
if (cmpCntMap.containsKey("" + compareContent.charAt(i))) {
Integer count = cmpCntMap.get("" + compareContent.charAt(i));
count = count + 1;
cmpCntMap.put("" + compareContent.charAt(i), count);
k = count;
} else {
cmpCntMap.put("" + compareContent.charAt(i), new Integer(1));
k = 1;
}

String tmpString = compareContent.charAt(i) + "" + k;
cmpCntSet.add(tmpString);
}
}

Set<String> tmpSet = new HashSet<String>();
tmpSet.addAll(cntSet);
cntSet.retainAll(cmpCntSet);
double intCount = cntSet.size();

tmpSet.addAll(cmpCntSet);

if (tmpSet.size() == 0)
return 0;
double uniCount = tmpSet.size();

return intCount / uniCount;
}

}