好友推荐—基于关系的java和spark代码实现

本文主要实现基于二度好友的推荐。
数学公式参考于：http://blog.csdn.net/qq_14950717/article/details/52197565
测试数据为自己随手画的关系图



把图片整理成文本信息如下：a b c d e f y
b c a f g
c a b d
d c a e h q r
e f h d a
f e a b g
g h f b
h e g i d
i j m n q h
j i k l
k j
l j
m i
n i o
o n p
p o q
q i d
r d s t w v
s r t u
t r s u
u s t
v r w x
w r v x
x w v
y a z
z y每行的第一个为用户，之后的为其好友。
在计算用户之间的推荐分上参考上面博客中的两个数学公式分为两个推荐分。
一个公式为


上面是每个共同好友默认的提供1分，但是在生活中如果次共同好友的好友数比较少，则说明次好友可能更加重要，所以有了下面对每个共同好友加权重的公式



如果好友数差距过大，要对好友数进行开方或对数之类的，这就是要看产品方面对我们的要求了。



说的比较简单，详细理论可以看上面链接的博客。
下面是单机版java的代码：
选创建一个来描述好友之间关系的类import java.util.List;

public class Score {
//A和B的所有好友数
private int Union;
//A和B的共同好友数
private int Intersection;
//A和B的共同好友列表
private List<Character> l;
//此对象是A和B之间的关系
private char A;
private char B;
//没加权重的得分
private float score;
//对每个共同好友加权重的得分
private double wscore;
public double getWscore() {
return wscore;
}
public void setWscore(double wscore) {
this.wscore = wscore;
}
public float getScore() {
return score;
}
public char getA() {
return A;
}
public void setA(char a) {
A = a;
}
public char getB() {
return B;
}
public void setB(char b) {
B = b;
}
public int getUnion() {
return Union;
}
public void setUnion(int union) {
Union = union;
if(Union!=0){
score=(float) Intersection/Union;
}
}
public int getIntersection() {
return Intersection;
}
public void setIntersection(int intersection) {
Intersection = intersection;
if(Union!=0){
score=(float) Intersection/Union;
}
}
public List getL() {
return l;
}
public void setL(List l) {
this.l = l;
}
@Override
public String toString() {
return "Score{" +
"Union=" + Union +
", Intersection=" + Intersection +
", l=" + l +
", A=" + A +
", B=" + B +
", score=" + score +
", wscore="+wscore+
'}';
}
}然后写代码对数据进行操作：import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class fof {
public static void main(String[] args) throws Exception {
BufferedReader bufferedReader=new BufferedReader(new InputStreamReader(new FileInputStream("F:\\friends.txt")));
List<Score> scores=new ArrayList<Score>();
String s=null;
List<char[]> l=new ArrayList<char[]>();
Map<Character,Integer> map=new HashMap<Character, Integer>();
while((s=bufferedReader.readLine())!=null){
char[] c = s.replace(" ", "").toCharArray();
l.add(c);
map.put(c[0],c.length-1);
}
for(int i=0;i<l.size();i++){
for(int j=0;j<l.size();j++){
if(i!=j){
Score score = aaa(l.get(i), l.get(j));
if(score!=null){
List<Character> l1 = score.getL();
double wscore=0.0;
for(char c:l1){
double sqrt = Math.sqrt(map.get(c));
double v = 1 / sqrt;
wscore+=v;
}
double v = wscore / score.getUnion();//得出加权的分数
score.setWscore(v);
scores.add(score);

}
}
}
}
for(Score score:scores){
System.out.println(score);
}
}
//此方法得出两个用户之间的关系类
public static Score aaa(char[] a,char[] b){
char x=a[0];
//直接好友返回null
for(char y:b){
if(x==y){
return null;
}
}
List<Character> l=new ArrayList<Character>();

for(int i=1;i<a.length;i++){
for(int j=1;j<b.length;j++){
if(a[i]==b[j]){
l.add(a[i]);
}
}
}
if(l.size()==0){
return null;//没有共同好友返回null
}
Score score = new Score();
score.setL(l);
score.setA(a[0]);
score.setB(b[0]);
score.setIntersection(l.size());
score.setUnion(a.length+b.length-l.size()-2);
return score;
}
}在运行后得出来的数据：Score{Union=7, Intersection=2, l=[b, f], A=a, B=g, score=0.2857143, wscore=0.14285714285714285}
Score{Union=8, Intersection=2, l=[d, e], A=a, B=h, score=0.25, wscore=0.11353103630798289}
Score{Union=7, Intersection=1, l=[d], A=a, B=q, score=0.14285715, wscore=0.058321184351980436}
Score{Union=10, Intersection=1, l=[d], A=a, B=r, score=0.1, wscore=0.040824829046386304}
Score{Union=6, Intersection=1, l=[y], A=a, B=z, score=0.16666667, wscore=0.1178511301977579}
Score{Union=8, Intersection=2, l=[c, a], A=b, B=d, score=0.25, wscore=0.12319981995668611}
Score{Union=6, Intersection=2, l=[a, f], A=b, B=e, score=0.33333334, wscore=0.1513747150773105}
Score{Union=7, Intersection=1, l=[g], A=b, B=h, score=0.14285715, wscore=0.08247860988423227}
Score{Union=5, Intersection=1, l=[a], A=b, B=y, score=0.2, wscore=0.08164965809277261}
Score{Union=5, Intersection=2, l=[a, d], A=c, B=e, score=0.4, wscore=0.16329931618554522}
Score{Union=5, Intersection=2, l=[a, b], A=c, B=f, score=0.4, wscore=0.18164965809277261}
Score{Union=5, Intersection=1, l=[b], A=c, B=g, score=0.2, wscore=0.1}
Score{Union=6, Intersection=1, l=[d], A=c, B=h, score=0.16666667, wscore=0.06804138174397718}
Score{Union=4, Intersection=1, l=[d], A=c, B=q, score=0.25, wscore=0.10206207261596577}
Score{Union=7, Intersection=1, l=[d], A=c, B=r, score=0.14285715, wscore=0.058321184351980436}
Score{Union=4, Intersection=1, l=[a], A=c, B=y, score=0.25, wscore=0.10206207261596577}
Score{Union=8, Intersection=2, l=[c, a], A=d, B=b, score=0.25, wscore=0.12319981995668611}
Score{Union=8, Intersection=2, l=[a, e], A=d, B=f, score=0.25, wscore=0.11353103630798289}
Score{Union=8, Intersection=1, l=[h], A=d, B=g, score=0.125, wscore=0.0625}
Score{Union=9, Intersection=2, l=[h, q], A=d, B=i, score=0.22222222, wscore=0.13412297568739417}
Score{Union=7, Intersection=1, l=[q], A=d, B=p, score=0.14285715, wscore=0.10101525445522107}
Score{Union=8, Intersection=1, l=[r], A=d, B=s, score=0.125, wscore=0.05590169943749474}
Score{Union=8, Intersection=1, l=[r], A=d, B=t, score=0.125, wscore=0.05590169943749474}
Score{Union=8, Intersection=1, l=[r], A=d, B=v, score=0.125, wscore=0.05590169943749474}
Score{Union=8, Intersection=1, l=[r], A=d, B=w, score=0.125, wscore=0.05590169943749474}
Score{Union=7, Intersection=1, l=[a], A=d, B=y, score=0.14285715, wscore=0.058321184351980436}
Score{Union=6, Intersection=2, l=[f, a], A=e, B=b, score=0.33333334, wscore=0.1513747150773105}
Score{Union=5, Intersection=2, l=[d, a], A=e, B=c, score=0.4, wscore=0.16329931618554522}
Score{Union=5, Intersection=2, l=[f, h], A=e, B=g, score=0.4, wscore=0.2}
Score{Union=8, Intersection=1, l=[h], A=e, B=i, score=0.125, wscore=0.0625}
Score{Union=5, Intersection=1, l=[d], A=e, B=q, score=0.2, wscore=0.08164965809277261}
Score{Union=8, Intersection=1, l=[d], A=e, B=r, score=0.125, wscore=0.051031036307982884}
Score{Union=5, Intersection=1, l=[a], A=e, B=y, score=0.2, wscore=0.08164965809277261}
Score{Union=5, Intersection=2, l=[a, b], A=f, B=c, score=0.4, wscore=0.18164965809277261}
Score{Union=8, Intersection=2, l=[e, a], A=f, B=d, score=0.25, wscore=0.11353103630798289}
Score{Union=6, Intersection=2, l=[e, g], A=f, B=h, score=0.33333334, wscore=0.17955837819827095}
Score{Union=5, Intersection=1, l=[a], A=f, B=y, score=0.2, wscore=0.08164965809277261}
Score{Union=7, Intersection=2, l=[f, b], A=g, B=a, score=0.2857143, wscore=0.14285714285714285}
Score{Union=5, Intersection=1, l=[b], A=g, B=c, score=0.2, wscore=0.1}
Score{Union=8, Intersection=1, l=[h], A=g, B=d, score=0.125, wscore=0.0625}
Score{Union=5, Intersection=2, l=[h, f], A=g, B=e, score=0.4, wscore=0.2}
Score{Union=7, Intersection=1, l=[h], A=g, B=i, score=0.14285715, wscore=0.07142857142857142}
Score{Union=8, Intersection=2, l=[e, d], A=h, B=a, score=0.25, wscore=0.11353103630798289}
Score{Union=7, Intersection=1, l=[g], A=h, B=b, score=0.14285715, wscore=0.08247860988423227}
Score{Union=6, Intersection=1, l=[d], A=h, B=c, score=0.16666667, wscore=0.06804138174397718}
Score{Union=6, Intersection=2, l=[e, g], A=h, B=f, score=0.33333334, wscore=0.17955837819827095}
Score{Union=6, Intersection=1, l=[i], A=h, B=j, score=0.16666667, wscore=0.07453559924999299}
Score{Union=4, Intersection=1, l=[i], A=h, B=m, score=0.25, wscore=0.11180339887498948}
Score{Union=5, Intersection=1, l=[i], A=h, B=n, score=0.2, wscore=0.08944271909999159}
Score{Union=4, Intersection=2, l=[i, d], A=h, B=q, score=0.5, wscore=0.21386547149095525}
Score{Union=8, Intersection=1, l=[d], A=h, B=r, score=0.125, wscore=0.051031036307982884}
Score{Union=9, Intersection=2, l=[q, h], A=i, B=d, score=0.22222222, wscore=0.13412297568739417}
Score{Union=8, Intersection=1, l=[h], A=i, B=e, score=0.125, wscore=0.0625}
Score{Union=7, Intersection=1, l=[h], A=i, B=g, score=0.14285715, wscore=0.07142857142857142}
Score{Union=5, Intersection=1, l=[j], A=i, B=k, score=0.2, wscore=0.11547005383792516}
Score{Union=5, Intersection=1, l=[j], A=i, B=l, score=0.2, wscore=0.11547005383792516}
Score{Union=6, Intersection=1, l=
, A=i, B=o, score=0.16666667, wscore=0.1178511301977579}
Score{Union=6, Intersection=1, l=[q], A=i, B=p, score=0.16666667, wscore=0.1178511301977579}
Score{Union=6, Intersection=1, l=[i], A=j, B=h, score=0.16666667, wscore=0.07453559924999299}
Score{Union=3, Intersection=1, l=[i], A=j, B=m, score=0.33333334, wscore=0.14907119849998599}
Score{Union=4, Intersection=1, l=[i], A=j, B=n, score=0.25, wscore=0.11180339887498948}
Score{Union=4, Intersection=1, l=[i], A=j, B=q, score=0.25, wscore=0.11180339887498948}
Score{Union=5, Intersection=1, l=[j], A=k, B=i, score=0.2, wscore=0.11547005383792516}
Score{Union=1, Intersection=1, l=[j], A=k, B=l, score=1.0, wscore=0.5773502691896258}
Score{Union=5, Intersection=1, l=[j], A=l, B=i, score=0.2, wscore=0.11547005383792516}
Score{Union=1, Intersection=1, l=[j], A=l, B=k, score=1.0, wscore=0.5773502691896258}
Score{Union=4, Intersection=1, l=[i], A=m, B=h, score=0.25, wscore=0.11180339887498948}
Score{Union=3, Intersection=1, l=[i], A=m, B=j, score=0.33333334, wscore=0.14907119849998599}
Score{Union=2, Intersection=1, l=[i], A=m, B=n, score=0.5, wscore=0.22360679774997896}
Score{Union=2, Intersection=1, l=[i], A=m, B=q, score=0.5, wscore=0.22360679774997896}
Score{Union=5, Intersection=1, l=[i], A=n, B=h, score=0.2, wscore=0.08944271909999159}
Score{Union=4, Intersection=1, l=[i], A=n, B=j, score=0.25, wscore=0.11180339887498948}
Score{Union=2, Intersection=1, l=[i], A=n, B=m, score=0.5, wscore=0.22360679774997896}
Score{Union=3, Intersection=1, l=[o], A=n, B=p, score=0.33333334, wscore=0.2357022603955158}
Score{Union=3, Intersection=1, l=[i], A=n, B=q, score=0.33333334, wscore=0.14907119849998599}
Score{Union=6, Intersection=1, l=
, A=o, B=i, score=0.16666667, wscore=0.1178511301977579}
Score{Union=7, Intersection=1, l=[q], A=p, B=d, score=0.14285715, wscore=0.10101525445522107}
Score{Union=6, Intersection=1, l=[q], A=p, B=i, score=0.16666667, wscore=0.1178511301977579}
Score{Union=3, Intersection=1, l=[o], A=p, B=n, score=0.33333334, wscore=0.2357022603955158}
Score{Union=7, Intersection=1, l=[d], A=q, B=a, score=0.14285715, wscore=0.058321184351980436}
Score{Union=4, Intersection=1, l=[d], A=q, B=c, score=0.25, wscore=0.10206207261596577}
Score{Union=5, Intersection=1, l=[d], A=q, B=e, score=0.2, wscore=0.08164965809277261}
Score{Union=4, Intersection=2, l=[i, d], A=q, B=h, score=0.5, wscore=0.21386547149095525}
Score{Union=4, Intersection=1, l=[i], A=q, B=j, score=0.25, wscore=0.11180339887498948}
Score{Union=2, Intersection=1, l=[i], A=q, B=m, score=0.5, wscore=0.22360679774997896}
Score{Union=3, Intersection=1, l=[i], A=q, B=n, score=0.33333334, wscore=0.14907119849998599}
Score{Union=6, Intersection=1, l=[d], A=q, B=r, score=0.16666667, wscore=0.06804138174397718}
Score{Union=10, Intersection=1, l=[d], A=r, B=a, score=0.1, wscore=0.040824829046386304}
Score{Union=7, Intersection=1, l=[d], A=r, B=c, score=0.14285715, wscore=0.058321184351980436}
Score{Union=8, Intersection=1, l=[d], A=r, B=e, score=0.125, wscore=0.051031036307982884}
Score{Union=8, Intersection=1, l=[d], A=r, B=h, score=0.125, wscore=0.051031036307982884}
Score{Union=6, Intersection=1, l=[d], A=r, B=q, score=0.16666667, wscore=0.06804138174397718}
Score{Union=5, Intersection=2, l=[s, t], A=r, B=u, score=0.4, wscore=0.23094010767585033}
Score{Union=5, Intersection=2, l=[w, v], A=r, B=x, score=0.4, wscore=0.23094010767585033}
Score{Union=8, Intersection=1, l=[r], A=s, B=d, score=0.125, wscore=0.05590169943749474}
Score{Union=5, Intersection=1, l=[r], A=s, B=v, score=0.2, wscore=0.08944271909999159}
Score{Union=5, Intersection=1, l=[r], A=s, B=w, score=0.2, wscore=0.08944271909999159}
Score{Union=8, Intersection=1, l=[r], A=t, B=d, score=0.125, wscore=0.05590169943749474}
Score{Union=5, Intersection=1, l=[r], A=t, B=v, score=0.2, wscore=0.08944271909999159}
Score{Union=5, Intersection=1, l=[r], A=t, B=w, score=0.2, wscore=0.08944271909999159}
Score{Union=5, Intersection=2, l=[s, t], A=u, B=r, score=0.4, wscore=0.23094010767585033}
Score{Union=8, Intersection=1, l=[r], A=v, B=d, score=0.125, wscore=0.05590169943749474}
Score{Union=5, Intersection=1, l=[r], A=v, B=s, score=0.2, wscore=0.08944271909999159}
Score{Union=5, Intersection=1, l=[r], A=v, B=t, score=0.2, wscore=0.08944271909999159}
Score{Union=8, Intersection=1, l=[r], A=w, B=d, score=0.125, wscore=0.05590169943749474}
Score{Union=5, Intersection=1, l=[r], A=w, B=s, score=0.2, wscore=0.08944271909999159}
Score{Union=5, Intersection=1, l=[r], A=w, B=t, score=0.2, wscore=0.08944271909999159}
Score{Union=5, Intersection=2, l=[w, v], A=x, B=r, score=0.4, wscore=0.23094010767585033}
Score{Union=5, Intersection=1, l=[a], A=y, B=b, score=0.2, wscore=0.08164965809277261}
Score{Union=4, Intersection=1, l=[a], A=y, B=c, score=0.25, wscore=0.10206207261596577}
Score{Union=7, Intersection=1, l=[a], A=y, B=d, score=0.14285715, wscore=0.058321184351980436}
Score{Union=5, Intersection=1, l=[a], A=y, B=e, score=0.2, wscore=0.08164965809277261}
Score{Union=5, Intersection=1, l=[a], A=y, B=f, score=0.2, wscore=0.08164965809277261}
Score{Union=6, Intersection=1, l=[y], A=z, B=a, score=0.16666667, wscore=0.1178511301977579}根据个人需求在根据得分进行排序就好了。
下面是在spark中的代码。
spark没有计算加权的分数（懒得写了），用的推荐分类还是上面的同一个Score类。import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function;
import scala.Tuple2;

import java.util.LinkedList;

public class Sparkfof {

public static void main(String[] args){
JavaSparkContext sparkContext = new JavaSparkContext(new SparkConf().setAppName("fof"));
JavaRDD<String> stringJavaRDD = sparkContext.textFile(args[0]);
JavaRDD<char[]> map = stringJavaRDD.map(new Function<String, char[]>() {
@Override
public char[] call(String s) throws Exception {
return s.replace(" ", "").toCharArray();
}
});
JavaPairRDD<char[], char[]> cartesian = map.cartesian(map);
JavaRDD<Score> scoreJavaRDD = cartesian.flatMap(new FlatMapFunction<Tuple2<char[], char[]>, Score>() {
@Override
public Iterable<Score> call(Tuple2<char[], char[]> tuple2) throws Exception {
Score aaa = fof.aaa(tuple2._1, tuple2._2);
LinkedList<Score> scores = new LinkedList<>();
scores.add(aaa);
return scores;
}
});
JavaRDD<Score> filter = scoreJavaRDD.filter(new Function<Score, Boolean>() {
@Override
public Boolean call(Score score) throws Exception {

return score != null;
}
});

filter.saveAsTextFile("/tmp/spark-tmp/fof");

}

}然后把程序打包放到spark机器上运行。
去hdfs中读取跑完的数据。hadoop fs -cat /tmp/spark-tmp/fof/part-00000得出数据如下和单机java数据一样。



由于自己刚接触spark，请各位老师指点！