探花08
文章目录
圈子推荐


圈子和视频积分表
RecommendQuanZi
package com.tanhua.recommend.pojo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.bson.types.ObjectId;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class RecommendQuanZi {
private ObjectId id;
private Long userId;// 用户id
private Long publishId; //动态id,需要转化为Long类型
private Double score; //得分
private Long date; //时间戳
}
RecommendVideo
package com.tanhua.recommend.pojo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.bson.types.ObjectId;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class RecommendVideo {
private ObjectId id;
private Long userId;// 用户id
private Long videoId; //视频id,需要转化为Long类型
private Double score; //得分
private Long date; //时间戳
}
搭建spark系统
paalication.properties
spark.mongodb.input.uri = mongodb://192.168.200.146:27017/tanhua.recommend_quanzi_20191001?readPreference=primaryPreferred
spark.video.mongodb.input.uri = mongodb://192.168.200.146:27017/tanhua.recommend_video_20191010?readPreference=primaryPreferred
spark.mongodb.output.user.uri = mongodb://192.168.200.146:27017/tanhua.recommend_user?readPreference=primaryPreferred
redis.cluster.nodes = 192.168.200.146:6379,192.168.200.146:6380,192.168.200.146:6381
jdbc.driver-class-name=com.mysql.jdbc.Driver
jdbc.url=jdbc:mysql://192.168.200.146:3306/mytanhua?useUnicode=true&characterEncoding=utf8&autoReconnect=true&allowMultiQueries=true&useSSL=false
jdbc.username=root
jdbc.password=123456计算推荐圈子SparkQuanZi
package com.tanhua.spark.mongo;
import com.mongodb.spark.MongoSpark;
import com.mongodb.spark.rdd.api.java.JavaMongoRDD;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.recommendation.MatrixFactorizationModel;
import org.apache.spark.mllib.recommendation.Rating;
import org.apache.spark.sql.sources.In;
import org.bson.Document;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisCluster;
import scala.Tuple2;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
public class SparkQuanZi {
public static void main(String[] args) throws IOException {
//加载外部配置文件
InputStream inputStream = SparkQuanZi.class.getClassLoader().getResourceAsStream("app.properties");
Properties properties = new Properties();
properties.load(inputStream);
//构建spark的配置
SparkConf sparkConf = new SparkConf()
.setAppName("SparkQuanZi")
.setMaster("local[*]")
.set("spark.mongodb.input.uri", properties.getProperty("spark.mongodb.input.uri"));
//获取spark上下文
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
//加载mongodb的数据
JavaMongoRDD<Document> rdd = MongoSpark.load(jsc);
//获取用户列表
JavaRDD<Long> userIdRdd = rdd.map(document -> document.getLong("userId"));
List<Long> userIdList = userIdRdd.distinct().collect();
//打印下数据
/**
* {
* "_id": {
* "$oid": "60741762ed6c4c4210bb1ffb"
* },
* "userId": {
* "$numberLong": "1"
* },
* "publishId": {
* "$numberLong": "1041"
* },
* "score": 12,
* "date": {
* "$numberLong": "1618220898066"
* },
* "_class": "com.tanhua.recommend.pojo.RecommendQuanZi"
* }
*/
//rdd.foreach(k-> System.out.println(k.toJson()));
JavaPairRDD<String, Document> rdd2 = rdd.mapToPair(document -> {
Long userId = document.getLong("userId");
Long publishId = document.getLong("publishId");
return new Tuple2<>(userId + "_" + publishId, document);
});
/**
* (2_1381498501455683584,Document{{_id=6073ec7bed6c4c4a3c64f6a9, userId=2, publishId=1381498501455683584, score=3.0, date=1618209884721, _class=com.tanhua.recommend.pojo.RecommendQuanZi}})
*/
//rdd2.foreach(v-> System.out.println(v.toString()));
JavaPairRDD<String, Document> rdd3 = rdd2.reduceByKey((v1, v2) -> {
double score = v1.getDouble("score") + v2.getDouble("score");
v1.put("score", score);
return v1;
});
/**
* (5_1084,Document{{_id=60741762ed6c4c4210bb22b4, userId=5, publishId=1084, score=7.0, date=1618220898448, _class=com.tanhua.recommend.pojo.RecommendQuanZi}})
*/
//rdd3.foreach(g-> System.out.println(g.toString()));
JavaRDD<Document> rdd4 = rdd3.values();
/**
* {
* "_id": {
* "$oid": "60741762ed6c4c4210bb2140"
* },
* "userId": {
* "$numberLong": "7"
* },
* "publishId": {
* "$numberLong": "1005"
* },
* "score": 35,
* "date": {
* "$numberLong": "1618220898257"
* },
* "_class": "com.tanhua.recommend.pojo.RecommendQuanZi"
* }
*
*/
// rdd4.foreach(h-> {
// System.out.println(h.toJson());
// } );
JavaPairRDD<Long, Rating> ratings = rdd4.mapToPair(document -> {
Long date = document.getLong("date");
int userId = document.getLong("userId").intValue();
int publishId = document.getLong("publishId").intValue();
Double score = document.getDouble("score");
Rating rating = new Rating(userId, publishId, score);
return new Tuple2<>(date % 10, rating);
});
MLlibRecommend mLlibRecommend = new MLlibRecommend();
MatrixFactorizationModel bestModel = mLlibRecommend.bestModel(ratings);
//连接redis做存储
String redisNodesStr = properties.getProperty("redis.cluster.nodes");
String[] redisNodesStrs = StringUtils.split(redisNodesStr, ",");
Set<HostAndPort> nodes = new HashSet<>();
for (String nodesStr : redisNodesStrs) {
String[] ss = StringUtils.split(nodesStr, ":");
nodes.add(new HostAndPort(ss[0],Integer.valueOf(ss[1])));
}
JedisCluster jedisCluster = new JedisCluster(nodes);
if(!CollectionUtils.isEmpty(userIdList)){
for (Long userId : userIdList) {
Rating[] recommendProducts = bestModel.recommendProducts(userId.intValue(), 20);
List<Integer> products = new ArrayList<>();
for (Rating product : recommendProducts) {
products.add(product.product());
}
String key = "QUANZI_PUBLISH_RECOMMEND_" + userId;
jedisCluster.set(key,StringUtils.join(products,","));
}
}
//关闭资源
jedisCluster.close();
jsc.stop();
}
}
SparkVideo
package com.tanhua.spark.mongo;
import com.mongodb.spark.MongoSpark;
import com.mongodb.spark.rdd.api.java.JavaMongoRDD;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.recommendation.MatrixFactorizationModel;
import org.apache.spark.mllib.recommendation.Rating;
import org.bson.Document;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisCluster;
import scala.Tuple2;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
public class SparkVideo {
public static void main(String[] args) throws IOException {
//加载外部配置文件
InputStream inputStream = SparkQuanZi.class.getClassLoader().getResourceAsStream("app.properties");
Properties properties = new Properties();
properties.load(inputStream);
//构建spark的配置
SparkConf sparkConf = new SparkConf()
.setAppName("SparkQuanZi")
.setMaster("local[*]")
.set("spark.mongodb.input.uri", properties.getProperty("spark.video.mongodb.input.uri"));
//获取spark上下文
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
//加载mongodb的数据
JavaMongoRDD<Document> rdd = MongoSpark.load(jsc);
//获取用户列表
JavaRDD<Long> userIdRdd = rdd.map(document -> document.getLong("userId"));
List<Long> userIdList = userIdRdd.distinct().collect();
//打印下数据
//rdd.foreach(k-> System.out.println(k.toJson()));
JavaPairRDD<String, Document> rdd2 = rdd.mapToPair(document -> {
Long userId = document.getLong("userId");
Long videoId = document.getLong("videoId");
return new Tuple2<>(userId + "_" + videoId, document);
});
//rdd2.foreach(v-> System.out.println(v.toString()));
JavaPairRDD<String, Document> rdd3 = rdd2.reduceByKey((v1, v2) -> {
double score = v1.getDouble("score") + v2.getDouble("score");
v1.put("score", score);
return v1;
});
//rdd3.foreach(g-> System.out.println(g.toString()));
JavaRDD<Document> rdd4 = rdd3.values();
// rdd4.foreach(h-> {
// System.out.println(h.toJson());
// } );
JavaPairRDD<Long, Rating> ratings = rdd4.mapToPair(document -> {
Long date = document.getLong("date");
int userId = document.getLong("userId").intValue();
int videoId = document.getLong("videoId").intValue();
Double score = document.getDouble("score");
Rating rating = new Rating(userId, videoId, score);
return new Tuple2<>(date % 10, rating);
});
MLlibRecommend mLlibRecommend = new MLlibRecommend();
MatrixFactorizationModel bestModel = mLlibRecommend.bestModel(ratings);
//连接redis做存储
String redisNodesStr = properties.getProperty("redis.cluster.nodes");
String[] redisNodesStrs = StringUtils.split(redisNodesStr, ",");
Set<HostAndPort> nodes = new HashSet<>();
for (String nodesStr : redisNodesStrs) {
String[] ss = StringUtils.split(nodesStr, ":");
nodes.add(new HostAndPort(ss[0],Integer.valueOf(ss[1])));
}
JedisCluster jedisCluster = new JedisCluster(nodes);
if(!CollectionUtils.isEmpty(userIdList)){
for (Long userId : userIdList) {
Rating[] recommendProducts = bestModel.recommendProducts(userId.intValue(), 20);
List<Integer> products = new ArrayList<>();
for (Rating product : recommendProducts) {
products.add(product.product());
}
String key = "VIDEO_PUBLISH_RECOMMEND_" + userId;
jedisCluster.set(key,StringUtils.join(products,","));
}
}
//关闭资源
jedisCluster.close();
jsc.stop();
}
}
评论
登录后参与技术讨论,分享你的经验与见解。
登录 / 注册暂无评论