探花08

圈子推荐

image.png
image.png

圈子和视频积分表

RecommendQuanZi

package com.tanhua.recommend.pojo;

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.bson.types.ObjectId;

@Data
@NoArgsConstructor
@AllArgsConstructor
public class RecommendQuanZi {

    private ObjectId id;
    private Long userId;// 用户id
    private Long publishId; //动态id,需要转化为Long类型
    private Double score; //得分
    private Long date; //时间戳
}

RecommendVideo

package com.tanhua.recommend.pojo;

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.bson.types.ObjectId;

@Data
@NoArgsConstructor
@AllArgsConstructor
public class RecommendVideo {
    private ObjectId id;
    private Long userId;// 用户id
    private Long videoId; //视频id,需要转化为Long类型
    private Double score; //得分
    private Long date; //时间戳
}

搭建spark系统

paalication.properties

spark.mongodb.input.uri = mongodb://192.168.200.146:27017/tanhua.recommend_quanzi_20191001?readPreference=primaryPreferred
spark.video.mongodb.input.uri = mongodb://192.168.200.146:27017/tanhua.recommend_video_20191010?readPreference=primaryPreferred
spark.mongodb.output.user.uri = mongodb://192.168.200.146:27017/tanhua.recommend_user?readPreference=primaryPreferred

redis.cluster.nodes = 192.168.200.146:6379,192.168.200.146:6380,192.168.200.146:6381

jdbc.driver-class-name=com.mysql.jdbc.Driver
jdbc.url=jdbc:mysql://192.168.200.146:3306/mytanhua?useUnicode=true&characterEncoding=utf8&autoReconnect=true&allowMultiQueries=true&useSSL=false
jdbc.username=root
jdbc.password=123456

计算推荐圈子SparkQuanZi

package com.tanhua.spark.mongo;

import com.mongodb.spark.MongoSpark;
import com.mongodb.spark.rdd.api.java.JavaMongoRDD;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.recommendation.MatrixFactorizationModel;
import org.apache.spark.mllib.recommendation.Rating;
import org.apache.spark.sql.sources.In;
import org.bson.Document;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisCluster;
import scala.Tuple2;

import java.io.IOException;
import java.io.InputStream;
import java.util.*;

public class SparkQuanZi {
    public static void main(String[] args) throws IOException {
        //加载外部配置文件
        InputStream inputStream = SparkQuanZi.class.getClassLoader().getResourceAsStream("app.properties");
        Properties properties = new Properties();
        properties.load(inputStream);

        //构建spark的配置
        SparkConf sparkConf = new SparkConf()
                .setAppName("SparkQuanZi")
                .setMaster("local[*]")
                .set("spark.mongodb.input.uri", properties.getProperty("spark.mongodb.input.uri"));

        //获取spark上下文
        JavaSparkContext jsc = new JavaSparkContext(sparkConf);

        //加载mongodb的数据
        JavaMongoRDD<Document> rdd = MongoSpark.load(jsc);

        //获取用户列表
        JavaRDD<Long> userIdRdd = rdd.map(document -> document.getLong("userId"));
        List<Long> userIdList = userIdRdd.distinct().collect();

        //打印下数据
        /**
         * {
         *     "_id": {
         *         "$oid": "60741762ed6c4c4210bb1ffb"
         *     },
         *     "userId": {
         *         "$numberLong": "1"
         *     },
         *     "publishId": {
         *         "$numberLong": "1041"
         *     },
         *     "score": 12,
         *     "date": {
         *         "$numberLong": "1618220898066"
         *     },
         *     "_class": "com.tanhua.recommend.pojo.RecommendQuanZi"
         * }
         */
        //rdd.foreach(k-> System.out.println(k.toJson()));


        JavaPairRDD<String, Document> rdd2 = rdd.mapToPair(document -> {
            Long userId = document.getLong("userId");
            Long publishId = document.getLong("publishId");
            return new Tuple2<>(userId + "_" + publishId, document);
        });
        /**
         * (2_1381498501455683584,Document{{_id=6073ec7bed6c4c4a3c64f6a9, userId=2, publishId=1381498501455683584, score=3.0, date=1618209884721, _class=com.tanhua.recommend.pojo.RecommendQuanZi}})
         */
        //rdd2.foreach(v-> System.out.println(v.toString()));

        JavaPairRDD<String, Document> rdd3 = rdd2.reduceByKey((v1, v2) -> {
            double score = v1.getDouble("score") + v2.getDouble("score");
            v1.put("score", score);
            return v1;
        });

        /**
         * (5_1084,Document{{_id=60741762ed6c4c4210bb22b4, userId=5, publishId=1084, score=7.0, date=1618220898448, _class=com.tanhua.recommend.pojo.RecommendQuanZi}})
         */
        //rdd3.foreach(g-> System.out.println(g.toString()));

        JavaRDD<Document> rdd4 = rdd3.values();

        /**
         * {
         *     "_id": {
         *         "$oid": "60741762ed6c4c4210bb2140"
         *     },
         *     "userId": {
         *         "$numberLong": "7"
         *     },
         *     "publishId": {
         *         "$numberLong": "1005"
         *     },
         *     "score": 35,
         *     "date": {
         *         "$numberLong": "1618220898257"
         *     },
         *     "_class": "com.tanhua.recommend.pojo.RecommendQuanZi"
         * }
         *
         */
//        rdd4.foreach(h-> {
//            System.out.println(h.toJson());
//        } );

        JavaPairRDD<Long, Rating> ratings = rdd4.mapToPair(document -> {
            Long date = document.getLong("date");
            int userId = document.getLong("userId").intValue();
            int publishId = document.getLong("publishId").intValue();
            Double score = document.getDouble("score");
            Rating rating = new Rating(userId, publishId, score);
            return new Tuple2<>(date % 10, rating);
        });

        MLlibRecommend mLlibRecommend = new MLlibRecommend();
        MatrixFactorizationModel bestModel = mLlibRecommend.bestModel(ratings);

        //连接redis做存储
        String redisNodesStr = properties.getProperty("redis.cluster.nodes");
        String[] redisNodesStrs = StringUtils.split(redisNodesStr, ",");
        Set<HostAndPort> nodes = new HashSet<>();
        for (String nodesStr : redisNodesStrs) {
            String[] ss = StringUtils.split(nodesStr, ":");
            nodes.add(new HostAndPort(ss[0],Integer.valueOf(ss[1])));
        }
        JedisCluster jedisCluster = new JedisCluster(nodes);
        if(!CollectionUtils.isEmpty(userIdList)){
            for (Long userId : userIdList) {
                Rating[] recommendProducts = bestModel.recommendProducts(userId.intValue(), 20);
                List<Integer> products = new ArrayList<>();
                for (Rating product : recommendProducts) {
                    products.add(product.product());
                }
                String key = "QUANZI_PUBLISH_RECOMMEND_" + userId;
                jedisCluster.set(key,StringUtils.join(products,","));
            }
        }

        //关闭资源
        jedisCluster.close();
        jsc.stop();
    }
}

SparkVideo

package com.tanhua.spark.mongo;

import com.mongodb.spark.MongoSpark;
import com.mongodb.spark.rdd.api.java.JavaMongoRDD;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.recommendation.MatrixFactorizationModel;
import org.apache.spark.mllib.recommendation.Rating;
import org.bson.Document;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisCluster;
import scala.Tuple2;

import java.io.IOException;
import java.io.InputStream;
import java.util.*;

public class SparkVideo {
    public static void main(String[] args) throws IOException {
        //加载外部配置文件
        InputStream inputStream = SparkQuanZi.class.getClassLoader().getResourceAsStream("app.properties");
        Properties properties = new Properties();
        properties.load(inputStream);

        //构建spark的配置
        SparkConf sparkConf = new SparkConf()
                .setAppName("SparkQuanZi")
                .setMaster("local[*]")
                .set("spark.mongodb.input.uri", properties.getProperty("spark.video.mongodb.input.uri"));

        //获取spark上下文
        JavaSparkContext jsc = new JavaSparkContext(sparkConf);

        //加载mongodb的数据
        JavaMongoRDD<Document> rdd = MongoSpark.load(jsc);

        //获取用户列表
        JavaRDD<Long> userIdRdd = rdd.map(document -> document.getLong("userId"));
        List<Long> userIdList = userIdRdd.distinct().collect();

        //打印下数据
        //rdd.foreach(k-> System.out.println(k.toJson()));


        JavaPairRDD<String, Document> rdd2 = rdd.mapToPair(document -> {
            Long userId = document.getLong("userId");
            Long videoId = document.getLong("videoId");
            return new Tuple2<>(userId + "_" + videoId, document);
        });

        //rdd2.foreach(v-> System.out.println(v.toString()));

        JavaPairRDD<String, Document> rdd3 = rdd2.reduceByKey((v1, v2) -> {
            double score = v1.getDouble("score") + v2.getDouble("score");
            v1.put("score", score);
            return v1;
        });

        //rdd3.foreach(g-> System.out.println(g.toString()));

        JavaRDD<Document> rdd4 = rdd3.values();


//        rdd4.foreach(h-> {
//            System.out.println(h.toJson());
//        } );

        JavaPairRDD<Long, Rating> ratings = rdd4.mapToPair(document -> {
            Long date = document.getLong("date");
            int userId = document.getLong("userId").intValue();
            int videoId = document.getLong("videoId").intValue();
            Double score = document.getDouble("score");
            Rating rating = new Rating(userId, videoId, score);
            return new Tuple2<>(date % 10, rating);
        });

        MLlibRecommend mLlibRecommend = new MLlibRecommend();
        MatrixFactorizationModel bestModel = mLlibRecommend.bestModel(ratings);

        //连接redis做存储
        String redisNodesStr = properties.getProperty("redis.cluster.nodes");
        String[] redisNodesStrs = StringUtils.split(redisNodesStr, ",");
        Set<HostAndPort> nodes = new HashSet<>();
        for (String nodesStr : redisNodesStrs) {
            String[] ss = StringUtils.split(nodesStr, ":");
            nodes.add(new HostAndPort(ss[0],Integer.valueOf(ss[1])));
        }
        JedisCluster jedisCluster = new JedisCluster(nodes);
        if(!CollectionUtils.isEmpty(userIdList)){
            for (Long userId : userIdList) {
                Rating[] recommendProducts = bestModel.recommendProducts(userId.intValue(), 20);
                List<Integer> products = new ArrayList<>();
                for (Rating product : recommendProducts) {
                    products.add(product.product());
                }
                String key = "VIDEO_PUBLISH_RECOMMEND_" + userId;
                jedisCluster.set(key,StringUtils.join(products,","));
            }
        }

        //关闭资源
        jedisCluster.close();
        jsc.stop();
    }
}

评论

暂无评论