探花09
文章目录
用户推荐
从mysql中读取数据写入mongodb中
配置文件
spark.mongodb.input.uri = mongodb://192.168.200.146:27017/tanhua.recommend_quanzi_20191001?readPreference=primaryPreferred
spark.video.mongodb.input.uri = mongodb://192.168.200.146:27017/tanhua.recommend_video_20191010?readPreference=primaryPreferred
spark.mongodb.output.user.uri = mongodb://192.168.200.146:27017/tanhua.recommend_user?readPreference=primaryPreferred
redis.cluster.nodes = 192.168.200.146:6379,192.168.200.146:6380,192.168.200.146:6381
jdbc.driver-class-name=com.mysql.jdbc.Driver
jdbc.url=jdbc:mysql://192.168.200.146:3306/mytanhua?useUnicode=true&characterEncoding=utf8&autoReconnect=true&allowMultiQueries=true&useSSL=false
jdbc.username=root
jdbc.password=123456生成推荐模型
MLlibRecommend
package com.tanhua.spark.mongo;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.mllib.recommendation.ALS;
import org.apache.spark.mllib.recommendation.MatrixFactorizationModel;
import org.apache.spark.mllib.recommendation.Rating;
import scala.Tuple2;
public class MLlibRecommend {
public MatrixFactorizationModel bestModel(JavaPairRDD<Long, Rating> ratings){
//统计有用户数量和动态数量以及用户对动态的评分数目
Long numRatings = ratings.count();
Long numUsers = ratings.map(v1 -> (v1._2()).user()).distinct().count();
Long numMovies = ratings.map(v1 -> (v1._2()).product()).distinct().count();
System.out.println("用户:" + numUsers + "动态:" + numMovies + "评论:" + numRatings);
//将样本评分表以key值切分成3个部分,分别用于训练 (60%,并加入用户评分), 校验 (20%), and 测试 (20%)
//该数据在计算过程中要多次应用到,所以cache到内存
Integer numPartitions = 4; // 分区数
// 训练集
JavaRDD<Rating> training = ratings
.filter(v -> v._1() < 6)
.values()
.repartition(numPartitions)
.cache();
// 校验集
JavaRDD<Rating> validation = ratings
.filter(v -> v._1() >= 6 && v._1() < 8)
.values()
.repartition(numPartitions).cache();
// 测试集
JavaRDD<Rating> test = ratings
.filter(v -> v._1() >= 8)
.values()
.cache();
Long numTraining = training.count();
Long numValidation = validation.count();
Long numTest = test.count();
System.out.println("训练集:" + numTraining + " 校验集:" + numValidation + " 测试集:" + numTest);
//训练不同参数下的模型,并在校验集中验证,获取最佳参数下的模
int[] ranks = new int[]{10, 11, 12};
// double[] lambdas = new double[]{0.01, 0.03, 0.1, 0.3, 1, 3};
double[] lambdas = new double[]{0.01};
// int[] numIters = new int[]{8, 9, 10, 11, 12, 13, 14, 15};
int[] numIters = new int[]{8, 9, 10};
MatrixFactorizationModel bestModel = null;
double bestValidationRmse = Double.MAX_VALUE;
int bestRank = 0;
double bestLambda = -0.01;
int bestNumIter = 0;
for (int rank : ranks) {
for (int numIter : numIters) {
for (double lambda : lambdas) {
MatrixFactorizationModel model = ALS.train(training.rdd(), rank, numIter, lambda);
Double validationRmse = computeRmse(model, validation, numValidation);
System.out.println("RMSE(校验集) = " + validationRmse + ", rank = " + rank + ", lambda = " + lambda + ", numIter = " + numIter);
if (validationRmse < bestValidationRmse) {
bestModel = model;
bestValidationRmse = validationRmse;
bestRank = rank;
bestLambda = lambda;
bestNumIter = numIter;
}
}
}
}
double testRmse = computeRmse(bestModel, test, numTest);
System.out.println("测试数据集在 最佳训练模型 rank = " + bestRank + ", lambda = " + bestLambda + ", numIter = " + bestNumIter + ", RMSE = " + testRmse);
// 计算均值
Double meanRating = training.union(validation).mapToDouble(v -> v.rating()).mean();
// 计算标准误差值
Double baselineRmse = Math.sqrt(test.map(v -> (meanRating - v.rating()) * (meanRating - v.rating())).reduce((v1, v2) -> (v1 + v2) / numTest));
// 计算准确率提升了多少
double improvement = (baselineRmse - testRmse) / baselineRmse * 100;
System.out.println("最佳训练模型的准确率提升了:" + String.format("%.2f", improvement) + "%.");
// 构建最佳训练模型
bestModel = ALS.train(ratings.values().rdd(), bestRank, bestNumIter, bestLambda);
return bestModel;
}
/**
* 校验集预测数据和实际数据之间的均方根误差
**/
public Double computeRmse(MatrixFactorizationModel model, JavaRDD<Rating> data, Long n) {
// 进行预测
JavaRDD<Rating> predictions = model.predict(data.mapToPair(v -> new Tuple2<>(v.user(), v.product())));
JavaRDD<Tuple2<Double, Double>> predictionsAndRatings = predictions
.mapToPair(v -> new Tuple2<>(new Tuple2<>(v.user(), v.product()), v.rating()))
.join(data.mapToPair(v -> new Tuple2<>(new Tuple2<>(v.user(), v.product()), v.rating()))).values();
Double reduce = predictionsAndRatings.map(v -> (v._1 - v._2) * (v._1 - v._2))
.reduce((v1, v2) -> (v1 + v2) / n);
//正平方根
return Math.sqrt(reduce);
}
}
SparkUserRecommend
package com.tanhua.spark.mongo;
import com.mongodb.spark.MongoSpark;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.recommendation.MatrixFactorizationModel;
import org.apache.spark.mllib.recommendation.Rating;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.joda.time.DateTime;
import scala.Tuple2;
import java.io.InputStream;
import java.math.BigDecimal;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
public class SparkUserRecommend {
public static void main(String[] args) throws Exception {
//加载外部的配置文件,app.properties
InputStream inputStream = SparkQunaZi.class.getClassLoader().getResourceAsStream("app.properties");
Properties properties = new Properties();
properties.load(inputStream);
//构建Spark配置
SparkConf sparkConf = new SparkConf()
.setAppName("SparkUserRecommend")
.setMaster("local[*]")
.set("spark.mongodb.output.uri", properties.getProperty("spark.mongodb.output.user.uri"));
//加载mysql数据
SparkSession sparkSession = SparkSession.builder().config(sparkConf).getOrCreate();
String url = properties.getProperty("jdbc.url");
// 设置数据库连接信息
Properties connectionProperties = new Properties();
connectionProperties.put("driver", properties.getProperty("jdbc.driver-class-name"));
connectionProperties.put("user", properties.getProperty("jdbc.username"));
connectionProperties.put("password", properties.getProperty("jdbc.password"));
JavaRDD<Row> userInfoRdd = sparkSession.read().jdbc(url, "tb_user_info", connectionProperties).toJavaRDD();
//用户列表
List<Long> userIds = userInfoRdd.map(v -> v.getLong(1)).collect();
//计算出这张表数据的 笛卡尔积
JavaPairRDD<Row, Row> cartesian = userInfoRdd.cartesian(userInfoRdd);
//计算用户的相似度
JavaPairRDD<Long, Rating> javaPairRDD = cartesian.mapToPair(row -> {
Row row1 = row._1();
Row row2 = row._2();
Long userId1 = row1.getLong(1);
Long userId2 = row2.getLong(1);
Long key = userId1 + userId2 + RandomUtils.nextLong();
// 自己与自己对比
if (userId1.longValue() == userId2.longValue()) {
return new Tuple2<>(key % 10, new Rating(userId1.intValue(), userId2.intValue(), 0));
}
double score = 0;
//计算年龄差
int ageDiff = Math.abs(row1.getInt(6) - row2.getInt(6));
if (ageDiff <= 2) {
score += 30;
} else if (ageDiff >= 3 && ageDiff <= 5) {
score += 20;
} else if (ageDiff > 5 && ageDiff <= 10) {
score += 10;
}
// 计算性别
if (row1.getInt(5) != row2.getInt(5)) {
score += 30;
}
// 计算城市
String city1 = StringUtils.substringBefore(row1.getString(8), "-");
String city2 = StringUtils.substringBefore(row2.getString(8), "-");
if (StringUtils.equals(city1, city2)) {
score += 20;
}
// 计算学历
String edu1 = row1.getString(7);
String edu2 = row2.getString(7);
if (StringUtils.equals(edu1, edu2)) {
score += 20;
}
Rating rating = new Rating(userId1.intValue(), userId2.intValue(), score);
return new Tuple2<>(key % 10, rating);
});
//MLlib进行计算最佳的推荐模型
MLlibRecommend mLlibRecommend = new MLlibRecommend();
MatrixFactorizationModel bestModel = mLlibRecommend.bestModel(javaPairRDD);
//将数据写入到MongoDB中
JavaSparkContext jsc = new JavaSparkContext(sparkSession.sparkContext());
for (Long userId : userIds) {
Rating[] ratings = bestModel.recommendProducts(userId.intValue(), 50);
JavaRDD<Document> documentJavaRDD = jsc.parallelize(Arrays.asList(ratings)).map(v1 -> {
Document document = new Document();
document.put("_id", ObjectId.get());
document.put("userId", v1.product());
document.put("toUserId", v1.user());
//得分,保留2位小数
double score = new BigDecimal(v1.rating()).setScale(2, BigDecimal.ROUND_DOWN).doubleValue();
document.put("score", score);
document.put("date", new DateTime().toString("yyyy/MM/dd"));
return document;
});
MongoSpark.save(documentJavaRDD);
}
}
}

上报地理位置
地理位置
package com.tanhua.dubbo.server.pojo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.bson.types.ObjectId;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.geo.GeoJsonPoint;
import org.springframework.data.mongodb.core.index.CompoundIndex;
import org.springframework.data.mongodb.core.index.Indexed;
import org.springframework.data.mongodb.core.mapping.Document;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Document(collection = "user_location")
@CompoundIndex(name = "location_index", def = "{'location': '2dsphere'}")
public class UserLocation implements java.io.Serializable{
private static final long serialVersionUID = 4508868382007529970L;
@Id
private ObjectId id;
@Indexed
private Long userId; //用户id
private GeoJsonPoint location; //x:经度 y:纬度
private String address; //位置描述
private Long created; //创建时间
private Long updated; //更新时间
private Long lastUpdated; //上次更新时间
}
常见索引介绍
@Indexed
声明该字段需要加索引,加索引后以该字段为条件检索将大大提高速度。
唯一索引的话是@Indexed(unique = true)。
也可以对数组进行索引,如果被索引的列是数组时,MongoDB会索引这个数组中的每一个元素。
也可以对整个Document进行索引,排序是预定义的按插入BSON数据的先后升序排列。
也可以对关联的对象的字段进行索引,譬如User对关联的address.city进行索引。(注解怎么写还不清楚,待查)
@Field
代表一个字段,可以不加,不加的话默认以参数名为列名。
给映射存储到 mongodb 的字段取别名
在 java bean 中字段名为 firstName,存储到 mongo 中 key 为 fName
@Field("fName")
private String firstName; @CompoundIndex
复合索引,加复合索引后通过复合索引字段查询将大大提高速度。
@Document
@CompoundIndexes({
@CompoundIndex(name = "age_idx", def = "{'lastName': 1, 'age': -1}")
})
public class Person<T extends Address> {
}mongodb增删改查总结
列表分页排序
public static PageRequest of(int page, int size) {
return of(page, size, Sort.unsorted());
}
public static PageRequest of(int page, int size, Sort sort) {
return new PageRequest(page, size, sort);
} //分页page从0开始
Pageable pageable = PageRequest.of(page - 1, pageSize, Sort.by(Sort.Order.desc("date")));
Query query = new Query().with(pageable);
Query queryPublish = Query.query(Criteria.where("id").in(ids)).with(Sort.by(Sort.Order.desc("created")));
//查询动态信息
List<Publish> publishList = this.mongoTemplate.find(queryPublish, Publish.class);更新文档
//更新
Query query = Query.query(Criteria.where("userId").is(userId));
Update update = Update.update("location", userLocation.getLocation())
.set("updated", System.currentTimeMillis())
.set("lastUpdated", ul.getUpdated())
.set("address", userLocation.getAddress());
this.mongoTemplate.updateFirst(query, update, UserLocation.class);删除文档
Criteria criteria = Criteria.where("userId").is(userId)
.and("publishId").is(new ObjectId(publishId))
.and("commentType").is(commentType);
Query query = Query.query(criteria);
DeleteResult deleteResult = this.mongoTemplate.remove(query, Comment.class);
return deleteResult.getDeletedCount() > 0;获取文档数量
//判断是否已经点赞,如果已经点赞就返回
Criteria criteria = Criteria.where("userId").is(userId)
.and("publishId").is(new ObjectId(publishId))
.and("commentType").is(1);
Query query = Query.query(criteria);
long count = this.mongoTemplate.count(query, Comment.class);查看单个文档
Publish publish = this.mongoTemplate.findById(comment.getPublishId(), Publish.class);保存文档(第二个参数是集合名)
Visitors v= this.mongoTemplate.save(visitors, "visitors");
评论
登录后参与技术讨论,分享你的经验与见解。
登录 / 注册暂无评论