探花07
文章目录
sparkmlib
最小二乘法ALS
7.4.2、数据建模
ALS算法的第二步就是数据建模,其实在MLlib算法库中有可以直接使用的训练算法,ALS.tran方法源码
如下:
package cn.itcast.spark;
import jdk.nashorn.internal.ir.IdentNode;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.recommendation.ALS;
import org.apache.spark.mllib.recommendation.MatrixFactorizationModel;
import org.apache.spark.mllib.recommendation.Rating;
public class MyRecommend {
public static void main(String[] args) {
SparkConf sparkConf = new SparkConf()
.setAppName("MyRecommend")
.setMaster("local[*]");
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
jsc.setLogLevel("WARN"); //设置日志级别
//读取数据文件
JavaRDD<String> sourceRDD = jsc.textFile("F:\\ml-100k\\u.data");
//对文件做分割的处理 \t
JavaRDD<String[]> lineRdd = sourceRDD.map(v -> v.split("\t"));
// 转化得到Rating对象集
JavaRDD<Rating> ratingRDD = lineRdd.map(v -> new Rating(Integer.valueOf(v[0]), Integer.valueOf(v[1]), Double.valueOf(v[2])));
// 训练模型
MatrixFactorizationModel model = ALS.train(ratingRDD.rdd(), 8, 10);
//商品推荐
Rating[] products = model.recommendProducts(301, 10);
// 打印结果集
for (Rating product : products) {
System.out.println("userId = "+ product.user() +", product = " + product.product());
}
jsc.stop();
}
}
推荐结果
MyRecommend2
package cn.itcast.spark;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.recommendation.ALS;
import org.apache.spark.mllib.recommendation.MatrixFactorizationModel;
import org.apache.spark.mllib.recommendation.Rating;
import scala.Int;
import scala.Tuple2;
import java.util.List;
public class MyRecommend2 {
public static void main(String[] args) {
SparkConf sparkConf = new SparkConf()
.setAppName("MyRecommend")
.setMaster("local[*]");
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
jsc.setLogLevel("WARN"); //设置日志级别
//读取数据文件
JavaRDD<String> sourceRDD = jsc.textFile("F:\\ml-100k\\u.data");
//对文件做分割的处理 \t
JavaRDD<String[]> lineRdd = sourceRDD.map(v -> v.split("\t"));
//装载样本评分数据,其中最后一列Timestamp取除10的余数作为key,Rating为值,即(Int,Rating)
JavaPairRDD<Integer, Rating> ratings = lineRdd.mapToPair(v -> {
Rating rating = new Rating(Integer.valueOf(v[0]), Integer.valueOf(v[1]), Double.valueOf(v[2]));
Integer key = Integer.valueOf(v[3]) % 10;
return new Tuple2<Integer, Rating>(key, rating);
});
//装载电影目录对照表(电影ID->电影标题)
List<Tuple2> movies = jsc.textFile("F:\\ml-100k\\u.item").map(v -> {
String[] ss = v.split("\\|");
return new Tuple2(ss[0], ss[1]);
}).collect();
//统计有用户数量和电影数量以及用户对电影的评分数目
long numUsers = ratings.map(v -> v._2().user()).distinct().count();
Long numRatings = ratings.count();
Long numMovies = ratings.map(v1 -> (v1._2()).product()).distinct().count();
System.out.println("用户:" + numUsers + "电影:" + numMovies + "评论:" + numRatings);
//将样本评分表以key值切分成3个部分,分别用于训练 (60%,并加入用户评分), 校验 (20%), and 测试 (20%)
// key :0 ~ 9 60% -> <6 20% -> 6<=x<8 20% x>=8
Integer numPartitions = 4; // 分区数
//训练集
JavaRDD<Rating> training = ratings.filter(v -> v._1() < 6)
.values()
.repartition(numPartitions)
.cache();
// 校验集
JavaRDD<Rating> validation = ratings
.filter(v -> v._1() >= 6 && v._1() < 8)
.values()
.repartition(numPartitions).cache();
// 测试集
JavaRDD<Rating> test = ratings
.filter(v -> v._1() >= 8)
.values()
.cache();
Long numTraining = training.count();
Long numValidation = validation.count();
Long numTest = test.count();
System.out.println("训练集:" + numTraining + " 校验集:" + numValidation + " 测试集:" + numTest);
int[] ranks = new int[]{10, 11, 12};
// double[] lambdas = new double[]{0.01, 0.03, 0.1, 0.3, 1, 3};
double[] lambdas = new double[]{0.01};
// int[] numIters = new int[]{8, 9, 10, 11, 12, 13, 14, 15};
int[] numIters = new int[]{8, 9, 10};
MatrixFactorizationModel bestModel = null;
double bestValidationRmse = Double.MAX_VALUE;
int bestRank = 0;
double bestLambda = -0.01;
int bestNumIter = 0;
//对上面的三个参数做循环,这里使用三层嵌套循环来测试
for (int rank : ranks) {
for (double lambda : lambdas) {
for (int numIter : numIters) {
MatrixFactorizationModel model = ALS.train(training.rdd(), rank, numIter, lambda);
Double rmse = computeRmse(model, validation, numValidation);
System.out.println("RMSE(校验集) = " + rmse + ", rank = " + rank + ", lambda = " + lambda + ", numIter = " + numIter);
if(rmse < bestValidationRmse){
bestValidationRmse = rmse;
bestModel = model;
bestRank = rank;
bestLambda = lambda;
bestNumIter = numIter;
}
}
}
}
double testRmse = computeRmse(bestModel, test, numTest);
System.out.println("测试数据集在 最佳训练模型 rank = " + bestRank + ", lambda = " + bestLambda + ", numIter = " + bestNumIter + ", RMSE = " + testRmse);
// 计算均值
Double meanRating = training.union(validation).mapToDouble(v -> v.rating()).mean();
// 计算标准误差值
Double baselineRmse = Math.sqrt(test.map(v -> (meanRating - v.rating()) * (meanRating - v.rating())).reduce((v1, v2) -> (v1 + v2) / numTest));
// 计算准确率提升了多少
double improvement = (baselineRmse - testRmse) / baselineRmse * 100;
System.out.println("最佳训练模型的准确率提升了:" + String.format("%.2f", improvement) + "%.");
//构建最佳训练模型
MatrixFactorizationModel model = ALS.train(ratings.values().rdd(), bestRank, bestNumIter, bestLambda);
//使用最佳的训练模型进行推荐
Rating[] recommendProducts = model.recommendProducts(789, 10);
//打印推荐结果
for (Rating rating : recommendProducts) {
System.out.println(rating.user() + "->" + rating.product()+": " + rating.rating());
}
jsc.stop();
}
/**
* 校验集预测数据和实际数据之间的均方根误差
**/
public static Double computeRmse(MatrixFactorizationModel model, JavaRDD<Rating> data, Long n) {
// 进行预测
JavaRDD<Rating> predictions = model.predict(data.mapToPair(v -> new Tuple2<>(v.user(), v.product())));
JavaRDD<Tuple2<Double, Double>> predictionsAndRatings = predictions
.mapToPair(v -> new Tuple2<>(new Tuple2<>(v.user(), v.product()), v.rating()))
.join(data.mapToPair(v -> new Tuple2<>(new Tuple2<>(v.user(), v.product()), v.rating()))).values();
Double reduce = predictionsAndRatings.map(v -> (v._1 - v._2) * (v._1 - v._2))
.reduce((v1, v2) -> (v1 + v2) / n);
//正平方根
return Math.sqrt(reduce);
}
}
TestMLlib
package cn.itcast.spark;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.linalg.Matrices;
import org.apache.spark.mllib.linalg.Matrix;
import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.mllib.linalg.distributed.*;
import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.junit.Test;
import scala.Tuple2;
import java.util.List;
public class TestMLlib {
@Test
public void testDense() {
Vector dense = Vectors.dense(9, 5, 2, 7);//定义密集型数据集
double v = dense.apply(1); //根据下标获取值
System.out.println(v);
}
@Test //测试稀疏型的数据集
public void testSparse() {
int[] indexes = new int[]{0, 1, 5, 9};
double[] values = new double[]{9, 5, 2, 7};
Vector sparse = Vectors.sparse(10, indexes, values); //定义稀疏型数据集
System.out.println(sparse.apply(2));
System.out.println(sparse.apply(3));
System.out.println(sparse.apply(9));
System.out.println(sparse.apply(7));
}
@Test //测试向量标签
public void testLabeledPoint() {
// 密集型标签
Vector dense = Vectors.dense(9, 5, 2, 7);//定义密集型数据集
LabeledPoint lp = new LabeledPoint(1, dense);//定义向量标签
System.out.println(lp.label()); //打印标签
System.out.println(lp.features()); //打印特征,数据集
System.out.println(lp);
int[] indexes = new int[]{0, 1, 5, 9};
double[] values = new double[]{9, 5, 2, 7};
Vector sparse = Vectors.sparse(10, indexes, values); //定义稀疏型数据集
LabeledPoint lp2 = new LabeledPoint(2, sparse);// 定义向量标签
System.out.println(lp2);
System.out.println(lp2.label());
System.out.println(lp2.features());
}
@Test //测试本地矩阵
public void testMatrix() {
double[] values = new double[]{1, 2, 3, 4, 5, 6};
Matrix matrix = Matrices.dense(2, 3, values);
System.out.println(matrix);
}
@Test //测试分布式矩阵,行矩阵
public void testRowMatrix(){
SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("TestMLlib");
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
//读取文件
JavaRDD<String> fileRdd = jsc.textFile("F:\\code\\files\\a.txt");
JavaRDD<Vector> vectorJavaRDD = fileRdd.map(v -> {
String[] ss = v.split(" ");
double[] ds = new double[ss.length];
for (int i = 0; i < ss.length; i++) {
ds[i] = Double.valueOf(ss[i]);
}
return Vectors.dense(ds);
});
RowMatrix rowMatrix = new RowMatrix(vectorJavaRDD.rdd()); //行矩阵
System.out.println(rowMatrix.numRows()); //行数
System.out.println(rowMatrix.numCols()); //列数
}
@Test //测试分布式矩阵,行索引矩阵
public void testIndexedRowMatrix(){
SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("TestMLlib");
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
//读取文件
JavaRDD<String> fileRdd = jsc.textFile("F:\\code\\files\\a.txt");
JavaRDD<Vector> rdd1 = fileRdd.map(v -> {
String[] ss = v.split(" ");
double[] ds = new double[ss.length];
for (int i = 0; i < ss.length; i++) {
ds[i] = Double.valueOf(ss[i]);
}
return Vectors.dense(ds);
});
JavaRDD<IndexedRow> rdd2 = rdd1.map(v -> new IndexedRow(v.size(), v));
//行索引矩阵
IndexedRowMatrix indexedRowMatrix = new IndexedRowMatrix(rdd2.rdd());
System.out.println(indexedRowMatrix.numRows());
System.out.println(indexedRowMatrix.numCols());
JavaRDD<IndexedRow> rdd3 = indexedRowMatrix.rows().toJavaRDD();
List<IndexedRow> list = rdd3.collect();
list.forEach(indexedRow -> System.out.println(indexedRow));
}
@Test //测试分布式矩阵,坐标矩阵
public void testCoordinateMatrix(){
SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("TestMLlib");
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
//读取文件
JavaRDD<String> fileRdd = jsc.textFile("F:\\code\\files\\a.txt");
JavaRDD<MatrixEntry> rdd = fileRdd.map(v -> {
String[] ss = v.split(" ");
return new MatrixEntry(Long.valueOf(ss[0]), Long.valueOf(ss[1]),Double.valueOf(ss[2]));
});
//坐标矩阵
CoordinateMatrix coordinateMatrix = new CoordinateMatrix(rdd.rdd());
List<MatrixEntry> collect = coordinateMatrix.entries().toJavaRDD().collect();
collect.forEach(matrixEntry -> System.out.println(matrixEntry));
}
@Test //测试分布式矩阵,分块矩阵
public void testBlockMatrix(){
SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("TestMLlib");
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
//读取文件
JavaRDD<String> fileRdd = jsc.textFile("F:\\code\\files\\a.txt");
JavaRDD<MatrixEntry> rdd = fileRdd.map(v -> {
String[] ss = v.split(" ");
return new MatrixEntry(Long.valueOf(ss[0]), Long.valueOf(ss[1]),Double.valueOf(ss[2]));
});
//坐标矩阵
CoordinateMatrix coordinateMatrix = new CoordinateMatrix(rdd.rdd());
// 转化得到分块矩阵
BlockMatrix blockMatrix = coordinateMatrix.toBlockMatrix();
// 校验分块是否正确,如果不正确会抛出异常
blockMatrix.validate();
System.out.println(blockMatrix.numCols());
System.out.println(blockMatrix.numRows());
System.out.println("--------");
// 两个矩阵相加
BlockMatrix addBlockMatrix = blockMatrix.add(blockMatrix);
System.out.println(addBlockMatrix.numCols());
System.out.println(addBlockMatrix.numRows());
List<Tuple2<Tuple2<Object, Object>, Matrix>> collect = addBlockMatrix.blocks().toJavaRDD().collect();
collect.forEach(tuple2MatrixTuple2 -> System.out.println(tuple2MatrixTuple2));
}
@Test
public void testDataSet(){
SparkSession sparkSession = SparkSession
.builder()
.master("local[*]")
.appName("TestMLlib")
.getOrCreate();
JavaRDD<String> rdd1 = sparkSession.read().textFile("F:\\code\\files\\users.txt").javaRDD();
JavaRDD<Student> rdd2 = rdd1.map(v -> {
String[] ss = v.split(",");
Student student = new Student();
student.setId(Long.valueOf(ss[0]));
student.setName(ss[1]);
student.setAge(Integer.valueOf(ss[2]));
return student;
});
Dataset<Row> dataSet = sparkSession.createDataFrame(rdd2, Student.class); //转化得到dataSet
// dataSet.show();
dataSet.select("id","name").orderBy(dataSet.col("id").desc()).show();
}
}

评论
登录后参与技术讨论,分享你的经验与见解。
登录 / 注册暂无评论