探花07

sparkmlib

最小二乘法ALS

7.4.2、数据建模

ALS算法的第二步就是数据建模,其实在MLlib算法库中有可以直接使用的训练算法,ALS.tran方法源码
如下:
image.png

package cn.itcast.spark;

import jdk.nashorn.internal.ir.IdentNode;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.recommendation.ALS;
import org.apache.spark.mllib.recommendation.MatrixFactorizationModel;
import org.apache.spark.mllib.recommendation.Rating;

public class MyRecommend {

    public static void main(String[] args) {

        SparkConf sparkConf = new SparkConf()
                .setAppName("MyRecommend")
                .setMaster("local[*]");
        JavaSparkContext jsc = new JavaSparkContext(sparkConf);
        jsc.setLogLevel("WARN"); //设置日志级别

        //读取数据文件
        JavaRDD<String> sourceRDD = jsc.textFile("F:\\ml-100k\\u.data");

        //对文件做分割的处理 \t
        JavaRDD<String[]> lineRdd = sourceRDD.map(v -> v.split("\t"));

        // 转化得到Rating对象集
        JavaRDD<Rating> ratingRDD = lineRdd.map(v -> new Rating(Integer.valueOf(v[0]), Integer.valueOf(v[1]), Double.valueOf(v[2])));

        // 训练模型
        MatrixFactorizationModel model = ALS.train(ratingRDD.rdd(), 8, 10);

        //商品推荐
        Rating[] products = model.recommendProducts(301, 10);

        // 打印结果集
        for (Rating product : products) {
            System.out.println("userId = "+ product.user() +", product = " + product.product());
        }

        jsc.stop();

    }

}

推荐结果
image.png

MyRecommend2

package cn.itcast.spark;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.recommendation.ALS;
import org.apache.spark.mllib.recommendation.MatrixFactorizationModel;
import org.apache.spark.mllib.recommendation.Rating;
import scala.Int;
import scala.Tuple2;

import java.util.List;

public class MyRecommend2 {

    public static void main(String[] args) {

        SparkConf sparkConf = new SparkConf()
                .setAppName("MyRecommend")
                .setMaster("local[*]");
        JavaSparkContext jsc = new JavaSparkContext(sparkConf);
        jsc.setLogLevel("WARN"); //设置日志级别

        //读取数据文件
        JavaRDD<String> sourceRDD = jsc.textFile("F:\\ml-100k\\u.data");

        //对文件做分割的处理 \t
        JavaRDD<String[]> lineRdd = sourceRDD.map(v -> v.split("\t"));

        //装载样本评分数据,其中最后一列Timestamp取除10的余数作为key,Rating为值,即(Int,Rating)
        JavaPairRDD<Integer, Rating> ratings = lineRdd.mapToPair(v -> {
            Rating rating = new Rating(Integer.valueOf(v[0]), Integer.valueOf(v[1]), Double.valueOf(v[2]));
            Integer key = Integer.valueOf(v[3]) % 10;
            return new Tuple2<Integer, Rating>(key, rating);
        });

        //装载电影目录对照表(电影ID->电影标题)
        List<Tuple2> movies = jsc.textFile("F:\\ml-100k\\u.item").map(v -> {
            String[] ss = v.split("\\|");
            return new Tuple2(ss[0], ss[1]);
        }).collect();

        //统计有用户数量和电影数量以及用户对电影的评分数目
        long numUsers = ratings.map(v -> v._2().user()).distinct().count();
        Long numRatings = ratings.count();
        Long numMovies = ratings.map(v1 -> (v1._2()).product()).distinct().count();
        System.out.println("用户:" + numUsers + "电影:" + numMovies + "评论:" + numRatings);

        //将样本评分表以key值切分成3个部分,分别用于训练 (60%,并加入用户评分), 校验 (20%), and 测试 (20%)
        // key :0 ~ 9  60% -> <6 20% -> 6<=x<8 20% x>=8

        Integer numPartitions = 4; // 分区数

        //训练集
        JavaRDD<Rating> training = ratings.filter(v -> v._1() < 6)
                .values()
                .repartition(numPartitions)
                .cache();

        // 校验集
        JavaRDD<Rating> validation = ratings
                .filter(v -> v._1() >= 6 && v._1() < 8)
                .values()
                .repartition(numPartitions).cache();

        // 测试集
        JavaRDD<Rating> test = ratings
                .filter(v -> v._1() >= 8)
                .values()
                .cache();

        Long numTraining = training.count();
        Long numValidation = validation.count();
        Long numTest = test.count();
        System.out.println("训练集:" + numTraining + " 校验集:" + numValidation + " 测试集:" + numTest);

        int[] ranks = new int[]{10, 11, 12};
//        double[] lambdas = new double[]{0.01, 0.03, 0.1, 0.3, 1, 3};
        double[] lambdas = new double[]{0.01};
//        int[] numIters = new int[]{8, 9, 10, 11, 12, 13, 14, 15};
        int[] numIters = new int[]{8, 9, 10};

        MatrixFactorizationModel bestModel = null;
        double bestValidationRmse = Double.MAX_VALUE;
        int bestRank = 0;
        double bestLambda = -0.01;
        int bestNumIter = 0;

        //对上面的三个参数做循环,这里使用三层嵌套循环来测试
        for (int rank : ranks) {
            for (double lambda : lambdas) {
                for (int numIter : numIters) {
                    MatrixFactorizationModel model = ALS.train(training.rdd(), rank, numIter, lambda);
                    Double rmse = computeRmse(model, validation, numValidation);
                    System.out.println("RMSE(校验集) = " + rmse + ", rank = " + rank + ", lambda = " + lambda + ", numIter = " + numIter);

                    if(rmse < bestValidationRmse){
                        bestValidationRmse = rmse;
                        bestModel = model;
                        bestRank = rank;
                        bestLambda = lambda;
                        bestNumIter = numIter;
                    }

                }
            }
        }

        double testRmse = computeRmse(bestModel, test, numTest);
        System.out.println("测试数据集在 最佳训练模型 rank = " + bestRank + ", lambda = " + bestLambda + ", numIter = " + bestNumIter + ", RMSE = " + testRmse);

        // 计算均值
        Double meanRating = training.union(validation).mapToDouble(v -> v.rating()).mean();

        // 计算标准误差值
        Double baselineRmse = Math.sqrt(test.map(v -> (meanRating - v.rating()) * (meanRating - v.rating())).reduce((v1, v2) -> (v1 + v2) / numTest));

        // 计算准确率提升了多少
        double improvement = (baselineRmse - testRmse) / baselineRmse * 100;

        System.out.println("最佳训练模型的准确率提升了:" + String.format("%.2f", improvement) + "%.");

        //构建最佳训练模型
        MatrixFactorizationModel model = ALS.train(ratings.values().rdd(), bestRank, bestNumIter, bestLambda);

        //使用最佳的训练模型进行推荐
        Rating[] recommendProducts = model.recommendProducts(789, 10);
        //打印推荐结果
        for (Rating rating : recommendProducts) {
            System.out.println(rating.user() + "->" + rating.product()+": " + rating.rating());
        }

        jsc.stop();

    }

    /**
     * 校验集预测数据和实际数据之间的均方根误差
     **/
    public static Double computeRmse(MatrixFactorizationModel model, JavaRDD<Rating> data, Long n) {
        // 进行预测
        JavaRDD<Rating> predictions = model.predict(data.mapToPair(v -> new Tuple2<>(v.user(), v.product())));

        JavaRDD<Tuple2<Double, Double>> predictionsAndRatings = predictions
                .mapToPair(v -> new Tuple2<>(new Tuple2<>(v.user(), v.product()), v.rating()))
                .join(data.mapToPair(v -> new Tuple2<>(new Tuple2<>(v.user(), v.product()), v.rating()))).values();

        Double reduce = predictionsAndRatings.map(v -> (v._1 - v._2) * (v._1 - v._2))
                .reduce((v1, v2) -> (v1 + v2) / n);
        //正平方根
        return Math.sqrt(reduce);
    }

}

TestMLlib

package cn.itcast.spark;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.linalg.Matrices;
import org.apache.spark.mllib.linalg.Matrix;
import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.mllib.linalg.distributed.*;
import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.junit.Test;
import scala.Tuple2;

import java.util.List;

public class TestMLlib {

    @Test
    public void testDense() {
        Vector dense = Vectors.dense(9, 5, 2, 7);//定义密集型数据集
        double v = dense.apply(1); //根据下标获取值
        System.out.println(v);
    }

    @Test //测试稀疏型的数据集
    public void testSparse() {
        int[] indexes = new int[]{0, 1, 5, 9};
        double[] values = new double[]{9, 5, 2, 7};
        Vector sparse = Vectors.sparse(10, indexes, values); //定义稀疏型数据集
        System.out.println(sparse.apply(2));
        System.out.println(sparse.apply(3));
        System.out.println(sparse.apply(9));
        System.out.println(sparse.apply(7));

    }

    @Test //测试向量标签
    public void testLabeledPoint() {
        // 密集型标签
        Vector dense = Vectors.dense(9, 5, 2, 7);//定义密集型数据集
        LabeledPoint lp = new LabeledPoint(1, dense);//定义向量标签
        System.out.println(lp.label()); //打印标签
        System.out.println(lp.features()); //打印特征,数据集
        System.out.println(lp);


        int[] indexes = new int[]{0, 1, 5, 9};
        double[] values = new double[]{9, 5, 2, 7};
        Vector sparse = Vectors.sparse(10, indexes, values); //定义稀疏型数据集
        LabeledPoint lp2 = new LabeledPoint(2, sparse);// 定义向量标签
        System.out.println(lp2);
        System.out.println(lp2.label());
        System.out.println(lp2.features());
    }

    @Test //测试本地矩阵
    public void testMatrix() {
        double[] values = new double[]{1, 2, 3, 4, 5, 6};
        Matrix matrix = Matrices.dense(2, 3, values);
        System.out.println(matrix);
    }

    @Test //测试分布式矩阵,行矩阵
    public void testRowMatrix(){

        SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("TestMLlib");

        JavaSparkContext jsc = new JavaSparkContext(sparkConf);

        //读取文件
        JavaRDD<String> fileRdd = jsc.textFile("F:\\code\\files\\a.txt");

        JavaRDD<Vector> vectorJavaRDD = fileRdd.map(v -> {
            String[] ss = v.split(" ");
            double[] ds = new double[ss.length];
            for (int i = 0; i < ss.length; i++) {
                ds[i] = Double.valueOf(ss[i]);
            }
            return Vectors.dense(ds);
        });


        RowMatrix rowMatrix = new RowMatrix(vectorJavaRDD.rdd()); //行矩阵
        System.out.println(rowMatrix.numRows()); //行数
        System.out.println(rowMatrix.numCols()); //列数

    }

    @Test //测试分布式矩阵,行索引矩阵
    public void testIndexedRowMatrix(){

        SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("TestMLlib");

        JavaSparkContext jsc = new JavaSparkContext(sparkConf);

        //读取文件
        JavaRDD<String> fileRdd = jsc.textFile("F:\\code\\files\\a.txt");

        JavaRDD<Vector> rdd1 = fileRdd.map(v -> {
            String[] ss = v.split(" ");
            double[] ds = new double[ss.length];
            for (int i = 0; i < ss.length; i++) {
                ds[i] = Double.valueOf(ss[i]);
            }
            return Vectors.dense(ds);
        });

        JavaRDD<IndexedRow> rdd2 = rdd1.map(v -> new IndexedRow(v.size(), v));


        //行索引矩阵
        IndexedRowMatrix indexedRowMatrix = new IndexedRowMatrix(rdd2.rdd());
        System.out.println(indexedRowMatrix.numRows());
        System.out.println(indexedRowMatrix.numCols());

        JavaRDD<IndexedRow> rdd3 = indexedRowMatrix.rows().toJavaRDD();
        List<IndexedRow> list = rdd3.collect();
        list.forEach(indexedRow -> System.out.println(indexedRow));

    }

    @Test //测试分布式矩阵,坐标矩阵
    public void testCoordinateMatrix(){

        SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("TestMLlib");

        JavaSparkContext jsc = new JavaSparkContext(sparkConf);

        //读取文件
        JavaRDD<String> fileRdd = jsc.textFile("F:\\code\\files\\a.txt");

        JavaRDD<MatrixEntry> rdd = fileRdd.map(v -> {
            String[] ss = v.split(" ");
            return new MatrixEntry(Long.valueOf(ss[0]), Long.valueOf(ss[1]),Double.valueOf(ss[2]));
        });


        //坐标矩阵
        CoordinateMatrix coordinateMatrix = new CoordinateMatrix(rdd.rdd());

        List<MatrixEntry> collect = coordinateMatrix.entries().toJavaRDD().collect();
        collect.forEach(matrixEntry -> System.out.println(matrixEntry));

    }

    @Test //测试分布式矩阵,分块矩阵
    public void testBlockMatrix(){

        SparkConf sparkConf = new SparkConf().setMaster("local[*]").setAppName("TestMLlib");

        JavaSparkContext jsc = new JavaSparkContext(sparkConf);

        //读取文件
        JavaRDD<String> fileRdd = jsc.textFile("F:\\code\\files\\a.txt");

        JavaRDD<MatrixEntry> rdd = fileRdd.map(v -> {
            String[] ss = v.split(" ");
            return new MatrixEntry(Long.valueOf(ss[0]), Long.valueOf(ss[1]),Double.valueOf(ss[2]));
        });


        //坐标矩阵
        CoordinateMatrix coordinateMatrix = new CoordinateMatrix(rdd.rdd());

        // 转化得到分块矩阵
        BlockMatrix blockMatrix = coordinateMatrix.toBlockMatrix();

        // 校验分块是否正确,如果不正确会抛出异常
        blockMatrix.validate();

        System.out.println(blockMatrix.numCols());
        System.out.println(blockMatrix.numRows());
        System.out.println("--------");

        // 两个矩阵相加
        BlockMatrix addBlockMatrix = blockMatrix.add(blockMatrix);

        System.out.println(addBlockMatrix.numCols());
        System.out.println(addBlockMatrix.numRows());

        List<Tuple2<Tuple2<Object, Object>, Matrix>> collect = addBlockMatrix.blocks().toJavaRDD().collect();
        collect.forEach(tuple2MatrixTuple2 -> System.out.println(tuple2MatrixTuple2));

    }

    @Test
    public void testDataSet(){
        SparkSession sparkSession = SparkSession
                .builder()
                .master("local[*]")
                .appName("TestMLlib")
                .getOrCreate();

        JavaRDD<String> rdd1 = sparkSession.read().textFile("F:\\code\\files\\users.txt").javaRDD();

        JavaRDD<Student> rdd2 = rdd1.map(v -> {
            String[] ss = v.split(",");

            Student student = new Student();
            student.setId(Long.valueOf(ss[0]));
            student.setName(ss[1]);
            student.setAge(Integer.valueOf(ss[2]));
            return student;
        });

        Dataset<Row> dataSet = sparkSession.createDataFrame(rdd2, Student.class); //转化得到dataSet

//        dataSet.show();

        dataSet.select("id","name").orderBy(dataSet.col("id").desc()).show();



    }


}

评论

暂无评论