ALS（python pyspark）

简介

ALS是alternating least squares的缩写 , 意为交替最小二乘法。该方法常用于基于矩阵分解的推荐系统中。例如：将用户(user)对商品(item)的评分矩阵分解为两个矩阵：一个是用户对商品隐含特征的偏好矩阵，另一个是商品所包含的隐含特征的矩阵。在这个矩阵分解的过程中，评分缺失项得到了填充，也就是说我们可以基于这个填充的评分来给用户最商品推荐了

公式推导

1. 固定Y，求解X
   
2. 固定X，求解Y
   

python实现

import numpydef mf_als(R, K, steps=15, beta=0.0002):    M, N = R.shape    X = numpy.random.rand(M, K)    X = numpy.mat(X)    Y = numpy.random.rand(N, K)    Y = numpy.mat(Y)    I = numpy.mat(numpy.eye(K))    err = []    it = []    for step in xrange(steps):        X1 = []        Y1 = []        P = (Y.T * Y + beta * I).I * Y.T        for i in xrange(M):            X1.append([j[0, 0] for j in (P * R[i, :].T)])        X = numpy.mat(X1)        Q = (X.T * X + beta * I).I * X.T        for i in xrange(N):            Y1.append([j[0, 0] for j in (Q * R[:, i])])        Y = numpy.mat(Y1)        it.append(step)        err.append(numpy.sqrt(numpy.sum(pow(numpy.array(R - X * Y.T), 2)) / (M * N)))    return it, errif __name__ == "__main__":    M = 100    F = 30    U = 300    R = numpy.matrix(numpy.random.rand(M, F)) * numpy.matrix(numpy.random.rand(U, F)).T    it, err1 = mf_als(R, F, beta=0.1)    it, err2 = mf_als(R, F, beta=0.01)    it, err3 = mf_als(R, F, beta=0.001)    it, err4 = mf_als(R, F, beta=0.0001)    import matplotlib.pyplot as plt    fig = plt.figure()    ax = fig.add_subplot(111)    ax.plot(it, err1, 'r', label='beta=0.1')    ax.plot(it, err2, 'b', label='beta=0.01')    ax.plot(it, err3, 'y', label='beta=0.001')    ax.plot(it, err4, 'g', label='beta=0.0001')    ax.legend(loc='upper right')    plt.xlabel('iterations')    plt.ylabel('rsme')    plt.show()

实验效果

pyspark实现
整体思路就是把矩阵拆成行向量，分别来做最小二乘参数估计

1. 并行求解
2. 并行求解
3. 两者串行求解

# coding:utf-8"""SimpleApp"""from pyspark import SparkContext, SparkConfimport numpy as npappName = "qinq"master = "local"conf = SparkConf().setAppName(appName).setMaster(master)sc = SparkContext(conf=conf)M = 100U = 200F = 30ITERATIONS = 50LAMBDA = 0.01# M： item个数， U： user个数， F： 分解矩阵的秩# 初始化评分矩阵R = np.mat(np.random.rand(M, F)) * np.mat(np.random.rand(U, F)).Tms = np.mat(np.random.rand(M, F))us = np.mat(np.random.rand(U, F))# 将评分矩阵，item矩阵，user矩阵广播到所有节点Rb = sc.broadcast(R)msb = sc.broadcast(ms)usb = sc.broadcast(us)def update(mat, rating):    # 变成可逆矩阵    XtX = mat.T * mat    Xty = mat.T * rating    # 正则项    for j in range(F):        XtX[j, j] += LAMBDA * 1    return np.linalg.solve(XtX, Xty)def rmse(R, ms, us):    return np.sqrt(np.sum(pow(np.array(R - ms * us.T), 2)) / (M * U))steps = []errors = []for i in range(ITERATIONS):    ms = sc.parallelize(range(M)).map(lambda x: update(usb.value, Rb.value[x, :].T)).collect()    ms = np.mat(np.array(ms)[:, :, 0])    msb = sc.broadcast(ms)    us = sc.parallelize(range(U)).map(lambda x: update(msb.value, Rb.value[:, x])).collect()    us = np.mat(np.array(us)[:, :, 0])    usb = sc.broadcast(us)    error = rmse(R, ms, us)    errors.append(error)    steps.append(i)    print i, errorimport matplotlib.pyplot as pltplt.plot(steps, errors, 'r')plt.xlabel('iterations')plt.ylabel('rsme')plt.show()

实现效果