基于模型融合的推荐系统实现(2)：迭代式SVD分解

SVD算法的原理网络上也有很多,不再细说了,关键是我们得到的数据是不完整的数据,所以要算SVD就必须做一次矩阵补全。补全的方式有很多,这里推荐使用均值补全的方法(用每一行均值和每一列均值的平均来代替空白处)，然后可以计算SVD,作PCA分析,然后就可以得到预测结果。

但是我们这里有一个极为关键的思路,迭代是SVD,我们用第一次预测得到的SVD的值来原来的均值预测,然后继续做SVD分解,直到收敛。这里的方法非常有效,最后得到的效果也不错(RMSE在0.87左右，第一次迭代的RMSE接近0.98)

同样将中间结果保存到文本文件里面,使得程序可以中断之后继续计算。

import numpy as npfrom queue import PriorityQueuefrom collections import Iterable,Counter,namedtuple,ChainMap,defaultdictfrom functools import reducefrom itertools import groupby,chain,compressfrom statistics import meanfrom code import read_filefrom PCA import get_traindef get_mean(train):    mean_u,mean_i,cnt = {},defaultdict(lambda:0),defaultdict(lambda:0)    for u,user_items in train.items():        mean_u[u] = mean(user_items.values())        for item,r in user_items.items():            mean_i[item]+=r            cnt[item]+=1    sum = 0    for each,mean_r in mean_i.items():        mean_i[each] = mean_r/cnt[each]        sum+=mean_i[each]    return mean_u,mean_i,sum/len(mean_i)def construct_matrix(train=get_train(path=r'smaller_train.txt')):#get train data from smaller data set    row = max(train)    col = 0    mean_u,mean_i,all_mean = get_mean(train)    for u,i in train.items():        col = max(col,max(i))    matrix = np.zeros((row,col))    for u,user_items in train.items():        for i in range(col):            mean_r = (mean_u[u]+mean_i[i+1])/2            if (i+1) in user_items:                matrix[u-1][i] = round(user_items[i+1]-all_mean,3)            else:                matrix[u-1][i] = round(mean_r-all_mean,3)    return matrixdef save_svd_predict(k):    initial = construct_matrix()    n = get_svd_predict(index=k)#get last result    print('svd start')    train = get_train(path = r'smaller_train.txt')    mean_u,mean_i,all_mean = get_mean(train)    u,s,v = None,None,None    for step in range(10):        print(step)        u,s,v = np.linalg.svd(n)        u = u[:,:k]        s = s[:k]        v = v[:k,:]        S = np.diag(s)        n = np.dot(u,np.dot(S,v))        np.savetxt('u{}.txt'.format(k),u)        np.savetxt('s{}.txt'.format(k),S)        np.savetxt('v{}.txt'.format(k),v)        RMES(get_svd_predict(index=k),all_mean)        for row_index in range(len(n)):            user = train[row_index+1]            row_ini = initial[row_index]            row_iter = n[row_index]            for col in range(len(n[0])):                if col+1 in user:#recover value rated                    row_iter[col] = row_ini[col]    print('svd finished')def get_svd_predict(index):    u = np.loadtxt('u{0}.txt'.format(index))    s = np.loadtxt('s{0}.txt'.format(index))    v = np.loadtxt('v{0}.txt'.format(index))    return np.dot(u,np.dot(s,v))def svd_predict(u,i,predictions):    try:        x = predictions[u-1][i-1]        return x    except:        return Nonedef write_ans(w_path,data):    with open(w_path,'w'):        pass    with open(w_path,'a') as file:            for r in data:                file.write('{0:.3f}\n'.format(r))