最小二乘法python实现

最小二乘法回归参数梯度

代码

import pandas as pdimport numpy as npdf = pd.read_csv('https://archive.ics.uci.edu/ml/'                     'machine-learning-databases/iris/iris.data', header = None)import matplotlib.pyplot as plty = df.iloc[0:400,4].valuesy = np.where(y == 'Iris-setosa', -1, 1)X = df.iloc[0:400, [0,2]].valuesclass AdaLineGD(object):    def __init__(self,eta = 0.01, n_iter = 100):        self.eta = eta        self.n_iter = n_iter    def net_input(self,X):        return np.dot(X, self.w[1:]) + self.w[0]    def fit(self, X, y):        self.w = np.zeros(1 + X.shape[1])        self.cost = []        for _ in range(self.n_iter):            output = self.net_input(X)            error = y - output            self.w[1:] += self.eta * X.T.dot(error)            self.w[0] += self.eta * error.sum()            cost = (error ** 2).sum()/ 2.0            self.cost.append(cost)        return self    def activation(self,X):        return self.net_input(X)    def predict(self,xi):        return np.where(self.activation(xi) >= 0.0,1,-1)fig, ax = plt.subplots(nrows =1, ncols = 2, figsize=(8,3))ada1 = AdaLineGD( eta= 0.0002).fit(X, y)ax[0].plot(range(1, 1 + len(ada1.cost)), np.log10(ada1.cost), marker = 'o')ax[0].set_xlabel("Epoch")ax[0].set_ylabel("log(Sum-squared-error)")ax[0].set_title('Adaline - Learning rate 0.0002')ada1 = AdaLineGD(eta= 0.0001, ).fit(X, y)ax[1].plot(range(1, 1 + len(ada1.cost)), np.log10(ada1.cost), marker = 'o')ax[1].set_xlabel("Epoch")ax[1].set_ylabel("log(Sum-squared-error)")ax[1].set_title('Adaline - Learning rate 0.0001')plt.show()

训练迭代

对比三张图明确learning rate学习率的意义:失之毫厘,差之千里
且并不是学习率越小越小,学习率为0.0002时80迭代基本收敛,学习率为0.0001时,收敛较慢,跌代100次还未完全收敛(学习率过小,每步学到的东西变少,到达收敛位置的迭代成本增加)

训练数据规范化

数据规范化之后,迭代次数显著减少,30次左右即收敛

X_std = np.copy(X)X_std[:,0] = (X[:,0] - X[:,0].mean())/ X[:,0].std()X_std[:,1] = (X[:,1] - X[:,1].mean())/ X[:,1].std()

随机梯度下降

import pandas as pdimport numpy as npimport matplotlib.pyplot as pltfrom numpy.random import seedclass AdaLineGD(object):    def __init__(self,eta = 0.01, n_iter = 30,shuffle = True, random_state=None):        self.eta = eta        self.n_iter = n_iter        self.shuffle = shuffle        self.w_inited = False        if random_state:            seed(random_state)    def net_input(self,X):        return np.dot(X, self._w[1:]) + self._w[0]    def _shuffle(self, X, y):        r = np.random.permutation(len(y))        return X[r],y[r]    def _init_weight(self, m):        self._w = np.zeros(m + 1)        self.w_inited = True    def _update_weight(self, xi, target):        output = self.net_input(xi)        error = target - output        self._w[1:] += self.eta * xi.dot(error)        self._w[0] += self.eta * error        cost =  0.5 * error ** 2        return cost    def fit(self, X, y):        self._init_weight(X.shape[1])        self.cost = []        for _ in range(self.n_iter):            if self.shuffle:                X, y = self._shuffle(X, y)            cost = []            for xi, target in zip(X, y):                cost.append(self._update_weight(xi, target))            avg_cost = sum(cost) / len(y)            self.cost.append(avg_cost)        return self    def partial_fit(self, X, y):        if not self.w_inited:            self._init_weight(X.shape[1])        if y.ravel().shape[0] > 1:            for xi, target in zip(X, y):                self._update_weight(xi, target)        else:            self._update_weight(X, y)        return self    def activation(self,X):        return self.net_input(X)    def predict(self,xi):        return np.where(self.activation(xi) >= 0.0,1,-1)df = pd.read_csv('https://archive.ics.uci.edu/ml/'                     'machine-learning-databases/iris/iris.data', header = None)y = df.iloc[0:400,4].valuesy = np.where(y == 'Iris-setosa', -1, 1)X = df.iloc[0:400, [0,2]].valuesX_std = np.copy(X)X_std[:,0] = (X[:,0] - X[:,0].mean())/ X[:,0].std()X_std[:,1] = (X[:,1] - X[:,1].mean())/ X[:,1].std()fig, ax = plt.subplots(nrows =1, ncols = 1, figsize=(8,3))ada1 = AdaLineGD(eta= 0.001).fit(X_std, y)ax.plot(range(1, 1 + len(ada1.cost)), np.log10(ada1.cost), marker = 'o')ax.set_xlabel("Epoch")ax.set_ylabel("log(Sum-squared-error)")ax.set_title('Adaline - Learning rate 0.001')plt.show()

学习率过大出现局部震荡