机器学习笔记——Neural Network

神经网路算法Neural Network

神经网络包含输入层input layer、隐藏层hidden layer、输出层output layer三部分。多层神经网络中常用的优化参数算法，backpropagation/反向传播算法。

多层神经网络结构如图：

其中隐藏层通常有多层组成；

神经网络中每个节点i的输入为：

在节点处进行非线性化处理：

f为激活函数，常用的形式有：tanh(x) 和 Sigmoid(x)

tanh(x)函数:

tanh(x)函数图像为：

tanh(x)导数形式为：

Sigmoid(x)函数:

Sigmoid(x)函数图像为：

Sigmoid(x)导数形式为：

选择Sigmoid函数作为激活函数时，则
对于输出层反向传递的误差为：

对于隐藏层反向传递的误差为：

权重w的更新方式为：

Bias的更新方式为：

Python实现：

#!/usr/bin/python# -*- coding: utf-8-*-# #Neural Network算法import numpy as npdef tanh(x):    return np.tanh(x)# #tanh()导数def tanh_deriv(x):    return 1.0 -np.tanh(x)*np.tanh(x)def logistic(x):    return 1/(1 + np.exp(-x))# #logistic()导数def logistic_derivative(x):    return logistic(x)*(1 - logistic(x))# #定义类NeuralNetworkclass NeuralNetwork:    # #__init__ 构造函数   self相当于this当前类的指针    def __init__(self, layers, activation = 'tanh'):        """        :param layers:a list 包含几个数表示有几层；数值表示对应层的神经元个数；        :param activation: 指定函数；默认为tanh函数；        """        if activation == 'logistic':            self.activation = logistic            self.activation_deriv = logistic_derivative        elif activation == 'tanh':            self.activation = tanh            self.activation_deriv = tanh_deriv        self.weights = []  # 初始化weights        for i in range(1, len(layers)-1):            self.weights.append((2*np.random.random((layers[i - 1] + 1,layers[i] + 1)) - 1)*0.25)            self.weights.append((2*np.random.random((layers[i] + 1,layers[i + 1])) - 1)*0.25)    # #X为数据集，每一行为一个实例；y为label；抽取X中的一个样本进行更新，循环epochs次    def fit(self, X, y, learning_rate=0.2, epochs=10000):        # #确定X至少是二维的数据        X = np.atleast_2d(X)        # #初始化temp为1；与X同行数，列数比X多1        temp = np.ones([X.shape[0], X.shape[1]+1])        temp[:, 0:-1] = X        X = temp        # #转换y的数据类型        y = np.array(y)        # #每次循环从X中随机抽取1行，对神经网络进行更新        for k in range(epochs):            i = np.random.randint(X.shape[0])            # #取X中的1行，即一个实例放入a中            a = [X[i]]            for l in range(len(self.weights)):                # # a与weights内积                a.append(self.activation(np.dot(a[l], self.weights[l])))            # #实际的标记值减去标签值            error = y[i] - a[-1]            deltas = [error * self.activation_deriv(a[-1])]            # #反向            for l in range(len(a) - 2, 0, -1):                deltas.append(deltas[-1].dot(self.weights[l].T)*self.activation_deriv(a[l]))            deltas.reverse()            for i in range(len(self.weights)):                layer = np.atleast_2d(a[i])                # #权重的更新                delta =np.atleast_2d(deltas[i])                self.weights[i] += learning_rate * layer.T.dot(delta)    def predict(self,x):        x = np.array(x)        temp = np.ones(x.shape[0] + 1)        temp[0:-1] = x        a = temp        for l in range(0, len(self.weights)):            a = self.activation(np.dot(a,self.weights[l]))        return a

写入NeuralNetwork.py文件。

示例1、调用NeuralNetwork中的算法，查看Neural Network算法实现效果：

#!/usr/bin/python# -*- coding: utf-8 -*-# #Neural Network算法实例from NeuralNetwork import NeuralNetworkimport numpy as np# #2个输入，2层神经网络，1个输出nn = NeuralNetwork([2,2,1],'tanh')X = np.array([[0,0],[0,1],[1,0],[1,1]])y = np.array([0, 1, 1, 0])nn.fit(X, y)for i in [[0,0],[0,1],[1,0],[1,1]]:    print i, nn.predict(i)

输出结果：

[0, 0] [ 0.00873859][0, 1] [ 0.99834637][1, 0] [ 0.99827345][1, 1] [ 0.01632346]

可以设置一个threshold（如0.5）即可知四个的分类为[0, 1, 1, 0]。

示例、调用NeuralNetwork中的算法和sklearn中自带的手写数字数据集，查看Neural Network算法实现效果：

# !/usr/bin/python# -*- coding: utf-8 -*-# #手写数字数据集分类import numpy as npfrom sklearn.datasets import load_digitsfrom sklearn.metrics import confusion_matrix,classification_reportfrom sklearn.preprocessing import LabelBinarizerfrom NeuralNetwork import NeuralNetworkfrom sklearn.model_selection import train_test_split# #交叉验证# from sklearn.cross_validation import train_test_split# #下载数据集digits = load_digits()# #将特征量放入XX = digits.data# #将标签值放入yy =digits.target# #预处理：将X标准化到0/1之间；X -= X.min()X /= X.max()# #实例化一个神经网络；64个像素点，10个输出类别，通常隐藏层层数要比输入节点多一些nn = NeuralNetwork([64, 100, 10], 'logistic')# #将数据集分为训练集和测试集X_train, X_test, y_train, y_test = train_test_split(X, y)# #将label数据转换为符合sklearn库的输入要求{即，数字对应位为1其余为0}labels_train = LabelBinarizer().fit_transform(y_train)labels_test = LabelBinarizer().fit_transform(y_test)print "start fitting"# #循环3000次nn.fit(X_train, labels_train,epochs=3000)predictions = []for i in range(X_test.shape[0]):    o = nn.predict(X_test[i])    # #选择1个概率最大的作为预测结果    predictions.append(np.argmax(o))# #呈现预测结果矩阵print confusion_matrix(y_test, predictions)# #打印出预测准确度的度量结果print classification_report(y_test, predictions)

打印结果：

start fitting[[36  0  0  0  1  0  0  0  0  0] [ 0 46  0  0  1  0  2  0  4  1] [ 0  1 52  0  0  0  0  0  0  0] [ 0  0  0 38  0  1  0  1  1  1] [ 1  3  0  0 38  0  0  0  1  0] [ 0  0  0  1  1 40  0  0  0  0] [ 1  0  0  0  0  0 46  0  0  0] [ 0  0  0  0  0  0  0 53  1  0] [ 0  5  1  0  0  0  1  1 28  0] [ 0  1  0  4  1  1  0  0  0 35]]             precision    recall  f1-score   support          0       0.95      0.97      0.96        37          1       0.82      0.85      0.84        54          2       0.98      0.98      0.98        53          3       0.88      0.90      0.89        42          4       0.90      0.88      0.89        43          5       0.95      0.95      0.95        42          6       0.94      0.98      0.96        47          7       0.96      0.98      0.97        54          8       0.80      0.78      0.79        36          9       0.95      0.83      0.89        42avg / total       0.92      0.92      0.92       450

Hierarchical Clustering　层次聚类　

利用层次聚类以图像颜色对图片进行聚类：
其中’map’文件夹中放入110张jpg格式的图片．

#!/usr/bin/python# coding:utf-8from PIL import Image, ImageDrawfrom clustering import hclusterfrom clustering import getheightfrom clustering import getdepthimport numpy as npimport osdef drawdendrogram(clust, imlist, jpeg= 'clusters.jpg'):    h = getheight(clust)*20    w = 1200    depth = getdepth(clust)    scaling = float(w - 150)/depth    img = Image.new('RGB', (w, h), (255, 255, 255))    draw = ImageDraw.Draw(img)    draw.line((0, h/2, 10, h/2), fill=(255, 0, 0))    drawnode(draw, clust, 10, int(h/2), scaling, imlist, img)    img.save(jpeg)def drawnode(draw,clust,x,y,scaling,imlist,img):    if clust.id < 0:        h1 = getheight(clust.left)*20        h2 = getheight(clust.right)*20        top = y - (h1 + h2)/2        bottom = y + (h1 + h2)/2        ll = clust.distance * scaling        #　若不是叶子节点则画线        draw.line((x, top + h1/2, x, bottom - h2/2), fill=(255, 0, 0))        draw.line((x, top + h1/2, x + ll, top + h1/2), fill=(255, 0, 0))        draw.line((x, bottom - h2/2, x + ll, bottom - h2/2), fill=(255, 0, 0))        drawnode(draw, clust.left, x + ll, top + h1/2, scaling, imlist, img)        drawnode(draw, clust.right, x + ll, bottom - h2/2, scaling, imlist, img)    else:        nodeim = Image.open(imlist[clust.id])        # 画出缩略图        nodeim.thumbnail((20, 20))        ns = nodeim.size        print x,y - ns[1]//2        print x + ns[0]        print        img.paste(nodeim, (int(x), int(y - ns[1]//2), int(x + ns[0]),int(y + ns[1] - ns[1]//2)))imlist = []folderPath = r'map'for filename in os.listdir(folderPath):    if os.path.splitext(filename)[1] == '.jpg':        imlist.append(os.path.join(folderPath, filename))n = len(imlist)print nfeatures = np.zeros((n, 3))for i in range(n):    im = np.array(Image.open(imlist[i]))    R = np.mean(im[:, :, 0].flatten())    G = np.mean(im[:, :, 1].flatten())    B = np.mean(im[:, :, 2].flatten())    features[i] = np.array([R, G, B])tree = hcluster(features)drawdendrogram(tree, imlist, jpeg='map.jpg')

在clustering.py文件中编写层次聚类所使用的类及函数．

#!/usr/bin/python# coding:utf-8from numpy import *"""层次聚类"""# 创建节点类及对应属性class cluster_mode:    def __init__(self, vec, left = None, right = None, distance =0.0, id = None, count =1):        self.left = left        self.right = right        self.vec = vec        self.id = id        self.distance = distance        self.cont = countdef L2dist(v1 ,v2):    return sqrt(sum((v1 - v2)**2))def L1dist(v1, v2):    return sum(abs(v1 - v2))# 传入features为数据矩阵，每一行为一个实例点def hcluster(features, distance = L2dist):    distances = {}    currentclustid = -1    # 每一点自成一类,按features的行数进行分类    clust = [cluster_mode(array(features[i]), id=i) for i in range(len(features))]    # 当类数大于1时继续循环    while len(clust) > 1:        # 初始化，取前两个点        lowestpair = (0, 1)        closest = distance(clust[0].vec, clust[1].vec)        for i in range(len(clust)):            for j in range(i+1, len(clust)):                if (clust[i].id, clust[j].id) not in distances:                    distances[(clust[i].id, clust[j].id)] = distance(clust[i].vec, clust[j].vec)                d = distances[(clust[i].id, clust[j].id)]                #　若当前距离小于最小距离，则将当前距离作为closest                if d < closest:                    closest = d                    lowestpair = (i, j)        #　通过计算两两之间的平均值来衡量类之间的相似度        mergevec = [(clust[lowestpair[0]].vec[i] + clust[lowestpair[1]].vec[i]) / 2.0                    for i in range(len(clust[0].vec))]        # 选最近的距离创建新的节点/类        newcluster = cluster_mode(array(mergevec), left= clust[lowestpair[0]],                                  right=clust[lowestpair[1]], distance= closest, id= currentclustid)        currentclustid -= 1        # 删除旧的clust并添加新的clust        del clust[lowestpair[1]]        del clust[lowestpair[0]]        clust.append(newcluster)    return clust[0]# 通过设置阈值dist将cluster取出def extract_clusters(clust, dist):    clusts = {}    # 若树状结构的distance比给定的阈值小，则直接返回    if clust.distance < dist:        return [clust]    # 若树状结构的distance大于给定阈值，则继续往下看左右的分支节点距离    else:        cl = []        cr = []        #　若节点不为None，则取出其dist，递归返回        if clust.left != None:            cl = extract_clusters(clust.left, dist= dist)        if clust.right != None:            cr = extract_clusters(clust.right, dist= dist)        return cl + cr#　将cluster中的元素取出def get_cluster_elements(clust):    if clust.id >= 0:        return [clust.id]    # 若id小于0,进行递归调用    else:        cl = []        cr = []        if clust.left != None:            cl = get_cluster_elements(clust.left)        if clust.right != None:            cr = get_cluster_elements(clust.right)#def printclust(clust, labels= None, n= 0):    for i in range(n): print ' ',    # 若当前的id小于0则,当前clust为支点    if clust.id < 0:        print '-'    # 若当前id大于等于0则表示当前节点为枝叶    else:        if labels == None: print clust.id        # 若不为None则打印出其id        else: print labels[clust.id]    #　通过递归的方法将左边和右边的分支都打印出来    if clust.left != None: printclust(clust.left, labels= labels, n= n+1)    if clust.right != None: printclust(clust.right, labels= labels, n= n+1)#　获取树的高度def getheight(clust):    #　若树没有左右支点，即只有一个原点，则返回树的高度为1    if clust.left== None and clust.right== None: return 1    # 否则返回左右分支的高度之和    return getheight(clust.left) + getheight(clust.right)#　获取树的深度def getdepth(clust):    # 　若树没有左右支点，即只有一个原点，则返回树的深度为0    if clust.left== None and clust.right== None: return 0    # 否则返回左右分支的深度最大的一个加上distance    return max(getdepth(clust.left), getdepth(clust.right)) + clust.distance

输出结果为：

110671.889031583 0690.889031583671.889031583 20691.889031583726.292592208 44746.292592208726.292592208 64746.292592208741.216242107 83761.216242107741.216242107 104761.216242107655.135915648 124675.135915648...1009.04888336 21231029.048883361009.04888336 21451029.04888336951.812241088 2163971.812241088951.812241088 2180971.812241088

.