神经网络初探

一.BP神经网络:曾经的最著名、最经典的非线性学习算法

  (1)BP神经网络基本结构

    

   (2)传递激活函数

    

 (3)训练过程

     a.正向传播过程

   

     b.计算期望与实际分类的误差

  

     c.计算反向传播过程

   

     d.修正各层的权值

   

(4)代码实现

# -*- coding: UTF-8 -*-from numpy import *import operatorimport Untilsimport matplotlib.pyplot as plt # 传递函数:def logistic(inX):    return 1.0/(1.0+exp(-inX))# 传递函数的导函数def dlogit(inX1,inX2):    return multiply(inX2,(1.0-inX2))# 矩阵各元素平方之和def errorfunc(inX):    return sum(power(inX,2))/2.0    # 加载student.txt数据集def loadDataSet(filename):    dataMat = []; labelMat = []    fr = open(filename) #testSet.txt    for line in fr.readlines():        lineArr = line.strip().split()        dataMat.append([float(lineArr[0]), float(lineArr[1]), 1.0])        labelMat.append(int(lineArr[2]))    return dataMat,labelMat   # 数据标准化(归一化):student.txt数据集def normalize(dataMat):    # 标准化    dataMat[:,0] = (dataMat[:,0]-mean(dataMat[:,0]))/std(dataMat[:,0])    dataMat[:,1] = (dataMat[:,1]-mean(dataMat[:,1]))/std(dataMat[:,1])    return dataMatdef bpNet(dataSet,classLabels):    # 数据集矩阵化    SampIn = mat(dataSet).T    expected = mat(classLabels)    m,n = shape(dataSet)     # 网络参数    eb = 0.01                   # 误差容限     eta = 0.05                  # 学习率     mc = 0.3                    # 动量因子     maxiter = 2000              # 最大迭代次数     errlist = []                # 误差列表        # 构造网络        # 初始化网络    nSampNum = m;    # 样本数量    nSampDim = n-1;  # 样本维度    nHidden = 4;   # 隐含层神经元     nOut = 1;      # 输出层        # 隐含层参数    hi_w = 2.0*(random.rand(nHidden,nSampDim)-0.5)      hi_b = 2.0*(random.rand(nHidden,1)-0.5)     hi_wb = mat(Untils.mergMatrix(mat(hi_w),mat(hi_b)))        # 输出层参数    out_w = 2.0*(random.rand(nOut,nHidden)-0.5)     out_b = 2.0*(random.rand(nOut,1)-0.5)    out_wb = mat(Untils.mergMatrix(mat(out_w),mat(out_b)))    # 默认旧权值    dout_wbOld = 0.0 ; dhi_wbOld = 0.0     for i in xrange(maxiter):           #1. 工作信号正向传播                #1.1 输入层到隐含层        hi_input = hi_wb*SampIn        hi_output = logistic(hi_input)                hi2out  = Untils.mergMatrix(hi_output.T, ones((nSampNum,1))).T            #1.2 隐含层到输出层            out_input = out_wb*hi2out        out_output = logistic(out_input)                #2. 误差计算             err = expected - out_output         sse = errorfunc(err)        errlist.append(sse);        #2.1 判断是否收敛        if sse <= eb:            print "iteration:",i+ 1                break;                #3.误差信号反向传播        #3.1 DELTA为输出层到隐含层梯度          DELTA = multiply(err,dlogit(out_input,out_output))        wDelta = out_wb[:,:-1].T*DELTA                 #3.2 delta为隐含层到输入层梯度        delta = multiply(wDelta,dlogit(hi_input,hi_output))                dout_wb = DELTA*hi2out.T                #3.3 输入层的权值更新        dhi_wb = delta*SampIn.T                    #3.4 更新输出层和隐含层权值        if i == 0:              out_wb = out_wb + eta * dout_wb             hi_wb = hi_wb + eta * dhi_wb        else :                out_wb = out_wb + (1.0 - mc)*eta*dout_wb  + mc * dout_wbOld            hi_wb = hi_wb + (1.0 - mc)*eta*dhi_wb + mc * dhi_wbOld        dout_wbOld = dout_wb        dhi_wbOld = dhi_wb         return errlist,out_wb,hi_wbdef BPClassfier(start,end,WEX,wex):    x = linspace(start,end,30)    xx = mat(ones((30,30)))    xx[:,0:30] = x     yy = xx.T    z = ones((len(xx),len(yy))) ;    for i in range(len(xx)):    for j in range(len(yy)):         xi = []; tauex=[] ; tautemp=[]         mat(xi.append([xx[i,j],yy[i,j],1]))          hi_input = wex*(mat(xi).T)         hi_out = logistic(hi_input)          taumrow,taucol= shape(hi_out)         tauex = mat(ones((1,taumrow+1)))         tauex[:,0:taumrow] = (hi_out.T)[:,0:taumrow]         HM = WEX*(mat(tauex).T)         out = logistic(HM)          z[i,j] = out     return x,z             

from numpy import *import operatorfrom bpNet import *import matplotlib.pyplot as plt # 数据集bpnet = BPNet() bpnet.loadDataSet("testSet2.txt")bpnet.dataMat = bpnet.normalize(bpnet.dataMat)# 绘制数据集散点图bpnet.drawClassScatter(plt)# BP神经网络进行数据分类bpnet.bpTrain()print bpnet.out_wbprint bpnet.hi_wb# 计算和绘制分类线x,z = bpnet.BPClassfier(-3.0,3.0)bpnet.classfyLine(plt,x,z)plt.show()# 绘制误差曲线bpnet.TrendLine(plt)plt.show()

测试输出结果如下:

二.自组织映射神经网络:(SOM)一种无监督的聚类算法,用于比较强的簇结构，只有两层即输入和输出层。数据松散运用KMeans.

 (1)结构图

  

(2)代码实现:

import sysimport randomimport mathclass SOM:    def __init__(self, inputDim, mapWidth, mapHeight, inputPatternSet):        self.__eta0   = 0.1        self.__sigma0 = 2.0        self.__tau1   = 250        self.__tau2   = 500        self.__patternLength = inputDim        self.__mapWidth      = mapWidth        self.__mapHeight     = mapHeight        self.__w = [[[random.random() * 0.1 for i in range(self.__patternLength)]                                            for i in range(self.__mapWidth)]                                            for i in range(self.__mapHeight)]    def coordinates(self, pattern):        winI = 0        winJ = 0        min = sys.float_info.max        for i in range(self.__mapHeight):            for j in range(self.__mapWidth):                s = 0.0                for k in range(3):                    s = s + abs(pattern.patternItem[k] - self.__w[i][j][k])                if min > s:                    min = s                    winI = i                    winJ = j        return (winI, winJ)    def clustering(self, patterns):#        for pattern in patterns:#            if not isinstance(pattern, IPattern):#                raise TypeError("Pattern must conform to IPattern")        #debug        self.__clu = [[] for i in range(len(patterns))]        self.__patterns = patterns        self.__clusteringProcess()        #self.__writeClu()    #debug    def __writeClu(self):        import csv        w = csv.writer(file(r'/Users/alexander/temp/data1.csv','wb'))        w.writerows(self.__clu)#        for i in range(len(self.__clu)):#            for j in range(len(self.__clu[i])):#                print i, j, self.__clu[i][j]    def __clusteringProcess(self):        error = 0.0        e = sys.float_info.max        de = e - error        end = False        i = 0        while not end:            error = self.__clusteringEpoch(self.__patterns, i)            de = abs(e - error)            if de < 0.0001 or i > 10000:                end = True            print "Iteration: ", i            print "Error: ", e            print "Error delta: ", de            e = error            i = i + 1    def __clusteringEpoch(self, patterns, n):        error = 0.0        randOrder = range(len(patterns))        for i in range(len(patterns)):            placeToSwap = random.randint(1, len(patterns) - 1)            temp = randOrder[i]            randOrder[i] = randOrder[placeToSwap]            randOrder[placeToSwap] = temp        for patIndex in range(len(patterns)):            currentPatternIndex = randOrder[patIndex]            pattern = self.__patterns[currentPatternIndex]            winI = 0            winJ = 0            min = sys.float_info.max            for i in range(self.__mapHeight):                for j in range(self.__mapWidth):                    s = 0.0                    for k in range(3):                        s = s + pow(pattern.patternItem[k] - self.__w[i][j][k], 2)                    s = math.sqrt(s)                    if min > s:                        min = s                        winI = i                        winJ = j            #self.__clu[currentPatternIndex].append((winI, winJ))            self.__clu[currentPatternIndex].append(winI * self.__mapWidth + winJ)            #print "(winI, winJ): ", (winI, winJ)            self.__printW()            e = 0.0            eta = self.__eta(n)            for i in range(self.__mapHeight):                for j in range(self.__mapWidth):                    nbh = self.__neighbourhood(i, j, winI, winJ, n)                    for k in range(3):                        dif = self.__patterns[currentPatternIndex].patternItem[k] - self.__w[i][j][k]                        e = nbh * dif                        #print "w[", i, "][", j, "][", k, "]: ", self.__w[i][j][k]                        self.__w[i][j][k] += eta * e;                        #print "w[", i, "][", j, "][", k, "]: ", self.__w[i][j][k]                        error += abs(e)            self.__printW()        return error    #for debug    def __printW(self):        return        for i in range(self.__mapHeight):            for j in range(self.__mapWidth):                print i * self.__mapWidth + j, " ", self.__w[i][j]    def __eta(self, n):        n = n * 1.0        eta = self.__eta0 * math.exp(-n / self.__tau2)        #print "Eta(", n, "): ", eta        return eta    def __neighbourhood(self, i, j, centerI, centerJ, n):        di = i - centerI        dj = j - centerJ        distance2 = di * di + dj * dj                nbh = math.exp(- distance2 / (2.0 * math.pow(self.__sigma0 * math.exp(-n/self.__tau1), 2)))        #print "Nbh(", "i: ", i, "j: ", j, "n: ", n, "): ", nbh        return  nbh

import numpy as np from Kohonen import *from numpy import *import matplotlib.pyplot as plt # 矩阵各元素平方之和def errorfunc(inX):return sum(power(inX,2))*0.5# 加载坐标数据文件SOMNet = Kohonen()SOMNet.loadDataSet("dataset2.txt");SOMNet.train()print SOMNet.wSOMNet.showCluster(plt)SOMNet.TrendLine(plt,SOMNet.lratelist)SOMNet.TrendLine(plt,SOMNet.rlist)

运行结果:

三.模拟退火算法(Boltzmann机)：无监督网络学习.

 (1)简单的代码实现:

import operatorimport Untilsfrom numpy import *import copyimport matplotlib.pyplot as plt # 计算矩阵各向量之间的距离:返回一个对称的n*n矩阵def distM(matA,matB):ma,na = shape(matA);mb,nb = shape(matB);rtnmat= zeros((ma,nb))for i in xrange(ma):for j in xrange(nb):rtnmat[i,j] = sqrt(sum(power(matA[i,:] - matB[:,j].T,2)))return rtnmatdef pathLen(dist,path):# dist:N*N邻接矩阵# 长度为N的向量，包含从1-N的整数N = len(path)plen = 0;for i in xrange(0,N-1):plen += dist[path[i], path[i+1]]plen +=  dist[path[0], path[N-1]]return plendef changePath(old_path):# 在oldpath附近产生新的路径if type(old_path) is not list :old_path = old_path.tolist()N = len(old_path)if random.rand() < 0.25:   # 产生两个位置，并交换chpos = floor(random.rand(1,2)*N) # random.rand(1,2)chpos = chpos.tolist()[0]new_path = copy.deepcopy(old_path)new_path[int(chpos[0])] = old_path[int(chpos[1])]new_path[int(chpos[1])] = old_path[int(chpos[0])]  else: # 产生三个位置，交换a-b和b-c段d = ceil(random.rand(1,3)*N);d = d.tolist()[0]d.sort()a = int(d[0]); b = int(d[1]); c = int(d[2])if a != b and b != c:new_path = copy.deepcopy(old_path)new_path[a:c-1] = old_path[b-1:c-1] + old_path[a:b-1]else:new_path = changePath(old_path)return new_pathdef boltzmann(cityPosition,MAX_ITER = 2000,T0 = 1000,Lambda = 0.97):  m,n = shape(cityPosition)  pn = m  # 将城市的坐标矩阵转换为邻接矩阵（城市间距离矩阵）  dist = distM(cityPosition,cityPosition.T)    # 初始化  MAX_M = m;  # 构造一个初始可行解  x0 = arange(m)  random.shuffle(x0)  #   T = T0;  iteration = 0;  x = x0;                   # 路径变量  xx = x0.tolist();         # 每个路径  di = []  di.append(pathLen(dist, x0))   # 每个路径对应的距离  k = 0;                  # 路径计数    # 外循环  while iteration <= MAX_ITER:  # 内循环迭代器  m = 0;  # 内循环  while m <= MAX_M:  # 产生新路径  newx =  changePath(x)  # 计算距离  oldl = pathLen(dist,x)  newl = pathLen(dist,newx)  if ( oldl > newl):   # 如果新路径优于原路径，选择新路径作为下一状态  x = newx  xx.append(x)    # xx[n,:] = x  di.append(newl) # di[n] = newl  k += 1              else: # 如果新路径比原路径差，则执行概率操作  tmp = random.rand()  sigmod = exp(-(newl - oldl)/T)  if tmp < sigmod:  x = newx  xx.append(x)    # xx[n,:] = x  di.append(newl) # di[n]= newl  k += 1  m += 1            # 内循环次数加1  # 内循环  iteration += 1      # 外循环次数加1  T = T*Lambda        # 降温    # 计算最优值  bestd = min(di)   indx = argmin(di)  bestx = xx[indx]  print "循环迭代",k,"次"  print "最优解:",bestd  print "最佳路线:",bestx  return bestx,di

import operatorimport copyimport Untilsimport Boltzmannfrom numpy import *import matplotlib.pyplot as plt dataSet = Untils.loadDataSet("dataSet25.txt")cityPosition = mat(dataSet)m,n = shape(cityPosition)bestx,di = Boltzmann.boltzmann(cityPosition,MAX_ITER = 1000,T0 = 100)# 优化前城市图,路径图Untils.drawScatter(cityPosition,flag=False)Untils.drawPath(range(m),cityPosition)# 显示优化后城市图,路径图Untils.drawScatter(cityPosition,flag=False)Untils.drawPath(bestx,cityPosition,color='b')# 绘制误差趋势线x0 = range(len(di));Untils.TrendLine(x0,di)

输出结果如下: