# VARIATION OF Code from Chapter 3 of Machine Learning: An Algorithmic Perspective # by Stephen Marsland (http://seat.massey.ac.nz/personal/s.r.marsland/MLBook.html) # here, on-line (not batch) learning; the weight matrices are transposed w.r.t. Marsland's original code # here, Elman network with weights3 from hidden to hidden layer added # You are free to use, change, or redistribute the code in any way you wish for # non-commercial purposes, but please maintain the name of the original author. # This code comes with no warranty of any kind. # Stephen Marsland, 2008 from numpy import * import numpy class mlp: """ A Multi-Layer Perceptron""" def __init__(self,inputs,targets,nhidden,beta=1,momentum=0.9,outtype='linear'): """ Constructor """ # Set up network size self.nin = shape(inputs)[1] self.nout = shape(targets)[1] self.ndata = shape(inputs)[0] self.nhidden = nhidden self.beta = beta self.momentum = momentum self.outtype = outtype # Initialise network self.weights1 = (random.rand(self.nhidden,self.nin+1)-0.5)*2/sqrt(self.nin) self.weights2 = (random.rand(self.nout,self.nhidden+1)-0.5)*2/sqrt(self.nhidden) self.weights3 = (random.rand(self.nhidden,self.nhidden)-0.5)*2/sqrt(self.nhidden) def mlptrain(self,inputs,targets,eta,niterations): """ Train the thing """ # Add the inputs that match the bias node inputs = concatenate((inputs,-ones((self.ndata,1))),axis=1) updatew1 = zeros((shape(self.weights1))) updatew2 = zeros((shape(self.weights2))) updatew3 = zeros((shape(self.weights3))) for n in range(niterations): error = 0.0 self.hidden = zeros((self.ndata,self.nhidden)) self.hidden[0] = 0.5 * ones(self.nhidden) for self.iter in range(self.ndata - 1): currenttar = targets[self.iter + 1] self.outputs = self.mlpfwd(inputs[self.iter]) if self.iter > 0: error += 0.5*sum((currenttar-self.outputs)**2) # Different types of output neurons if self.outtype == 'linear': deltao = (currenttar-self.outputs) elif self.outtype == 'logistic': deltao = (currenttar-self.outputs)*self.outputs*(1.0-self.outputs) elif self.outtype == 'softmax': #deltao = (targets-self.outputs)*self.outputs/self.ndata deltao = (currenttar-self.outputs) else: print "error" deltah0 = self.hiddenplusone*(1.0-self.hiddenplusone)*(dot(transpose(self.weights2),deltao)) # not correct for the threshold weight if (self.iter > 0): deltah1 = self.hidden[self.iter-1]*(1.0-self.hidden[self.iter-1])*(dot(transpose(self.weights3),deltah0[:-1])) updatew1 = eta*(outer(deltah0[:-1],inputs[self.iter])) if (self.iter > 0): updatew1 += eta*(outer(deltah1,inputs[self.iter-1])) updatew2 = eta*(outer(deltao,self.hiddenplusone)) updatew3 = numpy.zeros((shape(self.weights3))) if (self.iter > 0): updatew3 += eta*(outer(deltah0[:-1],self.hidden[self.iter-1])) if (self.iter > 1): updatew3 += eta*(outer(deltah1,self.hidden[self.iter-2])) self.weights1 += updatew1 + self.momentum*updatew1 self.weights2 += updatew2 + self.momentum*updatew2 self.weights3 += updatew3 + self.momentum*updatew3 print "Error:", error, " W1:%.1f %.1f W2:%.1f %.1f W3:%.1f %.1f" % (numpy.min(self.weights1), numpy.max(self.weights1), numpy.min(self.weights2), numpy.max(self.weights2), numpy.min(self.weights3), numpy.max(self.weights3)) def mlpfwd(self,inputs): """ Run the network forward """ if self.iter > 0: hidden_old = self.hidden[self.iter-1] else: hidden_old = self.hidden[0] self.hidden[self.iter] = dot(self.weights1,inputs) + dot(self.weights3,hidden_old) self.hidden[self.iter] = 1.0/(1.0+exp(-self.beta*self.hidden[self.iter])) self.hiddenplusone = append(self.hidden[self.iter],-ones(1)) outputs = dot(self.weights2,self.hiddenplusone) # Different types of output neurons if self.outtype == 'linear': return outputs elif self.outtype == 'logistic': return 1.0/(1.0+exp(-self.beta*outputs)) elif self.outtype == 'softmax': normaliser = sum(exp(outputs)) return exp(outputs)/normaliser else: print "error" def mlpplot(self,inputs,targets,t_given): """ writes two files, to be plotted using the following commands in gnuplot: set style data lines plot "out0.dat", "out1.dat" """ f0 = open("out0.dat", 'wb') f1 = open("out1.dat", 'wb') inputs = concatenate((inputs,-ones((self.ndata,1))),axis=1) for self.iter in range(self.ndata - 1): currentin = inputs[self.iter] if self.iter > t_given: # after t_given time steps currentin = self.outputs # feed outputs back into the inputs currentin = append(currentin,-ones(1)) self.outputs = self.mlpfwd(currentin) # out0.dat contains the targets val_ch = str(targets[self.iter + 1][0]) + " " f0.write(val_ch) f0.write("\n") # out1.dat contains the outputs (plus 0.02 for better visibility in gnuplot) val_ch = str(self.outputs[0]) + " " f1.write(val_ch) f1.write("\n") f0.close() f1.close() def mlpplotreber(self,inputs,targets): """ visualize outputs by drawing the number of the most active unit writes two files, to be plotted using the following commands in gnuplot: set style data histeps set ytics ("B" 0, "T" 1, "P" 2, "X" 3, "S" 4, "V" 5, "E" 6) plot "out0.dat", "out1.dat" """ f0 = open("out0.dat", 'wb') f1 = open("out1.dat", 'wb') currentin = inputs[0] for self.iter in range(self.ndata - 1): if self.iter > 0: # after the first time point currentin = self.outputs # feed outputs back into the inputs currentin = append(currentin,-ones(1)) self.outputs = self.mlpfwd(currentin) # out0.dat contains the targets val_ch = str(numpy.argmax(targets[self.iter + 1])) + " " f0.write(val_ch) f0.write("\n") # out1.dat contains the outputs (plus 0.02 for better visibility in gnuplot) val_ch = str(0.02 + numpy.argmax(self.outputs)) + " " f1.write(val_ch) f1.write("\n") f0.close() f1.close()