Source Code for Module ML.Neural.Trainers

  1  # 
  2  #  Copyright (C) 2000  greg Landrum 
  3  # 
  4  """ Training algorithms for feed-forward neural nets 
  5   
  6    Unless noted otherwise, algorithms and notation are taken from: 
  7    "Artificial Neural Networks: Theory and Applications", 
  8      Dan W. Patterson, Prentice Hall, 1996 
  9     
 10  """ 
 11  from Numeric import * 
 12   
 13     
 14 -class Trainer(object): 
 15    """ "virtual base class" for network trainers 
 16   
 17    """ 
 18    pass 
 19   
 20 -class BackProp(Trainer): 
 21    """implement back propagation (algorithm on pp 153-154 of Patterson) 
 22   
 23     I don't *think* that I've made any assumptions about the connectivity of 
 24       the net (i.e. full connectivity between layers is not required). 
 25   
 26     **NOTE:** this code is currently making the assumption that the activation 
 27       functions on the nodes in the network are capable of calculating their 
 28       derivatives using only their values (i.e. a DerivFromVal method should 
 29       exist).  This shouldn't be too hard to change. 
 30      
 31    """ 
 32 -  def StepUpdate(self,example,net,resVect=None): 
 33      """ does a BackProp step based upon the example 
 34   
 35        **Arguments** 
 36   
 37          - example: a 2-tuple: 
 38             1) a list of variable values values 
 39             2) a list of result values (targets) 
 40   
 41          - net: a _Network_ (or something supporting the same API) 
 42   
 43          - resVect: if this is nonzero, then the network is not required to 
 44            classify the _example_ 
 45   
 46        **Returns** 
 47   
 48          the backprop error from _network_ **before the update** 
 49   
 50        **Note** 
 51   
 52          In case it wasn't blindingly obvious, the weights in _network_ are modified 
 53          in the course of taking a backprop step. 
 54           
 55      """ 
 56      totNumNodes = net.GetNumNodes() 
 57      if self.oldDeltaW is None: 
 58        self.oldDeltaW = zeros(totNumNodes,Float64) 
 59      outputNodeList = net.GetOutputNodeList() 
 60      nOutput = len(outputNodeList) 
 61      targetVect = array(example[-nOutput:],Float64) 
 62      trainVect = example[:-nOutput] 
 63      if resVect is None: 
 64        # classify the example 
 65        net.ClassifyExample(trainVect) 
 66        resVect = net.GetLastOutputs() 
 67      outputs = take(resVect,outputNodeList) 
 68      errVect = targetVect - outputs 
 69   
 70      delta = zeros(totNumNodes,Float64) 
 71      # start with the output layer 
 72      for i in xrange(len(outputNodeList)): 
 73        idx = outputNodeList[i] 
 74        node = net.GetNode(idx) 
 75        # the deltas here are easy 
 76        delta[idx] = errVect[i]*node.actFunc.DerivFromVal(resVect[idx]) 
 77        # use these results to start working on the deltas of the preceding layer 
 78        inputs = node.GetInputs() 
 79        weights = delta[idx]*node.GetWeights() 
 80        for j in xrange(len(inputs)): 
 81          idx2 = inputs[j] 
 82          delta[idx2] = delta[idx2] + weights[j] 
 83   
 84      # now propagate the deltas backwards 
 85      for layer in xrange(net.GetNumHidden()-1,-1,-1): 
 86        nodesInLayer = net.GetHiddenLayerNodeList(layer) 
 87        for idx in nodesInLayer: 
 88          node = net.GetNode(idx) 
 89          # start by finishing off the error term for this guy 
 90          delta[idx] = delta[idx]*node.actFunc.DerivFromVal(resVect[idx]) 
 91   
 92          # and then propagate our errors to the preceding layer 
 93          if layer != 0: 
 94            inputs = node.GetInputs() 
 95            weights = delta[idx]*node.GetWeights() 
 96            for i in xrange(len(inputs)): 
 97              idx2 = inputs[i] 
 98              delta[idx2] = delta[idx2] + weights[i] 
 99           
100      # okey dokey... we've now got the deltas for each node, use those 
101      #  to update the weights (whew!) 
102      nHidden = net.GetNumHidden() 
103      for layer in xrange(0,nHidden+1): 
104        if layer == nHidden: 
105          idxList = net.GetOutputNodeList() 
106        else: 
107          idxList = net.GetHiddenLayerNodeList(layer) 
108        for idx in idxList: 
109          node = net.GetNode(idx) 
110          dW = self.speed * delta[idx] * take(resVect,node.GetInputs()) 
111          newWeights = node.GetWeights() + dW 
112          node.SetWeights(newWeights) 
113         
114      # return the RMS error from the OLD network 
115      return sqrt(errVect*errVect)[0] 
116       
117   
118 -  def TrainOnLine(self,examples,net,maxIts=5000,errTol=0.1,useAvgErr=1, 
119                    silent=0): 
120      """ carries out online training of a neural net 
121   
122        The definition of online training is that the network is updated after 
123          each example is presented. 
124   
125        **Arguments** 
126   
127          - examples: a list of 2-tuple: 
128             1) a list of variable values values 
129             2) a list of result values (targets) 
130   
131          - net: a _Network_ (or something supporting the same API) 
132   
133          - maxIts: the maximum number of *training epochs* (see below for definition) to be 
134            run 
135   
136          - errTol: the tolerance for convergence 
137   
138          - useAvgErr: if this toggle is nonzero, then the error at each step will be 
139            divided by the number of training examples for the purposes of checking 
140            convergence. 
141   
142          - silent: controls the amount of visual noise produced as this runs. 
143   
144   
145        **Note** 
146   
147           a *training epoch* is one complete pass through all the training examples 
148   
149      """     
150      nExamples = len(examples) 
151      converged = 0 
152      cycle = 0 
153   
154      while (not converged) and (cycle < maxIts): 
155        maxErr = 0 
156        newErr = 0 
157        #print 'bp: ',cycle 
158        for example in examples: 
159          localErr = self.StepUpdate(example,net) 
160          newErr += localErr 
161          if localErr > maxErr: 
162            maxErr = localErr 
163        if useAvgErr == 1: 
164          newErr = newErr / nExamples 
165        else: 
166          newErr = maxErr 
167        #print '\t',newErr,errTol 
168           
169        if newErr <= errTol: 
170          converged = 1 
171   
172  #      if cycle % 10 == 0 and not silent: 
173        if not silent: 
174          print 'epoch %d, error: % 6.4f'%(cycle,newErr) 
175   
176        cycle = cycle + 1 
177      if not silent: 
178        if converged: 
179          print 'Converged after %d epochs.'%cycle 
180        else: 
181          print 'NOT Converged after %d epochs.'%cycle 
182        print 'final error: % 6.4f'%newErr 
183   
184 -  def __init__(self,speed=0.5,momentum=0.7): 
185      """ Constructor 
186   
187        **Arguments** 
188         
189          - speed: the speed parameter for back prop training 
190   
191          - momentum: the momentum term for back prop training 
192            *Not currently used* 
193   
194      """ 
195      self.speed = speed 
196      self.momentum = momentum 
197      self.oldDeltaW = None 
198   
199   
200   
201  if __name__ == '__main__': 
202    from ML.Neural import Network 
203   
204 -  def testAnd(): 
205      examples = [ 
206            [[0,0,1], [0.1]], 
207            [[0,1,1], [.1]], 
208            [[1,0,1], [.1]], 
209            [[1,1,1], [.9]] 
210        ] 
211      net = Network.Network([3,1]) 
212      t = BackProp() 
213      t.TrainOnLine(examples,net) 
214      return net 
215   
216 -  def testOr(): 
217      examples = [ 
218            [[0,0,1], [0.1]], 
219            [[0,1,1], [.9]], 
220            [[1,0,1], [.9]], 
221            [[1,1,1], [.9]] 
222        ] 
223      net = Network.Network([3,1]) 
224      t = BackProp() 
225      t.TrainOnLine(examples,net,maxIts=1000,useAvgErr=0) 
226      print 'classifications:' 
227      for example in examples: 
228        res = net.ClassifyExample(example[0]) 
229        print '%f -> %f'%(example[1][0],res) 
230   
231      return net 
232   
233 -  def testXor(): 
234      examples = [ 
235            [[0,0,1], [.1]], 
236            [[0,1,1], [.9]], 
237            [[1,0,1], [.9]], 
238            [[1,1,1], [.1]] 
239        ] 
240      net = Network.Network([3,3,1]) 
241   
242      t = BackProp(speed=.8) 
243      t.TrainOnLine(examples,net,errTol=0.2) 
244      return net 
245   
246   
247 -  def testLinear(): 
248      examples = [ 
249            [.1,.1], 
250            [.2,.2], 
251            [.3,.3], 
252            [.4,.4], 
253            [.8,.8], 
254        ] 
255      net = Network.Network([1,2,1]) 
256      t = BackProp(speed=.8) 
257      t.TrainOnLine(examples,net,errTol=0.1,useAvgErr=0) 
258      print 'classifications:' 
259      for example in examples: 
260        res = net.ClassifyExample(example[:-1]) 
261        print '%f -> %f'%(example[-1],res) 
262   
263      return net 
264   
265 -  def runProfile(command): 
266      import RandomArray 
267      RandomArray.seed(23,42) 
268      import profile,pstats 
269      datFile = '%s.prof.dat'%(command) 
270      profile.run('%s()'%command,datFile) 
271      stats = pstats.Stats(datFile) 
272      stats.strip_dirs() 
273      stats.sort_stats('time').print_stats() 
274       
275    if 0: 
276      net = testXor() 
277      print 'Xor:', net 
278      import cPickle 
279      outF = open('xornet.pkl','wb+') 
280      cPickle.dump(net,outF) 
281      outF.close() 
282    else: 
283      #runProfile('testLinear') 
284      net = testLinear() 
285      #net = testOr() 
286