ML.Neural.Trainers

Package ML :: Package Neural :: Module Trainers |
[hide private]
[frames] | no frames]

# Source Code for Module ML.Neural.Trainers

      1  #
      2  #  Copyright (C) 2000  greg Landrum
      3  #
      4  """ Training algorithms for feed-forward neural nets
      5
      6    Unless noted otherwise, algorithms and notation are taken from:
      7    "Artificial Neural Networks: Theory and Applications",
      8      Dan W. Patterson, Prentice Hall, 1996
      9
     10  """
     11  from Numeric import *
     12
     13
     14 -class Trainer(object):
     15    """ "virtual base class" for network trainers
     16
     17    """
     18    pass
     19
     20 -class BackProp(Trainer):
     21    """implement back propagation (algorithm on pp 153-154 of Patterson)
     22
     23     I don't *think* that I've made any assumptions about the connectivity of
     24       the net (i.e. full connectivity between layers is not required).
     25
     26     **NOTE:** this code is currently making the assumption that the activation
     27       functions on the nodes in the network are capable of calculating their
     28       derivatives using only their values (i.e. a DerivFromVal method should
     29       exist).  This shouldn't be too hard to change.
     30
     31    """
     32 -  def StepUpdate(self,example,net,resVect=None):
     33      """ does a BackProp step based upon the example
     34
     35        **Arguments**
     36
     37          - example: a 2-tuple:
     38             1) a list of variable values values
     39             2) a list of result values (targets)
     40
     41          - net: a _Network_ (or something supporting the same API)
     42
     43          - resVect: if this is nonzero, then the network is not required to
     44            classify the _example_
     45
     46        **Returns**
     47
     48          the backprop error from _network_ **before the update**
     49
     50        **Note**
     51
     52          In case it wasn't blindingly obvious, the weights in _network_ are modified
     53          in the course of taking a backprop step.
     54
     55      """
     56      totNumNodes = net.GetNumNodes()
     57      if self.oldDeltaW is None:
     58        self.oldDeltaW = zeros(totNumNodes,Float64)
     59      outputNodeList = net.GetOutputNodeList()
     60      nOutput = len(outputNodeList)
     61      targetVect = array(example[-nOutput:],Float64)
     62      trainVect = example[:-nOutput]
     63      if resVect is None:
     64        # classify the example
     65        net.ClassifyExample(trainVect)
     66        resVect = net.GetLastOutputs()
     67      outputs = take(resVect,outputNodeList)
     68      errVect = targetVect - outputs
     69
     70      delta = zeros(totNumNodes,Float64)
     71      # start with the output layer
     72      for i in xrange(len(outputNodeList)):
     73        idx = outputNodeList[i]
     74        node = net.GetNode(idx)
     75        # the deltas here are easy
     76        delta[idx] = errVect[i]*node.actFunc.DerivFromVal(resVect[idx])
     77        # use these results to start working on the deltas of the preceding layer
     78        inputs = node.GetInputs()
     79        weights = delta[idx]*node.GetWeights()
     80        for j in xrange(len(inputs)):
     81          idx2 = inputs[j]
     82          delta[idx2] = delta[idx2] + weights[j]
     83
     84      # now propagate the deltas backwards
     85      for layer in xrange(net.GetNumHidden()-1,-1,-1):
     86        nodesInLayer = net.GetHiddenLayerNodeList(layer)
     87        for idx in nodesInLayer:
     88          node = net.GetNode(idx)
     89          # start by finishing off the error term for this guy
     90          delta[idx] = delta[idx]*node.actFunc.DerivFromVal(resVect[idx])
     91
     92          # and then propagate our errors to the preceding layer
     93          if layer != 0:
     94            inputs = node.GetInputs()
     95            weights = delta[idx]*node.GetWeights()
     96            for i in xrange(len(inputs)):
     97              idx2 = inputs[i]
     98              delta[idx2] = delta[idx2] + weights[i]
     99
    100      # okey dokey... we've now got the deltas for each node, use those
    101      #  to update the weights (whew!)
    102      nHidden = net.GetNumHidden()
    103      for layer in xrange(0,nHidden+1):
    104        if layer == nHidden:
    105          idxList = net.GetOutputNodeList()
    106        else:
    107          idxList = net.GetHiddenLayerNodeList(layer)
    108        for idx in idxList:
    109          node = net.GetNode(idx)
    110          dW = self.speed * delta[idx] * take(resVect,node.GetInputs())
    111          newWeights = node.GetWeights() + dW
    112          node.SetWeights(newWeights)
    113
    114      # return the RMS error from the OLD network
    115      return sqrt(errVect*errVect)[0]
    116
    117
    118 -  def TrainOnLine(self,examples,net,maxIts=5000,errTol=0.1,useAvgErr=1,
    119                    silent=0):
    120      """ carries out online training of a neural net
    121
    122        The definition of online training is that the network is updated after
    123          each example is presented.
    124
    125        **Arguments**
    126
    127          - examples: a list of 2-tuple:
    128             1) a list of variable values values
    129             2) a list of result values (targets)
    130
    131          - net: a _Network_ (or something supporting the same API)
    132
    133          - maxIts: the maximum number of *training epochs* (see below for definition) to be
    134            run
    135
    136          - errTol: the tolerance for convergence
    137
    138          - useAvgErr: if this toggle is nonzero, then the error at each step will be
    139            divided by the number of training examples for the purposes of checking
    140            convergence.
    141
    142          - silent: controls the amount of visual noise produced as this runs.
    143
    144
    145        **Note**
    146
    147           a *training epoch* is one complete pass through all the training examples
    148
    149      """
    150      nExamples = len(examples)
    151      converged = 0
    152      cycle = 0
    153
    154      while (not converged) and (cycle < maxIts):
    155        maxErr = 0
    156        newErr = 0
    157        #print 'bp: ',cycle
    158        for example in examples:
    159          localErr = self.StepUpdate(example,net)
    160          newErr += localErr
    161          if localErr > maxErr:
    162            maxErr = localErr
    163        if useAvgErr == 1:
    164          newErr = newErr / nExamples
    165        else:
    166          newErr = maxErr
    167        #print '\t',newErr,errTol
    168
    169        if newErr <= errTol:
    170          converged = 1
    171
    172  #      if cycle % 10 == 0 and not silent:
    173        if not silent:
    174          print 'epoch %d, error: % 6.4f'%(cycle,newErr)
    175
    176        cycle = cycle + 1
    177      if not silent:
    178        if converged:
    179          print 'Converged after %d epochs.'%cycle
    180        else:
    181          print 'NOT Converged after %d epochs.'%cycle
    182        print 'final error: % 6.4f'%newErr
    183
    184 -  def __init__(self,speed=0.5,momentum=0.7):
    185      """ Constructor
    186
    187        **Arguments**
    188
    189          - speed: the speed parameter for back prop training
    190
    191          - momentum: the momentum term for back prop training
    192            *Not currently used*
    193
    194      """
    195      self.speed = speed
    196      self.momentum = momentum
    197      self.oldDeltaW = None
    198
    199
    200
    201  if __name__ == '__main__':
    202    from ML.Neural import Network
    203
    204 -  def testAnd():
    205      examples = [
    206            [[0,0,1], [0.1]],
    207            [[0,1,1], [.1]],
    208            [[1,0,1], [.1]],
    209            [[1,1,1], [.9]]
    210        ]
    211      net = Network.Network([3,1])
    212      t = BackProp()
    213      t.TrainOnLine(examples,net)
    214      return net
    215
    216 -  def testOr():
    217      examples = [
    218            [[0,0,1], [0.1]],
    219            [[0,1,1], [.9]],
    220            [[1,0,1], [.9]],
    221            [[1,1,1], [.9]]
    222        ]
    223      net = Network.Network([3,1])
    224      t = BackProp()
    225      t.TrainOnLine(examples,net,maxIts=1000,useAvgErr=0)
    226      print 'classifications:'
    227      for example in examples:
    228        res = net.ClassifyExample(example[0])
    229        print '%f -> %f'%(example[1][0],res)
    230
    231      return net
    232
    233 -  def testXor():
    234      examples = [
    235            [[0,0,1], [.1]],
    236            [[0,1,1], [.9]],
    237            [[1,0,1], [.9]],
    238            [[1,1,1], [.1]]
    239        ]
    240      net = Network.Network([3,3,1])
    241
    242      t = BackProp(speed=.8)
    243      t.TrainOnLine(examples,net,errTol=0.2)
    244      return net
    245
    246
    247 -  def testLinear():
    248      examples = [
    249            [.1,.1],
    250            [.2,.2],
    251            [.3,.3],
    252            [.4,.4],
    253            [.8,.8],
    254        ]
    255      net = Network.Network([1,2,1])
    256      t = BackProp(speed=.8)
    257      t.TrainOnLine(examples,net,errTol=0.1,useAvgErr=0)
    258      print 'classifications:'
    259      for example in examples:
    260        res = net.ClassifyExample(example[:-1])
    261        print '%f -> %f'%(example[-1],res)
    262
    263      return net
    264
    265 -  def runProfile(command):
    266      import RandomArray
    267      RandomArray.seed(23,42)
    268      import profile,pstats
    269      datFile = '%s.prof.dat'%(command)
    270      profile.run('%s()'%command,datFile)
    271      stats = pstats.Stats(datFile)
    272      stats.strip_dirs()
    273      stats.sort_stats('time').print_stats()
    274
    275    if 0:
    276      net = testXor()
    277      print 'Xor:', net
    278      import cPickle
    279      outF = open('xornet.pkl','wb+')
    280      cPickle.dump(net,outF)
    281      outF.close()
    282    else:
    283      #runProfile('testLinear')
    284      net = testLinear()
    285      #net = testOr()
    286