rdkit.ML.DecTree.PruneTree

Package rdkit :: Package ML :: Package DecTree :: Module PruneTree |
[hide private]
[frames] | no frames]

# Source Code for Module rdkit.ML.DecTree.PruneTree

      1  #
      2  #  Copyright (C) 2000-2008  greg Landrum and Rational Discovery LLC
      3  #
      4  """ Contains functionality for doing tree pruning
      5
      6  """
      7  import numpy
      8  from rdkit.ML.DecTree import CrossValidate, DecTree
      9  import copy
     10
     11  _verbose = 0
     12
     13 -def MaxCount(examples):
     14    """ given a set of examples, returns the most common result code
     15
     16     **Arguments**
     17
     18        examples: a list of examples to be counted
     19
     20     **Returns**
     21
     22       the most common result code
     23
     24    """
     25    resList = [x[-1] for x in examples]
     26    maxVal = max(resList)
     27    counts = [None]*(maxVal+1)
     28    for i in xrange(maxVal+1):
     29      counts[i] = sum([x==i for x in resList])
     30
     31    return numpy.argmax(counts)
     32
     33 -def _GetLocalError(node):
     34    nWrong = 0
     35    for example in node.GetExamples():
     36      pred = node.ClassifyExample(example,appendExamples=0)
     37      if pred != example[-1]:
     38        nWrong +=1
     39        #if _verbose: print '------------------>MISS:',example,pred
     40    return nWrong
     41
     42 -def _Pruner(node,level=0):
     43    """Recursively finds and removes the nodes whose removals improve classification
     44
     45       **Arguments**
     46
     47         - node: the tree to be pruned.  The pruning data should already be contained
     48           within node (i.e. node.GetExamples() should return the pruning data)
     49
     50         - level: (optional) the level of recursion, used only in _verbose printing
     51
     52
     53       **Returns**
     54
     55          the pruned version of node
     56
     57
     58       **Notes**
     59
     60        - This uses a greedy algorithm which basically does a DFS traversal of the tree,
     61          removing nodes whenever possible.
     62
     63        - If removing a node does not affect the accuracy, it *will be* removed.  We
     64          favor smaller trees.
     65
     66    """
     67    if _verbose: print '  '*level,'<%d>  '%level,'>>> Pruner'
     68    children = node.GetChildren()[:]
     69
     70    bestTree = copy.deepcopy(node)
     71    bestErr = 1e6
     72    emptyChildren=[]
     73    #
     74    # Loop over the children of this node, removing them when doing so
     75    #  either improves the local error or leaves it unchanged (we're
     76    #  introducing a bias for simpler trees).
     77    #
     78    for i in range(len(children)):
     79      child = children[i]
     80      examples = child.GetExamples()
     81      if _verbose:
     82        print '  '*level,'<%d>  '%level,' Child:',i,child.GetLabel()
     83        bestTree.Print()
     84        print
     85      if len(examples):
     86        if _verbose: print '  '*level,'<%d>  '%level,'  Examples',len(examples)
     87        if not child.GetTerminal():
     88          if _verbose: print '  '*level,'<%d>  '%level,'    Nonterminal'
     89
     90          workTree = copy.deepcopy(bestTree)
     91          #
     92          # First recurse on the child (try removing things below it)
     93          #
     94          newNode = _Pruner(child,level=level+1)
     95          workTree.ReplaceChildIndex(i,newNode)
     96          tempErr = _GetLocalError(workTree)
     97          if tempErr<=bestErr:
     98            bestErr = tempErr
     99            bestTree = copy.deepcopy(workTree)
    100            if _verbose:
    101              print '  '*level,'<%d>  '%level,'>->->->->->'
    102              print '  '*level,'<%d>  '%level,'replacing:',i,child.GetLabel()
    103              child.Print()
    104              print '  '*level,'<%d>  '%level,'with:'
    105              newNode.Print()
    106              print '  '*level,'<%d>  '%level,'<-<-<-<-<-<'
    107          else:
    108            workTree.ReplaceChildIndex(i,child)
    109          #
    110          # Now try replacing the child entirely
    111          #
    112          bestGuess = MaxCount(child.GetExamples())
    113          newNode = DecTree.DecTreeNode(workTree,'L:%d'%(bestGuess),
    114                                        label=bestGuess,isTerminal=1)
    115          newNode.SetExamples(child.GetExamples())
    116          workTree.ReplaceChildIndex(i,newNode)
    117          if _verbose:
    118            print '  '*level,'<%d>  '%level,'ATTEMPT:'
    119            workTree.Print()
    120          newErr = _GetLocalError(workTree)
    121          if _verbose: print '  '*level,'<%d>  '%level,'---> ',newErr,bestErr
    122          if newErr <= bestErr:
    123            bestErr = newErr
    124            bestTree = copy.deepcopy(workTree)
    125            if _verbose:
    126              print '  '*level,'<%d>  '%level,'PRUNING:'
    127              workTree.Print()
    128          else:
    129            if _verbose: print '  '*level,'<%d>  '%level,'FAIL'
    130            # whoops... put the child back in:
    131            workTree.ReplaceChildIndex(i,child)
    132        else:
    133          if _verbose: print '  '*level,'<%d>  '%level,'    Terminal'
    134      else:
    135        if _verbose: print '  '*level,'<%d>  '%level,'  No Examples',len(examples)
    136        #
    137        # FIX:  we need to figure out what to do here (nodes that contain
    138        #   no examples in the testing set).  I can concoct arguments for
    139        #   leaving them in and for removing them.  At the moment they are
    140        #   left intact.
    141        #
    142        pass
    143
    144    if _verbose: print '  '*level,'<%d>  '%level,'<<< out'
    145    return bestTree
    146
    147 -def PruneTree(tree,trainExamples,testExamples,minimizeTestErrorOnly=1):
    148    """ implements a reduced-error pruning of decision trees
    149
    150     This algorithm is described on page 69 of Mitchell's book.
    151
    152     Pruning can be done using just the set of testExamples (the validation set)
    153     or both the testExamples and the trainExamples by setting minimizeTestErrorOnly
    154     to 0.
    155
    156     **Arguments**
    157
    158       - tree: the initial tree to be pruned
    159
    160       - trainExamples: the examples used to train the tree
    161
    162       - testExamples: the examples held out for testing the tree
    163
    164       - minimizeTestErrorOnly: if this toggle is zero, all examples (i.e.
    165         _trainExamples_ + _testExamples_ will be used to evaluate the error.
    166
    167     **Returns**
    168
    169       a 2-tuple containing:
    170
    171          1) the best tree
    172
    173          2) the best error (the one which corresponds to that tree)
    174
    175    """
    176    if minimizeTestErrorOnly:
    177      testSet = testExamples
    178    else:
    179      testSet = trainExamples + testExamples
    180
    181    # remove any stored examples the tree may have
    182    tree.ClearExamples()
    183
    184    #
    185    # screen the test data through the tree so that we end up with the
    186    #  appropriate points stored at each node of the tree
    187    #
    188    totErr,badEx = CrossValidate.CrossValidate(tree,testSet,appendExamples=1)
    189
    190
    191    #
    192    # Prune
    193    #
    194    newTree = _Pruner(tree)
    195
    196    #
    197    # And recalculate the errors
    198    #
    199    totErr,badEx = CrossValidate.CrossValidate(newTree,testSet)
    200    newTree.SetBadExamples(badEx)
    201
    202    return newTree,totErr
    203
    204
    205  # -------
    206  #  testing code
    207  # -------
    208 -def _testRandom():
    209    from rdkit.ML.DecTree import randomtest
    210    #examples,attrs,nPossibleVals = randomtest.GenRandomExamples(nVars=20,randScale=0.25,nExamples = 200)
    211    examples,attrs,nPossibleVals = randomtest.GenRandomExamples(nVars=10,randScale=0.5,nExamples = 200)
    212    tree,frac = CrossValidate.CrossValidationDriver(examples,attrs,nPossibleVals)
    213    tree.Print()
    214    tree.Pickle('orig.pkl')
    215    print 'original error is:', frac
    216
    217    print '----Pruning'
    218    newTree,frac2 = PruneTree(tree,tree.GetTrainingExamples(),tree.GetTestExamples())
    219    newTree.Print()
    220    print 'pruned error is:',frac2
    221    newTree.Pickle('prune.pkl')
    222
    223
    224 -def _testSpecific():
    225    from rdkit.ML.DecTree import ID3
    226    oPts= [ \
    227      [0,0,1,0],
    228      [0,1,1,1],
    229      [1,0,1,1],
    230      [1,1,0,0],
    231      [1,1,1,1],
    232      ]
    233    tPts = oPts+[[0,1,1,0],[0,1,1,0]]
    234
    235    tree = ID3.ID3Boot(oPts,attrs=range(3),nPossibleVals=[2]*4)
    236    tree.Print()
    237    err,badEx = CrossValidate.CrossValidate(tree,oPts)
    238    print 'original error:',err
    239
    240
    241    err,badEx = CrossValidate.CrossValidate(tree,tPts)
    242    print 'original holdout error:',err
    243    newTree,frac2 = PruneTree(tree,oPts,tPts)
    244    newTree.Print()
    245    err,badEx = CrossValidate.CrossValidate(newTree,tPts)
    246    print 'pruned holdout error is:',err
    247    print badEx
    248
    249    print len(tree),len(newTree)
    250
    251 -def _testChain():
    252    from rdkit.ML.DecTree import ID3
    253    oPts= [ \
    254      [1,0,0,0,1],
    255      [1,0,0,0,1],
    256      [1,0,0,0,1],
    257      [1,0,0,0,1],
    258      [1,0,0,0,1],
    259      [1,0,0,0,1],
    260      [1,0,0,0,1],
    261      [0,0,1,1,0],
    262      [0,0,1,1,0],
    263      [0,0,1,1,1],
    264      [0,1,0,1,0],
    265      [0,1,0,1,0],
    266      [0,1,0,0,1],
    267      ]
    268    tPts = oPts
    269
    270    tree = ID3.ID3Boot(oPts,attrs=range(len(oPts[0])-1),nPossibleVals=[2]*len(oPts[0]))
    271    tree.Print()
    272    err,badEx = CrossValidate.CrossValidate(tree,oPts)
    273    print 'original error:',err
    274
    275
    276    err,badEx = CrossValidate.CrossValidate(tree,tPts)
    277    print 'original holdout error:',err
    278    newTree,frac2 = PruneTree(tree,oPts,tPts)
    279    newTree.Print()
    280    err,badEx = CrossValidate.CrossValidate(newTree,tPts)
    281    print 'pruned holdout error is:',err
    282    print badEx
    283
    284
    285  if __name__ == '__main__':
    286    _verbose=1
    287    #_testRandom()
    288
    289    _testChain()
    290