rdkit.ML.KNN.CrossValidate

Package rdkit :: Package ML :: Package KNN :: Module CrossValidate |
[hide private]
[frames] | no frames]

# Source Code for Module rdkit.ML.KNN.CrossValidate

      1  #
      2  #  Copyright (C) 2000-2008  greg Landrum
      3  #
      4  """ handles doing cross validation with k-nearest neighbors model
      5
      6  and evaluation of individual models
      7
      8  """
      9
     10  from rdkit.ML.KNN.KNNClassificationModel import KNNClassificationModel
     11  from rdkit.ML.KNN.KNNRegressionModel import KNNRegressionModel
     12  from rdkit import RDRandom
     13  from rdkit.ML.KNN import DistFunctions
     14  from rdkit.ML.Data import SplitData
     15
     16 -def makeClassificationModel(numNeigh, attrs, distFunc) :
     17    return KNNClassificationModel(numNeigh, attrs, distFunc)
     18 -def makeRegressionModel(numNeigh, attrs, distFunc) :
     19    return KNNRegressionModel(numNeigh, attrs, distFunc)
     20
     21 -def CrossValidate(knnMod,testExamples,appendExamples=0):
     22    """
     23    Determines the classification error for the testExamples
     24
     25    **Arguments**
     26
     27      - tree: a decision tree (or anything supporting a _ClassifyExample()_ method)
     28
     29      - testExamples: a list of examples to be used for testing
     30
     31      - appendExamples: a toggle which is passed along to the tree as it does
     32        the classification. The trees can use this to store the examples they
     33        classify locally.
     34
     35    **Returns**
     36
     37      a 2-tuple consisting of:
     38        """
     39    nTest = len(testExamples)
     40
     41    if isinstance(knnMod,KNNClassificationModel):
     42      badExamples = []
     43      nBad = 0
     44      for i in xrange(nTest):
     45        testEx = testExamples[i]
     46        trueRes = testEx[-1]
     47        res = knnMod.ClassifyExample(testEx, appendExamples)
     48        if (trueRes != res) :
     49          badExamples.append(testEx)
     50          nBad += 1
     51      return float(nBad)/nTest, badExamples
     52    elif isinstance(knnMod,KNNRegressionModel):
     53      devSum=0.0
     54      for i in xrange(nTest):
     55        testEx = testExamples[i]
     56        trueRes = testEx[-1]
     57        res = knnMod.PredictExample(testEx, appendExamples)
     58        devSum += abs(trueRes-res)
     59      return devSum/nTest,None
     60    raise ValueError,"Unrecognized Model Type"
     61
     62 -def CrossValidationDriver(examples, attrs, nPossibleValues, numNeigh,
     63                            modelBuilder=makeClassificationModel,
     64                            distFunc=DistFunctions.EuclideanDist,
     65                            holdOutFrac=0.3,
     66                            silent=0,
     67                            calcTotalError=0,
     68                            **kwargs) :
     69    """ Driver function for building a KNN model of a specified type
     70
     71    **Arguments**
     72
     73      - examples: the full set of examples
     74
     75      - numNeigh: number of neighbors for the KNN model (basically k in k-NN)
     76
     77      - knnModel: the type of KNN model (a classification vs regression model)
     78
     79      - holdOutFrac: the fraction of the data which should be reserved for the hold-out set
     80        (used to calculate error)
     81
     82      - silent: a toggle used to control how much visual noise this makes as it goes
     83
     84      - calcTotalError: a toggle used to indicate whether the classification error
     85        of the tree should be calculated using the entire data set (when true) or just
     86        the training hold out set (when false)
     87        """
     88
     89    nTot = len(examples)
     90    if not kwargs.get('replacementSelection',0):
     91      testIndices,trainIndices = SplitData.SplitIndices(nTot,holdOutFrac,
     92                                                        silent=1,legacy=1,
     93                                                        replacement=0)
     94    else:
     95      testIndices,trainIndices = SplitData.SplitIndices(nTot,holdOutFrac,
     96                                                        silent=1,legacy=0,
     97                                                        replacement=1)
     98    trainExamples = [examples[x] for x in trainIndices]
     99    testExamples = [examples[x] for x in testIndices]
    100
    101
    102    nTrain = len(trainExamples)
    103
    104    if not silent:
    105      print "Training with %d examples"%(nTrain)
    106
    107    knnMod = modelBuilder(numNeigh, attrs, distFunc)
    108
    109    knnMod.SetTrainingExamples(trainExamples)
    110    knnMod.SetTestExamples(testExamples)
    111
    112    if not calcTotalError:
    113      xValError,badExamples = CrossValidate(knnMod, testExamples,appendExamples=1)
    114    else:
    115      xValError,badExamples = CrossValidate(knnMod, examples,appendExamples=0)
    116
    117    if not silent :
    118      'Validation error was %%%4.2f'%(100*xValError)
    119
    120    knnMod._trainIndices = trainIndices
    121    return knnMod, xValError
    122