ML.KNN.CrossValidate

Package ML :: Package KNN :: Module CrossValidate |
[hide private]
[frames] | no frames]

# Source Code for Module ML.KNN.CrossValidate

      1  #
      2  #  Copyright (C) 2000  greg Landrum
      3  #
      4  """ handles doing cross validation with k-nearest neighbors model
      5
      6  and evaluation of individual models
      7
      8  """
      9
     10  from Numeric import *
     11  from ML.KNN.KNNClassificationModel import KNNClassificationModel
     12  from ML.KNN.KNNRegressionModel import KNNRegressionModel
     13  import RDRandom
     14  from ML.KNN import DistFunctions
     15  from ML.Data import SplitData
     16
     17 -def makeClassificationModel(numNeigh, attrs, distFunc) :
     18    return KNNClassificationModel(numNeigh, attrs, distFunc)
     19 -def makeRegressionModel(numNeigh, attrs, distFunc) :
     20    return KNNRegressionModel(numNeigh, attrs, distFunc)
     21
     22 -def CrossValidate(knnMod,testExamples,appendExamples=0):
     23    """
     24    Determines the classification error for the testExamples
     25
     26    **Arguments**
     27
     28      - tree: a decision tree (or anything supporting a _ClassifyExample()_ method)
     29
     30      - testExamples: a list of examples to be used for testing
     31
     32      - appendExamples: a toggle which is passed along to the tree as it does
     33        the classification. The trees can use this to store the examples they
     34        classify locally.
     35
     36    **Returns**
     37
     38      a 2-tuple consisting of:
     39        """
     40    nTest = len(testExamples)
     41
     42    if isinstance(knnMod,KNNClassificationModel):
     43      badExamples = []
     44      nBad = 0
     45      for i in xrange(nTest):
     46        testEx = testExamples[i]
     47        trueRes = testEx[-1]
     48        res = knnMod.ClassifyExample(testEx, appendExamples)
     49        if (trueRes != res) :
     50          badExamples.append(testEx)
     51          nBad += 1
     52      return float(nBad)/nTest, badExamples
     53    elif isinstance(knnMod,KNNRegressionModel):
     54      devSum=0.0
     55      for i in xrange(nTest):
     56        testEx = testExamples[i]
     57        trueRes = testEx[-1]
     58        res = knnMod.PredictExample(testEx, appendExamples)
     59        devSum += abs(trueRes-res)
     60      return devSum/nTest,None
     61    raise ValueError,"Unrecognized Model Type"
     62
     63 -def CrossValidationDriver(examples, attrs, nPossibleValues, numNeigh,
     64                            modelBuilder=makeClassificationModel,
     65                            distFunc=DistFunctions.EuclideanDist,
     66                            holdOutFrac=0.3,
     67                            silent=0,
     68                            calcTotalError=0,
     69                            **kwargs) :
     70    """ Driver function for building a KNN model of a specified type
     71
     72    **Arguments**
     73
     74      - examples: the full set of examples
     75
     76      - numNeigh: number of neighbors for the KNN model (basically k in k-NN)
     77
     78      - knnModel: the type of KNN model (a classification vs regression model)
     79
     80      - holdOutFrac: the fraction of the data which should be reserved for the hold-out set
     81        (used to calculate error)
     82
     83      - silent: a toggle used to control how much visual noise this makes as it goes
     84
     85      - calcTotalError: a toggle used to indicate whether the classification error
     86        of the tree should be calculated using the entire data set (when true) or just
     87        the training hold out set (when false)
     88        """
     89
     90    nTot = len(examples)
     91    if not kwargs.get('replacementSelection',0):
     92      testIndices,trainIndices = SplitData.SplitIndices(nTot,holdOutFrac,
     93                                                        silent=1,legacy=1,
     94                                                        replacement=0)
     95    else:
     96      testIndices,trainIndices = SplitData.SplitIndices(nTot,holdOutFrac,
     97                                                        silent=1,legacy=0,
     98                                                        replacement=1)
     99    trainExamples = [examples[x] for x in trainIndices]
    100    testExamples = [examples[x] for x in testIndices]
    101
    102
    103    nTrain = len(trainExamples)
    104
    105    if not silent:
    106      print "Training with %d examples"%(nTrain)
    107
    108    knnMod = modelBuilder(numNeigh, attrs, distFunc)
    109
    110    knnMod.SetTrainingExamples(trainExamples)
    111    knnMod.SetTestExamples(testExamples)
    112
    113    if not calcTotalError:
    114      xValError,badExamples = CrossValidate(knnMod, testExamples,appendExamples=1)
    115    else:
    116      xValError,badExamples = CrossValidate(knnMod, examples,appendExamples=0)
    117
    118    if not silent :
    119      'Validation error was %%%4.2f'%(100*xValError)
    120
    121    knnMod._trainIndices = trainIndices
    122    return knnMod, xValError
    123