机器学习---决策树

一、决策树概念

二、决策树的构造

<span style="font-size:14px;">检测数据集中每个子项是否属于同一分类：
if so return 类标签；
else
寻找划分数据集的最好特征
划分数据集
创建分支节点
for 每个划分的子集
调用函数createBranch并增加返回结果到分支节点中
return 分支节点</span>

三、python实现

<span style="font-size:14px;">from math import log
import operator
#计算香农熵
def calcShannonEnt(dataSet):
numEntries=len(dataSet)
labelCounts={}
for featVec in dataSet:#为所有分类创建字典
currentLabel=featVec[-1]#the last
if currentLabel not in labelCounts.keys():
labelCounts[currentLabel]=0
labelCounts[currentLabel]+=1
shannonEnt=0.0
for key in labelCounts:
prob=float(labelCounts[key])/numEntries
shannonEnt-=prob*log(prob,2)
return shannonEnt
#测试样例
def createDataSet():
dataSet=[[1,1,'yes'],
[1,1,'yes'],
[1,0,'no'],
[0,1,'no'],
[0,1,'no']]
labels=['no surfacin','flippers']
return dataSet,labels
#划分数据集
def splitDataSet(dataSet,axis,value):
retDataSet=[]
for featVec in dataSet:
if featVec[axis]==value:
reducedFeatVec=featVec[:axis]#0~axis
reducedFeatVec.extend(featVec[axis+1:])#注意两个函数区别
retDataSet.append(reducedFeatVec)
return retDataSet
#选择最好的特征
def chooseBestFeatureToSplit(dataSet):
numFeatures=len(dataSet[0])-1#不包括类标签
baseEntropy=calcShannonEnt(dataSet)
bestInfoGain=0.0;bestFeature=-1
for i in range(numFeatures):
featList=[example[i] for example in dataSet]
uniqueVals=set(featList)
newEntropy=0.0
for value in uniqueVals:#计算信息熵
subDataSet = splitDataSet(dataSet,i,value)
prob = len(subDataSet)/float(len(dataSet))
newEntropy += prob*calcShannonEnt(subDataSet)
infoGain=baseEntropy-newEntropy
if (infoGain>bestInfoGain):#找到最好的信息增益
bestInfoGain = infoGain
bestFeature = i
return bestFeature
#分支结束若叶子节点分类不唯一，采取投票方式
def majorityCht(classList):
classCount={}
for vote in classList:
if vote not in classCount.key():
classCount[vote]=0
classCount[vote]+=1
sortedClassCount=sorted(classCount.iteritems(),
key=operator.itemgetter(1),reverse=True)
return sortedClassCount[0][0]
#构造树
def createTree(dataSet,labels):
classList = [example[-1] for example in dataSet]
if classList.count(classList[0])==len(classList):
return classList[0]
if len(dataSet[0])==1:# no more features
return majorityCnt(classList)
bestFeat = chooseBestFeatureToSplit(dataSet)#bestFeat is the index of best feature
bestFeatLabel = labels[bestFeat]
myTree = {bestFeatLabel:{}}
del (labels[bestFeat])
featValues = [example[bestFeat] for example in dataSet]
uniqueFeatVals = set(featValues)
for value in uniqueFeatVals:
subLabels=labels[:]
myTree[bestFeatLabel][value] = createTree(splitDataSet\
(dataSet,bestFeat,value),subLabels)
return myTree</span>