Adaboost从原理到实现（Python）

# -*- coding:utf-8 -*- 
from numpy import*
class Adaboosting(object):
    def loadSimpData(self):
        datMat = matrix(
            [[1., 2.1],
             [2., 1.1],
             [1.3, 1.],
             [1., 1.],
             [2., 1.]])
        classLabels = [1.0, 1.0, -1.0, -1.0, 1.0]
        return datMat, classLabels
    def stumpClassify(self, datMat, dimen, threshVal, threshIneq):
        # print "-----data-----"
        # print datMat
        retArr = ones((shape(datMat)[0], 1))  
        if threshIneq == 'lt':
            retArr[datMat[:, dimen] <= threshVal] = -1.0  # 小于阈值的列都为-1
        else:
            retArr[datMat[:, dimen] > threshVal] = -1.0  # 大于阈值的列都为-1
        # print "---------retArr------------"
        # print retArr
        return retArr
    def buildStump(self, dataArr, classLables, D):
        """
        单层决策树生成函数
        """
        dataMatrix = mat(dataArr)
        lableMat = mat(classLables).T
        m, n = shape(dataMatrix)
        numSteps = 10.0  # 步数，影响的是迭代次数，步长
        bestStump = {}  # 存储分类器的信息
        bestClassEst = mat(zeros((m, 1)))  # 最好的分类器
        minError = inf  # 迭代寻找最小错误率
        for i in range(n):
            # 求出每一列数据的最大最小值计算步长
            rangeMin = dataMatrix[:, i].min()
            rangeMax = dataMatrix[:, i].max()
            stepSize = (rangeMax - rangeMin) / numSteps
            # j唯一的作用用步数去生成阈值，从最小值大最大值都与数据比较一边了一遍
            for j in range(-1, int(numSteps) + 1):
                threshVal = rangeMin + float(j) * stepSize  # 阈值
                for inequal in ['lt', 'gt']:
                    predictedVals = self.stumpClassify(
                        dataMatrix, i, threshVal, inequal)
                    errArr = mat(ones((m, 1)))
                    errArr[predictedVals == lableMat] = 0  # 为1的 表示i分错的
                    weightedError = D.T * errArr  # 分错的个数*权重(开始权重=1/M行)
                    # print "split: dim %d, thresh %.2f, thresh ineqal:
#%s,the weighted error is %.3f" % (i, threshVal, inequal, weightedError)
                    if weightedError < minError:  # 寻找最小的加权错误率然后保存当前的信息
                        minError = weightedError
                        bestClassEst = predictedVals.copy()  # 分类结果
                        bestStump['dim'] = i
                        bestStump['thresh'] = threshVal
                        bestStump['ineq'] = inequal
        # print bestStump
        # print minError
        # print bestClassEst  # 类别估计
        return bestStump, minError, bestClassEst
    def adaBoostingDs(self, dataArr, classLables, numIt=40):
        """
        基于单层决策树的AdaBoosting训练过程：
        """
        weakClassArr = []  # 最佳决策树数组
        m = shape(dataArr)[0]
        D = mat(ones((m, 1)) / m)
        aggClassEst = mat(zeros((m, 1)))
        for i in range(numIt):
            bestStump, minError, bestClassEst = self.buildStump(
                dataArr, classLables, D)
            print "bestStump:", bestStump
            print "D:", D.T
            alpha = float(
                0.5 * log((1.0 - minError) / max(minError, 1e-16)))
            bestStump['alpha'] = alpha
            weakClassArr.append(bestStump)
            print "alpha:", alpha
            print "classEst:", bestClassEst.T  # 类别估计
            expon = multiply(-1 * alpha * mat(classLables).T, bestClassEst)
            D = multiply(D, exp(expon))
            D = D / D.sum()
            aggClassEst += alpha * bestClassEst
            print "aggClassEst ；", aggClassEst.T
            # 累加错误率
            aggErrors = multiply(sign(aggClassEst) !=
                                 mat(classLables).T, ones((m, 1)))
            # 错误率平均值
            errorsRate = aggErrors.sum() / m
            print "total error:", errorsRate, "n"
            if errorsRate == 0.0:
                break
        print "weakClassArr:", weakClassArr
        return weakClassArr
    def adClassify(self, datToClass, classifierArr):
        """
        预测分类：
        datToClass：待分类数据
        classifierArr: 训练好的分类器数组
        """
        dataMatrix = mat(datToClass)
        m = shape(dataMatrix)[0]
        aggClassEst = mat(zeros((m, 1)))
        print
        for i in range(len(classifierArr)):  # 有多少个分类器迭代多少次
            # 调用第一个分类器进行分类
            classEst = self.stumpClassify(dataMatrix, classifierArr[i]['dim'],
                                          classifierArr[i]['thresh'],
                                          classifierArr[i]['ineq']
                                          )
            # alpha 表示每个分类器的权重，
            print classEst
            aggClassEst += classifierArr[i]['alpha'] * classEst
            print aggClassEst
        return sign(aggClassEst)
if __name__ == "__main__":
    adaboosting = Adaboosting()
    D = mat(ones((5, 1)) / 5)
    dataMat, lableMat = adaboosting.loadSimpData()
    # 训练分类器
    classifierArr = adaboosting.adaBoostingDs(dataMat, lableMat, 40)
    # 预测数据
    result = adaboosting.adClassify([0, 0], classifierArr)
    print result