1.支持向量机简介
支持向量机总的特点：泛化错误率低，计算开销不大，结果易解释，缺点：对参数调节和核函数的选择敏感，原始分类器不加修改仅适用二类问题。适用数据类型：数值型和标称型数据。
什么叫支持向量？什么又叫做支持向量机？支持向量的本质就是离分割超平面最近的那些点，就是数据集的部分样本点，如果包括全部的样本点，那就是kNN算法了。支持向量机是一种分类器，之所以称之为“机”是因为它会产生一个二值决策结果，即它是一种决策“机”。
支持向量机的核心思想：我们必须找到具有最小间隔的数据点，这些数据点也就是之前提到的支持向量，一旦找到这些最小间隔的数据点，就需要对该间距最大化。核心的公式如下所示：
argmaxw,b{minn(label⋅(wTx+b))⋅1∥w∥}
对于线性可分的二分类问题，我们可以直接通过上述方法对数据进行分类。但是对于线性不可分的数据集呢？
那我们就需要使用核函数的方法，什么叫核函数？我们将一个特征空间转换到另一个空间的映射就是通过核函数来实现的。简单的来说，核函数其实就是定义的一个函数，用来改变整个数据集在空间中的分布的。
2.python实现SMO算法
先贴出代码和数据集下载链接: /s/1bpCK4d5 密码: w4by
#coding:utf-8#导入对应的依赖库，LZ这里没明白为什么要导入sleep，程序中好像不太需要from numpy import *from time import sleep#下载数据，这个和之前的操作都是一致的，逐行解析，每个样本有两个特征，最后一列是labeldef loadDataSet(fileName):dataMat = []; labelMat = []fr = open(fileName)for line in fr.readlines():lineArr = line.strip().split('\t')dataMat.append([float(lineArr[0]), float(lineArr[1])])labelMat.append(float(lineArr[2]))return dataMat,labelMat#随机选择一个j，要求j和i不相等def selectJrand(i,m):j=i #we want to select any J not equal to iwhile (j==i):j = int(random.uniform(0,m))return j#限定alpha的上下界def clipAlpha(aj,H,L):if aj > H: aj = Hif L > aj:aj = Lreturn aj#简化版SMO算法def smoSimple(dataMatIn, classLabels, C, toler, maxIter):dataMatrix = mat(dataMatIn); labelMat = mat(classLabels).transpose()b = 0; m,n = shape(dataMatrix)alphas = mat(zeros((m,1)))iter = 0while (iter < maxIter):alphaPairsChanged = 0for i in range(m):fXi = float(multiply(alphas,labelMat).T*(dataMatrix*dataMatrix[i,:].T)) + bEi = fXi - float(labelMat[i])#if checks if an example violates KKT conditionsif ((labelMat[i]*Ei < -toler) and (alphas[i] < C)) or ((labelMat[i]*Ei > toler) and (alphas[i] > 0)):j = selectJrand(i,m)fXj = float(multiply(alphas,labelMat).T*(dataMatrix*dataMatrix[j,:].T)) + bEj = fXj - float(labelMat[j])alphaIold = alphas[i].copy(); alphaJold = alphas[j].copy();if (labelMat[i] != labelMat[j]):L = max(0, alphas[j] - alphas[i])H = min(C, C + alphas[j] - alphas[i])else:L = max(0, alphas[j] + alphas[i] - C)H = min(C, alphas[j] + alphas[i])if L==H: print "L==H"; continueeta = 2.0 * dataMatrix[i,:]*dataMatrix[j,:].T - dataMatrix[i,:]*dataMatrix[i,:].T - dataMatrix[j,:]*dataMatrix[j,:].Tif eta >= 0: print "eta>=0"; continuealphas[j] -= labelMat[j]*(Ei - Ej)/etaalphas[j] = clipAlpha(alphas[j],H,L)if (abs(alphas[j] - alphaJold) < 0.00001): print "j not moving enough"; continuealphas[i] += labelMat[j]*labelMat[i]*(alphaJold - alphas[j])#update i by the same amount as j#the update is in the oppostie directionb1 = b - Ei- labelMat[i]*(alphas[i]-alphaIold)*dataMatrix[i,:]*dataMatrix[i,:].T - labelMat[j]*(alphas[j]-alphaJold)*dataMatrix[i,:]*dataMatrix[j,:].Tb2 = b - Ej- labelMat[i]*(alphas[i]-alphaIold)*dataMatrix[i,:]*dataMatrix[j,:].T - labelMat[j]*(alphas[j]-alphaJold)*dataMatrix[j,:]*dataMatrix[j,:].Tif (0 < alphas[i]) and (C > alphas[i]): b = b1elif (0 < alphas[j]) and (C > alphas[j]): b = b2else: b = (b1 + b2)/2.0alphaPairsChanged += 1print "iter: %d i:%d, pairs changed %d" % (iter,i,alphaPairsChanged)if (alphaPairsChanged == 0): iter += 1else: iter = 0print "iteration number: %d" % iterreturn b,alphas#核转换函数def kernelTrans(X, A, kTup): #calc the kernel or transform data to a higher dimensional spacem,n = shape(X)K = mat(zeros((m,1)))if kTup[0]=='lin': K = X * A.T #linear kernelelif kTup[0]=='rbf':for j in range(m):deltaRow = X[j,:] - AK[j] = deltaRow*deltaRow.TK = exp(K/(-1*kTup[1]**2)) #divide in NumPy is element-wise not matrix like Matlabelse: raise NameError('Houston We Have a Problem -- \That Kernel is not recognized')return K#定义一个class，和其内部属性class optStruct:def __init__(self,dataMatIn, classLabels, C, toler, kTup): # Initialize the structure with the parameters self.X = dataMatInself.labelMat = classLabelsself.C = Cself.tol = tolerself.m = shape(dataMatIn)[0]self.alphas = mat(zeros((self.m,1)))self.b = 0self.eCache = mat(zeros((self.m,2))) #first column is valid flagself.K = mat(zeros((self.m,self.m)))for i in range(self.m):self.K[:,i] = kernelTrans(self.X, self.X[i,:], kTup)#计算误差 def calcEk(oS, k):fXk = float(multiply(oS.alphas,oS.labelMat).T*oS.K[:,k] + oS.b)Ek = fXk - float(oS.labelMat[k])return Ek#启发式的选择j def selectJ(i, oS, Ei): #this is the second choice -heurstic, and calcs EjmaxK = -1; maxDeltaE = 0; Ej = 0oS.eCache[i] = [1,Ei] #set valid #choose the alpha that gives the maximum delta EvalidEcacheList = nonzero(oS.eCache[:,0].A)[0]if (len(validEcacheList)) > 1:for k in validEcacheList: #loop through valid Ecache values and find the one that maximizes delta Eif k == i: continue #don't calc for i, waste of timeEk = calcEk(oS, k)deltaE = abs(Ei - Ek)if (deltaE > maxDeltaE):maxK = k; maxDeltaE = deltaE; Ej = Ekreturn maxK, Ejelse: #in this case (first time around) we don't have any valid eCache valuesj = selectJrand(i, oS.m)Ej = calcEk(oS, j)return j, Ej#更新误差def updateEk(oS, k):#after any alpha has changed update the new value in the cacheEk = calcEk(oS, k)oS.eCache[k] = [1,Ek]#内循环函数 def innerL(i, oS):Ei = calcEk(oS, i)if ((oS.labelMat[i]*Ei < -oS.tol) and (oS.alphas[i] < oS.C)) or ((oS.labelMat[i]*Ei > oS.tol) and (oS.alphas[i] > 0)):j,Ej = selectJ(i, oS, Ei) #this has been changed from selectJrandalphaIold = oS.alphas[i].copy(); alphaJold = oS.alphas[j].copy();if (oS.labelMat[i] != oS.labelMat[j]):L = max(0, oS.alphas[j] - oS.alphas[i])H = min(oS.C, oS.C + oS.alphas[j] - oS.alphas[i])else:L = max(0, oS.alphas[j] + oS.alphas[i] - oS.C)H = min(oS.C, oS.alphas[j] + oS.alphas[i])if L==H: print "L==H"; return 0eta = 2.0 * oS.K[i,j] - oS.K[i,i] - oS.K[j,j] #changed for kernelif eta >= 0: print "eta>=0"; return 0oS.alphas[j] -= oS.labelMat[j]*(Ei - Ej)/etaoS.alphas[j] = clipAlpha(oS.alphas[j],H,L)updateEk(oS, j) #added this for the Ecacheif (abs(oS.alphas[j] - alphaJold) < 0.00001): print "j not moving enough"; return 0oS.alphas[i] += oS.labelMat[j]*oS.labelMat[i]*(alphaJold - oS.alphas[j])#update i by the same amount as jupdateEk(oS, i) #added this for the Ecache#the update is in the oppostie directionb1 = oS.b - Ei- oS.labelMat[i]*(oS.alphas[i]-alphaIold)*oS.K[i,i] - oS.labelMat[j]*(oS.alphas[j]-alphaJold)*oS.K[i,j]b2 = oS.b - Ej- oS.labelMat[i]*(oS.alphas[i]-alphaIold)*oS.K[i,j]- oS.labelMat[j]*(oS.alphas[j]-alphaJold)*oS.K[j,j]if (0 < oS.alphas[i]) and (oS.C > oS.alphas[i]): oS.b = b1elif (0 < oS.alphas[j]) and (oS.C > oS.alphas[j]): oS.b = b2else: oS.b = (b1 + b2)/2.0return 1else: return 0#完整的SMO算法def smoP(dataMatIn, classLabels, C, toler, maxIter,kTup=('lin', 0)): #full Platt SMOoS = optStruct(mat(dataMatIn),mat(classLabels).transpose(),C,toler, kTup)iter = 0entireSet = True; alphaPairsChanged = 0while (iter < maxIter) and ((alphaPairsChanged > 0) or (entireSet)):alphaPairsChanged = 0if entireSet: #go over allfor i in range(oS.m): alphaPairsChanged += innerL(i,oS)print "fullSet, iter: %d i:%d, pairs changed %d" % (iter,i,alphaPairsChanged)iter += 1else:#go over non-bound (railed) alphasnonBoundIs = nonzero((oS.alphas.A > 0) * (oS.alphas.A < C))[0]for i in nonBoundIs:alphaPairsChanged += innerL(i,oS)print "non-bound, iter: %d i:%d, pairs changed %d" % (iter,i,alphaPairsChanged)iter += 1if entireSet: entireSet = False #toggle entire set loopelif (alphaPairsChanged == 0): entireSet = True print "iteration number: %d" % iterreturn oS.b,oS.alphas#计算权重Wdef calcWs(alphas,dataArr,classLabels):X = mat(dataArr); labelMat = mat(classLabels).transpose()m,n = shape(X)w = zeros((n,1))for i in range(m):w += multiply(alphas[i]*labelMat[i],X[i,:].T)return w#测试函数def testRbf(k1=1.3):dataArr,labelArr = loadDataSet('testSetRBF.txt')b,alphas = smoP(dataArr, labelArr, 200, 0.0001, 10000, ('rbf', k1)) #C=200 importantdatMat=mat(dataArr); labelMat = mat(labelArr).transpose()svInd=nonzero(alphas.A>0)[0]sVs=datMat[svInd] #get matrix of only support vectorslabelSV = labelMat[svInd];print "there are %d Support Vectors" % shape(sVs)[0]m,n = shape(datMat)errorCount = 0for i in range(m):kernelEval = kernelTrans(sVs,datMat[i,:],('rbf', k1))predict=kernelEval.T * multiply(labelSV,alphas[svInd]) + bif sign(predict)!=sign(labelArr[i]): errorCount += 1print "the training error rate is: %f" % (float(errorCount)/m)dataArr,labelArr = loadDataSet('testSetRBF2.txt')errorCount = 0datMat=mat(dataArr); labelMat = mat(labelArr).transpose()m,n = shape(datMat)for i in range(m):kernelEval = kernelTrans(sVs,datMat[i,:],('rbf', k1))predict=kernelEval.T * multiply(labelSV,alphas[svInd]) + bif sign(predict)!=sign(labelArr[i]): errorCount += 1 print "the test error rate is: %f" % (float(errorCount)/m) #图像转化成向量 def img2vector(filename):returnVect = zeros((1,1024))fr = open(filename)for i in range(32):lineStr = fr.readline()for j in range(32):returnVect[0,32*i+j] = int(lineStr[j])return returnVect#下载图像def loadImages(dirName):from os import listdirhwLabels = []trainingFileList = listdir(dirName) #load the training setm = len(trainingFileList)trainingMat = zeros((m,1024))for i in range(m):fileNameStr = trainingFileList[i]fileStr = fileNameStr.split('.')[0]#take off .txtclassNumStr = int(fileStr.split('_')[0])if classNumStr == 9: hwLabels.append(-1)else: hwLabels.append(1)trainingMat[i,:] = img2vector('%s/%s' % (dirName, fileNameStr))return trainingMat, hwLabels #数字集测试函数def testDigits(kTup=('rbf', 10)):dataArr,labelArr = loadImages('trainingDigits')b,alphas = smoP(dataArr, labelArr, 200, 0.0001, 10000, kTup)datMat=mat(dataArr); labelMat = mat(labelArr).transpose()svInd=nonzero(alphas.A>0)[0]sVs=datMat[svInd] labelSV = labelMat[svInd];print "there are %d Support Vectors" % shape(sVs)[0]m,n = shape(datMat)errorCount = 0for i in range(m):kernelEval = kernelTrans(sVs,datMat[i,:],kTup)predict=kernelEval.T * multiply(labelSV,alphas[svInd]) + bif sign(predict)!=sign(labelArr[i]): errorCount += 1print "the training error rate is: %f" % (float(errorCount)/m)dataArr,labelArr = loadImages('testDigits')errorCount = 0datMat=mat(dataArr); labelMat = mat(labelArr).transpose()m,n = shape(datMat)for i in range(m):kernelEval = kernelTrans(sVs,datMat[i,:],kTup)predict=kernelEval.T * multiply(labelSV,alphas[svInd]) + bif sign(predict)!=sign(labelArr[i]): errorCount += 1 print "the test error rate is: %f" % (float(errorCount)/m) '''#######********************************Non-Kernel VErsions below'''#######********************************#不使用核函数的SMO完整的算法class optStructK:def __init__(self,dataMatIn, classLabels, C, toler): # Initialize the structure with the parameters self.X = dataMatInself.labelMat = classLabelsself.C = Cself.tol = tolerself.m = shape(dataMatIn)[0]self.alphas = mat(zeros((self.m,1)))self.b = 0self.eCache = mat(zeros((self.m,2))) #first column is valid flagdef calcEkK(oS, k):fXk = float(multiply(oS.alphas,oS.labelMat).T*(oS.X*oS.X[k,:].T)) + oS.bEk = fXk - float(oS.labelMat[k])return Ekdef selectJK(i, oS, Ei): #this is the second choice -heurstic, and calcs EjmaxK = -1; maxDeltaE = 0; Ej = 0oS.eCache[i] = [1,Ei] #set valid #choose the alpha that gives the maximum delta EvalidEcacheList = nonzero(oS.eCache[:,0].A)[0]if (len(validEcacheList)) > 1:for k in validEcacheList: #loop through valid Ecache values and find the one that maximizes delta Eif k == i: continue #don't calc for i, waste of timeEk = calcEk(oS, k)deltaE = abs(Ei - Ek)if (deltaE > maxDeltaE):maxK = k; maxDeltaE = deltaE; Ej = Ekreturn maxK, Ejelse: #in this case (first time around) we don't have any valid eCache valuesj = selectJrand(i, oS.m)Ej = calcEk(oS, j)return j, Ejdef updateEkK(oS, k):#after any alpha has changed update the new value in the cacheEk = calcEk(oS, k)oS.eCache[k] = [1,Ek]def innerLK(i, oS):Ei = calcEk(oS, i)if ((oS.labelMat[i]*Ei < -oS.tol) and (oS.alphas[i] < oS.C)) or ((oS.labelMat[i]*Ei > oS.tol) and (oS.alphas[i] > 0)):j,Ej = selectJ(i, oS, Ei) #this has been changed from selectJrandalphaIold = oS.alphas[i].copy(); alphaJold = oS.alphas[j].copy();if (oS.labelMat[i] != oS.labelMat[j]):L = max(0, oS.alphas[j] - oS.alphas[i])H = min(oS.C, oS.C + oS.alphas[j] - oS.alphas[i])else:L = max(0, oS.alphas[j] + oS.alphas[i] - oS.C)H = min(oS.C, oS.alphas[j] + oS.alphas[i])if L==H: print "L==H"; return 0eta = 2.0 * oS.X[i,:]*oS.X[j,:].T - oS.X[i,:]*oS.X[i,:].T - oS.X[j,:]*oS.X[j,:].Tif eta >= 0: print "eta>=0"; return 0oS.alphas[j] -= oS.labelMat[j]*(Ei - Ej)/etaoS.alphas[j] = clipAlpha(oS.alphas[j],H,L)updateEk(oS, j) #added this for the Ecacheif (abs(oS.alphas[j] - alphaJold) < 0.00001): print "j not moving enough"; return 0oS.alphas[i] += oS.labelMat[j]*oS.labelMat[i]*(alphaJold - oS.alphas[j])#update i by the same amount as jupdateEk(oS, i) #added this for the Ecache#the update is in the oppostie directionb1 = oS.b - Ei- oS.labelMat[i]*(oS.alphas[i]-alphaIold)*oS.X[i,:]*oS.X[i,:].T - oS.labelMat[j]*(oS.alphas[j]-alphaJold)*oS.X[i,:]*oS.X[j,:].Tb2 = oS.b - Ej- oS.labelMat[i]*(oS.alphas[i]-alphaIold)*oS.X[i,:]*oS.X[j,:].T - oS.labelMat[j]*(oS.alphas[j]-alphaJold)*oS.X[j,:]*oS.X[j,:].Tif (0 < oS.alphas[i]) and (oS.C > oS.alphas[i]): oS.b = b1elif (0 < oS.alphas[j]) and (oS.C > oS.alphas[j]): oS.b = b2else: oS.b = (b1 + b2)/2.0return 1else: return 0def smoPK(dataMatIn, classLabels, C, toler, maxIter): #full Platt SMOoS = optStruct(mat(dataMatIn),mat(classLabels).transpose(),C,toler)iter = 0entireSet = True; alphaPairsChanged = 0while (iter < maxIter) and ((alphaPairsChanged > 0) or (entireSet)):alphaPairsChanged = 0if entireSet: #go over allfor i in range(oS.m): alphaPairsChanged += innerL(i,oS)print "fullSet, iter: %d i:%d, pairs changed %d" % (iter,i,alphaPairsChanged)iter += 1else:#go over non-bound (railed) alphasnonBoundIs = nonzero((oS.alphas.A > 0) * (oS.alphas.A < C))[0]for i in nonBoundIs:alphaPairsChanged += innerL(i,oS)print "non-bound, iter: %d i:%d, pairs changed %d" % (iter,i,alphaPairsChanged)iter += 1if entireSet: entireSet = False #toggle entire set loopelif (alphaPairsChanged == 0): entireSet = True print "iteration number: %d" % iterreturn oS.b,oS.alphas
这当中牵扯到很多理论知识，就需要小伙伴自行进行学习啦O(∩_∩)O
