python学习-数据分析2（分类型数据处理+SVM建模）

来源：互联网发布：在线打电话软件编辑：程序博客网时间：2024/06/08 07:46

原文地址 http://blog.sina.com.cn/s/blog_727a704c0102vn2c.html

研究多种数据处理方式（拆分，整合，哑变量，重编码，分段变量处理等）

测试SVM模型

示例：

研究房屋相关信息预测价格区间

[转载]python学习-数据分析2（分类型数据处理+SVM建模）

#!/usr/bin/env python# -*- coding:utf-8 -*-#1.读取数据库import MySQLdbimport pandas as pdimport numpy as npimport pandas.io.sql as sqlconn = MySQLdb.connect(host='', port=3306,charset='utf8',user='', passwd='', db='')df = sql.read_sql('select * from test',conn)conn.close()[转载]python学习-数据分析2（分类型数据处理+SVM建模）#2.数据处理(分词+表达式)#拆分detail成多个变量detailframe = pd.DataFrame()detailnarry = df['detail']for d in detailnarry:   ds =pd.DataFrame(d.split('/')).T   detailframe = detailframe.append(ds)detailframe.columns=['bnum','floor','allfloor','dir','years']detailframe.index=detailnarry.index#整合数据prodf = pd.merge(df.ix[:,[2,3,4,5,6,7,9,10]],detailframe,left_index=True,right_index=True)#缺失值处理，删除有空值的样本prodf = prodf[(prodf.years<>"")&(prodf.bnum<>"")&(prodf.dir<>"")]#哑变量处理（回归模型需要，有些模型可不做）dumdir = pd.get_dummies(prodf['dir'],prefix='dir')dumbn = pd.get_dummies(prodf['bnum'],prefix='bnum')#保留数字，删除多余文字import repattern = 'd+'regx = re.compile(pattern)r = lambda x: int(regx.findall(x)[0])ss = prodf.ix[:,[3,4,5,9,10,12]].applymap(r)fliterdf = pd.merge(prodf.ix[:,[0,1,2,6,7,8,11]],ss,left_index=True,right_index=True)fliterdf = fliterdf.join(dumdir).join(dumbn)#print fliterdf[:10]fliterdf.price = fliterdf['price'].astype(float)fliterdf = fliterdf.dropna()#3.描述统计print fliterdf.describe()print fliterdf['floor'].value_counts()by_ad = fliterdf.groupby(['floor','dir'])print by_ad.size().unstack().fillna(0)import matplotlib.pyplot as pltplt.scatter(fliterdf['sizes'],fliterdf['price'])plt.show()#离散化预测变量plt.hist(fliterdf['price'],bins=50,color='k')plt.show()bins=[0,200,500,1000,2000,4000]fliterdf['pricecats'] = pd.cut(fliterdf['price'],bins)#中文文本变量重新编码bnum_to_bnm = {u'4室0厅': '1', u'4室1厅': '2', u'4室2厅': '3', u'4室3厅': '4', u'4室4厅': '5'}fliterdf['bnm'] = fliterdf['bnum'].map(bnum_to_bnm)dir_to_drm = {u'北向': '1', u'西向': '2', u'东向': '3', u'西北向': '4', u'东北向': '5',               u'东西向': '6',u'西南向': '7',u'东南向': '8',u'南向': '9',u'南北向': '10'}fliterdf['drm'] = fliterdf['dir'].map(dir_to_drm)#4.建模import sklearn.linear_modelimport numpy as np#建模,测试数据分组sampleR=0.7nsample = len(y)sampleBoundary = int(nsample*sampleR)#SVM（svc,rbf,poly三种方式）import sklearn.svm as svmx_c=fliterdf.ix[:,[7,10,11,29,30]]y_c= fliterdf['pricecats']train_xc = x_c.ix[shffleIdx[:sampleBoundary]]test_xc = x_c.ix[shffleIdx[sampleBoundary:]]train_yc = y_c[shffleIdx[:sampleBoundary]]test_yc = y_c[shffleIdx[sampleBoundary:]]#svc = svm.SVC(kernel='linear',C=1.0).fit(train_xc,train_yc)rbf_svc = svm.SVC(kernel='rbf',gamma=0.7,C=1.0).fit(train_xc,train_yc)#poly_svc = svm.SVC(kernel='poly',degree=3,C=1.0).fit(train_xc,train_yc)#预测数据，准确率计算pre_yc = rbf_svc.predict(test_xc)correctnum =0test_yc.index = range(len(pre_yc))for i in range(len(pre_yc)):    if (pre_yc[i]==test_yc.iloc[i]):        correctnum =correctnum +1accuracy = correctnum*1.0/len(pre_yc)print accuracy#结果存入数据库conn = MySQLdb.connect(host='', port=3306,charset='utf8',user='', passwd='', db='')#sql.write_frame(res,"temp_test",conn)cur = conn.cursor()wsql = 'INSERT INTO  temp_test(y) VALUES (%s)'for py in pre_y:    cur.execute(wsql,py)    conn.commit()cur.close()conn.close()

0 0