用Python研究了三千套房子，告诉你究竟是什么抬高了房价？

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sn
import missingno as msno
%matplotlib inlinetrain = pd.read_csv('train.csv',index_col=0) 
#导入训练集test = pd.read_csv('test.csv',index_col=0) 
#导入测试集train.head(3)

print('train训练集缺失数据分布图')msno.matrix(train)

print('test测试集缺失数据分布图')msno.matrix(test)

##分割Y和X数据y=train['SalePrice']
#看一下y的值分布prices = pd.DataFrame({'price':y,'log(price+1)':np.log1p(y)})
prices.hist()

y = np.log1p(y)#+1的目的是防止对数转化后的值无意义

#合并训练特征和测试集
all_df = pd.concat((X,test),axis=0)
print('all_df缺失数据图')
msno.matrix(all_df)

#定义缺失统计函数
def show_missing(feature):    
    missing = feature.columns[feature.isnull().any()].tolist()    
    return missingprint('缺失特征的数据缺失量统计：')
    all_df[show_missing(all_df)].isnull().sum()

#先处理numeric数值型数据
#挨个儿看一下分布
fig,axs = plt.subplots(3,2,figsize=(16,9))
all_df['BsmtFinSF1'].hist(ax = axs[0,0])
#众数填充all_df['BsmtFinSF2'].hist(ax = axs[0,1])
#众数all_df['BsmtUnfSF'].hist(ax =  axs[1,0])
#中位数all_df['TotalBsmtSF'].hist(ax = axs[1,1])
#均值填充all_df['BsmtFullBath'].hist(ax = axs[2,0])
#众数all_df['BsmtHalfBath'].hist(ax = axs[2,1])
#众数#lotfrontage用均值填充
mean_lotfrontage = all_df.LotFrontage.mean()
all_df.LotFrontage.hist()
rint('用均值填充:')
cat_input(all_df,'LotFrontage',mean_lotfrontage)

cat_input(all_df,'BsmtFinSF1',0.0)
cat_input(all_df,'BsmtFinSF2',0.0)
cat_input(all_df,'BsmtFullBath',0.0)
cat_input(all_df,'BsmtHalfBath',0.0)
cat_input(all_df,'BsmtUnfSF',467.00)
cat_input(all_df,'TotalBsmtSF',1051.78)

#在处理字符型，同样，挨个看下分布
fig,axs = plt.subplots(4,2,figsize=(16,9))
all_df['MSZoning'].hist(ax = axs[0,0])
#众数填充
all_df['Utilities'].hist(ax = axs[0,1])
#众数
all_df['Exterior1st'].hist(ax =  axs[1,0])
#众数
all_df['Exterior2nd'].hist(ax = axs[1,1])
#众数填充
all_df['KitchenQual'].hist(ax = axs[2,0])
#众数
all_df['Functional'].hist(ax = axs[2,1])
#众数
all_df['SaleType'].hist(ax = axs[3,0])
#众数

cat_input(all_df,'MSZoning','RL')
cat_input(all_df,'Utilities','AllPub')
cat_input(all_df,'Exterior1st','VinylSd')
cat_input(all_df,'Exterior2nd','VinylSd')
cat_input(all_df,'KitchenQual','TA')
cat_input(all_df,'Functional','Typ')
cat_input(all_df,'SaleType','WD')
#再看一下缺失分布
msno.matrix(all_df)

#观察特征属性发现，MSSubClass是分类特征，但是数据给的是数值型，需要对其做转换
all_df['MSSubClass']=all_df['MSSubClass'].astype(str)
#将分类变量转变成数值变量
all_df = pd.get_dummies(all_df)
print('分类变量转换完成后有{}行{}列'.format(*all_df.shape))

#标准化处理
numeric_cols = all_df.columns[all_df.dtypes !='uint8']
#x-mean(x)/std(x)numeric_mean = all_df.loc[:,numeric_cols].mean()
numeric_std = all_df.loc[:,numeric_cols].std()
all_df.loc[:,numeric_cols] = (all_df.loc[:,numeric_cols]-numeric_mean)/numeric_std

train_df = all_df.ix[0:1460]
#训练集
test_df = all_df.ix[1461:] 
#测试集

from sklearn import cross_validation
from sklearn import linear_model
from sklearn.learning_curve 
import learning_curvefrom sklearn.metrics 
import explained_variance_scorefrom sklearn.grid_search 
import GridSearchCVfrom sklearn.model_selection 
import cross_val_scorefrom sklearn.ensemble 
import RandomForestRegressory = y.values 
#转换成array数组
X = train_df.values #
转换成array数组

cv = cross_validation.ShuffleSplit(len(X),n_iter=3,test_size=0.2)
print('岭回归交叉验证结果：')
for train_index,test_index in cv:    
    ridge = linear_model.Ridge(alpha=1).fit(X,y)    
    print('train_score:{0:.3f},test_score:{1:.3f}\n'.format(ridge.score(X[train_index],y[train_index]),                                                            
    ridge.score(X[test_index],y[test_index])))
    print('随机森林交叉验证结果：')
    for train_index,test_index in cv:    
        rf = RandomForestRegressor().fit(X,y)   
        print('train_score:{0:.3f},test_score:{1:.3f}\n'.format(rf.score(X[train_index],y[train_index]),                                                            
        rf.score(X[test_index],y[test_index])))

alphas =[0.01,0.1,1,10,20,50,100,300]
test_scores = []
for alp in alphas:    
    clf = linear_model.Ridge(alp)    
    test_score = -cross_val_score(clf,X,y,cv=10,scoring='neg_mean_squared_error')    
    test_scores.append(np.mean(test_score))
import matplotlib.pyplot as plt
%matplotlib inlineplt.plot(alphas,test_scores)plt.title('alpha vs test_score')

#随机森林的深度参数max_depth=[2,4,6,8,10]
test_scores_depth = []
for depth in max_depth:    
    clf = RandomForestRegressor(max_depth=depth)    
    test_score_depth = -cross_val_score(clf,X,y,cv=10,scoring='neg_mean_squared_error')   
    test_scores_depth.append(np.mean(test_score_depth))
    #随机森林的特征个数参数
    max_features =[.1, .3, .5, .7, .9, .99]
    test_scores_feature = []
    for feature in max_features:    
        clf = RandomForestRegressor(max_features=feature)    
        test_score_feature = -cross_val_score(clf,X,y,cv=10,scoring='neg_mean_squared_error')    
        test_scores_feature.append(np.mean(test_score_feature))
        #随机森林的估计器个位数参数
        n_estimators =[10,50,100,200,500]
        test_scores_n = []
        for n in n_estimators:    
            clf = RandomForestRegressor(n_estimators=n)    
            test_score_n = -cross_val_score(clf,X,y,cv=10,scoring='neg_mean_squared_error')    
            test_scores_n.append(np.mean(test_score_n))

#预测
ridge = linear_model.Ridge(alpha=10).fit(X,y)
rf = RandomForestRegressor(n_estimators=500,max_depth=8,max_features=.5).fit(X,y)
y_ridge = np.expm1(ridge.predict(test_df.values))
y_rf = np.expm1(rf.predict(test_df.values))
y_final = (y_ridge + y_rf)/2