特征选择

# 加载 IRIS 数据集做演示
import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from sklearn.preprocessing import LabelEncoder
%matplotlib inline

df = sns.load_dataset('iris')
print(df.shape)
df['species'] = LabelEncoder().fit_transform(df.iloc[:, 4])
df.head()

(150, 5)

# 方差选择法
# 自己手写理论公式来实现功能
def VarianceThreshold(df, threshold=0.):
    dfc = df.iloc[:, :4].copy()
    print('>>>特征名：\n', dfc.columns.tolist())
    # 1 求方差
    var = np.sum(np.power(np.matrix(dfc.values)-np.matrix(dfc.mean()), 2), axis=0)/(dfc.shape[0]-1)
    T = []
    # 2 筛选大于阈值的特征
    for index, v in enumerate(var.reshape(-1, 1)):
        if v > threshold:
            T.append(index)
    dfc = dfc.iloc[:, T]
    return var, dfc

# 阈值设置为 0.6
var, dfc = VarianceThreshold(df, 0.60)
print('\n>>>原始特征对应的方差值：\n', var)
print('\n>>>方差阈值选择后的特征名：\n', dfc.columns)
dfc.head()

>>>特征名：
 ['sepal_length', 'sepal_width', 'petal_length', 'petal_width']

>>>原始特征对应的方差值：
 [[0.68569351 0.18997942 3.11627785 0.58100626]]

>>>方差阈值选择后的特征名：
 Index(['sepal_length', 'petal_length'], dtype='object')

# 调用 sklearn 模块 API 接口
from sklearn.feature_selection import VarianceThreshold

vt = VarianceThreshold(0.6)
x_vt = vt.fit_transform(df.iloc[:,:4])
print(vt.variances_)
x_vt[:5]

[0.68112222 0.18871289 3.09550267 0.57713289]





array([[5.1, 1.4],
       [4.9, 1.4],
       [4.7, 1.3],
       [4.6, 1.5],
       [5. , 1.4]])

# 相关系数--特征与特征
# 自己手写理论公式实现功能
def corr_selector(df):
    dfc = df.copy().iloc[:,:4]
    CORR = np.zeros((dfc.shape[1], dfc.shape[1])) 
    delete, save = [], []
    for i in range(dfc.shape[1]):
        if dfc.columns.tolist()[i] not in delete:
            save.append(dfc.columns.tolist()[i])
        for j in range(i+1, dfc.shape[1]):
            # 计算特征与特征之间的相关系数
            cov = np.sum((dfc.iloc[:,i]-dfc.iloc[:,i].mean()) * (df.iloc[:,j]-df.iloc[:,j].mean()))
            std = np.sqrt(np.sum((df.iloc[:,i]-df.iloc[:,i].mean())**2)) * np.sqrt(np.sum((df.iloc[:,j]-df.iloc[:,j].mean())**2))
            corr = cov/std
            CORR[i][j] = corr
            # 筛选掉高线性相关两两特征中的某一个特征
            if (np.abs(corr) > 0.89) and (dfc.columns.tolist()[j] not in delete):
                delete.append(dfc.columns.tolist()[j])
    dfc_ = dfc[save].copy()
    return CORR, dfc_

corr,dfc_ = corr_selector(df)
print(corr)
dfc_.head()

[[ 0.         -0.11756978  0.87175378  0.81794113]
 [ 0.          0.         -0.4284401  -0.36612593]
 [ 0.          0.          0.          0.96286543]
 [ 0.          0.          0.          0.        ]]

# 相关系数--特征与目标变量
# 自己手写理论公式实现功能
def corr_selector(df):
    X, y = df.iloc[:, :4], df.iloc[:, 4]
    cor_list = []
    for i in X.columns.tolist():
        cor = np.corrcoef(X[i], y)[0, 1]
        cor_list.append(cor)
    print(X.columns.tolist())
    print(cor_list)
    return cor_list

corr_selector(df)

['sepal_length', 'sepal_width', 'petal_length', 'petal_width']
[0.7825612318100814, -0.4266575607811243, 0.9490346990083884, 0.9565473328764028]





[0.7825612318100814,
 -0.4266575607811243,
 0.9490346990083884,
 0.9565473328764028]

df.plot()
plt.savefig('corr.png')

# 调用 sklearn 模块 API 接口
from sklearn.feature_selection import SelectKBest
from scipy.stats import pearsonr
from numpy import array

fun = lambda X, Y: tuple(map(tuple, array(list(map(lambda x: pearsonr(x, Y), X.T))).T))
sb = SelectKBest(fun, k=2)
x_sb = sb.fit_transform(df.iloc[:,:4], df.iloc[:, 4])
print('>>>检验统计值(相关系数)：\n', sb.scores_)
print('\n>>>P值：\n', sb.pvalues_)
x_sb[:5]

>>>检验统计值(相关系数)：
 [ 0.78256123 -0.42665756  0.9490347   0.95654733]

>>>P值：
 [2.89047835e-32 5.20156326e-08 4.20187315e-76 4.15531102e-81]


array([[1.4, 0.2],
       [1.4, 0.2],
       [1.3, 0.2],
       [1.5, 0.2],
       [1.4, 0.2]])

# 调用 sklearn 模块 API 接口
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2

skb = SelectKBest(chi2, k=2)
x_skb = skb.fit_transform(df.iloc[:,:4], df.iloc[:,4])
print('>>>检验统计值(卡方值)：\n', skb.scores_)
print('\n>>>P值：\n', skb.pvalues_)
x_skb[:5]

>>>检验统计值(卡方值)：
 [ 10.81782088   3.7107283  116.31261309  67.0483602 ]

>>>P值：
 [4.47651499e-03 1.56395980e-01 5.53397228e-26 2.75824965e-15]

array([[1.4, 0.2],
       [1.4, 0.2],
       [1.3, 0.2],
       [1.5, 0.2],
       [1.4, 0.2]])

# 互信息法
# 自己手写理论公式实现功能
# np.where(condition, x, y)
# 满足条件(condition)，输出x，不满足输出y。
# numpy.intersect1d(ar1, ar2, assume_unique=False) [资源]
# 找到两个数组的交集。
# 返回两个输入数组中已排序的唯一值。
# math.log(x[, base])
# x -- 数值表达式。
# base -- 可选，底数，默认为 e
def MI_and_NMI():
    import math
    from sklearn import metrics
    A = np.array([1,1,1,1,1,1,2,2,2,2,2,2,3,3,3,3,3])
    B = np.array([1,2,1,1,1,1,1,2,2,2,2,3,1,1,3,3,3])
    N, A_ids, B_ids = len(A), set(A), set(B)
    # print(N, A_ids, B_ids)

    # 互信息计算
    MI, eps = 0, 1.4e-45
    # 你说的这种应该是为了避免log0的结果
    # 所以添加一个非常小的数字,避免无穷的...log(0)的情况
    for i in A_ids:
        for j in B_ids:
            ida = np.where(A==i)  # 返回索引
            idb = np.where(B==j)
            idab = np.intersect1d(ida, idb)  # 返回相同的部分
            #print(ida,idb,idab)

            # 概率值
            px = 1.0*len(ida[0])/N  # 出现的次数/总样本数
            py = 1.0*len(idb[0])/N
            pxy = 1.0*len(idab)/N
            # MI 值
            MI += pxy * math.log(pxy/(px*py)+eps, 2)

    # 标准互信息计算
    Hx = 0
    for i in A_ids:
        ida = np.where(A==i)
        px = 1.0*len(ida[0])/N
        Hx -= px * math.log(px+eps, 2)
    Hy = 0
    for j in B_ids:
        idb = np.where(B==j)
        py = 1.0*len(idb[0])/N
        Hy -= py * math.log(py+eps, 2)
    NMI = 2.0*MI/(Hx+Hy)

    return MI, NMI, metrics.normalized_mutual_info_score(A,B)

MI_and_NMI()

(0.565445018842856, 0.3645617718571898, 0.36456177185718985)

from sklearn import metrics

A = [1, 1, 1, 2, 3, 3]
B = [1, 2, 3, 1, 2, 3]
result_NMI=metrics.normalized_mutual_info_score(A, B)
print("result_NMI:",result_NMI)

result_NMI: 0.3016631159962042

from sklearn.svm import SVC

X1, X2, Y = df[['petal_width']], df[['sepal_width']], df['species']
svc = SVC()
for X in [X1, X2]:
    svc.fit(X,Y)
    print(svc.score(X,Y))

0.96
0.5533333333333333

from sklearn.feature_selection import RFE
from sklearn.linear_model import LinearRegression
from sklearn.datasets import load_boston

boston = load_boston()
X = boston["data"]
Y = boston["target"]
names = boston["feature_names"]

#use linear regression as the model
lr = LinearRegression()
#rank all features, i.e continue the elimination until the last one
rfe = RFE(lr, n_features_to_select=2)
rfe.fit(X,Y)

print("Features sorted by their rank:")
print(sorted(zip(rfe.ranking_, names)))

Features sorted by their rank:
[(1, 'NOX'), (1, 'RM'), (2, 'CHAS'), (3, 'PTRATIO'), (4, 'DIS'), (5, 'LSTAT'), (6, 'RAD'), (7, 'CRIM'), (8, 'INDUS'), (9, 'ZN'), (10, 'TAX'), (11, 'B'), (12, 'AGE')]

from sklearn.linear_model import LinearRegression

X = df.iloc[:,:4]
Y = df.iloc[:,4]
lr = LinearRegression()
lr.fit(X,Y)
print(X.columns.tolist())
print(lr.coef_)

['sepal_length', 'sepal_width', 'petal_length', 'petal_width']
[-0.11190585 -0.04007949  0.22864503  0.60925205]

from sklearn.linear_model import Lasso

X = df.iloc[:,:4]
Y = df.iloc[:,4]
lasso = Lasso(alpha=0.3)
lasso.fit(X,Y)
print(X.columns.tolist())
print(lasso.coef_)

['sepal_length', 'sepal_width', 'petal_length', 'petal_width']
[ 0.        -0.         0.3435091  0.       ]

from sklearn.linear_model import Ridge

X = df.iloc[:,:4]
Y = df.iloc[:,4]
ridge = Ridge(alpha=0.3)
ridge.fit(X,Y)
print(X.columns.tolist())
print(ridge.coef_)

['sepal_length', 'sepal_width', 'petal_length', 'petal_width']
[-0.11327412 -0.03665944  0.23969479  0.58451798]

# 平均不纯度减少
from sklearn.datasets import load_boston
from sklearn.ensemble import RandomForestRegressor
import numpy as np

#Load boston housing dataset as an example
boston = load_boston()
X = boston["data"]
Y = boston["target"]
names = boston["feature_names"]
# 训练随机森林模型，并通过feature_importances_属性获取每个特征的重要性分数。
rf = RandomForestRegressor()
rf.fit(X, Y)
print("Features sorted by their score:")
print(sorted(zip(map(lambda x: round(x, 4), rf.feature_importances_), names),
             reverse=True))

Features sorted by their score:
[(0.4368, 'RM'), (0.3694, 'LSTAT'), (0.0655, 'DIS'), (0.0384, 'CRIM'), (0.0248, 'NOX'), (0.0156, 'PTRATIO'), (0.0141, 'TAX'), (0.0132, 'AGE'), (0.0117, 'B'), (0.0051, 'INDUS'), (0.0036, 'RAD'), (0.001, 'ZN'), (0.0008, 'CHAS')]

# 通过sklearn中的随机森林返回特征的重要性：

from sklearn.datasets import load_iris
from sklearn.ensemble import RandomForestClassifier
import pandas as pd
import numpy as np
iris = load_iris()
df = pd.DataFrame(iris.data, columns=iris.feature_names)
df['is_train'] = np.random.uniform(0, 1, len(df)) <= .75
df['species'] = pd.Categorical.from_codes(iris.target, iris.target_names)
df.head()
train, test = df[df['is_train']==True], df[df['is_train']==False]
features = df.columns[:4]
clf = RandomForestClassifier(n_jobs=2)
y, _ = pd.factorize(train['species'])
clf.fit(train[features], y)
preds = iris.target_names[clf.predict(test[features])]
pd.crosstab(test['species'], preds, rownames=['actual'], colnames=['preds'])
clf.feature_importances_

array([0.08850542, 0.0250167 , 0.39281297, 0.49366491])

# 生成不平衡分类数据集
from collections import Counter
from sklearn.datasets import make_classification
X, y = make_classification(n_samples=3000, n_features=2, n_informative=2,
                           n_redundant=0, n_repeated=0, n_classes=3,
                           n_clusters_per_class=1,
                           weights=[0.1, 0.05, 0.85],
                           class_sep=0.8, random_state=2018)
Counter(y)

Counter({2: 2532, 1: 163, 0: 305})

# 使用RandomOverSampler从少数类的样本中进行随机采样来增加新的样本使各个分类均衡
from imblearn.over_sampling import RandomOverSampler

ros = RandomOverSampler(random_state=0)
X_resampled, y_resampled = ros.fit_sample(X, y)
sorted(Counter(y_resampled).items())

Using TensorFlow backend.





[(0, 2532), (1, 2532), (2, 2532)]

# SMOTE: 对于少数类样本a, 随机选择一个最近邻的样本b, 然后从a与b的连线上随机选取一个点c作为新的少数类样本
from imblearn.over_sampling import SMOTE

X_resampled_smote, y_resampled_smote = SMOTE().fit_sample(X, y)

sorted(Counter(y_resampled_smote).items())
# [(0, 2532), (1, 2532), (2, 2532)]

[(0, 2532), (1, 2532), (2, 2532)]

# ADASYN: 关注的是在那些基于K最近邻分类器被错误分类的原始样本附近生成新的少数类样本
from imblearn.over_sampling import ADASYN

X_resampled_adasyn, y_resampled_adasyn = ADASYN().fit_sample(X, y)

sorted(Counter(y_resampled_adasyn).items())
# [(0, 2522), (1, 2520), (2, 2532)]

[(0, 2522), (1, 2520), (2, 2532)]

# RandomUnderSampler函数是一种快速并十分简单的方式来平衡各个类别的数据: 随机选取数据的子集.
from imblearn.under_sampling import RandomUnderSampler
rus = RandomUnderSampler(random_state=0)
X_resampled, y_resampled = rus.fit_sample(X, y)

sorted(Counter(y_resampled).items())

[(0, 163), (1, 163), (2, 163)]

# 在之前的SMOTE方法中, 当由边界的样本与其他样本进行过采样差值时, 很容易生成一些噪音数据. 因此, 在过采样之后需要对样本进行清洗. 
# 这样TomekLink 与 EditedNearestNeighbours方法就能实现上述的要求.
from imblearn.combine import SMOTEENN
smote_enn = SMOTEENN(random_state=0)
X_resampled, y_resampled = smote_enn.fit_sample(X, y)

sorted(Counter(y_resampled).items())

[(0, 2111), (1, 2099), (2, 1893)]

from imblearn.combine import SMOTETomek
smote_tomek = SMOTETomek(random_state=0)
X_resampled, y_resampled = smote_tomek.fit_sample(X, y)

sorted(Counter(y_resampled).items())
# [(0, 2412), (1, 2414), (2, 2396)]

[(0, 2412), (1, 2414), (2, 2396)]

# 使用SVM的权重调节处理不均衡样本 权重为balanced 意味着权重为各分类数据量的反比
from sklearn.svm import SVC  

svm_model = SVC(class_weight='balanced')
svm_model.fit(X, y)

SVC(C=1.0, break_ties=False, cache_size=200, class_weight='balanced', coef0=0.0,
    decision_function_shape='ovr', degree=3, gamma='scale', kernel='rbf',
    max_iter=-1, probability=False, random_state=None, shrinking=True,
    tol=0.001, verbose=False)

# BalancedBaggingClassifier 允许在训练每个基学习器之前对每个子集进行重抽样. 简而言之, 该方法结合了EasyEnsemble采样器与分类器(如BaggingClassifier)的结果.
from sklearn.tree import DecisionTreeClassifier
from imblearn.ensemble import BalancedBaggingClassifier

bbc = BalancedBaggingClassifier(base_estimator=DecisionTreeClassifier(),
                                replacement=False,
                                random_state=0)
bbc.fit(X, y) 

BalancedBaggingClassifier(base_estimator=DecisionTreeClassifier(ccp_alpha=0.0,
                                                                class_weight=None,
                                                                criterion='gini',
                                                                max_depth=None,
                                                                max_features=None,
                                                                max_leaf_nodes=None,
                                                                min_impurity_decrease=0.0,
                                                                min_impurity_split=None,
                                                                min_samples_leaf=1,
                                                                min_samples_split=2,
                                                                min_weight_fraction_leaf=0.0,
                                                                presort='deprecated',
                                                                random_state=None,
                                                                splitter='best'),
                          bootstrap=True, bootstrap_features=False,
                          max_features=1.0, max_samples=1.0, n_estimators=10,
                          n_jobs=None, oob_score=False, random_state=0,
                          replacement=False, sampling_strategy='auto',
                          verbose=0, warm_start=False)