# 训练和测试数据的观察

``` 1import gc
2import itertools
3from copy import deepcopy
4import numpy as np
5import pandas as pd
6from tqdm import tqdm
7from scipy.stats import ks_2samp
8from sklearn.preprocessing import scale, MinMaxScaler
9from sklearn.manifold import TSNE
10from sklearn.decomposition import TruncatedSVD
11from sklearn.decomposition import FastICA
12from sklearn.random_projection import GaussianRandomProjection
13from sklearn.random_projection import SparseRandomProjection
14from sklearn import manifold
15from sklearn.ensemble import ExtraTreesClassifier
16from sklearn.model_selection import cross_val_predict
17from sklearn.metrics import classification_report
18from sklearn.model_selection import StratifiedKFold
19import matplotlib.pyplot as plt
20from matplotlib.ticker import NullFormatter
21%matplotlib inline```

1.t-SNE分布概述

1.0 数据预处理

• 从训练集和测试集中获取4459行并将它们连接起来
• 删除了训练集中标准差为0的列
• 删除了训练集中重复的列
• 对包含异常值（> 3x标准差）的所有列进行对数变换
• 创建数据集：
• 均值 - 方差比例缩放所有列，包括0值！
• 均值 - 方差比例所有列不包括 0值！ 合并数据集
``` 1def combined_data(train, test):
2"""
3Get the combined data
4:param train pandas.dataframe:
5:param test pandas.dataframe:
6:return pandas.dataframe:
7"""
8A = set(train.columns.values)
9B = set(test.columns.values)
10colToDel = A.difference(B)
11total_df = pd.concat([train.drop(colToDel, axis=1), test], axis=0)

``` 1def remove_duplicate_columns(total_df):
2"""
3Removing duplicate columns
4"""
5colsToRemove = []
6columns = total_df.columns
7for i in range(len(columns) - 1):
8v = total_df[columns[i]].values
9for j in range(i + 1, len(columns)):
10    if np.array_equal(v, total_df[columns[j]].values):
11        colsToRemove.append(columns[j])
12colsToRemove = list(set(colsToRemove))
13total_df.drop(colsToRemove, axis=1, inplace=True)
14print(f">> Dropped {len(colsToRemove)} duplicate columns")

``` 1def log_significant_outliers(total_df):
2"""
3frist master fill na
4Log-transform all columns which have significant outliers (> 3x standard deviation)
5:return pandas.dataframe:
6"""
7total_df_all = deepcopy(total_df).select_dtypes(include=[np.number])
8total_df_all.fillna(0, inplace=True)  # ********
9for col in total_df_all.columns:
10# print(col)
11data = total_df_all[col].values
12data_mean, data_std = np.mean(data), np.std(data)
13cut_off = data_std * 3
14lower, upper = data_mean - cut_off, data_mean + cut_off
15outliers = [x for x in data if x < lower or x > upper]
16
17if len(outliers) > 0:
18    non_zero_index = data != 0
19    total_df_all.loc[non_zero_index, col] = np.log(data[non_zero_index])
20
21non_zero_rows = total_df[col] != 0
22total_df_all.loc[non_zero_rows, col] = scale(total_df_all.loc[non_zero_rows, col])
23gc.collect()
24

1.1 执行PCA

``` 1def test_pca(data, train_idx, test_idx, create_plots=True):
2"""
3data, panda.DataFrame
4train_idx = range(0, len(train_df))
5test_idx = range(len(train_df), len(total_df))
6Run PCA analysis, return embeding
7"""
8data = data.select_dtypes(include=[np.number])
9data = data.fillna(0)
10# Create a PCA object, specifying how many components we wish to keep
11pca = PCA(n_components=len(data.columns))
12
13# Run PCA on scaled numeric dataframe, and retrieve the  projected data
14pca_trafo = pca.fit_transform(data)
15
16# The transformed data is in a numpy matrix. This may be inconvenient if we want to further
17# process the data, and have a more visual impression of what each column is etc. We therefore
18# put transformed/projected data into new dataframe, where we specify column names and index
19pca_df = pd.DataFrame(
20pca_trafo,
21index=data.index,
22columns=['PC' + str(i + 1) for i in range(pca_trafo.shape[1])]
23)
24
25if create_plots:
26# Create two plots next to each other
27_, axes = plt.subplots(2, 2, figsize=(20, 15))
28axes = list(itertools.chain.from_iterable(axes))
29
30# Plot the explained variance# Plot t
31axes[0].plot(
32    pca.explained_variance_ratio_, "--o", linewidth=2,
33    label="Explained variance ratio"
34)
35
36# Plot the explained variance# Plot t
37axes[0].plot(
38    pca.explained_variance_ratio_.cumsum(), "--o", linewidth=2,
39    label="Cumulative explained variance ratio"
40)
41
42# show legend
43axes[0].legend(loc='best', frameon=True)
44
45# show biplots
46for i in range(1, 4):
47    # Components to be plottet
48    x, y = "PC" + str(i), "PC" + str(i + 1)
49
50    # plot biplots
51    settings = {'kind': 'scatter', 'ax': axes[i], 'alpha': 0.2, 'x': x, 'y': y}
52
53    pca_df.iloc[train_idx].plot(label='Train', c='#ff7f0e', **settings)
54    pca_df.iloc[test_idx].plot(label='Test', c='#1f77b4', **settings)
55return pca_df
56train_idx = range(0, len(train_df))
57test_idx = range(len(train_df), len(total_df))
58
59pca_df = test_pca(total_df, train_idx, test_idx)
60pca_df_all = test_pca(total_df_all, train_idx, test_idx)
61print(">> PCA : (only for np.number)", pca_df.shape, pca_df_all.shape)```

1.2 运行t-SNE

``` 1def test_tsne(data, ax=None, title='t-SNE'):
2"""Run t-SNE and return embedding"""
3
4# Run t-SNE
5tsne = TSNE(n_components=2, init='pca')
6Y = tsne.fit_transform(data)
7
8# Create plot
9for name, idx in zip(["Train", "Test"], [train_idx, test_idx]):
10ax.scatter(Y[idx, 0], Y[idx, 1], label=name, alpha=0.2)
11ax.set_title(title)
12ax.xaxis.set_major_formatter(NullFormatter())
13ax.yaxis.set_major_formatter(NullFormatter())
14ax.legend()
15return Y
16
17# Run t-SNE on PCA embedding
18_, axes = plt.subplots(1, 2,  figsize=(20, 8))
19
20tsne_df = test_tsne(
21pca_df, axes[0],
22title='t-SNE: Scaling on non-zeros'
23)
24tsne_df_unique = test_tsne(
25pca_df_all, axes[1],
26title='t-SNE: Scaling on all entries'
27)
28
29plt.axis('tight')
30plt.show()```

1.2.1 t-SNE由行索引或零计数着色

1.2.2 t-SNE的不同参数

``` 1_, axes = plt.subplots(1, 4, figsize=(20, 5))
2for i, perplexity in enumerate([5, 30, 50, 100]):
3
4# Create projection
5Y = TSNE(init='pca', perplexity=perplexity).fit_transform(pca_df)
6
7# Plot t-SNE
8for name, idx in zip(["Train", "Test"], [train_idx, test_idx]):
9axes[i].scatter(Y[idx, 0], Y[idx, 1], label=name, alpha=0.2)
10axes[i].set_title("Perplexity=%d" % perplexity)
11axes[i].xaxis.set_major_formatter(NullFormatter())
12axes[i].yaxis.set_major_formatter(NullFormatter())
13axes[i].legend()
14plt.show()```

2.Test vs.Train

``` 1def test_prediction(data):
2"""Try to classify train/test samples from total dataframe"""
3# Create a target which is 1 for training rows, 0 for test rows
4y = np.zeros(len(data))
5y[train_idx] = 1
6
7# Perform shuffled CV predictions of train/test label
8predictions = cross_val_predict(
9 ExtraTreesClassifier(n_estimators=100, n_jobs=4),
10data, y,
11cv=StratifiedKFold(
12    n_splits=10,
13    shuffle=True,
14    random_state=42
15)
16)
17
18# Show the classification report
19print(classification_report(y, predictions))
20
21# Run classification on total raw data
22test_prediction(total_df_all)```

```1>> Prediction Train or Test
2     precision    recall  f1-score   support
3
40.0       0.86      0.46      0.60      4459
51.0       0.63      0.92      0.75      4459
6
7avg / total       0.75      0.69      0.68      8918```

＃3 每个特征的分布相似性

``` 1def get_diff_columns(train_df, test_df, show_plots=True, show_all=False, threshold=0.1):
2"""Use KS to estimate columns where distributions differ a lot from each other"""
3
4# Find the columns where the distributions are very different
5diff_data = []
6for col in tqdm(train_df.columns):
7statistic, pvalue = ks_2samp(
8    train_df[col].values,
9    test_df[col].values
10)
11if pvalue <= 0.05 and np.abs(statistic) > threshold:
12    diff_data.append({'feature': col, 'p': np.round(pvalue, 5), 'statistic': np.round(np.abs(statistic), 2)})
13
14# Put the differences into a dataframe
15diff_df = pd.DataFrame(diff_data).sort_values(by='statistic', ascending=False)
16
17if show_plots:
18# Let us see the distributions of these columns to confirm they are indeed different
19n_cols = 7
20if show_all:
21    n_rows = int(len(diff_df) / 7)
22else:
23    n_rows = 2
24_, axes = plt.subplots(n_rows, n_cols, figsize=(20, 3*n_rows))
25axes = [x for l in axes for x in l]
26
27# Create plots
28for i, (_, row) in enumerate(diff_df.iterrows()):
29    if i >= len(axes):
30        break
31    extreme = np.max(np.abs(train_df[row.feature].tolist() + test_df[row.feature].tolist()))
32    train_df.loc[:, row.feature].apply(np.log1p).hist(
33        ax=axes[i], alpha=0.5, label='Train', density=True,
34        bins=np.arange(-extreme, extreme, 0.25)
35    )
36    test_df.loc[:, row.feature].apply(np.log1p).hist(
37        ax=axes[i], alpha=0.5, label='Test', density=True,
38        bins=np.arange(-extreme, extreme, 0.25)
39    )
40    axes[i].set_title(f"Statistic = {row.statistic}, p = {row.p}")
41    axes[i].set_xlabel(f'Log({row.feature})')
42    axes[i].legend()
43
44plt.tight_layout()
45plt.show()
46
47return diff_df
48
49# Get the columns which differ a lot between test and train
50diff_df = get_diff_columns(total_df.iloc[train_idx], total_df.iloc[test_idx])```
```1>> Dropping 22 features based on KS tests
2     precision    recall  f1-score   support
3
40.0       0.85      0.45      0.59      4459
51.0       0.63      0.92      0.75      4459
6
7avg / total       0.74      0.68      0.67      8918```

＃4 分解特征

``` 1COMPONENTS = 20
2
3# List of decomposition methods to use
4methods = [
5TruncatedSVD(n_components=COMPONENTS),
6PCA(n_components=COMPONENTS),
7FastICA(n_components=COMPONENTS),
8GaussianRandomProjection(n_components=COMPONENTS, eps=0.1),
9SparseRandomProjection(n_components=COMPONENTS, dense_output=True)
10# Run all the methods
11embeddings = []
12for method in methods:
13name = method.__class__.__name__
14embeddings.append(
15pd.DataFrame(method.fit_transform(total_df), columns=[f"{name}_{i}" for i in range(COMPONENTS)])
16)
17print(f">> Ran {name}")
18
19# Put all components into one dataframe
20components_df = pd.concat(embeddings, axis=1)
21
22# Prepare plot
23, axes = plt.subplots(1, 3, figsize=(20, 5))
2425# Run t-SNE on components
26tsne_df = test_tsne(
27components_df, axes[0],
28title='t-SNE: with decomposition features'
29)
30
31# Color by index
32sc = axes[1].scatter(tsne_df[:, 0], tsne_df[:, 1], alpha=0.2, c=range(len(tsne_df)), cmap=cm)
33cbar = fig.colorbar(sc, ax=axes[1])
34cbar.set_label('Entry index')
35axes[1].set_title("t-SNE colored by index")
36axes[1].xaxis.set_major_formatter(NullFormatter())
37axes[1].yaxis.set_major_formatter(NullFormatter())
38
39# Color by target
40sc = axes[2].scatter(tsne_df[train_idx, 0], tsne_df[train_idx, 1], alpha=0.2, c=np.log1p(train_df.target), cmap=cm)
41cbar = fig.colorbar(sc, ax=axes[2])
42cbar.set_label('Log1p(target)')
43axes[2].set_title("t-SNE colored by target")
44axes[2].xaxis.set_major_formatter(NullFormatter())
45axes[2].yaxis.set_major_formatter(NullFormatter())
46plt.axis('tight')
47plt.show()```

0 条评论

• ### 译文 | 在使用过采样或欠采样处理类别不均衡数据后，如何正确做交叉验证？

最近读的一篇英文博客，讲的很不错，于是便抽空翻译成了中文。 [关于我在这篇文章中使用的术语可以在 Physionet （http://www.physionet...

• ### 使用Python+Tensorflow的CNN技术快速识别验证码

近年来，机器学习变得愈加火热，中国选手柯洁与AlphaGo的人机大战更是引起热议。目前，在图像识别和视觉分析研究中，卷积神经网络（CNN）技术的使用越来越多。T...

• ### 这几个问题答不对，就别说精通TCP

网络协议那么多，为什么面试喜欢问TCP？原因无外乎两个：1、TCP协议直接与进程打交道，写网络程序要用；2、TCP协议设计十分精巧，在一个不可靠的IP网络上实现...

• ### 解决python pandas读取excel中多个不同sheet表格存在的问题

以上这篇解决python pandas读取excel中多个不同sheet表格存在的问题就是小编分享给大家的全部内容了，希望能给大家一个参考。

• ### Python如何优雅地处理NaN

很多数据不可避免的会遗失掉，或者采集的时候采集对象不愿意透露，这就造成了很多NaN（Not a Number）的出现。这些NaN会造成大部分模型运行出错，所以对...

• ### System Generator系列之多速率系统的使用（下）

前两步完成了不同时钟域层次结构的创建以及异步通道的创建，本次将完成多速率系统设计的最后一步，给每个层次指定不同的时钟域，直接在上一篇的基础上进行本次的设计。

• ### 致Android开发者：APP 瘦身经验总结

小编说：随着移动端产品功能的逐渐增加，APP 的体积也不可避免地呈现上升趋势，如果不加以重视，几个版本迭代下来，可能你的 APP 体积会达到用户不能忍受的程度。

• ### IDEA Pycharm WebStorm JetBranis全版本 2020年 最新激活方式

2020-09-30 亲测有效：IntelliJ IDEA 2020.1 最新全家桶系列产品激活破解方法, JetBrains 全家桶激活

• ### 我的电子学习之路

本科专业是测控技术与仪器，研究生专业是微电子学与固体电子学。回顾整个学生生涯，觉得有必要整理一下我的「电子学习之路」，算是对学生时代的总结吧！

• ### 【投资中的那些坑】开栏语

很多人在投资中喜欢财聚人聚，财散人散，总往有钱的地方钻营。就象游牧民族，哪里水草肥美，就到哪里去。一旦情况不好，就会迅速退场。只能同甘，不能共苦。