[数据科学] 基于基因表达监测预测肿瘤

import numpy as npimport pandas as pdimport matplotlib.pyplot as plt%matplotlib inline

labels_df = pd.read_csv('actual.csv', index_col = 'patient')test_df = pd.read_csv('data_set_ALL_AML_independent.csv')data_df = pd.read_csv('data_set_ALL_AML_train.csv')print('train_data: ',  data_df.shape, '\n test_data: ',  test_df.shape, '\n labels: ', labels_df.shape)# labels_df.shapedata_df.head()   #查看前5行（默认前5行）

test_cols_to_drop = [c for c in test_df.columns if 'call' in c]test_df = test_df.drop(test_cols_to_drop, axis=1)test_df = test_df.drop(['Gene Description', 'Gene Accession Number'], axis=1 )
data_cols_to_drop = [c for c in data_df.columns if 'call' in c]data_df = data_df.drop(data_cols_to_drop, axis=1)data_df = data_df.drop(['Gene Description', 'Gene Accession Number'], axis=1 )print('train_data ', data_df.shape, '\n test_data: ',  test_df.shape,  '\n labels: ', labels_df.shape)data_df.head()

labels_df = labels_df.replace({'ALL':0, 'AML':1})train_labels = labels_df[labels_df.index <= 38]test_labels = labels_df[labels_df.index > 38]print(train_labels.shape, test_labels.shape)# labels_df.indextest_df = test_df.Ttrain_df = data_df.T

print('Columns containing null values in train and test data are ', data_df.isnull().values.sum(),  test_df.isnull().values.sum())

full_df = train_df.append(test_df, ignore_index=True)print(full_df.shape)full_df.head()

# Standardizationfrom sklearn import preprocessingX_std = preprocessing.StandardScaler().fit_transform(full_df)# Check how the standardized data look likegene_index = 1print('mean is : ',  np.mean(X_std[:, gene_index] ) )print('std is :', np.std(X_std[:, gene_index]))
fig= plt.figure(figsize=(10,10))plt.hist(X_std[:, gene_index], bins=10)plt.xlim((-4, 4))plt.xlabel('rescaled expression', size=30)plt.ylabel('frequency', size=30)

# PCA
from sklearn.decomposition import PCApca = PCA(n_components=50, random_state=42)X_pca = pca.fit_transform(X_std)print(X_pca.shape)

cum_sum = pca.explained_variance_ratio_.cumsum()cum_sum = cum_sum*100
fig = plt.figure(figsize=(10,10))plt.bar(range(50), cum_sum)plt.xlabel('PCA', size=30)plt.ylabel('Cumulative Explained Varaince', size=30)plt.title("Around 90% of variance is explained by the First 50 columns ", size=30)

labels = labels_df['cancer'].values
colors = np.where(labels==0, 'red', 'blue')

from mpl_toolkits.mplot3d import Axes3Dplt.clf()fig = plt.figure(1, figsize=(15,15 ))ax = Axes3D(fig, elev=-150, azim=110,)ax.scatter(X_pca[:, 0], X_pca[:, 1], X_pca[:, 2], c=colors, cmap=plt.cm.Paired,linewidths=10)ax.set_title("First three PCA directions")ax.set_xlabel("PCA1")ax.w_xaxis.set_ticklabels([])ax.set_ylabel("PCA2")ax.w_yaxis.set_ticklabels([])ax.set_zlabel("PCA3")ax.w_zaxis.set_ticklabels([])plt.show()

X = X_pcay = labelsprint(X.shape, y.shape)

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)print(X_train.shape, y_train.shape)

import seaborn as snsfrom sklearn.ensemble import RandomForestClassifierfrom sklearn.metrics import accuracy_scorerfc = RandomForestClassifier(random_state=42)rfc.fit(X_train, y_train)y_pred = rfc.predict(X_test)print(accuracy_score(y_test, y_pred))

labs = ['PCA'+str(i+1) for i in range(X_train.shape[1])]importance_df = pd.DataFrame({    'feature':labs,    'importance': rfc.feature_importances_})
importance_df_sorted = importance_df.sort_values('importance', ascending=False)importance_df_sorted.head()
fig = plt.figure(figsize=(25,10))sns.barplot(data=importance_df_sorted, x='feature', y='importance')plt.xlabel('PCAs', size=30)plt.ylabel('Feature Importance', size=30)plt.title('RF Feature Importance', size=30)plt.savefig('RF Feature Importance.png', dpi=300)plt.show

from sklearn.metrics import confusion_matrixcm = confusion_matrix(y_test, y_pred)### Normalize cm, np.newaxis makes to devide each row by the sumcm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

print(np.newaxis)cmap=plt.cm.Blues
plt.imshow(cm, interpolation='nearest', cmap=cmap)print(cm)

from sklearn.ensemble import GradientBoostingClassifiergbc = GradientBoostingClassifier(random_state=42)gbc.fit(X_train, y_train)y_gbc_pred = gbc.predict(X_test)print(accuracy_score(y_test, y_gbc_pred))