信用卡欺诈检测｜用启发式搜索优化XGBoost超参数

import numpy as np 
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
import random
random.seed(1234)

plt.style.use('ggplot')
dat = pd.read_csv('creditcard.csv')
dat.head()

df['Class'].value_counts()/len(df)

0    0.998273
1    0.001727
Name: Class, dtype: float64

total = dat.isnull().sum().sort_values(ascending = False)
percent = (dat.isnull().sum()/dat.isnull().count()*100
          ).sort_values(ascending = False)
pd.concat([total, percent], axis=1, 
          keys=['Total', 'Percent']).transpose()

import plotly.graph_objs as go
from plotly.offline import iplot
temp = dat["Class"].value_counts()
df = pd.DataFrame({'Class': temp.index,'values': temp.values})

trace = go.Bar(
    x = df['Class'],y = df['values'],
    name="信用卡欺诈类别-数据不平衡 \n(非欺诈 = 0, 欺诈 = 1)",
    marker=dict(color="Blue"),
    text=df['values'])
temp_data = [trace]
layout = dict(title = '信用卡欺诈类别-数据不平衡 (非欺诈 = 0, 欺诈 = 1)',
          xaxis = dict(title = 'Class', showticklabels=True), 
          yaxis = dict(title = '交易数量'),
          hovermode = 'closest',width=1000, height=600)
fig = dict(data=temp_data, layout=layout)
iplot(fig, filename='class')

amt99 = 1018
# 99%的交易
dat99 = dat.iloc[ list(dat['Amount']<=amt99) ]
# 1%的交易
dat01 = dat.iloc[ list(dat['Amount']>amt99) ]
class_0 = dat99.loc[dat['Class'] == 0]["Time"]
class_1 = dat99.loc[dat['Class'] == 1]["Time"]

max_amount=int(round(max(dat.Amount) *1.04,-3))
bins=list(range(0,1601,100))
bins.append(max_amount)
dat['Amt']=pd.cut(dat.Amount,bins)
all_trans = dat.pivot_table(index="Amt",columns="Class",values="V1",aggfunc=len) 
all_trans['All'] = all_trans[0] +all_trans[1]
all_trans = all_trans.drop( 0, axis=1)
all_trans.columns=['欺诈','总体']
all_trans['欺诈 %'] = all_trans['欺诈']  / len( dat[dat.Class==1])*100
all_trans['总体 %'] = all_trans['总体']  / len( dat.Class)*100

sns.heatmap(dat.corr(), cmap="YlGnBu")

color = sns.color_palette("Set3", 6)
plt.figure(figsize=(20,20))
i=1

for col in dat.columns:
    plt.subplot(8,4,i)
    ax=sns.boxplot(x=dat['Class'],y=dat[col],  palette=color)
    
    for p in ax.patches:
        ax.annotate(format(p.get_height(), '.2f'), 
                (p.get_x() + p.get_width() / 2.,
                 p.get_height()), ha = 'center', va = 'center', 
                fontsize=8,
                xytext = (0, 10), textcoords = '离群点')
    i+=1
plt.tight_layout()

plt.figure(figsize=(20,80))
i=1
for col in dat.columns:
    plt.subplot(16,2,i)
    
    sns.regplot(y=dat[dat.Class==0].Amount,
                x=dat[dat.Class==0][col], color="g")
    ax = sns.regplot(y=dat[dat.Class==1].Amount,
                    x=dat[dat.Class==1][col], color="r")
    ax.tick_params(labelsize=15)
    ax.set_xlabel(col,fontsize=15)

    i+=1
plt.tight_layout()

from scipy import stats
from sklearn import preprocessing

dat['Time'] = preprocessing.scale(dat['Time'])
dat['Amount'] = preprocessing.scale(dat['Amount'])
pt = pd.pivot_table(dat, values=dat.columns, columns = 'Class', aggfunc='mean')

print('\n韦尔奇t检验和形状统计的p值:\n')
for i in range(30):
    col_name = dat.columns[i]
    t, p_val = stats.ttest_ind(dat.loc[ dat['Class']==0, col_name], 
                               dat.loc[ dat['Class']==1, col_name],equal_var=False)  
    skewness = dat.loc[:,col_name].skew()
    kurtosis = stats.kurtosis(dat.loc[:,col_name])
    print('Variable: {:7s}'.format(col_name),end='')    
    print('p-value: {:6.3f}  skewness: {:6.3f}  kurtosis: {:6.3f}'.format(p_val, skewness, kurtosis))

for i in range(30):
    axes[i].hist(dat[dat.columns[i]], bins=50,facecolor='b',alpha=0.5)

韦尔奇t检验和形状统计的p值:
Variable: Time p-value: 0.000 skewness: -0.036  kurtosis: -1.294
Variable: V1   p-value: 0.000 skewness: -3.281  kurtosis: 32.486
Variable: V2   p-value: 0.000 skewness: -4.625  kurtosis: 95.771
Variable: V3   p-value: 0.000 skewness: -2.240  kurtosis: 26.619
...
Variable: Amount p-value:  0.004  skewness: 16.978  kurtosis: 845.078

Class = dat['Class'].values

allIndices = np.arange(len(Class))
np.random.shuffle(allIndices) # 打乱观察的索引

numTrain = int(round(0.40*len(Class)))
numValid = int(round(0.30*len(Class)))
numTest = len(Class)-numTrain-numValid

inTrain = allIndices[:numTrain]
inValid = allIndices[numTrain:(numTrain+numValid)]
inTest =  allIndices[(numTrain+numValid):]

train = dat.iloc[inTrain,:30]
valid= dat.iloc[inValid,:30]
test =  dat.iloc[inTest,:30]

trainY = Class[inTrain]
validY = Class[inValid]
testY = Class[inTest]

import xgboost as xgb
# 将数据转换为xgboost所需的矩阵
dtrain = xgb.DMatrix(train, label=trainY)
dvalid = xgb.DMatrix(valid, label=validY)
dtest = xgb.DMatrix(test, label=testY)

# 固定一些参数
# boosting迭代数
num_rounds=20 
param = {'silent':1,
         'min_child_weight':1,
         'objective':'binary:logistic',
         'eval_metric':'auc',
         'seed' : 1234}  

from collections import OrderedDict

ratio_neg_to_pos = sum(trainY==0)/sum(trainY==1)
print('负与正实例的比率: {:6.1f}'.format(ratio_neg_to_pos))

# 需要调节的参数
tune_dic = OrderedDict()

tune_dic['max_depth']= [5,10,15,20,25] # 树的最大深度
tune_dic['subsample']=[0.5,0.6,0.7,0.8,0.9,1.0] # 在树中使用的训练实例的比例
tune_dic['colsample_bytree']= [0.5,0.6,0.7,0.8,0.9,1.0] # 列(特征)的子样本比率
tune_dic['eta']= [0.01,0.05,0.10,0.20,0.30,0.40]  # 学习率
tune_dic['gamma']= [0.00,0.05,0.10,0.15,0.20]  # 分裂所需的最小损失函数减少值
tune_dic['scale_pos_weight']=[30,40,50,300,400,500,600,700] # 正面/负面实例的相对权重

lengths = [len(lst) for lst in tune_dic.values()]
combs=1
for i in range(len(lengths)):
    combs *= lengths[i]
print('所有组合总数: {:16d}'.format(combs))  
maxiter=100
columns=[*tune_dic.keys()]+['F-Score','Best F-Score']
results = pd.DataFrame(index=range(maxiter), columns=columns) # dataframe保存训练结果

负与正实例的比率:  580.2
所有组合总数:     43200

def perf_measures(preds, labels, print_conf_matrix=False):
    
    act_pos=sum(labels==1) # 实际正样本数
    act_neg=len(labels) - act_pos # 实际负样本数
    
    pred_pos=sum(1 for i in range(len(preds)) if (preds[i]>=0.5)) # 预测真样本数
    true_pos=sum(1 for i in range(len(preds)) if (preds[i]>=0.5) & (labels[i]==1)) ## 预测负样本数
    
    false_pos=pred_pos - true_pos # 假正数
    false_neg=act_pos-true_pos # 假负数
    true_neg=act_neg-false_pos # 正负数
      
    precision = true_pos/pred_pos ## tp/(tp+fp) 预测为正样本数中正确分类的百分比
    recall = true_pos/act_pos ## tp/(tp+fn) 正确分类的正样本百分比
    
    f_score = 2*precision*recall/(precision+recall) 
    
    if print_conf_matrix:
        print('\n混淆矩阵')
        print('----------------')
        print( 'tn:{:6d} fp:{:6d}'.format(true_neg,false_pos))
        print( 'fn:{:6d} tp:{:6d}'.format(false_neg,true_pos))    
    return(f_score)

def do_train(cur_choice, param, train,train_s,trainY,valid,valid_s,validY,print_conf_matrix=False):
  """
  使用给定的固定和可变参数进行训练，并报告验证数据集上的F-score
  """
    print('参数:')
    for (key,value) in cur_choice.items():
        print(key,': ',value,' ',end='')
        param[key]=value
    print('\n')    
    
#    下面注释掉的部分使用一个观察列表来监控推进迭代的进程 
#    evallist  = [(train,train_s), (valid,valid_s)]
#    model = xgb.train( param, train, num_boost_round=num_rounds,
#                      evals=evallist,verbose_eval=False)  
    
    model = xgb.train( param, train, num_boost_round=num_rounds)  
    
    preds = model.predict(valid)
    labels = valid.get_label()
      
    f_score = perf_measures(preds, labels,print_conf_matrix)
    return(f_score, model)

def next_choice(cur_params=None):
  """
  返回变量参数的随机组合(如果cur_params=None)或cur_params的随机相邻组合
  """
    if cur_params:
        # 选择要更改的参数
        # 参数名称和当前值
        choose_param_name, cur_value = random.choice(list(cur_choice.items())) # 参数名称
       
        all_values =  list(tune_dic[choose_param_name]) # 所选参数的所有值
        cur_index = all_values.index(cur_value) # 选定参数的当前索引
        
        if cur_index==0: 
       # 如果它是范围中的第一个，则选择第二个
            next_index=1
        elif cur_index==len(all_values)-1: 
            # 如果它是范围中的最后一个，则选择前一个
            next_index=len(all_values)-2
        else: 
            # 否则随机选择左或右值
            direction=np.random.choice([-1,1])
            next_index=cur_index + direction

        next_params = dict((k,v) for k,v in cur_params.items())
        next_params[choose_param_name] = all_values[next_index] 
        # 更改所选参数的值
        print('参数的选择移动到: {:10s}: from {:6.2f} to {:6.2f}'.
              format(choose_param_name, cur_value, all_values[next_index] ))
    else: 
       # 生成参数的随机组合
        next_params=dict()
        for i in range(len(tune_dic)):
            key = [*tune_dic.keys()][i] 
            values = [*tune_dic.values()][i]
            next_params[key] = np.random.choice(values)
    return(next_params)  

import time
t0 = time.clock()
T=0.40
best_params = dict() # 初始化字典用以保存最佳参数
best_f_score = -1.   # 初始化最佳F-score
prev_f_score = -1.   # 初始化先前的F-score
prev_choice = None   # 初始化先前选择的参数
weights = list(map(lambda x: 10**x, [0,1,2,3,4])) # 哈希函数的权值
hash_values=set()

for iter in range(maxiter):
    print('\nIteration = {:5d}  T = {:12.6f}'.format(iter,T))
    # 找到以前没有访问过的参数的下一个选择
    while True:
        # 第一个选择或选择prev_choice的邻居
        cur_choice=next_choice(prev_choice)          
        # 按参数的字母顺序排列的选择索引  
        indices=[tune_dic[name].index(cur_choice[name]) for name in sorted([*tune_dic.keys()])]      
        # 检查选择是否已经被访问
        hash_val = sum([i*j for (i, j) in zip(weights, indices)])
        if hash_val in hash_values:
            print('\n再次访问组合 -- 再次搜索')
#        tmp=abs(results.loc[:,[*cur_choice.keys()]] - list(cur_choice.values()))
#        tmp=tmp.sum(axis=1)
#        if any(tmp==0): ## selection has already been visited
#            print('\n再次访问组合 -- 再次搜索')
        else:
            hash_values.add(hash_val)
            break # 跳出while循环
     
    # 训练模型并在验证数据集上获得F-score
    f_score,model=do_train(cur_choice, param, dtrain,'train',trainY,dvalid,'valid',validY)  
    # 存储参数
    results.loc[iter,[*cur_choice.keys()]]=list(cur_choice.values())   
    print('F-Score: {:6.2f}  先前值: {:6.2f} 目前最佳值: {:6.2f}'.format(f_score, prev_f_score, best_f_score))
 
    if f_score > prev_f_score:
        print('局部改进')   
        # 接受这个组合作为新的起点
        prev_f_score = f_score
        prev_choice = cur_choice     
        # 如果F-score全局较好，则更新最佳参数
        if f_score > best_f_score:         
            best_f_score = f_score
            print('全局改进 - 最佳F-score已更新')
            for (key,value) in prev_choice.items():
                best_params[key]=value
    else: ## F-score比先前值小        
        ## 接受这个组合作为新的概率起点 exp(-(1.6 x f-score 下降)/温度) 
        rnd = random.random()
        diff = f_score-prev_f_score
        thres=np.exp(1.3*diff/T)
        if rnd <= thres:
            print('更糟糕的结果。 F-Score 变化: {:8.4f}  阈值: {:6.4f}  随机数: {:6.4f} -> 已接受'.
                  format(diff, thres, rnd))
            prev_f_score = f_score
            prev_choice = cur_choice
        else:
            # 不更新以前的F-score和以前选择的参数
            print('更糟糕的结果。 F-Score 变化: {:8.4f}  阈值: {:6.4f}  随机数: {:6.4f} -> 已拒绝'.
                 format(diff, thres, rnd))
    # 存储结果
    results.loc[iter,'F-Score']=f_score
    results.loc[iter,'最佳F-Score']=best_f_score
    if iter % 5 == 0: T=0.85*T  # 每5次迭代降低温度并继续        
print('\n{:6.1f} 分钟处理时间\n'.format((time.clock() - t0)/60))    
print('找到最佳变量参数:\n')
print(best_params)

Iteration = 0  T =  0.400000
参数:
eta :  0.05  colsample_bytree :  0.9  
scale_pos_weight :  300  gamma :  0.2  
max_depth :  20  subsample :  0.5  

F-Score: 0.80  先前值: -1.00 目前最佳值: -1.00
局部改进
全局改进 - 最佳F-score已更新
...

19.2 分钟处理时间
找到最佳变量参数:
{'eta': 0.4, 'colsample_bytree': 1.0, 
'scale_pos_weight': 50, 'gamma': 0.1, 
'max_depth': 15, 'subsample': 0.5}

print('\n找到的最佳参数:\n')  
print(best_params)
print('\n对测试数据集的评估\n')  
best_f_score,best_model=do_train(best_params, param, dtrain,'train',trainY,dtest,'test',testY,print_conf_matrix=True)
print('\n测试数据集上的F-score: {:6.2f}'.format(best_f_score))
f, (ax1,ax2) = plt.subplots(nrows=1, ncols=2, sharey=False, figsize=(8,5))
ax1.plot(results['F-Score'])
ax2.plot(results['Best F-Score'])
ax1.set_xlabel('Iterations',fontsize=11)
ax2.set_xlabel('Iterations',fontsize=11)
ax1.set_ylabel('F-Score',fontsize=11)
ax2.set_ylabel('Best F-Score',fontsize=11)
ax1.set_ylim([0.7,0.9])
ax2.set_ylim([0.7,0.9])
plt.tight_layout()
plt.show()
print('\n特征重要性:\n')  
p = xgb.plot_importance(best_model) 
plt.show()

找到的最佳参数:

{'eta': 0.4, 'colsample_bytree': 1.0, 
 'scale_pos_weight': 50, 'gamma': 0.1, 
 'max_depth': 15, 'subsample': 0.5}

对测试数据集的评估

Parameters:
eta :  0.4  colsample_bytree :  1.0  
scale_pos_weight :  50  gamma :  0.1  
max_depth :  15  subsample :  0.5  

混淆矩阵
----------------
 tn: 85265 fp:    18
 fn:    25 tp:   134