首页手记【机器学习】综合案例——交易数...

【机器学习】综合案例——交易数据异常检测（Python版）

标签：

Python 机器学习算法

【案例描述】

注意：本案例为唐宇迪数据挖掘实战课程案例
通过Logistic回归分类信用卡交易异常数据，属于二分类问题，通过sklearn库函数实现数据预处理以及模型构建。
数据集链接：https://pan.baidu.com/s/1l56R4JvTzWguC4KlcZw3JA 提取码: r2zc

【涉及内容】

本案例所涉及的机器学习内容如下：

数据均衡（下采样与过采样）
数据标准化（将数据置为[-1,1]区间）
混淆矩阵
recall评估以及精度评估等方法
交叉验证
SMOTE算法，生成数据
sigmoid函数阈值对结果的影响

【具体代码】

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
data=pd.read_csv('creditcard.csv')
data.head()

count_classes = pd.value_counts(data['Class'],sort=True).sort_index()#统计Class属性的value类别以及数量，并按照value值进行排序
count_classes

count_classes.plot(kind='bar')
plt.title('Fraud class histogram')
plt.xlabel('Class')
plt.ylabel('Frequency')

from sklearn.preprocessing import StandardScaler#sklearn数据预处理库

data['normAmount']=StandardScaler().fit_transform(data['Amount'].values.reshape(-1,1))
data=data.drop(['Time','Amount'],axis=1)
data.head()
data['normAmount'].values
X=data.iloc[:,data.columns!='Class']#loc和iloc的区别，在于loc通过index获取数据，iloc通过行号获取数据
Y=data.iloc[:,data.columns=='Class']
number_records_fraud=len(data[data.Class==1])
fraud_indexes=np.array(data[data.Class==1].index)
fraud_indexes
normal_indexes=data[data.Class==0].index
normal_indexes

random_normal_indexes=np.random.choice(normal_indexes,number_records_fraud,replace=False)
random_normal_indexes

under_sample_indexes=np.concatenate([fraud_indexes,random_normal_indexes])#将两个数组合并成一个数组
under_sample_data =data.iloc[under_sample_indexes,:]
under_sample_data

X_underSample = under_sample_data.iloc[:,under_sample_data.columns!='Class']
Y_underSample = under_sample_data.iloc[:,under_sample_data.columns=='Class']

print('正样本比例---',len(under_sample_data[under_sample_data.Class==0])/len(under_sample_data))
print('负样本比例---',len(under_sample_data[under_sample_data.Class==1])/len(under_sample_data))

print('总样本个数---',len(under_sample_data))

from sklearn.cross_validation import train_test_split #导入数据切分的工具包

#对整个数据集进行一个分割
X_train,X_test,y_train,y_test=train_test_split(X,Y,test_size=0.3,random_state=0)
X_train

print('总训练样本---',len(X_train))
print('总测试样本---',len(X_test))
print('样本总数---',len(X_train)+len(X_test))
#under_sample dataset进行交叉验证分割
X_train_undersample,X_test_undersample,Y_train_undersample,Y_test_undersample=train_test_split(X_underSample,Y_underSample,test_size=0.3,random_state=0)
print('下采样：总训练样本---',len(X_train_undersample))
print('下采样：总测试样本---',len(X_test_undersample))
print('下采样：样本总数---',len(X_train_undersample)+len(X_test_undersample))

#Recall=TP/(TP/FP)
from sklearn.linear_model import LogisticRegression #逻辑回归
from sklearn.cross_validation import KFold,cross_val_score #交叉验证
from sklearn.metrics import confusion_matrix,recall_score,classification_report#混淆矩阵

fold = KFold(len(Y_train_undersample),5,shuffle=False)
fold
help(KFold)

for train_index, test_index in fold:
    print("TRAIN:", train_index, "TEST:", test_index)
#核心函数
def printing_Kfold_scores(X_train_data,Y_train_data):
    fold =KFold(len(Y_train_data),5,shuffle=False)
    #正则化中不同的惩罚项参数
    c_param_range =[0.01,0.1,1,10,100]
    results_table=pd.DataFrame(index=range(len(c_param_range)),columns=['C_parameter','Mean recall score'])
    results_table['C_parameter'] = c_param_range
    #the k-fold will gives 2 lists: train_indexes=indexes[0],test_indexes=indexes[1]
    j=0
    for c_param in c_param_range:
        print('----------------------------------------------------')
        print('C parameter:',c_param)
        print('----------------------------------------------------')
        print('')
        recall_accs=[]
        for iteration,indexes in enumerate(fold,start=1):
            # Call the logistic regression model with a certain C parameter
            lr=LogisticRegression(C=c_param,penalty='l1')#L1惩罚，L2惩罚
            # Use the training data to fit the model. In this case, we use the portion of the fold to train the model
            # with indices[0]. We then predict on the portion assigned as the 'test cross validation' with indices[1]
            lr.fit(X_train_data.iloc[indexes[0],:],Y_train_data.iloc[indexes[0],:].values.ravel())
            # Predict values using the test indices in the training data
            Y_pred_undersample=lr.predict(X_train_data.iloc[indexes[1],:].values)
            # Calculate the recall score and append it to a list for recall scores representing the current c_parameter
            recall_acc=recall_score(Y_train_data.iloc[indexes[1],:].values,Y_pred_undersample)
            recall_accs.append(recall_acc)
            print('Iteration',iteration,':recall score =',recall_acc)
        results_table.loc[j,'Mean recall score']=np.mean(recall_accs)
        j+=1
        print('')
        print('Mean recall score',np.mean(recall_accs))
        print('')
    best_c=results_table.loc[results_table['Mean recall score'].astype('float64').idxmax()]['C_parameter']
    #Finally,we can check which C parameter is the best amongst the chosen
    print('***************************************************')
    print('Best model to choose from cross validation is with C parameter =',best_c)
    print('***************************************************')
    return best_c
best_c=printing_Kfold_scores(X_train_undersample,Y_train_undersample)
#混淆矩阵
def plot_confusion_matrix(cm, classes,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    """
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=0)
    plt.yticks(tick_marks, classes)

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')


import itertools
lr = LogisticRegression(C = best_c, penalty = 'l1')
lr.fit(X_train_undersample,Y_train_undersample.values.ravel())
Y_pred_undersample = lr.predict(X_test_undersample.values)

# Compute confusion matrix
cnf_matrix = confusion_matrix(Y_test_undersample,Y_pred_undersample)
np.set_printoptions(precision=2)

print("Recall metric in the testing dataset: ", cnf_matrix[1,1]/(cnf_matrix[1,0]+cnf_matrix[1,1]))

# Plot non-normalized confusion matrix
class_names = [0,1]
plt.figure()
plot_confusion_matrix(cnf_matrix
                      , classes=class_names
                      , title='Confusion matrix')
plt.show()


lr = LogisticRegression(C = best_c, penalty = 'l1')
lr.fit(X_train_undersample,Y_train_undersample.values.ravel())
Y_pred = lr.predict(X_test.values)

# Compute confusion matrix
cnf_matrix = confusion_matrix(y_test,Y_pred)
np.set_printoptions(precision=2)

print("Recall metric in the testing dataset: ", cnf_matrix[1,1]/(cnf_matrix[1,0]+cnf_matrix[1,1]))

# Plot non-normalized confusion matrix
class_names = [0,1]
plt.figure()
plot_confusion_matrix(cnf_matrix
                      , classes=class_names
                      , title='Confusion matrix')
plt.show()

best_c = printing_Kfold_scores(X_train,y_train)

lr = LogisticRegression(C = best_c, penalty = 'l1')
lr.fit(X_train,y_train.values.ravel())
y_pred_undersample = lr.predict(X_test.values)

# Compute confusion matrix
cnf_matrix = confusion_matrix(y_test,y_pred_undersample)
np.set_printoptions(precision=2)

print("Recall metric in the testing dataset: ", cnf_matrix[1,1]/(cnf_matrix[1,0]+cnf_matrix[1,1]))

# Plot non-normalized confusion matrix
class_names = [0,1]
plt.figure()
plot_confusion_matrix(cnf_matrix
                      , classes=class_names
                      , title='Confusion matrix')
plt.show()


lr = LogisticRegression(C = 0.01, penalty = 'l1')
lr.fit(X_train_undersample,Y_train_undersample.values.ravel())
y_pred_undersample_proba = lr.predict_proba(X_test_undersample.values)

thresholds = [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]

plt.figure(figsize=(10,10))

j = 1
for i in thresholds:
    y_test_predictions_high_recall = y_pred_undersample_proba[:,1] > i
    
    plt.subplot(3,3,j)
    j += 1
    
    # Compute confusion matrix
    cnf_matrix = confusion_matrix(Y_test_undersample,y_test_predictions_high_recall)
    np.set_printoptions(precision=2)

    print("Recall metric in the testing dataset: ", cnf_matrix[1,1]/(cnf_matrix[1,0]+cnf_matrix[1,1]))

    # Plot non-normalized confusion matrix
    class_names = [0,1]
    plot_confusion_matrix(cnf_matrix
                          , classes=class_names
                          , title='Threshold >= %s'%i) 


#过采样具体过程
from imblearn.over_sampling import SMOTE
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import  confusion_matrix
from sklearn.model_selection import train_test_split

credit_cards=pd.read_csv('creditcard.csv')
columns =credit_cards.columns
#删除最后一列
features_columns =columns.delete(len(columns)-1)

features =credit_cards[features_columns]
labels=credit_cards['Class']

feature_train,feature_test,labels_train,labels_test=train_test_split(features,labels,test_size=0.2,random_state=0)

oversampler=SMOTE(random_state=0)
os_features,os_labels=oversampler.fit_sample(feature_train,labels_train)

len(os_labels[os_labels.values==1])

os_features=pd.DataFrame(os_features)

os_labels=pd.DataFrame(os_labels)

best_c=printing_Kfold_scores(os_features,os_labels)

lr = LogisticRegression(C = best_c, penalty = 'l1')
lr.fit(os_features,os_labels.values.ravel())
y_pred = lr.predict(feature_test.values)

# Compute confusion matrix
cnf_matrix = confusion_matrix(labels_test,y_pred)
np.set_printoptions(precision=2)

print("Recall metric in the testing dataset: ", cnf_matrix[1,1]/(cnf_matrix[1,0]+cnf_matrix[1,1]))

# Plot non-normalized confusion matrix
class_names = [0,1]
plt.figure()
plot_confusion_matrix(cnf_matrix
                      , classes=class_names
                      , title='Confusion matrix')
plt.show()