# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
from scipy.io import arff
import numpy as np
import math
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV, train_test_split, cross_val_score, cross_validate
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score
from sklearn.naive_bayes import MultinomialNB
from sklearn.tree import DecisionTreeClassifier
import os
import time
from tqdm.notebook import trange, tqdm

import warnings 
warnings.filterwarnings('ignore')

file_pwd = os.getcwd() + "\Data"
res = os.walk(file_pwd)
file_list = [i[2] for i in res][0]
file_list


# +
#输入data,输出log2D列的数据
def preprocess(df):
    head_list = df.columns.values.tolist()
    #标准化
    data_without_YN = df.drop("Defective",axis = 1)
    data_normalize = (data_without_YN-data_without_YN.mean())/(data_without_YN.std())
    data_normalize['Defective'] = df.Defective
    
    row_yes_data = df[df.Defective == b'Y']
    row_yes_data = row_yes_data.drop("Defective",axis = 1).values
    row_no_data =  df[df.Defective == b'N']
    row_no_data = row_no_data.drop("Defective",axis = 1).values
    
    yes_samples = data_normalize[data_normalize.Defective == b"Y"]
    yes_samples = yes_samples.drop("Defective",axis = 1)
    no_samples = data_normalize[data_normalize.Defective == b"N"]
    no_samples = no_samples.drop("Defective",axis = 1)
    k = len(no_samples)//len(yes_samples)
    
    yes_samples_array = yes_samples.values
    no_samples_array = no_samples.values
    array = [[np.sqrt(np.sum(np.square(x-y))) for y in no_samples_array]for x in yes_samples_array]
    array = np.array(array).argsort()[:,:k]
    w = {i:0 for i in range(yes_samples.shape[1])}

    for i in range(array.shape[0]):
        for j in array[i]:
            ds = np.abs(row_yes_data[i,:] - row_no_data[j,:])
            ds = pd.Series(ds).rank(method='min')
            for index in range(len(ds)):
                w[index] += ds[index]
    
    a = sorted(w.items(),key=lambda x:x[1],reverse=True)
    b = [i[0] for i in a ]
    c = np.array(head_list)
    column = list(c[b])
    df2 = df.loc[:,column].copy()
    
#     d = df2.shape[1]
#     log2d = math.ceil(math.log2(d))
#     df2 = df2.iloc[:,:log2d]
    return df2


# -

#返回十次十折交叉验证的平均auc
def SVM(data,label):
    clf = SVC(gamma='auto')
    auc_list = []
    data["label"] = label
    for i in tqdm(range(10)):
        data = data.sample(frac=1)
        scores = cross_val_score(clf,data.iloc[:,:-1],data.label,cv=10,scoring="roc_auc")
        auc_list.append(scores.mean())
    return np.mean(auc_list)


#贝叶斯分类
def NB(data,label):
    clf = MultinomialNB()
    auc_list = []
    data["label"] = label
    for i in tqdm(range(10)):
        data = data.sample(frac=1)
        scores = cross_val_score(clf,data.iloc[:,:-1],data.label,cv=10,scoring="roc_auc")
        auc_list.append(scores.mean())
    return np.mean(auc_list)


#决策树分类
def DT(data,label):
    clf = DecisionTreeClassifier()
    auc_list = []
    data["label"] = label
    for i in tqdm(range(10)):
        data = data.sample(frac=1)
        scores = cross_val_score(clf,data.iloc[:,:-1],data.label,cv=10,scoring="roc_auc")
        auc_list.append(scores.mean())
    return np.mean(auc_list)



real_start = time.clock()
for each in tqdm(file_list):
    res_list = []
    data = arff.loadarff('./data/{}'.format(each))
    df = pd.DataFrame(data[0])
    if df.columns[-1] == "label":
        df.rename(columns={'label':'Defective'},inplace=True) 
    defective = df.Defective.copy()
    defective[defective==b'N'] = 0
    defective[defective==b'Y'] = 1

    #得到排好序的数据
    data = preprocess(df)
    head_list = data.columns
    
    for every_feature in tqdm(head_list):
        start = time.clock()
        X = data.loc[:,head_list[0]:every_feature]
        label = defective.astype(int)
        svm_auc = SVM(X.copy(),label)
        destree_auc = DT(X.copy(),label)
        nb_auc = NB(X.copy(),label)
        print("*"*20)
        print("数据尺寸:{}".format(X.shape))
        print("文件名:{}".format("CM1"))
        print("feature:{}:{}".format(head_list[0],every_feature))
        print("SVM--->{}:".format(svm_auc))
        print("决策树--->{}:".format(destree_auc))
        print("贝叶斯--->{}".format(nb_auc))
        spend = (time.clock()-start)
        print("use time:{}".format(spend))
        print("="*20)
        make_dic = {
            "size":X.shape,
            "feature":every_feature,
            "SVM":svm_auc,
            "tree":destree_auc,
            "nb":nb_auc
        }
        res_list.append(make_dic)
    print(res_list)

    info = {key:[]for key in res_list[0].keys()}
    for one in res_list:
        for key,value in one.items():
            info[key].append(value)
    info = pd.DataFrame(info)
    info.to_csv("{}.csv".format(each))
print("总共耗时:",(time.clock()-real_start))

# +
# real_start = time.clock()
# res_list = []
# for each in file_list:
#     data = arff.loadarff('./data/{}'.format(each))
#     df = pd.DataFrame(data[0])
#     if df.columns[-1] == "label":
#         df.rename(columns={'label':'Defective'},inplace=True) 
#     defective = df.Defective.copy()
#     defective[defective==b'N'] = 0
#     defective[defective==b'Y'] = 1
    
#     start = time.clock()
#     #分为数据和标签
#     data = preprocess(df)
#     label = defective.astype(int)
#     svm_auc = SVM(data,label)
#     destree_auc = DT(data,label)
#     nb_auc = NB(data,label)
#     print("*"*20)
#     print("数据尺寸:{}".format(data.shape))
#     print("文件名:{}".format(each))
#     print("log2D:{}".format(data.shape[1]))
#     print("SVM--->{}:".format(svm_auc))
#     print("决策树--->{}:".format(destree_auc))
#     print("贝叶斯--->{}".format(nb_auc))
#     spend = (time.clock()-start)
#     print("use time:{}".format(spend))
#     print("="*20)
#     make_dic = {
#         "size":data.shape,
#         "name":each,
#         "log2D":data.shape[1],
#         "SVM":svm_auc,
#         "tree":destree_auc,
#         "nb":nb_auc
#     }
#     res_list.append(make_dic)
# print("总共耗时:",(time.clock()-real_start))
# print(res_list)


# +
# data = {key:[]for key in res_list[0].keys()}
# for one in res_list:
#     for key,value in one.items():
#         data[key].append(value)
# data.pop("size")
# data = pd.DataFrame(data,index=range(1,13))
# data.to_csv("log2D.csv")
# -





StarcoderdataJupyterScriptsDedupFiltered