# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

from sklearn.linear_model            import LogisticRegression
from sklearn.model_selection         import GridSearchCV, train_test_split, cross_val_score
from sklearn.metrics                 import accuracy_score, recall_score, roc_auc_score, f1_score
from sklearn.feature_extraction.text import TfidfVectorizer
from konlpy.tag                      import Okt; t=Okt()
import warnings
warnings.filterwarnings(action='ignore')

train = pd.read_csv('./new_train.csv', index_col=0)
dev = pd.read_csv('../../data/dev.hate.csv')
test = pd.read_csv('../../data/test.hate.no_label.csv')
train.head()

train.head()

train = pd.concat([train, dev], axis=0)
train.head()

train.info()

# 트레인 데이터 레이블대로 분류
none_df = train[train['label'] == 'none']['comments']
offensive_df = train[train['label'] == 'offensive']['comments']
hate_df = train[train['label'] == 'hate']['comments']
print(len(none_df), len(offensive_df), len(hate_df), "\n",
      "sum: {}".format(sum([len(none_df), len(offensive_df), len(hate_df)])))

# +
# TfIdf 벡터라이즈
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf_vectorizer = TfidfVectorizer()

# 훈련 데이터 전체 벡터라이즈 fit
tfidf_vectorizer.fit(train['comments'])

# 각각 레이블 벡터라이즈 transform
none_matrix = tfidf_vectorizer.transform(none_df)
offensive_matrix = tfidf_vectorizer.transform(offensive_df)
hate_matrix = tfidf_vectorizer.transform(hate_df)

none_matrix.shape, offensive_matrix.shape, hate_matrix.shape

# +
# 각각의 평균 벡터값(위치) 산출
none_vec = []
offensive_vec = []
hate_vec = []

for i in range(none_matrix.shape[1]):
    none_vec.append(none_matrix[:,i].mean())

for i in range(offensive_matrix.shape[1]):
    offensive_vec.append(offensive_matrix[:,i].mean())
    
for i in range(hate_matrix.shape[1]):
    hate_vec.append(hate_matrix[:,i].mean())

# 벡터라이즈 잘 되었는지 길이 확인
len(none_vec), len(offensive_vec), len(hate_vec)

# +
# 어레이 형태로 변환
none_vec = np.array(none_vec)
offensive_vec = np.array(offensive_vec)
hate_vec = np.array(hate_vec)
# 2차원 어레이로 변환
none_vec = none_vec.reshape(1, -1)
offensive_vec = offensive_vec.reshape(1, -1)
hate_vec = hate_vec.reshape(1, -1)

# 테스트 코멘트 벡터라이즈 적용
test_matrix = tfidf_vectorizer.transform(test['comments'])
test_matrix.shape

# +
# 코멘트 <-> 각 레이블 간의 유사도 측정 후 가장 유사한 레이블 산출
from sklearn.metrics.pairwise import cosine_similarity

preds = []

for i in range(test_matrix.shape[0]):
    distances = {cosine_similarity(test_matrix[i,:], none_vec)[0][0] : "none",
                 cosine_similarity(test_matrix[i,:], offensive_vec)[0][0] : "offensive",
                 cosine_similarity(test_matrix[i,:], hate_vec)[0][0] : "hate"}
    preds.append( distances[max(distances.keys())] )
    
preds
# -





test['label'] = preds
test.head()


def kaggle_format(df):
    df['label'][df['label'] == 'none'] = 0
    df['label'][df['label'] == 'offensive'] = 1
    df['label'][df['label'] == 'hate'] = 2
    return df


kaggle_format(test)

test.head()

test.info()

test.to_csv('./0116_cos_jc.csv', index=False)

pd.read_csv('./0116_cos_jc.csv')







# dd



StarcoderdataJupyterScriptsDedupFiltered