# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import pandas as pd

with open('C:/Users/win8/PycharmProjects/textmining/venv/1111stop.txt', encoding='utf8') as file:
    stopWord_list = [k.strip() for k in file.readlines()]

train_df = pd.read_csv('C:/Users/win8/PycharmProjects/textmining/venv/train.txt',sep='\t', header=None,)
train_df = train_df.astype(str)
train_df.shape

# +
import jieba
import time

train_df.columns = ['分类', '文章']
stopword_list = [k.strip() for k in open('C:/Users/win8/Desktop/1111stop.txt', encoding='utf8').readlines() if k.strip() != '']
cutWords_list = []
i = 0
startTime = time.time()
for article in train_df['文章']:
    cutWords = [k for k in jieba.cut(article) if k not in stopword_list]
    i += 1
    if i % 1000 == 0:
        print('前%d篇文章分词共花费%.2f秒' %(i, time.time()-startTime))
    cutWords_list.append(cutWords)
# -

with open('C:/Users/win8/PycharmProjects/textmining/venv/cutWords_list.txt', 'w',encoding='utf-8') as file: 
    for cutWords in cutWords_list:
        file.write(' '.join(cutWords) + '\n')
with open('C:/Users/win8/PycharmProjects/textmining/venv/cutWords_list.txt',encoding='utf-8') as file:
    cutWords_list = [k.split() for k in file.readlines()]

# +
from sklearn.feature_extraction.text import TfidfVectorizer

tfidf = TfidfVectorizer(cutWords_list, min_df=20, max_df=0.3)
# -

X = tfidf.fit_transform(train_df['文章'])
print(X.shape)

# +
from sklearn.preprocessing import LabelEncoder
import pandas as pd

train_df = pd.read_csv('C:/Users/win8/PycharmProjects/textmining/venv/train.txt', sep='\t', header=None)
train_df = train_df.astype(str)
labelEncoder = LabelEncoder()
y = labelEncoder.fit_transform(train_df[0].values)
y.shape
# -

train_df[0].value_counts()
import time  
from sklearn import metrics  
import pickle as pickle  
import pandas as pd

# +
#逻辑回归模型 LR
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split

train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.2)
# -

logistic_model = LogisticRegression(multi_class='multinomial', solver='lbfgs')
logistic_model.fit(train_X, train_y)
logistic_model.score(test_X, test_y)

# +
#保存模型的代码块
import pickle

with open('C:/Users/win8/PycharmProjects/textmining/venv/lr_40_0.3.model', 'rb') as file:
    tfidf_model = pickle.load(file)
    tfidfVectorizer = tfidf_model['tfidfVectorizer']
    labelEncoder = tfidf_model['labelEncoder']
    logistic_model = tfidf_model['logistic_model']

# +
import pandas as pd

train_df = pd.read_csv('C:/Users/win8/PycharmProjects/textmining/venv/train.txt', sep='\t', header=None)
train_df = train_df.astype(str)
X = tfidfVectorizer.transform(train_df[1])
y = labelEncoder.transform(train_df[0])

# +
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import ShuffleSplit
from sklearn.model_selection import cross_val_score

logistic_model = LogisticRegression(multi_class='multinomial', solver='lbfgs')
cv_split = ShuffleSplit(n_splits=5, test_size=0.3)
score_ndarray = cross_val_score(logistic_model, X, y, cv=cv_split)
print(score_ndarray)
print(score_ndarray.mean())

# +
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix
import pandas as pd

train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.2)
logistic_model = LogisticRegression(multi_class='multinomial', solver='lbfgs')
logistic_model.fit(train_X, train_y)
predict_y = logistic_model.predict(test_X)
pd.DataFrame(confusion_matrix(test_y, predict_y), 
             columns=labelEncoder.classes_, 
             index=labelEncoder.classes_)
# -

from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
print(classification_report(test_y, predict_y))

# +
import pandas as pd

test_df = pd.read_csv('C:/Users/win8/PycharmProjects/textmining/venv/train.txt', sep='\t', header=None)
test_X = tfidfVectorizer.transform(test_df[1])
test_y = labelEncoder.transform(test_df[0])
predict_y = logistic_model.predict(test_X)
print(classification_report(test_y, predict_y))
# -



StarcoderdataJupyterScriptsDedupFiltered