"""
# Baseline notebook

**Let's try to predict theme of a joke by body text**
"""
!pip install pymorphy2[fast]
from pymorphy2 import MorphAnalyzer
from stop_words import get_stop_words
import string

from tqdm import tqdm_notebook

from sklearn.preprocessing import LabelEncoder
from sklearn.svm import LinearSVC
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import classification_report
from sklearn.model_selection import cross_val_score

import pandas as pd
import numpy as np

import warnings
warnings.filterwarnings("ignore")


morpher = MorphAnalyzer()
sw = set(get_stop_words("ru"))
exclude = set(string.punctuation)
"""
### Common functions
"""
def preprocess_txt(line):
  spls = "".join(i for i in line.strip() if i not in exclude).split()
  spls = [morpher.parse(i.lower())[0].normal_form for i in spls]
  spls = [i for i in spls if i not in sw and i != ""]
  return ' '.join(spls)

def calc_score(model, X, y):
  cv = cross_val_score(
      model,
      X, y,
      cv=5,
      scoring='f1_weighted'
  )
  print(f'F1_weighted cv: {np.mean(cv)}')
  print(f'\nClassification report\n{classification_report(y, model.predict(X))}')
"""
### Load dataset
"""
%%time

df = pd.read_csv('../input/russian-jokes/jokes.csv')
%%time

for index, row in tqdm_notebook(df.iterrows(), total=df.shape[0]):
    df.loc[index, 'text_token'] = preprocess_txt(row['text'])

df.head()
"""
### Encode target feature
"""
le = LabelEncoder()
df['theme'] = le.fit_transform(df['theme'])
df.head()
"""
### Split data
"""
X_train, X_test, y_train, y_test = train_test_split(
  df['text_token'], df['theme'],
  test_size=0.2,
  stratify=df['theme'],
  random_state=100
)
"""
### TFIDF
"""
%%time

tfidf = TfidfVectorizer(
  ngram_range=(1, 2),
  max_features=10000
)

X_train_idf = tfidf.fit_transform(X_train)
X_test_idf = tfidf.transform(X_test)
"""
### Train LinearSVC model
"""
%%time

svc = LinearSVC(
    random_state=100,
    max_iter=1000,
    loss='squared_hinge',
    dual=False,
)
svc.fit(X_train_idf, y_train)
"""
### Scores
"""
%%time

calc_score(svc, X_test_idf, y_test)