train_data = pd.read_csv('./data/evo_train.csv', sep=',', header=0)
test_data = pd.read_csv('./data/evo_test.csv', sep=',', header=0)

print train_data.shape
print test_data.shape

train_data.head()

"""
Explanation: Загрузка данных
End of explanation
"""


train_data.GROUP_ID.value_counts()

"""
Explanation: Выходная переменная
End of explanation
"""


train_data[train_data.GROUP_ID == 6]['NAME'].head(10)

"""
Explanation: Посмотрим самые частые и самые редкие категории.
End of explanation
"""


# удаление спецсимволов
import re

def remove_special_character(somestring):
    pattern = re.compile(u'[^a-zA-Zа-яА-Я0-9_]+')
    tokens = pattern.sub(' ', somestring).strip()
    return tokens

train_data['NAME'] = train_data['NAME'].str.decode('utf-8')
train_data['NAME'] = train_data['NAME'].apply(remove_special_character)

test_data['NAME'] = test_data['NAME'].str.decode('utf-8')
test_data['NAME'] = test_data['NAME'].apply(remove_special_character)

# лемматизация
from pymystem3 import Mystem

m = Mystem()
def lemmatize(text):
    lemmas = m.lemmatize(text)
    return (''.join(lemmas)).strip()

train_data['NAME'] = train_data['NAME'].apply(lemmatize)
test_data['NAME'] = test_data['NAME'].apply(lemmatize)

train_data['NAME'].head(10)

"""
Explanation: 7 = сигареты
6 = продукты питания
32 = позиция по свободной цене

Предобработка данных
!!По-хорошему:
1. К нижнему регистру
2. Вычистить стоп-слова и знаки препинания
3. Лемматизация
4. CountVectorizer, TFIDF, Word2Vec
End of explanation
"""


from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier

max_features = range(100, 1000, 100)

for mf in max_features:
    cv = CountVectorizer(max_features=mf)
    cv_train_data = cv.fit_transform(train_data['NAME'])
    rf = RandomForestClassifier(n_estimators=200, random_state=11)
    scores = cross_val_score(rf, cv_train_data, train_data['GROUP_ID'], cv=2, scoring='accuracy')
    
    print 'Max features: ', mf
    print 'Score: ', scores.mean()

# пробное решение
cv = CountVectorizer(max_features=1200)
cv_train_data = cv.fit_transform(train_data['NAME'])
cv_test_data = cv.transform(test_data['NAME'])

rf = RandomForestClassifier(n_estimators=400, random_state=11)
rf.fit(cv_train_data, train_data['GROUP_ID'])

predict_rf = rf.predict(cv_test_data)
test_data['GROUP_ID'] = predict_rf

test_data.head()

test_data[['id', 'GROUP_ID']].to_csv('res_3105.csv', sep=',', header=True, index=False)

"""
Explanation: Эксперимент с CountVectorizer и RF
End of explanation
"""


from sklearn.naive_bayes import MultinomialNB

mnb = MultinomialNB()
scores = cross_val_score(mnb, cv_train_data, train_data['GROUP_ID'], cv=3, scoring='accuracy')

print scores

"""
Explanation: NaiveBayes
End of explanation
"""


