# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="emR6ejxnsMXI"
import pandas as pd
import numpy as np

# + colab={"base_uri": "https://localhost:8080/"} id="KZIE3PGuHVKA" outputId="b8ee491c-ccd8-4463-8317-d05b363501c6"
df = pd.read_csv('labeled_french_sentences.csv');
print(df.columns)

# + colab={"base_uri": "https://localhost:8080/"} id="CpvEG_FvfVnk" outputId="a6e583f3-3a8b-4bbe-b1a6-02fcff923fe2"
df.columns = ['text', 'level']
print(df.columns)
df['text'] = df['text'].str.strip()
df['level'] = df['level'].str.strip()
sentences = df['text'].values
print(sentences[0])

# + colab={"base_uri": "https://localhost:8080/"} id="rL9OcuuGfXGs" outputId="37224b50-fd7b-44d4-970f-f5fb158b5bfb"
import nltk
nltk.download('punkt')

# + id="PVmg6uNxfYvM"
from nltk.tokenize import word_tokenize
import string

tokenized_senctences = []
for s in sentences:
  tokenized_senctences.append(word_tokenize(s, language='french'))

# + colab={"base_uri": "https://localhost:8080/"} id="Fa6wHfzrfaI5" outputId="25ed887d-0225-41a8-98e5-c3658a928e96"
# !pip install wordstats

# + id="xqXIIDbgfbji"
from wordstats import Word
# frequency is the calcualted by taking the log from it. 
# frequency of 'de' = round(math.log(7225478), 2) = 15.79
def difficulty(sentence):
  freq_sum = 0
  
  for word in sentence:
   freq_sum = freq_sum + Word.stats(word, 'fr').frequency
  
  freq_mean = freq_sum/len(sentence)
  return freq_mean


# + id="fpZ0DpYm4evk" colab={"base_uri": "https://localhost:8080/"} outputId="bd54b830-7a18-46f9-b832-dcc8eb34f4dc"
print(difficulty('de'))

# + id="llbrVWsafhgG"
X = []

# calculate avarage word difficulty and length
for s in tokenized_senctences:
  sum = 0
  count = 0
  for t in s:
    d = difficulty(t)
    if (d > 0):
      sum += d
      count += 1
  X.append([sum/count, len(s)])

X = np.array(X)




# + id="DcVWI5VnjM3L"

y = df['level'].values

# + colab={"base_uri": "https://localhost:8080/", "height": 514} id="cL2TMGtHfkJ-" outputId="bbc0d4ff-c364-41b2-d069-1e4f019f0e5e"
import matplotlib.pyplot as plt 

fig, ax = plt.subplots(figsize = (8, 8))
colors = {'A1':'red', 'A2':'green', 'B1':'blue', 'B2':'yellow', 'C1':'black', 'C2': 'cyan'}
ax.scatter(X[:, 0],X[:, 1], c=[colors[k] for k in y])

plt.xlabel("Avarage difficality")
plt.ylabel("word count")


# + [markdown] id="ubYZ5qxF_ntx"
# ## Logistic regression

# + id="Vi5QQYO8_teI"
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)

# + colab={"base_uri": "https://localhost:8080/"} id="QPY-bF7B_ubT" outputId="6b3b04e2-5312-46df-e1f7-daf860a3dc16"
from sklearn.linear_model import LogisticRegression
logisticRegr = LogisticRegression()
logisticRegr.fit(x_train, y_train)

# + colab={"base_uri": "https://localhost:8080/"} id="mAu_DSv0AXtu" outputId="e872f0b2-b949-4b06-e173-055edb5d23e8"
# Use score method to get accuracy of model
score = logisticRegr.score(x_test, y_test)
print(score)

# + id="3qMFJe4ZAk9M"
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import metrics

# + colab={"base_uri": "https://localhost:8080/"} id="gUxCNzffAshw" outputId="c6f4add2-7579-46ed-c80b-e7992fbfd173"
y_pred = logisticRegr.predict(x_test)
cm = metrics.confusion_matrix(y_test, y_pred)
print(cm)

# + colab={"base_uri": "https://localhost:8080/"} id="dTkpB1-GAuZ5" outputId="b06c7417-7fe8-46e0-f6a2-d1876f399d6d"
print(metrics.classification_report(y_test, y_pred))

# + [markdown] id="_VmkdhgXDUVl"
# **results are bad**

# + [markdown] id="dVxoDoyDOaPr"
# ## Naive Bayes

# + id="Jk49Le4SOarK"
import pdb
from nltk.corpus import stopwords, twitter_samples
import numpy as np
import pandas as pd
import nltk
import string
from nltk.tokenize import TweetTokenizer
from os import getcwd

# + colab={"base_uri": "https://localhost:8080/"} id="4WDvuTtZ2QqQ" outputId="fa3bd293-96a2-4f6f-dfec-16ba0ff066ce"
nltk.download('stopwords')

# + id="G9kr47CJIqxL"
from nltk.tokenize import word_tokenize
import string
from nltk.stem.snowball import SnowballStemmer

stimmer = SnowballStemmer('french', ignore_stopwords=True)

def process_senctence(senctence):
    '''
    Input:
        senctence: a string containing a french sentence
    Output:
        senctence_out: a list of words containing the processed sentence

    '''
    sentence_tokens = word_tokenize(senctence, language='french')

    senctence_out = []
    for word in sentence_tokens:
        if (word not in string.punctuation):  # remove punctuation
            # senctence_out.append(stimmer.stem(word))
            senctence_out.append(word)

    return senctence_out


# + colab={"base_uri": "https://localhost:8080/"} id="1ChSyJeVX_Rb" outputId="b3294862-746a-4272-e848-3434b96a95b3"
print(process_senctence(df['text'][0]))


# + id="95fv428BYBrc"
def count_words(result, sentences, ys):
    '''
    Input:
        result: a dictionary that will be used to map each pair to its frequency
        sentences: a list of senctences
        ys: a list corresponding to the level of each senctence (A1, A2, B1, B2,
          C1, C2)
    Output:
        result: a dictionary mapping each pair to its frequency
    '''

    for y, senctence in zip(ys, sentences):
      if type(senctence) == str:
        for word in process_senctence(senctence):
            pair = (word, y)
            if pair in result:
                result[pair] += 1
            else:
                result[pair] = 1

    return result


# + id="QU8qFh-2fHji"
def train_naive_bayes(freqs, train_x, train_y):
    '''
    Input:
        freqs: dictionary from (word, label) to how often the word appears
        train_x: a list of sentences
        train_y: a list of labels correponding to the sentences (0,1)
    Output:
        logprior: the log prior. (equation 3 above)
        loglikelihood: the log likelihood of you Naive bayes equation. (equation 6 above)
    '''
    loglikelihood = {}
    logprior = 0

    # calculate V, the number of unique words in the vocabulary
    vocab = set([pair[0] for pair in freqs.keys()])
    V = len(vocab)

    # calculate N_pos and N_neg
    N_pos = N_neg = 0
    for pair in freqs.keys():
        # if the label is positive (greater than zero)
        if pair[1] == 1:

            # Increment the number of positive words by the count for this (word, label) pair
            N_pos += freqs[pair]

        # else, the label is negative
        else:

            # increment the number of negative words by the count for this (word,label) pair
            N_neg += freqs[pair]

    # Calculate D, the number of documents
    D = len(train_y)

    # Calculate D_pos, the number of positive documents (*hint: use sum(<np_array>))
    D_pos = np.sum((train_y).astype(int))

    # Calculate D_neg, the number of negative documents (*hint: compute using D and D_pos)
    D_neg = D - D_pos

    # Calculate logprior
    logprior = np.log(D_pos) - np.log(D_neg)

    # For each word in the vocabulary...
    for word in vocab:
        # get the positive and negative frequency of the word
        freq_pos = freqs[(word, 1)] if (word, 1) in freqs else 0
        freq_neg = freqs[(word, 0)] if (word, 0) in freqs else 0

        # calculate the probability that each word is positive, and negative
        p_w_pos = (freq_pos + 1) / (N_pos + V)
        p_w_neg = (freq_neg + 1) / (N_neg + V)

        # calculate the log likelihood of the word
        loglikelihood[word] = np.log(p_w_pos) - np.log(p_w_neg)

    ### END CODE HERE ###

    return logprior, loglikelihood



# + id="eva0wyhshfA1"
def naive_bayes_predict(sentence, logprior, loglikelihood):
    '''
    Input:
        sentence: a string
        logprior: a number
        loglikelihood: a dictionary of words mapping to numbers
    Output:
        p: the sum of all the logliklihoods of each word in the sentence (if found in the dictionary) + logprior (a number)

    '''
    # process the sentence to get a list of words
    word_l = process_senctence(sentence)

    # initialize probability to zero
    p = 0

    # add the logprior
    p += logprior

    for word in word_l:

        # check if the word exists in the loglikelihood dictionary
        if word in loglikelihood:
            # add the log likelihood of that word to the probability
            p += loglikelihood[word]

    return p


# + id="xpITUpfDkgU5"
def test_naive_bayes(test_x, test_y, logprior, loglikelihood):
    """
    Input:
        test_x: List of sentences
        test_y: Labels corresponding to the list of sentences
        logprior: Log prior
        loglikelihood: Dictionary mapping words to their loglikelihoods
    Output:
        accuracy: (# of sentences classified correctly) / (total # of sentences)
    """

    # START CODE HERE

    # List of predicted labels:
    y_hats = []

    for sentence in test_x:
        # Prediction is > 0:
        if naive_bayes_predict(sentence, logprior, loglikelihood) > 0:
            # Predicted label is 1:
            y_hat_i = 1
        else:
            # Predicted label is 0:
            y_hat_i = 0
        # Append the predicted label to the list y_hats:
        y_hats.append(y_hat_i)

    # error is the average of the absolute values of the differences between
    # test_y and y_hats:
    error = test_y - np.array(y_hats)
    error = np.mean(np.absolute(error))

    # accuracy is 1 minus the error:
    accuracy = 1 - error

    # END CODE HERE

    return accuracy


# + colab={"base_uri": "https://localhost:8080/"} id="bWB3l7S3aIbQ" outputId="906bdf7a-0610-4f6d-e194-afd8a3e67d3b"
result = {}
temp_sentences = df['text'][1:3]
ys = df['level'][1:3]
print(count_words(result, temp_sentences, ys))

# + id="Y13BThUsbeEb"
all_sentences = df['text'].str.strip().to_numpy()
all_levels = df['level'].str.strip().to_numpy()

# + id="Mw8zcGzTd2cU"
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(all_sentences, all_levels, test_size=0.2, random_state=0)


# + id="hPd8Kogw3MHv"
def train(level, x_train, y_train, x_test, y_test):
  a1_y_train = (y_train == level).astype(int)
  a1_y_test = (y_test == level).astype(int)

  freqs = count_words({}, x_train, a1_y_train)

  logprior, loglikelihood = train_naive_bayes(freqs, x_train, a1_y_train)
  accuracy = test_naive_bayes(x_test, a1_y_test, logprior, loglikelihood)
  return logprior, loglikelihood, accuracy


# + colab={"base_uri": "https://localhost:8080/"} id="LNPiTyu3e7g6" outputId="a0260a41-9c8d-4283-8c3e-4b6842c2c06e"
level = 'B2'
logprior, loglikelihood, accuracy = train('A1', x_train, y_train, x_test, y_test)
print("A1 - Naive Bayes accuracy = %0.4f" % accuracy)

logprior, loglikelihood, accuracy = train('A2', x_train, y_train, x_test, y_test)
print("A2 - Naive Bayes accuracy = %0.4f" % accuracy)

logprior, loglikelihood, accuracy = train('B1', x_train, y_train, x_test, y_test)
print("B1 - Naive Bayes accuracy = %0.4f" % accuracy)

logprior, loglikelihood, accuracy = train('B1', x_train, y_train, x_test, y_test)
print("B2 - Naive Bayes accuracy = %0.4f" % accuracy)

logprior, loglikelihood, accuracy = train('C1', x_train, y_train, x_test, y_test)
print("C1 - Naive Bayes accuracy = %0.4f" % accuracy)

logprior, loglikelihood, accuracy = train('C2', x_train, y_train, x_test, y_test)
print("C2 - Naive Bayes accuracy = %0.4f" % accuracy)

# + [markdown] id="13G9lNfB6Yeb"
# # Classification with more features
#

StarcoderdataJupyterScriptsDedupFiltered