# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px
from matplotlib import dates
import matplotlib.style as style

import matplotlib
import statsmodels.tsa.api as tsa

%matplotlib inline

matplotlib.style.use("Solarize_Light2")

from pylab import rcParams
import statsmodels.api as sm

import warnings
warnings.simplefilter(action='ignore', category=Warning)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
"""
<center style="font-family:verdana;"><h1 style="font-size:200%; padding: 10px; background: #DC143C;"><b style="color:#006400;">Darija - Moroccan Arabic</b></h1></center>

"Moroccan Arabic, known as Darija in Morocco, is a form of vernacular Arabic spoken in Morocco. It is part of the Maghrebi Arabic dialect continuum and as such is mutually intelligible to some extent with Algerian Arabic and to a lesser extent with Tunisian Arabic."

"It is spoken as a first language by about 50% to 75% of Morocco's population. Most other Moroccans natively speak one of the Tamazight languages. Educated Moroccan Tamazight speakers can communicate in mainstream Moroccan Arabic."

https://en.wikipedia.org/wiki/Moroccan_Arabic
"""
"""
![](https://moroccanarabiclanguage.com/wp-content/uploads/2020/07/learn-moroccan-arabic-transcription-with-prononciation-online-courses.jpg)moroccanarabiclanguage.com
"""
df1 = pd.read_csv("/kaggle/input/moroccan-darija/darija-data/proverbs.csv", delimiter=',', encoding='utf8')
df1.tail(10)
df = pd.read_csv("/kaggle/input/moroccan-darija/darija-data/imagenet_b_darija.csv", delimiter=',', encoding='utf8')
df.head()
df.isnull().sum()
import nltk 
from nltk import word_tokenize
from nltk.corpus import stopwords
import re
import string
import matplotlib.pyplot as plt
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import normalize
from sklearn.metrics import f1_score
from sklearn.metrics import accuracy_score
from sklearn.naive_bayes import GaussianNB
##Code by Taha07  https://www.kaggle.com/taha07/data-scientists-jobs-analysis-visualization/notebook

from wordcloud import WordCloud
from wordcloud import STOPWORDS
stopwords = set(STOPWORDS)
wordcloud = WordCloud(background_color = 'red',
                      height =2000,
                      width = 2000
                     ).generate(str(df["general_darija"]))
plt.rcParams['figure.figsize'] = (12,12)
plt.axis("off")
plt.imshow(wordcloud)
plt.title("Imagenet General Darija")
plt.show()
##Code by Taha07  https://www.kaggle.com/taha07/data-scientists-jobs-analysis-visualization/notebook

from wordcloud import WordCloud
from wordcloud import STOPWORDS
stopwords = set(STOPWORDS)
wordcloud = WordCloud(background_color = 'black',
                      height =2000,
                      width = 2000
                     ).generate(str(df1["darija"]))
plt.rcParams['figure.figsize'] = (12,12)
plt.axis("off")
plt.imshow(wordcloud)
plt.title("Darija")
plt.show()
#Code by Andrea Sindico  https://www.kaggle.com/asindico/love-for-poems/notebook

def removePunctuation(x):
    x = x.lower()
    x = re.sub(r'[^\x00-\x7f]',r' ',x)
    x = x.replace('\r','')
    x = x.replace('\n','')
    x = x.replace('  ','')
    x = x.replace('\'','')
    return re.sub("["+string.punctuation+"]", " ", x)
#https://stackoverflow.com/questions/51534586/add-and-remove-words-from-the-nltk-stopwords-list

from nltk.corpus import stopwords

stop_words = set(stopwords.words('english'))

#add words that aren't in the NLTK stopwords list
new_stopwords = ['and', 'at', 'so', 'er']
new_stopwords_list = stop_words.union(new_stopwords)

#remove words that are in NLTK stopwords list
not_stopwords = {'snake', 'lion', 'pigs'} 
final_stop_words = set([word for word in new_stopwords_list if word not in not_stopwords])

print(final_stop_words)
#Code by Andrea Sindico  https://www.kaggle.com/asindico/love-for-poems/notebook

def processText(x):
    x= removePunctuation(x)
    #x= removeStopwords(x)
    return x
from nltk.tokenize import sent_tokenize, word_tokenize
tin = pd.Series([word_tokenize(processText(x)) for x in df1['eng']])
tin.head(10)
sentences = [ "hurry and you will be late","a man bitten by a snake is afraid of rope", "he knocks and says who is there", "a friend in need is a friend indeed", "better luck next time"]

vocab = [s.encode('utf-8').split() for s in sentences]

voc_vec = word2vec.Word2Vec(vocab, min_count=1)
#Code by Andrea Sindico  https://www.kaggle.com/asindico/love-for-poems/notebook

from gensim.models import word2vec
num_features = 300    # Word vector dimensionality                      
min_word_count = 4   # Minimum word count #Changed 40 to 4 only avoid build a vocabulary 1st                       
num_workers = 4       # Number of threads to run in parallel
context = 10          # Context window size                                                                                    
downsampling = 1e-3   # Downsample setting for frequent words
model = word2vec.Word2Vec(tin, workers=num_workers, 
                          min_count = min_word_count,
                          window = context, sample = downsampling)
from gensim import utils
import logging
from timeit import default_timer
import threading
from six.moves import range
from six import itervalues, string_types
from gensim import matutils
from numpy import float32 as REAL, ones, random, dtype
from types import GeneratorType
from gensim.utils import deprecated
import os
import copy
#Code by Andrea Sindico  https://www.kaggle.com/asindico/love-for-poems/notebook

def most_similar(self, positive=None, negative=None, topn=10, restrict_vocab=None, indexer=None):
        """Deprecated, use self.wv.most_similar() instead.
        Refer to the documentation for :meth:`~gensim.models.keyedvectors.WordEmbeddingsKeyedVectors.most_similar`.
        """
        return self.wv.most_similar(positive, negative, topn, restrict_vocab, indexer)
#Code by Andrea Sindico  https://www.kaggle.com/asindico/love-for-poems/notebook

#KeyError: "Key 'further' not present"  Any word didn't work her

#model.wv.most_similar('further')
df1["darija"].value_counts()
labels = 'zreb t3TTel', 'kaydo99 wigoul chkoun', 'lli khaf nja', 'khirha f ghirha'
sizes = [1247, 182, 326, 1170]  #must have same number labels, sizes and explode
explode = (0, 0.2, 0, 0)  # only "explode" the 2nd slice 

fig1, ax1 = plt.subplots(figsize=(10,10))
ax1.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%',
        shadow=True, startangle=90)
ax1.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.

plt.show()
"""
#Since all proverbs has only one darija, that pie above doesn't make any sense.
"""
len(df1['darija'].unique())
#Code by Andrea Sindico  https://www.kaggle.com/asindico/love-for-poems/notebook

from nltk.tokenize import sent_tokenize, word_tokenize
tin = pd.Series([word_tokenize(processText(x)) for x in df1['darija']])
tin.head(10)
sentences = [ "zreb t3TTel","lli 3DDo l7nch kaykhaf mn l7bel", "kaydo99 wigoul chkoun", "sdd tl9a mat7ell", "khirha f ghirha"]

vocab = [s.encode('utf-8').split() for s in sentences]

voc_vec = word2vec.Word2Vec(vocab, min_count=1)
#Code by Andrea Sindico  https://www.kaggle.com/asindico/love-for-poems/notebook

from gensim.models import word2vec
num_features = 300    # Word vector dimensionality                      
min_word_count = 1   # Minimum word count #Changed 40 to 1 only                        
num_workers = 4       # Number of threads to run in parallel
context = 10          # Context window size                                                                                    
downsampling = 1e-3   # Downsample setting for frequent words
model = word2vec.Word2Vec(tin, workers=num_workers, #size=num_features - I removed unexpected argument
                          min_count = min_word_count,
                          window = context, sample = downsampling)
#Since most similar is now in KeyedVectors I copied the snippet below

#https://github.com/RaRe-Technologies/gensim/blob/develop/gensim/models/keyedvectors.py

from gensim import models
from gensim.models import KeyedVectors


import gensim.downloader as api
word_vectors = api.load("glove-wiki-gigaword-100")  # load pre-trained word-vectors from gensim-data 
# Check the "most similar words", using the default "cosine similarity" measure.
result = word_vectors.most_similar(positive=['zreb', 'walaw'], negative=['tart'])

most_similar_key, similarity = result[0]  # look at the first match
print(f"{most_similar_key}: {similarity:.4f}")
"""
#Seems that they don't speak Darija.
"""
#https://github.com/RaRe-Technologies/gensim/blob/develop/gensim/models/keyedvectors.py

result = word_vectors.most_similar(positive=['friend', 'luck'], negative=['late'])

most_similar_key, similarity = result[0]  # look at the first match
print(f"{most_similar_key}: {similarity:.4f}")
#https://github.com/RaRe-Technologies/gensim/blob/develop/gensim/models/keyedvectors.py

result = word_vectors.most_similar(positive=['time', 'rope'], negative=['bitten'])

most_similar_key, similarity = result[0]  # look at the first match
print(f"{most_similar_key}: {similarity:.4f}")
"""
That's all for now. Algorithms learn Darija.
"""