content: 
  import os
  import sys
  import itertools
  import operator
  import nltk
  import numpy as np
  import matplotlib.pyplot as plt
  
  from nltk.util import ngrams
  
  from collections import Counter
  
  from spell_checker import SpellChecker
  from sklearn.decomposition import TruncatedSVD
  from sklearn.pipeline import make_pipeline
  from sklearn.preprocessing import Normalizer
  
  sys.path.insert(0, '/Users/diakite_w/Documents/Dev/ExperimentationsACA/FrenchLefffLemmatizer')
  from FrenchLefffLemmatizer import FrenchLefffLemmatizer
  
  
  
  def extract_ngrams(documents, n):
      '''
              Return list of n-grams
      '''
      chained_documents = list(itertools.chain.from_iterable(documents))
      return Counter(ngrams(chained_documents, n))
  
  
  def tokenize(text):
  
      fll = FrenchLefffLemmatizer()
      splck = SpellChecker()
  
      contracted_pronouns = ["l'", "m'", "n'", "d'", "c'", "j'", "qu'", "s'"]
      dictionnary = []
      stopwords = [w.rstrip() for w in open('stopwords-fr.txt')]
  
      # Put everything to lower case
      text = text.lower()
  
      # Tokenize text
      tokens = nltk.tokenize.word_tokenize(text)
      print('Nombre de tokens dans le texte :', len(tokens))
  
      #tokens = [splck.correct(t) if t not in dictionnary else t for t in tokens]
  
      # Remove contacted pronous from tokens
      tokens = [t[2:] if t[:2] in contracted_pronouns else t for t in tokens]
  
      tokens = [t for t in tokens if len(t) > 2]
  
      tokens = [t for t in tokens if t not in stopwords]
  
      tokens = [fll.lemmatize(t) for t in tokens]
  
      print('Nombre de tokens apres traitement :', len(tokens), '\n')
      return tokens
  
  
  def tokens_to_vec(tokens):
  	vec = np.zeros(len(word_index_map))
  	for token in tokens:
  		idx = word_index_map[token]
  		vec[idx] = 1
  	return vec
  
  
  def read_txt(textfile):
  	with open(textfile, 'r') as f:
  	    text = f.read()
  	    text = text.replace('\n', ' ')
  	    text = text.replace('- ', '')
  	    text = text.replace('.', '')
  	    text = text.replace('-', '')
  	    text = text.replace("‘l'", 'ï')
  	return text
  
  
  def get_all_doc(directory):
  	'''
  		Read all txt documents and append them in string
  	'''
  	documents = [] 
  	counter = 1
  	for filename in os.listdir(directory):
  		if filename.endswith('.txt'):
  			print('\n[...] Reading document', counter)
  			filename = 'data/' + filename
  			documents.append(read_txt(filename))
  			counter += 1
  	return documents
  
  
  documents = get_all_doc('data/')
  all_tokens = [tokenize(doc) for doc in documents]
  vocabulary = list(set(itertools.chain.from_iterable(all_tokens)))
  print ('\nVocab size:', len(vocabulary))
  
  
  # Computing n-grams
  bigrams = extract_ngrams(all_tokens, 2)
  trigrams = extract_ngrams(all_tokens, 3)
  [print(t) for t in trigrams.most_common(5)]
  print('\n')
  [print(t) for t in bigrams.most_common(10)]
  
  
  '''
  # Key: word - value: index
  word_index_map = {j: i for i, j in enumerate(vocabulary)}
  
  # Key: index - value: word
  index_word_map = sorted(word_index_map.items(), key=operator.itemgetter(1))
  index_word_map = [t[0] for t in index_word_map]
  
  N = len(documents)
  D = len(word_index_map)
  X = np.zeros((D,N))
  i = 0
  for tokens in all_tokens:
  	X[:,i] = tokens_to_vec(tokens)
  	i += 1
  print(X.shape)
  
  svd = TruncatedSVD()
  Z = svd.fit_transform(X)
  print('Z shape', Z.shape)
  
  plt.scatter(Z[:,0], Z[:,1])
  print('D:', D)
  for i in range(D):
  	plt.annotate(s=index_word_map[i], xy=(Z[i,0], Z[i,1]))
  plt.show()
  '''
  
  
  
  
  
  
  
  	
  
  
  
  
  
  
  
  
  