# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + colab={"autoexec": {"startup": false, "wait_interval": 0}} colab_type="code" id="uDMNGj81eiaR"
# !conda install spacy
# -

# !python -m spacy download fr_core_news_md

# + [markdown] colab_type="text" id="22g96wDxrb1j"
# # Extractions de différentes informations
#
# Voici les tag de *part of speech* :
#

# + colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 34} colab_type="code" executionInfo={"elapsed": 24739, "status": "ok", "timestamp": 1529313482727, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-WekCrd6tYzo/AAAAAAAAAAI/AAAAAAAAAAg/I6TLjrt6Pq8/s50-c-k-no/photo.jpg", "userId": "108344344344240562730"}, "user_tz": -120} id="iSHCfS_lf6PN" outputId="02c4dab1-b291-44fd-dfe3-5721a74bcd1b"
import spacy
nlp = spacy.load("fr_core_news_md")

doc = nlp(u"Bonjour le monde de Linux Mag")
print([(word.text, word.pos_) for word in doc])

# + [markdown] colab_type="text" id="VS_F7CpqrwY7"
# Et maintenant, regardons les *Lemma*

# + colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 34} colab_type="code" executionInfo={"elapsed": 528, "status": "ok", "timestamp": 1529313495263, "user": {"displayName": "Ugo Cupcic", "photoUrl": "//lh4.googleusercontent.com/-WekCrd6tYzo/AAAAAAAAAAI/AAAAAAAAAAg/I6TLjrt6Pq8/s50-c-k-no/photo.jpg", "userId": "108344344344240562730"}, "user_tz": -120} id="wsbIjQuOhP1_" outputId="ee0c932d-cad1-494d-9ee9-1cc54dfe1e06"
doc = nlp(u"Nous adorons ce magazine qui est très intéressant.")
print(" ".join([word.lemma_ + " " for  word in doc]))

# + [markdown] colab_type="text" id="VglUDO2yr5OE"
# Nous pouvons aussi identifier des groupes de mots qui représente un seul concept: les *noun chunks*.

# + colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 34} colab_type="code" executionInfo={"elapsed": 555, "status": "ok", "timestamp": 1528986564231, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-WekCrd6tYzo/AAAAAAAAAAI/AAAAAAAAAAg/I6TLjrt6Pq8/s50-c-k-no/photo.jpg", "userId": "108344344344240562730"}, "user_tz": -120} id="wqtn7tBXiqEL" outputId="30f0c089-7600-4b47-f406-f560244200d6"
doc = nlp(u"Nous allons parler d'intelligence artificielle.")

for noun_chunk in doc.noun_chunks:
    if " " in noun_chunk.text:
        print noun_chunk.text

# + [markdown] colab_type="text" id="psuBBbAGsJ44"
# # Travail sur la similarité
#
# ## Distance entre des mots
#
# Etudions la similarité de ces quelques mots:

# + colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 68} colab_type="code" executionInfo={"elapsed": 541, "status": "ok", "timestamp": 1528986568055, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-WekCrd6tYzo/AAAAAAAAAAI/AAAAAAAAAAg/I6TLjrt6Pq8/s50-c-k-no/photo.jpg", "userId": "108344344344240562730"}, "user_tz": -120} id="s7FPy384i3Oe" outputId="4d45c6ba-9f5d-43d2-d12e-a3b9a4ef7030"
fraise = nlp(u"fraise")
framboise = nlp(u"framboise")
journal = nlp(u"journal")

print "similarité fraise-framboise: " + str(fraise.similarity(framboise))
print "similarité fraise-journal: " + str(fraise.similarity(journal))
print "similarité framboise-journal: " + str(framboise.similarity(journal))

# + [markdown] colab_type="text" id="URGoMLCljZjV"
# Utilisation un peu plus poussée des vecteurs word2vec à la base de notre similarité:

# + colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 102} colab_type="code" executionInfo={"elapsed": 540, "status": "ok", "timestamp": 1529313196333, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-WekCrd6tYzo/AAAAAAAAAAI/AAAAAAAAAAg/I6TLjrt6Pq8/s50-c-k-no/photo.jpg", "userId": "108344344344240562730"}, "user_tz": -120} id="wrywQtHRlrzu" outputId="7deac1fb-7e8b-4e57-a6ad-8807b89c6d46"
print norm(nlp(u'Paris').vector - nlp(u'France').vector)
print norm(nlp(u'Rome').vector - nlp(u'Italie').vector)
print norm(nlp(u'Madrid').vector - nlp(u'Espagne').vector)
print norm(nlp(u'Moscou').vector - nlp(u'Russie').vector)

print cosine(nlp(u'Paris').vector - nlp(u'France').vector, nlp(u'capitale').vector)

# + [markdown] colab_type="text" id="055gCkQqsXnb"
# ## Distance entre des phrases
#
# La mesure de similarité s'applique également à des phrases

# + colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 51} colab_type="code" executionInfo={"elapsed": 575, "status": "ok", "timestamp": 1528986570752, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-WekCrd6tYzo/AAAAAAAAAAI/AAAAAAAAAAg/I6TLjrt6Pq8/s50-c-k-no/photo.jpg", "userId": "108344344344240562730"}, "user_tz": -120} id="sQvx6ja-jOPh" outputId="925558b2-0865-43e9-da79-a40912d31975"

phrase_1 = nlp(u"Ce journal est très intéressant.")
phrase_2 = nlp(u"Ce magazine est vraiment passionnant.")
phrase_3 = nlp(u"Je bois un thé en écrivant mon article.")

sim_1_2 = phrase_1.similarity(phrase_2)
sim_1_3 = phrase_1.similarity(phrase_3)
print "similarité entre les deux premières phrases: " + str(sim_1_2)
print "similarité entre la première et la dernière phrases: " + str(sim_1_3)


# + [markdown] colab_type="text" id="Zjo3fYoosidz"
# Mais elle présente des limitations:

# + colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 51} colab_type="code" executionInfo={"elapsed": 569, "status": "ok", "timestamp": 1528986573303, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-WekCrd6tYzo/AAAAAAAAAAI/AAAAAAAAAAg/I6TLjrt6Pq8/s50-c-k-no/photo.jpg", "userId": "108344344344240562730"}, "user_tz": -120} id="2Fqdj6OeqN79" outputId="35465f60-7832-4368-f723-22dba40578c8"

phrase_1 = nlp(u"Ce journal est très intéressant.")
phrase_2 = nlp(u"Je trouve cette revue passionante.")
phrase_3 = nlp(u"Ce thé est vraiment délicieux.")

sim_1_2 = phrase_1.similarity(phrase_2)
sim_1_3 = phrase_1.similarity(phrase_3)
print "similarité entre les deux premières phrases: " + str(sim_1_2)
print "similarité entre la première et la dernière phrases: " + str(sim_1_3)


# + [markdown] colab_type="text" id="ijuaOh2XsuMO"
# Simplifions nos phrases et recalculons des similarités:

# + colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 153} colab_type="code" executionInfo={"elapsed": 533, "status": "ok", "timestamp": 1528986576241, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-WekCrd6tYzo/AAAAAAAAAAI/AAAAAAAAAAg/I6TLjrt6Pq8/s50-c-k-no/photo.jpg", "userId": "108344344344240562730"}, "user_tz": -120} id="8BSJPRrowZ0z" outputId="c01dbf21-2d3e-4acd-fdeb-18ac272f2e95"
def simplifier(sentence):
    sentence_as_NLP = nlp(sentence)
    simplified_sentence = ""
    for word in sentence_as_NLP:
        if word.pos_ in ["PROPN", "NOUN", "VERB", "ADJ"]:
            simplified_sentence += word.lemma_ + " "
            
    print ("Phrase complete: \"" + sentence + "\" => simplication: \"" + simplified_sentence + "\"")
    return simplified_sentence

def similarity(sentence1, sentence2):
    return nlp(sentence1).similarity(nlp(sentence2))

sentence1 = u"Ce thé est vraiment délicieux"
sentence2 = u"Je trouve cette revue passionante"
sentence3 = u"Ce thé est un délice."

print "----"
print "Similarité:" + str(similarity(simplifier(sentence1), simplifier(sentence2)))

print "----"
print "Similarité:" + str(similarity(simplifier(sentence1), simplifier(sentence3)))

# + [markdown] colab_type="text" id="N8k3kQlrs870"
# Nous avons toujours des limitations:

# + colab={"autoexec": {"startup": false, "wait_interval": 0}, "base_uri": "https://localhost:8080/", "height": 68} colab_type="code" executionInfo={"elapsed": 577, "status": "ok", "timestamp": 1528986608087, "user": {"displayName": "<NAME>", "photoUrl": "//lh4.googleusercontent.com/-WekCrd6tYzo/AAAAAAAAAAI/AAAAAAAAAAg/I6TLjrt6Pq8/s50-c-k-no/photo.jpg", "userId": "108344344344240562730"}, "user_tz": -120} id="ucvhZXI-zCBU" outputId="c4ab1d5a-6cee-4432-a1b8-3bb3b774f237"

print "Similarité:" + str(similarity(simplifier(u"J'aime le thé."), simplifier(u"Je n'aime pas le thé")))


StarcoderdataJupyterScriptsDedupFiltered