# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
# !pip install sentencepiece

from transformers import CamembertTokenizer, CamembertForCausalLM

tokenizer = CamembertTokenizer.from_pretrained("camembert-base")
#Model Comenbert Causal
model = CamembertForCausalLM.from_pretrained("camembert-base", is_decoder=True, add_cross_attention=True)

# -

#Causal language modeling: Pour predire le prochain token dans une phrase.
#Masked language modeling: Pour predire le token masquée dans le input.
model.eval()

#Tester tokens especial
tokenizer.special_tokens_map

#Encoder la phrase
token_encode = tokenizer.encode('La phrase encodé par tokenizer, est la ...')
token_encode

#Decoder la phrase
tokenizer.decode(token_encode)

# +
# sentiment analysis
from transformers import pipeline


classifier = pipeline("sentiment-analysis")

# -

classifier('Dans la lumière de ton sourire')

import torch
from transformers import *

# Tokeniser, Model
tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')
modele = BertForNextSentencePrediction.from_pretrained('bert-base-multilingual-cased')
modele.eval()

text = "Dans la lumière de ton sourire, Peu importe ta destination, Je meurs si tu es triste."
# Texte tokenisé
tokenized_text = tokenizer.tokenize(text)
# Convertir le texte en indexs
tokens_ind = tokenizer.convert_tokens_to_ids(tokenized_text)
segments_ids = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]

# Transformer en tenseurs
tenseurs_tokens = torch.tensor([tokens_ind])
segments_tensors = torch.tensor([segments_ids])

predictions = model(tokens_tensor, segments_tensors)

import numpy as np
if np.argmax(predictions) == 0:
    print("Suite")
else:
    print("Pas la suite")

tokenizer.vocab_size

# +
#En francais
camembert_fill_mask = pipeline("fill-mask")

results = camembert_fill_mask("je suis <mask>")
# -

#phrase en Anglais
nlp = pipeline('fill-mask')
nlp('I am <mask> ')

summarizer = pipeline("summarization", model="moussaKam/barthez-orangesum-abstract", tokenizer="moussaKam/barthez",)
summarizer("Dans la lumière de ton sourire, Peu importe ta destination, Je meurs si tu es triste.") 



StarcoderdataJupyterScriptsDedupFiltered