Merge branch 'main' into Ling

.gitignore

@@ -1,6 +1,6 @@
 .venv/**
 data/**
 src/__pycache__
-model/**
+model/model.pt
 html5up-helios/**
 **/__pycache__/**

Dockerfile

@@ -8,4 +8,6 @@ RUN pip install --no-cache-dir --upgrade -r requirements.txt
 
 COPY . .
 
-CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "7860"]
+CMD ["uvicorn", "src.api:app", "--host", "0.0.0.0", "--port", "7860"]
+
+#CMD python3 -m uvicorn --app-dir ./src api:app --host 0.0.0.0 --port 3001

README.md

@@ -1,7 +1,7 @@
 ---
 title: SummaryProject
 sdk: docker
-app_file: app.py
+app_file: src/app.py
 pinned: false
 ---
 # Initialisation

api.py

@@ -0,0 +1,51 @@
+import uvicorn
+from fastapi import FastAPI, Form, Request
+from fastapi.staticfiles import StaticFiles
+from fastapi.templating import Jinja2Templates
+
+from inference import inferenceAPI
+
+
+# ------ MODELE --------------------------------------------------------------
+# appel de la fonction inference, adaptee pour une entree txt
+def summarize(text: str):
+    return " ".join(inferenceAPI(text))
+
+
+# ----------------------------------------------------------------------------------
+
+
+# -------- API ---------------------------------------------------------------------
+app = FastAPI()
+
+# static files pour envoi du css au navigateur
+templates = Jinja2Templates(directory="templates")
+app.mount("/", StaticFiles(directory="templates", html=True), name="templates")
+
+
+@app.get("/")
+async def index(request: Request):
+    return templates.TemplateResponse("index.html.jinja", {"request": request})
+
+
+# retourner le texte, les predictions et message d'erreur si formulaire envoye vide
+@app.post("/")
+async def prediction(request: Request, text: str = Form(None)):
+    if not text:
+        error = "Merci de saisir votre texte."
+        return templates.TemplateResponse(
+            "index.html.jinja", {"request": request, "text": error}
+        )
+    else:
+        summary = summarize(text)
+        return templates.TemplateResponse(
+            "index.html.jinja", {"request": request, "text": text, "summary": summary}
+        )
+
+
+# ------------------------------------------------------------------------------------
+
+
+# lancer le serveur et le recharger a chaque modification sauvegardee
+# if __name__ == "__main__":
+#     uvicorn.run("api:app", port=8000, reload=True)

requirements.txt

@@ -6,7 +6,6 @@ anyascii==0.3.1
 anyio==3.6.2
 async-timeout==4.0.2
 attrs==22.2.0
-autopep8==2.0.2
 banal==1.0.6
 blis==0.7.9
 catalogue==2.0.8
@@ -33,6 +32,11 @@ fsspec==2023.3.0
 greenlet==2.0.2
 h11==0.14.0
 huggingface-hub==0.12.1
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+fastapi==0.92.0
+filelock==3.9.0
 idna==3.4
 importlib-metadata==6.0.0
 importlib-resources==5.12.0
@@ -47,7 +51,6 @@ mccabe==0.7.0
 multidict==6.0.4
 multiprocess==0.70.14
 murmurhash==1.0.9
-nltk==3.8.1
 numpy==1.24.2
 nvidia-cublas-cu11==11.10.3.66
 nvidia-cuda-nvrtc-cu11==11.7.99
@@ -77,7 +80,6 @@ scikit-learn==1.2.1
 scipy==1.10.0
 sentencepiece==0.1.97
 six==1.16.0
-sklearn==0.0.post1
 smart-open==6.3.0
 sniffio==1.3.0
 spacy==3.5.0
@@ -98,6 +100,12 @@ transformers==4.26.1
 typer==0.7.0
 typing-extensions==4.4.0
 urllib3==1.26.14
+starlette==0.25.0
+tokenizers==0.13.2
+torch==1.13.1
+tqdm==4.65.0
+typing_extensions==4.5.0
+urllib3==1.26.15
 uvicorn==0.20.0
 wasabi==1.1.1
 xxhash==3.2.0

src/api.py

@@ -2,17 +2,33 @@ import uvicorn
 from fastapi import FastAPI, Form, Request
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
+import re
 
-from inference import inferenceAPI
+from src.inference import inferenceAPI
+from src.inference_t5 import inferenceAPI_t5
 
 
-# ------ MODELE --------------------------------------------------------------
+# ------ INFERENCE MODEL --------------------------------------------------------------
 # appel de la fonction inference, adaptee pour une entree txt
 def summarize(text: str):
-    return " ".join(inferenceAPI(text))
+    if choisir_modele.var == 'lstm' :
+        return " ".join(inferenceAPI(text))
+    elif choisir_modele.var == "fineTunedT5":
+        text = inferenceAPI_t5(text)
+
 # ----------------------------------------------------------------------------------
 
 
+def choisir_modele(choixModele):
+    print("ON A RECUP LE CHOIX MODELE")
+    if choixModele == "lstm" :
+        choisir_modele.var ='lstm'
+    elif choixModele == "fineTunedT5":
+        choisir_modele.var = "fineTunedT5"
+    else :
+        "le modele n'est pas defini"
+
+
 # -------- API ---------------------------------------------------------------------
 app = FastAPI()
 
@@ -20,26 +36,54 @@ app = FastAPI()
 templates = Jinja2Templates(directory="templates")
 app.mount("/templates", StaticFiles(directory="templates"), name="templates")
 
+
 @app.get("/")
 async def index(request: Request):
     return templates.TemplateResponse("index.html.jinja", {"request": request})
 
+@app.get("/model")
+async def index(request: Request):
+    return templates.TemplateResponse("index.html.jinja", {"request": request})
+
+@app.get("/predict")
+async def index(request: Request):
+    return templates.TemplateResponse("index.html.jinja", {"request": request})
+
+
+@app.post("/model")
+async def choix_model(request: Request, choixModel:str = Form(None)):
+    print(choixModel)
+    if not choixModel:
+        erreur_modele = "Merci de saisir un modèle."
+        return templates.TemplateResponse(
+        "index.html.jinja", {"request": request, "text": erreur_modele}
+        )
+    else :
+        choisir_modele(choixModel)
+        print("C'est bon on utilise le modèle demandé")
+        return templates.TemplateResponse(
+        "index.html.jinja", {"request": request}
+        )
+
+
 # retourner le texte, les predictions et message d'erreur si formulaire envoye vide
-@app.post("/")
+@app.post("/predict")
 async def prediction(request: Request, text: str = Form(None)):
-    if not text :
+    if not text:
         error = "Merci de saisir votre texte."
         return templates.TemplateResponse(
-        "index.html.jinja", {"request": request, "text": error}
-    )
-    else :
+            "index.html.jinja", {"request": request, "text": error}
+        )
+    else:
         summary = summarize(text)
         return templates.TemplateResponse(
             "index.html.jinja", {"request": request, "text": text, "summary": summary}
         )
+
+
 # ------------------------------------------------------------------------------------
 
 
 # lancer le serveur et le recharger a chaque modification sauvegardee
-if __name__ == "__main__":
-    uvicorn.run("api:app", port=8000, reload=True)
+# if __name__ == "__main__":
+#     uvicorn.run("api:app", port=8000, reload=True)

src/dataloader.py

@@ -11,17 +11,15 @@
     Création d'un Vectoriserà partir du vocabulaire :
 
 """
+import pickle
 import string
 from collections import Counter
 
 import pandas as pd
 import torch
-from nltk import word_tokenize
 
-# nltk.download('punkt')
 
-
-class Data:
+class Data(torch.utils.data.Dataset):
     """
     A class used to get data from file
     ...
@@ -44,8 +42,27 @@ class Data:
         create a dataset with cleaned data
     """
 
-    def __init__(self, path: str) -> None:
+    def __init__(self, path: str, transform=None) -> None:
         self.path = path
+        self.data = pd.read_json(path_or_buf=self.path, lines=True)
+        self.transform = transform
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        row = self.data.iloc[idx]
+        text = row["text"].translate(str.maketrans("", "", string.punctuation)).split()
+        summary = (
+            row["summary"].translate(str.maketrans("", "", string.punctuation)).split()
+        )
+        summary = ["<start>", *summary, "<end>"]
+        sample = {"text": text, "summary": summary}
+
+        if self.transform:
+            sample = self.transform(sample)
+
+        return sample
 
     def open(self) -> pd.DataFrame:
         """
@@ -85,26 +102,13 @@ class Data:
         # - s'occuper des noms propres (mots commençant par une majuscule qui se suivent)
         for text in texts:
             text = text.translate(str.maketrans("", "", string.punctuation))
-            text = word_tokenize(text)
+            text = text.split()
             tokenized_texts.append(text)
 
         if text_type == "summary":
             return [["<start>", *summary, "<end>"] for summary in tokenized_texts]
         return tokenized_texts
 
-    def pad_sequence(self):
-        """
-        pad summary with empty token
-        """
-        texts = self.clean_data("text")
-        summaries = self.clean_data("summary")
-        padded_summary = []
-        for text, summary in zip(texts, summaries):
-            if len(summary) != len(text):
-                summary += ["<empty>"] * (len(text) - len(summary))
-            padded_summary.append(summary)
-        return texts, padded_summary
-
     def get_words(self) -> list:
         """
         Create a dictionnary of the data vocabulary
@@ -114,15 +118,20 @@ class Data:
         summary_words = [word for text in summaries for word in text]
         return text_words + summary_words
 
-    def make_dataset(self) -> pd.DataFrame:
-        """
-        Create a Pandas Dataframe with cleaned data
-        --------------------
-        param: self: Data
-        return: pd.DataFrame
-        """
-        texts, summaries = self.clean_data("text"), self.clean_data("summary")
-        return pd.DataFrame(list(zip(texts, summaries)), columns=["text", "summary"])
+
+def pad_collate(data):
+    text_batch = [element[0] for element in data]
+    summary_batch = [element[1] for element in data]
+    max_len = max([len(element) for element in summary_batch + text_batch])
+    text_batch = [
+        torch.nn.functional.pad(element, (0, max_len - len(element)), value=-100)
+        for element in text_batch
+    ]
+    summary_batch = [
+        torch.nn.functional.pad(element, (0, max_len - len(element)), value=-100)
+        for element in summary_batch
+    ]
+    return text_batch, summary_batch
 
 
 class Vectoriser:
@@ -146,12 +155,25 @@ class Vectoriser:
         encode an entire row from the dataset
     """
 
-    def __init__(self, vocab) -> None:
+    def __init__(self, vocab=None) -> None:
         self.vocab = vocab
         self.word_count = Counter(word.lower().strip(",.\\-") for word in self.vocab)
         self.idx_to_token = sorted([t for t, c in self.word_count.items() if c > 1])
         self.token_to_idx = {t: i for i, t in enumerate(self.idx_to_token)}
 
+    def load(self, path):
+        with open(path, "rb") as file:
+            self.vocab = pickle.load(file)
+            self.word_count = Counter(
+                word.lower().strip(",.\\-") for word in self.vocab
+            )
+            self.idx_to_token = sorted([t for t, c in self.word_count.items() if c > 1])
+            self.token_to_idx = {t: i for i, t in enumerate(self.idx_to_token)}
+
+    def save(self, path):
+        with open(path, "wb") as file:
+            pickle.dump(self.vocab, file)
+
     def encode(self, tokens) -> torch.tensor:
         """
         Encode une phrase selon les mots qu'elle contient
@@ -165,7 +187,7 @@ class Vectoriser:
         :return: words_idx : tensor
             Un tensor contenant les index des mots de la phrase
         """
-        if type(tokens) == list:
+        if isinstance(tokens, list):
             words_idx = torch.tensor(
                 [
                     self.token_to_idx.get(t.lower(), len(self.token_to_idx))
@@ -175,7 +197,7 @@ class Vectoriser:
             )
 
         # Permet d'encoder mots par mots
-        elif type(tokens) == str:
+        elif isinstance(tokens, str):
             words_idx = torch.tensor(self.token_to_idx.get(tokens.lower()))
 
         return words_idx
@@ -184,9 +206,9 @@ class Vectoriser:
         """
         Decode une phrase selon le procédé inverse que la fonction encode
         """
-        words_idx_tensor = words_idx_tensor.argmax(dim=-1)
+
         idxs = words_idx_tensor.tolist()
-        if type(idxs) == int:
+        if isinstance(idxs, int):
             words = [self.idx_to_token[idxs]]
         else:
             words = []
@@ -195,10 +217,7 @@ class Vectoriser:
                     words.append(self.idx_to_token[idx])
         return words
 
-    def beam_search(self, words_idx_tensor) -> list:
-        pass
-
-    def vectorize(self, row) -> torch.tensor:
+    def __call__(self, row) -> torch.tensor:
         """
         Encode les données d'une ligne du dataframe
         ----------

src/fine_tune_t5.py

@@ -0,0 +1,204 @@
+import torch
+import datasets
+from datasets import Dataset, DatasetDict
+import pandas as pd
+from tqdm import tqdm
+import re
+import os
+import nltk
+import string
+import contractions
+from transformers import pipeline
+import evaluate
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer,AutoConfig
+from transformers import Seq2SeqTrainingArguments ,Seq2SeqTrainer
+from transformers import DataCollatorForSeq2Seq
+
+# cuda out of memory
+os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "max_split_size_mb:200"
+
+nltk.download('stopwords')
+nltk.download('punkt')
+
+
+def clean_data(texts):
+    texts = texts.lower()
+    texts = contractions.fix(texts)
+    texts = texts.translate(str.maketrans("", "", string.punctuation))
+    texts = re.sub(r'\n',' ',texts)
+    return texts
+
+def datasetmaker (path=str):
+    data = pd.read_json(path, lines=True)
+    df = data.drop(['url','archive','title','date','compression','coverage','density','compression_bin','coverage_bin','density_bin'],axis=1)
+    tqdm.pandas()
+    df['text'] = df.text.apply(lambda texts : clean_data(texts))
+    df['summary'] = df.summary.apply(lambda summary : clean_data(summary))
+    # df['text'] = df['text'].map(str)
+    # df['summary'] = df['summary'].map(str)
+    dataset = Dataset.from_dict(df)
+    return dataset
+
+#voir si le model par hasard esr déjà bien
+
+# test_text = dataset['text'][0]
+# pipe = pipeline('summarization',model = model_ckpt)
+# pipe_out = pipe(test_text)
+# print (pipe_out[0]['summary_text'].replace('.<n>','.\n'))
+# print(dataset['summary'][0])
+
+def generate_batch_sized_chunks(list_elements, batch_size):
+    """split the dataset into smaller batches that we can process simultaneously
+    Yield successive batch-sized chunks from list_of_elements."""
+    for i in range(0, len(list_elements), batch_size):
+        yield list_elements[i : i + batch_size]
+
+def calculate_metric(dataset, metric, model, tokenizer,
+                               batch_size, device,
+                               column_text='text',
+                               column_summary='summary'):
+    article_batches = list(str(generate_batch_sized_chunks(dataset[column_text], batch_size)))
+    target_batches = list(str(generate_batch_sized_chunks(dataset[column_summary], batch_size)))
+
+    for article_batch, target_batch in tqdm(
+        zip(article_batches, target_batches), total=len(article_batches)):
+
+        inputs = tokenizer(article_batch, max_length=1024,  truncation=True,
+                        padding="max_length", return_tensors="pt")
+
+        summaries = model.generate(input_ids=inputs["input_ids"].to(device),
+                         attention_mask=inputs["attention_mask"].to(device),
+                         length_penalty=0.8, num_beams=8, max_length=128)
+        ''' parameter for length penalty ensures that the model does not generate sequences that are too long. '''
+
+        # Décode les textes
+        # renplacer les tokens, ajouter des textes décodés avec les rédéfences vers la métrique.
+        decoded_summaries = [tokenizer.decode(s, skip_special_tokens=True,
+                                clean_up_tokenization_spaces=True)
+               for s in summaries]
+
+        decoded_summaries = [d.replace("", " ") for d in decoded_summaries]
+
+
+        metric.add_batch(predictions=decoded_summaries, references=target_batch)
+
+    #compute et return les ROUGE scores.
+    results = metric.compute()
+    rouge_names = ['rouge1','rouge2','rougeL','rougeLsum']
+    rouge_dict = dict((rn, results[rn] ) for rn in rouge_names )
+    return pd.DataFrame(rouge_dict, index = ['T5'])
+
+
+def convert_ex_to_features(example_batch):
+    input_encodings = tokenizer(example_batch['text'],max_length = 1024,truncation = True)
+
+    labels =tokenizer(example_batch['summary'], max_length = 128, truncation = True )
+
+    return {
+        'input_ids' : input_encodings['input_ids'],
+        'attention_mask': input_encodings['attention_mask'],
+        'labels': labels['input_ids']
+    }
+
+if __name__=='__main__':
+
+    train_dataset = datasetmaker('data/train_extract_100.jsonl')
+
+    dev_dataset = datasetmaker('data/dev_extract_100.jsonl')
+
+    test_dataset = datasetmaker('data/test_extract_100.jsonl')
+
+    dataset = datasets.DatasetDict({'train':train_dataset,'dev':dev_dataset ,'test':test_dataset})
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
+    mt5_config = AutoConfig.from_pretrained(
+    "google/mt5-small",
+    max_length=128,
+    length_penalty=0.6,
+    no_repeat_ngram_size=2,
+    num_beams=15,
+    )
+    model = (AutoModelForSeq2SeqLM
+            .from_pretrained("google/mt5-small", config=mt5_config)
+            .to(device))
+
+    dataset_pt= dataset.map(convert_ex_to_features,remove_columns=["summary", "text"],batched = True,batch_size=128)
+
+    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model,return_tensors="pt")
+
+
+    training_args = Seq2SeqTrainingArguments(
+        output_dir = "mt5_sum",
+        log_level = "error",
+        num_train_epochs = 10,
+        learning_rate = 5e-4,
+        #   lr_scheduler_type = "linear",
+        warmup_steps = 0,
+        optim = "adafactor",
+        weight_decay = 0.01,
+        per_device_train_batch_size = 2,
+        per_device_eval_batch_size = 1,
+        gradient_accumulation_steps = 16,
+        evaluation_strategy = "steps",
+        eval_steps = 100,
+        predict_with_generate=True,
+        generation_max_length = 128,
+        save_steps = 500,
+        logging_steps = 10,
+        # push_to_hub = True
+    )
+
+
+    trainer = Seq2SeqTrainer(
+        model = model,
+        args = training_args,
+        data_collator = data_collator,
+        # compute_metrics = calculate_metric,
+        train_dataset=dataset_pt['train'],
+        eval_dataset=dataset_pt['dev'].select(range(10)),
+        tokenizer = tokenizer,
+    )
+
+    trainer.train()
+    rouge_metric = evaluate.load("rouge")
+
+    score = calculate_metric(test_dataset, rouge_metric, trainer.model, tokenizer,
+                                batch_size=2, device=device,
+                                column_text='text',
+                                column_summary='summary')
+    print (score)
+
+
+    #Fine Tuning terminés et à sauvgarder
+
+
+
+    # save fine-tuned model in local
+    os.makedirs("./summarization_t5", exist_ok=True)
+    if hasattr(trainer.model, "module"):
+        trainer.model.module.save_pretrained("./summarization_t5")
+    else:
+        trainer.model.save_pretrained("./summarization_t5")
+    tokenizer.save_pretrained("./summarization_t5")
+    # load local model
+    model = (AutoModelForSeq2SeqLM
+            .from_pretrained("./summarization_t5")
+            .to(device))
+
+
+    # mettre en usage : TEST
+
+
+    # gen_kwargs = {"length_penalty": 0.8, "num_beams":8, "max_length": 128}
+    # sample_text = dataset["test"][0]["text"]
+    # reference = dataset["test"][0]["summary"]
+    # pipe = pipeline("summarization", model='./summarization_t5')
+
+    # print("Text:")
+    # print(sample_text)
+    # print("\nReference Summary:")
+    # print(reference)
+    # print("\nModel Summary:")
+    # print(pipe(sample_text, **gen_kwargs)[0]["summary_text"])

src/inference.py

@@ -1,22 +1,16 @@
 """
  Allows to predict the summary for a given entry text
 """
-import torch
-from nltk import word_tokenize
+import pickle
 
-import dataloader
-from model import Decoder, Encoder, EncoderDecoderModel
+import torch
 
-# On doit loader les données pour avoir le Vectoriser > sauvegarder "words" dans un fichiers et le loader par la suite ??
-### À CHANGER POUR N'AVOIR À LOADER QUE LE VECTORISER
-data1 = dataloader.Data("data/train_extract.jsonl")
-data2 = dataloader.Data("data/dev_extract.jsonl")
-train_dataset = data1.make_dataset()
-dev_dataset = data2.make_dataset()
-words = data1.get_words()
+from src import dataloader
+from src.model import Decoder, Encoder, EncoderDecoderModel
 
+with open("model/vocab.pkl", "rb") as vocab:
+    words = pickle.load(vocab)
 vectoriser = dataloader.Vectoriser(words)
-word_counts = vectoriser.word_count
 
 
 def inferenceAPI(text: str) -> str:
@@ -30,22 +24,20 @@ def inferenceAPI(text: str) -> str:
         str
             The summary for the input text
     """
-    text = word_tokenize(text)
+    text = text.split()
     # On défini les paramètres d'entrée pour le modèle
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    encoder = Encoder(len(vectoriser.idx_to_token) + 1, 256, 512, 0.5, device).to(
-        device
-    )
-    decoder = Decoder(len(vectoriser.idx_to_token) + 1, 256, 512, 0.5, device).to(
-        device
-    )
+    encoder = Encoder(len(vectoriser.idx_to_token) + 1, 256, 512, 0.5, device)
+    encoder.to(device)
+    decoder = Decoder(len(vectoriser.idx_to_token) + 1, 256, 512, 0.5, device)
+    decoder.to(device)
 
     # On instancie le modèle
-    model = EncoderDecoderModel(encoder, decoder, device)
+    model = EncoderDecoderModel(encoder, decoder, vectoriser, device)
 
-    model.load_state_dict(torch.load("model/model.pt", map_location=device))
-    model.eval()
-    model.to(device)
+    # model.load_state_dict(torch.load("model/model.pt", map_location=device))
+    # model.eval()
+    # model.to(device)
 
     # On vectorise le texte
     source = vectoriser.encode(text)
@@ -55,6 +47,7 @@ def inferenceAPI(text: str) -> str:
     with torch.no_grad():
         output = model(source).to(device)
         output.to(device)
+        output = output.argmax(dim=-1)
     return vectoriser.decode(output)
 
 

src/inference_t5.py

@@ -27,6 +27,7 @@ def inferenceAPI(text: str) -> str:
         str
             The summary for the input text
     """
+
     # On défini les paramètres d'entrée pour le modèle
     text = clean_text(text)
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
@@ -35,6 +36,7 @@ def inferenceAPI(text: str) -> str:
     model = (AutoModelForSeq2SeqLM
              .from_pretrained("Linggg/t5_summary")
              .to(device))
+
     text_encoding = tokenizer(
         text,
         max_length=1024,
@@ -60,8 +62,6 @@ def inferenceAPI(text: str) -> str:
     return "".join(preds)
 
 
-if __name__ == "__main__":
-    '''
-    '''
-    text = input('Entrez votre phrase à résumer : ')
-    print('summary:', inferenceAPI(text))
+# if __name__ == "__main__":
+#     text = input('Entrez votre phrase à résumer : ')
+#     print('summary:', inferenceAPI(text))

src/model.py

@@ -6,14 +6,8 @@ import logging
 
 import torch
 
-import dataloader
-
 logging.basicConfig(level=logging.DEBUG)
 
-data1 = dataloader.Data("data/train_extract.jsonl")
-words = data1.get_words()
-vectoriser = dataloader.Vectoriser(words)
-
 
 class Encoder(torch.nn.Module):
     def __init__(
@@ -86,51 +80,59 @@ class Decoder(torch.nn.Module):
 
 
 class EncoderDecoderModel(torch.nn.Module):
-    def __init__(self, encoder, decoder, device):
+    def __init__(self, encoder, decoder, vectoriser, device):
         # Une idiosyncrasie de torch, pour qu'iel puisse faire sa magie
         super().__init__()
         self.encoder = encoder
         self.decoder = decoder
+        self.vectoriser = vectoriser
         self.device = device
 
-    def forward(self, source, num_beams=3):
-        # CHANGER LA TARGET LEN POUR QQCH DE MODULABLE
-        target_len = int(1 * source.shape[0])  # Taille du texte que l'on recherche
-        target_vocab_size = self.decoder.vocab_size  # Taille du mot
-
-        # tensor to store decoder outputs
-        outputs = torch.zeros(target_len, target_vocab_size).to(
-            self.device
-        )  # Instenciation d'une matrice de zeros de taille (taille du texte, taille du mot)
-        outputs.to(
-            self.device
-        )  # Une idiosyncrasie de torch pour mettre le tensor sur le GPU
+    def forward(self, source, num_beams=3, summary_len=0.2):
+        """
+        :param source: tensor
+            the input text
+        :param num_beams: int
+            the number of outputs to iterate on for beam_search
+        :param summary_len: int
+            length ratio of the summary compared to the text
+        """
+        # The ratio must be inferior to 1 to allow text compression
+        assert summary_len < 1, f"number lesser than 1 expected, got {summary_len}"
+
+        target_len = int(
+            summary_len * source.shape[0]
+        )  # Expected summary length (in words)
+        target_vocab_size = self.decoder.vocab_size  # Word Embedding length
+
+        # Output of the right format (expected summmary length x word embedding length)
+        # filled with zeros. On each iteration, we will replace one of the row of this
+        # matrix with the choosen word embedding
+        outputs = torch.zeros(target_len, target_vocab_size)
+
+        # put the tensors on the device (useless if CPU bus very useful in case of GPU)
+        outputs.to(self.device)
+        source.to(self.device)
 
         # last hidden state of the encoder is used as the initial hidden state of the decoder
-        source.to(
-            self.device
-        )  # Une idiosyncrasie de torch pour mettre le tensor sur le GPU
-        hidden, cell = self.encoder(source)  # Encode le texte sous forme de vecteur
-        hidden.to(
-            self.device
-        )  # Une idiosyncrasie de torch pour mettre le tensor sur le GPU
-        cell.to(
-            self.device
-        )  # Une idiosyncrasie de torch pour mettre le tensor sur le GPU
-
-        # first input to the decoder is the <start> token.
-        input = vectoriser.encode("<start>")  # Mot de départ du MOdèle
-        input.to(self.device)  # idiosyncrasie de torch pour mmettre sur GPU
-
-        ### DÉBUT DE L'INSTANCIATION TEST ###
+        hidden, cell = self.encoder(source)  # Encode the input text
+        input = self.vectoriser.encode(
+            "<start>"
+        )  # Encode the first word of the summary
+
+        # put the tensors on the device
+        hidden.to(self.device)
+        cell.to(self.device)
+        input.to(self.device)
+
+        ### BEAM SEARCH ###
         # If you wonder, b stands for better
         values = None
         b_outputs = torch.zeros(target_len, target_vocab_size).to(self.device)
         b_outputs.to(self.device)
 
-        for i in range(
-            1, target_len
-        ):  # On va déterminer autant de mot que la taille du texte souhaité
+        for i in range(1, target_len):
+            # On va déterminer autant de mot que la taille du texte souhaité
             # insert input token embedding, previous hidden and previous cell states
             # receive output tensor (predictions) and new hidden and cell states.
 

src/script.py

@@ -1,90 +0,0 @@
-"""
-    DONE :
-    - Separer la partie vectoriser du Classifeur
-    - Ajouter un LSTM au Classifieur
-    - entrainer le Classifieur
-    TO DO :
-    - Améliorer les résultats du modèle
-"""
-import logging
-import random
-from typing import Sequence
-
-import torch
-
-import dataloader
-from model import Decoder, Encoder, EncoderDecoderModel
-from train import train_network
-
-# logging INFO, WARNING, ERROR, CRITICAL, DEBUG
-logging.basicConfig(level=logging.INFO)
-logging.disable(level=10)
-
-import os
-
-os.environ[
-    "CUBLAS_WORKSPACE_CONFIG"
-] = ":16:8"  # pour que ça marche en deterministe sur mon pc boulot
-# variable environnement dans git bash export CUBLAS_WORKSPACE_CONFIG=:16:8
-# from datasets import load_dataset
-
-### OPEN DATASET###
-# dataset = load_dataset("newsroom", data_dir=DATA_PATH, data_files="data/train.jsonl")
-
-data1 = dataloader.Data("data/train_extract.jsonl")
-data2 = dataloader.Data("data/dev_extract.jsonl")
-train_dataset = data1.make_dataset()
-dev_dataset = data2.make_dataset()
-words = data1.get_words()
-
-vectoriser = dataloader.Vectoriser(words)
-word_counts = vectoriser.word_count
-
-
-def predict(model, tokens: Sequence[str]) -> Sequence[str]:
-    """Predict the POS for a tokenized sequence"""
-    words_idx = vectoriser.encode(tokens).to(device)
-    # Pas de calcul de gradient ici : c'est juste pour les prédictions
-    with torch.no_grad():
-        # equivalent to model(input) when called out of class
-        out = model(words_idx).to(device)
-    out_predictions = out.to(device)
-    return vectoriser.decode(out_predictions)
-
-
-if __name__ == "__main__":
-    ### NEURAL NETWORK ###
-    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    print("Device check. You are using:", device)
-
-    ### RÉSEAU ENTRAÎNÉ ###
-    # Pour s'assurer que les résultats seront les mêmes à chaque run du notebook
-    torch.use_deterministic_algorithms(True)
-    torch.manual_seed(0)
-    random.seed(0)
-
-    # On peut également entraîner encoder séparemment
-    encoder = Encoder(len(vectoriser.idx_to_token) + 1, 256, 512, 0.5, device)
-    decoder = Decoder(len(vectoriser.idx_to_token) + 1, 256, 512, 0.5, device)
-    # S'ils sont entraînés, on peut les sauvegarder
-    torch.save(encoder.state_dict(), "model/encoder.pt")
-    torch.save(encoder.state_dict(), "model/encoder.pt")
-
-    trained_classifier = EncoderDecoderModel(encoder, decoder, device).to(device)
-
-    print(next(trained_classifier.parameters()).device)
-    # print(train_dataset.is_cuda)
-
-    train_network(
-        trained_classifier,
-        [vectoriser.vectorize(row) for index, row in train_dataset.iterrows()],
-        [vectoriser.vectorize(row) for index, row in dev_dataset.iterrows()],
-        5,
-    )
-
-    torch.save(trained_classifier.state_dict(), "model/model.pt")
-
-    print(f'test text : {dev_dataset.iloc[6]["summary"]}')
-    print(
-        f'test prediction : {predict(trained_classifier, dev_dataset.iloc[6]["text"])}'
-    )

src/train.py

@@ -3,21 +3,19 @@ Training the network
 """
 import datetime
 import logging
+import random
 import time
 from typing import Sequence, Tuple
 
 import torch
 
 import dataloader
+from model import Decoder, Encoder, EncoderDecoderModel
 
 # logging INFO, WARNING, ERROR, CRITICAL, DEBUG
 logging.basicConfig(level=logging.INFO)
 logging.disable(level=10)
 
-data1 = dataloader.Data("data/train_extract.jsonl")
-words = data1.get_words()
-vectoriser = dataloader.Vectoriser(words)
-
 
 def train_network(
     model: torch.nn.Module,
@@ -47,7 +45,6 @@ def train_network(
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
     model = model.to(device)
     print("Device check. You are using:", model.device)
-    model.train()
 
     # with torch.no_grad():
 
@@ -81,10 +78,12 @@ def train_network(
 
             out = model(source).to(device)
             logging.debug(f"outputs = {out.shape}")
+
             target = torch.nn.functional.pad(
                 target, (0, len(out) - len(target)), value=-100
             )
-            # logging.debug(f"predition : {vectoriser.decode(output_predictions)}")
+
+            # logging.debug(f"prediction : {vectoriser.decode(output_predictions)}")
             loss = torch.nn.functional.nll_loss(out, target).to(device)
             loss.backward()
             torch.nn.utils.clip_grad_norm_(model.parameters(), clip)
@@ -131,3 +130,73 @@ def train_network(
         print(
             f"{epoch_n}\t{epoch_loss/epoch_length:.5}\t{abs(dev_correct/dev_total):.2%}\t\t{datetime.timedelta(seconds=epoch_compute_time)}"
         )
+
+
+def predict(model, tokens: Sequence[str]) -> Sequence[str]:
+    """Predict the POS for a tokenized sequence"""
+    words_idx = vectoriser.encode(tokens).to(device)
+    # Pas de calcul de gradient ici : c'est juste pour les prédictions
+    with torch.no_grad():
+        # equivalent to model(input) when called out of class
+        out = model(words_idx).to(device)
+    out_predictions = out.to(device)
+    print(out_predictions)
+    out_predictions = out_predictions.argmax(dim=-1)
+    return vectoriser.decode(out_predictions)
+
+
+if __name__ == "__main__":
+    train_dataset = dataloader.Data("data/train_extract.jsonl")
+    words = train_dataset.get_words()
+    vectoriser = dataloader.Vectoriser(words)
+
+    train_dataset = dataloader.Data("data/train_extract.jsonl", transform=vectoriser)
+    dev_dataset = dataloader.Data("data/dev_extract.jsonl", transform=vectoriser)
+
+    train_dataloader = torch.utils.data.DataLoader(
+        train_dataset, batch_size=2, shuffle=True, collate_fn=dataloader.pad_collate
+    )
+
+    dev_dataloader = torch.utils.data.DataLoader(
+        dev_dataset, batch_size=4, shuffle=True, collate_fn=dataloader.pad_collate
+    )
+
+    for i_batch, batch in enumerate(train_dataloader):
+        print(i_batch, batch[0], batch[1])
+
+    ### NEURAL NETWORK ###
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    print("Device check. You are using:", device)
+
+    ### RÉSEAU ENTRAÎNÉ ###
+    # Pour s'assurer que les résultats seront les mêmes à chaque run du notebook
+    torch.use_deterministic_algorithms(True)
+    torch.manual_seed(0)
+    random.seed(0)
+
+    # On peut également entraîner encoder séparemment
+    encoder = Encoder(len(vectoriser.idx_to_token) + 1, 256, 512, 0.5, device)
+    decoder = Decoder(len(vectoriser.idx_to_token) + 1, 256, 512, 0.5, device)
+
+    trained_classifier = EncoderDecoderModel(encoder, decoder, vectoriser, device).to(
+        device
+    )
+
+    print(next(trained_classifier.parameters()).device)
+    # print(train_dataset.is_cuda)
+
+    train_network(
+        trained_classifier,
+        train_dataset,
+        dev_dataset,
+        2,
+    )
+
+    torch.save(trained_classifier.state_dict(), "model/model.pt")
+    vectoriser.save("model/vocab.pkl")
+    trained_classifier.push_to_hub("SummaryProject-LSTM")
+
+    print(f"test summary : {vectoriser.decode(dev_dataset[6][1])}")
+    print(
+        f"test prediction : {predict(trained_classifier, vectoriser.decode(dev_dataset[6][0]))}"
+    )

templates/index.html.jinja

@@ -4,7 +4,7 @@
         <title>Text summarization API</title>
         <meta charset="utf-8" />
 		<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no" />
-		<link rel="stylesheet" href="{{ url_for('templates', path='site_style/css/main.css') }}" />
+		<link rel="stylesheet" href="{{ url_for('templates', path='templates/site_style/css/main.css') }}" />
         <script>
             function customReset()
             {
@@ -13,6 +13,23 @@
                 document.getElementById("summary").value = "";
             }
         </script>
+        <script>
+        function submitBothForms()
+            {
+            document.getElementById("my_form").submit();
+            document.getElementById("choixModel").submit();
+            }
+        </script>
+        <script>
+            function getValue() {
+                var e = document.getElementById("choixModel");
+                var value = e.value;
+                var text = e.options[e.selectedIndex].text;
+                return text}
+        </script>
+        <script type="text/javascript">
+            document.getElementById('choixModel').value = "<?php echo $_GET['choixModel'];?>";
+        </script>
     </head>
     <body>
         <div id="header">
@@ -28,18 +45,21 @@
             </nav>
 
             <div class="choixModel">
-                <label for="model-select">Choose a model :</label>
-                <select name="model" id="model-select">
-                    <option value="lstm">LSTM</option>
-                    <option value="autre">Autre</option>
-                </select>
+                <form id="choixModel" method="post" action="/model">
+                    <label for="selectModel">Choose a model :</label>
+                    <select name="choixModel" class="selectModel" id="choixModel">
+                        <option value="lstm">LSTM</option>
+                        <option value="fineTunedT5">Fine-tuned T5</option>
+                    </select>
+                </form>
+                <button form ="choixModel" class='search_bn' type="submit" class="btn btn-primary btn-block btn-large" rows="1" cols="50">Select model</button>
             </div>
 
             <div>
                 <table>
                     <tr>
                         <td>
-                            <form id = "my_form" action="/"  method="post" class="formulaire">
+                            <form id = "my_form" action="/predict"  method="post" class="formulaire">
                                 <textarea id="text" name="text" placeholder="Enter your text here!" rows="15" cols="75">{{text}}</textarea>
                                 <input type="hidden" name="textarea_value" value="{{ text }}">
                             </form>
@@ -51,8 +71,9 @@
                 </table>
             </div>
             <div class="buttons">
-                    <button form ="my_form" class='search_bn' type="submit" class="btn btn-primary btn-block btn-large" rows="1" cols="50">Go !</button>
-                    <button form ="my_form" type="button" value="Reset" onclick="customReset();">Reset</button>
+                <!-- <button id="submit" type="submit" onclick=submitBothForms()>SUBMIT</button> -->
+                <button form ="my_form" class='search_bn' type="submit" class="btn btn-primary btn-block btn-large" rows="1" cols="50">Go !</button>
+                <button form ="my_form" type="button" value="Reset" onclick="customReset();">Reset</button>
             </div>
 
             <div class="copyright">