Merge branch 'Ling' of https://github.com/EveSa/SummaryProject into Ling

api.py

@@ -1,51 +0,0 @@
-import uvicorn
-from fastapi import FastAPI, Form, Request
-from fastapi.staticfiles import StaticFiles
-from fastapi.templating import Jinja2Templates
-
-from inference import inferenceAPI
-
-
-# ------ MODELE --------------------------------------------------------------
-# appel de la fonction inference, adaptee pour une entree txt
-def summarize(text: str):
-    return " ".join(inferenceAPI(text))
-
-
-# ----------------------------------------------------------------------------------
-
-
-# -------- API ---------------------------------------------------------------------
-app = FastAPI()
-
-# static files pour envoi du css au navigateur
-templates = Jinja2Templates(directory="templates")
-app.mount("/", StaticFiles(directory="templates", html=True), name="templates")
-
-
-@app.get("/")
-async def index(request: Request):
-    return templates.TemplateResponse("index.html.jinja", {"request": request})
-
-
-# retourner le texte, les predictions et message d'erreur si formulaire envoye vide
-@app.post("/")
-async def prediction(request: Request, text: str = Form(None)):
-    if not text:
-        error = "Merci de saisir votre texte."
-        return templates.TemplateResponse(
-            "index.html.jinja", {"request": request, "text": error}
-        )
-    else:
-        summary = summarize(text)
-        return templates.TemplateResponse(
-            "index.html.jinja", {"request": request, "text": text, "summary": summary}
-        )
-
-
-# ------------------------------------------------------------------------------------
-
-
-# lancer le serveur et le recharger a chaque modification sauvegardee
-# if __name__ == "__main__":
-#     uvicorn.run("api:app", port=8000, reload=True)

requirements.txt

@@ -1,56 +1,16 @@
-absl-py==1.4.0
-aiohttp==3.8.4
-aiosignal==1.3.1
-alembic==1.9.4
 anyascii==0.3.1
 anyio==3.6.2
-async-timeout==4.0.2
-attrs==22.2.0
-banal==1.0.6
-blis==0.7.9
-catalogue==2.0.8
 certifi==2022.12.7
-charset-normalizer==3.0.1
+charset-normalizer==3.1.0
 click==8.1.3
-confection==0.0.4
-contourpy==1.0.7
 contractions==0.1.73
-cycler==0.11.0
-cymem==2.0.7
-dataloader==2.0
-dataset==1.6.0
-datasets==2.10.1
-dill==0.3.6
-en-core-web-lg==3.5.0
-evaluate==0.4.0
-fastapi==0.91.0
+fastapi==0.94.0
 filelock==3.9.0
-flake8==6.0.0
-fonttools==4.38.0
-frozenlist==1.3.3
-fsspec==2023.3.0
-greenlet==2.0.2
 h11==0.14.0
-huggingface-hub==0.12.1
-certifi==2022.12.7
-charset-normalizer==3.1.0
-click==8.1.3
-fastapi==0.92.0
-filelock==3.9.0
+huggingface-hub==0.13.2
 idna==3.4
-importlib-metadata==6.0.0
-importlib-resources==5.12.0
 Jinja2==3.1.2
-joblib==1.2.0
-kiwisolver==1.4.4
-langcodes==3.3.0
-Mako==1.2.4
 MarkupSafe==2.1.2
-matplotlib==3.7.0
-mccabe==0.7.0
-multidict==6.0.4
-multiprocess==0.70.14
-murmurhash==1.0.9
 numpy==1.24.2
 nvidia-cublas-cu11==11.10.3.66
 nvidia-cuda-nvrtc-cu11==11.7.99
@@ -58,56 +18,22 @@ nvidia-cuda-runtime-cu11==11.7.99
 nvidia-cudnn-cu11==8.5.0.96
 packaging==23.0
 pandas==1.5.3
-pathy==0.10.1
-Pillow==9.4.0
-preshed==3.0.8
-protobuf==3.20.0
 pyahocorasick==2.0.0
-pyarrow==11.0.0
-pycodestyle==2.10.0
-pydantic==1.10.4
-pyflakes==3.0.1
-pyparsing==3.0.9
+pydantic==1.10.6
 python-dateutil==2.8.2
-python-multipart==0.0.5
+python-multipart==0.0.6
 pytz==2022.7.1
 PyYAML==6.0
 regex==2022.10.31
 requests==2.28.2
-responses==0.18.0
-rouge-score==0.1.2
-scikit-learn==1.2.1
-scipy==1.10.0
-sentencepiece==0.1.97
 six==1.16.0
-smart-open==6.3.0
 sniffio==1.3.0
-spacy==3.5.0
-spacy-legacy==3.0.12
-spacy-loggers==1.0.4
-SQLAlchemy==1.4.46
-srsly==2.4.5
-starlette==0.24.0
-summarizer==0.0.7
+starlette==0.26.1
 textsearch==0.0.24
-thinc==8.1.7
-threadpoolctl==3.1.0
-tokenizers==0.13.2
-tomli==2.0.1
-torch==1.13.1
-tqdm==4.64.1
-transformers==4.26.1
-typer==0.7.0
-typing-extensions==4.4.0
-urllib3==1.26.14
-starlette==0.25.0
 tokenizers==0.13.2
 torch==1.13.1
 tqdm==4.65.0
+transformers==4.26.1
 typing_extensions==4.5.0
 urllib3==1.26.15
-uvicorn==0.20.0
-wasabi==1.1.1
-xxhash==3.2.0
-yarl==1.8.2
-zipp==3.14.0
+uvicorn==0.21.0

src/api.py

@@ -1,31 +1,30 @@
-import uvicorn
 from fastapi import FastAPI, Form, Request
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
-import re
 
-from src.inference import inferenceAPI
-from src.inference_t5 import inferenceAPI_t5
+from src.inference_lstm import inference_lstm
+from src.inference_t5 import inference_t5
 
 
 # ------ INFERENCE MODEL --------------------------------------------------------------
 # appel de la fonction inference, adaptee pour une entree txt
 def summarize(text: str):
-    if choisir_modele.var == 'lstm' :
-        return " ".join(inferenceAPI(text))
+    if choisir_modele.var == "lstm":
+        return " ".join(inference_lstm(text))
     elif choisir_modele.var == "fineTunedT5":
-        text = inferenceAPI_t5(text)
+        text = inference_t5(text)
+
 
 # ----------------------------------------------------------------------------------
 
 
 def choisir_modele(choixModele):
     print("ON A RECUP LE CHOIX MODELE")
-    if choixModele == "lstm" :
-        choisir_modele.var ='lstm'
+    if choixModele == "lstm":
+        choisir_modele.var = "lstm"
     elif choixModele == "fineTunedT5":
         choisir_modele.var = "fineTunedT5"
-    else :
+    else:
         "le modele n'est pas defini"
 
 
@@ -41,29 +40,29 @@ app.mount("/templates", StaticFiles(directory="templates"), name="templates")
 async def index(request: Request):
     return templates.TemplateResponse("index.html.jinja", {"request": request})
 
+
 @app.get("/model")
 async def index(request: Request):
     return templates.TemplateResponse("index.html.jinja", {"request": request})
 
+
 @app.get("/predict")
 async def index(request: Request):
     return templates.TemplateResponse("index.html.jinja", {"request": request})
 
 
 @app.post("/model")
-async def choix_model(request: Request, choixModel:str = Form(None)):
+async def choix_model(request: Request, choixModel: str = Form(None)):
     print(choixModel)
     if not choixModel:
         erreur_modele = "Merci de saisir un modèle."
         return templates.TemplateResponse(
-        "index.html.jinja", {"request": request, "text": erreur_modele}
+            "index.html.jinja", {"request": request, "text": erreur_modele}
         )
-    else :
+    else:
         choisir_modele(choixModel)
         print("C'est bon on utilise le modèle demandé")
-        return templates.TemplateResponse(
-        "index.html.jinja", {"request": request}
-        )
+        return templates.TemplateResponse("index.html.jinja", {"request": request})
 
 
 # retourner le texte, les predictions et message d'erreur si formulaire envoye vide

src/dataloader.py

@@ -52,10 +52,15 @@ class Data(torch.utils.data.Dataset):
 
     def __getitem__(self, idx):
         row = self.data.iloc[idx]
-        text = row["text"].translate(str.maketrans("", "", string.punctuation)).split()
+        text = row["text"].translate(
+            str.maketrans(
+                "", "", string.punctuation)).split()
         summary = (
-            row["summary"].translate(str.maketrans("", "", string.punctuation)).split()
-        )
+            row["summary"].translate(
+                str.maketrans(
+                    "",
+                    "",
+                    string.punctuation)).split())
         summary = ["<start>", *summary, "<end>"]
         sample = {"text": text, "summary": summary}
 
@@ -106,7 +111,8 @@ class Data(torch.utils.data.Dataset):
             tokenized_texts.append(text)
 
         if text_type == "summary":
-            return [["<start>", *summary, "<end>"] for summary in tokenized_texts]
+            return [["<start>", *summary, "<end>"]
+                    for summary in tokenized_texts]
         return tokenized_texts
 
     def get_words(self) -> list:
@@ -157,8 +163,10 @@ class Vectoriser:
 
     def __init__(self, vocab=None) -> None:
         self.vocab = vocab
-        self.word_count = Counter(word.lower().strip(",.\\-") for word in self.vocab)
-        self.idx_to_token = sorted([t for t, c in self.word_count.items() if c > 1])
+        self.word_count = Counter(word.lower().strip(",.\\-")
+                                  for word in self.vocab)
+        self.idx_to_token = sorted(
+            [t for t, c in self.word_count.items() if c > 1])
         self.token_to_idx = {t: i for i, t in enumerate(self.idx_to_token)}
 
     def load(self, path):
@@ -167,7 +175,8 @@ class Vectoriser:
             self.word_count = Counter(
                 word.lower().strip(",.\\-") for word in self.vocab
             )
-            self.idx_to_token = sorted([t for t, c in self.word_count.items() if c > 1])
+            self.idx_to_token = sorted(
+                [t for t, c in self.word_count.items() if c > 1])
             self.token_to_idx = {t: i for i, t in enumerate(self.idx_to_token)}
 
     def save(self, path):

src/fine_tune_T5.py

@@ -146,7 +146,11 @@ if __name__ == '__main__':
     train_dataset = datasetmaker('data/train_extract.jsonl')
 
 
+<<<<<<< HEAD
     dev_dataset = datasetmaker("data/dev_extract.jsonl")
+=======
+    test_dataset = datasetmaker("data/test_extract.jsonl")
+>>>>>>> 4e410f4bdcd6de645d9e73bb207d8a9170dfc3e1
 
     test_dataset = datasetmaker('data/test_extract.jsonl')
 

src/{inference.py → inference_lstm.py}

@@ -1,5 +1,6 @@
 """
  Allows to predict the summary for a given entry text
+ using LSTM model
 """
 import pickle
 
@@ -7,13 +8,14 @@ import torch
 
 from src import dataloader
 from src.model import Decoder, Encoder, EncoderDecoderModel
+# from transformers import AutoModel
 
 with open("model/vocab.pkl", "rb") as vocab:
     words = pickle.load(vocab)
 vectoriser = dataloader.Vectoriser(words)
 
 
-def inferenceAPI(text: str) -> str:
+def inference_lstm(text: str) -> str:
     """
     Predict the summary for an input text
     --------
@@ -34,6 +36,7 @@ def inferenceAPI(text: str) -> str:
 
     # On instancie le modèle
     model = EncoderDecoderModel(encoder, decoder, vectoriser, device)
+    # model = AutoModel.from_pretrained("EveSa/SummaryProject-LSTM")
 
     # model.load_state_dict(torch.load("model/model.pt", map_location=device))
     # model.eval()

src/inference_t5.py

@@ -1,20 +1,23 @@
 """
  Allows to predict the summary for a given entry text
 """
-import torch
 import re
 import string
+
+import contractions
+import torch
 from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
 
 
 def clean_text(texts: str) -> str:
     texts = texts.lower()
     texts = texts.translate(str.maketrans("", "", string.punctuation))
-    texts = re.sub(r'\n', ' ', texts)
+    texts = re.sub(r"\n", " ", texts)
     return texts
 
 
-def inferenceAPI_T5(text: str) -> str:
+
+def inference_t5(text: str) -> str:
     """
     Predict the summary for an input text
     --------
@@ -29,32 +32,36 @@ def inferenceAPI_T5(text: str) -> str:
     # On défini les paramètres d'entrée pour le modèle
     text = clean_text(text)
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
     tokenizer = (AutoTokenizer.from_pretrained("Linggg/t5_summary",use_auth_token=True))
     # load local model
     model = (AutoModelForSeq2SeqLM
              .from_pretrained("Linggg/t5_summary",use_auth_token=True)
              .to(device))
 
+
     text_encoding = tokenizer(
         text,
         max_length=1024,
-        padding='max_length',
+        padding="max_length",
         truncation=True,
         return_attention_mask=True,
         add_special_tokens=True,
-        return_tensors='pt'
+        return_tensors="pt",
     )
     generated_ids = model.generate(
-        input_ids=text_encoding['input_ids'],
-        attention_mask=text_encoding['attention_mask'],
+        input_ids=text_encoding["input_ids"],
+        attention_mask=text_encoding["attention_mask"],
         max_length=128,
         num_beams=8,
         length_penalty=0.8,
-        early_stopping=True
+        early_stopping=True,
     )
 
     preds = [
-        tokenizer.decode(gen_id, skip_special_tokens=True, clean_up_tokenization_spaces=True)
+        tokenizer.decode(
+            gen_id, skip_special_tokens=True, clean_up_tokenization_spaces=True
+        )
         for gen_id in generated_ids
     ]
     return "".join(preds)

src/model.py

@@ -25,7 +25,8 @@ class Encoder(torch.nn.Module):
         # on s'en servira pour les mots inconnus
         self.embeddings = torch.nn.Embedding(vocab_size, embeddings_dim)
         self.embeddings.to(device)
-        self.hidden = torch.nn.LSTM(embeddings_dim, hidden_size, dropout=dropout)
+        self.hidden = torch.nn.LSTM(
+            embeddings_dim, hidden_size, dropout=dropout)
         # Comme on va calculer la log-vraisemblance,
         # c'est le log-softmax qui nous intéresse
         self.dropout = torch.nn.Dropout(dropout)
@@ -61,7 +62,8 @@ class Decoder(torch.nn.Module):
         # on s'en servira pour les mots inconnus
         self.vocab_size = vocab_size
         self.embeddings = torch.nn.Embedding(vocab_size, embeddings_dim)
-        self.hidden = torch.nn.LSTM(embeddings_dim, hidden_size, dropout=dropout)
+        self.hidden = torch.nn.LSTM(
+            embeddings_dim, hidden_size, dropout=dropout)
         self.output = torch.nn.Linear(hidden_size, vocab_size)
         # Comme on va calculer la log-vraisemblance,
         # c'est le log-softmax qui nous intéresse
@@ -100,32 +102,36 @@ class EncoderDecoderModel(torch.nn.Module):
         # The ratio must be inferior to 1 to allow text compression
         assert summary_len < 1, f"number lesser than 1 expected, got {summary_len}"
 
-        target_len = int(
-            summary_len * source.shape[0]
-        )  # Expected summary length (in words)
-        target_vocab_size = self.decoder.vocab_size  # Word Embedding length
+        # Expected summary length (in words)
+        target_len = int(summary_len * source.shape[0])
+        # Word Embedding length
+        target_vocab_size = self.decoder.vocab_size
 
-        # Output of the right format (expected summmary length x word embedding length)
-        # filled with zeros. On each iteration, we will replace one of the row of this
-        # matrix with the choosen word embedding
+        # Output of the right format (expected summmary length x word
+        # embedding length) filled with zeros. On each iteration, we
+        # will replace one of the row of this matrix with the choosen
+        # word embedding
         outputs = torch.zeros(target_len, target_vocab_size)
 
-        # put the tensors on the device (useless if CPU bus very useful in case of GPU)
+        # put the tensors on the device (useless if CPU bus very useful in
+        # case of GPU)
         outputs.to(self.device)
         source.to(self.device)
 
-        # last hidden state of the encoder is used as the initial hidden state of the decoder
-        hidden, cell = self.encoder(source)  # Encode the input text
-        input = self.vectoriser.encode(
-            "<start>"
-        )  # Encode the first word of the summary
+        # last hidden state of the encoder is used
+        # as the initial hidden state of the decoder
+
+        # Encode the input text
+        hidden, cell = self.encoder(source)
+        # Encode the first word of the summary
+        input = self.vectoriser.encode("<start>")
 
         # put the tensors on the device
         hidden.to(self.device)
         cell.to(self.device)
         input.to(self.device)
 
-        ### BEAM SEARCH ###
+        #   BEAM SEARCH   #
         # If you wonder, b stands for better
         values = None
         b_outputs = torch.zeros(target_len, target_vocab_size).to(self.device)
@@ -134,14 +140,16 @@ class EncoderDecoderModel(torch.nn.Module):
         for i in range(1, target_len):
             # On va déterminer autant de mot que la taille du texte souhaité
             # insert input token embedding, previous hidden and previous cell states
-            # receive output tensor (predictions) and new hidden and cell states.
+            # receive output tensor (predictions) and new hidden and cell
+            # states.
 
             # replace predictions in a tensor holding predictions for each token
             # logging.debug(f"output : {output}")
 
             ####### DÉBUT DU BEAM SEARCH ##########
             if values is None:
-                # On calcule une première fois les premières probabilité de mot après <start>
+                # On calcule une première fois les premières probabilité de mot
+                # après <start>
                 output, hidden, cell = self.decoder(input, hidden, cell)
                 output.to(self.device)
                 b_hidden = hidden
@@ -152,7 +160,8 @@ class EncoderDecoderModel(torch.nn.Module):
                 values, indices = output.topk(num_beams, sorted=True)
 
             else:
-                # On instancie le dictionnaire qui contiendra les scores pour chaque possibilité
+                # On instancie le dictionnaire qui contiendra les scores pour
+                # chaque possibilité
                 scores = {}
 
                 # Pour chacune des meilleures valeurs, on va calculer l'output
@@ -160,7 +169,8 @@ class EncoderDecoderModel(torch.nn.Module):
                     indice.to(self.device)
 
                     # On calcule l'output
-                    b_output, b_hidden, b_cell = self.decoder(indice, b_hidden, b_cell)
+                    b_output, b_hidden, b_cell = self.decoder(
+                        indice, b_hidden, b_cell)
 
                     # On empêche le modèle de se répéter d'un mot sur l'autre en mettant
                     # de force la probabilité du mot précédent à 0
@@ -179,7 +189,8 @@ class EncoderDecoderModel(torch.nn.Module):
                 # Et du coup on rempli la place de i-1 à la place de i
                 b_outputs[i - 1] = b_output.to(self.device)
 
-                # On instancies nos nouvelles valeurs pour la prochaine itération
+                # On instancies nos nouvelles valeurs pour la prochaine
+                # itération
                 values, indices = b_output.topk(num_beams, sorted=True)
 
             ##################################

src/train.py

@@ -150,16 +150,24 @@ if __name__ == "__main__":
     words = train_dataset.get_words()
     vectoriser = dataloader.Vectoriser(words)
 
-    train_dataset = dataloader.Data("data/train_extract.jsonl", transform=vectoriser)
-    dev_dataset = dataloader.Data("data/dev_extract.jsonl", transform=vectoriser)
+    train_dataset = dataloader.Data(
+        "data/train_extract.jsonl",
+        transform=vectoriser)
+    dev_dataset = dataloader.Data(
+        "data/dev_extract.jsonl",
+        transform=vectoriser)
 
     train_dataloader = torch.utils.data.DataLoader(
-        train_dataset, batch_size=2, shuffle=True, collate_fn=dataloader.pad_collate
-    )
+        train_dataset,
+        batch_size=2,
+        shuffle=True,
+        collate_fn=dataloader.pad_collate)
 
     dev_dataloader = torch.utils.data.DataLoader(
-        dev_dataset, batch_size=4, shuffle=True, collate_fn=dataloader.pad_collate
-    )
+        dev_dataset,
+        batch_size=4,
+        shuffle=True,
+        collate_fn=dataloader.pad_collate)
 
     for i_batch, batch in enumerate(train_dataloader):
         print(i_batch, batch[0], batch[1])
@@ -169,7 +177,8 @@ if __name__ == "__main__":
     print("Device check. You are using:", device)
 
     ### RÉSEAU ENTRAÎNÉ ###
-    # Pour s'assurer que les résultats seront les mêmes à chaque run du notebook
+    # Pour s'assurer que les résultats seront les mêmes à chaque run du
+    # notebook
     torch.use_deterministic_algorithms(True)
     torch.manual_seed(0)
     random.seed(0)
@@ -178,9 +187,8 @@ if __name__ == "__main__":
     encoder = Encoder(len(vectoriser.idx_to_token) + 1, 256, 512, 0.5, device)
     decoder = Decoder(len(vectoriser.idx_to_token) + 1, 256, 512, 0.5, device)
 
-    trained_classifier = EncoderDecoderModel(encoder, decoder, vectoriser, device).to(
-        device
-    )
+    trained_classifier = EncoderDecoderModel(
+        encoder, decoder, vectoriser, device).to(device)
 
     print(next(trained_classifier.parameters()).device)
     # print(train_dataset.is_cuda)
@@ -194,7 +202,6 @@ if __name__ == "__main__":
 
     torch.save(trained_classifier.state_dict(), "model/model.pt")
     vectoriser.save("model/vocab.pkl")
-    trained_classifier.push_to_hub("SummaryProject-LSTM")
 
     print(f"test summary : {vectoriser.decode(dev_dataset[6][1])}")
     print(