Merge pull request #8 from EveSa/Estelle

src/api.py

@@ -2,19 +2,33 @@ import uvicorn
 from fastapi import FastAPI, Form, Request
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
+import re
 
 from src.inference import inferenceAPI
+from src.inference_t5 import inferenceAPI_t5
 
 
-# ------ MODELE --------------------------------------------------------------
+# ------ INFERENCE MODEL --------------------------------------------------------------
 # appel de la fonction inference, adaptee pour une entree txt
 def summarize(text: str):
-    return " ".join(inferenceAPI(text))
-
+    if choisir_modele.var == 'lstm' :
+        return " ".join(inferenceAPI(text))
+    elif choisir_modele.var == "fineTunedT5":
+        text = inferenceAPI_t5(text)
 
 # ----------------------------------------------------------------------------------
 
 
+def choisir_modele(choixModele):
+    print("ON A RECUP LE CHOIX MODELE")
+    if choixModele == "lstm" :
+        choisir_modele.var ='lstm'
+    elif choixModele == "fineTunedT5":
+        choisir_modele.var = "fineTunedT5"
+    else :
+        "le modele n'est pas defini"
+
+
 # -------- API ---------------------------------------------------------------------
 app = FastAPI()
 
@@ -27,9 +41,33 @@ app.mount("/templates", StaticFiles(directory="templates"), name="templates")
 async def index(request: Request):
     return templates.TemplateResponse("index.html.jinja", {"request": request})
 
+@app.get("/model")
+async def index(request: Request):
+    return templates.TemplateResponse("index.html.jinja", {"request": request})
+
+@app.get("/predict")
+async def index(request: Request):
+    return templates.TemplateResponse("index.html.jinja", {"request": request})
+
+
+@app.post("/model")
+async def choix_model(request: Request, choixModel:str = Form(None)):
+    print(choixModel)
+    if not choixModel:
+        erreur_modele = "Merci de saisir un modèle."
+        return templates.TemplateResponse(
+        "index.html.jinja", {"request": request, "text": erreur_modele}
+        )
+    else :
+        choisir_modele(choixModel)
+        print("C'est bon on utilise le modèle demandé")
+        return templates.TemplateResponse(
+        "index.html.jinja", {"request": request}
+        )
+
 
 # retourner le texte, les predictions et message d'erreur si formulaire envoye vide
-@app.post("/")
+@app.post("/predict")
 async def prediction(request: Request, text: str = Form(None)):
     if not text:
         error = "Merci de saisir votre texte."

src/fine_tune_t5.py

@@ -0,0 +1,204 @@
+import torch
+import datasets
+from datasets import Dataset, DatasetDict
+import pandas as pd
+from tqdm import tqdm
+import re
+import os
+import nltk
+import string
+import contractions
+from transformers import pipeline
+import evaluate
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer,AutoConfig
+from transformers import Seq2SeqTrainingArguments ,Seq2SeqTrainer
+from transformers import DataCollatorForSeq2Seq
+
+# cuda out of memory
+os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "max_split_size_mb:200"
+
+nltk.download('stopwords')
+nltk.download('punkt')
+
+
+def clean_data(texts):
+    texts = texts.lower()
+    texts = contractions.fix(texts)
+    texts = texts.translate(str.maketrans("", "", string.punctuation))
+    texts = re.sub(r'\n',' ',texts)
+    return texts
+
+def datasetmaker (path=str):
+    data = pd.read_json(path, lines=True)
+    df = data.drop(['url','archive','title','date','compression','coverage','density','compression_bin','coverage_bin','density_bin'],axis=1)
+    tqdm.pandas()
+    df['text'] = df.text.apply(lambda texts : clean_data(texts))
+    df['summary'] = df.summary.apply(lambda summary : clean_data(summary))
+    # df['text'] = df['text'].map(str)
+    # df['summary'] = df['summary'].map(str)
+    dataset = Dataset.from_dict(df)
+    return dataset
+
+#voir si le model par hasard esr déjà bien
+
+# test_text = dataset['text'][0]
+# pipe = pipeline('summarization',model = model_ckpt)
+# pipe_out = pipe(test_text)
+# print (pipe_out[0]['summary_text'].replace('.<n>','.\n'))
+# print(dataset['summary'][0])
+
+def generate_batch_sized_chunks(list_elements, batch_size):
+    """split the dataset into smaller batches that we can process simultaneously
+    Yield successive batch-sized chunks from list_of_elements."""
+    for i in range(0, len(list_elements), batch_size):
+        yield list_elements[i : i + batch_size]
+
+def calculate_metric(dataset, metric, model, tokenizer,
+                               batch_size, device,
+                               column_text='text',
+                               column_summary='summary'):
+    article_batches = list(str(generate_batch_sized_chunks(dataset[column_text], batch_size)))
+    target_batches = list(str(generate_batch_sized_chunks(dataset[column_summary], batch_size)))
+
+    for article_batch, target_batch in tqdm(
+        zip(article_batches, target_batches), total=len(article_batches)):
+
+        inputs = tokenizer(article_batch, max_length=1024,  truncation=True,
+                        padding="max_length", return_tensors="pt")
+
+        summaries = model.generate(input_ids=inputs["input_ids"].to(device),
+                         attention_mask=inputs["attention_mask"].to(device),
+                         length_penalty=0.8, num_beams=8, max_length=128)
+        ''' parameter for length penalty ensures that the model does not generate sequences that are too long. '''
+
+        # Décode les textes
+        # renplacer les tokens, ajouter des textes décodés avec les rédéfences vers la métrique.
+        decoded_summaries = [tokenizer.decode(s, skip_special_tokens=True,
+                                clean_up_tokenization_spaces=True)
+               for s in summaries]
+
+        decoded_summaries = [d.replace("", " ") for d in decoded_summaries]
+
+
+        metric.add_batch(predictions=decoded_summaries, references=target_batch)
+
+    #compute et return les ROUGE scores.
+    results = metric.compute()
+    rouge_names = ['rouge1','rouge2','rougeL','rougeLsum']
+    rouge_dict = dict((rn, results[rn] ) for rn in rouge_names )
+    return pd.DataFrame(rouge_dict, index = ['T5'])
+
+
+def convert_ex_to_features(example_batch):
+    input_encodings = tokenizer(example_batch['text'],max_length = 1024,truncation = True)
+
+    labels =tokenizer(example_batch['summary'], max_length = 128, truncation = True )
+
+    return {
+        'input_ids' : input_encodings['input_ids'],
+        'attention_mask': input_encodings['attention_mask'],
+        'labels': labels['input_ids']
+    }
+
+if __name__=='__main__':
+
+    train_dataset = datasetmaker('data/train_extract_100.jsonl')
+
+    dev_dataset = datasetmaker('data/dev_extract_100.jsonl')
+
+    test_dataset = datasetmaker('data/test_extract_100.jsonl')
+
+    dataset = datasets.DatasetDict({'train':train_dataset,'dev':dev_dataset ,'test':test_dataset})
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
+    mt5_config = AutoConfig.from_pretrained(
+    "google/mt5-small",
+    max_length=128,
+    length_penalty=0.6,
+    no_repeat_ngram_size=2,
+    num_beams=15,
+    )
+    model = (AutoModelForSeq2SeqLM
+            .from_pretrained("google/mt5-small", config=mt5_config)
+            .to(device))
+
+    dataset_pt= dataset.map(convert_ex_to_features,remove_columns=["summary", "text"],batched = True,batch_size=128)
+
+    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model,return_tensors="pt")
+
+
+    training_args = Seq2SeqTrainingArguments(
+        output_dir = "mt5_sum",
+        log_level = "error",
+        num_train_epochs = 10,
+        learning_rate = 5e-4,
+        #   lr_scheduler_type = "linear",
+        warmup_steps = 0,
+        optim = "adafactor",
+        weight_decay = 0.01,
+        per_device_train_batch_size = 2,
+        per_device_eval_batch_size = 1,
+        gradient_accumulation_steps = 16,
+        evaluation_strategy = "steps",
+        eval_steps = 100,
+        predict_with_generate=True,
+        generation_max_length = 128,
+        save_steps = 500,
+        logging_steps = 10,
+        # push_to_hub = True
+    )
+
+
+    trainer = Seq2SeqTrainer(
+        model = model,
+        args = training_args,
+        data_collator = data_collator,
+        # compute_metrics = calculate_metric,
+        train_dataset=dataset_pt['train'],
+        eval_dataset=dataset_pt['dev'].select(range(10)),
+        tokenizer = tokenizer,
+    )
+
+    trainer.train()
+    rouge_metric = evaluate.load("rouge")
+
+    score = calculate_metric(test_dataset, rouge_metric, trainer.model, tokenizer,
+                                batch_size=2, device=device,
+                                column_text='text',
+                                column_summary='summary')
+    print (score)
+
+
+    #Fine Tuning terminés et à sauvgarder
+
+
+
+    # save fine-tuned model in local
+    os.makedirs("./summarization_t5", exist_ok=True)
+    if hasattr(trainer.model, "module"):
+        trainer.model.module.save_pretrained("./summarization_t5")
+    else:
+        trainer.model.save_pretrained("./summarization_t5")
+    tokenizer.save_pretrained("./summarization_t5")
+    # load local model
+    model = (AutoModelForSeq2SeqLM
+            .from_pretrained("./summarization_t5")
+            .to(device))
+
+
+    # mettre en usage : TEST
+
+
+    # gen_kwargs = {"length_penalty": 0.8, "num_beams":8, "max_length": 128}
+    # sample_text = dataset["test"][0]["text"]
+    # reference = dataset["test"][0]["summary"]
+    # pipe = pipeline("summarization", model='./summarization_t5')
+
+    # print("Text:")
+    # print(sample_text)
+    # print("\nReference Summary:")
+    # print(reference)
+    # print("\nModel Summary:")
+    # print(pipe(sample_text, **gen_kwargs)[0]["summary_text"])

src/inference_t5.py

@@ -0,0 +1,65 @@
+"""
+ Allows to predict the summary for a given entry text
+"""
+import torch
+import nltk
+import contractions
+import re
+import string
+nltk.download('stopwords')
+nltk.download('punkt')
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+def clean_data(texts):
+    texts = texts.lower()
+    texts = contractions.fix(texts)
+    texts = texts.translate(str.maketrans("", "", string.punctuation))
+    texts = re.sub(r'\n',' ',texts)
+    return texts
+
+def inferenceAPI_t5(text: str) -> str:
+    """
+    Predict the summary for an input text
+    --------
+    Parameter
+        text: str
+            the text to sumarize
+    Return
+        str
+            The summary for the input text
+    """
+    # definition des parametres d'entree pour le modèle
+    text = clean_data(text)
+    device = torch.device("cpu" if torch.cuda.is_available() else "cpu")
+    tokenizer= (AutoTokenizer.from_pretrained("./summarization_t5"))
+    # chargement du modele local
+    model = (AutoModelForSeq2SeqLM
+            .from_pretrained("./summarization_t5")
+            .to(device))
+    text_encoding = tokenizer(
+        text,
+        max_length=1024,
+        padding='max_length',
+        truncation=True,
+        return_attention_mask=True,
+        add_special_tokens=True,
+        return_tensors='pt'
+    )
+    generated_ids = model.generate(
+        input_ids=text_encoding['input_ids'],
+        attention_mask=text_encoding['attention_mask'],
+        max_length=128,
+        num_beams=8,
+        length_penalty=0.8,
+        early_stopping=True
+    )
+
+    preds = [
+            tokenizer.decode(gen_id, skip_special_tokens=True, clean_up_tokenization_spaces=True)
+            for gen_id in generated_ids
+    ]
+    return "".join(preds)
+
+if __name__ == "__main__":
+     text = input('Entrez votre phrase à résumer : ')
+     print('summary:',inferenceAPI(text))

templates/index.html.jinja

@@ -13,6 +13,23 @@
                 document.getElementById("summary").value = "";
             }
         </script>
+        <script>
+        function submitBothForms()
+            {
+            document.getElementById("my_form").submit();
+            document.getElementById("choixModel").submit();
+            }
+        </script>
+        <script>
+            function getValue() {
+                var e = document.getElementById("choixModel");
+                var value = e.value;
+                var text = e.options[e.selectedIndex].text;
+                return text}
+        </script>
+        <script type="text/javascript">
+            document.getElementById('choixModel').value = "<?php echo $_GET['choixModel'];?>";
+        </script>
     </head>
     <body>
         <div id="header">
@@ -28,18 +45,21 @@
             </nav>
 
             <div class="choixModel">
-                <label for="model-select">Choose a model :</label>
-                <select name="model" id="model-select">
-                    <option value="lstm">LSTM</option>
-                    <option value="autre">Autre</option>
-                </select>
+                <form id="choixModel" method="post" action="/model">
+                    <label for="selectModel">Choose a model :</label>
+                    <select name="choixModel" class="selectModel" id="choixModel">
+                        <option value="lstm">LSTM</option>
+                        <option value="fineTunedT5">Fine-tuned T5</option>
+                    </select>
+                </form>
+                <button form ="choixModel" class='search_bn' type="submit" class="btn btn-primary btn-block btn-large" rows="1" cols="50">Select model</button>
             </div>
 
             <div>
                 <table>
                     <tr>
                         <td>
-                            <form id = "my_form" action="/"  method="post" class="formulaire">
+                            <form id = "my_form" action="/predict"  method="post" class="formulaire">
                                 <textarea id="text" name="text" placeholder="Enter your text here!" rows="15" cols="75">{{text}}</textarea>
                                 <input type="hidden" name="textarea_value" value="{{ text }}">
                             </form>
@@ -51,8 +71,9 @@
                 </table>
             </div>
             <div class="buttons">
-                    <button form ="my_form" class='search_bn' type="submit" class="btn btn-primary btn-block btn-large" rows="1" cols="50">Go !</button>
-                    <button form ="my_form" type="button" value="Reset" onclick="customReset();">Reset</button>
+                <!-- <button id="submit" type="submit" onclick=submitBothForms()>SUBMIT</button> -->
+                <button form ="my_form" class='search_bn' type="submit" class="btn btn-primary btn-block btn-large" rows="1" cols="50">Go !</button>
+                <button form ="my_form" type="button" value="Reset" onclick="customReset();">Reset</button>
             </div>
 
             <div class="copyright">