add

requirements.txt

@@ -9,4 +9,5 @@ seqeval==1.2.2
 pandas==2.1.4
 gradio==4.13.0
 pydantic_settings==2.1.0
-sentencepiece==0.1.99
+sentencepiece==0.1.99
+umap-learn==0.5.5

source/services/predicting_effective_arguments/train/model.py

@@ -7,6 +7,8 @@ import torch.nn.functional as F
 import matplotlib.pyplot as plt
 from typing import List
 from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix
+from umap import UMAP
+from sklearn.preprocessing import MinMaxScaler
 
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
@@ -17,19 +19,41 @@ class TransformersSequenceClassifier:
                  model_output_dir,
                  num_labels,
                  tokenizer : AutoTokenizer,
+                 id2label,
+                 label2id,
                  model_checkpoint="distilbert-base-uncased"
                  ):
         self.model_output_dir = model_output_dir
-        self.tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
-        self.model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=num_labels).to(device)
+        self.tokenizer = tokenizer
+        self.model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=num_labels, id2label=id2label, label2id=label2id).to(device)
 
     def tokenizer_batch(self, batch):
-        return self.tokenizer(batch["inputs"], truncation=True) #, max_len=386
+        return self.tokenizer(batch["inputs"], truncation=True, padding=True, return_tensors="pt") #, max_len=386
 
     def tokenize_dataset(self, dataset):
         return dataset.map(self.tokenizer_batch, batched=True, remove_columns=('inputs', '__index_level_0__'))
+
+    @staticmethod
+    def extract_hidden_states(batch, tokenizer, model):
+        # Place model inputs on the GPU
+        inputs = {k:v for k,v in batch.items() if k in tokenizer.model_input_names} #.to(device)
+        # Extract last hidden states
+        with torch.no_grad():
+            last_hidden_state = model(**inputs).last_hidden_state
+        # Return vector for [CLS] token
+        return {"hidden_state": last_hidden_state[:,0].cpu().numpy()}
+    
+    @staticmethod
+    def fit_umap(df_x):
+        # Scale features to [0,1] range
+        X_scaled = MinMaxScaler().fit_transform(df_x)
+        # Initialize and fit UMAP
+        mapper = UMAP(n_components=2, metric="cosine").fit(X_scaled)
+        return mapper.embedding_
+        # Create a DataFrame of 2D embeddings
+    
     def train(self, train_dataset, eval_dataset, batch_size, epochs):
-        data_collator = DataCollatorWithPadding(tokenizer=self.tokenizer, padding='longest')
+        #data_collator = DataCollatorWithPadding(tokenizer=self.tokenizer, padding='longest')
         training_args = TrainingArguments(output_dir=self.model_output_dir,
                                           num_train_epochs=epochs,
                                           learning_rate=2e-5,
@@ -39,7 +63,7 @@ class TransformersSequenceClassifier:
                                           evaluation_strategy="epoch",
                                           save_strategy='epoch',
                                           disable_tqdm=False,
-                                          logging_steps=len(train_dataset)// batch_size,
+                                          logging_steps=len(train_dataset)//batch_size,
                                           push_to_hub=True,
                                           load_best_model_at_end=True,
                                           log_level="error")
@@ -50,7 +74,7 @@ class TransformersSequenceClassifier:
             train_dataset=train_dataset,
             eval_dataset=eval_dataset,
             tokenizer=self.tokenizer,
-            data_collator=data_collator
+            #data_collator=data_collator
         )
         self.trainer.train()
         self.trainer.push_to_hub(commit_message="Training completed!")
@@ -83,15 +107,15 @@ class TransformersSequenceClassifier:
         return valid_dataset.map(self.forward_pass_with_label, batched=True, batch_size=16)
 
     @staticmethod
-    def plot_confusion_matrix(y_preds, y_true, labels):
+    def plot_confusion_matrix(y_preds, y_true, label_names):
         cm = confusion_matrix(y_true, y_preds, normalize="true")
         fig, ax = plt.subplots(figsize=(6, 6))
-        disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels)
+        disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=label_names)
         disp.plot(cmap="Blues", values_format=".2f", ax=ax, colorbar=False)
         plt.title("Normalized confusion matrix")
         plt.show()
 
-    def predict_valid_data(self, valid_dataset):
+    def predict_argmax_logit(self, valid_dataset):
         #trainer = Trainer(model=self.model)
         preds_output = self.trainer.predict(valid_dataset)
         print(preds_output.metrics)
@@ -99,7 +123,7 @@ class TransformersSequenceClassifier:
         return y_preds
     
     @staticmethod
-    def predict_test_data(model_checkpoint, test_list: List[str]) -> List:
+    def predict_pipeline(model_checkpoint, test_list: List[str]) -> List:
         pipe_classifier = pipeline("text-classification", model=model_checkpoint)
         preds = pipe_classifier(test_list, return_all_scores=True)
         return preds

source/services/predicting_effective_arguments/train/train_seq_classification.py

@@ -0,0 +1,119 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from datasets import load_dataset
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from datasets import Dataset, load_metric
+from sklearn.model_selection import train_test_split
+from source.services.predicting_effective_arguments.train.model import TransformersSequenceClassifier
+
+class CFG:
+    TARGET = 'discourse_effectiveness'
+    TEXT = "discourse_text"
+    MODEL_CHECKPOINT = "distilbert-base-uncased"
+    MODEL_OUTPUT_DIR ='source/services/predicting_effective_arguments/model/hf_textclassification/predicting_effective_arguments_distilbert'
+    model_name="debertav3base"
+    learning_rate=1.5e-5
+    weight_decay=0.02
+    hidden_dropout_prob=0.007
+    attention_probs_dropout_prob=0.007
+    num_train_epochs=10
+    n_splits=4
+    batch_size=12
+    random_seed=42
+    save_steps=100
+    max_length=512
+
+
+def seed_everything(seed: int):
+    import random, os
+    import numpy as np
+    import torch
+    
+    random.seed(seed)
+    os.environ['PYTHONHASHSEED'] = str(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = True
+
+
+def prepare_input_text(df, sep_token):
+    df['inputs'] = df.discourse_type.str.lower() + ' ' + sep_token + ' ' + df.discourse_text.str.lower()
+    return df
+
+
+if __name__ == '__main__':
+
+    config = CFG()
+    tokenizer = AutoTokenizer.from_pretrained(config.MODEL_CHECKPOINT)
+
+    data = pd.read_csv("data/raw_data/train.csv")[:100]
+    label_names = list(data[config.TARGET].unique())
+    #score_df = pd.read_csv("data/raw_data/test.csv")
+
+    """
+    data[TARGET].value_counts(ascending=True).plot.barh()
+    plt.title("Frequency of Classes")
+    plt.show()
+
+    data['discourse_type'].value_counts(ascending=True).plot.barh()
+    plt.title("Frequency of discourse_type")
+    plt.show()
+
+    data["Words Per text"] = data[TEXT].str.split().apply(len)
+    data.boxplot("Words Per text", by=TARGET, grid=False, showfliers=False,
+            color="black")
+    plt.suptitle("")
+    plt.xlabel("")
+    plt.show()
+    """
+
+    train_size = 0.7
+    valid_size = 0.2
+    test_size = 0.1
+
+    # First split: Separate out the training set
+    train_df, temp_df = train_test_split(data, test_size=1 - train_size, random_state=5600)
+
+    # Second split: Separate out the validation and test sets
+    valid_df, test_df = train_test_split(temp_df, test_size=test_size / (test_size + valid_size), random_state=5600)
+
+
+    train_df = prepare_input_text(train_df, sep_token=tokenizer.sep_token)
+    valid_df = prepare_input_text(valid_df, sep_token=tokenizer.sep_token)
+    test_df = prepare_input_text(test_df, sep_token=tokenizer.sep_token)
+    
+    train_dataset = Dataset.from_pandas(train_df[['inputs', config.TARGET]]).rename_column(config.TARGET, 'label').class_encode_column("label")
+    val_dataset = Dataset.from_pandas(valid_df[['inputs', config.TARGET]]).rename_column(config.TARGET, 'label').class_encode_column("label")
+    test_dataset = Dataset.from_pandas(test_df[['inputs', config.TARGET]]).rename_column(config.TARGET, 'label').class_encode_column("label")
+
+    id2label = {i: label for i, label in enumerate(label_names)}
+    label2id = {v: k for k, v in id2label.items()}
+    seqClassifer = TransformersSequenceClassifier(model_output_dir=config.MODEL_OUTPUT_DIR,
+                                                  tokenizer=tokenizer,
+                                                  model_checkpoint="distilbert-base-uncased",
+                                                  num_labels=3,
+                                                  id2label=id2label,
+                                                  label2id=label2id)
+
+    train_tok_dataset = seqClassifer.tokenize_dataset(dataset=train_dataset)
+    val_tok_dataset = seqClassifer.tokenize_dataset(dataset=val_dataset)
+    test_tok_dataset = seqClassifer.tokenize_dataset(dataset=test_dataset)
+    
+
+    
+    seqClassifer.train(train_dataset=train_tok_dataset, eval_dataset=val_tok_dataset, epochs=1, batch_size=16)
+
+    y_test_pred = seqClassifer.predict_argmax_logit(test_tok_dataset)
+    seqClassifer.plot_confusion_matrix(y_preds=y_test_pred, y_true=test_dataset['label'], label_names=label_names)
+
+    y_pred = seqClassifer.predict_pipeline(model_checkpoint=config.MODEL_OUTPUT_DIR, test_list=test_df['inputs'].tolist())
+    #hidden = train_tok_dataset.map(seqClassifer.extract_hidden_states, 
+    #                                        batched=True, 
+    #                                        fn_kwargs={'tokenizer': AutoTokenizer.from_pretrained(config.MODEL_OUTPUT_DIR),
+    #                                                   'model': AutoModelForSequenceClassification.from_pretrained(config.MODEL_OUTPUT_DIR)})
+    
+    pass
+