Add other files

CustomBERTModel.py

@@ -0,0 +1,33 @@
+import torch
+import torch.nn as nn
+from src.bert import BERT
+
+class CustomBERTModel(nn.Module):
+    def __init__(self, vocab_size, output_dim, pre_trained_model_path):
+        super(CustomBERTModel, self).__init__()
+        hidden_size = 768
+        self.bert = BERT(vocab_size=vocab_size, hidden=hidden_size, n_layers=4, attn_heads=8, dropout=0.1)
+
+        # Load the pre-trained model's state_dict
+        checkpoint = torch.load(pre_trained_model_path, map_location=torch.device('cpu'))
+        if isinstance(checkpoint, dict):
+            self.bert.load_state_dict(checkpoint)
+        else:
+            raise TypeError(f"Expected state_dict, got {type(checkpoint)} instead.")
+
+        # Fully connected layer with input size 768 (matching BERT hidden size)
+        self.fc = nn.Linear(hidden_size, output_dim)
+
+    def forward(self, sequence, segment_info):
+        sequence = sequence.to(next(self.parameters()).device)
+        segment_info = segment_info.to(sequence.device)
+
+        x = self.bert(sequence, segment_info)
+        print(f"BERT output shape: {x.shape}") 
+
+        cls_embeddings = x[:, 0]  # Extract CLS token embeddings
+        print(f"CLS Embeddings shape: {cls_embeddings.shape}") 
+
+        logits = self.fc(cls_embeddings)  # Pass tensor of size (batch_size, 768) to the fully connected layer
+        
+        return logits

data_preprocessor.py

@@ -0,0 +1,170 @@
+import time
+import pandas as pd
+
+import sys
+
+class DataPreprocessor:
+    def __init__(self, input_file_path):
+        self.input_file_path = input_file_path
+        self.unique_students = None
+        self.unique_problems = None
+        self.unique_prob_hierarchy = None
+        self.unique_steps = None
+        self.unique_kcs = None
+
+    def analyze_dataset(self):
+        file_iterator = self.load_file_iterator()
+
+        start_time = time.time()
+        self.unique_students = {"st"}
+        self.unique_problems = {"pr"}
+        self.unique_prob_hierarchy = {"ph"}
+        self.unique_kcs = {"kc"}
+        for chunk_data in file_iterator:
+            for student_id, std_groups in chunk_data.groupby('Anon Student Id'):
+                self.unique_students.update({student_id})
+                prob_hierarchy = std_groups.groupby('Level (Workspace Id)')
+                for hierarchy, hierarchy_groups in prob_hierarchy:
+                    self.unique_prob_hierarchy.update({hierarchy})
+                    prob_name = hierarchy_groups.groupby('Problem Name')
+                    for problem_name, prob_name_groups in prob_name:
+                        self.unique_problems.update({problem_name})
+                        sub_skills = prob_name_groups['KC Model(MATHia)']
+                        for a in sub_skills:
+                            if str(a) != "nan":
+                                temp = a.split("~~")
+                                for kc in temp:
+                                    self.unique_kcs.update({kc})
+        self.unique_students.remove("st")
+        self.unique_problems.remove("pr")
+        self.unique_prob_hierarchy.remove("ph")
+        self.unique_kcs.remove("kc")
+        end_time = time.time()
+        print("Time Taken to analyze dataset = ", end_time - start_time)
+        print("Length of unique students->", len(self.unique_students))
+        print("Length of unique problems->", len(self.unique_problems))
+        print("Length of unique problem hierarchy->", len(self.unique_prob_hierarchy))
+        print("Length of Unique Knowledge components ->", len(self.unique_kcs))
+
+    def analyze_dataset_by_section(self, workspace_name):
+        file_iterator = self.load_file_iterator()
+        
+        start_time = time.time()
+        self.unique_students = {"st"}
+        self.unique_problems = {"pr"}
+        self.unique_prob_hierarchy = {"ph"}
+        self.unique_steps = {"s"}
+        self.unique_kcs = {"kc"}
+        # with open("workspace_info.txt", 'a') as f:
+        #     sys.stdout = f
+        for chunk_data in file_iterator:
+            for student_id, std_groups in chunk_data.groupby('Anon Student Id'):
+                prob_hierarchy = std_groups.groupby('Level (Workspace Id)')
+                for hierarchy, hierarchy_groups in prob_hierarchy:
+                    if workspace_name == hierarchy:
+                        # print("Workspace : ", hierarchy)
+                        self.unique_students.update({student_id})   
+                        self.unique_prob_hierarchy.update({hierarchy})
+                        prob_name = hierarchy_groups.groupby('Problem Name')
+                        for problem_name, prob_name_groups in prob_name:
+                            self.unique_problems.update({problem_name})
+                            step_names = prob_name_groups['Step Name']
+                            sub_skills = prob_name_groups['KC Model(MATHia)']
+                            for step in step_names:
+                                if str(step) != "nan":
+                                    self.unique_steps.update({step})
+                            for a in sub_skills:
+                                if str(a) != "nan":
+                                    temp = a.split("~~")
+                                    for kc in temp:
+                                        self.unique_kcs.update({kc})
+        self.unique_problems.remove("pr")
+        self.unique_prob_hierarchy.remove("ph")
+        self.unique_steps.remove("s")
+        self.unique_kcs.remove("kc")
+        end_time = time.time()
+        print("Time Taken to analyze dataset = ", end_time - start_time)
+        print("Workspace-> ",workspace_name)
+        print("Length of unique students->", len(self.unique_students))
+        print("Length of unique problems->", len(self.unique_problems))
+        print("Length of unique problem hierarchy->", len(self.unique_prob_hierarchy))
+        print("Length of unique step names ->", len(self.unique_steps))
+        print("Length of unique knowledge components ->", len(self.unique_kcs))
+        #     f.close()
+        # sys.stdout = sys.__stdout__
+        
+    def analyze_dataset_by_school(self, workspace_name, school_id=None):
+        file_iterator = self.load_file_iterator(sep=",")
+
+        start_time = time.time()
+        self.unique_schools = set()
+        self.unique_class = set()
+        self.unique_students = set()
+        self.unique_problems = set()
+        self.unique_steps = set()
+        self.unique_kcs = set()
+        self.unique_actions = set()
+        self.unique_outcomes = set()
+        self.unique_new_steps_w_action_attempt = set()
+        self.unique_new_steps_w_kcs = set()
+        self.unique_new_steps_w_action_attempt_kcs = set()
+        
+        for chunk_data in file_iterator:
+            for school, school_group in chunk_data.groupby('CF (Anon School Id)'):
+                # if school and school == school_id:
+                self.unique_schools.add(school)
+                for class_id, class_group in school_group.groupby('CF (Anon Class Id)'):
+                    self.unique_class.add(class_id)
+                    for student_id, std_group in class_group.groupby('Anon Student Id'):
+                        self.unique_students.add(student_id)
+                        for prob, prob_group in std_group.groupby('Problem Name'):
+                            self.unique_problems.add(prob)
+                            
+                            step_names = set(prob_group['Step Name'])
+                            sub_skills = set(prob_group['KC Model(MATHia)'])
+                            actions = set(prob_group['Action'])
+                            outcomes = set(prob_group['Outcome'])
+                            
+                            self.unique_steps.update(step_names)
+                            self.unique_kcs.update(sub_skills)
+                            self.unique_actions.update(actions)
+                            self.unique_outcomes.update(outcomes)
+                            
+                            for step in step_names:                                
+                                if pd.isna(step):
+                                    step_group = prob_group[pd.isna(prob_group['Step Name'])]
+                                else:
+                                    step_group = prob_group[prob_group['Step Name']==step]
+                                    
+                                for kc in set(step_group['KC Model(MATHia)']):
+                                    new_step = f"{step}:{kc}"
+                                    self.unique_new_steps_w_kcs.add(new_step)
+
+                                for action, action_group in step_group.groupby('Action'):
+                                    for attempt, attempt_group in action_group.groupby('Attempt At Step'):
+                                        new_step = f"{step}:{action}:{attempt}"
+                                        self.unique_new_steps_w_action_attempt.add(new_step)
+
+                                        for kc in set(attempt_group["KC Model(MATHia)"]):
+                                            new_step = f"{step}:{action}:{attempt}:{kc}"
+                                            self.unique_new_steps_w_action_attempt_kcs.add(new_step)
+                                        
+
+        end_time = time.time()
+        print("Time Taken to analyze dataset = ", end_time - start_time)
+        print("Workspace-> ",workspace_name)
+        print("Length of unique students->", len(self.unique_students))
+        print("Length of unique problems->", len(self.unique_problems))
+        print("Length of unique classes->", len(self.unique_class))
+        print("Length of unique step names ->", len(self.unique_steps))
+        print("Length of unique knowledge components ->", len(self.unique_kcs))
+        print("Length of unique actions ->", len(self.unique_actions))
+        print("Length of unique outcomes ->", len(self.unique_outcomes))
+        print("Length of unique new step names with actions and attempts ->", len(self.unique_new_steps_w_action_attempt))
+        print("Length of unique new step names with actions, attempts and kcs ->", len(self.unique_new_steps_w_action_attempt_kcs))
+        print("Length of unique new step names with kcs ->", len(self.unique_new_steps_w_kcs))
+
+    def load_file_iterator(self, sep="\t"):
+        chunk_iterator = pd.read_csv(self.input_file_path, sep=sep, header=0, iterator=True, chunksize=1000000)
+        return chunk_iterator
+

hint_fine_tuning.py

@@ -0,0 +1,382 @@
+import argparse
+import os
+import sys
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, random_split, TensorDataset
+from src.dataset import TokenizerDataset
+from src.bert import BERT
+from src.pretrainer import BERTFineTuneTrainer1
+from src.vocab import Vocab
+import pandas as pd
+
+
+# class CustomBERTModel(nn.Module):
+#     def __init__(self, vocab_size, output_dim, pre_trained_model_path):
+#         super(CustomBERTModel, self).__init__()
+#         hidden_size = 768
+#         self.bert = BERT(vocab_size=vocab_size, hidden=hidden_size, n_layers=12, attn_heads=12, dropout=0.1)
+#         checkpoint = torch.load(pre_trained_model_path, map_location=torch.device('cpu'))
+#         if isinstance(checkpoint, dict):
+#             self.bert.load_state_dict(checkpoint)
+#         elif isinstance(checkpoint, BERT):
+#             self.bert = checkpoint
+#         else:
+#             raise TypeError(f"Expected state_dict or BERT instance, got {type(checkpoint)} instead.")
+#         self.fc = nn.Linear(hidden_size, output_dim)
+
+#     def forward(self, sequence, segment_info):
+#         sequence = sequence.to(next(self.parameters()).device)
+#         segment_info = segment_info.to(sequence.device)
+
+#         if sequence.size(0) == 0 or sequence.size(1) == 0:
+#             raise ValueError("Input sequence tensor has 0 elements. Check data preprocessing.")
+
+#         x = self.bert(sequence, segment_info)
+#         print(f"BERT output shape: {x.shape}")
+
+#         if x.size(0) == 0 or x.size(1) == 0:
+#             raise ValueError("BERT output tensor has 0 elements. Check input dimensions.")
+
+#         cls_embeddings = x[:, 0]
+#         logits = self.fc(cls_embeddings)
+#         return logits
+
+# class CustomBERTModel(nn.Module):
+#     def __init__(self, vocab_size, output_dim, pre_trained_model_path):
+#         super(CustomBERTModel, self).__init__()
+#         hidden_size = 764  # Ensure this is 768
+#         self.bert = BERT(vocab_size=vocab_size, hidden=hidden_size, n_layers=12, attn_heads=12, dropout=0.1)
+        
+#         # Load the pre-trained model's state_dict
+#         checkpoint = torch.load(pre_trained_model_path, map_location=torch.device('cpu'))
+#         if isinstance(checkpoint, dict):
+#             self.bert.load_state_dict(checkpoint)
+#         else:
+#             raise TypeError(f"Expected state_dict, got {type(checkpoint)} instead.")
+        
+#         # Fully connected layer with input size 768
+#         self.fc = nn.Linear(hidden_size, output_dim)
+
+#     def forward(self, sequence, segment_info):
+#         sequence = sequence.to(next(self.parameters()).device)
+#         segment_info = segment_info.to(sequence.device)
+
+#         x = self.bert(sequence, segment_info)
+#         print(f"BERT output shape: {x.shape}")  # Should output (batch_size, seq_len, 768)
+
+#         cls_embeddings = x[:, 0]  # Extract CLS token embeddings
+#         print(f"CLS Embeddings shape: {cls_embeddings.shape}")  # Should output (batch_size, 768)
+
+#         logits = self.fc(cls_embeddings)  # Should now pass a tensor of size (batch_size, 768) to `fc`
+        
+#         return logits
+
+
+# for test
+class CustomBERTModel(nn.Module):
+    def __init__(self, vocab_size, output_dim, pre_trained_model_path):
+        super(CustomBERTModel, self).__init__()
+        self.hidden = 764  # Ensure this is defined correctly
+        self.bert = BERT(vocab_size=vocab_size, hidden=self.hidden, n_layers=12, attn_heads=12, dropout=0.1)
+
+        # Load the pre-trained model's state_dict
+        checkpoint = torch.load(pre_trained_model_path, map_location=torch.device('cpu'))
+        if isinstance(checkpoint, dict):
+            self.bert.load_state_dict(checkpoint)
+        else:
+            raise TypeError(f"Expected state_dict, got {type(checkpoint)} instead.")
+
+        self.fc = nn.Linear(self.hidden, output_dim)
+
+    def forward(self, sequence, segment_info):
+        x = self.bert(sequence, segment_info)
+        cls_embeddings = x[:, 0]  # Extract CLS token embeddings
+        logits = self.fc(cls_embeddings)  # Pass to fully connected layer
+        return logits
+
+def preprocess_labels(label_csv_path):
+    try:
+        labels_df = pd.read_csv(label_csv_path)
+        labels = labels_df['last_hint_class'].values.astype(int)
+        return torch.tensor(labels, dtype=torch.long)
+    except Exception as e:
+        print(f"Error reading dataset file: {e}")
+        return None
+
+
+def preprocess_data(data_path, vocab, max_length=128):
+    try:
+        with open(data_path, 'r') as f:
+            sequences = f.readlines()
+    except Exception as e:
+        print(f"Error reading data file: {e}")
+        return None, None
+
+    if len(sequences) == 0:
+        raise ValueError(f"No sequences found in data file {data_path}. Check the file content.")
+
+    tokenized_sequences = []
+
+    for sequence in sequences:
+        sequence = sequence.strip()
+        if sequence:
+            encoded = vocab.to_seq(sequence, seq_len=max_length)
+            encoded = encoded[:max_length] + [vocab.vocab.get('[PAD]', 0)] * (max_length - len(encoded))
+            segment_label = [0] * max_length
+
+            tokenized_sequences.append({
+                'input_ids': torch.tensor(encoded),
+                'segment_label': torch.tensor(segment_label)
+            })
+
+    if not tokenized_sequences:
+        raise ValueError("Tokenization resulted in an empty list. Check the sequences and tokenization logic.")
+
+    tokenized_sequences = [t for t in tokenized_sequences if len(t['input_ids']) == max_length]
+
+    if not tokenized_sequences:
+        raise ValueError("All tokenized sequences are of unexpected length. This suggests an issue with the tokenization logic.")
+
+    input_ids = torch.cat([t['input_ids'].unsqueeze(0) for t in tokenized_sequences], dim=0)
+    segment_labels = torch.cat([t['segment_label'].unsqueeze(0) for t in tokenized_sequences], dim=0)
+
+    print(f"Input IDs shape: {input_ids.shape}")
+    print(f"Segment labels shape: {segment_labels.shape}")
+
+    return input_ids, segment_labels
+
+
+def collate_fn(batch):
+    inputs = []
+    labels = []
+    segment_labels = []
+
+    for item in batch:
+        if item is None:
+            continue
+
+        if isinstance(item, dict):
+            inputs.append(item['input_ids'].unsqueeze(0))
+            labels.append(item['label'].unsqueeze(0))
+            segment_labels.append(item['segment_label'].unsqueeze(0))
+
+    if len(inputs) == 0 or len(segment_labels) == 0:
+        print("Empty batch encountered. Returning None to skip this batch.")
+        return None
+
+    try:
+        inputs = torch.cat(inputs, dim=0)
+        labels = torch.cat(labels, dim=0)
+        segment_labels = torch.cat(segment_labels, dim=0)
+    except Exception as e:
+        print(f"Error concatenating tensors: {e}")
+        return None
+
+    return {
+        'input': inputs,
+        'label': labels,
+        'segment_label': segment_labels
+    }
+
+def custom_collate_fn(batch):
+    processed_batch = collate_fn(batch)
+    
+    if processed_batch is None or len(processed_batch['input']) == 0:
+        # Return a valid batch with at least one element instead of an empty one
+        return {
+            'input': torch.zeros((1, 128), dtype=torch.long),
+            'label': torch.zeros((1,), dtype=torch.long),
+            'segment_label': torch.zeros((1, 128), dtype=torch.long)
+        }
+    
+    return processed_batch
+
+
+def train_without_progress_status(trainer, epoch, shuffle):
+    for epoch_idx in range(epoch):
+        print(f"EP_train:{epoch_idx}:")
+        for batch in trainer.train_data:
+            if batch is None:
+                continue
+
+            # Check if batch is a string (indicating an issue)
+            if isinstance(batch, str):
+                print(f"Error: Received a string instead of a dictionary in batch: {batch}")
+                raise ValueError(f"Unexpected string in batch: {batch}")
+
+            # Validate the batch structure before passing to iteration
+            if isinstance(batch, dict):
+                # Verify that all expected keys are present and that the values are tensors
+                if all(key in batch for key in ['input_ids', 'segment_label', 'labels']):
+                    if all(isinstance(batch[key], torch.Tensor) for key in batch):
+                        try:
+                            print(f"Batch Structure: {batch}")  # Debugging batch before iteration
+                            trainer.iteration(epoch_idx, batch)
+                        except Exception as e:
+                            print(f"Error during batch processing: {e}")
+                            sys.stdout.flush()
+                            raise e  # Propagate the exception for better debugging
+                    else:
+                        print(f"Error: Expected all values in batch to be tensors, but got: {batch}")
+                        raise ValueError("Batch contains non-tensor values.")
+                else:
+                    print(f"Error: Batch missing expected keys. Batch keys: {batch.keys()}")
+                    raise ValueError("Batch does not contain expected keys.")
+            else:
+                print(f"Error: Expected batch to be a dictionary but got {type(batch)} instead.")
+                raise ValueError(f"Invalid batch structure: {batch}")
+
+# def main(opt):
+#     # device = torch.device("cpu")
+#     device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+#     vocab = Vocab(opt.vocab_file)
+#     vocab.load_vocab()
+
+#     input_ids, segment_labels = preprocess_data(opt.data_path, vocab, max_length=128)
+#     labels = preprocess_labels(opt.dataset)
+
+#     if input_ids is None or segment_labels is None or labels is None:
+#         print("Error in preprocessing data. Exiting.")
+#         return
+
+#     dataset = TensorDataset(input_ids, segment_labels, torch.tensor(labels, dtype=torch.long))
+#     val_size = len(dataset) - int(0.8 * len(dataset))
+#     val_dataset, train_dataset = random_split(dataset, [val_size, len(dataset) - val_size])
+
+#     train_dataloader = DataLoader(
+#         train_dataset,
+#         batch_size=32,
+#         shuffle=True,
+#         collate_fn=custom_collate_fn
+#     )
+#     val_dataloader = DataLoader(
+#         val_dataset,
+#         batch_size=32,
+#         shuffle=False,
+#         collate_fn=custom_collate_fn
+#     )
+
+#     custom_model = CustomBERTModel(
+#         vocab_size=len(vocab.vocab),
+#         output_dim=2,
+#         pre_trained_model_path=opt.pre_trained_model_path
+#     ).to(device)
+
+#     trainer = BERTFineTuneTrainer1(
+#         bert=custom_model.bert,
+#         vocab_size=len(vocab.vocab),
+#         train_dataloader=train_dataloader,
+#         test_dataloader=val_dataloader,
+#         lr=5e-5,
+#         num_labels=2,
+#         with_cuda=torch.cuda.is_available(),
+#         log_freq=10,
+#         workspace_name=opt.output_dir,
+#         log_folder_path=opt.log_folder_path 
+#     )
+
+#     trainer.train(epoch=20)
+
+#     # os.makedirs(opt.output_dir, exist_ok=True)
+#     # output_model_file = os.path.join(opt.output_dir, 'fine_tuned_model.pth')
+#     # torch.save(custom_model.state_dict(), output_model_file)
+#     # print(f'Model saved to {output_model_file}')
+    
+#     os.makedirs(opt.output_dir, exist_ok=True)
+#     output_model_file = os.path.join(opt.output_dir, 'fine_tuned_model_2.pth')
+#     torch.save(custom_model, output_model_file)
+#     print(f'Model saved to {output_model_file}')
+
+
+def main(opt):
+    # Set device to GPU if available, otherwise use CPU
+    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+    
+    print(torch.cuda.is_available())  # Should return True if GPU is available
+    print(torch.cuda.device_count())  
+
+    # Load vocabulary
+    vocab = Vocab(opt.vocab_file)
+    vocab.load_vocab()
+
+    # Preprocess data and labels
+    input_ids, segment_labels = preprocess_data(opt.data_path, vocab, max_length=128)
+    labels = preprocess_labels(opt.dataset)
+
+    if input_ids is None or segment_labels is None or labels is None:
+        print("Error in preprocessing data. Exiting.")
+        return
+
+    # Transfer tensors to the correct device (GPU/CPU)
+    input_ids = input_ids.to(device)
+    segment_labels = segment_labels.to(device)
+    labels = torch.tensor(labels, dtype=torch.long).to(device)
+
+    # Create TensorDataset and split into train and validation sets
+    dataset = TensorDataset(input_ids, segment_labels, labels)
+    val_size = len(dataset) - int(0.8 * len(dataset))
+    val_dataset, train_dataset = random_split(dataset, [val_size, len(dataset) - val_size])
+
+    # Create DataLoaders for training and validation
+    train_dataloader = DataLoader(
+        train_dataset,
+        batch_size=32,
+        shuffle=True,
+        collate_fn=custom_collate_fn
+    )
+    val_dataloader = DataLoader(
+        val_dataset,
+        batch_size=32,
+        shuffle=False,
+        collate_fn=custom_collate_fn
+    )
+
+    # Initialize custom BERT model and move it to the device
+    custom_model = CustomBERTModel(
+        vocab_size=len(vocab.vocab),
+        output_dim=2,
+        pre_trained_model_path=opt.pre_trained_model_path
+    ).to(device)
+
+    # Initialize the fine-tuning trainer
+    trainer = BERTFineTuneTrainer1(
+        bert=custom_model.bert,
+        vocab_size=len(vocab.vocab),
+        train_dataloader=train_dataloader,
+        test_dataloader=val_dataloader,
+        lr=5e-5,
+        num_labels=2,
+        with_cuda=torch.cuda.is_available(),
+        log_freq=10,
+        workspace_name=opt.output_dir,
+        log_folder_path=opt.log_folder_path 
+    )
+
+    # Train the model
+    trainer.train(epoch=20)
+
+    # Save the model to the specified output directory
+    # os.makedirs(opt.output_dir, exist_ok=True)
+    # output_model_file = os.path.join(opt.output_dir, 'fine_tuned_model_2.pth')
+    # torch.save(custom_model.state_dict(), output_model_file)
+    # print(f'Model saved to {output_model_file}')
+    os.makedirs(opt.output_dir, exist_ok=True)
+    output_model_file = os.path.join(opt.output_dir, 'fine_tuned_model_2.pth')
+    torch.save(custom_model, output_model_file)
+    print(f'Model saved to {output_model_file}')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Fine-tune BERT model.')
+    parser.add_argument('--dataset', type=str, default='/home/jupyter/bert/dataset/hint_based/ratio_proportion_change_3/er/er_train.csv', help='Path to the dataset file.')
+    parser.add_argument('--data_path', type=str, default='/home/jupyter/bert/ratio_proportion_change3_1920/_Aug23/gt/er.txt', help='Path to the input sequence file.')
+    parser.add_argument('--output_dir', type=str, default='/home/jupyter/bert/ratio_proportion_change3_1920/_Aug23/output/hint_classification', help='Directory to save the fine-tuned model.')
+    parser.add_argument('--pre_trained_model_path', type=str, default='/home/jupyter/bert/ratio_proportion_change3_1920/output/pretrain:1800ms:64hs:4l:8a:50s:64b:1000e:-5lr/bert_trained.seq_encoder.model.ep68', help='Path to the pre-trained BERT model.')
+    parser.add_argument('--vocab_file', type=str, default='/home/jupyter/bert/ratio_proportion_change3_1920/_Aug23/pretraining/vocab.txt', help='Path to the vocabulary file.')
+    parser.add_argument('--log_folder_path', type=str, default='/home/jupyter/bert/ratio_proportion_change3_1920/logs/oct_logs', help='Path to the folder for saving logs.')
+
+
+    opt = parser.parse_args()
+    main(opt)

main.py

@@ -0,0 +1,322 @@
+import argparse
+
+from torch.utils.data import DataLoader
+import torch
+import torch.nn as nn
+
+from src.bert import BERT
+from src.pretrainer import BERTTrainer, BERTFineTuneTrainer, BERTAttention
+from src.dataset import PretrainerDataset, TokenizerDataset
+from src.vocab import Vocab
+
+import time
+import os
+import tqdm
+import pickle
+
+def train():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('-workspace_name', type=str, default=None)
+    parser.add_argument('-code', type=str, default=None, help="folder for pretraining outputs and logs")
+    parser.add_argument('-finetune_task', type=str, default=None, help="folder inside finetuning")
+    parser.add_argument("-attention", type=bool, default=False, help="analyse attention scores")
+    parser.add_argument("-diff_test_folder", type=bool, default=False, help="use for different test folder")
+    parser.add_argument("-embeddings", type=bool, default=False, help="get and analyse embeddings")
+    parser.add_argument('-embeddings_file_name', type=str, default=None, help="file name of embeddings")
+    parser.add_argument("-pretrain", type=bool, default=False, help="pretraining: true, or false")
+    # parser.add_argument('-opts', nargs='+', type=str, default=None, help='List of optional steps')
+    parser.add_argument("-max_mask", type=int, default=0.15, help="% of input tokens selected for masking") 
+    # parser.add_argument("-p", "--pretrain_dataset", type=str, default="pretraining/pretrain.txt", help="pretraining dataset for bert")
+    # parser.add_argument("-pv", "--pretrain_val_dataset", type=str, default="pretraining/test.txt", help="pretraining validation dataset for bert")
+# default="finetuning/test.txt",
+    parser.add_argument("-vocab_path", type=str, default="pretraining/vocab.txt", help="built vocab model path with bert-vocab")
+
+    parser.add_argument("-train_dataset_path", type=str, default="train.txt", help="fine tune train dataset for progress classifier")
+    parser.add_argument("-val_dataset_path", type=str, default="val.txt", help="test set for evaluate fine tune train set")
+    parser.add_argument("-test_dataset_path", type=str, default="test.txt", help="test set for evaluate fine tune train set")
+    parser.add_argument("-num_labels", type=int, default=2, help="Number of labels") 
+    parser.add_argument("-train_label_path", type=str, default="train_label.txt", help="fine tune train dataset for progress classifier")
+    parser.add_argument("-val_label_path", type=str, default="val_label.txt", help="test set for evaluate fine tune train set")
+    parser.add_argument("-test_label_path", type=str, default="test_label.txt", help="test set for evaluate fine tune train set")
+    ##### change Checkpoint for finetuning
+    parser.add_argument("-pretrained_bert_checkpoint", type=str, default=None, help="checkpoint of saved pretrained bert model")  #."output_feb09/bert_trained.model.ep40"
+    parser.add_argument('-check_epoch', type=int, default=None)
+
+    parser.add_argument("-hs", "--hidden", type=int, default=64, help="hidden size of transformer model") #64
+    parser.add_argument("-l", "--layers", type=int, default=4, help="number of layers") #4
+    parser.add_argument("-a", "--attn_heads", type=int, default=4, help="number of attention heads") #8
+    parser.add_argument("-s", "--seq_len", type=int, default=50, help="maximum sequence length")
+
+    parser.add_argument("-b", "--batch_size", type=int, default=500, help="number of batch_size") #64
+    parser.add_argument("-e", "--epochs", type=int, default=50)#1501, help="number of epochs") #501
+    # Use 50 for pretrain, and 10 for fine tune
+    parser.add_argument("-w", "--num_workers", type=int, default=4, help="dataloader worker size")
+
+    # Later run with cuda
+    parser.add_argument("--with_cuda", type=bool, default=True, help="training with CUDA: true, or false")
+    parser.add_argument("--log_freq", type=int, default=10, help="printing loss every n iter: setting n")
+    # parser.add_argument("--corpus_lines", type=int, default=None, help="total number of lines in corpus")
+    parser.add_argument("--cuda_devices", type=int, nargs='+', default=None, help="CUDA device ids")
+    # parser.add_argument("--on_memory", type=bool, default=False, help="Loading on memory: true or false")
+
+    parser.add_argument("--dropout", type=float, default=0.1, help="dropout of network")
+    parser.add_argument("--lr", type=float, default=1e-05, help="learning rate of adam") #1e-3
+    parser.add_argument("--adam_weight_decay", type=float, default=0.01, help="weight_decay of adam")
+    parser.add_argument("--adam_beta1", type=float, default=0.9, help="adam first beta value")
+    parser.add_argument("--adam_beta2", type=float, default=0.98, help="adam first beta value") #0.999
+
+    parser.add_argument("-o", "--output_path", type=str, default="bert_trained.seq_encoder.model", help="ex)output/bert.model")    
+    # parser.add_argument("-o", "--output_path", type=str, default="output/bert_fine_tuned.model", help="ex)output/bert.model")
+    
+    args = parser.parse_args()
+    for k,v in vars(args).items():
+        if 'path' in k:
+            if v:
+                if k == "output_path":
+                    if args.code:
+                        setattr(args, f"{k}", args.workspace_name+f"/output/{args.code}/"+v)
+                    elif args.finetune_task:
+                        setattr(args, f"{k}", args.workspace_name+f"/output/{args.finetune_task}/"+v)
+                    else:
+                        setattr(args, f"{k}", args.workspace_name+"/output/"+v)
+                elif k != "vocab_path":
+                    if args.pretrain:
+                        setattr(args, f"{k}", args.workspace_name+"/pretraining/"+v)
+                    else:
+                        if args.code:
+                            setattr(args, f"{k}", args.workspace_name+f"/{args.code}/"+v)
+                        elif args.finetune_task:
+                            if args.diff_test_folder and "test" in k:
+                                setattr(args, f"{k}", args.workspace_name+f"/finetuning/"+v)
+                            else:
+                                setattr(args, f"{k}", args.workspace_name+f"/finetuning/{args.finetune_task}/"+v)
+                        else:
+                            setattr(args, f"{k}", args.workspace_name+"/finetuning/"+v)
+                else:
+                    setattr(args, f"{k}", args.workspace_name+"/"+v)
+                
+                print(f"args.{k} : {getattr(args, f'{k}')}")
+
+    print("Loading Vocab", args.vocab_path)
+    vocab_obj = Vocab(args.vocab_path)
+    vocab_obj.load_vocab()
+    print("Vocab Size: ", len(vocab_obj.vocab))
+    
+    if args.attention:
+        print(f"Attention aggregate...... code: {args.code}, dataset: {args.finetune_task}")
+        if args.code:
+            new_folder = f"{args.workspace_name}/plots/{args.code}/"
+            if not os.path.exists(new_folder):
+                os.makedirs(new_folder)
+        
+        train_dataset = TokenizerDataset(args.train_dataset_path, None, vocab_obj, seq_len=args.seq_len)
+        train_data_loader = DataLoader(train_dataset, batch_size=args.batch_size, num_workers=args.num_workers)
+        print("Load Pre-trained BERT model")
+        cuda_condition = torch.cuda.is_available() and args.with_cuda
+        device = torch.device("cuda:0" if cuda_condition else "cpu")
+        bert = torch.load(args.pretrained_bert_checkpoint, map_location=device)
+        trainer = BERTAttention(bert, vocab_obj, train_dataloader = train_data_loader, workspace_name = args.workspace_name, code=args.code, finetune_task = args.finetune_task)
+        trainer.getAttention()
+        
+    elif args.embeddings:
+        print("Get embeddings... and cluster... ")
+        train_dataset = TokenizerDataset(args.test_dataset_path, None, vocab_obj, seq_len=args.seq_len)
+        train_data_loader = DataLoader(train_dataset, batch_size=args.batch_size, num_workers=args.num_workers)
+        print("Load Pre-trained BERT model")
+        cuda_condition = torch.cuda.is_available() and args.with_cuda
+        device = torch.device("cuda:0" if cuda_condition else "cpu")
+        bert = torch.load(args.pretrained_bert_checkpoint).to(device)
+        available_gpus = list(range(torch.cuda.device_count()))
+        if torch.cuda.device_count() > 1:
+            print("Using %d GPUS for BERT" % torch.cuda.device_count())
+            bert = nn.DataParallel(bert, device_ids=available_gpus)
+
+        data_iter = tqdm.tqdm(enumerate(train_data_loader), 
+                              desc="Model: %s" % (args.pretrained_bert_checkpoint.split("/")[-1]), 
+                              total=len(train_data_loader), bar_format="{l_bar}{r_bar}")
+        all_embeddings = []
+        for i, data in data_iter:
+            data = {key: value.to(device) for key, value in data.items()}
+            embedding = bert(data["input"], data["segment_label"])
+            # print(embedding.shape, embedding[:, 0].shape)
+            embeddings = [h for h in embedding[:,0].cpu().detach().numpy()]
+            all_embeddings.extend(embeddings)
+            
+        new_emb_folder = f"{args.workspace_name}/embeddings"
+        if not os.path.exists(new_emb_folder):
+            os.makedirs(new_emb_folder)
+        pickle.dump(all_embeddings, open(f"{new_emb_folder}/{args.embeddings_file_name}.pkl", "wb"))
+    else:
+        if args.pretrain:
+            print("Pre-training......")
+            print("Loading Pretraining Train Dataset", args.train_dataset_path)
+            print(f"Workspace: {args.workspace_name}")
+            pretrain_dataset = PretrainerDataset(args.train_dataset_path, vocab_obj, seq_len=args.seq_len, max_mask = args.max_mask)
+
+            print("Loading Pretraining Validation Dataset", args.val_dataset_path)
+            pretrain_valid_dataset = PretrainerDataset(args.val_dataset_path, vocab_obj, seq_len=args.seq_len, max_mask = args.max_mask) \
+                if args.val_dataset_path is not None else None
+            
+            print("Loading Pretraining Test Dataset", args.test_dataset_path)
+            pretrain_test_dataset = PretrainerDataset(args.test_dataset_path, vocab_obj, seq_len=args.seq_len, max_mask = args.max_mask) \
+                if args.test_dataset_path is not None else None
+
+            print("Creating Dataloader")
+            pretrain_data_loader = DataLoader(pretrain_dataset, batch_size=args.batch_size, num_workers=args.num_workers)
+            pretrain_val_data_loader = DataLoader(pretrain_valid_dataset, batch_size=args.batch_size, num_workers=args.num_workers)\
+                if pretrain_valid_dataset is not None else None
+            pretrain_test_data_loader = DataLoader(pretrain_test_dataset, batch_size=args.batch_size, num_workers=args.num_workers)\
+                if pretrain_test_dataset is not None else None
+            
+            print("Building BERT model")
+            bert = BERT(len(vocab_obj.vocab), hidden=args.hidden, n_layers=args.layers, attn_heads=args.attn_heads, dropout=args.dropout)
+
+            if args.pretrained_bert_checkpoint:
+                print(f"BERT model : {args.pretrained_bert_checkpoint}")
+                bert = torch.load(args.pretrained_bert_checkpoint)
+            
+            new_log_folder = f"{args.workspace_name}/logs"
+            new_output_folder = f"{args.workspace_name}/output"
+            if args.code: # is sent almost all the time
+                new_log_folder = f"{args.workspace_name}/logs/{args.code}"
+                new_output_folder = f"{args.workspace_name}/output/{args.code}"
+                            
+            if not os.path.exists(new_log_folder):
+                os.makedirs(new_log_folder)
+            if not os.path.exists(new_output_folder):
+                os.makedirs(new_output_folder)
+                
+            print(f"Creating BERT Trainer .... masking: True, max_mask: {args.max_mask}")
+            trainer = BERTTrainer(bert, len(vocab_obj.vocab), train_dataloader=pretrain_data_loader, 
+                                  val_dataloader=pretrain_val_data_loader, test_dataloader=pretrain_test_data_loader,
+                                  lr=args.lr, betas=(args.adam_beta1, args.adam_beta2), weight_decay=args.adam_weight_decay, 
+                                  with_cuda=args.with_cuda, cuda_devices=args.cuda_devices, log_freq=args.log_freq, 
+                                  log_folder_path=new_log_folder)
+
+            start_time = time.time()
+            print(f'Pretraining Starts, Time: {time.strftime("%D %T", time.localtime(start_time))}')
+            # if need to pretrain from a check-point, need :check_epoch
+            repoch = range(args.check_epoch, args.epochs) if args.check_epoch else range(args.epochs)
+            counter = 0
+            patience = 20
+            for epoch in repoch:
+                print(f'Training Epoch {epoch} Starts, Time: {time.strftime("%D %T", time.localtime(time.time()))}')
+                trainer.train(epoch)
+                print(f'Training Epoch {epoch} Ends, Time: {time.strftime("%D %T", time.localtime(time.time()))} \n')
+
+                if pretrain_val_data_loader is not None:
+                    print(f'Validation Epoch {epoch} Starts, Time: {time.strftime("%D %T", time.localtime(time.time()))}')
+                    trainer.val(epoch)
+                    print(f'Validation Epoch {epoch} Ends, Time: {time.strftime("%D %T", time.localtime(time.time()))} \n')
+
+                if trainer.save_model: #  or epoch%10 == 0 and epoch > 4  
+                    trainer.save(epoch, args.output_path)
+                    counter = 0
+                    if pretrain_test_data_loader is not None:
+                        print(f'Test Epoch {epoch} Starts, Time: {time.strftime("%D %T", time.localtime(time.time()))}')
+                        trainer.test(epoch)
+                        print(f'Test Epoch {epoch} Ends, Time: {time.strftime("%D %T", time.localtime(time.time()))} \n')
+                else:
+                    counter +=1
+                    if counter >= patience:
+                        print(f"Early stopping at epoch {epoch}")
+                        break
+
+            end_time = time.time()
+            print("Time Taken to pretrain model = ", end_time - start_time)
+            print(f'Pretraining Ends, Time: {time.strftime("%D %T", time.localtime(end_time))}')
+        else:
+            print("Fine Tuning......")
+            print("Loading Train Dataset", args.train_dataset_path)            
+            train_dataset = TokenizerDataset(args.train_dataset_path, args.train_label_path, vocab_obj, seq_len=args.seq_len)
+
+#             print("Loading Validation Dataset", args.val_dataset_path)            
+#             val_dataset = TokenizerDataset(args.val_dataset_path, args.val_label_path, vocab_obj, seq_len=args.seq_len) \
+#                 if args.val_dataset_path is not None else None
+            
+            print("Loading Test Dataset", args.test_dataset_path)
+            test_dataset = TokenizerDataset(args.test_dataset_path, args.test_label_path, vocab_obj, seq_len=args.seq_len) \
+                if args.test_dataset_path is not None else None
+
+            print("Creating Dataloader...")
+            train_data_loader = DataLoader(train_dataset, batch_size=args.batch_size, num_workers=args.num_workers)
+            # val_data_loader = DataLoader(val_dataset, batch_size=args.batch_size, num_workers=args.num_workers) \
+            #     if val_dataset is not None else None
+            test_data_loader = DataLoader(test_dataset, batch_size=args.batch_size, num_workers=args.num_workers) \
+                if test_dataset is not None else None
+
+            print("Load Pre-trained BERT model")
+            # bert = BERT(len(vocab_obj.vocab), hidden=args.hidden, n_layers=args.layers, attn_heads=args.attn_heads)
+            cuda_condition = torch.cuda.is_available() and args.with_cuda
+            device = torch.device("cuda:0" if cuda_condition else "cpu")
+            bert = torch.load(args.pretrained_bert_checkpoint, map_location=device)
+
+    #         if args.finetune_task == "SL":
+    #             if args.workspace_name == "ratio_proportion_change4":
+    #                 num_labels = 9
+    #             elif args.workspace_name == "ratio_proportion_change3":
+    #                 num_labels = 9
+    #             elif args.workspace_name == "scale_drawings_3":
+    #                 num_labels = 9
+    #             elif args.workspace_name == "sales_tax_discounts_two_rates":
+    #                 num_labels = 3
+    #         else:
+            # num_labels = 2
+    #         # num_labels = 1
+            # print(f"Number of Labels : {args.num_labels}")
+            new_log_folder = f"{args.workspace_name}/logs"
+            new_output_folder = f"{args.workspace_name}/output"
+            if args.finetune_task: # is sent almost all the time
+                new_log_folder = f"{args.workspace_name}/logs/{args.finetune_task}"
+                new_output_folder = f"{args.workspace_name}/output/{args.finetune_task}"
+                            
+            if not os.path.exists(new_log_folder):
+                os.makedirs(new_log_folder)
+            if not os.path.exists(new_output_folder):
+                os.makedirs(new_output_folder)
+                
+            print("Creating BERT Fine Tune Trainer")
+            trainer = BERTFineTuneTrainer(bert, len(vocab_obj.vocab), 
+                          train_dataloader=train_data_loader, test_dataloader=test_data_loader, 
+                          lr=args.lr, betas=(args.adam_beta1, args.adam_beta2), weight_decay=args.adam_weight_decay, 
+                          with_cuda=args.with_cuda, cuda_devices = args.cuda_devices, log_freq=args.log_freq, 
+                          workspace_name = args.workspace_name, num_labels=args.num_labels, log_folder_path=new_log_folder)
+
+            print("Fine-tune training Start....")
+            start_time = time.time()
+            repoch = range(args.check_epoch, args.epochs) if args.check_epoch else range(args.epochs)
+            counter = 0
+            patience = 10
+            for epoch in repoch:
+                print(f'Training Epoch {epoch} Starts, Time: {time.strftime("%D %T", time.localtime(time.time()))}')
+                trainer.train(epoch)
+                print(f'Training Epoch {epoch} Ends, Time: {time.strftime("%D %T", time.localtime(time.time()))} \n')
+
+                if test_data_loader is not None:
+                    print(f'Test Epoch {epoch} Starts, Time: {time.strftime("%D %T", time.localtime(time.time()))}')
+                    trainer.test(epoch)
+                    # pickle.dump(trainer.probability_list, open(f"{args.workspace_name}/output/aaai/change4_mid_prob_{epoch}.pkl","wb"))
+                    print(f'Test Epoch {epoch} Ends, Time: {time.strftime("%D %T", time.localtime(time.time()))} \n')
+
+                # if val_data_loader is not None:
+                #     print(f'Validation Epoch {epoch} Starts, Time: {time.strftime("%D %T", time.localtime(time.time()))}')
+                #     trainer.val(epoch)
+                #     print(f'Validation Epoch {epoch} Ends, Time: {time.strftime("%D %T", time.localtime(time.time()))} \n')
+                
+                if trainer.save_model: #  or epoch%10 == 0
+                    trainer.save(epoch, args.output_path)
+                    counter = 0
+                else:
+                    counter +=1
+                    if counter >= patience:
+                        print(f"Early stopping at epoch {epoch}")
+                        break
+                    
+            end_time = time.time()
+            print("Time Taken to fine-tune model = ", end_time - start_time)
+            print(f'Pretraining Ends, Time: {time.strftime("%D %T", time.localtime(end_time))}')
+            
+
+if __name__ == "__main__":
+    train()

metrics.py

@@ -0,0 +1,149 @@
+import numpy as np
+from scipy.special import softmax
+
+
+class CELoss(object):
+
+    def compute_bin_boundaries(self, probabilities = np.array([])):
+
+        #uniform bin spacing
+        if probabilities.size == 0:
+            bin_boundaries = np.linspace(0, 1, self.n_bins + 1)
+            self.bin_lowers = bin_boundaries[:-1]
+            self.bin_uppers = bin_boundaries[1:]
+        else:
+            #size of bins 
+            bin_n = int(self.n_data/self.n_bins)
+
+            bin_boundaries = np.array([])
+
+            probabilities_sort = np.sort(probabilities)  
+
+            for i in range(0,self.n_bins):
+                bin_boundaries = np.append(bin_boundaries,probabilities_sort[i*bin_n])
+            bin_boundaries = np.append(bin_boundaries,1.0)
+
+            self.bin_lowers = bin_boundaries[:-1]
+            self.bin_uppers = bin_boundaries[1:]
+
+
+    def get_probabilities(self, output, labels, logits):
+        #If not probabilities apply softmax!
+        if logits:
+            self.probabilities = softmax(output, axis=1)
+        else:
+            self.probabilities = output
+
+        self.labels = labels
+        self.confidences = np.max(self.probabilities, axis=1)
+        self.predictions = np.argmax(self.probabilities, axis=1)
+        self.accuracies = np.equal(self.predictions,labels)
+
+    def binary_matrices(self):
+        idx = np.arange(self.n_data)
+        #make matrices of zeros
+        pred_matrix = np.zeros([self.n_data,self.n_class])
+        label_matrix = np.zeros([self.n_data,self.n_class])
+        #self.acc_matrix = np.zeros([self.n_data,self.n_class])
+        pred_matrix[idx,self.predictions] = 1
+        label_matrix[idx,self.labels] = 1
+
+        self.acc_matrix = np.equal(pred_matrix, label_matrix)
+
+
+    def compute_bins(self, index = None):
+        self.bin_prop = np.zeros(self.n_bins)
+        self.bin_acc = np.zeros(self.n_bins)
+        self.bin_conf = np.zeros(self.n_bins)
+        self.bin_score = np.zeros(self.n_bins)
+
+        if index == None:
+            confidences = self.confidences
+            accuracies = self.accuracies
+        else:
+            confidences = self.probabilities[:,index]
+            accuracies = self.acc_matrix[:,index]
+
+
+        for i, (bin_lower, bin_upper) in enumerate(zip(self.bin_lowers, self.bin_uppers)):
+            # Calculated |confidence - accuracy| in each bin
+            in_bin = np.greater(confidences,bin_lower.item()) * np.less_equal(confidences,bin_upper.item())
+            self.bin_prop[i] = np.mean(in_bin)
+
+            if self.bin_prop[i].item() > 0:
+                self.bin_acc[i] = np.mean(accuracies[in_bin])
+                self.bin_conf[i] = np.mean(confidences[in_bin])
+                self.bin_score[i] = np.abs(self.bin_conf[i] - self.bin_acc[i])
+
+class MaxProbCELoss(CELoss):
+    def loss(self, output, labels, n_bins = 15, logits = True):
+        self.n_bins = n_bins
+        super().compute_bin_boundaries()
+        super().get_probabilities(output, labels, logits)
+        super().compute_bins()
+
+#http://people.cs.pitt.edu/~milos/research/AAAI_Calibration.pdf
+class ECELoss(MaxProbCELoss):
+
+    def loss(self, output, labels, n_bins = 15, logits = True):
+        super().loss(output, labels, n_bins, logits)
+        return np.dot(self.bin_prop,self.bin_score)
+
+class MCELoss(MaxProbCELoss):
+    
+    def loss(self, output, labels, n_bins = 15, logits = True):
+        super().loss(output, labels, n_bins, logits)
+        return np.max(self.bin_score)
+
+#https://arxiv.org/abs/1905.11001
+#Overconfidence Loss (Good in high risk applications where confident but wrong predictions can be especially harmful)
+class OELoss(MaxProbCELoss):
+
+    def loss(self, output, labels, n_bins = 15, logits = True):
+        super().loss(output, labels, n_bins, logits)
+        return np.dot(self.bin_prop,self.bin_conf * np.maximum(self.bin_conf-self.bin_acc,np.zeros(self.n_bins)))
+
+
+#https://arxiv.org/abs/1904.01685
+class SCELoss(CELoss):
+
+    def loss(self, output, labels, n_bins = 15, logits = True):
+        sce = 0.0
+        self.n_bins = n_bins
+        self.n_data = len(output)
+        self.n_class = len(output[0])
+
+        super().compute_bin_boundaries()
+        super().get_probabilities(output, labels, logits)
+        super().binary_matrices()
+
+        for i in range(self.n_class):
+            super().compute_bins(i)
+            sce += np.dot(self.bin_prop,self.bin_score)
+
+        return sce/self.n_class
+
+class TACELoss(CELoss):
+
+    def loss(self, output, labels, threshold = 0.01, n_bins = 15, logits = True):
+        tace = 0.0
+        self.n_bins = n_bins
+        self.n_data = len(output)
+        self.n_class = len(output[0])
+
+        super().get_probabilities(output, labels, logits)
+        self.probabilities[self.probabilities < threshold] = 0
+        super().binary_matrices()
+
+        for i in range(self.n_class):
+            super().compute_bin_boundaries(self.probabilities[:,i]) 
+            super().compute_bins(i)
+            tace += np.dot(self.bin_prop,self.bin_score)
+
+        return tace/self.n_class
+
+#create TACELoss with threshold fixed at 0
+class ACELoss(TACELoss):
+
+    def loss(self, output, labels, n_bins = 15, logits = True):
+        return super().loss(output, labels, 0.0 , n_bins, logits)

new_fine_tuning/README.md

@@ -0,0 +1,197 @@
+## Pre-training Data
+
+### ratio_proportion_change3 : Calculating Percent Change and Final Amounts
+> clear;python3 prepare_pretraining_input_vocab_file.py -analyze_dataset_by_section True -workspace_name ratio_proportion_change3 -opt_step1 OptionalTask_1 EquationAnswer NumeratorFactor DenominatorFactor -opt_step2 OptionalTask_2 FirstRow1:1 FirstRow1:2 FirstRow2:1 FirstRow2:2 SecondRow ThirdRow -pretrain True -train_file_path pretraining/pretrain1000.txt -train_info_path pretraining/pretrain1000_info.txt -test_file_path pretraining/test1000.txt -test_info_path pretraining/test1000_info.txt
+
+> clear;python3 prepare_pretraining_input_vocab_file.py -workspace_name ratio_proportion_change3 -opt_step1 OptionalTask_1 EquationAnswer NumeratorFactor DenominatorFactor -opt_step2 OptionalTask_2 FirstRow1:1 FirstRow1:2 FirstRow2:1 FirstRow2:2 SecondRow ThirdRow -pretrain True -train_file_path pretraining/pretrain2000.txt -train_info_path pretraining/pretrain2000_info.txt -test_file_path pretraining/test2000.txt -test_info_path pretraining/test2000_info.txt
+
+#### Test simple
+> clear;python3 prepare_pretraining_input_vocab_file.py -workspace_name ratio_proportion_change3 -code full -opt_step1 OptionalTask_1 EquationAnswer NumeratorFactor DenominatorFactor -opt_step2 OptionalTask_2 FirstRow1:1 FirstRow1:2 FirstRow2:1 FirstRow2:2 SecondRow ThirdRow -train_file_path full.txt -train_info_path full_info.txt
+
+> clear;python3 prepare_pretraining_input_vocab_file.py -workspace_name ratio_proportion_change3 -code gt -opt_step1 OptionalTask_1 EquationAnswer NumeratorFactor DenominatorFactor -opt_step2 OptionalTask_2 FirstRow1:1 FirstRow1:2 FirstRow2:1 FirstRow2:2 SecondRow ThirdRow -train_file_path er.txt -train_info_path er_info.txt -test_file_path me.txt -test_info_path me_info.txt
+
+> clear;python3 prepare_pretraining_input_vocab_file.py -workspace_name ratio_proportion_change3 -code correct -opt_step1 OptionalTask_1 EquationAnswer NumeratorFactor DenominatorFactor -opt_step2 OptionalTask_2 FirstRow1:1 FirstRow1:2 FirstRow2:1 FirstRow2:2 SecondRow ThirdRow -train_file_path correct.txt -train_info_path correct_info.txt -test_file_path incorrect.txt -test_info_path incorrect_info.txt -final_step FinalAnswer
+
+> clear;python3 prepare_pretraining_input_vocab_file.py -workspace_name ratio_proportion_change3 -code progress -opt_step1 OptionalTask_1 EquationAnswer NumeratorFactor DenominatorFactor -opt_step2 OptionalTask_2 FirstRow1:1 FirstRow1:2 FirstRow2:1 FirstRow2:2 SecondRow ThirdRow -train_file_path graduated.txt -train_info_path graduated_info.txt -test_file_path promoted.txt -test_info_path promoted_info.txt 
+
+### ratio_proportion_change4 : Using Percents and Percent Change
+> clear;python3 prepare_pretraining_input_vocab_file.py -analyze_dataset_by_section True -workspace_name ratio_proportion_change4 -opt_step1 OptionalTask_1 EquationAnswer NumeratorFactor DenominatorFactor NumeratorLabel1 DenominatorLabel1 -opt_step2 OptionalTask_2 FirstRow1:1 FirstRow1:2 FirstRow2:1 FirstRow2:2 SecondRow ThirdRow -pretrain True -train_file_path pretraining/pretrain1000.txt -train_info_path pretraining/pretrain1000_info.txt -test_file_path pretraining/test1000.txt -test_info_path pretraining/test1000_info.txt
+
+> clear;python3 prepare_pretraining_input_vocab_file.py -workspace_name ratio_proportion_change4 -opt_step1 OptionalTask_1 EquationAnswer NumeratorFactor DenominatorFactor NumeratorLabel1 DenominatorLabel1 -opt_step2 OptionalTask_2 FirstRow1:1 FirstRow1:2 FirstRow2:1 FirstRow2:2 SecondRow ThirdRow -pretrain True -train_file_path pretraining/pretrain2000.txt -train_info_path pretraining/pretrain2000_info.txt -test_file_path pretraining/test2000.txt -test_info_path pretraining/test2000_info.txt
+
+#### Test simple
+> clear;python3 prepare_pretraining_input_vocab_file.py -workspace_name ratio_proportion_change4 -code full -opt_step1 OptionalTask_1 EquationAnswer NumeratorFactor DenominatorFactor -opt_step2 OptionalTask_2 FirstRow1:1 FirstRow1:2 FirstRow2:1 FirstRow2:2 SecondRow ThirdRow -train_file_path full.txt -train_info_path full_info.txt
+
+> clear;python3 prepare_pretraining_input_vocab_file.py -workspace_name ratio_proportion_change4 -code gt -opt_step1 OptionalTask_1 EquationAnswer NumeratorFactor DenominatorFactor -opt_step2 OptionalTask_2 FirstRow1:1 FirstRow1:2 FirstRow2:1 FirstRow2:2 SecondRow ThirdRow -train_file_path er.txt -train_info_path er_info.txt -test_file_path me.txt -test_info_path me_info.txt
+
+> clear;python3 prepare_pretraining_input_vocab_file.py -workspace_name ratio_proportion_change4 -code correct -opt_step1 OptionalTask_1 EquationAnswer NumeratorFactor DenominatorFactor -opt_step2 OptionalTask_2 FirstRow1:1 FirstRow1:2 FirstRow2:1 FirstRow2:2 SecondRow ThirdRow -train_file_path correct.txt -train_info_path correct_info.txt -test_file_path incorrect.txt -test_info_path incorrect_info.txt -final_step FinalAnswer
+
+> clear;python3 prepare_pretraining_input_vocab_file.py -workspace_name ratio_proportion_change4 -code progress -opt_step1 OptionalTask_1 EquationAnswer NumeratorFactor DenominatorFactor -opt_step2 OptionalTask_2 FirstRow1:1 FirstRow1:2 FirstRow2:1 FirstRow2:2 SecondRow ThirdRow -train_file_path graduated.txt -train_info_path graduated_info.txt -test_file_path promoted.txt -test_info_path promoted_info.txt 
+
+## Pretraining
+
+### ratio_proportion_change3 : Calculating Percent Change and Final Amounts
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3_1920 -code pretrain1000 --pretrain_dataset pretraining/pretrain1000.txt --pretrain_val_dataset pretraining/test1000.txt
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000 --pretrain_dataset pretraining/pretrain2000.txt --pretrain_val_dataset pretraining/test2000.txt
+
+#### Test simple models
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l1h-5lr --pretrain_dataset pretraining/pretrain2000.txt --pretrain_val_dataset pretraining/test2000.txt --layers 1 --attn_heads 1
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l2h-5lr --pretrain_dataset pretraining/pretrain2000.txt --pretrain_val_dataset pretraining/test2000.txt --layers 1 --attn_heads 2
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_2l2h-5lr --pretrain_dataset pretraining/pretrain2000.txt --pretrain_val_dataset pretraining/test2000.txt --layers 2 --attn_heads 2
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_2l4h-5lr --pretrain_dataset pretraining/pretrain2000.txt --pretrain_val_dataset pretraining/test2000.txt --layers 2 --attn_heads 4
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l4h-5lr --pretrain_dataset pretraining/pretrain2000.txt --pretrain_val_dataset pretraining/test2000.txt --layers 4 --attn_heads 4
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l8h-5lr --pretrain_dataset pretraining/pretrain2000.txt --pretrain_val_dataset pretraining/test2000.txt --layers 4 --attn_heads 8
+
+
+
+### ratio_proportion_change4 : Using Percents and Percent Change
+> clear;python3 src/main.py -workspace_name ratio_proportion_change4 -code pretrain1000 --pretrain_dataset pretraining/pretrain1000.txt --pretrain_val_dataset pretraining/test1000.txt
+> clear;python3 src/main.py -workspace_name ratio_proportion_change4 -code pretrain2000 --pretrain_dataset pretraining/pretrain2000.txt --pretrain_val_dataset pretraining/test2000.txt
+
+#### Test simple models
+> clear;python3 src/main.py -workspace_name ratio_proportion_change4 -code pretrain2000_1l1h-5lr --pretrain_dataset pretraining/pretrain2000.txt --pretrain_val_dataset pretraining/test2000.txt --layers 1 --attn_heads 1
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change4 -code pretrain2000_1l2h-5lr --pretrain_dataset pretraining/pretrain2000.txt --pretrain_val_dataset pretraining/test2000.txt --layers 1 --attn_heads 2
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change4 -code pretrain2000_2l2h-5lr --pretrain_dataset pretraining/pretrain2000.txt --pretrain_val_dataset pretraining/test2000.txt --layers 2 --attn_heads 2
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change4 -code pretrain2000_2l4h-5lr --pretrain_dataset pretraining/pretrain2000.txt --pretrain_val_dataset pretraining/test2000.txt --layers 2 --attn_heads 4
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change4 -code pretrain2000_4l4h-5lr --pretrain_dataset pretraining/pretrain2000.txt --pretrain_val_dataset pretraining/test2000.txt --layers 4 --attn_heads 4
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change4 -code pretrain2000_4l8h-5lr --pretrain_dataset pretraining/pretrain2000.txt --pretrain_val_dataset pretraining/test2000.txt --layers 4 --attn_heads 8
+
+
+## Preparing Fine Tuning Data
+
+### ratio_proportion_change3 : Calculating Percent Change and Final Amounts
+> clear;python3 prepare_pretraining_input_vocab_file.py -workspace_name ratio_proportion_change3 -opt_step1 OptionalTask_1 EquationAnswer NumeratorFactor DenominatorFactor -opt_step2 OptionalTask_2 FirstRow1:1 FirstRow1:2 FirstRow2:1 FirstRow2:2 SecondRow ThirdRow -final_step FinalAnswer
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -finetune_task check2 --train_dataset finetuning/check2/train.txt --test_dataset finetuning/check2/test.txt --train_label finetuning/check2/train_label.txt --test_label finetuning/check2/test_label.txt --pretrained_bert_checkpoint ratio_proportion_change3/output/bert_trained.seq_encoder.model.ep279 --epochs 51
+
+#### Attention Head Check
+<!-- > PercentChange	NumeratorQuantity2	NumeratorQuantity1	DenominatorQuantity1	OptionalTask_1	EquationAnswer	NumeratorFactor	EquationAnswer	NumeratorFactor	EquationAnswer	NumeratorFactor	DenominatorFactor	NumeratorFactor	DenominatorFactor	NumeratorFactor	DenominatorFactor	FirstRow1:2	FirstRow1:1	FirstRow2:1	FirstRow2:2	FirstRow2:1	SecondRow	ThirdRow	FinalAnswerDirection	ThirdRow	FinalAnswer -->
+
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l1h-5lr --train_dataset full/full.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_1l1h-5lr/bert_trained.seq_encoder.model.ep598 --attention True -finetune_task full;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l1h-5lr --train_dataset gt/er.txt --pretrained_bert_checkpoint ratio_proportion_change3/output/pretrain2000_1l1h-5lr/bert_trained.seq_encoder.model.ep598 --attention True -finetune_task er ;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l1h-5lr --train_dataset gt/me.txt --pretrained_bert_checkpoint ratio_proportion_change3/output/pretrain2000_1l1h-5lr/bert_trained.seq_encoder.model.ep598 --attention True -finetune_task me;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l1h-5lr --train_dataset correct/correct.txt --pretrained_bert_checkpoint ratio_proportion_change3/output/pretrain2000_1l1h-5lr/bert_trained.seq_encoder.model.ep598 --attention True -finetune_task correct ;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l1h-5lr --train_dataset correct/incorrect.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_1l1h-5lr/bert_trained.seq_encoder.model.ep598 --attention True -finetune_task incorrect;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l1h-5lr --train_dataset progress/graduated.txt --pretrained_bert_checkpoint ratio_proportion_change3/output/pretrain2000_1l1h-5lr/bert_trained.seq_encoder.model.ep598 --attention True -finetune_task graduated;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l1h-5lr --train_dataset progress/promoted.txt --pretrained_bert_checkpoint ratio_proportion_change3/output/pretrain2000_1l1h-5lr/bert_trained.seq_encoder.model.ep598 --attention True -finetune_task promoted 
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l2h-5lr --train_dataset full/full.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_1l2h-5lr/bert_trained.seq_encoder.model.ep823 --attention True -finetune_task full;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l2h-5lr --train_dataset gt/er.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_1l2h-5lr/bert_trained.seq_encoder.model.ep823 --attention True -finetune_task er;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l2h-5lr --train_dataset gt/me.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_1l2h-5lr/bert_trained.seq_encoder.model.ep823 --attention True -finetune_task me;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l2h-5lr --train_dataset correct/correct.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_1l2h-5lr/bert_trained.seq_encoder.model.ep823 --attention True -finetune_task correct;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l2h-5lr --train_dataset correct/incorrect.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_1l2h-5lr/bert_trained.seq_encoder.model.ep823 --attention True -finetune_task incorrect;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l2h-5lr --train_dataset progress/graduated.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_1l2h-5lr/bert_trained.seq_encoder.model.ep823 --attention True -finetune_task graduated;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l2h-5lr --train_dataset progress/promoted.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_1l2h-5lr/bert_trained.seq_encoder.model.ep823 --attention True -finetune_task promoted 
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_2l2h-5lr --train_dataset full/full.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_2l2h-5lr/bert_trained.seq_encoder.model.ep1045 --attention True -finetune_task full;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_2l2h-5lr --train_dataset gt/er.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_2l2h-5lr/bert_trained.seq_encoder.model.ep1045 --attention True -finetune_task er;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_2l2h-5lr --train_dataset gt/me.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_2l2h-5lr/bert_trained.seq_encoder.model.ep1045 --attention True -finetune_task me;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_2l2h-5lr --train_dataset correct/correct.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_2l2h-5lr/bert_trained.seq_encoder.model.ep1045 --attention True -finetune_task correct;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_2l2h-5lr --train_dataset correct/incorrect.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_2l2h-5lr/bert_trained.seq_encoder.model.ep1045 --attention True -finetune_task incorrect;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_2l2h-5lr --train_dataset progress/graduated.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_2l2h-5lr/bert_trained.seq_encoder.model.ep1045 --attention True -finetune_task graduated;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_2l2h-5lr --train_dataset progress/promoted.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_2l2h-5lr/bert_trained.seq_encoder.model.ep1045 --attention True -finetune_task promoted
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_2l4h-5lr --train_dataset full/full.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_2l4h-5lr/bert_trained.seq_encoder.model.ep1336 --attention True -finetune_task full;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_2l4h-5lr --train_dataset gt/er.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_2l4h-5lr/bert_trained.seq_encoder.model.ep1336 --attention True -finetune_task er;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_2l4h-5lr --train_dataset gt/me.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_2l4h-5lr/bert_trained.seq_encoder.model.ep1336 --attention True -finetune_task me;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_2l4h-5lr --train_dataset correct/correct.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_2l4h-5lr/bert_trained.seq_encoder.model.ep1336 --attention True -finetune_task correct;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_2l4h-5lr --train_dataset correct/incorrect.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_2l4h-5lr/bert_trained.seq_encoder.model.ep1336 --attention True -finetune_task incorrect;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_2l4h-5lr --train_dataset progress/graduated.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_2l4h-5lr/bert_trained.seq_encoder.model.ep1336 --attention True -finetune_task graduated;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_2l4h-5lr --train_dataset progress/promoted.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_2l4h-5lr/bert_trained.seq_encoder.model.ep1336 --attention True -finetune_task promoted
+
+<!-- > clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l4h-5lr --train_dataset full/full.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_4l4h-5lr/bert_trained.seq_encoder.model.ep923 --attention True -->
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l4h-5lr --train_dataset full/full.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_4l4h-5lr/bert_trained.seq_encoder.model.ep871 --attention True -finetune_task full;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l4h-5lr --train_dataset gt/er.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_4l4h-5lr/bert_trained.seq_encoder.model.ep871 --attention True -finetune_task er;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l4h-5lr --train_dataset gt/me.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_4l4h-5lr/bert_trained.seq_encoder.model.ep871 --attention True -finetune_task me;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l4h-5lr --train_dataset correct/correct.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_4l4h-5lr/bert_trained.seq_encoder.model.ep871 --attention True -finetune_task correct;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l4h-5lr --train_dataset correct/incorrect.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_4l4h-5lr/bert_trained.seq_encoder.model.ep871 --attention True -finetune_task incorrect;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l4h-5lr --train_dataset progress/graduated.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_4l4h-5lr/bert_trained.seq_encoder.model.ep871 --attention True -finetune_task graduated;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l4h-5lr --train_dataset progress/promoted.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_4l4h-5lr/bert_trained.seq_encoder.model.ep871 --attention True -finetune_task promoted
+
+clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l8h-5lr --train_dataset full/full_attn.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_4l8h-5lr/bert_trained.seq_encoder.model.ep1349 --attention True -finetune_task full
+
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l8h-5lr --train_dataset full/full.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_4l8h-5lr/bert_trained.seq_encoder.model.ep1349 --attention True -finetune_task full;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l8h-5lr --train_dataset gt/er.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_4l8h-5lr/bert_trained.seq_encoder.model.ep1349 --attention True -finetune_task er;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l8h-5lr --train_dataset gt/me.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_4l8h-5lr/bert_trained.seq_encoder.model.ep1349 --attention True -finetune_task me;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l8h-5lr --train_dataset correct/correct.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_4l8h-5lr/bert_trained.seq_encoder.model.ep1349 --attention True -finetune_task correct;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l8h-5lr --train_dataset correct/incorrect.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_4l8h-5lr/bert_trained.seq_encoder.model.ep1349 --attention True -finetune_task incorrect;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l8h-5lr --train_dataset progress/graduated.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_4l8h-5lr/bert_trained.seq_encoder.model.ep1349 --attention True -finetune_task graduated;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_4l8h-5lr --train_dataset progress/promoted.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_4l8h-5lr/bert_trained.seq_encoder.model.ep1349 --attention True -finetune_task promoted
+
+
+<!-- PercentChange	NumeratorQuantity2	NumeratorQuantity1	DenominatorQuantity1	OptionalTask_2	FirstRow2:1	FirstRow2:2	FirstRow1:1	SecondRow	ThirdRow	FinalAnswer	FinalAnswerDirection --> me
+
+<!-- PercentChange	NumeratorQuantity2	NumeratorQuantity1	DenominatorQuantity1	OptionalTask_1	DenominatorFactor	NumeratorFactor	OptionalTask_2	EquationAnswer	FirstRow1:1	FirstRow1:2	FirstRow2:2	FirstRow2:1	FirstRow1:2	SecondRow	ThirdRow	FinalAnswer --> er
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l1h-5lr --train_dataset pretraining/attention_train.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_1l1h-5lr/bert_trained.seq_encoder.model.ep273 --attention True
+
+<!-- PercentChange	NumeratorQuantity2	NumeratorQuantity1	DenominatorQuantity1	OptionalTask_1	DenominatorFactor	NumeratorFactor	OptionalTask_2	EquationAnswer	FirstRow1:1	FirstRow1:2	FirstRow2:2	FirstRow2:1	FirstRow1:2	SecondRow	ThirdRow	FinalAnswer -->
+
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -code pretrain2000_1l2h-5lr --train_dataset pretraining/attention_train.txt --pretrained_bert_checkpoint  ratio_proportion_change3/output/pretrain2000_1l2h-5lr/bert_trained.seq_encoder.model.ep1021 --attention True
+
+
+
+### ratio_proportion_change4 : Using Percents and Percent Change
+> clear;python3 prepare_pretraining_input_vocab_file.py -workspace_name ratio_proportion_change4 -opt_step1 OptionalTask_1 EquationAnswer NumeratorFactor DenominatorFactor NumeratorLabel1 DenominatorLabel1 -opt_step2 OptionalTask_2 FirstRow1:1 FirstRow1:2 FirstRow2:1 FirstRow2:2 SecondRow ThirdRow -final_step FinalAnswer
+
+### scale_drawings_3 : Calculating Measurements Using a Scale
+> clear;python3 prepare_pretraining_input_vocab_file.py -workspace_name scale_drawings_3 -opt_step1 opt1-check opt1-ratio-L-n opt1-ratio-L-d opt1-ratio-R-n opt1-ratio-R-d opt1-me2-top-3 opt1-me2-top-4 opt1-me2-top-2 opt1-me2-top-1 opt1-me2-middle-1 opt1-me2-bottom-1 -opt_step2 opt2-check opt2-ratio-L-n opt2-ratio-L-d opt2-ratio-R-n opt2-ratio-R-d opt2-me2-top-3 opt2-me2-top-4 opt2-me2-top-1 opt2-me2-top-2 opt2-me2-middle-1 opt2-me2-bottom-1 -final_step unk-value1 unk-value2
+
+### sales_tax_discounts_two_rates : Solving Problems with Both Sales Tax and Discounts
+> clear;python3 prepare_pretraining_input_vocab_file.py -workspace_name sales_tax_discounts_two_rates -opt_step1 optionalTaskGn salestaxFactor2 discountFactor2 multiplyOrderStatementGn -final_step totalCost1
+
+
+# Fine Tuning Pre-trained model
+
+## ratio_proportion_change3 : Calculating Percent Change and Final Amounts
+> Selected Pretrained model: **ratio_proportion_change3/output/bert_trained.seq_encoder.model.ep279**
+> New **bert/ratio_proportion_change3/output/pretrain2000/bert_trained.seq_encoder.model.ep731**
+
+### 10per
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -finetune_task 10per --train_dataset finetuning/10per/train.txt --test_dataset finetuning/10per/test.txt --train_label finetuning/10per/train_label.txt --test_label finetuning/10per/test_label.txt --pretrained_bert_checkpoint ratio_proportion_change3/output/pretrain2000/bert_trained.seq_encoder.model.ep731 --epochs 51
+
+### IS
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -finetune_task IS --train_dataset finetuning/IS/train.txt --test_dataset finetuning/FS/train.txt --train_label finetuning/IS/train_label.txt --test_label finetuning/FS/train_label.txt --pretrained_bert_checkpoint ratio_proportion_change3/output/pretrain2000/bert_trained.seq_encoder.model.ep731 --epochs 51
+
+### FS
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -finetune_task FS --train_dataset finetuning/FS/train.txt --test_dataset finetuning/IS/train.txt --train_label finetuning/FS/train_label.txt --test_label finetuning/IS/train_label.txt --pretrained_bert_checkpoint ratio_proportion_change3/output/pretrain2000/bert_trained.seq_encoder.model.ep731 --epochs 51
+ 
+### correctness
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -finetune_task correctness --train_dataset finetuning/correctness/train.txt --test_dataset finetuning/correctness/test.txt --train_label finetuning/correctness/train_label.txt --test_label finetuning/correctness/test_label.txt --pretrained_bert_checkpoint ratio_proportion_change3/output/bert_trained.seq_encoder.model.ep279 --epochs 51
+
+### SL
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -finetune_task SL --train_dataset finetuning/SL/train.txt --test_dataset finetuning/SL/test.txt --train_label finetuning/SL/train_label.txt --test_label finetuning/SL/test_label.txt --pretrained_bert_checkpoint ratio_proportion_change3/output/bert_trained.seq_encoder.model.ep279 --epochs 51
+
+### effectiveness
+> clear;python3 src/main.py -workspace_name ratio_proportion_change3 -finetune_task effectiveness --train_dataset finetuning/effectiveness/train.txt --test_dataset finetuning/effectiveness/test.txt --train_label finetuning/effectiveness/train_label.txt --test_label finetuning/effectiveness/test_label.txt --pretrained_bert_checkpoint ratio_proportion_change3/output/bert_trained.seq_encoder.model.ep279 --epochs 51
+
+
+## ratio_proportion_change4 : Using Percents and Percent Change
+> Selected Pretrained model: **ratio_proportion_change4/output/bert_trained.seq_encoder.model.ep287**
+### 10per
+> clear;python3 src/main.py -workspace_name ratio_proportion_change4 -finetune_task 10per --train_dataset finetuning/10per/train.txt --test_dataset finetuning/10per/test.txt --train_label finetuning/10per/train_label.txt --test_label finetuning/10per/test_label.txt --pretrained_bert_checkpoint ratio_proportion_change4/output/bert_trained.seq_encoder.model.ep287 --epochs 51
+
+### IS
+
+### FS
+
+### correctness
+> clear;python3 src/main.py -workspace_name ratio_proportion_change4 -finetune_task correctness --train_dataset finetuning/correctness/train.txt --test_dataset finetuning/correctness/test.txt --train_label finetuning/correctness/train_label.txt --test_label finetuning/correctness/test_label.txt --pretrained_bert_checkpoint ratio_proportion_change4/output/bert_trained.seq_encoder.model.ep287 --epochs 51
+
+### SL
+> clear;python3 src/main.py -workspace_name ratio_proportion_change4 -finetune_task SL --train_dataset finetuning/SL/train.txt --test_dataset finetuning/SL/test.txt --train_label finetuning/SL/train_label.txt --test_label finetuning/SL/test_label.txt --pretrained_bert_checkpoint ratio_proportion_change4/output/bert_trained.seq_encoder.model.ep287 --epochs 51
+
+### effectiveness
+> clear;python3 src/main.py -workspace_name ratio_proportion_change4 -finetune_task effectiveness --train_dataset finetuning/effectiveness/train.txt --test_dataset finetuning/effectiveness/test.txt --train_label finetuning/effectiveness/train_label.txt --test_label finetuning/effectiveness/test_label.txt --pretrained_bert_checkpoint ratio_proportion_change4/output/bert_trained.seq_encoder.model.ep287 --epochs 51
+
+
+## scale_drawings_3 : Calculating Measurements Using a Scale
+> Selected Pretrained model: **scale_drawings_3/output/bert_trained.seq_encoder.model.ep252**
+### 10per
+> clear;python3 src/main.py -workspace_name scale_drawings_3 -finetune_task 10per --train_dataset finetuning/10per/train.txt --test_dataset finetuning/10per/test.txt --train_label finetuning/10per/train_label.txt --test_label finetuning/10per/test_label.txt --pretrained_bert_checkpoint scale_drawings_3/output/bert_trained.seq_encoder.model.ep252 --epochs 51
+
+### IS
+
+### FS
+
+### correctness
+> clear;python3 src/main.py -workspace_name scale_drawings_3 -finetune_task correctness --train_dataset finetuning/correctness/train.txt --test_dataset finetuning/correctness/test.txt --train_label finetuning/correctness/train_label.txt --test_label finetuning/correctness/test_label.txt --pretrained_bert_checkpoint scale_drawings_3/output/bert_trained.seq_encoder.model.ep252 --epochs 51
+
+### SL
+> clear;python3 src/main.py -workspace_name scale_drawings_3 -finetune_task SL --train_dataset finetuning/SL/train.txt --test_dataset finetuning/SL/test.txt --train_label finetuning/SL/train_label.txt --test_label finetuning/SL/test_label.txt --pretrained_bert_checkpoint scale_drawings_3/output/bert_trained.seq_encoder.model.ep252 --epochs 51
+
+### effectiveness
+
+## sales_tax_discounts_two_rates : Solving Problems with Both Sales Tax and Discounts
+> Selected Pretrained model: **sales_tax_discounts_two_rates/output/bert_trained.seq_encoder.model.ep255**
+
+### 10per
+> clear;python3 src/main.py -workspace_name sales_tax_discounts_two_rates -finetune_task 10per --train_dataset finetuning/10per/train.txt --test_dataset finetuning/10per/test.txt --train_label finetuning/10per/train_label.txt --test_label finetuning/10per/test_label.txt --pretrained_bert_checkpoint sales_tax_discounts_two_rates/output/bert_trained.seq_encoder.model.ep255 --epochs 51
+
+### IS
+
+### FS
+
+### correctness
+> clear;python3 src/main.py -workspace_name sales_tax_discounts_two_rates -finetune_task correctness --train_dataset finetuning/correctness/train.txt --test_dataset finetuning/correctness/test.txt --train_label finetuning/correctness/train_label.txt --test_label finetuning/correctness/test_label.txt --pretrained_bert_checkpoint sales_tax_discounts_two_rates/output/bert_trained.seq_encoder.model.ep255 --epochs 51
+
+### SL
+
+### effectiveness

new_hint_fine_tuned.py

@@ -0,0 +1,131 @@
+import argparse
+import os
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, random_split, TensorDataset
+from src.dataset import TokenizerDataset
+from src.bert import BERT
+from src.pretrainer import BERTFineTuneTrainer1
+from src.vocab import Vocab
+import pandas as pd
+
+def preprocess_labels(label_csv_path):
+    try:
+        labels_df = pd.read_csv(label_csv_path)
+        labels = labels_df['last_hint_class'].values.astype(int)
+        return torch.tensor(labels, dtype=torch.long)
+    except Exception as e:
+        print(f"Error reading dataset file: {e}")
+        return None
+
+def preprocess_data(data_path, vocab, max_length=128):
+    try:
+        with open(data_path, 'r') as f:
+            sequences = f.readlines()
+    except Exception as e:
+        print(f"Error reading data file: {e}")
+        return None, None
+
+    tokenized_sequences = []
+    for sequence in sequences:
+        sequence = sequence.strip()
+        if sequence:
+            encoded = vocab.to_seq(sequence, seq_len=max_length)
+            encoded = encoded[:max_length] + [vocab.vocab.get('[PAD]', 0)] * (max_length - len(encoded))
+            segment_label = [0] * max_length
+
+            tokenized_sequences.append({
+                'input_ids': torch.tensor(encoded),
+                'segment_label': torch.tensor(segment_label)
+            })
+
+    input_ids = torch.cat([t['input_ids'].unsqueeze(0) for t in tokenized_sequences], dim=0)
+    segment_labels = torch.cat([t['segment_label'].unsqueeze(0) for t in tokenized_sequences], dim=0)
+
+    print(f"Input IDs shape: {input_ids.shape}")
+    print(f"Segment labels shape: {segment_labels.shape}")
+
+    return input_ids, segment_labels
+
+def custom_collate_fn(batch):
+    inputs = [item['input_ids'].unsqueeze(0) for item in batch]
+    labels = [item['label'].unsqueeze(0) for item in batch]
+    segment_labels = [item['segment_label'].unsqueeze(0) for item in batch]
+
+    inputs = torch.cat(inputs, dim=0)
+    labels = torch.cat(labels, dim=0)
+    segment_labels = torch.cat(segment_labels, dim=0)
+
+    return {
+        'input': inputs,
+        'label': labels,
+        'segment_label': segment_labels
+    }
+
+def main(opt):
+    # Set device to GPU if available, otherwise use CPU
+    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+    # Load vocabulary
+    vocab = Vocab(opt.vocab_file)
+    vocab.load_vocab()
+
+    # Preprocess data and labels
+    input_ids, segment_labels = preprocess_data(opt.data_path, vocab, max_length=50)  # Using sequence length 50
+    labels = preprocess_labels(opt.dataset)
+
+    if input_ids is None or segment_labels is None or labels is None:
+        print("Error in preprocessing data. Exiting.")
+        return
+
+    # Create TensorDataset and split into train and validation sets
+    dataset = TensorDataset(input_ids, segment_labels, labels)
+    val_size = len(dataset) - int(0.8 * len(dataset))
+    val_dataset, train_dataset = random_split(dataset, [val_size, len(dataset) - val_size])
+
+    # Create DataLoaders for training and validation
+    train_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True, collate_fn=custom_collate_fn)
+    val_dataloader = DataLoader(val_dataset, batch_size=32, shuffle=False, collate_fn=custom_collate_fn)
+
+    # Initialize custom BERT model and move it to the device
+    custom_model = CustomBERTModel(
+        vocab_size=len(vocab.vocab),
+        output_dim=2,
+        pre_trained_model_path=opt.pre_trained_model_path
+    ).to(device)
+
+    # Initialize the fine-tuning trainer
+    trainer = BERTFineTuneTrainer1(
+        bert=custom_model,
+        vocab_size=len(vocab.vocab),
+        train_dataloader=train_dataloader,
+        test_dataloader=val_dataloader,
+        lr=1e-5,  # Using learning rate 10^-5 as specified
+        num_labels=2,
+        with_cuda=torch.cuda.is_available(),
+        log_freq=10,
+        workspace_name=opt.output_dir,
+        log_folder_path=opt.log_folder_path
+    )
+
+    # Train the model
+    trainer.train(epoch=20)
+
+    # Save the model
+    os.makedirs(opt.output_dir, exist_ok=True)
+    output_model_file = os.path.join(opt.output_dir, 'fine_tuned_model_3.pth')
+    torch.save(custom_model, output_model_file)
+    print(f'Model saved to {output_model_file}')
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Fine-tune BERT model.')
+    parser.add_argument('--dataset', type=str, default='/home/jupyter/bert/dataset/hint_based/ratio_proportion_change_3/er/er_train.csv', help='Path to the dataset file.')
+    parser.add_argument('--data_path', type=str, default='/home/jupyter/bert/ratio_proportion_change3_1920/_Aug23/gt/er.txt', help='Path to the input sequence file.')
+    parser.add_argument('--output_dir', type=str, default='/home/jupyter/bert/ratio_proportion_change3_1920/_Aug23/output/hint_classification', help='Directory to save the fine-tuned model.')
+    parser.add_argument('--pre_trained_model_path', type=str, default='/home/jupyter/bert/ratio_proportion_change3_1920/output/pretrain:1800ms:64hs:4l:8a:50s:64b:1000e:-5lr/bert_trained.seq_encoder.model.ep68', help='Path to the pre-trained BERT model.')
+    parser.add_argument('--vocab_file', type=str, default='/home/jupyter/bert/ratio_proportion_change3_1920/_Aug23/pretraining/vocab.txt', help='Path to the vocabulary file.')
+    parser.add_argument('--log_folder_path', type=str, default='/home/jupyter/bert/ratio_proportion_change3_1920/logs/oct', help='Path to the folder for saving logs.')
+
+
+    opt = parser.parse_args()
+    main(opt)

new_test_saved_finetuned_model.py

@@ -0,0 +1,613 @@
+import argparse
+import os
+import torch
+import torch.nn as nn
+from torch.optim import Adam
+from torch.utils.data import DataLoader
+import pickle
+print("here1",os.getcwd())
+from src.dataset import TokenizerDataset, TokenizerDatasetForCalibration
+from src.vocab import Vocab
+print("here3",os.getcwd())
+from src.bert import BERT
+from src.seq_model import BERTSM
+from src.classifier_model import BERTForClassification, BERTForClassificationWithFeats
+# from src.new_finetuning.optim_schedule import ScheduledOptim
+import metrics, recalibration, visualization
+from recalibration import ModelWithTemperature
+import tqdm
+import sys
+import time
+import numpy as np
+
+from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, roc_curve, roc_auc_score
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+from collections import defaultdict
+print("here3",os.getcwd())
+class BERTFineTuneTrainer:
+    
+    def __init__(self, bertFinetunedClassifierwithFeats: BERT, #BERTForClassificationWithFeats
+                 vocab_size: int, test_dataloader: DataLoader = None,
+                 lr: float = 1e-4, betas=(0.9, 0.999), weight_decay: float = 0.01, warmup_steps=10000,
+                 with_cuda: bool = True, cuda_devices=None, log_freq: int = 10, workspace_name=None, 
+                 num_labels=2, log_folder_path: str = None):
+        """
+        :param bert: BERT model which you want to train
+        :param vocab_size: total word vocab size
+        :param test_dataloader: test dataset data loader [can be None]
+        :param lr: learning rate of optimizer
+        :param betas: Adam optimizer betas
+        :param weight_decay: Adam optimizer weight decay param
+        :param with_cuda: traning with cuda
+        :param log_freq: logging frequency of the batch iteration
+        """
+
+        # Setup cuda device for BERT training, argument -c, --cuda should be true
+        # cuda_condition = torch.cuda.is_available() and with_cuda
+        # self.device = torch.device("cuda:0" if cuda_condition else "cpu")
+        self.device = torch.device("cpu") #torch.device("cuda:0" if cuda_condition else "cpu")
+        # print(cuda_condition, " Device used = ", self.device)
+        print(" Device used = ", self.device)
+        
+        # available_gpus = list(range(torch.cuda.device_count()))
+
+        # This BERT model will be saved every epoch
+        self.model = bertFinetunedClassifierwithFeats.to("cpu")
+        print(self.model.parameters())
+        for param in self.model.parameters():
+            param.requires_grad = False
+        # Initialize the BERT Language Model, with BERT model
+        # self.model = BERTForClassification(self.bert, vocab_size, num_labels).to(self.device)
+        # self.model = BERTForClassificationWithFeats(self.bert, num_labels, 8).to(self.device)
+        # self.model = bertFinetunedClassifierwithFeats
+        # print(self.model.bert.parameters())
+        # for param in self.model.bert.parameters():
+        #     param.requires_grad = False
+        # BERTForClassificationWithFeats(self.bert, num_labels, 18).to(self.device)
+        
+        # self.model = BERTForClassificationWithFeats(self.bert, num_labels, 1).to(self.device)
+        # Distributed GPU training if CUDA can detect more than 1 GPU
+        # if with_cuda and torch.cuda.device_count() > 1:
+        #     print("Using %d GPUS for BERT" % torch.cuda.device_count())
+        #     self.model = nn.DataParallel(self.model, device_ids=available_gpus)
+
+        # Setting the train, validation and test data loader
+        # self.train_data = train_dataloader
+        # self.val_data = val_dataloader
+        self.test_data = test_dataloader
+    
+        # self.optim = Adam(self.model.parameters(), lr=lr, weight_decay=weight_decay) #, eps=1e-9
+        self.optim = Adam(self.model.parameters(), lr=lr, betas=betas, weight_decay=weight_decay)
+        # self.optim_schedule = ScheduledOptim(self.optim, self.model.bert.hidden, n_warmup_steps=warmup_steps)
+        # self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=2, factor=0.1)
+        self.criterion = nn.CrossEntropyLoss()
+
+        # if num_labels == 1:
+        #     self.criterion = nn.MSELoss()
+        # elif num_labels == 2:
+        #     self.criterion = nn.BCEWithLogitsLoss()
+        #     # self.criterion = nn.CrossEntropyLoss()
+        # elif num_labels > 2:
+            # self.criterion = nn.CrossEntropyLoss()
+            # self.criterion = nn.BCEWithLogitsLoss()
+        
+        
+        self.log_freq = log_freq
+        self.log_folder_path = log_folder_path
+        # self.workspace_name = workspace_name
+        # self.finetune_task = finetune_task
+        # self.save_model = False
+        # self.avg_loss = 10000
+        self.start_time = time.time()
+        # self.probability_list = []
+        for fi in ['test']: #'val', 
+            f = open(self.log_folder_path+f"/log_{fi}_finetuned.txt", 'w')
+            f.close()
+        print("Total Parameters:", sum([p.nelement() for p in self.model.parameters()]))
+
+    # def train(self, epoch):
+    #     self.iteration(epoch, self.train_data)
+
+    # def val(self, epoch):
+    #     self.iteration(epoch, self.val_data, phase="val")
+        
+    def test(self, epoch):
+        # if epoch == 0:
+        #     self.avg_loss = 10000
+        self.iteration(epoch, self.test_data, phase="test")
+
+    def iteration(self, epoch, data_loader, phase="train"):
+        """
+        loop over the data_loader for training or testing
+        if on train status, backward operation is activated
+        and also auto save the model every peoch
+
+        :param epoch: current epoch index
+        :param data_loader: torch.utils.data.DataLoader for iteration
+        :param train: boolean value of is train or test
+        :return: None
+        """
+        
+        # Setting the tqdm progress bar
+        data_iter = tqdm.tqdm(enumerate(data_loader),
+                              desc="EP_%s:%d" % (phase, epoch),
+                              total=len(data_loader),
+                              bar_format="{l_bar}{r_bar}")
+
+        avg_loss = 0.0
+        total_correct = 0
+        total_element = 0
+        plabels = []
+        tlabels = []
+        probabs = []
+        positive_class_probs=[]
+        if phase == "train":
+            self.model.train()
+        else:
+            self.model.eval()
+        # self.probability_list = []
+        
+        with open(self.log_folder_path+f"/log_{phase}_finetuned.txt", 'a') as f:
+            sys.stdout = f
+            for i, data in data_iter:
+                # 0. batch_data will be sent into the device(GPU or cpu)
+                data = {key: value.to(self.device) for key, value in data.items()}
+                if phase == "train":
+                    logits = self.model.forward(data["input"], data["segment_label"], data["feat"])
+                else:
+                    with torch.no_grad():
+                        logits = self.model.forward(data["input"].cpu(), data["segment_label"].cpu(), data["feat"].cpu())
+
+                logits = logits.cpu()
+                loss = self.criterion(logits, data["label"])
+                # if torch.cuda.device_count() > 1:
+                #     loss = loss.mean()
+
+                # 3. backward and optimization only in train
+                # if phase == "train":
+                #     self.optim_schedule.zero_grad()
+                #     loss.backward()
+                #     self.optim_schedule.step_and_update_lr()
+
+                # prediction accuracy
+                probs = nn.Softmax(dim=-1)(logits) # Probabilities
+                probabs.extend(probs.detach().cpu().numpy().tolist())
+                predicted_labels = torch.argmax(probs, dim=-1) #correct
+                # self.probability_list.append(probs)
+                # true_labels = torch.argmax(data["label"], dim=-1)
+                plabels.extend(predicted_labels.cpu().numpy())
+                tlabels.extend(data['label'].cpu().numpy())
+                positive_class_probs = [prob[1] for prob in probabs]
+                # Compare predicted labels to true labels and calculate accuracy
+                correct = (data['label'] == predicted_labels).sum().item()
+                
+                avg_loss += loss.item()
+                total_correct += correct
+                # total_element += true_labels.nelement()
+                total_element += data["label"].nelement()
+                # print(">>>>>>>>>>>>>>", predicted_labels, true_labels, correct, total_correct, total_element)
+                
+                post_fix = {
+                    "epoch": epoch,
+                    "iter": i,
+                    "avg_loss": avg_loss / (i + 1),
+                    "avg_acc": total_correct / total_element * 100 if total_element != 0 else 0,
+                    "loss": loss.item()
+                }
+                if i % self.log_freq == 0:
+                    data_iter.write(str(post_fix))
+            
+            precisions = precision_score(tlabels, plabels, average="weighted", zero_division=0)
+            recalls = recall_score(tlabels, plabels, average="weighted")
+            f1_scores = f1_score(tlabels, plabels, average="weighted")
+            cmatrix = confusion_matrix(tlabels, plabels)
+            end_time = time.time()
+            auc_score = roc_auc_score(tlabels, positive_class_probs)
+            final_msg = {
+                "avg_loss": avg_loss / len(data_iter),
+                "total_acc": total_correct * 100.0 / total_element,
+                "precisions": precisions,
+                "recalls": recalls,
+                "f1_scores": f1_scores,
+                # "confusion_matrix": f"{cmatrix}",
+                # "true_labels": f"{tlabels}",
+                # "predicted_labels": f"{plabels}",
+                "time_taken_from_start": end_time - self.start_time,
+                "auc_score":auc_score
+            }
+            with open("result.txt", 'w') as file:
+                for key, value in final_msg.items():
+                    file.write(f"{key}: {value}\n")
+            print(final_msg)
+            fpr, tpr, thresholds = roc_curve(tlabels, positive_class_probs)
+            with open("roc_data.pkl", "wb") as f:
+                pickle.dump((fpr, tpr, thresholds), f)
+            print(final_msg)
+            f.close()
+            with open(self.log_folder_path+f"/log_{phase}_finetuned_info.txt", 'a') as f1:
+                sys.stdout = f1
+                final_msg = {
+                "epoch": f"EP{epoch}_{phase}",
+                "confusion_matrix": f"{cmatrix}",
+                "true_labels": f"{tlabels if epoch == 0 else ''}",
+                "predicted_labels": f"{plabels}",
+                "probabilities": f"{probabs}",
+                "time_taken_from_start": end_time - self.start_time
+                }
+                print(final_msg)
+                f1.close()
+            sys.stdout = sys.__stdout__
+        sys.stdout = sys.__stdout__
+        
+
+        
+class BERTFineTuneCalibratedTrainer:
+    
+    def __init__(self, bertFinetunedClassifierwithFeats: BERT, #BERTForClassificationWithFeats
+                 vocab_size: int, test_dataloader: DataLoader = None,
+                 lr: float = 1e-4, betas=(0.9, 0.999), weight_decay: float = 0.01, warmup_steps=10000,
+                 with_cuda: bool = True, cuda_devices=None, log_freq: int = 10, workspace_name=None, 
+                 num_labels=2, log_folder_path: str = None):
+        """
+        :param bert: BERT model which you want to train
+        :param vocab_size: total word vocab size
+        :param test_dataloader: test dataset data loader [can be None]
+        :param lr: learning rate of optimizer
+        :param betas: Adam optimizer betas
+        :param weight_decay: Adam optimizer weight decay param
+        :param with_cuda: traning with cuda
+        :param log_freq: logging frequency of the batch iteration
+        """
+
+        # Setup cuda device for BERT training, argument -c, --cuda should be true
+        cuda_condition = torch.cuda.is_available() and with_cuda
+        self.device = torch.device("cuda:0" if cuda_condition else "cpu")
+        print(cuda_condition, " Device used = ", self.device)
+        
+        # available_gpus = list(range(torch.cuda.device_count()))
+
+        # This BERT model will be saved every epoch
+        self.model = bertFinetunedClassifierwithFeats
+        print(self.model.parameters())
+        for param in self.model.parameters():
+            param.requires_grad = False
+        # Initialize the BERT Language Model, with BERT model
+        # self.model = BERTForClassification(self.bert, vocab_size, num_labels).to(self.device)
+        # self.model = BERTForClassificationWithFeats(self.bert, num_labels, 8).to(self.device)
+        # self.model = bertFinetunedClassifierwithFeats
+        # print(self.model.bert.parameters())
+        # for param in self.model.bert.parameters():
+        #     param.requires_grad = False
+        # BERTForClassificationWithFeats(self.bert, num_labels, 18).to(self.device)
+        
+        # self.model = BERTForClassificationWithFeats(self.bert, num_labels, 1).to(self.device)
+        # Distributed GPU training if CUDA can detect more than 1 GPU
+        # if with_cuda and torch.cuda.device_count() > 1:
+        #     print("Using %d GPUS for BERT" % torch.cuda.device_count())
+        #     self.model = nn.DataParallel(self.model, device_ids=available_gpus)
+
+        # Setting the train, validation and test data loader
+        # self.train_data = train_dataloader
+        # self.val_data = val_dataloader
+        self.test_data = test_dataloader
+    
+        # self.optim = Adam(self.model.parameters(), lr=lr, weight_decay=weight_decay) #, eps=1e-9
+        self.optim = Adam(self.model.parameters(), lr=lr, betas=betas, weight_decay=weight_decay)
+        # self.optim_schedule = ScheduledOptim(self.optim, self.model.bert.hidden, n_warmup_steps=warmup_steps)
+        # self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=2, factor=0.1)
+        self.criterion = nn.CrossEntropyLoss()
+
+        # if num_labels == 1:
+        #     self.criterion = nn.MSELoss()
+        # elif num_labels == 2:
+        #     self.criterion = nn.BCEWithLogitsLoss()
+        #     # self.criterion = nn.CrossEntropyLoss()
+        # elif num_labels > 2:
+            # self.criterion = nn.CrossEntropyLoss()
+            # self.criterion = nn.BCEWithLogitsLoss()
+        
+        
+        self.log_freq = log_freq
+        self.log_folder_path = log_folder_path
+        # self.workspace_name = workspace_name
+        # self.finetune_task = finetune_task
+        # self.save_model = False
+        # self.avg_loss = 10000
+        self.start_time = time.time()
+        # self.probability_list = []
+        for fi in ['test']: #'val', 
+            f = open(self.log_folder_path+f"/log_{fi}_finetuned.txt", 'w')
+            f.close()
+        print("Total Parameters:", sum([p.nelement() for p in self.model.parameters()]))
+
+    # def train(self, epoch):
+    #     self.iteration(epoch, self.train_data)
+
+    # def val(self, epoch):
+    #     self.iteration(epoch, self.val_data, phase="val")
+        
+    def test(self, epoch):
+        # if epoch == 0:
+        #     self.avg_loss = 10000
+        self.iteration(epoch, self.test_data, phase="test")
+
+    def iteration(self, epoch, data_loader, phase="train"):
+        """
+        loop over the data_loader for training or testing
+        if on train status, backward operation is activated
+        and also auto save the model every peoch
+
+        :param epoch: current epoch index
+        :param data_loader: torch.utils.data.DataLoader for iteration
+        :param train: boolean value of is train or test
+        :return: None
+        """
+        
+        # Setting the tqdm progress bar
+        data_iter = tqdm.tqdm(enumerate(data_loader),
+                              desc="EP_%s:%d" % (phase, epoch),
+                              total=len(data_loader),
+                              bar_format="{l_bar}{r_bar}")
+
+        avg_loss = 0.0
+        total_correct = 0
+        total_element = 0
+        plabels = []
+        tlabels = []
+        probabs = []
+
+        if phase == "train":
+            self.model.train()
+        else:
+            self.model.eval()
+        # self.probability_list = []
+        
+        with open(self.log_folder_path+f"/log_{phase}_finetuned.txt", 'a') as f:
+            sys.stdout = f
+            for i, data in data_iter:
+                # 0. batch_data will be sent into the device(GPU or cpu)
+                # print(data_pair[0])
+                data = {key: value.to(self.device) for key, value in data[0].items()}
+                # print(f"data : {data}")
+                # data = {key: value.to(self.device) for key, value in data.items()}
+                
+                # if phase == "train":
+                #     logits = self.model.forward(data["input"], data["segment_label"], data["feat"])
+                # else:
+                with torch.no_grad():
+                    # logits = self.model.forward(data["input"], data["segment_label"], data["feat"])
+                    logits = self.model.forward(data)
+
+                loss = self.criterion(logits, data["label"])
+                if torch.cuda.device_count() > 1:
+                    loss = loss.mean()
+
+                # 3. backward and optimization only in train
+                # if phase == "train":
+                #     self.optim_schedule.zero_grad()
+                #     loss.backward()
+                #     self.optim_schedule.step_and_update_lr()
+
+                # prediction accuracy
+                probs = nn.Softmax(dim=-1)(logits) # Probabilities
+                probabs.extend(probs.detach().cpu().numpy().tolist())
+                predicted_labels = torch.argmax(probs, dim=-1) #correct
+                # self.probability_list.append(probs)
+                # true_labels = torch.argmax(data["label"], dim=-1)
+                plabels.extend(predicted_labels.cpu().numpy())
+                tlabels.extend(data['label'].cpu().numpy())
+                positive_class_probs = [prob[1] for prob in probabs]
+
+                # Compare predicted labels to true labels and calculate accuracy
+                correct = (data['label'] == predicted_labels).sum().item()
+                
+                avg_loss += loss.item()
+                total_correct += correct
+                # total_element += true_labels.nelement()
+                total_element += data["label"].nelement()
+                # print(">>>>>>>>>>>>>>", predicted_labels, true_labels, correct, total_correct, total_element)
+                
+                post_fix = {
+                    "epoch": epoch,
+                    "iter": i,
+                    "avg_loss": avg_loss / (i + 1),
+                    "avg_acc": total_correct / total_element * 100 if total_element != 0 else 0,
+                    "loss": loss.item()
+                }
+                if i % self.log_freq == 0:
+                    data_iter.write(str(post_fix))
+            
+            precisions = precision_score(tlabels, plabels, average="weighted", zero_division=0)
+            recalls = recall_score(tlabels, plabels, average="weighted")
+            f1_scores = f1_score(tlabels, plabels, average="weighted")
+            cmatrix = confusion_matrix(tlabels, plabels)
+            auc_score = roc_auc_score(tlabels, positive_class_probs)
+            end_time = time.time()
+            final_msg = {
+                "avg_loss": avg_loss / len(data_iter),
+                "total_acc": total_correct * 100.0 / total_element,
+                "precisions": precisions,
+                "recalls": recalls,
+                "f1_scores": f1_scores,
+                "auc_score":auc_score,
+                # "confusion_matrix": f"{cmatrix}",
+                # "true_labels": f"{tlabels}",
+                # "predicted_labels": f"{plabels}",
+                "time_taken_from_start": end_time - self.start_time
+            }
+            with open("result.txt", 'w') as file:
+                for key, value in final_msg.items():
+                    file.write(f"{key}: {value}\n")
+            with open("plabels.txt","w") as file:
+                file.write(plabels)
+                       
+            print(final_msg)
+            fpr, tpr, thresholds = roc_curve(tlabels, positive_class_probs)
+            f.close()
+            with open(self.log_folder_path+f"/log_{phase}_finetuned_info.txt", 'a') as f1:
+                sys.stdout = f1
+                final_msg = {
+                
+                "confusion_matrix": f"{cmatrix}",
+                "true_labels": f"{tlabels if epoch == 0 else ''}",
+                "predicted_labels": f"{plabels}",
+                "probabilities": f"{probabs}",
+                "time_taken_from_start": end_time - self.start_time
+                }
+                print(final_msg)
+                f1.close()
+            sys.stdout = sys.__stdout__
+        sys.stdout = sys.__stdout__
+        
+
+    
+def train():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('-workspace_name', type=str, default=None)
+    parser.add_argument('-code', type=str, default=None, help="folder for pretraining outputs and logs")
+    parser.add_argument('-finetune_task', type=str, default=None, help="folder inside finetuning")
+    parser.add_argument("-attention", type=bool, default=False, help="analyse attention scores")
+    parser.add_argument("-diff_test_folder", type=bool, default=False, help="use for different test folder")
+    parser.add_argument("-embeddings", type=bool, default=False, help="get and analyse embeddings")
+    parser.add_argument('-embeddings_file_name', type=str, default=None, help="file name of embeddings")
+    parser.add_argument("-pretrain", type=bool, default=False, help="pretraining: true, or false")
+    # parser.add_argument('-opts', nargs='+', type=str, default=None, help='List of optional steps')
+    parser.add_argument("-max_mask", type=int, default=0.15, help="% of input tokens selected for masking") 
+    # parser.add_argument("-p", "--pretrain_dataset", type=str, default="pretraining/pretrain.txt", help="pretraining dataset for bert")
+    # parser.add_argument("-pv", "--pretrain_val_dataset", type=str, default="pretraining/test.txt", help="pretraining validation dataset for bert")
+# default="finetuning/test.txt",
+    parser.add_argument("-vocab_path", type=str, default="pretraining/vocab.txt", help="built vocab model path with bert-vocab")
+
+    parser.add_argument("-train_dataset_path", type=str, default="train.txt", help="fine tune train dataset for progress classifier")
+    parser.add_argument("-val_dataset_path", type=str, default="val.txt", help="test set for evaluate fine tune train set")
+    parser.add_argument("-test_dataset_path", type=str, default="test.txt", help="test set for evaluate fine tune train set")
+    parser.add_argument("-num_labels", type=int, default=2, help="Number of labels") 
+    parser.add_argument("-train_label_path", type=str, default="train_label.txt", help="fine tune train dataset for progress classifier")
+    parser.add_argument("-val_label_path", type=str, default="val_label.txt", help="test set for evaluate fine tune train set")
+    parser.add_argument("-test_label_path", type=str, default="test_label.txt", help="test set for evaluate fine tune train set")
+    ##### change Checkpoint for finetuning
+    parser.add_argument("-pretrained_bert_checkpoint", type=str, default=None, help="checkpoint of saved pretrained bert model") 
+    parser.add_argument("-finetuned_bert_classifier_checkpoint", type=str, default=None, help="checkpoint of saved finetuned bert model")  #."output_feb09/bert_trained.model.ep40"
+    #."output_feb09/bert_trained.model.ep40"
+    parser.add_argument('-check_epoch', type=int, default=None)
+
+    parser.add_argument("-hs", "--hidden", type=int, default=64, help="hidden size of transformer model") #64
+    parser.add_argument("-l", "--layers", type=int, default=4, help="number of layers") #4
+    parser.add_argument("-a", "--attn_heads", type=int, default=4, help="number of attention heads") #8
+    parser.add_argument("-s", "--seq_len", type=int, default=128, help="maximum sequence length")
+
+    parser.add_argument("-b", "--batch_size", type=int, default=500, help="number of batch_size") #64
+    parser.add_argument("-e", "--epochs", type=int, default=1)#1501, help="number of epochs") #501
+    # Use 50 for pretrain, and 10 for fine tune
+    parser.add_argument("-w", "--num_workers", type=int, default=0, help="dataloader worker size")
+
+    # Later run with cuda
+    parser.add_argument("--with_cuda", type=bool, default=False, help="training with CUDA: true, or false")
+    parser.add_argument("--log_freq", type=int, default=10, help="printing loss every n iter: setting n")
+    # parser.add_argument("--corpus_lines", type=int, default=None, help="total number of lines in corpus")
+    parser.add_argument("--cuda_devices", type=int, nargs='+', default=None, help="CUDA device ids")
+    # parser.add_argument("--on_memory", type=bool, default=False, help="Loading on memory: true or false")
+    
+    parser.add_argument("--dropout", type=float, default=0.1, help="dropout of network")
+    parser.add_argument("--lr", type=float, default=1e-05, help="learning rate of adam") #1e-3
+    parser.add_argument("--adam_weight_decay", type=float, default=0.01, help="weight_decay of adam")
+    parser.add_argument("--adam_beta1", type=float, default=0.9, help="adam first beta value")
+    parser.add_argument("--adam_beta2", type=float, default=0.98, help="adam first beta value") #0.999
+
+    parser.add_argument("-o", "--output_path", type=str, default="bert_trained.seq_encoder.model", help="ex)output/bert.model")    
+    # parser.add_argument("-o", "--output_path", type=str, default="output/bert_fine_tuned.model", help="ex)output/bert.model")
+    
+    args = parser.parse_args()
+    for k,v in vars(args).items():
+        if 'path' in k:
+            if v:
+                if k == "output_path":
+                    if args.code:
+                        setattr(args, f"{k}", args.workspace_name+f"/output/{args.code}/"+v)
+                    elif args.finetune_task:
+                        setattr(args, f"{k}", args.workspace_name+f"/output/{args.finetune_task}/"+v)
+                    else:
+                        setattr(args, f"{k}", args.workspace_name+"/output/"+v)
+                elif k != "vocab_path":
+                    if args.pretrain:
+                        setattr(args, f"{k}", args.workspace_name+"/pretraining/"+v)
+                    else:
+                        if args.code:
+                            setattr(args, f"{k}", args.workspace_name+f"/{args.code}/"+v)
+                        elif args.finetune_task:
+                            if args.diff_test_folder and "test" in k:
+                                setattr(args, f"{k}", args.workspace_name+f"/finetuning/"+v)
+                            else:
+                                setattr(args, f"{k}", args.workspace_name+f"/finetuning/{args.finetune_task}/"+v)
+                        else:
+                            setattr(args, f"{k}", args.workspace_name+"/finetuning/"+v)
+                else:
+                    setattr(args, f"{k}", args.workspace_name+"/"+v)
+                
+                print(f"args.{k} : {getattr(args, f'{k}')}")
+
+    print("Loading Vocab", args.vocab_path)
+    vocab_obj = Vocab(args.vocab_path)
+    vocab_obj.load_vocab()
+    print("Vocab Size: ", len(vocab_obj.vocab))
+    
+    
+    print("Testing using finetuned model......")
+    print("Loading Test Dataset", args.test_dataset_path)            
+    test_dataset = TokenizerDataset(args.test_dataset_path, args.test_label_path, vocab_obj, seq_len=args.seq_len)
+    # test_dataset = TokenizerDatasetForCalibration(args.test_dataset_path, args.test_label_path, vocab_obj, seq_len=args.seq_len)
+
+    print("Creating Dataloader...")
+    test_data_loader = DataLoader(test_dataset, batch_size=args.batch_size, num_workers=args.num_workers)
+
+    print("Load fine-tuned BERT classifier model with feats")
+    # cuda_condition = torch.cuda.is_available() and args.with_cuda
+    device = torch.device("cpu") #torch.device("cuda:0" if cuda_condition else "cpu")
+    finetunedBERTclassifier = torch.load(args.finetuned_bert_classifier_checkpoint, map_location=device)
+    if isinstance(finetunedBERTclassifier, torch.nn.DataParallel):
+        finetunedBERTclassifier = finetunedBERTclassifier.module
+    
+    new_log_folder = f"{args.workspace_name}/logs"
+    new_output_folder = f"{args.workspace_name}/output"
+    if args.finetune_task: # is sent almost all the time
+        new_log_folder = f"{args.workspace_name}/logs/{args.finetune_task}"
+        new_output_folder = f"{args.workspace_name}/output/{args.finetune_task}"
+
+    if not os.path.exists(new_log_folder):
+        os.makedirs(new_log_folder)
+    if not os.path.exists(new_output_folder):
+        os.makedirs(new_output_folder)
+
+    print("Creating BERT Fine Tuned Test Trainer")
+    trainer = BERTFineTuneTrainer(finetunedBERTclassifier, 
+                    len(vocab_obj.vocab), test_dataloader=test_data_loader, 
+                  lr=args.lr, betas=(args.adam_beta1, args.adam_beta2), weight_decay=args.adam_weight_decay, 
+                  with_cuda=args.with_cuda, cuda_devices = args.cuda_devices, log_freq=args.log_freq, 
+                  workspace_name = args.workspace_name, num_labels=args.num_labels, log_folder_path=new_log_folder)
+
+    # trainer = BERTFineTuneCalibratedTrainer(finetunedBERTclassifier, 
+    #                 len(vocab_obj.vocab), test_dataloader=test_data_loader, 
+    #               lr=args.lr, betas=(args.adam_beta1, args.adam_beta2), weight_decay=args.adam_weight_decay, 
+    #               with_cuda=args.with_cuda, cuda_devices = args.cuda_devices, log_freq=args.log_freq, 
+    #               workspace_name = args.workspace_name, num_labels=args.num_labels, log_folder_path=new_log_folder)
+    print("Testing fine-tuned model Start....")
+    start_time = time.time()
+    repoch = range(args.check_epoch, args.epochs) if args.check_epoch else range(args.epochs)
+    counter = 0
+    # patience = 10
+    for epoch in repoch:
+            print(f'Test Epoch {epoch} Starts, Time: {time.strftime("%D %T", time.localtime(time.time()))}')
+            trainer.test(epoch)
+            # pickle.dump(trainer.probability_list, open(f"{args.workspace_name}/output/aaai/change4_mid_prob_{epoch}.pkl","wb"))
+            print(f'Test Epoch {epoch} Ends, Time: {time.strftime("%D %T", time.localtime(time.time()))} \n')
+    end_time = time.time()
+    print("Time Taken to fine-tune model = ", end_time - start_time)
+    print(f'Pretraining Ends, Time: {time.strftime("%D %T", time.localtime(end_time))}')
+    
+    
+    
+if __name__ == "__main__":
+    train()

ratio_proportion_change3/finetuning/test.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da39d07824b2cfc3a41445694ff65018b1ffdf3e9b844d464cdba3c0ad6a8b87
+size 6876678

ratio_proportion_change3/finetuning/test_in.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5029b0f965c4f6f1d5dd981778daf0b8f0f778dd71ecad7eb984e8461fa85b9
+size 1318665

ratio_proportion_change3/finetuning/test_in_info.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:044b445c06dbdecb8663e5db8d6f270799240f1b433a169c335c15e566dbba20
+size 1660506

ratio_proportion_change3/finetuning/test_in_label.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c035490a97515200f23348bf01bd3c16def88046a7c2215d9ef169ffc089d0d
+size 17202

ratio_proportion_change3/finetuning/test_label.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6fee24daa1523d1a8d7615c415fac559d0bf85ace5ab18d9db1a8dff533ff68
+size 79424

ratio_proportion_change3/finetuning/testr_in_label.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b95faf33529a8cdbcedfca3853be88f917e730c79261731c4860f0d57909f13f
+size 97701

ratio_proportion_change3/finetuning/testr_label.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29519e69e1ec480ae0440e23dcb57f97bbb33cdd9b91d18e5e999d3e7e58288c
+size 549160

ratio_proportion_change3/finetuning/train.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b397618386eb7cd21cf59251b4d48c1880330477c3186375a039047f181beae
+size 775465

ratio_proportion_change3/finetuning/train_in.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b935dcf7dbbe3ad66c2616ae3e6c342d9d1b162c4931c7a291386c5ce609ce0
+size 1656785

ratio_proportion_change3/finetuning/train_in_info.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5663b5706417ba65ec10abecf405f5644dfa637683fe1198ea937b8838cba6a
+size 2411977

ratio_proportion_change3/finetuning/train_in_label.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e38fd99af6313174626b81cad3f5a6b6e88711f9f66f57cb5c3b0e6bc2e8b4c
+size 17202

ratio_proportion_change3/finetuning/train_info.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9968e038b75a633b4957602e37d57b7c0cb561f9ae3c2b17ad0f9eb48b554c21
+size 1080190

ratio_proportion_change3/finetuning/train_label.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de505197183cefe6a1c5ff4f5cd8e07dc14ed1b601951d7c3e02947d603e58c6
+size 8932

ratio_proportion_change3/finetuning/trainr_in_label.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95e450636dcb476a258439c94249f1078e9186bfe00d8e70da7b9c339f4f728c
+size 129011

ratio_proportion_change3/finetuning/trainr_label.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0294122c85237764e51d69d2efc5233d2c3a0d1027b31b4f510ca68bd6e46bc1
+size 61542

ratio_proportion_change3/logs/masked/log_test_10per_finetuned.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d42d75c9a38be298f8ee1f022a544fe49804b72979a734b42aea08f7b31fb52
+size 671476

ratio_proportion_change3/logs/masked/log_test_FS_finetuned.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:858fc5350a9bf0c75d46b8af1dc3b0f310bab1a0afa92ca8bca1e829b57d0b73
+size 149839

ratio_proportion_change3/logs/masked/log_test_IS_finetuned.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c66fcb29fc9f3d92bed511d4a91530ad79a13860b93e418f0b8c6c1be0e54169
+size 149828

ratio_proportion_change3/logs/masked/log_test_pretrained.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df1193ce0490717b442303f51da68869c6419f461ce5044b5a275b40e7bfb368
+size 1055582

ratio_proportion_change3/logs/masked/log_train_10per_finetuned.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf58b6b3ba0d0a9562cfd510ce1a7bff20a4bb0ee1faa907397314333d26dcd2
+size 88900

ratio_proportion_change3/logs/masked/log_train_FS_finetuned.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e92c0a8722c7b21b36f5028493692ccf32b473c20d3f6027d54e5fd822960432
+size 167286

ratio_proportion_change3/logs/masked/log_train_IS_finetuned.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:644994580015b35979dce25d0e2b3be7b6ef6d02193a1b0ea6d10411412c5495
+size 167148

ratio_proportion_change3/logs/masked/log_train_pretrained.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6095cbd3be17925bc64b05902281c01c2c3255df63ea2e5cd48b5d402c06033b
+size 4116343

ratio_proportion_change3/output/FS/train.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:367628f1b9aa5047a07d5eb6e574e166e12d533d18a1634045424736bff9cc42
+size 1699339

ratio_proportion_change3/output/FS/train_label.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be1eeaf1d96b6010aec2db568d20170e79d5e53bb790e250074f877931ab23d3
+size 20636

ratio_proportion_change3/output/IS/train.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc804d5d3a54d0cbe69b295464378609916a2c5b2a8c0696757d20be185e1427
+size 1361007

ratio_proportion_change3/output/IS/train_label.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a394ba9e86a56b82422fd9b7a7212bde72eae95fbd8d899e0e9fb9c21132a605
+size 20636

ratio_proportion_change3/output/correctness/test.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0faf7af6b63c26cb29b586e087c84881365a94b22d71f1a8587bfa979f2d5794
+size 6253326

ratio_proportion_change3/output/correctness/test_label.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:708d181754296d2bbbe56ce509eb896ca69bd2d7a418839c0a09836bf1c31541
+size 75023