Add files via upload

aiprojecttest.py

@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+"""AiProjectTest.ipynb
+
+Automatically generated by Colaboratory.
+
+Original file is located at
+    https://colab.research.google.com/drive/1E4AHYbuRi_FbOMhQntdAMMZMY14hWh2e
+"""
+
+from pathlib import Path
+from sklearn.model_selection import train_test_split
+import torch
+from torch.utils.data import Dataset
+from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification
+from transformers import Trainer, TrainingArguments
+from torch.utils.data import DataLoader
+from transformers import AdamW
+import pandas as pd
+
+df_train = pd.read_csv('train.csv')
+df_test = pd.read_csv('test.csv')
+df_test_labels = pd.read_csv('test_labels.csv')
+
+model_name = "distilbert-base-uncased"
+
+def read_file(f):
+  texts = f['comment_text'].tolist()
+  labels = []
+  for i in range(len(f)):
+    temp = []
+    temp.append(f['toxic'][i])
+    temp.append(f['severe_toxic'][i])
+    temp.append(f['obscene'][i])
+    temp.append(f['threat'][i])
+    temp.append(f['insult'][i])
+    temp.append(f['identity_hate'][i])
+    labels.append(temp)
+  return texts, labels
+
+train_texts, train_labels = read_file(df_train)
+test_texts = df_test['comment_text'].tolist()
+test_labels = []
+for i in range(len(df_test_labels)):
+  temp = []
+  temp.append(df_test_labels['toxic'][i])
+  temp.append(df_test_labels['severe_toxic'][i])
+  temp.append(df_test_labels['obscene'][i])
+  temp.append(df_test_labels['threat'][i])
+  temp.append(df_test_labels['insult'][i])
+  temp.append(df_test_labels['identity_hate'][i])
+  test_labels.append(temp)
+
+train_texts, val_texts, train_labels, val_labels = train_test_split(train_texts, train_labels, test_size=.2)
+
+tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)
+
+ind = 0
+train_encodings = {'input_ids': [], 'attention_mask': []}
+
+for i in range(len(train_texts)//16):
+  temp = tokenizer(train_texts[ind:ind+16], truncation=True, padding=True)
+  train_encodings['input_ids'] += temp['input_ids']
+  train_encodings['attention_mask'] += temp['attention_mask']
+  ind += 16
+
+ind = 0
+val_encodings = {'input_ids': [], 'attention_mask': []}
+
+for i in range(len(val_texts)//16):
+  temp = tokenizer(val_texts[ind:ind+16], truncation=True, padding=True)
+  val_encodings['input_ids'] += temp['input_ids']
+  val_encodings['attention_mask'] += temp['attention_mask']
+  ind += 16
+
+ind = 0
+test_encodings = {'input_ids': [], 'attention_mask': []}
+
+for i in range(len(test_texts)//16):
+  temp = tokenizer(test_texts[ind:ind+16], truncation=True, padding=True)
+  test_encodings['input_ids'] += temp['input_ids']
+  test_encodings['attention_mask'] += temp['attention_mask']
+  ind += 16
+
+while True:
+  if len(train_labels) > len(train_encodings):
+    train_labels.pop()
+  else:
+    break
+  
+while True:
+  if len(val_labels) > len(val_encodings):
+    val_labels.pop()
+  else:
+    break
+
+while True:
+  if len(test_labels) > len(test_encodings):
+    test_labels.pop()
+  else:
+    break
+
+class dataset(Dataset):
+  def __init__(self, encodings, labels):
+    self.encodings = encodings
+    self.labels = labels
+  
+  def __getitem__(self, idx):
+    item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
+    item['labels'] = torch.tensor(self.labels[idx])
+    return item
+  
+  def __len__(self):
+    return(len(self.labels))
+
+train_dataset_list = [[], [], [], [], [], []]
+for i in train_labels:
+  for j in range(6):
+    train_dataset_list[j].append(i[j])
+  
+val_dataset_list = [[], [], [], [], [], []]
+for i in val_labels:
+  for j in range(6):
+    val_dataset_list[j].append(i[j])
+
+train_dataset_0 = dataset(train_encodings, train_dataset_list[0])
+train_dataset_1 = dataset(train_encodings, train_dataset_list[1])
+train_dataset_2 = dataset(train_encodings, train_dataset_list[2])
+train_dataset_3 = dataset(train_encodings, train_dataset_list[3])
+train_dataset_4 = dataset(train_encodings, train_dataset_list[4])
+train_dataset_5 = dataset(train_encodings, train_dataset_list[5])
+
+val_dataset_0 = dataset(val_encodings, val_dataset_list[0])
+val_dataset_1 = dataset(val_encodings, val_dataset_list[1])
+val_dataset_2 = dataset(val_encodings, val_dataset_list[2])
+val_dataset_3 = dataset(val_encodings, val_dataset_list[3])
+val_dataset_4 = dataset(val_encodings, val_dataset_list[4])
+val_dataset_5 = dataset(val_encodings, val_dataset_list[5])
+
+training_args = TrainingArguments(output_dir='./results', 
+                                  num_train_epochs=2, 
+                                  per_device_train_batch_size=16, 
+                                  per_device_eval_batch_size=64, 
+                                  warmup_steps=500, learning_rate=5e-5, 
+                                  weight_decay=.01, logging_dir='./logs', 
+                                  logging_steps=10)
+
+model = DistilBertForSequenceClassification.from_pretrained(model_name)
+
+trainer_0 = Trainer(model=model, args=training_args, train_dataset=train_dataset_0, eval_dataset=val_dataset_0)
+trainer_0.train()
+
+trainer_1 = Trainer(model=model, args=training_args, train_dataset=train_dataset_1, eval_dataset=val_dataset_1)
+trainer_1.train()
+
+trainer_2 = Trainer(model=model, args=training_args, train_dataset=train_dataset_2, eval_dataset=val_dataset_2)
+trainer_2.train()
+
+trainer_3 = Trainer(model=model, args=training_args, train_dataset=train_dataset_3, eval_dataset=val_dataset_3)
+trainer_3.train()
+
+trainer_4 = Trainer(model=model, args=training_args, train_dataset=train_dataset_4, eval_dataset=val_dataset_4)
+trainer_4.train()
+
+trainer_5 = Trainer(model=model, args=training_args, train_dataset=train_dataset_5, eval_dataset=val_dataset_5)
+trainer_5.train()
+
+# train_dataset = dataset(train_encodings, train_labels)
+# val_dataset = dataset(val_encodings, val_labels)
+# test_dataset = dataset(test_encodings, test_labels)
+
+# -----------------------------------------------------------------
+
+# test_dataset_list = [[], [], [], [], [], []]
+# for i in test_labels:
+#   for j in range(6):
+#     test_dataset_list[j].append(i[j])
+
+# -----------------------------------------------------------------
+
+# val_dataset = dataset(val_encodings, val_labels)
+
+# test_dataset_0 = dataset(test_encodings, test_dataset_list[0])
+# test_dataset_1 = dataset(test_encodings, test_dataset_list[1])
+# test_dataset_2 = dataset(test_encodings, test_dataset_list[2])
+# test_dataset_3 = dataset(test_encodings, test_dataset_list[3])
+# test_dataset_4 = dataset(test_encodings, test_dataset_list[4])
+# test_dataset_5 = dataset(test_encodings, test_dataset_list[5])
+
+# -----------------------------------------------------------------
+
+# device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
+
+# model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased')
+# model.to(device)
+# model.train()
+
+# train_loader = DataLoader(train_dataset_0, batch_size=16, shuffle=True)
+
+# optim = AdamW(model.parameters(), lr=5e-5)
+
+# num_train_epochs = 2
+# for epoch in range(num_train_epochs):
+#     for batch in train_loader:
+#         optim.zero_grad()
+#         input_ids = batch['input_ids'].to(device)
+#         attention_mask = batch['attention_mask'].to(device)
+#         labels = batch['labels'].to(device)
+
+#         outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
+
+#         loss = outputs[0]
+#         loss.backward()
+#         optim.step()
+
+# model.eval()