<reponame>stancld/MSc-Project<filename>Sentiment scoring/BERT_Train on a down-stream task/BERT - Fine-tuning.py
# -*- coding: utf-8 -*-
"""BERT - Fine-tuning.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1U8cXrKGMNMj-TaB2iwurllMqn6fDnVkG

# **BERT - Fine-tuning**

<hr>

**Author: <NAME>**

**Description:** The notebook used for fine-tuning BERT implemented by HuggingFace on a down-stream task, sentiment analysis of employee reviews from Glassdoor. This notebook is a part of the MSc project.

Fine-tuning is done using 1 GPU.

### **1. Settings & Import modules**

<hr>

- Install modules necessary for training on TPUs
- Install all non-standard modules
- Import libraries and mount the drive
"""

!pip install pytorch-lightning==0.8.5
!pip install tensorboard==2.3.0
!pip install transformers==2.10.0

# Commented out IPython magic to ensure Python compatibility.
# import libraries and settings
from os import listdir
from os.path import isfile, join
import numpy as np
from numpy.random import random, seed
import pandas as pd

import torch
from torch import nn, optim
import torch.nn.functional as F
from torch.utils.data import Dataset, TensorDataset, DataLoader
import pytorch_lightning as pl
from pytorch_lightning import LightningModule, Trainer

import transformers
from transformers import BertModel, BertTokenizer, AdamW, get_linear_schedule_with_warmup

from google.colab import drive
drive.mount('/content/drive/', force_remount=True)
# %cd 'drive/My Drive'

!nvidia-smi

"""### **2. BERT import**

<hr>

- Choose the dir
- Import BERT_SentimentClassifier module
- set parameters for BERT
- instantiate BERT_SentimetnClassifier
- set early stopping rule
"""

# Commented out IPython magic to ensure Python compatibility.
# choose the dir
# %cd 'MSc Project/'

from SentimentClassifier import BERT_SentimentClassifier

BERT_parameters = {
    'PRE_TRAINED_MODEL_NAME': 'bert-base-cased',
    'BATCH_SIZE': 16,
    'NUM_WORKERS': 40,
    'DROPOUT_PROB': 0.1,
    'N_CLASSES': 3,
    'N_EPOCHS': 4,
    'LEARNING_RATE': 4e-5,
    'DATA_PATH': '/content/drive/My Drive/MSc Project/Data/',
}

BERT_clf = BERT_SentimentClassifier(**BERT_parameters)

"""### **3. BERT training**

<hr>

- Instantiate training with desired parameters
- Run training
"""

# instantiate trainer
trainer = Trainer(
    gpus=1,
    max_epochs=BERT_parameters['N_EPOCHS']
)

for params in BERT_clf.bert.parameters():
  params.requires_grad = False
trainer.fit(BERT_clf)

"""### **4. Save fine-tuned model**"""

PATH = '/content/drive/My Drive/MSc Project/checkpoint.pt'

torch.save(BERT_clf.state_dict(), PATH)

model=BERT_SentimentClassifier(**BERT_parameters)
model.load_state_dict(torch.load(PATH))

BERT_clf

