# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/jonasmue/natural-language-generation/blob/master/Cover_Letter_Generation_with_GPT_2.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="-Thf8CT8rnx0" colab_type="text"
# # Generating Cover Letters with GPT-2

# + [markdown] id="N3b8m6rvrxAW" colab_type="text"
# ## Imports and Setup

# + id="bf_ylKqJOOGR" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="e35b2b8c-51d9-4a86-8d5f-3c21c356af90"
from google.colab import drive
drive.mount('/content/drive')

# + id="lGwIqrqyWEo8" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="c8cdbee0-a2e3-4d64-efe8-82531b985a56"
# %cd "/content/drive/My Drive"

# + id="GuTteNFsN79e" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="51ea45fb-fecb-4010-905a-b1807e5cd102"
# !git clone https://github.com/huggingface/transformers

# + id="xqIBvl_vWaZj" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="0e2a7e41-5a77-41eb-a29d-83cb1047b307"
# %cd "transformers"

# + id="y9r5ULJ4VSR_" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 1000} outputId="172a3dd1-3535-4eca-e9c1-250ec9a63560"
# !pip install .
# !pip install -r ./examples/requirements.txt

# + [markdown] id="_SKRz3YErudT" colab_type="text"
# ## Training

# + id="0h87l7RSS7oN" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 1000} outputId="8374ff47-1ff5-4b35-eb5a-f858d79fb8ca"
# !python examples/language-modeling/run_language_modeling.py --block_size=256 --output_dir="/content/drive/My Drive/data/cover_letter_model" --model_name_or_path=gpt2 --do_train --train_data_file="/content/drive/My Drive/data/letters.train.tokens" --do_eval --eval_data_file="/content/drive/My Drive/data/letters.train.tokens"

# + id="68tO-BZvXQrq" colab_type="code" colab={}


# + [markdown] id="76cG3B3yr0s2" colab_type="text"
# ## Text Generation

# + id="emnthhsNr20x" colab_type="code" colab={}
from transformers import AutoTokenizer, GPT2LMHeadModel

# + id="445-Q5kzs0hf" colab_type="code" colab={}
TEXT_START = "Human Computer Interaction, Deep Learning, Machine Learning, Psychology, User Experience, Natural Language Processing, Software Engineering\nDear Mr. Morley,"

# + id="gDj9a-3EsD2E" colab_type="code" colab={}
MODEL_PATH = "/content/drive/My Drive/data/cover_letter_model"

# + id="5NQ_2zokr-2q" colab_type="code" colab={}
tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
model = GPT2LMHeadModel.from_pretrained(MODEL_PATH)

# + id="9n663wv1szt_" colab_type="code" colab={}
input_ids = tokenizer.encode(TEXT_START, return_tensors="pt")

# + id="hl6C0cN9sJCz" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="aa7d56a9-e3d9-483d-edce-59744c799037"
sample_outputs = model.generate(
    input_ids,
    do_sample=True,
    max_length=512,
    top_k= 50,
    top_p=0.92,
    num_return_sequences=3
)

# + id="C9atChAotbGp" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 819} outputId="b2ec5f86-15c7-4496-fe97-1a90b351264e"
print("Output\n" + "=" * 80)
for i, sample_output in enumerate(sample_outputs):
  print("{}: {}".format(i, tokenizer.decode(sample_output, skip_special_tokens=True)))

# + id="paXAjh_nsJ7a" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered