# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# ### Creation of development ChEMBL corpus
# We create a smaller sampled ChEMBL corpus for the purpose of rapid iteration
# on the development on our model

# +
import pandas as pd

from src.drugexr.config.constants import PROC_DATA_PATH
# -

_SEED = 42
_DEV_CORPUS_SIZE = 1_000

chembl_df = pd.read_table(filepath_or_buffer=PROC_DATA_PATH / "chembl_corpus.txt")

dev_chembl_df = chembl_df.sample(n=_DEV_CORPUS_SIZE, random_state=_SEED, ignore_index=True)

dev_chembl_df.to_csv(path_or_buf= PROC_DATA_PATH / f"chembl_corpus_DEV_{_DEV_CORPUS_SIZE}.txt", encoding="utf-8", sep="\t" ,index=False)



StarcoderdataJupyterScriptsDedupFiltered