# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: anaconda
#     language: python
#     name: python3
# ---

# +
import csv

def to_tsv(infpath, outfpath):
    with open(infpath, 'r') as infile, open(outfpath, 'w+', newline='') as outfile:
        reader = csv.reader(infile, delimiter='\t')
        writer = csv.writer(outfile, delimiter='\t')
        for label, text in reader:
            tmp=text.split('/')
            tmp=[t for t in tmp if len(t.split()) > 3]
            for newtext in tmp:
                if len(newtext.strip()) > 10:
                    writer.writerow([label, newtext.replace(' & amp ; ', ' and ')])
                


# -

# ! mkdir bertified

# +
to_tsv('dev.tsv', 'bertified/dev.tsv')


# -

to_tsv('test.tsv', 'bertified/test.tsv')

to_tsv('train.tsv', 'bertified/train.tsv')



StarcoderdataJupyterScriptsDedupFiltered