# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import spacy
import re
import random

# ### Read data

df_trn = pd.read_csv('germeval2018.training.txt', sep='\t', header=None, names=['text', 'bin', 'detail'])
print(df_trn.head())
df_trn['bin'].value_counts()

# The data is given as UTF-8 text and is quite balanced (1/3 of rows is offensive)

# +
# generate a list of samples

#idx = [3, 27, 65]
idx = random.sample(range(len(df_trn)), 10)

# +
# set up spacy
nlp = spacy.load('de')

for i in idx:
    sample = df_trn['text'][i]
    print(f'\n---\nsample: {sample}')
    # replace twitter handles
    sample_opt = sample
    #sample_opt = re.sub('@(\S+)', lambda m: m.group(1).capitalize(), sample)
    #sample_opt = re.sub('#(\S+)', lambda m: m.group(1), sample)
    #if sample is not sample_opt:
    #    print(f'optimized sample: {sample_opt}')
    print('')    
    doc = nlp(sample_opt)
    for token in doc:
        print("{0:<20}\t{1:20}\t{2:4}\t{3:5}\t{4}".format(token.text, token.lemma_, token.pos_, token.tag_, token.has_vector))

# -

t = "@spdde kein verläßlicher Verhandlungspartner."
t2 = re.sub('@(\S+)', lambda m: '@' + m.group(1).capitalize(), t)
t2

StarcoderdataJupyterScriptsDedupFiltered