# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [conda env:tesurf] *
#     language: python
#     name: conda-env-tesurf-py
# ---

# + pycharm={"is_executing": false}
import sys
sys.path.append('..')

# + pycharm={"is_executing": false}
from tesufr import Processor, TextProcessParams, SummarySize
from tesufr.cores import SummaCore, FallbackCore
from tesufr.cores.em_core import EmCoresWrapper
from tesufr.corpora.providers import BbcNewsProvider, Krapivin2009Provider, LimitedProvider
from tesufr.keysum_evaluator import evaluate_processor_on_corpus
from tesufr.corpora import SetType, CorpusDocument, CorpusPurpose

# + pycharm={"is_executing": false}
processor_baseline = Processor([FallbackCore()])
processor_summa = Processor([SummaCore()])
processor_em = Processor([EmCoresWrapper()])


# -

def process_and_report(text, process_params, processor):
    doc = processor.process_text(text, process_params)
    print('====================================')
    print("Keywords: "+' | '.join([str(kw) for kw in doc.keywords]))
    print()
    print("Named entities:")
    for ne in doc.entities:
        print(f"{ne.lemma} ({ne.subkind})")
    print()
    print(f"Summary ({len(doc.summary)}):")
    for s in doc.summary:
        print("* "+s.lemma)


# https://www.theguardian.com/us-news/2019/may/02/why-we-are-addicted-to-conspiracy-theories
text_en = open('theguardian.txt', 'rt', encoding='utf-8').read()
print(text_en[:200])

tpp = TextProcessParams(SummarySize.new_relative(0.1), 10)
process_and_report(text_en, tpp, processor_baseline)

process_and_report(text_en, tpp, processor_summa)

process_and_report(text_en, tpp, processor_em)

# https://www.cicero.de/innenpolitik/grundgesetz-freiheit-demokratie-meinungsfreiheit-debattenkultur
text_de = open('cicero1.txt', encoding='utf-8').read()
print(text_de[:200])

tpp = TextProcessParams(SummarySize.new_relative(0.1), 10)
process_and_report(text_de, tpp, processor_em)



StarcoderdataJupyterScriptsDedupFiltered