# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# !pip install spacy --upgrade

import spacy
spacy.__version__

# !python -m spacy download ru_core_news_sm

# +
import spacy
from spacy.lang.ru.examples import sentences 

nlp = spacy.load("ru_core_news_sm")
doc = nlp(sentences[0])
print(doc.text)
for token in doc:
    print(token.text, token.pos_, token.dep_)

# +
import csv

cancel = []
cause  = []
rels = []
same = []

records = []
with open("ru_news_cause_v1.tsv", "r", encoding='utf-8') as r:
    reader = csv.reader(r, delimiter="\t")
    header = next(reader)
    for row in reader:
        r = dict(zip(header, row))
        if float(r["confidence"]) < 0.69:
            continue
        result = r["result"]
        mapping = {
#             "bad": 0,
            "rel": 0,
            "same": 0,
            "left_right_cause": 0,
            "left_right_cancel": 1,
            "right_left_cause": 0,
            "right_left_cancel": 1
        }
        if result not in mapping:
            continue
        if result.startswith('right'):
            r['left_title'], r['right_title'] = r['right_title'], r['left_title']
        if result.endswith('cause'):
            cause.append( (r['left_title'], r['right_title']) )
        elif result.endswith('cancel'):
            cancel.append( (r['left_title'], r['right_title']) )
        elif result.endswith('same'):
            same.append( (r['left_title'], r['right_title']) )
        elif result.endswith('rel'):
            rels.append( (r['left_title'], r['right_title']) )

        r["label"] = mapping[result]
        records.append(r)


# +
def get_verbs_lemmed(s):
    doc = nlp(s)
    negation_tokens = [tok for tok in doc if tok.dep_ == 'neg' or tok.lemma_ in ('не','передумать',
                            'отменить','отказаться')]
    negated_tokens = [child.lemma_ for token in negation_head_tokens for child in token.children]
    verbs = []
    for token in doc:
        if token.is_stop: continue
        if token.pos_ == 'VERB':
            if token.lemma_ in negated_tokens:
                verbs.append( 'не_'+token.lemma_ )
            else:
                verbs.append( token.lemma_ )
    return tuple(verbs)

def pairs2verbs(l):
    return tuple( [ (get_verbs_lemmed(s1), get_verbs_lemmed(s2)) for s1, s2 in l ] )

cancel_verbs = pairs2verbs(cancel)
cause_verbs = pairs2verbs(cause)
rels_verbs = pairs2verbs(rels)
same_verbs = pairs2verbs(same)
cancel_verbs

# +
from collections import Counter

def verbs2stats(pl):
    left_verbs = []
    right_verbs = []
    same_verbs = []
    pairs_verbs = []
    neg_pairs_verbs = []
    for vv1, vv2 in pl:
        left_verbs.extend(vv1)
        right_verbs.extend(vv2)
        same_verbs.extend(list(set(vv1)&set(vv2)))
        for v1 in vv1:
            for v2 in vv2:
                pairs_verbs.append( (v1,v2) )
                if (v1 in v2 or v2 in v1) and (v1.startswith('не_') or v2.startswith('не_')):
                    neg_pairs_verbs.append( (v1,v2) )
    print('\nleft verbs stats')
    print('\t'+'\n\t'.join(map(str, Counter(left_verbs).most_common(5))))
    print('\nright verbs stats')
    print('\t'+'\n\t'.join(map(str, Counter(right_verbs).most_common(5))))
    print('\nsame verbs stats')
    print('\t'+'\n\t'.join(map(str, Counter(same_verbs).most_common(5))))
    print('\npair verbs stats')
    print('\t'+'\n\t'.join(map(str, Counter(pairs_verbs).most_common(5))))
    print('\nneg pair verbs stats')
    print('\t'+'\n\t'.join(map(str, Counter(neg_pairs_verbs).most_common(5))))
    
    
print('CANCEL')
verbs2stats(cancel_verbs)
print('\nCAUSE')
verbs2stats(cause_verbs)
print('\nREL')
verbs2stats(rels_verbs)
print('\nSAME')
verbs2stats(same_verbs)


# +
cancel_sim = [ nlp(s1).similarity(nlp(s2)) for s1, s2 in cancel ]
cause_sim = [ nlp(s1).similarity(nlp(s2)) for s1, s2 in cause ]
rels_sim = [ nlp(s1).similarity(nlp(s2)) for s1, s2 in rels ]
same_sim = [ nlp(s1).similarity(nlp(s2)) for s1, s2 in same ]



# +
import numpy as np
import scipy.stats

def stat_tests(sample1,sample2):
    try:
        mw = scipy.stats.mannwhitneyu(sample1,sample2,alternative='two-sided')[1]
    except:
        mw = "nan"
    tt = scipy.stats.ttest_ind(sample1,sample2)[1]
#     print(mw, tt)
    return (float(mw),float(tt))

def compare_similarities(v1, v2):
    ctl_proj = v1
    exp_proj = v2
    print(f'means',np.mean(v1), np.mean(v2))
    print(f'means diff',float(np.mean(ctl_proj)-np.mean(exp_proj)))
    print(f'stat tests', stat_tests(ctl_proj,exp_proj))

for t1,c1 in (('CANCEL', cancel_sim), ('CAUSE', cause_sim), ('RELS', rels_sim), ('SAME', same_sim), ):
    for t2,c2 in (('CANCEL', cancel_sim), ('CAUSE', cause_sim), ('RELS', rels_sim), ('SAME', same_sim), ):
        print(f'{t1} VS {t2}')
        compare_similarities(c1,c2)
        print()

# +
from collections import Counter

def save_gdf(fn, pl):
    nodes = Counter()
    edges = Counter()
    for vv1, vv2 in pl:
        for v1 in vv1:
            nodes.update([v1,])
        for v2 in vv2:
            nodes.update([v2,])
        for v1 in vv1:
            for v2 in vv2:
                edges.update([(v1,v2)])
    with open(fn, 'w', encoding='utf-8') as fh:
        print('nodedef>name VARCHAR,label VARCHAR,cnt DOUBLE', file=fh)
        for n, c in nodes.items():
            print(f'{n},{n},{c}', file=fh)
        print('edgedef>node1 VARCHAR,node2 VARCHAR,directed BOOLEAN,label VARCHAR', file=fh)
        for nn, l  in edges.items():
            n1, n2 = nn
            print(f'{n1},{n2},true,{l}', file=fh)

save_gdf('cancel.gdf', cancel_verbs)
save_gdf('cause.gdf', cause_verbs)
save_gdf('rels.gdf', rels_verbs)
save_gdf('same.gdf', same_verbs)
# -

# ## Cancel
#
# ![image.png](attachment:image.png)
#
#
# ## Cause
#
# ![image-2.png](attachment:image-2.png)

StarcoderdataJupyterScriptsDedupFiltered