# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Transitivity

# ## Set up modules, variables, functions

# +
import sys
import collections
import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 500)
from tf.fabric import Fabric
from tf.app import use
import matplotlib.pyplot as plt

# import some project code from elsewhere
# Lord, please forgive me for my import sins
sys.path.append('/Users/cody/github/CambridgeSemiticsLab/time_collocations/tools/stats/')
from pca import apply_pca
from significance import apply_deltaP, apply_fishers
sys.path.append('/Users/cody/github/CambridgeSemiticsLab/Gesenius_data/analysis/')
from text_show import TextShower

# load Text-Fabric with datasets downloaded from Github
# NB that URLs of the original Github repos can be inferred
# from the pathnames. Just replace github/ with github.com/
locations = [
    '~/github/etcbc/bhsa/tf/c', # main ETCBC data
    '~/github/etcbc/heads/tf/c', # for phrase head data (i.e. for complements)
]

# load TF data, give main classes shortform names, instantiate the use app with gives convenient methods
TF = Fabric(locations=locations)
API = TF.load('''

vs vt pdp lex lex_utf8
rela code function freq_lex
typ nhead
label gloss number 
mother language
''')

A = use('bhsa', api=API)
F, E, T, L = A.api.F, A.api.E, A.api.T, A.api.L

sys.path.append('../')
from clause_relas import in_dep_calc as clause_relator


# -

def get_spread(array, n):
    """Retrieve an even spread of indices an array/Series.
    
    https://stackoverflow.com/a/50685454/8351428
    
    Args:
        array: either numpy array or Pandas Series 
            (with multiple indices allowed)
        n: number of indices to return
    Returns:
        indexed array or series
    """
    end = len(array) - 1
    spread = np.ceil(np.linspace(0, end, n)).astype(int)
    indices = np.unique(spread)
    try:
        return array[indices]
    except KeyError:
        
        return array.iloc[indices]


ts = TextShower(
    default=['ref','verb_id', 'text' , 'clause', 'has_obj'],
    stylize=['text', 'clause']
)


# ## Build the dataset

# +
def clause_has_object(verb, clause_atom, clause):
    """Search a given clause for any marked objects."""
    
    clause_phrases = L.d(clause, 'phrase')
    daughters = E.mother.t(clause_atom)
    daught_relas = set(F.rela.v(d) for d in daughters)
    daught_codes = set(F.code.v(d) for d in daughters)
    phrase_functs = set(F.function.v(p) for p in clause_phrases)
    
    # we count direct speech clauses following amar as direct objects
    amar_obj = F.lex.v(verb) == '>MR[' and 999 in daught_codes 
    
    # evaluate conditions for object presence
    return any([
        'Objc' in phrase_functs,
        'PreO' in phrase_functs,
        'PtcO' in phrase_functs,
        'Objc' in daught_relas,
        amar_obj,
    ])

def cmpl_data(clause):
    """Tag complement data within a clause"""
    data = {
        'has_cmpl': 0,
#         'cmpl_ph_type': np.nan,
#         'cmpl_heads': np.nan,
    }
    
    # check for complement
    cmpl_ph = [p for p in L.d(clause, 'phrase') if F.function.v(p) == 'Cmpl']
    
    # detect presence of complement
    if cmpl_ph:
        data['has_cmpl'] = 1
    
    return data
    
dataset = []

for verb in F.pdp.s('verb'):
    
    # ignore low frequency clauses
    if F.freq_lex.v(verb) < 10:
        continue
    
    clause = L.u(verb, 'clause')[0]
    ca_rela = clause_relator(clause)
    
    # ignore non-main clauses to avoid 
    # the problem of, e.g., אשֶׁר subjects / objects
    # which are not marked in the database, as well as other
    # potential complications
    if ca_rela != 'Main':
        continue
    
    # process the data
    book, chapter, verse = T.sectionFromNode(verb)
    clause_atom = L.u(verb, 'clause_atom')[0]
    ref = f'{book} {chapter}:{verse}'
    vs = F.vs.v(verb)
    lex = F.lex_utf8.v(verb)
    lex_node = L.u(verb, 'lex')[0]
    verb_id = f'{lex}.{vs}'
    
    has_obj = 1 * clause_has_object(verb, clause_atom, clause)
    
    cl_data = {
        'node': verb,
        'ref': ref,
        'lex_node': lex_node,
        'lex': lex,
        'text': T.text(verb),
        'clause': T.text(clause), 
        'clause_node': clause,
        'clause_atom': clause_atom,
        'verb_id': verb_id,
        'has_obj': has_obj,
    }
    cl_data.update(cmpl_data(clause))
    dataset.append(cl_data)
    
vdf = pd.DataFrame(dataset)
vdf = vdf.set_index('node')

print(vdf.shape)

vdf.head(5)
# -

ts.show(vdf, spread=10)

# ## Count verb lexeme object tendencies

# +
vo_ct = pd.pivot_table(
    vdf,
    index='verb_id',
    columns='has_obj',
    aggfunc='size',
    fill_value=0,
)

vo_ct = vo_ct.loc[vo_ct.sum(1).sort_values(ascending=False).index]

vo_ct.head(10)
# -

# ### Cull dataset down

print('present data shape:')
vo_ct.shape

vo_ct.sum(1).plot()

# +
vo_ct2 = vo_ct[vo_ct.sum(1) >= 30] # keep those with N observations

print('new shape of data:')
vo_ct2.shape

# +
vo_pr = vo_ct2.div(vo_ct2.sum(1), 0)

vo_pr.head(10)
# -

# # Cluster them

import seaborn as sns

# +
fig, ax = plt.subplots(figsize=(5, 8))

sns.stripplot(ax=ax, data=100*vo_pr.iloc[:, 1:2], jitter=0.2, edgecolor='black', linewidth=1, color='lightblue', size=10)
ax.set_ylabel('% of verb lexeme with an object', size=14)
plt.savefig('/Users/cody/Desktop/verb_objects.png', dpi=300, bbox_inches='tight', facecolor='white')
# -

# ## Select a subset for transitivity tagging
#
# We will select the prototypical, unambiguous cases.
#
# These seem to be those from 0-5% (intransitive)
# and those from 60-100%. We'll leave the rest of the cases.

tran = vo_pr[vo_pr[1] >= 0.6]
itran = vo_pr[vo_pr[1] <= 0.05]

tran

itran

# how many verbs would this account for?
vo_ct.loc[np.array(tran.index, itran.index)].sum(1).sum()

# ## Export for Inspection

# +
from pathlib import Path
from df2gspread import df2gspread as d2g
import gspread
from oauth2client.service_account import ServiceAccountCredentials

scope = ['https://spreadsheets.google.com/feeds',
         'https://www.googleapis.com/auth/drive']

credentials = ServiceAccountCredentials.from_json_keyfile_name(
    '/Users/cody/.config/gspread/service_account.json', scope
)

drive_id = Path.home().joinpath('github/CambridgeSemiticsLab/Gesenius_data/data/_private_/keys/tran_folder.txt').read_text().strip()
# -

gc = gspread.authorize(credentials)
tran_sh = gc.create("transitive_verbs", drive_id)
itran_sh = gc.create('intransitive_verbs', drive_id)
d2g.upload(tran.round(2), tran_sh.id, 'Sheet1', 
           row_names=True, col_names=True, credentials=credentials)
d2g.upload(itran.round(2), itran_sh.id, 'Sheet1', 
           row_names=True, col_names=True, credentials=credentials)

# ## Cluster on multivariate tendencies
#
# A lot of dynamic verbs are in the intransitive list. It would be nice to have some separation for categories close to a stative / dynamic dichotomy. It may be possible to do this with a PCA analysis by using the `Cmpl` arguments.

# +
va_ct = pd.pivot_table(
    vdf,
    index='verb_id',
    values=['has_obj', 'has_cmpl'],
    aggfunc='sum',
    fill_value=0,
)

vlex_ct = vdf.verb_id.value_counts()
vlex_ct = vlex_ct[vlex_ct >= 15] # restrict frquency to N and up

va_ct = va_ct.loc[vlex_ct.index]
va_pr = va_ct.div(vlex_ct, 0)

va_pr.head(25)
# -

va_pr.shape

va_pr.loc['ידע.qal']

va_pr[
    (va_pr.has_obj < 0.3)
    & (va_pr.has_cmpl < 0.1)
]



StarcoderdataJupyterScriptsDedupFiltered