# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: projectname
#     language: python
#     name: projectname
# ---

import sys
import json
sys.path.append('..')
import tagnews
import matplotlib.pyplot as plt
import datetime as dt
import numpy as np
import pandas as pd
pd.set_option('display.width', 150)
pd.set_option('max.columns', 15)

# # Determine the costs of processing existing articles
#
# Based on complete data files from through 2019-09-07.
#
# Each 1000 words of an article submitted is one "unit", rounded up.
#
# 1,496,665 units total = $2487 to process at once, or 300 months in free batches of 5k...

# +
crimetags = tagnews.CrimeTags()

df_all = tagnews.load_data()
df_all['read_date'] = df_all['created'].str.slice(0, 10)
### Limiting it to last two years because the data volume is unstable before that
df = df_all.loc[df_all['read_date'] >= '2017-01-01']
del df_all
### Number of units to process title and article through Google Cloud API
df['n_chars'] = df['title'].str.len() + df['bodytext'].str.len()
df['n_units'] = np.ceil(df['n_chars']/1000.)


# -

def calculate_google_nlp_price(total_units, verbose=True):
    '''Cost to run entity sentiment analysis on a given number of 
    units in a single month through in Google Cloud API.
    https://cloud.google.com/natural-language/#natural-language-api-pricing
    
    First 5000 = free
    5k-1M  = $2 per 1000 units
    1M-5M  = $1 per 1000 units
    5M-20M = $0.5 per 1000 units
    '''
    free_units        = min(5e3, total_units)
    first_tier_units  = min(1e6-5e3, total_units-free_units)
    second_tier_units = min(5e6-1e6, total_units-free_units-first_tier_units)
    third_tier_units  = max(0, total_units-free_units-first_tier_units-second_tier_units)
    units = [free_units, first_tier_units, second_tier_units, third_tier_units]
    costs = [0, 2., 1., 0.5]
    total_cost = sum([c*np.ceil(u/1e3) for (c, u) in zip(costs, units)])
    if verbose:
        print('{:.0f} units: {:.0f}*0 + {:.0f}*$2 + {:.0f}*$1 + {:.0f}*$0.50 = ${:.2f}'
              .format(total_units,
                      np.ceil(free_units/1e3),
                      np.ceil(first_tier_units/1e3),
                      np.ceil(second_tier_units/1e3),
                      np.ceil(third_tier_units/1e3),
                      total_cost))
    return total_cost


units = df['n_units'].sum()
cost = calculate_google_nlp_price(units)

units_per_day = (df
                 .groupby('read_date')
                 .agg({'url': 'count',
                       'n_units': 'sum'})
                )
print(units_per_day.index.min(), units_per_day.index.max())

### Number of units coming in per day
### Typically ranges from 800-2000 daily, so definitely >5000 monthly
f1, ax1 = plt.subplots(1, figsize=[15, 6])
ax1.plot(range(units_per_day.shape[0]), units_per_day['n_units'], label='# units')

# # Relevance scoring/binning

### Full dataset takes up too much memory, so dropping all but the most recent now
### This keeps 276122 of the original 1.5e6, or a little less than 1/5th of the total
df2 = df.loc[df['read_date'] >= '2019-03-01']
del df

new_units = df2['n_units'].sum()
downscale = new_units/units
print(new_units, downscale)

# +
### Assign a made-up CPD relevance score

### Words associated with CPD
cop_words = [
    "cpd",
    "police",
    "officer",
    "cop",
    "officers",
    "pigs",
    "policeofficer",
  ]
### Count number of times relevant words appear in title or text
df2['cop_word_counts'] = 0
for w in cop_words:
    df2['cop_word_counts'] += df2['bodytext'].str.lower().str.count(w)
    df2['cop_word_counts'] += df2['title'].str.lower().str.count(w)
# -

df2['cop_word_counts'].describe()

### Does the word count measure the same thing as the CPD_model column? 
### No, doesn't look very correlated actually...
f1, ax1 = plt.subplots(1, figsize=[14,6])
ax1.scatter(df2['cop_word_counts'], df2['CPD_model'], alpha=0.3, s=5)
ax1.set_xlabel('cop word count')
ax1.set_ylabel('CPD_model')

### See examples that use the relevant words but didn't score highly in CPD_model
### Some definitely look relevant (e.g. article 650870)
relevant_but_zero = df2.loc[(df2['CPD_model']==0) & ((df2['CPD']==0))].sort_values('cop_word_counts', ascending=False)
print(relevant_but_zero.loc[650870, 'title'])
print(relevant_but_zero.loc[650870, 'bodytext'])

### Basic relevance score:
### - 50% human tagged "CPD"
### - 25% "CPD_model"
### - 25% usage of above words
df2['CPD_relevance'] = (  0.5*df2['CPD'] # upweight because it means more
                       + 0.25*df2['CPD_model']
                       + 0.25*(df2['cop_word_counts']/(2*len(cop_words))).clip(upper=1.)
                      )
### 55% have relevance = 0
### 
df['relevance_tier'] = 0
df.head()

### What number/fraction have score > 0?
print(df2.loc[df2['CPD_relevance']>0, 'n_units'].sum(), (df2['CPD_relevance']>0).mean())
### What number/fraction have score = 0?
print(df2.loc[df2['CPD_relevance']==0, 'n_units'].sum(), (df2['CPD_relevance']==0).mean())

# +
### About half of scores are 0
### What is the distribution of the nonzero ones?
nonzero_scores = df2.loc[df2['CPD_relevance']>0].sort_values('CPD_relevance', ascending=False)

f1, ax1 = plt.subplots(1, figsize=[14, 6])
ax1.hist(nonzero_scores['CPD_relevance'], bins=20)
# -

5000*downscale

### Divide this sample into groups of 900 rows each, in order to get
### sizes needed for bins that would be ~5000 each.
### This ould actually be a bit too big, but you get the general idea
### Bins would have to get progressively smaller as we go down to stay equal in number
nonzero_scores['CPD_relevance'].iloc[[i*900 for i in range(1, int(np.ceil(nonzero_scores.shape[0]/900)))]]



StarcoderdataJupyterScriptsDedupFiltered