import pandas
import random
import csv
import seaborn as sns


"""
Explanation: This notebook analyzes a toughies.csv file. This file is generated by the Django management command named get_toughie_info.py with a production db backup
End of explanation
"""


LEXICAL_FAMILY = 'CSW'

toughies = pandas.read_csv('./toughies.csv')
len(toughies)

"""
Explanation: Set the following variables to CSW or NWL depending on what we are generating (and what the input is)
End of explanation
"""


better_toughies = toughies.loc[toughies['asked'] >= 26]


# Order of dictionary updates:
def lexkey_assigner(row):
    return {
        'OWL2': 1,
        'America': 2,
        'NWL18': 3,
        'NWL20': 4,
        ## CSW
        'CSW12': 5,
        'CSW15': 6,
        'CSW19': 7,
    }[row['lexicon']]

better_toughies = better_toughies.assign(
    lexkey=better_toughies.apply(lexkey_assigner, axis=1)).sort_values('lexkey')


# Determine which words have NOT been asked yet. 
# Can create these files like this, for example:
# sqlite> .mode csv
# sqlite> .output 7s_alphs.csv
# sqlite> select alphagram, contains_word_uniq_to_lex_split from alphagrams where length = 7;
# sqlite> .output 8s_alphs.csv
# sqlite> select alphagram, contains_word_uniq_to_lex_split from alphagrams where length = 8;

if LEXICAL_FAMILY == 'CSW':
    lexica = ['CSW12', 'CSW15', 'CSW19']
elif LEXICAL_FAMILY == 'NWL':
    lexica = ['OWL2', 'America', 'NWL18', 'NWL20']

LATEST_LEXICON = lexica[-1]

alphas_7s = pandas.read_csv(f'./{LATEST_LEXICON}_7s_alphs.csv', header=None, names=['Alphagram', 'lexuniq'], index_col='Alphagram')
alphas_8s = pandas.read_csv(f'./{LATEST_LEXICON}_8s_alphs.csv', header=None, names=['Alphagram', 'lexuniq'], index_col='Alphagram')


alphas_7s.loc['ACEIORT']['lexuniq']
# alphas_8s.loc['AEINOSTV']['lexuniq']

# These text files below were created like:
# sqlite> .header on
# sqlite> .output 7s_newin_CSW19.csv
# sqlite> select alphagram from alphagrams where length(alphagram) = 7 and contains_update_to_lex = 1;
# sqlite> .output 8s_newin_CSW19.csv
# sqlite> select alphagram from alphagrams where length(alphagram) = 8 and contains_update_to_lex = 1;



new_sevens_first_update = pandas.read_csv(f'./7s_newin_{lexica[1]}.csv', index_col='alphagram')
new_sevens_second_update = pandas.read_csv(f'./7s_newin_{lexica[2]}.csv', index_col='alphagram')
new_eights_first_update = pandas.read_csv(f'./8s_newin_{lexica[1]}.csv', index_col='alphagram')
new_eights_second_update = pandas.read_csv(f'./8s_newin_{lexica[2]}.csv', index_col='alphagram')


better_toughies.loc[better_toughies['Alphagram'] == 'AADHNPR']

better_toughies.loc[better_toughies['Alphagram'] == 'EEIRSTU']

asked_7s = set()
asked_8s = set()

# Start at the first lexicon. Toughies are sorted chronologically by lexicon.
last_lex = lexica[0]
for _, row in better_toughies.iterrows():
    lex = row['lexicon']
    alpha = row['Alphagram']
    if lex not in lexica:
        continue
    # Clear out questions that got new additions.
    if lex != last_lex:
        if lex == lexica[1]:
            for row in new_sevens_first_update.iterrows():
                if row[0] in asked_7s:
                    asked_7s.remove(row[0])
            for row in new_eights_first_update.iterrows():
                if row[0] in asked_8s:
                    asked_8s.remove(row[0])
        elif lex == lexica[2]:
            for row in new_sevens_second_update.iterrows():
                if row[0] in asked_7s:
                    asked_7s.remove(row[0])
            for row in new_eights_second_update.iterrows():
                if row[0] in asked_8s:
                    asked_8s.remove(row[0])
        
    if len(alpha) == 7 and alpha in alphas_7s.index:
        asked_7s.add(alpha)
    if len(alpha) == 8 and alpha in alphas_8s.index:
        asked_8s.add(alpha) 
        
    last_lex = lex
        
print(f'Asked {len(asked_7s)} out of {len(alphas_7s)} 7s')
print(f'Asked {len(asked_8s)} out of {len(alphas_8s)} 8s')



print((len(alphas_7s) - len(asked_7s)) / 50)
print((len(alphas_8s) - len(asked_8s)) / 50)

# We want to add asked_7s and asked_8s from NWL as long as the alphagram doesn't have any CSW-only words.
# i.e. imagine OUTRIDE has not been asked in CSW. We should not add the stats from NWL because ETOURDI# / IODURET# 
if LEXICAL_FAMILY == 'CSW':
    lexica = ['OWL2', 'America', 'NWL18', 'NWL20']
    for _, row in better_toughies.iterrows():
        lex = row['lexicon']
        if lex not in lexica:
            continue    
        alpha = row['Alphagram']
        if len(alpha) == 7:
            if alpha not in alphas_7s.index:
                # Skip the very few NWL-only bingos
                continue
            if alphas_7s.loc[alpha]['lexuniq']:
                # Skip if it has been asked in NWL, but the alphagram has a CSW-only solution
                continue
            asked_7s.add(alpha)
        if len(alpha) == 8:
            if alpha not in alphas_8s.index:
                # Skip the very few NWL-only bingos
                continue
            if alphas_8s.loc[alpha]['lexuniq']:
                continue

            asked_8s.add(alpha)
    
    print(f'Counting NWL, asked {len(asked_7s)} out of {len(alphas_7s)} 7s')
    print(f'Counting NWL, asked {len(asked_8s)} out of {len(alphas_8s)} 8s')
    print((len(alphas_7s) - len(asked_7s)) / 50)
    print((len(alphas_8s) - len(asked_8s)) / 50)

print('missing 7s')
print(len(set(alphas_7s.index) - asked_7s))

print('missing 8s')
print(len(set(alphas_8s.index) - asked_8s))

"""
Explanation: For better statistical significance, filter only bingos that were asked at least 30 times.
End of explanation
"""


bingos = {}
# These numbers are somewhat hand-wavy. They should still result in decent results in most cases.
weights = {
    'OWL2': 0.15,
    'America': 0.25,
    'NWL18': 0.3,
    'NWL20': 0.3,
    # CSW numbers. These might or might not combine with the above. 
    'CSW12': 0.15,
    'CSW15': 0.35,
    'CSW19': 0.5,
}
pandas.set_option("display.max_rows", 100, "display.max_columns", None)
def aggregation_fn(group):
    d = {}
    numerator = 0
    denominator = 0
    probability = 0
    for _, row in group.iterrows():
        lex = row['lexicon']
        if LEXICAL_FAMILY == 'CSW' and lex in ['OWL2', 'America', 'NWL18', 'NWL20']:
            # Don't count NWL weights where the alphagram has CSW-only solutions,
            # otherwise it's inaccurate:
            if len(row['Alphagram']) == 8 and alphas_8s.loc[row['Alphagram']]['lexuniq']:
                continue
            if len(row['Alphagram']) == 7 and alphas_7s.loc[row['Alphagram']]['lexuniq']:
                continue
        numerator += row['missed'] * weights[row['lexicon']]
        denominator += row['asked'] * weights[row['lexicon']]
    d['weighted_difficulty'] = numerator/denominator
    d['weighted_numerator'] = numerator
    d['weighted_denominator'] = denominator
    d['length'] = len(row['Alphagram'])
    return pandas.Series(d)

bingos = better_toughies.groupby('Alphagram').apply(aggregation_fn).reset_index()




bingos.sort_values('weighted_difficulty', ascending=False).head(100)[["Alphagram", "weighted_difficulty"]].to_clipboard()


# bdf = pandas.DataFrame.from_dict(bingos, orient='index')

# Now we can ask some questions. For example, what are the hardest 1000 bingos with probability < 15000?

# total = 100
# prob_limit = 50000
# alphas = bdf[bdf['probability'] <= prob_limit].sort_values('difficulty', ascending=False)[:total][['Alphagram', 'probability','difficulty']]
# ct = 0
# for _, row in alphas.iterrows():
#     if len(row['Alphagram']) == 7:
#         ct += 1
# print(f'There are {ct} 7s out of {total}')
# # for alpha in alphas:
# #     print(alpha)
# pandas.set_option("display.max_rows", None, "display.max_columns", None)

# print(alphas)

sevens = bingos[bingos["length"] == 7].copy()
eights = bingos[bingos["length"] == 8].copy()
nquantiles = 100

qlabels = [f'q{q}' for q in range(nquantiles)]

sevens.sort_values('weighted_difficulty', ascending=False)
sevens['quantile'] = pandas.qcut(sevens['weighted_difficulty'], nquantiles, labels=qlabels)
sevens.to_csv(f'{LATEST_LEXICON}_7s_difficulty.csv', index=True)
print(pandas.qcut(sevens['weighted_difficulty'], nquantiles))

eights.sort_values('weighted_difficulty', ascending=False)
eights['quantile'] = pandas.qcut(eights['weighted_difficulty'], nquantiles, labels=qlabels)
eights.to_csv(f'{LATEST_LEXICON}_8s_difficulty.csv', index=True)
print(pandas.qcut(eights['weighted_difficulty'], nquantiles))

sns.histplot(sevens['weighted_difficulty'])

sns.histplot(eights['weighted_difficulty'])

bingos.loc[bingos['Alphagram'] == 'ABGHOSTU']

"""
Explanation: Determine a list of all bingos by difficulty!
The methodology here is tricky. We can try a few things:


Since the df is sorted from oldest to newest lexicon, results from newer "asks" will supersede older asks. We can make it so that if the new bingo was asked at least X more times (7 seems like an ok number, so if the old bingo was asked 70 times and the new bingo was asked 77 times, more data is better, so take the newer number), then we default to the newer number.


The problem with 1 is that old Aerolith users skewed significantly more expert than new ones. Look up any tough bingo in the dataframe and the number wrong invariably goes up. So some very tough alphagrams, like ABGHOSTU, have only a 62.6% miss rate in the OWL2 days, but a 89.5% miss rate in the NWL18 days. We could strictly overwrite older data with newer data, but there is typically less newer data than there is older data, so the data might not be as good.


Straight average. Since there is more old data than new data, this will still skew it a bit towards old data, but it might not be as bad.


Weighted average. We can normalize the eras for the different lexica to the same toal number, or somehow count older data less than newer data.
e.g.
Alphagram    probability    asked    missed    difficulty    lexicon 
ABGHOSTU       15394        147      92        0.625850      OWL2    
ABGHOSTU       16059        77       61        0.792208      America 
ABGHOSTU       16088        57       51        0.894737      NWL18

weigh it 20/30/50
0.2 * 0.6285 + 0.3 * 0.7922 + 0.5 * 0.894737 = 0.8107
or by raw numbers
(0.2 * 92 + 0.3 * 61 + 0.5 * 51) / (0.2 * 147 + 0.3 * 77 + 0.5 * 57) = 0.7679
If we use NWL20, we can use weights: 0.15, 0.25, 0.3, 0.3 (for example)
(0.15 * 92 + 0.25 * 61 + 0.3 * 51) / (0.15 * 147 + 0.25 * 77 + 0.3 * 57) = 0.759
End of explanation
"""


missing_8s = list(set(alphas_8s.index) - asked_8s)
missing_7s = list(set(alphas_7s.index) - asked_7s)

asked_7s_new = list(asked_7s)
asked_8s_new = list(asked_8s)
random.shuffle(asked_7s_new)
random.shuffle(asked_8s_new)

# Extend the 8s by so we have 1500 exactly.
missing_8s.extend(asked_8s_new[:371])
# Extend the 7s so we have 200 exactly
missing_7s.extend(asked_7s_new[:110])
assert(len(missing_7s) == 200)
assert(len(missing_8s) == 1500)

random.shuffle(missing_7s)
random.shuffle(missing_8s)

i = 0
for seven in missing_7s:
    print(seven)
    i += 1
    if i % 50 == 0:
        print('-' * 6)

print ('-' * 12)
i = 0
for eight in missing_8s:
    print(eight)
    i += 1
    if i % 50 == 0:
        print('-' * 6)

"""
Explanation: Run the following cell to determine which questions are left to ask (maybe can use for future updates or CSW)
End of explanation
"""


num_days = 10 * 365   # We've been asking qs for roughly 10 years (since Jun 2011 -- update if changes)
num_qs = 50
num_alphas = 36549   # How many sevens or eights

alphas = set(range(num_alphas))

for i in range(num_days):
    todays = list(range(num_alphas))
    random.shuffle(todays)
    for q in todays[:num_qs]:
        if q in alphas:
            alphas.remove(q)

print(len(alphas))

"""
Explanation: Challenge simulator - how many unasked questions do we have after a certain time?
End of explanation
"""


