exps = ['neurips_1_eval1', ]
paths = [os.path.expanduser("~/nta/results/{}".format(e)) for e in exps]
df = load_many(paths)

df.head(5)

df.columns

df.shape

df.iloc[1]

df.groupby('model')['model'].count()

"""
Explanation: Load and check data
End of explanation
"""


# Did any  trials failed?
df[df["epochs"]<30]["epochs"].count()

# Removing failed trials
df_origin = df.copy()
df = df_origin[df_origin["epochs"]>=30]
df.shape

# which ones failed?
# failed, or still ongoing?
df_origin['failed'] = df_origin["epochs"]<100
df_origin[df_origin['failed']]['epochs']

# helper functions
def mean_and_std(s):
    return "{:.3f} ± {:.3f}".format(s.mean(), s.std())

def round_mean(s):
    return "{:.0f}".format(round(s.mean()))

stats = ['min', 'max', 'mean', 'std']

def agg(columns, filter=None, round=3):
    if filter is None:
        return (df.groupby(columns)
             .agg({'val_acc_max_epoch': round_mean,
                   'val_acc_max': stats,                
                   'model': ['count']})).round(round)
    else:
        return (df[filter].groupby(columns)
             .agg({'val_acc_max_epoch': round_mean,
                   'val_acc_max': stats,                
                   'model': ['count']})).round(round)


"""
Explanation: ## Analysis
Experiment Details
End of explanation
"""


agg(['model'])

high_sparsity = (df['on_perc']==0.05)
avg_sparsity = (df['on_perc']==0.1)
low_sparsity = (df['on_perc']==0.2)

agg(['kwinners'], low_sparsity)

agg(['kwinners'], high_sparsity)

"""
Explanation: Did Hebbian perform better than SET?
End of explanation
"""


agg(['on_perc'])

"""
Explanation: No evidence of significant difference. In networks with high sparsity, the impact of kWinners is worst, which is expected since kWinners (at 30%) will make the activations more sparse than ReLU (which is 50% sparse on average)

What is the optimal level of weight sparsity?
End of explanation
"""


agg(['hebbian_grow'])

agg(['hebbian_grow'], low_sparsity)

agg(['hebbian_grow'], high_sparsity)

"""
Explanation: Sparsity at 80 and 90% levels seem more or less equivalent, difference is 1 point in accuracy. The jump from 90 to 95% shows a drastic increase in acc, of 6 points.

Hebbian grow helps learning?
End of explanation
"""


agg(['hebbian_prune_perc'])

agg(['hebbian_prune_perc'], low_sparsity)

agg(['hebbian_prune_perc'], high_sparsity)

"""
Explanation: No strong evidence it helps in low sparsity case. In high sparsity (95%), seems very harmful

Hebbian pruning helps learning?
End of explanation
"""


no_magnitude = (df['weight_prune_perc'] == 0)
agg(['hebbian_prune_perc'], no_magnitude)

no_magnitude = (df['weight_prune_perc'] == 0)
agg(['hebbian_prune_perc'], (no_magnitude & low_sparsity))

"""
Explanation: There is good evidence it helps. The trend is very clear in the low sparsity (80% sparse) cases.
End of explanation
"""


agg(['weight_prune_perc'])

agg(['weight_prune_perc'], low_sparsity)

agg(['weight_prune_perc'], high_sparsity)

agg(['weight_prune_perc'], avg_sparsity)

"""
Explanation: Results seem similar even when no magnitude pruning is involved, only hebbian pruning

Magnitude pruning helps learning?
End of explanation
"""


no_hebbian = (df['hebbian_prune_perc'] == 0)
agg(['weight_prune_perc'], no_hebbian)

"""
Explanation: In low sparsity cases, results are the same for any amount of pruning. In average and high sparsity, there is a gaussian like curve, with the peak at around 0.2 (maybe extending to 0.3). 
Results are consistent with what has been seen in previous experiments and in related papers.
Worth note that although results are better at 0.2, it also takes slightly longer to achieve better results compared to m
End of explanation
"""


pd.pivot_table(df, 
              index='hebbian_prune_perc',
              columns='weight_prune_perc',
              values='val_acc_max',
              aggfunc=mean_and_std)

pd.pivot_table(df[low_sparsity], 
              index=['kwinners','hebbian_prune_perc'],
              columns='weight_prune_perc',
              values='val_acc_max',
              aggfunc=mean_and_std)

pd.pivot_table(df[avg_sparsity], 
              index=['kwinners','hebbian_prune_perc'],
              columns='weight_prune_perc',
              values='val_acc_max',
              aggfunc=mean_and_std)

"""
Explanation: Somewhat inconsistent result looking at cases where there is no hebbian learning, only pruning by magnitude. There is an anomaly at the last entry where 50% of the weights are pruned - results are similar to 20%.
Number of samples averaged from is a lot lower in this pivot

What is the optimal combination of weight and magnitude pruning?
End of explanation
"""


pd.pivot_table(df[avg_sparsity], 
              index='hebbian_prune_perc',
              columns='weight_prune_perc',
              values='val_acc_max',
              aggfunc=mean_and_std)

pd.pivot_table(df[high_sparsity], 
              index='hebbian_prune_perc',
              columns='weight_prune_perc',
              values='val_acc_max',
              aggfunc=mean_and_std)

"""
Explanation: There is a more clear trend in the low sparsity case. Results from high sparsity are inconclusive, with several runs failing to "converge"
Weight pruning alone improves the model by up to 0.7% from 10% pruning to 50% magnitude pruning
Hebbian pruning alone improves the model by 1.5%
Both combined can increase from 1.5% seem in hebbian only to 1.8% improvement. 
Comparisons above are from 0.1 to 0.5 pruning. There is a question left of why no pruning at both sides - the (0,0) point - it is an anomaly to the trend shown in the pivot.
End of explanation
"""


