# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3.8.3 64-bit
#     name: python38364bitd11d0bbfcc9343bf86e51fc58926f404
# ---

# +
import numpy as np
import pandas as pd

from pathlib import Path

# +
data_dir = Path("./plmdata/raw")

alsfrs_fn = data_dir/"user_ALSFRS_score.csv"
condition_fn = data_dir/"user_condition.csv"
onset_fn = data_dir/"user_onset_date.csv"
symptom_fn = data_dir/"user_symptom.csv"

als = "amyotrophic lateral sclerosis"

# +
alsfrs = pd.read_csv(alsfrs_fn, index_col=0)
alsfrs["report_date"] = pd.to_datetime(alsfrs["report_date"])

condition = pd.read_csv(condition_fn, index_col=0)
onset = pd.read_csv(onset_fn, index_col=0)

onset["first_definitive_diagnosis_date"] = pd.to_datetime(onset["first_definitive_diagnosis_date"])

symptom = pd.read_csv(symptom_fn, index_col=0)

# + tags=[]
print(alsfrs.columns)
print(condition.columns)
print(onset.columns)
print(symptom.columns)

# + tags=[]
print("Rows by table:")
print(f"  alsfrs:{len(alsfrs)}")
print(f"  condition:{len(condition)}")
print(f"  onset:{len(onset)}")
print(f"  symptom:{len(symptom)}")
# -

# # Check first few rows by table

# +
# alsfrs.head()

# +
# condition.head()

# +
# onset.head()

# +
# symptom.head()
# -

# # Check counts of rows by user_id

# +
alsfrs_users = alsfrs.user_id.unique()

len(alsfrs_users)
# -

condition_users = condition.user_id.unique()

len(condition_users)

# +
onset_users = onset.user_id.unique()

len(onset_users)

# +
symptom_users = symptom.user_id.unique()

len(symptom_users)

# + tags=[]
# Of 200 user_ids in alsfrs table, how many are in onset?

print(f"Number of alsfrs users (total {len(alsfrs_users)}) in onset: {np.sum(np.isin(onset_users, alsfrs_users))}")
print(f"Number of alsfrs users (total {len(alsfrs_users)}) in condition: {np.sum(np.isin(condition_users, alsfrs_users))}")
print(f"Number of alsfrs users (total {len(alsfrs_users)}) in symptom: {np.sum(np.isin(symptom_users, alsfrs_users))}")
print(f"Number of onset users (total {len(onset_users)}) in condition: {np.sum(np.isin(condition_users, onset_users))}")
print(f"Number of onset users (total {len(onset_users)}) in symptom: {np.sum(np.isin(symptom_users, onset_users))}")
print(f"Number of condition users (total {len(condition_users)}) in symptom (total {len(symptom_users)}): {np.sum(np.isin(symptom_users, condition_users))}")

# +
# CHANGED SINCE SUBMISSION!

C_als_ids = condition[condition.condition_name == "amyotrophic lateral sclerosis"].user_id.unique()

len(C_als_ids)
# -



# +
# CHANGED SINCE SUBMISSION!

# symptom[np.isin(symptom.user_id, C_als_ids)]
# -

# # What conditions do users in the alsfrs table have?

# +
condition_als = condition[np.isin(condition.user_id, alsfrs.user_id)]

condition_als
# -

condition_als.to_csv("./plmdata/raw/user_condition_alsfrs.csv", index=False)

conditions_all_alsfrs = condition_als.join(pd.get_dummies(condition_als["condition_name"])).drop("condition_name", axis=1).groupby("user_id").sum()

conditions_all_alsfrs.sum()

# +
# Making sure everything reconciles

conditions_all_alsfrs.sum().sum()
# -

# Of the 200 users with data in alsfrs, only 197 are present in the conditions table and 179 are present in the onset date table. 
#
# Those 197 users with entries in condition have 298 total conditions (rows in the condition table) among them. Oddly, only 2 of them are indicated as having ALS.

# # Plotting example score series

import seaborn as sns


# +
def get_user_score_series(user_id):
    return alsfrs[alsfrs.user_id == user_id].sort_values("report_date")
    

def get_user_onset_date(user_id):
    return onset[onset.user_id == user_id].first_definitive_diagnosis_date.item()

# CHANGED AFTER SUBMISSION!

def get_user_plot(user_id):
    user_series = get_user_score_series(user_id)

    try:
        onset_date = get_user_onset_date(user_id)
        render_onset_line = True
    except ValueError:
        render_onset_line = False

    plot = user_series.plot.scatter("report_date", "score")
    plot.tick_params(axis='x', labelrotation=45)
    plot.set_ylim((0,50))

    if render_onset_line:
        plot.axvline(onset_date,linewidth=2, color='r')

    return plot

def get_jointplot(df, plot_type="kde"):
    sns.jointplot(
        x="Days_since_first_report", y="score",
        data=df, kind=plot_type
    )

def get_jointplot_random_sample(n_users, alsfrs_df, plot_type="kde"):
    selected_users = np.random.permutation(
        alsfrs_df.user_id
    )[:n_users]

    get_jointplot(
        alsfrs_df[np.isin(alsfrs_df.user_id,selected_users)], plot_type
    )

def get_regplot_single_user(user, alsfrs_df, plot_type="kde"):
    sns.regplot(
        x="Days_since_first_report", y="score",
        data=alsfrs_df[alsfrs.user_id == user]
    )

def get_regplot_random_sample(n_users, alsfrs_df):
    selected_users = np.random.permutation(
        alsfrs_df.user_id
    )[:n_users]

    sns.regplot(
        x="Days_since_first_report", y="score",
        data=alsfrs_df[np.isin(alsfrs_df.user_id,selected_users)]
    )


# -

alsfrs_users

# +
# Linear 811 230 1757 2019 6276 *1206* 2909 2768 2514 1841 5589 350 509 4952
#     No ALS? 180 230
# Nonlinear 163 7326 1914 7150

get_user_plot(4952)

# +
# CHANGED AFTER SUBMISSION!

alsfrs.groupby("user_id").count()["score"].hist(bins=100)
# -

condition_als[condition_als.user_id == 1914]

symptom[symptom.user_id == 1914]

alsfrs_sorted = alsfrs.sort_values(["user_id", "report_date"])

alsfrs_sorted["report_date"] = alsfrs_sorted.report_date.apply(lambda x: x.toordinal())

min_by_user = alsfrs_sorted.groupby("user_id")["report_date"].min().rename("first_date")

alsfrs_first=alsfrs_sorted.merge(min_by_user, left_on="user_id", right_index=True)

alsfrs_first["Days_since_first_report"] = alsfrs_first.report_date - alsfrs_first.first_date

sns.jointplot(x="Days_since_first_report", y="score", data=alsfrs_first, kind="kde");

len(alsfrs_first), len(alsfrs_sorted)

# +
# linear 2768
# nonlinear

get_regplot_single_user), alsfrs_first, "reg")
# -

get_jointplot_random_sample(50, alsfrs_first)

get_regplot_random_sample(8, alsfrs_first)



StarcoderdataJupyterScriptsDedupFiltered