lama/stats/permutation_stats/run_permutation_stats.py
def annotate(thresholds: pd.DataFrame,
             lm_results: pd.DataFrame,
             lines_root_dir: Path,
             is_line_level: bool = True,
             label_info: Path = None,
             label_map: Path = None,
             write_thresholded_inv_labels=False,
             fdr_threshold: float=0.05,
             t_values: pd.DataFrame=None,
             organ_volumes: pd.DataFrame=None) -> pd.DataFrame:
    """
    Using the p_value thresholds and the linear model p-value results,
    create the following CSV files

        Line-level results
        specimen-level results

    Parameters
    ----------
    thresholds
        columns label(index), p_thresh, fdr, num_hits_across_all_lines/specimens
    lm_results
        The alternative distribution
        index: line/specimen id
        cols: labels (+ line_id for specimen_level)
    lines_root_dir
        The root directory to save the annotated CSV files. Each line to go in a subfolder
    is_line_level
        if not True, place results in specimen-level sub directory
    label_info
        CSV to map label number to name
    t_values
         same format as lm_results but with t-statistics
    organ_volumes
        All the organ volumes for baselines and mutants (as it was used in lm(), so probably normalised to whole embryo

    Returns
    -------
    Aggregated hit dataframe

    Notes
    -----
    TODO: Add file number prefixes so we don't overwrite mulyiple analyses done on the same day
    TODO: the organ_volumes folder name is hard-coded. What about if we add a new analysis type to the  permutation stats pipeline?
    """
    hit_dataframes = []

    if label_map:
        label_map = read_array(label_map)

    # Iterate over each line or specimen (for line or specimen-level analysis)
    for id_, row in lm_results.iterrows():

        # Create a dataframe containing a p-value column. each row an organ
        df = row.to_frame()

        if not is_line_level:
            # specimen-level has an extra line column we need to remove
            df = df.T.drop(columns=['line']).T

        # Rename the line_specimen column to be more informative
        df.rename(columns={id_: GENOTYPE_P_COL_NAME}, inplace=True)

        if is_line_level:
            line = id_
        else:
            line = row['line']
            spec_id = id_

        # Merge the permutation results (p-thresh, fdr, number of hit lines for this label) with the mutant results
        df.index = df.index.astype(np.int64)  # Index needs to be cast from object to enable merge
        df = df.merge(thresholds, left_index=True, right_index=True, validate='1:1')
        df.index.name = 'label'

        # Merge the t-statistics
        if t_values is not None:
            t_df = pd.DataFrame(t_values.loc[id_])

            if t_df.shape[1] != 1:  # We get multiple columns f there are duplicate specimen/line ids
                raise ValueError("Duplicate specimen names not allowed")

            t_df.columns = ['t']
            t_df.drop(columns=['line'], errors='ignore', inplace=True)  # this is for speciem-level results

            t_df.index = t_df.index.astype(np.int64)

            df = df.merge(t_df, left_index=True, right_index=True, validate='1:1')
            if len(df) < 1:
                logging.info(f'skipping {id_} no hits')  # Should we continue at this point?

        # Add mean organ vol difference and cohens d
        df['mean_vol_ratio'] = None
        if is_line_level:
            df['cohens_d'] = None

        for label, row in df.iterrows():
            # Organ vols are prefixed with x so it can work with statsmodels
            label_col = f'x{label}'
            label_organ_vol = organ_volumes[[label_col, 'line']]
            wt_ovs = label_organ_vol[label_organ_vol.line == 'baseline'][f'x{label}']
            mut_ovs = label_organ_vol[label_organ_vol.line == line][f'x{label}']

            df.loc[label, 'mean_vol_ratio'] =  mut_ovs.mean() / wt_ovs.mean()
            if is_line_level:
                cd = cohens_d(mut_ovs,wt_ovs)
                df.loc[label, 'cohens_d'] = cd

        output_name = f'{id_}_organ_volumes_{str(date.today())}.csv'

        line_output_dir = lines_root_dir / line
        line_output_dir.mkdir(exist_ok=True)

        if not is_line_level:
            # If dealing with specimen-level stats, make subfolder to put results in
            line_output_dir = line_output_dir / 'specimen_level' / id_
            line_output_dir.mkdir(parents=True, exist_ok=True)

        output_path = line_output_dir / output_name

        add_significance(df, fdr_threshold)

        if label_info:
            df = add_label_names(df, label_info)

        df.to_csv(output_path)

        hit_df = df[df['significant_cal_p'] == True]
        hit_df['line'] = line

        if not is_line_level:
            hit_df['specimen'] = spec_id

        hit_dataframes.append(hit_df)

        hit_labels_out = line_output_dir / f'{line}__hit_labels.nrrd'

        hits = hit_df.index

        if write_thresholded_inv_labels:
            _write_thresholded_label_map(label_map, hits, hit_labels_out)

    collated_df = pd.concat(hit_dataframes)
    return collated_df

    Using the p_value thresholds and the linear model p-value results,
    create the following CSV files

        Line-level results
        specimen-level results

    Parameters
    ----------
    thresholds
        columns label(index), p_thresh, fdr, num_hits_across_all_lines/specimens
    lm_results
        The alternative distribution
        index: line/specimen id
        cols: labels (+ line_id for specimen_level)
    lines_root_dir
        The root directory to save the annotated CSV files. Each line to go in a subfolder
    is_line_level
        if not True, place results in specimen-level sub directory
    label_info
        CSV to map label number to name
    t_values
         same format as lm_results but with t-statistics
    organ_volumes
        All the organ volumes for baselines and mutants (as it was used in lm(), so probably normalised to whole embryo

    Returns
    -------
    Aggregated hit dataframe

    Notes
    -----
    TODO: Add file number prefixes so we don't overwrite mulyiple analyses done on the same day
    TODO: the organ_volumes folder name is hard-coded. What about if we add a new analysis type to the  permutation stats pipeline?
    
Using the p_value thresholds and the linear model p-value results,
create the following CSV files

Line-level results
specimen-level results

Parameters

thresholds
columns label(index), p_thresh, fdr, num_hits_across_all_lines/specimens
lm_results
The alternative distribution
index: line/specimen id
cols: labels (+ line_id for specimen_level)
lines_root_dir
The root directory to save the annotated CSV files. Each line to go in a subfolder
is_line_level
if not True, place results in specimen-level sub directory
label_info
CSV to map label number to name
t_values
same format as lm_results but with t-statistics
organ_volumes
All the organ volumes for baselines and mutants (as it was used in lm(), so probably normalised to whole embryo

Returns

Aggregated hit dataframe

Notes

Add file number prefixes so we don't overwrite mulyiple analyses done on the same day
TODO: the organ_volumes folder name is hard-coded.
Using the p_value thresholds and the linear model p - value results create the following CSV files Line - level results specimen - level results Parameters thresholds columns label ( index ) p_thresh fdr num_hits_across_all_lines / specimens lm_results The alternative distribution index : line / specimen id cols : labels ( + line_id for specimen_level ) lines_root_dir The root directory to save the annotated CSV files . Each line to go in a subfolder is_line_level if not True place results in specimen - level sub directory label_info CSV to map label number to name t_values same format as lm_results but with t - statistics organ_volumes All the organ volumes for baselines and mutants ( as it was used in lm () so probably normalised to whole embryo Returns Aggregated hit dataframe Notes Add file number prefixes so we don ' t overwrite mulyiple analyses done on the same day TODO : the organ_volumes folder name is hard - coded .
def annotate(thresholds: pd.DataFrame,
             lm_results: pd.DataFrame,
             lines_root_dir: Path,
             is_line_level: bool = True,
             label_info: Path = None,
             label_map: Path = None,
             write_thresholded_inv_labels=False,
             fdr_threshold: float=0.05,
             t_values: pd.DataFrame=None,
             organ_volumes: pd.DataFrame=None) -> pd.DataFrame:
    hit_dataframes = []
    if label_map:
        label_map = read_array(label_map)
    for id_, row in lm_results.iterrows():
        df = row.to_frame()
        if not is_line_level:
            df = df.T.drop(columns=['line']).T
        df.rename(columns={id_: GENOTYPE_P_COL_NAME}, inplace=True)
        if is_line_level:
            line = id_
        else:
            line = row['line']
            spec_id = id_
        df.index = df.index.astype(np.int64)  
        df = df.merge(thresholds, left_index=True, right_index=True, validate='1:1')
        df.index.name = 'label'
        if t_values is not None:
            t_df = pd.DataFrame(t_values.loc[id_])
            if t_df.shape[1] != 1:  
                raise ValueError("Duplicate specimen names not allowed")
            t_df.columns = ['t']
            t_df.drop(columns=['line'], errors='ignore', inplace=True)  
            t_df.index = t_df.index.astype(np.int64)
            df = df.merge(t_df, left_index=True, right_index=True, validate='1:1')
            if len(df) < 1:
                logging.info(f'skipping {id_} no hits')  
        df['mean_vol_ratio'] = None
        if is_line_level:
            df['cohens_d'] = None
        for label, row in df.iterrows():
            label_col = f'x{label}'
            label_organ_vol = organ_volumes[[label_col, 'line']]
            wt_ovs = label_organ_vol[label_organ_vol.line == 'baseline'][f'x{label}']
            mut_ovs = label_organ_vol[label_organ_vol.line == line][f'x{label}']
            df.loc[label, 'mean_vol_ratio'] =  mut_ovs.mean() / wt_ovs.mean()
            if is_line_level:
                cd = cohens_d(mut_ovs,wt_ovs)
                df.loc[label, 'cohens_d'] = cd
        output_name = f'{id_}_organ_volumes_{str(date.today())}.csv'
        line_output_dir = lines_root_dir / line
        line_output_dir.mkdir(exist_ok=True)
        if not is_line_level:
            line_output_dir = line_output_dir / 'specimen_level' / id_
            line_output_dir.mkdir(parents=True, exist_ok=True)
        output_path = line_output_dir / output_name
        add_significance(df, fdr_threshold)
        if label_info:
            df = add_label_names(df, label_info)
        df.to_csv(output_path)
        hit_df = df[df['significant_cal_p'] == True]
        hit_df['line'] = line
        if not is_line_level:
            hit_df['specimen'] = spec_id
        hit_dataframes.append(hit_df)
        hit_labels_out = line_output_dir / f'{line}__hit_labels.nrrd'
        hits = hit_df.index
        if write_thresholded_inv_labels:
            _write_thresholded_label_map(label_map, hits, hit_labels_out)
    collated_df = pd.concat(hit_dataframes)
    return collated_df
def annotate ( thresholds : pd . DataFrame , lm_results : pd . DataFrame , lines_root_dir : Path , is_line_level : bool = True , label_info : Path = None , label_map : Path = None , write_thresholded_inv_labels = False , fdr_threshold : float = 0.05 , t_values : pd . DataFrame = None , organ_volumes : pd . DataFrame = None ) -> pd . DataFrame : hit_dataframes = [ ] if label_map : label_map = read_array ( label_map ) for id_ , row in lm_results . iterrows ( ) : df = row . to_frame ( ) if not is_line_level : df = df . T . drop ( columns = [ 'line' ] ) . T df . rename ( columns = { id_ : GENOTYPE_P_COL_NAME } , inplace = True ) if is_line_level : line = id_ else : line = row [ 'line' ] spec_id = id_ df . index = df . index . astype ( np . int64 ) df = df . merge ( thresholds , left_index = True , right_index = True , validate = '1:1' ) df . index . name = 'label' if t_values is not None : t_df = pd . DataFrame ( t_values . loc [ id_ ] ) if t_df . shape [ 1 ] != 1 : raise ValueError ( "Duplicate specimen names not allowed" ) t_df . columns = [ 't' ] t_df . drop ( columns = [ 'line' ] , errors = 'ignore' , inplace = True ) t_df . index = t_df . index . astype ( np . int64 ) df = df . merge ( t_df , left_index = True , right_index = True , validate = '1:1' ) if len ( df ) < 1 : logging . info ( f'skipping {id_} no hits' ) df [ 'mean_vol_ratio' ] = None if is_line_level : df [ 'cohens_d' ] = None for label , row in df . iterrows ( ) : label_col = f'x{label}' label_organ_vol = organ_volumes [ [ label_col , 'line' ] ] wt_ovs = label_organ_vol [ label_organ_vol . line == 'baseline' ] [ f'x{label}' ] mut_ovs = label_organ_vol [ label_organ_vol . line == line ] [ f'x{label}' ] df . loc [ label , 'mean_vol_ratio' ] = mut_ovs . mean ( ) / wt_ovs . mean ( ) if is_line_level : cd = cohens_d ( mut_ovs , wt_ovs ) df . loc [ label , 'cohens_d' ] = cd output_name = f'{id_}_organ_volumes_{str(date.today())}.csv' line_output_dir = lines_root_dir / line line_output_dir . mkdir ( exist_ok = True ) if not is_line_level : line_output_dir = line_output_dir / 'specimen_level' / id_ line_output_dir . mkdir ( parents = True , exist_ok = True ) output_path = line_output_dir / output_name add_significance ( df , fdr_threshold ) if label_info : df = add_label_names ( df , label_info ) df . to_csv ( output_path ) hit_df = df [ df [ 'significant_cal_p' ] == True ] hit_df [ 'line' ] = line if not is_line_level : hit_df [ 'specimen' ] = spec_id hit_dataframes . append ( hit_df ) hit_labels_out = line_output_dir / f'{line}__hit_labels.nrrd' hits = hit_df . index if write_thresholded_inv_labels : _write_thresholded_label_map ( label_map , hits , hit_labels_out ) collated_df = pd . concat ( hit_dataframes ) return collated_df
Using the p_value thresholds and the linear model p-value results,
create the following CSV files
Using the p_value thresholds and the linear model p - value results create the following CSV files
"""
    Using the p_value thresholds and the linear model p-value results,
    create the following CSV files

        Line-level results
        specimen-level results

    Parameters
    ----------
    thresholds
        columns label(index), p_thresh, fdr, num_hits_across_all_lines/specimens
    lm_results
        The alternative distribution
        index: line/specimen id
        cols: labels (+ line_id for specimen_level)
    lines_root_dir
        The root directory to save the annotated CSV files. Each line to go in a subfolder
    is_line_level
        if not True, place results in specimen-level sub directory
    label_info
        CSV to map label number to name
    t_values
         same format as lm_results but with t-statistics
    organ_volumes
        All the organ volumes for baselines and mutants (as it was used in lm(), so probably normalised to whole embryo

    Returns
    -------
    Aggregated hit dataframe

    Notes
    -----
    TODO: Add file number prefixes so we don't overwrite mulyiple analyses done on the same day
    TODO: the organ_volumes folder name is hard-coded. What about if we add a new analysis type to the  permutation stats pipeline?
    """ # Iterate over each line or specimen (for line or specimen-level analysis) # Create a dataframe containing a p-value column. each row an organ # specimen-level has an extra line column we need to remove # Rename the line_specimen column to be more informative # Merge the permutation results (p-thresh, fdr, number of hit lines for this label) with the mutant results # Index needs to be cast from object to enable merge # Merge the t-statistics # We get multiple columns f there are duplicate specimen/line ids # this is for speciem-level results # Should we continue at this point? # Add mean organ vol difference and cohens d # Organ vols are prefixed with x so it can work with statsmodels # If dealing with specimen-level stats, make subfolder to put results in
thresholds
pd.DataFrame
lm_results
pd.DataFrame
lines_root_dir
Path
is_line_level
bool
label_info
Path
label_map
Path
write_thresholded_inv_labels
fdr_threshold
float
t_values
pd.DataFrame
organ_volumes
pd.DataFrame