lama/stats/permutation_stats/run_permutation_stats.py
def prepare_data(wt_organ_vol: pd.DataFrame,
                 wt_staging: pd.DataFrame,
                 mut_organ_vol: pd.DataFrame,
                 mut_staging: pd.DataFrame,
                 label_meta: Path = None,
                 normalise_to_whole_embryo=False,
                 qc_file: Path = None) -> pd.DataFrame:
    """
    Merge the mutant and wildtype dtaframes
    Optionally normalise to staging metric (Usually whole embryo volume)
    Optionally remove any qc-flagged organs (These will be set to 'nan')

    Returns
    -------
    Dataframe with following columns:
        - a column for each label (prefixed with 'x' as statsmodels does not like integer ids)
        - line
        - genotype (baseline or mutant)
        - staging (whole embryo volume)
    """

    wt_staging.rename(columns={'value': 'staging'}, inplace=True)
    mut_staging.rename(columns={'value': 'staging'}, inplace=True)
    wt_staging.index = wt_staging.index.astype(str)

    # Ensure all indxes are same type
    for d in [wt_organ_vol, mut_organ_vol, wt_staging, mut_staging]:
        d.index = d.index.astype(str)

    if normalise_to_whole_embryo:
        logging.info('Normalising organ volume to whole embryo volume')
        wt_organ_vol = wt_organ_vol.divide(wt_staging['staging'], axis=0)
        mut_organ_vol = mut_organ_vol.divide(mut_staging['staging'], axis=0)



    # merge the organ vol
    organ_vols = pd.concat([wt_organ_vol, mut_organ_vol])

    # Drop any organ columns that has only zero values. These are the gaps in the label map caused by merging labels
    # in the atlas
    organ_vols = organ_vols.loc[:, (organ_vols != 0).any(axis=0)]

    # For the statsmodels linear mode to work, column names cannot start with a digit. Prefix with 'x'
    organ_vols.columns = [f'x{x}' if x.isdigit() else x for x in organ_vols.columns]

    staging = pd.concat([wt_staging, mut_staging])

    # Merge staging to the organvolume dataframe. First drop line so we don't get duplicate entries
    # staging.drop(columns=['line'], inplace=True)

    data = pd.concat([organ_vols, staging], axis=1)

    # Filter any labels that have been flagged at the label-level (for all specimens)
    if label_meta:

        label_meta = pd.read_csv(label_meta, index_col=0)

        if 'no_analysis' in label_meta:  # If we have a no_analysis column, drop labels that are flagged
            flagged_lables = label_meta[label_meta.no_analysis == True].index
            data.drop(columns=[f'x{x}' for x in flagged_lables if f'x{x}' in data] , inplace=True)

    # QC-flagged organs from specimens specified in QC file are set to None
    if qc_file:
        logging.info(f'Excluding organ volumes specified in: {qc_file}')
        qc = pd.read_csv(qc_file)

        for _, row in qc.iterrows():
            qc_id = str(row.id)

            if qc_id not in data.index:
                raise LamaDataException(f'QC flagged specimen {row.id} does not exist in dataset')

            if f'x{row.label}' not in data:
                raise LamaDataException(f'QC flagegd label, {row.label}, does not exist in dataset')

            data.loc[qc_id, f'x{row.label}'] = None

    return data

    Merge the mutant and wildtype dtaframes
    Optionally normalise to staging metric (Usually whole embryo volume)
    Optionally remove any qc-flagged organs (These will be set to 'nan')

    Returns
    -------
    Dataframe with following columns:
        - a column for each label (prefixed with 'x' as statsmodels does not like integer ids)
        - line
        - genotype (baseline or mutant)
        - staging (whole embryo volume)
    
Merge the mutant and wildtype dtaframes
Optionally normalise to staging metric (Usually whole embryo volume)
Optionally remove any qc-flagged organs (These will be set to 'nan')

Returns

Dataframe with following columns:
a column for each label (prefixed with 'x' as statsmodels does not like integer ids)
line
genotype (baseline or mutant)
staging (whole embryo volume)
Merge the mutant and wildtype dtaframes Optionally normalise to staging metric ( Usually whole embryo volume ) Optionally remove any qc - flagged organs ( These will be set to ' nan ' ) Returns Dataframe with following columns : a column for each label ( prefixed with ' x ' as statsmodels does not like integer ids ) line genotype ( baseline or mutant ) staging ( whole embryo volume )
def prepare_data(wt_organ_vol: pd.DataFrame,
                 wt_staging: pd.DataFrame,
                 mut_organ_vol: pd.DataFrame,
                 mut_staging: pd.DataFrame,
                 label_meta: Path = None,
                 normalise_to_whole_embryo=False,
                 qc_file: Path = None) -> pd.DataFrame:
    wt_staging.rename(columns={'value': 'staging'}, inplace=True)
    mut_staging.rename(columns={'value': 'staging'}, inplace=True)
    wt_staging.index = wt_staging.index.astype(str)
    for d in [wt_organ_vol, mut_organ_vol, wt_staging, mut_staging]:
        d.index = d.index.astype(str)
    if normalise_to_whole_embryo:
        logging.info('Normalising organ volume to whole embryo volume')
        wt_organ_vol = wt_organ_vol.divide(wt_staging['staging'], axis=0)
        mut_organ_vol = mut_organ_vol.divide(mut_staging['staging'], axis=0)
    organ_vols = pd.concat([wt_organ_vol, mut_organ_vol])
    organ_vols = organ_vols.loc[:, (organ_vols != 0).any(axis=0)]
    organ_vols.columns = [f'x{x}' if x.isdigit() else x for x in organ_vols.columns]
    staging = pd.concat([wt_staging, mut_staging])
    data = pd.concat([organ_vols, staging], axis=1)
    if label_meta:
        label_meta = pd.read_csv(label_meta, index_col=0)
        if 'no_analysis' in label_meta:  
            flagged_lables = label_meta[label_meta.no_analysis == True].index
            data.drop(columns=[f'x{x}' for x in flagged_lables if f'x{x}' in data] , inplace=True)
    if qc_file:
        logging.info(f'Excluding organ volumes specified in: {qc_file}')
        qc = pd.read_csv(qc_file)
        for _, row in qc.iterrows():
            qc_id = str(row.id)
            if qc_id not in data.index:
                raise LamaDataException(f'QC flagged specimen {row.id} does not exist in dataset')
            if f'x{row.label}' not in data:
                raise LamaDataException(f'QC flagegd label, {row.label}, does not exist in dataset')
            data.loc[qc_id, f'x{row.label}'] = None
    return data
def prepare_data ( wt_organ_vol : pd . DataFrame , wt_staging : pd . DataFrame , mut_organ_vol : pd . DataFrame , mut_staging : pd . DataFrame , label_meta : Path = None , normalise_to_whole_embryo = False , qc_file : Path = None ) -> pd . DataFrame : wt_staging . rename ( columns = { 'value' : 'staging' } , inplace = True ) mut_staging . rename ( columns = { 'value' : 'staging' } , inplace = True ) wt_staging . index = wt_staging . index . astype ( str ) for d in [ wt_organ_vol , mut_organ_vol , wt_staging , mut_staging ] : d . index = d . index . astype ( str ) if normalise_to_whole_embryo : logging . info ( 'Normalising organ volume to whole embryo volume' ) wt_organ_vol = wt_organ_vol . divide ( wt_staging [ 'staging' ] , axis = 0 ) mut_organ_vol = mut_organ_vol . divide ( mut_staging [ 'staging' ] , axis = 0 ) organ_vols = pd . concat ( [ wt_organ_vol , mut_organ_vol ] ) organ_vols = organ_vols . loc [ : , ( organ_vols != 0 ) . any ( axis = 0 ) ] organ_vols . columns = [ f'x{x}' if x . isdigit ( ) else x for x in organ_vols . columns ] staging = pd . concat ( [ wt_staging , mut_staging ] ) data = pd . concat ( [ organ_vols , staging ] , axis = 1 ) if label_meta : label_meta = pd . read_csv ( label_meta , index_col = 0 ) if 'no_analysis' in label_meta : flagged_lables = label_meta [ label_meta . no_analysis == True ] . index data . drop ( columns = [ f'x{x}' for x in flagged_lables if f'x{x}' in data ] , inplace = True ) if qc_file : logging . info ( f'Excluding organ volumes specified in: {qc_file}' ) qc = pd . read_csv ( qc_file ) for _ , row in qc . iterrows ( ) : qc_id = str ( row . id ) if qc_id not in data . index : raise LamaDataException ( f'QC flagged specimen {row.id} does not exist in dataset' ) if f'x{row.label}' not in data : raise LamaDataException ( f'QC flagegd label, {row.label}, does not exist in dataset' ) data . loc [ qc_id , f'x{row.label}' ] = None return data
Merge the mutant and wildtype dtaframes
Optionally normalise to staging metric (Usually whole embryo volume)
Optionally remove any qc-flagged organs (These will be set to 'nan')
Merge the mutant and wildtype dtaframes Optionally normalise to staging metric ( Usually whole embryo volume ) Optionally remove any qc - flagged organs ( These will be set to ' nan ' )
"""
    Merge the mutant and wildtype dtaframes
    Optionally normalise to staging metric (Usually whole embryo volume)
    Optionally remove any qc-flagged organs (These will be set to 'nan')

    Returns
    -------
    Dataframe with following columns:
        - a column for each label (prefixed with 'x' as statsmodels does not like integer ids)
        - line
        - genotype (baseline or mutant)
        - staging (whole embryo volume)
    """ # Ensure all indxes are same type # merge the organ vol # Drop any organ columns that has only zero values. These are the gaps in the label map caused by merging labels # in the atlas # For the statsmodels linear mode to work, column names cannot start with a digit. Prefix with 'x' # Merge staging to the organvolume dataframe. First drop line so we don't get duplicate entries # staging.drop(columns=['line'], inplace=True) # Filter any labels that have been flagged at the label-level (for all specimens) # If we have a no_analysis column, drop labels that are flagged # QC-flagged organs from specimens specified in QC file are set to None
wt_organ_vol
pd.DataFrame
wt_staging
pd.DataFrame
mut_organ_vol
pd.DataFrame
mut_staging
pd.DataFrame
label_meta
Path
normalise_to_whole_embryo
qc_file
Path