Delete Quality_Control.py

Quality_Control.py

@@ -1,1688 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-import warnings
-import os
-import plotly as plt
-import seaborn as sb
-import plotly.express as px
-import panel as pn
-import holoviews as hv
-import hvplot.pandas
-import pandas as pd
-import numpy as np
-import json
-import panel as pn
-import pandas as pd
-import random
-import asyncio
-import matplotlib.pyplot as plt
-from bokeh.plotting import figure
-from bokeh.io import push_notebook, show
-from bokeh.io.export import export_png
-from bokeh.resources import INLINE
-from bokeh.embed import file_html
-from bokeh.io import curdoc
-from bokeh.models import Span, Label
-from bokeh.models import ColumnDataSource, Button
-from my_modules import *
-from datasets import load_dataset
-
-#Silence FutureWarnings & UserWarnings
-warnings.filterwarnings('ignore', category= FutureWarning)
-warnings.filterwarnings('ignore', category= UserWarning)
-
-#input_path = os.path.join(present_dir, 'wetransfer_data-zip_2024-05-17_1431')
-present_dir = os.path.dirname(os.path.realpath(__file__))  
-# Construct the full path to the stored_variables.json file
-json_path = os.path.join(present_dir, 'stored_variables.json')  
-with open(json_path, 'r') as file:
-        stored_vars = json.load(file)
-        directory = stored_vars['base_dir']
-        input_path = os.path.join(present_dir,directory) 
-        set_path = stored_vars['set_path']
-        selected_metadata_files = stored_vars['selected_metadata_files']
-        ls_samples = stored_vars['ls_samples']
-base_dir = input_path
-
-#input_path = '/Users/harshithakolipaka/Desktop/CycIF/wetransfer_data-zip_2024-05-17_1431'
-#set_path = 'test'
-#selected_metadata_files = ['Slide_B_DD1s1.one_1.tif.csv', 'Slide_B_DD1s1.one_2.tif.csv']
-#ls_samples = ['Ashlar_Exposure_Time.csv', 'new_data.csv', 'DD3S1.csv', 'DD3S2.csv', 'DD3S3.csv', 'TMA.csv']
-pn.extension()
-
-update_button = pn.widgets.Button(name='CSV Files', button_type='primary')
-def update_samples(event):
-    with open(json_path, 'r') as file:
-        stored_vars = json.load(file)
-        print(stored_vars)
-        ls_samples = stored_vars['ls_samples']
-        return f'CSV Files Selected: {ls_samples}'
-update_button.on_click(update_samples)
-
-csv_files_button = pn.widgets.Button(icon="clipboard", button_type="primary")
-indicator = pn.indicators.LoadingSpinner(value=False, size=25)
-
-def handle_click(clicks):
-    with open(json_path, 'r') as file:
-        stored_vars = json.load(file)
-        print(stored_vars)
-        #ls_samples = stored_vars['ls_samples']
-    #return f'CSV Files Selected: {ls_samples}'
-
-# pn.Row(csv_files_button,pn.bind(
-# , csv_files_button.param.clicks),)
-
-
-# ## I.2. *DIRECTORIES
-
-#set_path = 'test'
-
-# Set base directory
-
-directorio_actual = os.getcwd()
-print(directorio_actual)
-
-##### MAC WORKSTATION #####
-#base_dir = r'/Volumes/LaboLabrie/Projets/OC_TMA_Pejovic/Temp/Zoe/CyCIF_pipeline/'
-###########################
-
-##### WINDOWS WORKSTATION #####
-#base_dir = r'C:\Users\LaboLabrie\gerz2701\cyCIF-pipeline\Set_B'
-###############################
-input_path = base_dir
-
-##### LOCAL WORKSTATION #####
-#base_dir = r'/Users/harshithakolipaka/Downloads/wetransfer_data-zip_2024-05-17_1431/'
-base_dir = input_path
-print(base_dir)
-#############################
-
-#set_name = 'Set_A'
-#set_name = 'test'
-set_name = set_path
-
-project_name = set_name              # Project name
-step_suffix = 'qc_eda'               # Curent part (here part I)
-previous_step_suffix_long = ""       # Previous part (here empty)
-
-# Initial input data directory
-input_data_dir = os.path.join(base_dir, project_name + "_data")
-
-# QC/EDA output directories
-# global output
-output_data_dir = os.path.join(base_dir, project_name + "_" + step_suffix)
-# images subdirectory
-output_images_dir = os.path.join(output_data_dir,"images")
-
-# Data and Metadata directories
-# global data
-metadata_dir = os.path.join(base_dir, project_name + "_metadata")
-# images subdirectory
-metadata_images_dir = os.path.join(metadata_dir,"images")
-
-# Create directories if they don't already exist
-for d in [base_dir, input_data_dir, output_data_dir, output_images_dir, metadata_dir, metadata_images_dir]:
-    if not os.path.exists(d):
-        print("Creation of the" , d, "directory...")
-        os.makedirs(d)
-    else :
-        print("The", d, "directory already exists !")
-
-os.chdir(input_data_dir)
-with open(json_path, 'r') as file:
-        stored_vars = json.load(file)
-#        ls_samples = stored_vars['ls_samples']
-        selected_metadata_files = stored_vars['selected_metadata_files']
-
-directories = []
-for i in [base_dir, input_data_dir, output_data_dir, output_images_dir, metadata_dir, metadata_images_dir]:
-    directories.append(i)
-
-directories
-
-def print_directories(directories):
-    
-    label_path = []
-    labels = [
-        "base_dir",
-        "input_data_dir",
-        "output_data_dir",
-        "output_images_dir",
-        "metadata_dir",
-        "metadata_images_dir"
-    ]
-
-    for label, path in zip(labels, directories):
-        label_path.append(f"{label} : {path}")
-        
-    return label_path
-
-print_directories
-
-
-# Verify paths
-print('base_dir :', base_dir)
-print('input_data_dir :', input_data_dir)
-print('output_data_dir :', output_data_dir)
-print('output_images_dir :', output_images_dir)
-print('metadata_dir :', metadata_dir)
-print('metadata_images_dir :', metadata_images_dir)
-
-
-# ## I.3. FILES
-
-# Listing all the .csv files in the metadata/data directory
-# Don't forget to move the csv files into the proj_data directory
-# if the data dir is empty it's not going to work 
-#ls_samples = [sample for sample in os.listdir(input_data_dir) if sample.endswith(".csv")]
-print("The following CSV files were detected:\n\n",[sample for sample in ls_samples], "\n\nin", input_data_dir, "directory.")
-
-
-# In[26]:
-
-
-import os
-import pandas as pd
-
-def combine_and_save_metadata_files(metadata_dir, selected_metadata_files):   
-    if len(selected_metadata_files) == []:
-        if not file:
-            warnings.warn("No Ashlar file uploaded. Please upload a valid file.", UserWarning)
-            return
-    
-    elif len(selected_metadata_files) > 1:
-        combined_metadata_df = pd.DataFrame()
-
-        for file in selected_metadata_files:
-            file_path = os.path.join(metadata_dir, file)
-            df = pd.read_csv(file_path)
-            combined_metadata_df = pd.concat([combined_metadata_df, df], ignore_index=True)
-
-        combined_metadata_df.to_csv(os.path.join(metadata_dir, "combined_metadata.csv"), index=False)
-        print(f"Combined metadata file saved as 'combined_metadata.csv' in {metadata_dir}")
-
-        return combined_metadata_df
-    
-    else:
-        '''if selected_metadata_files:
-            single_file_path = os.path.join(metadata_dir, selected_metadata_files[0])
-            single_file_df = pd.read_csv(single_file_path)
-            print(f"Only one file selected: {selected_metadata_files[0]}")
-            
-            return single_file_df'''
-        
-        if len(selected_metadata_files) == 1:
-            combined_metadata_path = os.path.join(metadata_dir, 'combined_metadata.csv')
-
-            if os.path.exists(combined_metadata_path):
-                print(f"Combined metadata file already exists: {combined_metadata_path}")
-                combined_metadata_df = pd.read_csv(combined_metadata_path)
-            else:
-                if selected_metadata_files:
-                    combined_metadata_df = pd.DataFrame()
-                    for file in selected_metadata_files:
-                        file_path = os.path.join(metadata_dir, file)
-                        metadata_df = pd.read_csv(file_path)
-                        combined_metadata_df = pd.concat([combined_metadata_df, metadata_df], ignore_index=True)
-
-                    combined_metadata_df.to_csv(combined_metadata_path, index=False)
-                    print(f"Combined metadata saved to: {combined_metadata_path}")
-                else:
-                    print("No metadata files selected.")
-                    combined_metadata_df = pd.DataFrame()
-
-            return combined_metadata_df
-
-print(combine_and_save_metadata_files(metadata_dir, selected_metadata_files))
-
-ls_samples
-
-path = os.path.join(input_data_dir, ls_samples[0])
-#df = load_dataset('csv', data_files = path )
-df = pd.read_csv(os.path.join(input_data_dir, ls_samples[0]),index_col = 0, nrows = 1)
-df.head(10)
-
-# First gather information on expected headers using first file in ls_samples
-# Read in the first row of the file corresponding to the first sample (index = 0) in ls_samples
-df = pd.read_csv(os.path.join(input_data_dir, ls_samples[0]) , index_col = 0, nrows = 1)
-
-
-# Make sure the file was imported correctly
-print("df :\n", df.head(), "\n")
-print("df's columns :\n", df.columns, "\n")
-print("df's index :\n", df.index, "\n")
-print("df's index name :\n", df.index.name)
-
-df.head()
-
-# Verify that the ID column in input file became the index
-# Verify that the index name column is "ID", if not, rename it
-if df.index.name != "ID":
-    print("Expected the first column in input file (index_col = 0) to be 'ID'. \n"
-          "This column will be used to set the index names (cell number for each sample). \n"
-          "It appears that the column '" + df.index.name + "' was actually the imported as the index column.")
-    #df.index.name = 'ID'
-    print("A new index name (first column) will be given ('ID') to replace the current one '" + df.index.name + "'\n")
-
-# Apply the changes to the headers as specified with apply_header_changes() function (in my_modules.py)
-# Apply the changes to the dataframe rows as specified with apply_df_changes() function (in my_modules.py)
-#df = apply_header_changes(df)
-print(df.index)
-df.index = df.index.str.replace(r'@1$', '')
-df = apply_df_changes(df)
-
-# Set variable to hold default header values
-expected_headers = df.columns.values
-expected_header = True 
-print(expected_header)
-
-intial_dataframe = df
-# Make sure the file is now formated correctly
-print("\ndf :\n", df.head(), "\n")
-print("df's columns :\n", df.columns, "\n")
-print("df's index :\n", df.index, "\n")
-print("df's index name :\n", df.index.name)
-
-df.head()
-
-
-df.head()
-
-print("Used " + ls_samples[0] + " to determine the expected and corrected headers for all files.\n")
-print("These headers are: \n" + ", ".join([h for h in expected_headers]))
-
-corrected_headers = True
-
-for sample in ls_samples:
-    file_path = os.path.join(input_data_dir,sample)
-    print(file_path)
-
-# Import all the others files
-dfs = {}
-###############################
-# !! This may take a while !! #
-###############################
-errors = []
-
-for sample in ls_samples:
-    file_path = os.path.join(input_data_dir,sample)
-    
-    try:
-        # Read the CSV file
-        df = load_dataset("csv", data_files = file_path)
-        df = pd.read_csv(file_path, index_col=0)
-        # Check if the DataFrame is empty, if so, don't continue trying to process df and remove it
-        
-        if not df.empty:
-            # Manipulations necessary for concatenation
-            df = apply_header_changes(df)
-            df = apply_df_changes(df)
-            # Reorder the columns to match the expected headers list
-            #df = df.reindex(columns=expected_headers)
-            print(df.head(1))
-            print(sample, "file is processed !\n")
-            #print(df) 
-    
-            # Compare df's header df against what is expected
-            compare_headers(expected_headers, df.columns.values, sample)
-            #print(df.columns.values)
-            # Add a new colunm to identify the csv file (sample) where the df comes from
-            df['Sample_ID'] = sample 
-   
-    except pd.errors.EmptyDataError:
-        errors.append(f'\nEmpty data error in {sample} file. Removing from analysis...')
-        print(f'\nEmpty data error in {sample} file. Removing from analysis...')
-        ls_samples.remove(sample)      
-    
-    # Add df to dfs 
-    dfs[sample] = df
-    
-print(dfs)
-
-
-dfs.values()
-
-# Merge dfs into one df
-df = pd.concat(dfs.values(), ignore_index=False , sort = False)
-del dfs
-merge = True 
-merged_dataframe = df
-df.head()
-
-# Set index to Sample_ID + cell number : 
-# create a new custom index for df based on the sample names and integer cell numbers, and then remove the temporary columns 'level_0' and 'index' that were introduced during the operations
-
-# Creates a copy of the DataFrame df and resets its index without creating a new column for the old index
-# This essentially removes the old index column and replaces it with a default integer index
-df = df.copy().reset_index(drop=True)
-
-#print(df)
-
-# Initializing an empty list index to store the new index labels for the DataFrame
-index = []
-
-for sample in ls_samples:
-    # Extract a chunk of data from the original df where the 'Sample_ID' column matches the current sample name
-    # This chunk is stored in the df_chunk df, which is a subset of the original data for that specific sample
-    df_chunk = df.loc[df['Sample_ID'] == sample,:].copy()
-    old_index = df_chunk.index
-    # Reset the index of the df_chunk df, removing the old index and replacing it with a default integer index
-    df_chunk = df_chunk.reset_index(drop=True)
-    # A new index is created for the df_chunk df. It combines the sample name with 'Cell_' and the integer index values, converting them to strings
-    # This new index will have labels like 'SampleName_Cell_0', 'SampleName_Cell_1', and so on.
-    sample = sample.split('.')[0]
-    df_chunk = df_chunk.set_index(f'{sample}_Cell_' + df_chunk.index.astype(str))
-    # The index values of df_chunk are then added to the index list
-    index = index + df_chunk.index.values.tolist()
-
-# After processing all the samples in the loop, assign the index list as the new index of the original df.
-df.index =  index
-# Remove the 'level_0' and 'index' columns from df
-df = df.loc[:,~df.columns.isin(['level_0','index'])]
-assigned_new_index = True 
-df.head()
-
-
-# ### I.3.2. NOT_INTENSITIES
-
-# not_intensities is the list of the columns unrelated to the markers fluorescence intensities
-# Can include items that aren't in a given header.
-#not_intensitiehttp://localhost:8888/lab/tree/Downloads/wetransfer_data-zip_2024-05-17_1431/1_qc_eda.ipynb
-#I.3.2.-NOT_INTENSITIESs = ['Nuc_X', 'Nuc_X_Inv', 'Nuc_Y', 'Nuc_Y_Inv', 'Nucleus_Roundness', 'Nucleus_Size', 'Cell_Size', 
-#                   'ROI_index', 'Sample_ID', 'replicate_ID', 'Cell_ID','cell_type', 'cell_subtype', 'cluster','ID', 
-#                  'Cytoplasm_Size', 'immune_checkpoint', 'Unique_ROI_index', 'Patient', 'Primary_chem(1)_vs_surg(0)']
-# not_intensities is the list of the columns unrelated to the markers fluorescence intensities
-# Can include items that aren't in a given header.
-#not_intensities = ['Nuc_X', 'Nuc_X_Inv', 'Nuc_Y', 'Nuc_Y_Inv', 'Nucleus_Roundness', 'Nucleus_Size', 'Cell_Size', 
-#                   'ROI_index', 'Sample_ID', 'replicate_ID', 'Cell_ID','cell_type', 'cell_subtype', 'cluster','ID', 
-#                   'Cytoplasm_Size', 'immune_checkpoint', 'Unique_ROI_index', 'Patient', 'Primary_chem(1)_vs_surg(0)']
-
-# Get all column names
-all_columns = df.columns.tolist()
-
-# Create a list to store non-intensity column names
-not_intensities = []
-intensity_columns = []
-# Iterate over each column name
-for column in all_columns:
-    # Check if the column name contains 'Intensity_Average'
-    if 'Intensity_Average' not in column:
-        print(not_intensities)
-        not_intensities.append(column)
-    else:
-        intensity_columns.append(column)
-    
-    
-# Create a new DataFrame with non-intensity columns
-not_intensities_df = pd.DataFrame(not_intensities)
-print("Non-intensity columns:")
-print(not_intensities)
-
-print("non-intensity DataFrame:")
-not_intensities
-#print(len(intensity_columns))
-
-
-pd.DataFrame(not_intensities)
-
-path_not_intensities = os.path.join(metadata_dir,"not_intensities.csv")
-
-# If this file already exists, add only not_intensities items of the list not already present in file
-if os.path.exists(path_not_intensities):
-    print("'not_intensities.csv' already exists.")
-    print("Reconciling file and Jupyter notebook lists.")
-    file_not_intensities = open(path_not_intensities, "r")
-    file_ni = file_not_intensities.read().splitlines()
-    # Set difference to identify items not already in file
-    to_add = set(not_intensities) - set(file_ni)
-    # We want not_intensities to the a complete list
-    not_intensities = list(set(file_ni) | set(not_intensities))
-    file_not_intensities.close()
-    file_not_intensities = open(path_not_intensities, "a")
-    for item in to_add:
-        file_not_intensities.write(item +"\n")
-    file_not_intensities.close()
-
-else:
-    # The file does not yet exist
-    print("Could not find " + path_not_intensities + ". Creating now.")
-    file_not_intensities = open(path_not_intensities, "w")
-    for item in not_intensities:
-        file_not_intensities.write(item + "\n")
-    file_not_intensities.close()
-
-not_intensities_df = pd.read_csv(path_not_intensities)
-not_intensities_df
-
-# Columns we want to keep: not_intensities, and any intensity column that contains 'Intensity_Average' (drop any intensity marker column that is not a mean intensity)
-to_keep = not_intensities + [x for x in df.columns.values[~df.columns.isin(not_intensities)] if 'Intensity_Average' in x]
-
-to_keep
-
-print(len(to_keep) - 1)
-
-# However, our to_keep list contains items that might not be in our df headers!
-# These items are from our not_intensities list. So let's ask for only those items from to_keep that are actually found in our df
-# Retains only the columns from the to_keep list that are found in the df's headers (columns). 
-# This ensures that we are only keeping the columns that exist in your df, avoiding any potential issues with non-existent column names. 
-# The result is a df containing only the specified columns.
-df = df[[x for x in to_keep if x in df.columns.values]]
-
-df.head()
-
-# Assuming you have a DataFrame named 'df'
-# df = pd.read_csv('your_file.csv')
-
-# Load or create the stored_variables.json file
-json_file_path = os.path.join(present_dir,"stored_variables.json")
-
-if os.path.exists(json_file_path):
-    with open(json_file_path, "r") as file:
-        stored_variables = json.load(file)
-else:
-    stored_variables = {}
-
-# Get all column names
-all_columns = df.columns.tolist()
-
-# Create an empty list to store intensity markers
-intensity_marker = []
-
-# Iterate over each column name
-for column in all_columns:
-    # Check if the column name contains 'Intensity_Average'
-    if 'Intensity_Average' in column:
-        # Split the column name by underscore
-        parts = column.split('_')
-        
-        # Extract the word before the first underscore
-        marker = parts[0]
-        
-        # Add the marker to the intensity_marker list
-        intensity_marker.append(marker)
-
-# Remove duplicates from the intensity_marker list
-intensity_marker = list(set(intensity_marker))
-
-print("Intensity Markers:")
-print(intensity_marker)
-
-# Create a DataFrame with the intensity markers and default values
-marker_options_df = pd.DataFrame({
-    'Marker': intensity_marker,
-    'Cell': [True] * len(intensity_marker),
-    'Cytoplasm': [False] * len(intensity_marker),
-    'Nucleus': [False] * len(intensity_marker)
-})
-
-# Define formatters for the Tabulator widget
-tabulator_formatters = {
-    'Cell': {'type': 'tickCross'},
-    'Cytoplasm': {'type': 'tickCross'},
-    'Nucleus': {'type': 'tickCross'}
-}
-
-# Create the Tabulator widget
-tabulator = pn.widgets.Tabulator(marker_options_df, formatters=tabulator_formatters, sizing_mode='stretch_width')
-
-# Create a DataFrame to store the initial intensities
-new_data = [{'Description': f"{marker}_Cell_Intensity_Average"} for marker in intensity_marker if True]
-new_data_df = pd.DataFrame(new_data)
-
-# Create a widget to display the new data as a DataFrame
-new_data_table = pn.widgets.Tabulator(new_data_df, name='New Data Table', sizing_mode='stretch_width')
-
-# Create a button to start the update process
-run_button = pn.widgets.Button(name="Save Selection", button_type='primary')
-
-# Function to update stored_variables.json
-def update_stored_variables(selected_columns):
-    stored_variables["selected_intensities"] = selected_columns
-    with open(json_file_path, "w") as file:
-        json.dump(stored_variables, file, indent=4)
-
-# Define the update_intensities function
-def update_intensities(event=None):
-    global new_data, new_data_df
-    new_data = []
-    selected_columns = []
-    for _, row in tabulator.value.iterrows():
-        marker = row['Marker']
-        if row['Cell']:
-            new_data.append({'Description': f"{marker}_Cell_Intensity_Average"})
-            selected_columns.append(f"{marker}_Cell_Intensity_Average")
-        if row['Cytoplasm']:
-            new_data.append({'Description': f"{marker}_Cytoplasm_Intensity_Average"})
-            selected_columns.append(f"{marker}_Cytoplasm_Intensity_Average")
-        if row['Nucleus']:
-            new_data.append({'Description': f"{marker}_Nucleus_Intensity_Average"})
-            selected_columns.append(f"{marker}_Nucleus_Intensity_Average")
-    new_data_df = pd.DataFrame(new_data)
-    new_data_table.value = new_data_df
-    update_stored_variables(selected_columns)
-    print("Updated intensities DataFrame:")
-    print(new_data_df)
-
-# Define the runner function
-async def runner(event):
-    update_intensities()
-
-# Bind the runner function to the button
-run_button.on_click(runner)
-
-# Attach the update_intensities function to changes in the Tabulator widget
-tabulator.param.watch(update_intensities, 'value')
-
-# Layout
-updated_intensities = pn.Column(tabulator, run_button, new_data_table, sizing_mode="stretch_width")
-
-'''
-# Iterate over each column name
-for column in all_columns:
-    # Check if the column name contains 'Intensity_Average'
-    if 'Intensity_Average' in column:
-        # Split the column name by underscore
-        parts = column.split('_')
-        
-        # Extract the word before the first underscore
-        marker = parts[0]
-        
-        # Add the marker to the intensity_marker list
-        intensity_marker.append(marker)
-
-# Remove duplicates from the intensity_marker list
-intensity_marker = list(set(intensity_marker))
-
-print("Intensity Markers:")
-print(intensity_marker)
-
-# Create a callback function to update the intensities array
-def update_intensities(event):
-    global intensities
-    global intensities_df
-    new_intensities = []
-    selected_columns = []
-    for marker, cell, cytoplasm, nucleus in zip(marker_options_df['Marker'], marker_options_df['Cell'], marker_options_df['Cytoplasm'], marker_options_df['Nucleus']):
-        if cell:
-            new_intensities.append(f"{marker}_Cell_Intensity_Average")
-            selected_columns.append(f"{marker}_Cell_Intensity_Average")
-        if cytoplasm:
-            new_intensities.append(f"{marker}_Cytoplasm_Intensity_Average")
-            selected_columns.append(f"{marker}_Cytoplasm_Intensity_Average")
-        if nucleus:
-            new_intensities.append(f"{marker}_Nucleus_Intensity_Average")
-            selected_columns.append(f"{marker}_Nucleus_Intensity_Average")
-    intensities = new_intensities
-    if selected_columns:
-        intensities_df = merged_dataframe[selected_columns]
-    else:
-        intensities_df = pd.DataFrame()
-    print("Updated intensities DataFrame:")
-    print(intensities_df)
-
-tabulator_formatters = {
-    'bool': {'type': 'tickCross'}
-}
-
-# Create a DataFrame with the intensity markers and default values
-marker_options_df = pd.DataFrame({
-    'Marker': intensity_marker,
-    'Cell': [False] * len(intensity_marker),
-    'Cytoplasm': [False] * len(intensity_marker),
-    'Nucleus': [False] * len(intensity_marker)
-})
-
-# Create the Tabulator widget and link the callback function
-tabulator = pn.widgets.Tabulator(marker_options_df, formatters=tabulator_formatters, sizing_mode='stretch_width')
-tabulator.param.watch(update_intensities,'value')
-
-# Create a Panel layout with the Tabulator widget
-marker_options_layout = pn.Column(tabulator, sizing_mode="stretch_width")
-# Initialize the Panel extension with Tabulator
-pn.extension('tabulator')
-
-# Create a DataFrame with the intensity markers and default values
-marker_options_df = pd.DataFrame({
-    'Marker': intensity_marker,
-    'Cell': [True] * len(intensity_marker),
-    'Cytoplasm': [False] * len(intensity_marker),
-    'Nucleus': [False] * len(intensity_marker)
-})
-
-# Define formatters for the Tabulator widget
-tabulator_formatters = {
-    'Cell': {'type': 'tickCross'},
-    'Cytoplasm': {'type': 'tickCross'},
-    'Nucleus': {'type': 'tickCross'}
-}
-
-# Create the Tabulator widget
-tabulator = pn.widgets.Tabulator(marker_options_df, formatters=tabulator_formatters, sizing_mode='stretch_width')
-
-# Create a DataFrame to store the initial intensities
-new_data = [{'Description': f"{marker}_Cell_Intensity_Average"} for marker in intensity_marker if True]
-new_data_df = pd.DataFrame(new_data)
-
-# Create a widget to display the new data as a DataFrame
-new_data_table = pn.widgets.Tabulator(new_data_df, name='New Data Table', sizing_mode='stretch_width')
-
-# Create a button to start the update process
-run_button = pn.widgets.Button(name="Save Selection", button_type='primary')
-
-# Define the update_intensities function
-def update_intensities():
-    global new_data, new_data_df
-    new_data = []
-    for _, row in tabulator.value.iterrows():
-        marker = row['Marker']
-        if row['Cell']:
-            new_data.append({'Description': f"{marker}_Cell_Intensity_Average"})
-        if row['Cytoplasm']:
-            new_data.append({'Description': f"{marker}_Cytoplasm_Intensity_Average"})
-        if row['Nucleus']:
-            new_data.append({'Description': f"{marker}_Nucleus_Intensity_Average"})
-    new_data_df = pd.DataFrame(new_data)
-    new_data_table.value = new_data_df
-
-# Define the runner function
-async def runner(event):
-    update_intensities()
-
-# Bind the runner function to the button
-run_button.on_click(runner)
-
-# Layout
-updated_intensities = pn.Column(tabulator, run_button, new_data_table, sizing_mode="stretch_width")
-
-pn.extension()'''
-# Serve the layout
-#updated_intensities.servable()
-
-
-intensities_df = new_data_table
-intensities_df = pn.pane.DataFrame(intensities_df)
-print(intensities_df)
-# ## I.4. QC CHECKS
-
-def quality_check_results(check_shape, check_no_null,check_zero_intensities):
-    results = [
-        f"Check Index: {check_index}",
-        f"Check Shape: {check_shape}",
-        f"Check No Null: {check_no_null}",
-        f"Check Zero Intensities: {check_zero_intensities}"
-    ]
-    return pn.Column(*[pn.Row(result) for result in results], sizing_mode="stretch_width")
-
-print(ls_samples)
-
-def check_index_format(index_str, ls_samples):
-    """
-    Checks if the given index string follows the specified format.
-
-    Args:
-        index_str (str): The index string to be checked.
-        ls_samples (list): A list of valid sample names.
-
-    Returns:
-        bool: True if the index string follows the format, False otherwise.
-    """
-    # Split the index string into parts
-    parts = index_str.split('_')
-
-    # Check if there are exactly 3 parts
-    if len(parts) != 3:
-        print(len(parts))
-        return False
-
-    # Check if the first part is in ls_samples
-    sample_name = parts[0]
-    if f'{sample_name}.csv' not in ls_samples:
-        print(sample_name)
-        return False
-
-    # Check if the second part is in ['cell', 'cytoplasm', 'nucleus']
-    location = parts[1]
-    valid_locations = ['Cell', 'Cytoplasm', 'Nucleus']
-    if location not in valid_locations:
-        print(location)
-        return False
-
-    # Check if the third part is a number
-    try:
-        index = int(parts[2])
-    except ValueError:
-        print(index)
-        return False
-
-    # If all checks pass, return True
-    return True
-
-# Let's take a look at a few features to make sure our dataframe is as expected
-df.index
-def check_format_ofindex(index):
-    for index in df.index:
-        check_index = check_index_format(index, ls_samples) 
-        if check_index is False:
-            index_format = "Bad"
-            return index_format
-        
-    index_format = "Good"   
-    return index_format
-print(check_format_ofindex(df.index))
-
-df.shape
-check_index = df.index
-check_shape = df.shape
-print(check_shape)
-
-# Check for NaN entries (should not be any unless columns do not align)
-# False means no NaN entries 
-# True means NaN entries 
-df.isnull().any().any()
-
-check_no_null = df.isnull().any().any()
-
-# Check that all expected files were imported into final dataframe
-if sorted(df.Sample_ID.unique()) == sorted(ls_samples):
-    print("All expected filenames are present in big df Sample_ID column.")
-    check_all_expected_files_present = "All expected filenames are present in big df Sample_ID column."
-else:
-    compare_headers(['no samples'], df.Sample_ID.unique(), "big df Sample_ID column")
-    check_all_expected_files_present = compare_headers(['no samples'], df.Sample_ID.unique(), "big df Sample_ID column")
-
-print(df.Sample_ID)
-
-# Delete rows that have 0 value mean intensities for intensity columns
-print("df.shape before removing 0 mean values: ", df.shape)
-
-# We use the apply method on df to calculate the mean intensity for each row. It's done this by applying a lambda function to each row. 
-# The lambda function excludes the columns listed in the not_intensities list (which are not to be considered for mean intensity calculations) 
-# and calculates the mean of the remaining values in each row.
-###############################
-# !! This may take a while !! #
-###############################
-# Calculate mean intensity excluding 'not_intensities' columns
-mean_intensity = df.loc[:, ~df.columns.isin(not_intensities)].mean(axis=1)
-
-# Check if there are any 0 mean intensity values
-if (mean_intensity == 0).any():
-    df = df.loc[mean_intensity > 0, :]
-    print("Shape after removing 0 mean values: ", df.shape) 
-    check_zero_intensities = f'df.shape after removing 0 mean values: {df.shape}'
-else:
-    print("No zero intensity values.")
-    check_zero_intensities = " No zero intensity values found in the DataFrame."
-
-
-
-# Get quantiles (5th, 50th, 95th)
-# List of nucleus size percentiles to extract 
-#qs = [0.05,0.50,0.95] 
-
-
-
-#df["Nucleus_Size"].quantile(q=qs)
-
-
-quality_control_df = df
-quality_control_df.head()
-
-# Function to perform quality checks
-def perform_quality_checks(df, ls_samples, not_intensities):
-    results = {}
-    errors = []
-    # Check index
-    results['index'] = df.index
-    
-    # Check shape
-    results['shape'] = df.shape
-    
-    # Check for NaN entries
-    results['nan_entries'] = df.isnull().any().any()
-
-    # Remove rows with 0 mean intensity values
-    mean_intensity = df.loc[:, ~df.columns.isin(not_intensities)].mean(axis=1)
-    if (mean_intensity == 0).any():
-        df = df.loc[mean_intensity > 0, :]
-        results['zero_intensity_removal'] = f"Zero intensity entires are found and removed. Shape after removing: {df.shape}"
-    else:
-        results['zero_intensity_removal'] = "No zero intensity values found in the DataFrame."
-    
-    return results
-
-# Example usage of the function
-quality_check_results = perform_quality_checks(df, ls_samples, not_intensities)
-
-# Print results
-for key, value in quality_check_results.items():
-    print(f"{key}: {value}")
-
-
-import panel as pn
-import pandas as pd
-
-def quality_check(file, not_intensities):
-    # Load the output file
-    df = file
-
-    # Check Index
-    check_index = check_format_ofindex(df.index)
-
-    # Check Shape
-    check_shape = df.shape
-
-    # Check for NaN entries
-    check_no_null = df.isnull().any().any()
-
-    mean_intensity = df.loc[:, ~df.columns.isin(not_intensities)].mean(axis=1)
-    if (mean_intensity == 0).any():
-        df = df.loc[mean_intensity > 0, :]
-        print("df.shape after removing 0 mean values: ", df.shape)
-        check_zero_intensities = f'df.shape after removing 0 mean values: {df.shape}'
-    else:
-        print("No zero intensity values found in the DataFrame.")
-        check_zero_intensities = "No zero intensities."
-
-    # Create a quality check results table
-    quality_check_results_table = pd.DataFrame({
-        'Check': ['Index', 'Shape', 'Check for NaN Entries', 'Check for Zero Intensities'],
-        'Result': [str(check_index), str(check_shape), str(check_no_null), check_zero_intensities]
-    })
-
-    # Create a quality check results component
-    quality_check_results_component = pn.Card(
-        pn.pane.DataFrame(quality_check_results_table),
-        title="Quality Control Results",
-        header_background="#2196f3",
-        header_color="white",
-    )
-
-    return quality_check_results_component
-
-quantile_slider = pn.widgets.FloatSlider(name='Quantile', start=0.01, end=0.99, step=0.01, value=0.05)
-
-
-# Function to calculate quantile values
-def calculate_quantiles(quantile):
-    quantile_value_intensity = df["AF555_Cell_Intensity_Average"].quantile(q=[quantile, 0.50, 1 - quantile])    
-    return quantile_value_intensity
-
-# Function to create the Panel app
-def create_app(quantile = quantile_slider.param.value):
-    quantiles = calculate_quantiles(quantile)
-    output = pd.DataFrame(quantiles)
-    
-    # Create a Markdown widget to display the output
-    output_widget = pn.pane.DataFrame(output)
-
-    return output_widget
-
-
-# Bind the create_app function to the quantile slider
-quantile_output_app = pn.bind(create_app, quantile_slider.param.value)
-#pn.Column(quantile_slider,quantile_output_app).servable()
-
-# Function to create the line graph plot using Bokeh
-def create_line_graph2(quantile):
-    # Calculate histogram
-    hist, edges = np.histogram(df['Nucleus_Size'], bins=30)
-    
-    # Calculate the midpoints of bins for plotting
-    midpoints = (edges[:-1] + edges[1:]) / 2
-
-    # Calculate quantiles
-    qs = [quantile, 0.50, 1.00 - quantile]
-    quantiles = df['Nucleus_Size'].quantile(q=qs).values
-
-    # Create Bokeh line graph plot
-    p = figure(title='Frequency vs. Nucleus_Size', 
-               x_axis_label='Nucleus_Size', 
-               y_axis_label='Frequency',
-               width=800, height=400)
-    
-    # Plotting histogram
-    p.quad(top=hist, bottom=0, left=edges[:-1], right=edges[1:],
-           fill_color='skyblue', line_color='black', alpha=0.6)
-    
-    # Plotting line graph
-    p.line(midpoints, hist, line_width=2, color='blue', alpha=0.7)
-    
-    # Add quantile lines
-    for q in quantiles:
-        span = Span(location=q, dimension='height', line_color='red', line_dash='dashed', line_width=2)
-        p.add_layout(span)
-        p.add_layout(Label(x=q, y=max(hist), text=f'{q:.1f}', text_color='red'))
-    
-    return p
-
-# Bind the create_line_graph function to the quantile slider
-nucleus_size_line_graph_with_histogram = pn.bind(create_line_graph2, quantile=quantile_slider.param.value)
-
-# Clean the 'Nucleus_Size' column by removing NaN and infinite values
-df = df[np.isfinite(df['Nucleus_Size'])]  # This will keep only finite values
-
-# Check if the DataFrame is not empty after cleaning
-if df.empty:
-    raise ValueError("No valid data available after cleaning.")
-else:
-    # Calculate the histogram
-    hist, edges = np.histogram(df['Nucleus_Size'], bins=30)
-    print("Histogram calculated successfully.")
-    print("Histogram:", hist)
-    print("Edges:", edges)
-    plot1 = pn.Column(quantile_slider, pn.pane.Bokeh(nucleus_size_line_graph_with_histogram))
-    
-#Removing cells based on nucleus size
-
-quantile = quantile_slider.value
-qs = [quantile, 0.50, 1.00 - quantile]
-quantiles = df['Nucleus_Size'].quantile(q=qs).values
-threshold = quantiles[2]
-
-print(threshold)
-
-import panel as pn
-import pandas as pd
-import numpy as np
-from bokeh.plotting import figure
-from bokeh.models import Span, Label
-# Define the quantile slider
-#quantile_slider = pn.widgets.FloatSlider(name='Quantile', start=0.01, end=0.99, step=0.01, value=0.05)
-
-# Function to update the threshold and display number of cells removed
-def update_threshold_and_display(quantile):
-    qs = [quantile, 0.50, 1.00 - quantile]
-    quantiles = df['Nucleus_Size'].quantile(q=qs).values
-    threshold = quantiles[2]
-    
-    # Filter the DataFrame based on the new threshold
-    df_filtered = df.loc[(df['Nucleus_Size'] > 42) & (df['Nucleus_Size'] < threshold)]
-    
-    # Calculate the number of cells removed
-    cells_before_filter = df.shape[0]
-    cells_after_filter = df_filtered.shape[0]
-    cells_removed = cells_before_filter - cells_after_filter
-    
-    # Display the results
-    results = pn.Column(
-        f"Number of cells before filtering: {cells_before_filter}",
-        f"Number of cells after filtering on nucleus size: {cells_after_filter}",
-        f"Number of cells removed: {cells_removed}"
-    )
-    
-    return results
-
-# Bind the update function to the quantile slider
-results_display = pn.bind(update_threshold_and_display, quantile_slider)
-
-# Layout the components in a Panel app
-layout2 = results_display
-
-print("Number of cells before filtering :", df.shape[0])
-cells_before_filter = f"Number of cells before filtering :{df.shape[0]}"
-# Delete small cells and objects w/high AF555 Signal (RBCs) 
-# We usually use the 95th percentile calculated during QC_EDA
-df = df.loc[(df['Nucleus_Size'] > 42 )]
-df = df.loc[(df['Nucleus_Size'] < threshold)]
-cells_after_filter_nucleus_shape = df.shape[0]
-print("Number of cells after filtering on nucleus size:", df.shape[0])
-
-df = df.loc[(df['AF555_Cell_Intensity_Average'] < 2000)]
-print("Number of cells after filtering on AF555A ___ intensity:", df.shape[0])
-cells_after_filter_intensity_shape = df.shape[0]
-cells_after_filter_nucleus = f"Number of cells after filtering on nucleus size: {cells_after_filter_nucleus_shape}"
-cells_after_filter_intensity = f"Number of cells after filtering on AF555A ___ intensity: {cells_after_filter_intensity_shape}"
-
-num_of_cell_removal_intensity = cells_after_filter_intensity
-
-print(num_of_cell_removal_intensity )
-
-num_of_cell_removal = pn.Column(cells_before_filter, cells_after_filter_nucleus)
-
-
-# Assuming you have a DataFrame 'df' with the intensity columns
-intensities = df.filter(like='Intensity').columns.tolist()
-
-# Create a ColumnDataSource from the DataFrame
-source = ColumnDataSource(df)
-
-# Function to calculate quantile values
-def calculate_quantiles(column, quantile):
-    quantiles = df[column].quantile(q=[quantile, 0.50, 1 - quantile]).values
-    return quantiles
-
-# Create the dropdown menu
-column_dropdown = pn.widgets.Select(name='Select Column', options=intensities)
-
-quantile_slider = pn.widgets.FloatSlider(name='Quantile', start=0.01, end=0.99, step=0.01, value=0.05)
-
-
-# Function to create the Bokeh plot
-def create_intensity_plot(column, quantile):
-    quantiles = calculate_quantiles(column, quantile)
-    hist, edges = np.histogram(df[column], bins = 30)
-    # Calculate the midpoints of bins for plotting
-    midpoints = (edges[:-1] + edges[1:]) / 2
-    
-    # Create Bokeh plot
-    p = figure(title=f'Distribution of {column} with Quantiles', 
-               x_axis_label=f'{column} Values', 
-               y_axis_label='Frequency',
-               width=800, height=400)
-    
-    
-    p.quad(top=hist, bottom=0, left=edges[:-1], right= edges[1:], 
-           fill_color='skyblue', line_color='black', alpha=0.7)
-
-    # Plotting line graph
-    p.line(midpoints, hist, line_width=2, color='blue', alpha=0.7)
-    
-    # Add quantile lines
-    for q in quantiles:
-        span = Span(location=q, dimension='height', line_color='red', line_dash='dashed', line_width=2)
-        p.add_layout(span)
-        p.add_layout(Label(x=q, y=max(hist), text=f'{q:.1f}', text_color='red'))
-    
-    return p
-
-
-# Bind the create_plot function to the quantile slider, column dropdown, and button click
-marker_intensity_with_histogram = pn.bind(create_intensity_plot,column_dropdown.param.value, quantile_slider.param.value, watch=True)
-
-# Create the button
-generate_plot_button = Button(label='Generate Plot', button_type='primary')
-
-def update_plot(column, quantile):
-    plot = create_intensity_plot(column, quantile)
-    plot.renderers[0].data_source = source  # Update the data source for the renderer
-    return plot
-
-#Display the dropdown menu, quantile slider, button, and plot
-#plot = update_plot(column_dropdown.param.value, quantile_slider.param.value)
-
-def generate_plot(event):
-    updated_plot = update_plot(column_dropdown.param.value, quantile_slider.param.value)
-    #pn.Column(pn.Row(column_dropdown, generate_plot_button), quantile_slider, updated_plot).servable()
-
-generate_plot_button.on_click(generate_plot)
-selected_marker_plot = pn.Column(pn.Row(pn.Column(column_dropdown, marker_intensity_with_histogram )))
-#pn.Column(pn.Row(pn.Column(column_dropdown, marker_intensity_with_histogram ), generate_plot_button)).servable()
-
-import panel as pn
-import numpy as np
-import pandas as pd
-from bokeh.plotting import figure
-from bokeh.models import ColumnDataSource, Button, Span, Label
-
-# Assuming you have a DataFrame 'df' with the intensity columns
-intensities = df.filter(like='Intensity').columns.tolist()
-
-# Create a ColumnDataSource from the DataFrame
-source = ColumnDataSource(df)
-
-# Function to calculate quantile values
-def calculate_quantiles(column, quantile):
-    quantiles = df[column].quantile(q=[quantile, 0.50, 1 - quantile])
-    return quantiles
-
-quantile_slider = pn.widgets.FloatSlider(name='Quantile', start=0.01, end=0.99, step=0.01, value=0.05)
-
-
-# Bind the create_line_graph function to the quantile slider
-#nucleus_size_line_graph = pn.bind(create_line_graph, quantile=quantile_slider.param.value)
-
-# Layout the components in a Panel app
-#nucleus_size_graph = pn.Column(nucleus_size_line_graph)
-
-len(intensities)
-
-df
-
-def calculate_cytoplasm_quantiles(column, quantile):
-    # Print the columns of the DataFrame
-    print("DataFrame columns:", df.columns)
-    
-    # Check if the column exists in the DataFrame
-    if column not in df.columns:
-        raise KeyError(f"Column '{column}' does not exist in the DataFrame.")
-    
-    quantiles = df[column].quantile(q=[quantile, 0.50, 1 - quantile])
-    return quantiles
-
-def create_cytoplasm_intensity_df(column, quantile):
-    quantiles = calculate_cytoplasm_quantiles(column, quantile)
-    output = pd.DataFrame(quantiles)
-    return pn.pane.DataFrame(output)
-
-# Bind the create_app function to the quantile slider
-cytoplasm_quantile_output_app = pn.bind(create_cytoplasm_intensity_df, column=df.columns[10], quantile=quantile_slider.param.value)
-
-pn.Column(quantile_slider, cytoplasm_quantile_output_app)
-def calculate_cytoplasm_quantiles(column, quantile):
-    quantiles = df[column].quantile(q=[quantile, 0.50, 1 - quantile])
-    return quantiles
-
-def create_cytoplasm_intensity_df(column, quantile):
-    quantiles = calculate_cytoplasm_quantiles(column, quantile)
-    output = pd.DataFrame(quantiles)
-    # Create a Dataframe widget to display the output
-    output_widget = pn.pane.DataFrame(output)
-    return output_widget
-
-
-# Bind the create_app function to the quantile slider
-cytoplasm_quantile_output_app = pn.bind(create_cytoplasm_intensity_df, column=df.columns[10], quantile = quantile_slider.param.value)
-pn.Column(quantile_slider,cytoplasm_quantile_output_app)
-
-
-# ## I.5. COLUMNS OF INTERESTS
-
-# Remove columns containing "DAPI"
-df = df[[x for x in df.columns.values if 'DAPI' not in x]]
-
-print("Columns are now...")
-print([c for c in df.columns.values])
-
-
-# Create lists of full names and shortened names to use in plotting
-full_to_short_names, short_to_full_names =  \
-    shorten_feature_names(df.columns.values[~df.columns.isin(not_intensities)])
-
-short_to_full_names
-
-
-# Save this data to a metadata file
-filename = os.path.join(metadata_dir, "full_to_short_column_names.csv")
-fh = open(filename, "w")
-fh.write("full_name,short_name\n")
-for k,v in full_to_short_names.items():
-    fh.write(k + "," + v + "\n")
-    
-fh.close()
-print("The full_to_short_column_names.csv file was created !")
-
-# Save this data to a metadata file
-filename = os.path.join(metadata_dir, "short_to_full_column_names.csv")
-fh = open(filename, "w")
-fh.write("short_name,full_name\n")
-for k,v in short_to_full_names.items():
-    fh.write(k + "," + v + "\n")
-    
-fh.close()
-print("The short_to_full_column_names.csv file was created !")
-
-
-# ## I.6. EXPOSURE TIME
-
-
-#import the ashlar analysis file 
-file_path = os.path.join(metadata_dir, 'combined_metadata.csv')
-ashlar_analysis = pd.read_csv(file_path)
-ashlar_analysis 
-# Extracting and renaming columns
-new_df = ashlar_analysis[['Name', 'Cycle', 'ChannelIndex', 'ExposureTime']].copy()
-new_df.rename(columns={
-    'Name': 'Target',
-    'Cycle': 'Round',
-    'ChannelIndex': 'Channel'
-}, inplace=True)
-
-# Applying suffixes to the columns
-new_df['Round'] = 'R' + new_df['Round'].astype(str)
-new_df['Channel'] = 'c' + new_df['Channel'].astype(str) 
-
-# Save to CSV
-new_df.to_csv('Ashlar_Exposure_Time.csv', index=False)
-
-# Print the new dataframe
-print(new_df)
-
-# Here, we want to end up with a data structure that incorporates metadata on each intensity marker column used in our big dataframe in an easy-to-use format. 
-# This is going to include the full name of the intensity marker columns in the big data frame, 
-# the corresponding round and channel, 
-# the target protein (e.g., CD45), 
-# and the segmentation localization information (cell, cytoplasm, nucleus)
-
-# We can use this data structure to assign unique colors to all channels and rounds, for example, for use in later visualizations
-# Exposure_time file from ASHLAR analysis
-filename = "Exposure_Time.csv"
-filename = os.path.join(metadata_dir, filename)
-exp_df = pd.read_csv(filename)
-
-print(exp_df)
-
-# Verify file imported correctly
-# File length
-print("df's shape: ", exp_df.shape)
-# Headers
-expected_headers =['Round','Target','Exp','Channel']
-compare_headers(expected_headers, exp_df.columns.values, "Imported metadata file")
-
-# Missingness
-if exp_df.isnull().any().any():
-    print("\nexp_df has null value(s) in row(s):")
-    print(exp_df[exp_df.isna().any(axis=1)])
-else:
-    print("\nNo null values detected.")
-
-
-if len(exp_df['Target']) > len(exp_df['Target'].unique()):
-    print("One or more non-unique Target values in exp_df. Currently not supported.")
-exp_df = exp_df.drop_duplicates(subset = 'Target').reindex()
-
-# sort exp_df by the values in the 'Target' column in ascending order and then retrieve the first few rows of the sorted df
-exp_df.sort_values(by = ['Target']).head()
-
-# Create lowercase version of target
-exp_df['target_lower'] = exp_df['Target'].str.lower()
-exp_df.head()
-
-# Create df that contains marker intensity columns in our df that aren't in not_intensities
-intensities = pd.DataFrame({'full_column':df.columns.values[~df.columns.isin(not_intensities)]})
-
-intensities
-# Extract the marker information from the `full_column`, which corresponds to full column in big dataframe
-# Use regular expressions (regex) to isolate the part of the field that begins (^) with an alphanumeric value (W), and ends with an underscore (_)
-# '$' is end of line
-intensities['marker'] = intensities['full_column'].str.extract(r'([^\W_]+)')
-# convert to lowercase
-intensities['marker_lower'] = intensities['marker'].str.lower()
-
-intensities
-# Subset the intensities df to exclude any column pertaining to DAPI
-intensities = intensities.loc[intensities['marker_lower'] != 'dapi']
-
-intensities.head()
-# Merge the intensities andexp_df together to create metadata
-metadata = pd.merge(exp_df, intensities, how = 'left', left_on = 'target_lower',right_on = 'marker_lower')
-metadata = metadata.drop(columns = ['marker_lower'])
-metadata = metadata.dropna()
-
-# Target is the capitalization from the Exposure_Time.csv
-# target_lower is Target in small caps
-# marker is the extracted first component of the full column in segmentation data, with corresponding capitalization
-metadata
-# Add a column to signify marker target localisation.
-# Use a lambda to determine segmented location of intensity marker column and update metadata accordingly
-# Using the add_metadata_location() function in my_modules.py
-metadata['localisation'] = metadata.apply(
-    lambda row: add_metadata_location(row), axis = 1)
-
-mlid = metadata
-
-# Save this data structure to the metadata folder
-# don't want to add color in because that's better off treating color the same for round, channel, and sample
-filename = "marker_intensity_metadata.csv"
-filename = os.path.join(metadata_dir, filename)
-metadata.to_csv(filename, index = False)
-print("The marker_intensity_metadata.csv file was created !")
-
-
-
-# ## I.7. COLORS WORKFLOW
-
-# ### I.7.1. CHANNELS COLORS
-
-
-# we want colors that are categorical, since Channel is a non-ordered category (yes, they are numbered, but arbitrarily). 
-# A categorical color palette will have dissimilar colors.
-# Get those unique colors
-if len(metadata.Channel.unique()) > 10:
-    print("WARNING: There are more unique channel values than \
-    there are colors to choose from. Select different palette, e.g., \
-    continuous palette 'husl'.")
-channel_color_values = sb.color_palette("bright",n_colors = len(metadata.Channel.unique()))
-# chose 'colorblind' because it is categorical and we're unlikely to have > 10
-
-# You can customize the colors for each channel here
-custom_colors = {
-    'c2': 'lightgreen',
-    'c3': 'tomato',
-    'c4': 'pink',
-    'c5': 'turquoise'
-}
-
-custom_colors_values = sb.palplot(sb.color_palette([custom_colors.get(ch, 'blue') for ch in metadata.Channel.unique()]))
-
-# Display those unique customs colors
-print("Unique channels are:", metadata.Channel.unique())
-sb.palplot(sb.color_palette(channel_color_values))
-
-# Function to create a palette plot with custom colors
-def create_palette_plot():
-    # Get unique channels
-    unique_channels = metadata.Channel.unique()
-    
-    # Define custom colors for each channel
-    custom_colors = {
-        'c2': 'lightgreen',
-        'c3': 'tomato',
-        'c4': 'pink',
-        'c5': 'turquoise'
-    }
-    
-    # Get custom colors for each channel
-    colors = [custom_colors.get(ch, 'blue') for ch in unique_channels]
-    
-    # Create a palette plot (palplot)
-    palette_plot = sb.palplot(sb.color_palette(colors))
-    channel_color_values = sb.color_palette("bright",n_colors = len(metadata.Channel.unique()))
-    channel_color_values = sb.palplot(channel_color_values)
-    return palette_plot, channel_color_values
-
-
-# Create the palette plot directly
-palette_plot = create_palette_plot()
-
-# Define the Panel app layout
-app_palette_plot = pn.Column(
-    pn.pane.Markdown("### Custom Color Palette"),
-    palette_plot, 
-)
-
-# Function to create a palette plot with custom colors
-def create_palette_plot(custom_colors):
-    # Get unique channels
-    unique_channels = metadata.Channel.unique()
-    
-    # Get custom colors for each channel
-    colors = [custom_colors.get(ch, 'blue') for ch in unique_channels]
-    
-    # Create a palette plot (palplot)
-    palette_plot = sb.palplot(sb.color_palette(colors))
-    
-    return palette_plot
-
-# Define custom colors for each channel
-custom_colors = {
-    'c2': 'lightgreen',
-    'c3': 'tomato',
-    'c4': 'pink',
-    'c5': 'turquoise'
-}
-
-# Display those unique customs colo
-print("Unique channels are:", metadata.Channel.unique())
-# Function to bind create_palette_plot
-app_palette_plot = create_palette_plot(custom_colors)
-
-
-#app_palette_plot.servable()
-
-
-# Store in a dictionary
-channel_color_dict = dict(zip(metadata.Channel.unique(), channel_color_values))
-channel_color_dict
-for k,v in channel_color_dict.items():
-    channel_color_dict[k] = np.float64(v)
-
-channel_color_dict
-
-color_df_channel = color_dict_to_df(channel_color_dict, "Channel")
-
-# Save to file in metadatadirectory
-filename = "channel_color_data.csv"
-filename = os.path.join(metadata_dir, filename)
-color_df_channel.to_csv(filename, index = False)
-
-color_df_channel
-
-# Legend of channel info only
-g  = plt.figure(figsize = (1,1)).add_subplot(111)
-g.axis('off')
-handles = []
-for item in channel_color_dict.keys():
-        h = g.bar(0,0, color = channel_color_dict[item],
-                  label = item, linewidth =0)
-        handles.append(h)
-first_legend = plt.legend(handles=handles, loc='upper right', title = 'Channel'),
-                            # box_to_anchor=(10,10), 
-                             #       bbox_transform=plt.gcf().transFigure)
-
-filename = "Channel_legend.png"
-filename = os.path.join(metadata_images_dir, filename)
-plt.savefig(filename, bbox_inches = 'tight')
-
-# ### I.7.2. ROUNDS COLORS
-
-
-# we want colors that are sequential, since Round is an ordered category. 
-# We can still generate colors that are easy to distinguish. Also, many of the categorical palettes cap at at about 10 or so unique colors, and repeat from there. 
-# We do not want any repeats!
-round_color_values = sb.cubehelix_palette(
-    len(metadata.Round.unique()), start=1, rot= -0.75, dark=0.19, light=.85, reverse=True)
-# round_color_values = sb.color_palette("cubehelix",n_colors = len(metadata.Round.unique()))
-# chose 'cubehelix' because it is sequential, and round is a continuous process
-# each color value is a tuple of three values: (R, G, B)
-print(metadata.Round.unique())
-
-sb.palplot(sb.color_palette(round_color_values))
-
-## TO-DO: write what these parameters mean
-
-# Store in a dictionary
-round_color_dict = dict(zip(metadata.Round.unique(), round_color_values))
-
-for k,v in round_color_dict.items():
-    round_color_dict[k] = np.float64(v)
-
-round_color_dict
-
-color_df_round = color_dict_to_df(round_color_dict, "Round")
-
-# Save to file in metadatadirectory
-filename = "round_color_data.csv"
-filename = os.path.join(metadata_dir, filename)
-color_df_round.to_csv(filename, index = False)
-
-color_df_round
-
-# Legend of round info only
-
-round_legend  = plt.figure(figsize = (1,1)).add_subplot(111)
-round_legend.axis('off')
-handles = []
-for item in round_color_dict.keys():
-        h = round_legend.bar(0,0, color = round_color_dict[item],
-                  label = item, linewidth =0)
-        handles.append(h)
-first_legend = plt.legend(handles=handles, loc='upper right', title = 'Round'),
-                            # bbox_to_anchor=(10,10), 
-                             #       bbox_transform=plt.gcf().transFigure)
-
-filename = "Round_legend.png"
-filename = os.path.join(metadata_images_dir, filename)
-plt.savefig(filename, bbox_inches = 'tight')
-
-
-# ### I.7.3. SAMPLES COLORS
-
-# we want colors that are neither sequential nor categorical. 
-# Categorical would be ideal if we could generate an arbitrary number of colors, but I do not think that we can. 
-# Hense, we will choose `n` colors from a continuous palette. First we will generate the right number of colors. Later, we will assign TMA samples to gray.
-
-# Get those unique colors
-color_values = sb.color_palette("husl",n_colors = len(ls_samples))#'HLS'
-# each color value is a tuple of three values: (R, G, B)
-
-# Display those unique colors
-sb.palplot(sb.color_palette(color_values))
-
-TMA_samples = [s for s in df.Sample_ID.unique() if 'TMA' in s]
-TMA_color_values = sb.color_palette(n_colors = len(TMA_samples),palette = "gray")
-sb.palplot(sb.color_palette(TMA_color_values))
-
-# Store in a dictionary
-color_dict = dict()
-color_dict = dict(zip(df.Sample_ID.unique(), color_values))
-
-# Replace all TMA samples' colors with gray
-i = 0
-for key in color_dict.keys():
-    if 'TMA' in key:
-        color_dict[key] = TMA_color_values[i]
-        i +=1
-
-color_dict
-
-color_df_sample = color_dict_to_df(color_dict, "Sample_ID")
-
-# Save to file in metadatadirectory
-filename = "sample_color_data.csv"
-filename = os.path.join(metadata_dir, filename)
-color_df_sample.to_csv(filename, index = False)
-
-color_df_sample
-
-
-# Legend of sample info only
-g  = plt.figure(figsize = (1,1)).add_subplot(111)
-g.axis('off')
-handles = []
-for item in color_dict.keys():
-        h = g.bar(0,0, color = color_dict[item],
-                  label = item, linewidth =0)
-        handles.append(h)
-first_legend = plt.legend(handles=handles, loc='upper right', title = 'Sample')
-
-filename = "Sample_legend.png"
-filename = os.path.join(metadata_images_dir, filename)
-plt.savefig(filename, bbox_inches = 'tight')
-
-
-# ### I.7.4. CLUSTERS COLORS
-
-'''if 'cluster' in df.columns:
-    cluster_color_values = sb.color_palette("hls",n_colors = len(df.cluster.unique()))
-
-    #print(sorted(test_df.cluster.unique()))
-    # Display those unique colors
-    sb.palplot(sb.color_palette(cluster_color_values))
-    
-    cluster_color_dict = dict(zip(sorted(test_df.cluster.unique()), cluster_color_values))
-    print(cluster_color_dict)
-    
-    # Create dataframe
-    cluster_color_df = color_dict_to_df(cluster_color_dict, "cluster")
-    cluster_color_df.head()
-
-    # Save to file in metadatadirectory
-    filename = "cluster_color_data.csv"
-    filename = os.path.join(metadata_dir, filename)
-    cluster_color_df.to_csv(filename, index = False) 
-
-
-
-# Legend of cluster info only
-
-if 'cluster' in df.columns:
-    g  = plt.figure(figsize = (1,1)).add_subplot(111)
-    g.axis('off')
-    handles = []
-    for item in sorted(cluster_color_dict.keys()):
-            h = g.bar(0,0, color = cluster_color_dict[item],
-                      label = item, linewidth =0)
-            handles.append(h)
-    first_legend = plt.legend(handles=handles, loc='upper right', title = 'Cluster'),
-
-
-    filename = "Clustertype_legend.png"
-    filename = os.path.join(metadata_images_dir, filename)
-    plt.savefig(filename, bbox_inches = 'tight')'''
-
-mlid.head()
-
-
-metadata
-
-
-
-import io
-import panel as pn
-pn.extension()
-
-file_input = pn.widgets.FileInput()
-
-file_input
-
-
-def transform_data(variable, window, sigma):
-    """Calculates the rolling average and identifies outliers"""
-    avg = metadata[variable].rolling(window=window).mean()
-    residual = metadata[variable] - avg
-    std = residual.rolling(window=window).std()
-    outliers = np.abs(residual) > std * sigma
-    return avg, avg[outliers]
-
-
-def get_plot(variable="Exp", window=30, sigma=10):
-    """Plots the rolling average and the outliers"""
-    avg, highlight = transform_data(variable, window, sigma)
-    return avg.hvplot(
-        height=300, legend=False,
-    ) * highlight.hvplot.scatter(padding=0.1, legend=False)
-
-
-variable_widget = pn.widgets.Select(name="Target", value="Exp", options=list(metadata.columns))
-window_widget = pn.widgets.IntSlider(name="window", value=30, start=1, end=60)
-sigma_widget = pn.widgets.IntSlider(name="sigma", value=10, start=0, end=20)
-
-# Function to save files
-def save_files(event):
-    for sample in ls_samples:
-        sample_id = sample.split('.csv')[0]
-        filename = os.path.join(output_data_dir, sample_id + "_" + step_suffix + ".csv")
-        
-        df_save = df.loc[df['Sample_ID'] == sample, :]
-        if os.path.exists(filename):
-            df_save.to_csv(filename, index=True, index_label='ID', mode='w')  # Overwrite by default
-            print(f"File {filename} was overwritten!")
-        else:
-            df_save.to_csv(filename, index=True, index_label='ID')  # Save normally if the file doesn't exist
-            print(f"File {filename} was created and saved!")
-
-# Button to download files
-download_button = pn.widgets.Button(name='Download Files', button_type='primary')
-download_button.on_click(save_files)
-
-app = pn.template.GoldenTemplate(
-    site="Cyc-IF",
-    title="Quality Control",
-    main=[
-        pn.Tabs(
-            ("Dataframes", pn.Column(
-                pn.Row(csv_files_button,pn.bind(handle_click, csv_files_button.param.clicks), ),
-                pn.pane.Markdown("### The Dataframe uploaded:"), pn.pane.DataFrame(intial_dataframe),
-                #pn.pane.Markdown("### The Exposure time DataFrame is :"), pn.pane.DataFrame(exp_df.head()),
-                pn.pane.Markdown("### The DataFrame after merging CycIF data x metadata :"), pn.pane.DataFrame(merged_dataframe.head(25)),
-            )),
-            ("Quality Control", pn.Column(
-                quality_check(quality_control_df, not_intensities)
-                #pn.pane.Markdown("### The Quality check results are:"), quality_check_results(check_shape, check_no_null, check_all_expected_files_present, check_zero_intensities)
-            )),
-            ("Intensities", pn.Column(
-                pn.pane.Markdown("### The Not Intensities DataFrame after processing is :"), pn.pane.DataFrame(not_intensities_df, height=250),
-                pn.pane.Markdown("### Select Intensities to be included"), updated_intensities,
-                #pn.pane.Markdown("### The Intensities DataFrame"), intensities_df,
-                #pn.pane.Markdown("### The metadata obtained that specifies the localisation:"), pn.pane.DataFrame(mlid.head())
-            )),
-            ("Plots", pn.Column(
-                #pn.pane.Markdown(" ### Nucleus Size Distribution: "), pn.Row(nucleus_size_line_graph_with_histogram, num_of_cell_removal),
-                pn.pane.Markdown(" ### Nucleus Size Distribution: "), pn.Row(plot1,layout2),
-                #pn.pane.Markdown("### Nucleus Distribution Plot:"), pn.Column(nucleus_size_plot, nucleus_size_graph),
-                pn.pane.Markdown(" ### Intensity Average Plot:"), pn.Row(selected_marker_plot,num_of_cell_removal_intensity ),                
-                #pn.Column(pn.Column(column_dropdown, generate_plot_button), quantile_slider, plot),
-                #pn.pane.Markdown("### Cytoplasm Intensity Plot:"), cytoplasm_intensity_plot,
-                #pn.pane.Markdown("### AF555_Cell_Intensity_Average:"), quantile_output_app,
-                #pn.pane.Markdown("### Distribution of AF555_Cell_Intensity_Average with Quantiles:"), quantile_intensity_plot),
-                pn.Column(download_button),
-            )),
-            
-),
-    ])
-    
-app.servable()
-
-if __name__ == "__main__":
-    pn.serve(app, port=5007)