Step5_Marker_Threshold_Classification.py

#!/usr/bin/env python
# coding: utf-8
# # IV. MARKERS TRESHOLDS NOTEBOOK
# ## IV.1. PACKAGES IMPORT

import os
import random
import re
import pandas as pd
import numpy as np
import seaborn as sb
import matplotlib.pyplot as plt
import matplotlib.colors as mplc
import subprocess
import warnings
import panel as pn 
import json
from scipy import signal
from scipy.stats import pearsonr
import plotly.figure_factory as ff
import plotly
import plotly.graph_objs as go
from plotly.subplots import make_subplots
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot 
import plotly.express as px
import sys
sys.setrecursionlimit(5000)
from my_modules import *
#Silence FutureWarnings & UserWarnings
warnings.filterwarnings('ignore', category= FutureWarning)
warnings.filterwarnings('ignore', category= UserWarning)


# ## IV.2. *DIRECTORIES 
# Set base directory
#input_path = '/Users/harshithakolipaka/Downloads/wetransfer_data-zip_2024-05-17_1431'
#set_path = 'test'
present_dir = os.path.dirname(os.path.realpath(__file__))    
stored_variables_path = os.path.join(present_dir,'stored_variables.json')
with open(stored_variables_path, 'r') as file:
        stored_vars = json.load(file)
        directory = stored_vars['base_dir']
        input_path = os.path.join(present_dir,directory) 
        set_path = stored_vars['set_path']
        selected_metadata_files = stored_vars['selected_metadata_files']
        ls_samples = stored_vars['ls_samples']
base_dir = input_path
set_name = set_path
project_name = set_name                 # Project name
step_suffix = 'mt'                      # Curent part (here part IV)
previous_step_suffix_long = "_zscore"   # Previous part (here ZSCORE NOTEBOOK)

# Initial input data directory
input_data_dir = os.path.join(base_dir, project_name + previous_step_suffix_long) 

# ZSCORE/LOG2 output directories
output_data_dir = os.path.join(base_dir, project_name + "_" + step_suffix)
# ZSCORE/LOG2 images subdirectory
output_images_dir = os.path.join(output_data_dir,"images")

# Data and Metadata directories
# Metadata directories
metadata_dir = os.path.join(base_dir, project_name + "_metadata")
# images subdirectory
metadata_images_dir = os.path.join(metadata_dir,"images")

# Create directories if they don't already exist
#for d in [base_dir, input_data_dir, output_data_dir, output_images_dir, metadata_dir, metadata_images_dir]:
#   if not os.path.exists(d):
        #print("Creation of the" , d, "directory...")
#        os.makedirs(d)
    #else :
    #    print("The", d, "directory already exists !")

#os.chdir(input_data_dir)


# Verify paths
#print('base_dir :', base_dir)
#print('input_data_dir :', input_data_dir)
#print('output_data_dir :', output_data_dir)
#print('output_images_dir :', output_images_dir)
#print('metadata_dir :', metadata_dir)
#print('metadata_images_dir :', metadata_images_dir)


# ## IV.3. FILES

# ### IV.3.1. METADATA


filename = "marker_intensity_metadata.csv"
filename = os.path.join(metadata_dir, filename)

# Check file exists
#if not os.path.exists(filename):
#    print("WARNING: Could not find desired file: "+filename)
#else :
#    print("The",filename,"file was imported for further analysis!")
    
# Open, read in information
metadata = pd.read_csv(filename)

# Verify size with verify_line_no() function in my_modules.py
#verify_line_no(filename, metadata.shape[0] + 1)

# Verify headers
exp_cols = ['Round','Target','Channel','target_lower','full_column','marker','localisation']
compare_headers(exp_cols, metadata.columns.values, "Marker metadata file")

metadata = metadata.dropna()
metadata.head()


# ### IV.3.2. NOT_INTENSITIES
filename = "not_intensities.csv"
filename = os.path.join(metadata_dir, filename)

# Check file exists
#if not os.path.exists(filename):
#    print("WARNING: Could not find desired file: "+filename)
#else :
#    print("The",filename,"file was imported for further analysis!")
    
not_intensities = []
with open(filename, 'r') as fh:
    not_intensities = fh.read().strip().split("\n")
    # take str, strip whitespace, split on new line character

# Verify size
#print("\nVerifying data read from file is the correct length...\n")
#verify_line_no(filename, len(not_intensities))

# Print to console
#print("not_intensities =\n", not_intensities)


# ### IV.3.3. FULL_TO_SHORT_COLUMN_NAMES

filename = "full_to_short_column_names.csv"
filename = os.path.join(metadata_dir, filename)

# Check file exists
#if not os.path.exists(filename):
#    print("WARNING: Could not find desired file: " + filename)
#else :
#    print("The",filename,"file was imported for further analysis!")

# Open, read in information
df = pd.read_csv(filename, header = 0)

# Verify size
print("Verifying data read from file is the correct length...\n")
#verify_line_no(filename, df.shape[0] + 1)

# Turn into dictionary
full_to_short_names = df.set_index('full_name').T.to_dict('records')[0]
#print('full_to_short_names =\n',full_to_short_names)


# ### IV.3.4. SHORT_TO_FULL_COLUMN_NAMES


filename = "short_to_full_column_names.csv"
filename = os.path.join(metadata_dir, filename)

# Check file exists
#if not os.path.exists(filename):
#    print("WARNING: Could not find desired file: " + filename)
#else :
#    print("The",filename,"file was imported for further analysis!")

# Open, read in information
df = pd.read_csv(filename, header = 0)

# Verify size
#print("Verifying data read from file is the correct length...\n")
#verify_line_no(filename, df.shape[0] + 1)

# Turn into dictionary
short_to_full_names = df.set_index('short_name').T.to_dict('records')[0]
# Print information
#print('short_to_full_names =\n',short_to_full_names)


# ### IV.3.10. DATA

# List files in the directory
# Check if the directory exists
if os.path.exists(input_data_dir):
    # List files in the directory
    ls_samples = [sample for sample in os.listdir(input_data_dir) if sample.endswith("_zscore.csv")]
#    print("The following CSV files were detected:")
#    print([sample for sample in ls_samples])
#else:
#    print(f"The directory {input_data_dir} does not exist.")
# Import all the others files
dfs = {}

# Set variable to hold default header values
# First gather information on expected headers using first file in ls_samples
# Read in the first row of the file corresponding to the first sample (index = 0) in ls_samples
df = pd.read_csv(os.path.join(input_data_dir, ls_samples[0]) , index_col = 0, nrows = 1)
expected_headers = df.columns.values
#print('Header order should be :\n', expected_headers, '\n')

###############################
# !! This may take a while !! #
###############################
for sample in ls_samples:
    file_path = os.path.join(input_data_dir,sample)
   
    try:
        # Read the CSV file
        df = pd.read_csv(file_path, index_col=0)
        # Check if the DataFrame is empty, if so, don't continue trying to process df and remove it
        
        if not df.empty:
            # Reorder the columns to match the expected headers list
            df = df.reindex(columns=expected_headers)
#            print(sample, "file is processed !\n")
            #print(df) 
   
    except pd.errors.EmptyDataError:
#       print(f'\nEmpty data error in {sample} file. Removing from analysis...')
        ls_samples.remove(sample)      
    
    # Add df to dfs 
    dfs[sample] = df

#print(dfs)

# Merge dfs into one df
df = pd.concat(dfs.values(), ignore_index=False , sort = False)
del dfs

print(df.head())

intial_df = pn.pane.DataFrame(df.head(40), width = 2500)


# ### Marker Classification

# ## IV.5. *DOTPLOTS

df
# Load existing data from stored_variables.json with error handling
try:
    with open(stored_variables_path, 'r') as file:
        data = json.load(file)
except json.JSONDecodeError as e:
#    print(f"Error reading JSON file: {e}")
    data = {}

# Debug: Print loaded data to verify keys
#print(data)

df
df.head()


# ### IV.7.2. DOTPLOTS-DETERMINED TRESHOLD
#Empty dict in stored_variables to store the cell type classification for each marker 
#stored_variables_path = '/Users/harshithakolipaka/Downloads/stored_variables.json'
try:
    with open(stored_variables_path, 'r') as f:
        stored_variables = json.load(f)
except FileNotFoundError:
    stored_variables = {}

# Check if 'thresholds' field is present, if not, add it
if 'cell_type_classification' not in stored_variables:
    cell_type_classification = {}
    stored_variables['cell_type_classification'] = cell_type_classification
    with open(stored_variables_path, 'w') as f:
        json.dump(stored_variables, f, indent=4)
        
#Empty dict in stored_variables to store the cell subtype classification for each marker         
#stored_variables_path = '/Users/harshithakolipaka/Downloads/stored_variables.json'
try:
    with open(stored_variables_path, 'r') as f:
        stored_variables = json.load(f)
except FileNotFoundError:
    stored_variables = {}

# Check if 'thresholds' field is present, if not, add it
if 'cell_subtype_classification' not in stored_variables:
    cell_type_classification = {}
    stored_variables['cell_subtype_classification'] = cell_type_classification
    with open(stored_variables_path, 'w') as f:
        json.dump(stored_variables, f, indent=4)

df
data = df


import json
import panel as pn

# Load existing stored variables
with open(stored_variables_path, 'r') as f:
    stored_variables = json.load(f)

# Initialize a dictionary to hold threshold inputs
threshold_inputs = {}

# Create widgets for each marker to get threshold inputs from the user
for marker in stored_variables['markers']:
    threshold_inputs[marker] = pn.widgets.FloatInput(name=f'{marker} Threshold', value=0.0, step=0.1)
    
# Load stored_variables.json
#stored_variables_path = '/Users/harshithakolipaka/Downloads/stored_variables.json'
try:
    with open(stored_variables_path, 'r') as f:
        stored_variables = json.load(f)
except FileNotFoundError:
    stored_variables = {}

# Check if 'thresholds' field is present, if not, add it
if 'thresholds' not in stored_variables:
    thresholds = {marker: input_widget.value for marker, input_widget in threshold_inputs.items()}
    stored_variables['thresholds'] = thresholds
    with open(stored_variables_path, 'w') as f:
        json.dump(stored_variables, f, indent=4)
    
# Save button to save thresholds to stored_variables.json
def save_thresholds(event):
    thresholds = {marker: input_widget.value for marker, input_widget in threshold_inputs.items()}
    stored_variables['thresholds'] = thresholds
    with open(stored_variables_path, 'w') as f:
        json.dump(stored_variables, f, indent=4)
    pn.state.notifications.success('Thresholds saved successfully!')

save_button2 = pn.widgets.Button(name='Save Thresholds', button_type='primary')
save_button2.on_click(save_thresholds)

# Create a GridSpec layout
grid = pn.GridSpec()

# Add the widgets to the grid with three per row
row = 0
col = 0
for marker in stored_variables['markers']:
    grid[row, col] = threshold_inputs[marker]
    col += 1
    if col == 5:
        col = 0
        row += 1

# Add the save button at the end
grid[row + 1, :5] = save_button2

# Panel layout
threshold_panel = pn.Column(
    pn.pane.Markdown("## Define Thresholds for Markers"),
    grid)


import pandas as pd
import json

# Load stored variables from the JSON file
with open(stored_variables_path, 'r') as file:
    stored_variables = json.load(file)
# Step 1: Identify intensities
intensities = list(df.columns)

def assign_cell_type(row):
    for intensity in intensities:
        marker = intensity.split('_')[0]  # Extract marker from intensity name
        if marker in stored_variables['thresholds']:
            threshold = stored_variables['thresholds'][marker]
            if row[intensity] > threshold:
                for cell_type, markers in stored_variables['cell_type_classification'].items():
                    if marker in markers:
                        return cell_type
        return 'STROMA'  # Default if no condition matches

# Step 5: Apply the classification function to the DataFrame
df['cell_type'] = df.apply(lambda row: assign_cell_type(row), axis=1)
df.head()
# Check if 'IMMUNE' is present in any row of the cell_type column
present_stroma = df['cell_type'].str.contains('STROMA').sum()
present_cancer = df['cell_type'].str.contains('CANCER').sum()
present_immune = df['cell_type'].str.contains('IMMUNE').sum()
present_endothelial = df['cell_type'].str.contains('ENDOTHELIAL').sum()
# Print the result
#print(present_stroma)
#print(present_cancer)
#print(present_immune)
#print(present_endothelial)
#print(len(df))
df.head(30)
df

# ## IV.8. *HEATMAPS
#print(df.columns)
# Assuming df_merged is your DataFrame
if 'Sample_ID.1' in df.columns: 
    df = df.rename(columns={'Sample_ID.1': 'Sample_ID'})
 #   print("After renaming Sample_ID", df.columns)
# Selecting a subset of rows from the DataFrame df based on the 'Sample_ID' column 
# and then randomly choosing 20,000 rows from that subset to create the DataFrame test_dfkeep = ['TMA.csv']
with open(stored_variables_path, 'r') as file:
        ls_samples = stored_vars['ls_samples']
keep = ls_samples

keep_cell_type = ['ENDOTHELIAL','CANCER', 'STROMA', 'IMMUNE']
#if 'Sample_ID' in df.columns:
#   print("The",df.loc[df['cell_type'].isin(keep_cell_type)])
test2_df = df.loc[(df['cell_type'].isin(keep_cell_type)) 
                & (df['Sample_ID'].isin(keep)), :].copy()
#print(test2_df.head())

random_rows = np.random.choice(len(test2_df),20000)
df2 = test2_df.iloc[random_rows,:].copy()

df2
#print(df2)


# ### COLORS

# #### SAMPLES COLORS
color_values = sb.color_palette("husl",n_colors = len(ls_samples))
sb.palplot(sb.color_palette(color_values))

TMA_samples = [s for s in df.Sample_ID.unique() if 'TMA' in s]
TMA_color_values = sb.color_palette(n_colors = len(TMA_samples),palette = "gray")
sb.palplot(sb.color_palette(TMA_color_values))

# Store in a dictionary
color_dict = dict()
color_dict = dict(zip(df.Sample_ID.unique(), color_values))

# Replace all TMA samples' colors with gray
i = 0
for key in color_dict.keys():
    if 'TMA' in key:
        color_dict[key] = TMA_color_values[i]
        i +=1

color_dict

color_df_sample = color_dict_to_df(color_dict, "Sample_ID")

# Save to file in metadatadirectory
filename = "sample_color_data.csv"
filename = os.path.join(metadata_dir, filename)
color_df_sample.to_csv(filename, index = False)

color_df_sample

# Legend of sample info only
g  = plt.figure(figsize = (1,1)).add_subplot(111)
g.axis('off')
handles = []
for item in color_dict.keys():
        h = g.bar(0,0, color = color_dict[item],
                  label = item, linewidth =0)
        handles.append(h)
first_legend = plt.legend(handles=handles, loc='upper right', title = 'Sample')

filename = "Sample_legend.png"
filename = os.path.join(metadata_images_dir, filename)
plt.savefig(filename, bbox_inches = 'tight')

filename = "sample_color_data.csv"
filename = os.path.join(metadata_dir, filename)

# Check file exists
#if not os.path.exists(filename):
#    print("WARNING: Could not find desired file: " + filename)
#else :
#    print("The",filename,"file was imported for further analysis!")
    
# Open, read in information
df = pd.read_csv(filename, header = 0)
df = df.drop(columns = ['hex'])

# our tuple of float values for rgb, (r, g, b) was read in 
# as a string '(r, g, b)'. We need to extract the r-, g-, and b-
# substrings and convert them back into floats
df['rgb'] = df.apply(lambda row: rgb_tuple_from_str(row['rgb']), axis = 1)

# Verify size
#print("Verifying data read from file is the correct length...\n")
#verify_line_no(filename, df.shape[0] + 1)

# Turn into dictionary
sample_color_dict = df.set_index('Sample_ID')['rgb'].to_dict()

# Print information
#print('sample_color_dict =\n',sample_color_dict)


# #### CELL TYPES COLORS

# Define your custom colors for each cell type
custom_colors = {
    'CANCER': (0.1333, 0.5451, 0.1333),
    'STROMA': (0.4, 0.4, 0.4),
    'IMMUNE': (1, 1, 0),
    'ENDOTHELIAL': (0.502, 0, 0.502)
}

# Retrieve the list of cell types
cell_types = list(custom_colors.keys())

# Extract the corresponding colors from the dictionary
color_values = [custom_colors[cell] for cell in cell_types]

# Display the colors
sb.palplot(sb.color_palette(color_values))

# Store in a dctionnary
celltype_color_dict = dict(zip(cell_types, color_values))
celltype_color_dict

# Save color information (mapping and legend) to metadata directory
# Create dataframe
celltype_color_df = color_dict_to_df(celltype_color_dict, "cell_type")
celltype_color_df.head()

# Save to file in metadatadirectory
filename = "celltype_color_data.csv"
filename = os.path.join(metadata_dir, filename)
celltype_color_df.to_csv(filename, index = False)
#print("File" + filename + " was created!")

# Legend of cell type info only
g  = plt.figure(figsize = (1,1)).add_subplot(111)
g.axis('off')
handles = []
for item in celltype_color_dict.keys():
        h = g.bar(0,0, color = celltype_color_dict[item],
                  label = item, linewidth =0)
        handles.append(h)
first_legend = plt.legend(handles=handles, loc='upper right', title = 'Cell type'),


filename = "Celltype_legend.png"
filename = os.path.join(metadata_images_dir, filename)
plt.savefig(filename, bbox_inches = 'tight')

filename = "celltype_color_data.csv"
filename = os.path.join(metadata_dir, filename)

# Check file exists
#if not os.path.exists(filename):
#    print("WARNING: Could not find desired file: "+filename)
#else :
#    print("The",filename,"file was imported for further analysis!")
    
# Open, read in information
df = pd.read_csv(filename, header = 0)
df = df.drop(columns = ['hex'])

# our tuple of float values for rgb, (r, g, b) was read in 
# as a string '(r, g, b)'. We need to extract the r-, g-, and b-
# substrings and convert them back into floats
df['rgb'] = df.apply(lambda row: rgb_tuple_from_str(row['rgb']), axis = 1)

# Verify size
#print("Verifying data read from file is the correct length...\n")
#verify_line_no(filename, df.shape[0] + 1)

# Turn into dictionary
cell_type_color_dict = df.set_index('cell_type')['rgb'].to_dict()

# Print information
#print('cell_type_color_dict =\n',cell_type_color_dict)   

# Colors dictionaries
sample_row_colors =df2.Sample_ID.map(sample_color_dict)
#print(sample_row_colors[1:5])

cell_type_row_colors = df2.cell_type.map(cell_type_color_dict)
#print(cell_type_row_colors[1:5])


# ## Cell Subtype Colours
import pandas as pd
import os

def rgb_tuple_from_str(rgb_str):
    # Cleaning the string to remove any unexpected 'np.float64'
    rgb_str = rgb_str.replace("(","").replace(")","").replace(" ","").replace("np.float64", "")
    try:
        rgb = list(map(float, rgb_str.split(",")))
        return tuple(rgb)
    except ValueError as e:
#        print(f"Error converting {rgb_str} to floats: {e}")
        return None  # or handle the error as needed

filename = "cellsubtype_color_data.csv"
filename = os.path.join(metadata_dir, filename)

# Check file exists
#if not os.path.exists(filename):
#    print("WARNING: Could not find desired file: " + filename)
#else:
#    print("The", filename, "file was imported for further analysis!")

# Open, read in information
df = pd.read_csv(filename, header=0)
df = df.drop(columns=['hex'])

# Clean the 'rgb' column to remove unexpected strings
df['rgb'] = df['rgb'].str.replace("np.float64", "", regex=False)

# Apply the function to convert string to tuple of floats
df['rgb'] = df.apply(lambda row: rgb_tuple_from_str(row['rgb']), axis=1)

# Verify size
#print("Verifying data read from file is the correct length...\n")
# verify_line_no(filename, df.shape[0] + 1)

# Turn into dictionary
cell_subtype_color_dict = df.set_index('cell_subtype')['rgb'].to_dict()

# Print information
#print('cell_subtype_color_dict =\n', cell_subtype_color_dict)

df2

# Colors dictionaries
sample_row_colors =df2.Sample_ID.map(sample_color_dict)
#print(sample_row_colors[1:5])

cell_subtype_row_colors = df2.cell_subtype.map(cell_subtype_color_dict)
#print(cell_subtype_row_colors[1:5])


# #### Cell Type
df
#print(f"Loaded sample files: {ls_samples}")
selected_intensities = list(df.columns)
selected_intensities = list(df.columns)
#print(selected_intensities)
df
df2
df = df2
df
import json
import pandas as pd
import numpy as np
import panel as pn
import plotly.graph_objects as go

pn.extension('plotly')
# Load the selected intensities from the JSON file
with open(stored_variables_path, 'r') as f:
    json_data = json.load(f)

ls_samples = json_data["ls_samples"]
#print(f"Loaded sample files: {ls_samples}")

# Checkbox group to select files
checkbox_group = pn.widgets.CheckBoxGroup(name='Select Files', options=ls_samples)

# Initially empty dropdowns for X and Y axis selection
x_axis_dropdown = pn.widgets.Select(name='Select X-Axis', options=[])
y_axis_dropdown = pn.widgets.Select(name='Select Y-Axis', options=[])

# Input field for the number of random samples
random_sample_input = pn.widgets.IntInput(name='Number of Random Samples', value=20000, step=100)

# Sliders for interactive X and Y lines
x_line_slider = pn.widgets.FloatSlider(name='X Axis Line Position', start=0, end=1, step=0.01)
y_line_slider = pn.widgets.FloatSlider(name='Y Axis Line Position', start=0, end=1, step=0.01)

# Placeholder for the dot plot
plot_placeholder = pn.pane.Plotly()

# Placeholder for the digital reconstruction plot
reconstruction_placeholder = pn.pane.Plotly()

# Function to create the dot plot
def create_dot_plot(selected_files, x_axis, y_axis, n_samples, x_line_pos, y_line_pos):
    if not selected_files:
#        print("No files selected.")
        return go.Figure()
    
    keep = selected_files
    
    test2_df = df.loc[df['Sample_ID'].isin(keep), :].copy()
#    print(f"Number of samples in test2_df: {len(test2_df)}")
    if len(test2_df) > n_samples:
        random_rows = np.random.choice(len(test2_df), n_samples)
        test_df = test2_df.iloc[random_rows, :].copy()
    else:
        test_df = test2_df
    
#    print(f"Number of samples in test_df: {len(test_df)}")
    
    if x_axis not in test_df.columns or y_axis not in test_df.columns:
#        print(f"Selected axes {x_axis} or {y_axis} not in DataFrame columns.")
        return go.Figure()
    
    fig = go.Figure()
    title = 'Threshold'
    
    fig.add_trace(go.Scatter(
        x=test_df[x_axis], 
        y=test_df[y_axis],
        mode='markers',
        marker=dict(color='LightSkyBlue', size=2)
    ))

    # Add vertical and horizontal lines
    fig.add_vline(x=x_line_pos, line_width=2, line_dash="dash", line_color="red")
    fig.add_hline(y=y_line_pos, line_width=2, line_dash="dash", line_color="red")
    
    fig.update_layout(
        title=title, 
        plot_bgcolor='white',
        autosize=True,
        margin=dict(l=20, r=20, t=40, b=20),
        xaxis=dict(title=x_axis, linecolor='black', range=[test_df[x_axis].min(), test_df[x_axis].max()]),
        yaxis=dict(title=y_axis, linecolor='black', range=[test_df[y_axis].min(), test_df[y_axis].max()])
    )
    return fig

def assign_cell_types_again():
    with open(stored_variables_path, 'r') as file:
        stored_variables = json.load(file)
    intensities = list(df.columns)
    def assign_cell_type(row):
        for intensity in intensities:
            marker = intensity.split('_')[0]  # Extract marker from intensity name
            if marker in stored_variables['thresholds']:
                threshold = stored_variables['thresholds'][marker]
                if row[intensity] > threshold:
                    for cell_type, markers in stored_variables['cell_type_classification'].items():
                        if marker in markers:
                            return cell_type
        return 'STROMA'  # Default if no condition matches
    df['cell_type'] = df.apply(lambda row: assign_cell_type(row), axis=1)
    return df 

# Function to create the digital reconstruction plot
def create_reconstruction_plot(selected_files):
    if not selected_files:
#       print("No files selected.")
        return go.Figure()
    df = assign_cell_types_again()
    fig = go.Figure()
    
    for sample in selected_files:
        sample_id = sample
        sample_id2 = sample.split('_')[0]
        location_colors = df.loc[df['Sample_ID'] == sample_id, ['Nuc_X', 'Nuc_Y_Inv', 'cell_type']]

        title = sample_id2 + " Background Subtracted XY Map cell types"
        
        for celltype in df.loc[df['Sample_ID'] == sample_id, 'cell_type'].unique():
            fig.add_scatter(
                mode='markers',
                marker=dict(size=3, opacity=0.5, color='rgb' + str(cell_type_color_dict[celltype])),
                x=location_colors.loc[location_colors['cell_type'] == celltype, 'Nuc_X'],
                y=location_colors.loc[location_colors['cell_type'] == celltype, 'Nuc_Y_Inv'],
                name=celltype
            )

        fig.update_layout(
            title=title,
            plot_bgcolor='white',
            autosize=True,
            margin=dict(l=20, r=20, t=40, b=20),
            legend=dict(
                title='Cell Types',
                font=dict(
                    family='Arial',
                    size=12,
                    color='black'
                ),
                bgcolor='white',
                bordercolor='black',
                borderwidth=0.4,
                itemsizing='constant'
            ),
            xaxis=dict(title='Nuc_X', linecolor='black', range=[location_colors['Nuc_X'].min(), location_colors['Nuc_X'].max()]),
            yaxis=dict(title='Nuc_Y_Inv', linecolor='black', range=[location_colors['Nuc_Y_Inv'].min(), location_colors['Nuc_Y_Inv'].max()])
        )

    return fig

def update_dropdown_options(event):
    selected_files = checkbox_group.value
#    print(f"Selected files in update_dropdown_options: {selected_files}")
    if selected_files:
        keep = selected_files
        test2_df = df.loc[df['Sample_ID'].isin(keep), :].copy()
        selected_intensities = list(test2_df.columns)
        selected_intensities = [col for col in selected_intensities if '_Intensity_Average' in col]
#        print(f"Updated dropdown options: {selected_intensities}")
        x_axis_dropdown.options = selected_intensities
        y_axis_dropdown.options = selected_intensities
    else:
        x_axis_dropdown.options = []
        y_axis_dropdown.options = []

def update_slider_ranges(event):
    selected_files = checkbox_group.value
    x_axis = x_axis_dropdown.value
    y_axis = y_axis_dropdown.value
#    print("Axis:",x_axis,y_axis) 
    if selected_files and x_axis and y_axis:
        keep = selected_files
        test2_df = df.loc[df['Sample_ID'].isin(keep), :].copy()
        x_range = (test2_df[x_axis].min(), test2_df[x_axis].max())
        y_range = (test2_df[y_axis].min(), test2_df[y_axis].max())
#        print("Ranges:",x_range,y_range)
        x_line_slider.start = -abs(x_range[1])
        x_line_slider.end = abs(x_range[1])
        y_line_slider.start = -abs(y_range[1])
        y_line_slider.end = abs(y_range[1])
        x_line_slider.value = 0
        y_line_slider.value = 0

def on_value_change(event):
    selected_files = checkbox_group.value
    x_axis = x_axis_dropdown.value
    y_axis = y_axis_dropdown.value
    n_samples = random_sample_input.value
    x_line_pos = x_line_slider.value
    y_line_pos = y_line_slider.value
#    print(f"Selected files: {selected_files}")
#    print(f"X-Axis: {x_axis}, Y-Axis: {y_axis}, Number of samples: {n_samples}, X Line: {x_line_pos}, Y Line: {y_line_pos}")
    plot = create_dot_plot(selected_files, x_axis, y_axis, n_samples, x_line_pos, y_line_pos)
    reconstruction_plot = create_reconstruction_plot(selected_files)
    plot_placeholder.object = plot
    reconstruction_placeholder.object = reconstruction_plot

# Link value changes to function
checkbox_group.param.watch(update_dropdown_options, 'value')
checkbox_group.param.watch(update_slider_ranges, 'value')
x_axis_dropdown.param.watch(update_slider_ranges, 'value')
y_axis_dropdown.param.watch(update_slider_ranges, 'value')
x_axis_dropdown.param.watch(on_value_change, 'value')
y_axis_dropdown.param.watch(on_value_change, 'value')
random_sample_input.param.watch(on_value_change, 'value')
x_line_slider.param.watch(on_value_change, 'value')
y_line_slider.param.watch(on_value_change, 'value')

# Layout
plot_with_reconstruction = pn.Column(
    "## Select Files to Construct Dot Plot",
    checkbox_group,
    x_axis_dropdown,
    y_axis_dropdown,
    random_sample_input,
    pn.Row(x_line_slider, y_line_slider),
    pn.Row(
        pn.Column(
            "## Dot Plot",
            pn.Column(plot_placeholder)),
        pn.Column(
            "## Digital Reconstruction Plot",
            reconstruction_placeholder),
))

# Serve the app
#plot_with_reconstruction.show()

# ## MAKE HEATMAPS

# ### Cell Subtype
# Create data structure to hold everything we need for row/column annotations
# annotations is a dictionary
## IMPORTANT - if you use 'annotations', it MUST have both 'rows' and 'cols'
## objects inside. These can be empty lists, but they must be there!
anns = {}

# create a data structure to hold everything we need for only row annotations
# row_annotations is a list, where each item therein is a dictioary corresponding
# to all of the data pertaining to that particular annotation
# Adding each item (e.g., Sample, then Cluster), one at a time to ensure ordering
# is as anticipated on figure
row_annotations = []
row_annotations.append({'label':'Sample',
                        'type':'row',
                        'mapping':sample_row_colors,
                        'dict':sample_color_dict,
                        'location':'center left',
                        'bbox_to_anchor':(0.1, 0.9)})
row_annotations.append({'label':'Cell type',
                        'type':'row',
                        'mapping':cell_type_row_colors,
                        'dict':cell_type_color_dict,
                        'location':'center left',
                        'bbox_to_anchor':(0.17, 0.9)})
anns['rows'] = row_annotations

# Now we repeat the process for column annotations
col_annotations = []
anns['cols'] = col_annotations
# To simplify marker display in the following figures (heatmap, etc)
figure_marker_names = {key: value.split('_')[0] for key, value in full_to_short_names.items()} 
not_intensities
df2
df2.drop('cell_subtype', axis = 'columns')
not_intensities = ['Nuc_X', 'Nuc_X_Inv', 'Nuc_Y', 'Nuc_Y_Inv', 'Nucleus_Roundness', 'Nucleus_Size', 'Cell_Size', 
                   'ROI_index', 'Sample_ID', 'replicate_ID', 'Cell_ID','cell_type', 'cell_subtype', 'cluster','ID', 
                   'Cytoplasm_Size', 'immune_checkpoint', 'Unique_ROI_index', 'Patient', 'Primary_chem(1)_vs_surg(0)']
df2 = assign_cell_types_again()
df2.drop('cell_subtype', axis = 'columns')
df2.head()
# Save one heatmap

data = df
data
#print(data.columns)
# Selecting a subset of rows from df based on the 'Sample_ID' column 
# and then random>ly choosing 50,000 rows from that subset to create the DataFrame test_df
with open(stored_variables_path, 'r') as file:
        ls_samples = stored_vars['ls_samples']
keep = list(ls_samples)
keep_cell_type = ['STROMA','CANCER','IMMUNE','ENDOTHELIAL']

# Check the individual conditions
cell_type_condition = data['cell_type'].isin(keep_cell_type)
sample_id_condition = data['Sample_ID'].isin(keep)
#print("Cell type condition:")
#print(cell_type_condition.head())
#print("Sample ID condition:")
#print(sample_id_condition.head())

# Combine the conditions
combined_condition = cell_type_condition & sample_id_condition
#print("Combined condition:")
#print(combined_condition.head())

# Apply the combined condition to filter the DataFrame
test2_df = data.loc[combined_condition].copy()
#print("Filtered DataFrame:")
#print(test2_df.head())

#test2_df = data.loc[data['cell_type'].isin(keep_cell_type) & data['Sample_ID'].isin(keep)].copy()
#print("Test2_df",test2_df.head())
#print(len(test2_df))

#random_rows = np.random.choice(len(test2_df),len(test2_df))
random_rows = np.random.choice(len(test2_df),1000)
test_df = test2_df.iloc[random_rows,:].copy()
#print(len(test_df))
test_df
import json
import panel as pn
import param
import pandas as pd

# Initialize Panel extension
pn.extension('tabulator')

# Path to the stored variables file
file_path = stored_variables_path

# Load existing data from stored_variables.json with error handling
def load_data():
    try:
        with open(file_path, 'r') as file:
            return json.load(file)
    except json.JSONDecodeError as e:
        print(f"Error reading JSON file: {e}")
        return {}

data = load_data()

# Define markers, cell types, and cell subtypes from the loaded data
markers = data.get('markers', [])
cell_types = data.get('cell_type', [])
cell_subtypes = data.get('cell_subtype', [])

# Sanitize option names
def sanitize_options(options):
    return [opt.replace(' ', '_').replace('+', 'plus').replace('α', 'a').replace("'", "") for opt in options]

sanitized_cell_types = sanitize_options(cell_types)
sanitized_cell_subtypes = sanitize_options(cell_subtypes)

# Helper function to create a Parameterized class and DataFrame
def create_classification_df(items, item_label):
    params = {item_label: param.String()}
    for marker in markers:
        params[marker] = param.Boolean(default=False)
    
    Classification = type(f'{item_label}Classification', (param.Parameterized,), params)
    
    classification_widgets = []
    for item in items:
        item_params = {marker: False for marker in markers}
        item_params[item_label] = item
        classification_widgets.append(Classification(**item_params))
    
    classification_df = pd.DataFrame([cw.param.values() for cw in classification_widgets])
    classification_df = classification_df[[item_label] + markers]
    return classification_df

# Create DataFrames for cell types and cell subtypes
cell_type_df = create_classification_df(sanitized_cell_types, 'CELL_TYPE')
cell_subtype_df = create_classification_df(sanitized_cell_subtypes, 'CELL_SUBTYPE')

# Define formatters for Tabulator widgets
tabulator_formatters = {marker: {'type': 'tickCross'} for marker in markers}

# Create Tabulator widgets
cell_type_table = pn.widgets.Tabulator(cell_type_df, formatters=tabulator_formatters)
cell_subtype_table = pn.widgets.Tabulator(cell_subtype_df, formatters=tabulator_formatters)

# Save functions for cell types and cell subtypes
def save_data(table, classification_key, item_label):
    current_data = table.value
    df_bool = current_data.replace({'✔': True, '✘': False})
    
    classification = {}
    for i, row in df_bool.iterrows():
        item = row[item_label]
        selected_markers = [marker for marker in markers if row[marker]]
        classification[item] = selected_markers
    
    data[classification_key] = classification
#    try:
    with open(file_path, 'w') as file:
        json.dump(data, file, indent=4)
#        print(f"{classification_key} saved successfully.")
#    except IOError as e:
#        print(f"Error writing JSON file: {e}")

# Button actions
def save_cell_type_selections(event):
    save_data(cell_type_table, 'cell_type_classification', 'CELL_TYPE')

def save_cell_subtype_selections(event):
    save_data(cell_subtype_table, 'cell_subtype_classification', 'CELL_SUBTYPE')

# Create save buttons
save_cell_type_button = pn.widgets.Button(name='Save Cell Type Selections', button_type='primary')
save_cell_type_button.on_click(save_cell_type_selections)

save_cell_subtype_button = pn.widgets.Button(name='Save Cell Subtype Selections', button_type='primary')
save_cell_subtype_button.on_click(save_cell_subtype_selections)
cell_type_classification_app_main = pn.Column(
        pn.pane.Markdown("# Cell Type Classification"),
        cell_type_table,
        save_cell_type_button
    )
cell_subtype_classification_app_main = pn.Column(
        pn.pane.Markdown("# Cell Subtype Classification"),
        cell_subtype_table,
        save_cell_subtype_button
    )
#cell_subtype_classification_app_main.show()

import json
import panel as pn

# Load existing stored variables
with open(stored_variables_path, 'r') as f:
    stored_variables = json.load(f)

# Initialize a dictionary to hold threshold inputs
subtype_threshold_inputs = {}

# Create widgets for each marker to get threshold inputs from the user
for marker in stored_variables['markers']:
    subtype_threshold_inputs[marker] = pn.widgets.FloatInput(name=f'{marker} Threshold', value=0.0, step=0.1)
    
try:
    with open(stored_variables_path, 'r') as f:
        stored_variables = json.load(f)
except FileNotFoundError:
    stored_variables = {}

# Check if 'thresholds' field is present, if not, add it
if 'subtype_thresholds' not in stored_variables:
    subtype_thresholds = {marker: input_widget.value for marker, input_widget in subtype_threshold_inputs.items()}
    stored_variables['subtype_thresholds'] = subtype_thresholds
    with open(stored_variables_path, 'w') as f:
        json.dump(stored_variables, f, indent=4)
    
# Save button to save thresholds to stored_variables.json
def save_thresholds(event):
    subtype_thresholds = {marker: input_widget.value for marker, input_widget in subtype_threshold_inputs.items()}
    stored_variables['subtype_thresholds'] = subtype_thresholds
    with open(stored_variables_path, 'w') as f:
        json.dump(stored_variables, f, indent=4)
save_button = pn.widgets.Button(name='Save Thresholds', button_type='primary')
save_button.on_click(save_thresholds)

# Create a GridSpec layout
subtype_grid = pn.GridSpec()

# Add the widgets to the grid with five per row
row = 0
col = 0
for marker in stored_variables['markers']:
    subtype_grid[row, col] = subtype_threshold_inputs[marker]
    col += 1
    if col == 5:
        col = 0
        row += 1

# Add the save button at the end, spanning across all columns of the new row
subtype_grid[row + 1, :5] = save_button

# Panel layout
subtype_threshold_panel = pn.Column(
    pn.pane.Markdown("## Define Thresholds for Markers"),
    subtype_grid)

# Display the panel
#subtype_threshold_panel.show()

with open(stored_variables_path, 'r') as file:
        stored_variables = json.load(file)
intensities = list(df.columns)
def assign_cell_subtypes(row):
    for intensity in intensities:
        marker = intensity.split('_')[0]  # Extract marker from intensity name
        if marker in stored_variables['subtype_thresholds']:
            threshold = stored_variables['subtype_thresholds'][marker]
            if row[intensity] > threshold:
                for cell_subtype, markers in stored_variables['cell_subtype_classification'].items():
                    if marker in markers:
                        return cell_subtype
            return 'DC' 

df = assign_cell_types_again()       
df['cell_subtype'] = df.apply(lambda row: assign_cell_subtypes(row), axis=1)

df
data
# Define a color dictionary
cell_subtype_color_dict = {
    'DC': (0.6509803921568628, 0.807843137254902, 0.8901960784313725),
    'B': (0.12156862745098039, 0.47058823529411764, 0.7058823529411765),
    'TCD4': (0.6980392156862745, 0.8745098039215686, 0.5411764705882353),
    'Exhausted TCD4': (0.2, 0.6274509803921569, 0.17254901960784313),
    'Exhausted TCD8': (0.984313725490196, 0.6039215686274509, 0.6),
    'TCD8': (0.8901960784313725, 0.10196078431372549, 0.10980392156862745),
    'M1': (0.9921568627450981, 0.7490196078431373, 0.43529411764705883),
    'M2': (1.0, 0.4980392156862745, 0.0),
    'Treg': (0.792156862745098, 0.6980392156862745, 0.8392156862745098),
    'Other CD45+': (0.41568627450980394, 0.23921568627450981, 0.6039215686274509),
    'Cancer': (1.0, 1.0, 0.6),
    'myCAF αSMA+': (0.6941176470588235, 0.34901960784313724, 0.1568627450980392),
    'Stroma': (0.6509803921568628, 0.807843137254902, 0.8901960784313725),
    'Endothelial': (0.12156862745098039, 0.47058823529411764, 0.7058823529411765)
}
# Add the 'rgb' prefix to the colors
cell_subtype_color_dict = {k: f"rgb{v}" for k, v in cell_subtype_color_dict.items()}

# Load stored variables from JSON file
def load_stored_variables(path):
    with open(path, 'r') as file:
        return json.load(file)

# Get subtype intensities columns
subtype_intensities = [col for col in df.columns if '_Intensity_Average' in col]

# Assign cell subtype based on thresholds and classifications
def assign_cell_subtype(row):
    #print("new_row")
    stored_variables = load_stored_variables(stored_variables_path)
    for subtype_intensity in subtype_intensities:
        marker = subtype_intensity.split('_')[0]
        if marker in stored_variables['subtype_thresholds']:
            subtype_threshold = stored_variables['subtype_thresholds'][marker]
            if row[subtype_intensity] > subtype_threshold:
                for cell_subtype, markers in stored_variables['cell_subtype_classification'].items():
                    #print(cell_subtype,marker,markers)
                    if marker in markers:
                        #print("Markers:",marker)
                        return cell_subtype  # Return the assigned subtype
    return 'DC'  # Default value if no conditions match

# Main function to assign cell subtypes to DataFrame
def assign_cell_subtypes_again():
    df['cell_subtype'] = df.apply(lambda row: assign_cell_subtype(row), axis=1)
    return df

import json
import pandas as pd
import numpy as np
import panel as pn
import plotly.graph_objects as go

pn.extension('plotly')

# Load the selected intensities from the JSON file
with open(stored_variables_path, 'r') as f:
    json_data = json.load(f)

subtype_ls_samples = json_data["ls_samples"]
#print(f"Loaded sample files: {subtype_ls_samples}")


# Checkbox group to select files
subtype_checkbox_group = pn.widgets.CheckBoxGroup(name='Select Files', options=subtype_ls_samples)

# Initially empty dropdowns for X and Y axis selection
subtype_x_axis_dropdown = pn.widgets.Select(name='Select X-Axis', options=[])
subtype_y_axis_dropdown = pn.widgets.Select(name='Select Y-Axis', options=[])

# Input field for the number of random samples
subtype_random_sample_input = pn.widgets.IntInput(name='Number of Random Samples', value=20000, step=100)

# Sliders for interactive X and Y lines
subtype_x_line_slider = pn.widgets.FloatSlider(name='X Axis Line Position', start=0, end=1, step=0.01)
subtype_y_line_slider = pn.widgets.FloatSlider(name='Y Axis Line Position', start=0, end=1, step=0.01)

# Placeholder for the dot plot
subtype_plot_placeholder = pn.pane.Plotly()

# Placeholder for the digital reconstruction plot
subtype_reconstruction_placeholder = pn.pane.Plotly()

def update_color_dict():
    # Define a color dictionary
    cell_subtype_color_dict = {
        'DC': (0.6509803921568628, 0.807843137254902, 0.8901960784313725),
        'B': (0.12156862745098039, 0.47058823529411764, 0.7058823529411765),
        'TCD4': (0.6980392156862745, 0.8745098039215686, 0.5411764705882353),
        'Exhausted TCD4': (0.2, 0.6274509803921569, 0.17254901960784313),
        'Exhausted TCD8': (0.984313725490196, 0.6039215686274509, 0.6),
        'TCD8': (0.8901960784313725, 0.10196078431372549, 0.10980392156862745),
        'M1': (0.9921568627450981, 0.7490196078431373, 0.43529411764705883),
        'M2': (1.0, 0.4980392156862745, 0.0),
        'Treg': (0.792156862745098, 0.6980392156862745, 0.8392156862745098),
        'Other CD45+': (0.41568627450980394, 0.23921568627450981, 0.6039215686274509),
        'Cancer': (1.0, 1.0, 0.6),
        'myCAF αSMA+': (0.6941176470588235, 0.34901960784313724, 0.1568627450980392),
        'Stroma': (0.6509803921568628, 0.807843137254902, 0.8901960784313725),
        'Endothelial': (0.12156862745098039, 0.47058823529411764, 0.7058823529411765)
    }
    # Add the 'rgb' prefix to the colors
    cell_subtype_color_dict = {k: f"rgb{v}" for k, v in cell_subtype_color_dict.items()}
    return cell_subtype_color_dict 
    
# Function to create the dot plot
def create_subtype_dot_plot(subtype_selected_files, subtype_x_axis, subtype_y_axis, subtype_n_samples, subtype_x_line_pos, subtype_y_line_pos):
    if not subtype_selected_files:
#        print("No files selected.")
        return go.Figure()
    subtype_keep = subtype_selected_files
#    print(df)
    subtype_test2_df = df.loc[df['Sample_ID'].isin(subtype_keep), :].copy()
    #subtype_test2_df = df.loc[df['Sample_ID'].isin('TMA.csv'), :].copy()
#    print(f"Number of samples in test2_df: {len(subtype_test2_df)}")
    if len(subtype_test2_df) > subtype_n_samples:
        subtype_random_rows = np.random.choice(len(subtype_test2_df), subtype_n_samples)
        subtype_test_df = subtype_test2_df.iloc[subtype_random_rows, :].copy()
    else:
        subtype_test_df = subtype_test2_df

#    print(f"Number of samples in test_df: {len(subtype_test_df)}")

    if subtype_x_axis not in subtype_test_df.columns or subtype_y_axis not in subtype_test_df.columns:
#        print(f"Selected axes {subtype_x_axis} or {subtype_y_axis} not in DataFrame columns.")
        return go.Figure()

    fig = go.Figure()
    title = 'Threshold'

    fig.add_trace(go.Scatter(
        x=subtype_test_df[subtype_x_axis],
        y=subtype_test_df[subtype_y_axis],
        mode='markers',
        marker=dict(color='LightSkyBlue', size=2)
    ))

    # Add vertical and horizontal lines
    fig.add_vline(x=subtype_x_line_pos, line_width=2, line_dash="dash", line_color="red")
    fig.add_hline(y=subtype_y_line_pos, line_width=2, line_dash="dash", line_color="red")

    fig.update_layout(
        title=title,
        plot_bgcolor='white',
        autosize=True,
        margin=dict(l=20, r=20, t=40, b=20),
        xaxis=dict(title=subtype_x_axis, linecolor='black', range=[subtype_test_df[subtype_x_axis].min(), subtype_test_df[subtype_x_axis].max()]),
        yaxis=dict(title=subtype_y_axis, linecolor='black', range=[subtype_test_df[subtype_y_axis].min(), subtype_test_df[subtype_y_axis].max()])
    )
    return fig

def create_subtype_reconstruction_plot(subtype_selected_files):
    cell_subtype_color_dict = update_color_dict()
#    print(subtype_selected_files)
    if not subtype_selected_files:
#        print("No files selected.")
        return go.Figure()
    df = assign_cell_subtypes_again()
    subtype_fig = go.Figure()

    for sample in subtype_selected_files:
        sample_id = sample
        sample_id2 = sample.split('_')[0]
        location_colors = df.loc[df['Sample_ID'] == sample_id, ['Nuc_X', 'Nuc_Y_Inv', 'cell_subtype']]
#        print(location_colors.head())
        title = sample_id2 + " Background Subtracted XY Map cell subtypes"
        for cellsubtype in df.loc[df['Sample_ID'] == sample_id, 'cell_subtype'].unique():
            color = str(cell_subtype_color_dict[cellsubtype])
            subtype_fig.add_scatter(
                mode='markers',
                marker=dict(size=3, opacity=0.5, color=color),
                x=location_colors.loc[location_colors['cell_subtype'] == cellsubtype, 'Nuc_X'],
                y=location_colors.loc[location_colors['cell_subtype'] == cellsubtype, 'Nuc_Y_Inv'],
                name=cellsubtype
            )

        subtype_fig.update_layout(title=title, plot_bgcolor='white')
        subtype_fig.update_xaxes(title_text='Nuc_X', linecolor='black')
        subtype_fig.update_yaxes(title_text='Nuc_Y_Inv', linecolor='black')

        # Adjust the size of the points
        for trace in subtype_fig.data:
            trace.marker.size = 2

        subtype_fig.update_layout(
            title=title,
            plot_bgcolor='white',
            legend=dict(
                title='Cell Subtypes',  # Legend title
                font=dict(
                    family='Arial',
                    size=12,
                    color='black'
                ),
                bgcolor='white',
                bordercolor='black',
                borderwidth=0.4,
                itemsizing='constant'
            )
        )
        # Save the figure as an image if needed
        #subtype_fig.write_image(output_images_dir + "/" + title.replace(" ", "_") + ".png", width=1200, height=800, scale=4)
#        print(sample_id, "processed!")

    return subtype_fig

def update_subtype_dropdown_options(event):
#    print(1)
    subtype_selected_files = subtype_checkbox_group.value
#    print(f"Selected files in update_dropdown_options: {subtype_selected_files}")
    if subtype_selected_files:
        subtype_keep = subtype_selected_files
        subtype_test2_df = df.loc[df['Sample_ID'].isin(subtype_keep), :].copy()
        subtype_selected_intensities = list(subtype_test2_df.columns)
        subtype_selected_intensities = [col for col in subtype_selected_intensities if '_Intensity_Average' in col]
#        print(f"Updated dropdown options: {subtype_selected_intensities}")
        subtype_x_axis_dropdown.options = subtype_selected_intensities
        subtype_y_axis_dropdown.options = subtype_selected_intensities
    else:
        subtype_x_axis_dropdown.options = []
        subtype_y_axis_dropdown.options = []

def update_subtype_slider_ranges(event):
    subtype_selected_files = subtype_checkbox_group.value
    subtype_x_axis = subtype_x_axis_dropdown.value
    subtype_y_axis = subtype_y_axis_dropdown.value

    if subtype_selected_files and subtype_x_axis and subtype_y_axis:
        subtype_keep = subtype_selected_files
        subtype_test2_df = df.loc[df['Sample_ID'].isin(subtype_keep), :].copy()
        subtype_x_range = (subtype_test2_df[subtype_x_axis].min(), subtype_test2_df[subtype_x_axis].max())
        subtype_y_range = (subtype_test2_df[subtype_y_axis].min(), subtype_test2_df[subtype_y_axis].max())
        subtype_x_line_slider.start = -abs(subtype_x_range[1])
        subtype_x_line_slider.end = abs(subtype_x_range[1])
        subtype_y_line_slider.start = -abs(subtype_y_range[1])
        subtype_y_line_slider.end = abs(subtype_y_range[1])
        subtype_x_line_slider.value = 0
        subtype_y_line_slider.value = 0

def on_subtype_value_change(event):
    subtype_selected_files = subtype_checkbox_group.value
    subtype_x_axis = subtype_x_axis_dropdown.value
    subtype_y_axis = subtype_y_axis_dropdown.value
    subtype_n_samples = subtype_random_sample_input.value
    subtype_x_line_pos = subtype_x_line_slider.value
    subtype_y_line_pos = subtype_y_line_slider.value
#    print(f"Selected files: {subtype_selected_files}")
#    print(f"X-Axis: {subtype_x_axis}, Y-Axis: {subtype_y_axis}, Number of samples: {subtype_n_samples}, X Line: {subtype_x_line_pos}, Y Line: {subtype_y_line_pos}")
    subtype_plot = create_subtype_dot_plot(subtype_selected_files, subtype_x_axis, subtype_y_axis, subtype_n_samples, subtype_x_line_pos, subtype_y_line_pos)
    subtype_reconstruction_plot = create_subtype_reconstruction_plot(subtype_selected_files)
    subtype_plot_placeholder.object = subtype_plot
    subtype_reconstruction_placeholder.object = subtype_reconstruction_plot

# Link value changes to function
subtype_checkbox_group.param.watch(update_subtype_dropdown_options, 'value')
subtype_checkbox_group.param.watch(update_subtype_slider_ranges, 'value')
subtype_x_axis_dropdown.param.watch(update_subtype_slider_ranges, 'value')
subtype_y_axis_dropdown.param.watch(update_subtype_slider_ranges, 'value')
subtype_x_axis_dropdown.param.watch(on_subtype_value_change, 'value')
subtype_y_axis_dropdown.param.watch(on_subtype_value_change, 'value')
subtype_random_sample_input.param.watch(on_subtype_value_change, 'value')
subtype_x_line_slider.param.watch(on_subtype_value_change, 'value')
subtype_y_line_slider.param.watch(on_subtype_value_change, 'value')

# Layout
plot_with_subtype_reconstruction = pn.Column(
    "## Select Files to Construct Dot Plot",
    subtype_checkbox_group,
    subtype_x_axis_dropdown,
    subtype_y_axis_dropdown,
    subtype_random_sample_input,
    pn.Row(subtype_x_line_slider, subtype_y_line_slider),
    pn.Row(
        pn.Column(
            "## Dot Plot",
           pn.Column(subtype_plot_placeholder)),
        pn.Column(
            "## Cell Subtype Digital Reconstruction Plot",
            subtype_reconstruction_placeholder),
)
)

subtype_x_axis = subtype_x_axis_dropdown.value
subtype_y_axis = subtype_y_axis_dropdown.value
#print(subtype_x_axis      ,subtype_y_axis)


# Normalize the values in df2.cell_subtype
df2['cell_subtype'] = df2['cell_subtype'].str.strip().str.lower()

# Normalize the keys in cell_subtype_color_dict
cell_subtype_color_dict = {k.strip().lower(): v for k, v in cell_subtype_color_dict.items()}

# Map the cell_subtype values to colors
cell_subtype_row_colors = df2.cell_subtype.map(cell_subtype_color_dict)

# Debugging: print the unique values and the resulting mapped colors
#print("Unique values in df2.cell_subtype:", df2.cell_subtype.unique())
#print("Keys in cell_subtype_color_dict:", cell_subtype_color_dict.keys())
#print(cell_subtype_row_colors[1:5])
data
cell_subtype_color_dict
# Remove the 'rgb' prefix

cell_subtype_color_dict = {k: v[3:] for k, v in cell_subtype_color_dict.items()}
cell_subtype_color_dict

# Colors dictionaries
sample_row_colors =df.Sample_ID.map(sample_color_dict)
#print(sample_row_colors[1:5])

cell_subtype_row_colors = df.cell_subtype.map(cell_subtype_color_dict)
#print(cell_subtype_row_colors[1:5])

# Count of each immune_checkpoint type by cell_subtype
counts = df.groupby(['cell_type', 'cell_subtype']).size().reset_index(name='count')
counts

total = sum(counts['count'])
counts['percentage'] = counts.groupby('cell_subtype')['count'].transform(lambda x: (x / total) * 100)

#print(counts)


# ## IV.10. SAVE

# Save the data by Sample_ID
# Check for the existence of the output file first
for sample in ls_samples:
    #sample_id = sample.split('_')[0]
    sample_id = sample 
    filename = os.path.join(output_data_dir,  sample_id + "_" + step_suffix + ".csv")
    if os.path.exists(filename):
        df_save = df.loc[df['Sample_ID'] == sample_id, :]
        df_save.to_csv(filename, index=True, index_label='ID', mode='w')  # 'mode='w'' overwrites the file
#        print("File " + filename + " was overwritten!") 
    else:
        df_save = df.loc[df['Sample_ID'] == sample_id, :]
        df_save.to_csv(filename, index=True, index_label='ID')  # Save normally if the file doesn't exist
#        print("File " + filename + " was created and saved !")

# All samples
filename = os.path.join(output_data_dir,   "all_Samples_" + project_name + ".csv")
# Save the DataFrame to a CSV file
df.to_csv(filename, index=True, index_label='ID')
#print("Merged file " + filename + " created!")

# ## Panel App
# Create widgets and panes
df_widget = pn.widgets.DataFrame(metadata, name="MetaData")
# Define the three tabs content
metadata_tab = pn.Column(pn.pane.Markdown("## Initial DataFrame"),intial_df)
dotplot_tab = pn.Column(plot_with_reconstruction)
celltype_classification_tab = pn.Column(cell_type_classification_app_main, threshold_panel)
cellsubtype_classification_tab = pn.Column(cell_subtype_classification_app_main, subtype_threshold_panel)
subtype_dotplot_tab = pn.Column(plot_with_subtype_reconstruction,)

app4_5 = pn.template.GoldenTemplate(
    site="Cyc-IF",
    title="Marker Threshold & Classification",
    main=[
        pn.Tabs(
            ("Metadata", metadata_tab),
            ("Classify-Celltype-Marker",celltype_classification_tab),
            ("Cell_Types", dotplot_tab),
            ("Classify-Cell Subtype-Marker",cellsubtype_classification_tab),
            ("Cell-Subtypes", subtype_dotplot_tab),
#           ("Heatmap",pn.Column(celltype_heatmap, cell_subtype_heatmap))
        )
    ]
)
app4_5.show()