cluster_identities = pd.read_table('macosko2015/retina_clusteridentities.txt', header=None,
                                   names=['barcode', 'cluster_id'], index_col=0, squeeze=True)
print(cluster_identities.shape)
cluster_identities.head()

cluster_identities.groupby(cluster_identities).size()

"""
Explanation: Subset cells based on cluster identies
End of explanation
"""


cluster_sizes_table1 = table1.groupby(cluster_identities, axis=0).size()
cluster_sizes_table1

del cluster_markers_dropna

big_clusters = cluster_sizes_table1[cluster_sizes_table1 > 100]
big_clusters


big_clusters.index = big_clusters.index.astype(int)
big_clusters

cells_in_big_clusters = cluster_identities.isin(big_clusters.index)
cells_in_big_clusters = cells_in_big_clusters[cells_in_big_clusters]
cells_in_big_clusters.sum()

cells_in_big_clusters.head()

"""
Explanation: What is the cluster representation in this file?
End of explanation
"""


table1_big_clusters, y = table1.align(cells_in_big_clusters, axis=0, join='inner')
print(table1_big_clusters.shape)
print(y.shape)

"""
Explanation: Perform the subset!
End of explanation
"""


# np.random.seed(2017)

# n_cells = 200
# random_cells = np.random.choice(x.index, size=n_cells, replace=False)

# #  Perform the subset
# table1_subset_cells = x.loc[random_cells, :]
# print(table1_subset_cells.shape)
# table1_subset_cells.groupby(cluster_identities).size()

"""
Explanation: Grab random genes and make sure this has decent cluster representation
End of explanation
"""


np.random.seed(2017)

n_cells = 50
table1_subset_cells = table1_big_clusters.groupby(
    cluster_identities, as_index=False, group_keys=False).apply(
        lambda x: x.loc[np.random.choice(x.index, size=n_cells, replace=False)])
print(table1_subset_cells.shape)
table1_subset_cells.head()

"""
Explanation: Just kidding... this gets too lopsided so we're going to take 50 cells from each cluster
End of explanation
"""


cluster_markers = pd.read_excel('macosko2015/mmc4.xlsx', 
                                sheetname='FINAL_MARKERS_FOR_EACH_CLUSTER.',
                                skiprows=3)
print(cluster_markers.shape)
cluster_markers.head()

cluster_markers.dtypes

cluster_markers.tail()

cluster_markers = cluster_markers.dropna()
print(cluster_markers.shape)
cluster_markers.tail()

big_clusters.index.values.dtype

cluster_markers.groupby('cluster #').size()

"""
Explanation: Subset genes by differential expression
Use Supplementary table 4, which has the information of which genes were differentially expressed in different clusters
End of explanation
"""


cluster_markers = pd.read_excel('macosko2015/mmc4_v2.xlsx', skiprows=3)
cluster_markers = cluster_markers.rename(columns={"Unnamed: 0": 'gene_symbol'})

# Remove any rows with NA because those are all header rows
print(cluster_markers.shape)
cluster_markers = cluster_markers.dropna()
print(cluster_markers.shape)
cluster_markers['cluster #'] = cluster_markers['cluster #'].astype(int)
cluster_markers.head()

cluster_markers.dtypes

cluster_markers.groupby('cluster #').size()

table1.groupby()

"""
Explanation: Okay this file is corrupted..... have to go in and manually change the file so the cluster numbers are in the right column
Use manually created file
End of explanation
"""


len(cluster_markers['gene_symbol'].unique())

cleaned_symbols = cluster_markers['gene_symbol'].str.split().str[0]
len(cleaned_symbols.unique())

"""
Explanation: Still have to clean the gene symbols because the AUC ended up in some of them
There should be fewer unique genes so let's check how many we have now
End of explanation
"""


cluster_markers['gene_symbol'] = cleaned_symbols

"""
Explanation: Assign the symbols to the column
End of explanation
"""


cluster_markers.to_csv(os.path.join(data_folder, 'cluster_markers_v2.csv'))

"""
Explanation: Save to a csv
End of explanation
"""


rows = cluster_markers['cluster #'].isin(big_clusters.index.values)
print(rows.sum())
cluster_markers_big_clusters = cluster_markers.loc[rows]
print(cluster_markers_big_clusters.shape)
cluster_markers_big_clusters.head()

gene_subset = cluster_markers_big_clusters['gene_symbol'].unique()
len(gene_subset)

"""
Explanation: Use only genes found in the clusters with lots of cells ("big clusters")
End of explanation
"""


table1_subset_cells_genes = table1_subset_cells.loc[:, gene_subset]
print(table1_subset_cells_genes.shape)
table1_subset_cells_genes.head()

table1_subset_cells_genes.to_csv(
    os.path.join(data_folder, 'expression_table1_subset.csv'))

"""
Explanation: Perform the subset
End of explanation
"""


