"""
# What is about ?

Dataset is downloaded from https://amp.pharm.mssm.edu/archs4/download.html
The methods are described in Nature Communications paper: https://www.nature.com/articles/s41467-018-03751-6

The ARCHS4 data provides user-friendly access to multiple gene expression data from the GEO database. (https://www.ncbi.nlm.nih.gov/geo/ ).  While in GEO database most of data is stored in raw formats,
ARCHS4 provides prepared count matrix expression data. While GEO contains data stored separately for each research paper,
ARCHS4 collects all the information in one single matrix. One may consult the main site for further information.  


Main data files are in H5 (HD5, Hierarchical Data Format ) file format https://en.wikipedia.org/wiki/Hierarchical_Data_Format
It contains expression data, as well as annotation data and futher meta-information. There are several other auxilliary files like TSNE 3d projection (in CSV format) and correlation matrices for genes for human and mouse in feather format. 

The notebook below gives examples how to work with such files. 

The ARCHS4 project is by :

'Alexander Lachmann', 'alexander.lachmann@mssm.edu', update: '2020-02-06'

"""
"""
## Some details on data

The file human_matrix.h5 - is version of 8 of the data file.
It contains expressions of 35238 genes for  238522 samples. Same file contains meta-information - gene names, sources of data etc 

Expression files for mouse and human in HDF5 format. All gene counts are on gene level (Entrez Gene Symbol). For compression purposes the Kallisto pseudocounts are rounded to integer values.


Small csv files like sample_human_tsne.csv may correspond to different version of the main file for example to version 2,
and thus contains data corresponding to a subset of data relatively to .h5. 

"""
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
import h5py
import matplotlib.pyplot as plt 
import seaborn as sns
import time

"""
# Open file way 1
"""
filename = '/kaggle/input/multiple-single-cell-rna-expressions-archs4/human_matrix.h5'
with h5py.File(filename, "r") as f:
    # List all groups
    print("Keys: %s" % f.keys())
    a_group_key = list(f.keys())[0]

    # Get the data
    # data = list(f[a_group_key])
"""
# Open file way 2 - more simple 
"""
f = h5py.File(filename,'r')#, mode)
for key in f.keys():
    print(key) #Names of the groups in HDF5 file.
"""
# Access expression data  
"""
"""
## Look for 
"""
key = 'data'
group = f[key]
print(group.keys())
for k in group.keys():
    print(group[k])
"""
## Access
"""
key = 'data'
key2 = 'expression'
X = f[key][key2]
print(X.shape)
X
X[:5,:8]
"""
# Exercise - show percent of zeros for several genes ( illustrate dropout)
"""
t0=time.time()
for gene in range((X.shape[0]))[:10]:
    v = X[:,gene]
    print('Gene N', gene, 'Percent of zeros:',np.round(100*np.sum(v==0)/len(v), 2),'%  ', 'seconds passed:', np.round(time.time()-t0,1))
    
    
"""
# Get meta data

"""
key = 'meta'
group = f[key]
l = []
for key in group.keys():
    l.append(key)
    #print(key)
print(len(l))    
print(l)    
l_gene = list(  filter(lambda x: 'gene' in x , l) )
print(len(l_gene))
print(l_gene)
"""
## Gene names etc,  35238 genes
"""
key = 'meta'
key2 = 'gene_name'
gene_names = f[key][key2]
print(len(gene_names))
gene_names[:10]
l_gene
key = 'meta'
for key2 in l_gene:
    dt = f[key][key2]
    print(key2, 'len = ', len(dt), 'First 5 elements:')
    print(dt[:5] )
"""
## Sample information
"""
l_sample = list(  filter(lambda x: 'Sample' in x , l) )
print(len(l_sample))
print(l_sample)
key = 'meta'
key2 = 'Sample_organism_ch1'
dt = f[key][key2]
s = pd.Series(dt)
print(s.value_counts())
dt[:10]
l_sample
key = 'meta'
for key2 in l_sample:
    dt = f[key][key2]
    print(key2, 'len = ', len(dt), 'First 5 elements:')
    print(dt[:5] )
"""
# Info on ARCHS4 authorship
"""
print( f['info'] )
print( f['info'].keys() )
for k in f['info']:
    print(f['info'][k][0])

"""
# Plot genes in TSNE 3d dimensional reduction 
"""
fn = '/kaggle/input/multiple-single-cell-rna-expressions-archs4/gene_human_tsne.csv'
df = pd.read_csv(fn)
df
fn = '/kaggle/input/multiple-single-cell-rna-expressions-archs4/gene_human_tsne.csv'
df = pd.read_csv(fn)

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure(figsize = (15,8))
ax = fig.add_subplot(111, projection='3d')
ax.scatter(df['x'], df['y'], df['z'])# , c=c, marker=m)

plt.title('Human genes in TSN3 3D space')
ax.set_xlabel('X Label')
ax.set_ylabel('Y Label')
ax.set_zlabel('Z Label')


plt.show()
fn = '/kaggle/input/multiple-single-cell-rna-expressions-archs4/sample_mouse_tsne.csv'
df = pd.read_csv(fn)

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure(figsize = (15,8))
ax = fig.add_subplot(111, projection='3d')
ax.scatter(df['x'], df['y'], df['z'])# , c=c, marker=m)

plt.title('Mouse genes in TSN3 3D space')
ax.set_xlabel('X Label')
ax.set_ylabel('Y Label')
ax.set_zlabel('Z Label')

plt.show()
"""
# Load and look at gene correlation matrices
"""
import pyarrow.feather as feather

fn = '/kaggle/input/multiple-single-cell-rna-expressions-archs4/mouse_correlation_archs4.f'
with open(fn, 'rb') as f:
    df = feather.read_feather(f)
    

df
v = df.values.ravel()

#corr_matr = df.values
#corr_matr_abs = np.abs(read_df.values )

plt.figure(figsize=(14,8))
t0 = time.time()
plt.hist(v, bins = 50)
plt.title('correlation coefficients distribution')
plt.show()
print(time.time() - t0, 'seconds passed')

print(np.min(v ), 'minimal correlation' )
print(np.mean(np.abs(v) ), 'average absolute correlation' )
print(np.median(np.abs(v)), 'median absolute correlation' )
print(np.min(np.abs(v) ), 'min absolute correlation' )
print(np.std(np.abs(v) ), 'std absolute correlation' )
for t in [0.5,0.6, 0.7,0.8,0.9,0.95,0.97,0.98,.99]:
    print( ((np.abs(v) < 0.99999999) & (np.abs(v) > t)).sum()/2 , 'number of pairs correlated more than', t  )
v.shape
# Strange extremelt highly correlated elements - will look on them below  in more details
t = 0.9999
print( ((np.abs(v) < 0.9999999999999) & (np.abs(v) > t)).sum()/2 , 'number of pairs correlated more than', t  )
t = 0.999
print( ((np.abs(v) < 0.9999999999999) & (np.abs(v) > t)).sum()/2 , 'number of pairs correlated more than', t  )

a = np.where( np.abs(np.triu(df.values,1)) > 0.9999 )
l = zip( a[0],a[1])
l = list(l)
print( len(l) ); print()
for t in l[:20]:
    print(t[0],t[1], df.columns[t[0]], df.columns[t[1]],'Correlation:',  df.iloc[t[0], t[1]] )

"""
## Graph of correlations
"""
import igraph

# Create graph
a = np.where( np.abs(np.triu(df.values,1)) > 0.5 )
l = list( zip( a[0],a[1]) )
print('Number of edges:', len(l) )

g = igraph.Graph()
g.add_vertices(len(df))
g.add_edges( list(l) )
# Analyse connected components:

g_clusters = g.clusters(mode='WEAK')
g_clusters
print('Number of components',len(g_clusters))

list_cluster_sizes = [len(t) for t  in g_clusters ]
l = np.sort(list_cluster_sizes)[::-1]
print('Sizes of 20 largest components:',  l[:20] )
print('Number of single node components', (np.array(l) == 1).sum()) 

# Visualize largest connected component
ix_max = np.argmax(list_cluster_sizes)
print(ix_max)
len(g_clusters[ix_max] )
g2 = g.subgraph( g_clusters[ix_max] )
visual_style = {}
visual_style["vertex_color"] = ['green' for v in g2.vs]
#visual_style["vertex_label"] = range(g.vcount()) 
visual_style["vertex_size"] = 2
igraph.plot(g2,bbox = (800,500), **visual_style )
