ls -1

"""
Explanation: So now we have the unzipped version of the file, "GSE41265_allGenesTPM.txt". I wonder how much space they saved by zipping it?
Let's use the flags "-l" for "long listing" which will show us the sizes
End of explanation
"""


! ls -1 | wc -l

! gunzip --help

"""
Explanation: oof, this is in pure bytes and I can't convert to multiples of 1024 easily in my head (1024 bytes = 1 kilobyte, 1024 kilobytes = 1 megabtye, etc -  the 1000/byte is a lie that the hard drive companies use!). So let's use the -h flag, which tells the computer to do th conversion for us. We can combine multiple flags with the same dash, so
ls -l -h

Can be shortened to:
ls -lh
End of explanation
"""


! gunzip -f *gz

3+3

asdf = 'beyonce'
asdf

asdf + ' runs the world'

"""
Explanation: See, "GSE41265_allGenesTPM.txt.gz" is there!
Since the file ends in ".gz", this tells us its a "gnu-zipped" or "gzipped" ("gee-zipped") file, which is a specific flavor of "zipping" or compressing a file. We need to use a gnu-zipping-aware program to decompress the file, which is "gunzip" ("gnu-unzip").
Run the next cell to unzip the file
End of explanation
"""


ls

! head GSM1657872_1772078217.C04.csv 

import glob

import pandas as pd

pd.read_table('GSM1657872_1772078217.C04.csv')

pd.read_table('GSM1657872_1772078217.C04.csv', index_col=0)

dataframe = pd.read_table('GSM1657872_1772078217.C04.csv', index_col=0, header=None)
dataframe

series = pd.read_table('GSM1657872_1772078217.C04.csv', index_col=0, header=None, squeeze=True)
series

dataframe.shape

series.shape

series.name

filename = 'GSM1657872_1772078217.C04.csv'
filename

filename.split('.')

filename.split('.csv')

filename.split('.csv')[0]

cells = []

for filename in glob.iglob('*.csv'):
    cell = pd.read_table(filename, index_col=0, squeeze=True, header=None)
    name = filename.split('_')[0]
    cell.name = name
    cells.append(cell)
expression = pd.concat(cells, axis=1)
expression.index = expression.index.map(lambda x: x.strip(' '))
print(expression.shape)
expression.head()

"""
Explanation: Let's "ls" again to see what files have changed
End of explanation
"""


! gunzip /Users/kirkreardon/Downloads/*_series_matrix.txt.gz

! head /Users/kirkreardon/Downloads/*_series_matrix.txt

! head -n 20 /Users/kirkreardon/Downloads/*_series_matrix.txt

"Whooo!!!!!!!!!".strip("!")

"Whooo!!!!!!!!!".strip("o")

metadata1 = pd.read_table('/Users/kirkreardon/Downloads/GSE67835-GPL15520_series_matrix.txt', 
                          skiprows=37, header=None, index_col=0)
metadata1.index = metadata1.index.map(lambda x: x.strip('!'))
# Transpose so each row is a cell
metadata1 = metadata1.T
metadata1.head()

metadata2 = pd.read_table('/Users/kirkreardon/Downloads/GSE67835-GPL18573_series_matrix.txt', 
                          skiprows=37, header=None, index_col=0)
metadata2.index = metadata2.index.map(lambda x: x.strip('!'))
# transpose
metadata2 = metadata2.T
metadata2.head()

dataframes = [metadata1, metadata2]
metadata = pd.concat(dataframes)
print(metadata.shape)
metadata.head()

metadata = metadata.set_index('Sample_geo_accession')
metadata.head()

mkdir -p ~/projects/darmanis2015/processed_data

expression.to_csv('~/projects/darmanis2015/processed_data/expression.csv')

metadata.to_csv('~/projects/darmanis2015/processed_data/metadata.csv')

expression.GSM1657884

bad_rows = ['no_feature', 'ambiguous', 'alignment_not_unique']
good_genes = expression.index[~expression.index.isin(bad_rows)]
good_genes

expression.shape

expression_actually_genes = expression.loc[good_genes]
expression_actually_genes.shape

expression_actually_genes.tail()

expression_actually_genes.to_csv("/Users/kirkreardon/projects/darmanis2015/processed_data/expression_actually_genes.csv")

expression_actually_genes.dtypes

expression_actually_genes.tail().index

"""
Explanation: Read metadata
End of explanation
"""


