network = pd.read_csv('/dacb/meta4_bins/analysis/network/network.txt', 
                      sep='\t')

network.head()

network['target_organism'] = network['target'].str.extract('([A-z0-9]+)_[0-9]+')
network['target_gene'] = network['target'].str.extract('[A-z0-9]+_([0-9]+)')
network['source_organism'] = network['source'].str.extract('([A-z0-9]+)_[0-9]+')
network['source_gene'] = network['source'].str.extract('[A-z0-9]+_([0-9]+)')

network.head()

network = network.rename(columns=lambda x: re.sub('source$', 'source_locus_tag', x))
network = network.rename(columns=lambda x: re.sub('target$', 'target_locus_tag', x))

network.head(2)

network['target_organism'].unique()

network['cross_species'] = network['source_organism'] != network['target_organism']

network.cross_species.describe()

network.cross_species.plot.hist()

network.weight.plot.hist()

locus_to_organism = pd.read_csv('/dacb/meta4_bins/data/genome_bins.locus_to_organism.tsv', sep='\t',
                               names=['locus', 'organism'])

locus_to_organism.head()

# Found a problem: 
# Expected exactly 2 organsm names, but we have 3
#   {'Methylobacter-123 (UID203) ', 'Methylobacter-123 (UID203)', 'Methylotenera mobilis-49 (UID203)'}
# http://pandas.pydata.org/pandas-docs/stable/generated/pandas.Series.str.strip.html
#   strips both left and right whitespace  :) 
locus_to_organism['organism'] = locus_to_organism['organism'].str.strip()

locus_to_organism['organism ID'] = locus_to_organism['locus'].str.extract('([A-z]+[0-9]+)_[0-9]+')

source_organism_names = locus_to_organism[['organism ID', 'organism']].drop_duplicates()
target_organism_names = locus_to_organism[['organism ID', 'organism']].drop_duplicates()

source_organism_names = source_organism_names.rename(columns={'organism ID':'source_organism', 'organism':'source_organism_name'})
target_organism_names = target_organism_names.rename(columns={'organism ID':'target_organism', 'organism':'target_organism_name'})

source_organism_names

merged = pd.merge(network, source_organism_names)

merged.source_organism_name.unique()

merged.head(2)

merged = pd.merge(merged, target_organism_names)
print(merged.shape)
print(network.shape)

merged.head()

merged.source_organism_name.unique()

print(merged.shape)
print(network.shape)

merged.tail()

# # d[(d['x']>2) & (d['y']>7)]
merged[(merged['source_organism_name'] == 'Methylotenera mobilis-49 (UID203)') &
       (merged['target_organism_name'] == 'Methylobacter-123 (UID203)')].head(2)

merged[(merged['source_organism_name'] == 'Methylobacter-123 (UID203)') &
       (merged['target_organism_name'] == 'Methylotenera mobilis-49 (UID203)')].head(2)

"""
Explanation: The network directory in this share (which is still uploading, btw) contains a pickle (data.pkl) and the code used to generate it (network.py). The lfdr_pcor object in the pickle has the partial correlations after pruning, but poor has all of them (the full network). network.txt has a text version of the network (after pruning) that can be sucked into cytoscape.
The network data was calculated from mapping to genome bins:
End of explanation
"""


genes = pd.read_csv('/dacb/meta4_bins/analysis/assemble_summaries/summary_counts.xls', sep = '\t', usecols=[1, 2])

genes.tail(3)

genes.tail()

genes[genes['locus_tag'] == 'Ga0081607_11219']

merged.head(2)

source_genes = genes[['locus_tag', 'product']].rename(columns={'locus_tag':'source_locus_tag', 'product':'source_gene_product'})
target_genes = genes[['locus_tag', 'product']].rename(columns={'locus_tag':'target_locus_tag', 'product':'target_gene_product'})

source_genes.head(2)

network.shape

merged.shape

merged = pd.merge(merged, source_genes)

merged.shape

merged = pd.merge(merged, target_genes)

merged.shape

merged.head(2)

merged.head(3)

merged['sort'] = merged.weight.abs()
merged = merged.sort(columns='sort', ascending=False).drop('sort', axis=1)

merged['weight'].describe()

merged.head(2)

merged['source_gene_product'].unique()[0:4]

filename = 'Methylobacter-123--Methylotenera_mobilis-49_network'

! ls ../data

! mkdir ../data/Methylobacter--Methylotenera

dirname = '../data/Methylobacter--Methylotenera/' 

merged.to_csv(dirname + filename + '.tsv', sep='\t', index=False)

# The CSV isn't a good idea because of the gene names. 
#merged.to_csv(dirname + filename + '.csv')

merged.head(100).to_csv(dirname + filename + '--100' + '.tsv', sep='\t', index=False)

merged.shape

top_genes = list(merged['source_gene_product'].unique()[0:3]) + list(merged['target_gene_product'].unique()[0:3])

has_top_genes = merged[(merged['source_gene_product'].isin(top_genes)) | (merged['target_gene_product'].isin(top_genes))]

has_top_genes.shape

has_top_genes.to_csv(dirname + filename + '--some_top_genes--1268_rows' + '.tsv', sep='\t', index=False)

"""
Explanation: Use summary_counts, not summary_rpkm for gene names.
jmatsen@waffle:/dacb/meta4_bins/analysis/assemble_summaries$ ag Ga0081607_11219 summary_rpkm.xls | head -n 10
jmatsen@waffle:/dacb/meta4_bins/analysis/assemble_summaries$ ag Ga0081607_11219 summary_counts.xls | head -n 10
2015:Methylobacter-123 (UID203)         Ga0081607_11219 hypothetical protein    243652  6660    160     285587  448     89      94      4893    13
        66994   47733   163     301     3       146     1851    26      53125   249288  21      14249   28      12      42296   23538   2778    1918      2061    217     173983  164307  398     450     1170    10410   30      344     2224    2164    1452    810     338     656     70      222       3475    1143    2672    1313    1246    930     54      23      9942    9603    2381    8196    29      49      23721   7808    33195   17291     5825    6609    36      83      40661   28629   17949   12227   15478   15054   125     1010    10214   66875   40225   944     11993   9572      56      9375
End of explanation
"""


merged.head()

network.head()

def extract_nodes(df, source=True):
    if source:
        string = 'source'
    else:
        string = 'target'
    cols = df.columns.str.contains(string)
    df_subset = df.iloc[:, cols]
    df_subset = df_subset.rename(columns=lambda x: x.replace(string + "_", ''))
    df_subset = df_subset.rename(columns=lambda x: x.replace(string, 'locus_tag'))
    return df_subset
    

extract_nodes(merged, True).head(3)

extract_nodes(merged, False).head(3)

nodes = pd.concat([extract_nodes(merged, True).drop_duplicates(), 
                   extract_nodes(merged, False).drop_duplicates()], axis = 0)

nodes.shape

nodes.drop_duplicates(inplace=True)

nodes.shape

nodes.to_csv('../data/Methylobacter--Methylotenera/nodes.tsv', sep='\t', index=False)

"""
Explanation: Make a file with just nodes.
Start with the merged stuff since I have to take the source and target nodes separately no matter which DF I start with.
End of explanation
"""


