text: """
  geuvadis_lab_ethnicity_sex_revised.py
  
  Reads a CSV version of
  ftp://ftp.1000genomes.ebi.ac.uk/vol1/ftp/technical/working/
  20130606_sample_info/20130606_sample_info.xlsx and 
  http://www.ebi.ac.uk/arrayexpress/files/E-GEUV-3/E-GEUV-3.sdrf.txt
  to determine sample metadata for all of GEUVADIS manifest
  (GEUVADIS_all_samples_revised.manifest). Outputs GEUVADIS samples with
  descriptive names -- comment lines provide metadata. Some data are missing
  from http://www.ebi.ac.uk/arrayexpress/files/E-GEUV-1/E-GEUV-1.sdrf.txt
  
  Make the eval directory the current working directory before executing.
  
  Default values of command-line parameters were used for Rail simulation.
  
  This file is almost exactly the same as geuvadis_lab_ethnicity_sex_all.py;
  the only difference is it starts from GEUVADIS_all_samples_revised.manifest
  rather than GEUVADIS_all_samples.manifest to generate a descriptive manifest
  file. GEUVADIS_all_samples_revised.manifest has an extra GEUVADIS sample that
  was missing from GEUVADIS_all_samples.manifest, a manifest file we sample from
  to perform our scaling experiments.
  """
  import argparse
  import sys
  from collections import defaultdict
  
  # Print file's docstring if -h is invoked
  parser = argparse.ArgumentParser(description=__doc__, 
              formatter_class=argparse.RawDescriptionHelpFormatter)
  # Add command-line arguments
  parser.add_argument('--seed', type=int,
          default=0,
          help='Random seed to use'
      )
  parser.add_argument('--out', type=str,
          default='GEUVADIS_all_descriptive_revised',
          help='Output filename'
      )
  
  args = parser.parse_args()
  
  fastq_to_sample = {}
  sample_to_metadata = {}
  fastq_to_lab = {}
  
  with open('20130606_sample_info/Sample Info-Table 1.csv') \
      as sample_to_metadata_stream:
      line = sample_to_metadata_stream.readline()
      line = sample_to_metadata_stream.readline()
      while line:
          tokens = line.strip().split(',')
          if 'female' in tokens:
              sex = 'female'
          else:
              assert 'male' in tokens
              sex = 'male'
          sample_to_metadata[tokens[0]] = (sex, tokens[2]) #sex, hapmap pop
          line = sample_to_metadata_stream.readline()
  
  with open('E-GEUV-3.sdrf.txt') as geuvadis_data_stream:
      geuvadis_data_stream.readline() # labels
      for line in geuvadis_data_stream:
          tokens = line.strip().split('\t')
          fastq_to_sample[tokens[29].rpartition('/')[-1].partition('_')[0]] = \
              tokens[0]
          fastq_to_lab[tokens[29].rpartition('/')[-1].partition('_')[0]] = \
              (tokens[22], tokens[-1])
  
  manifest_line_to_metadata = {}
  
  with open('GEUVADIS_all_samples_revised.manifest') as geuvadis_stream:
      for line in geuvadis_stream:
          if line[0] == '#' or not line.strip(): # comment line
              continue
          tokens = line.strip().split('\t')
          fastq_name = tokens[0].rpartition('/')[-1].partition('_')[0]
          if fastq_name not in fastq_to_sample: continue
          manifest_line_to_metadata[line] = (fastq_to_sample[fastq_name],) + \
              sample_to_metadata[fastq_to_sample[fastq_name]] + \
              fastq_to_lab[fastq_name]
  
  from collections import defaultdict
  lab_to_manifest_lines = defaultdict(list)
  for line in manifest_line_to_metadata:
      lab_to_manifest_lines[manifest_line_to_metadata[line][-1]].append(line)
  
  def write_output(out):
      replicate_numbers = defaultdict(int)
      with open(out, 'w') as geuvadis_stream:
          print >>geuvadis_stream, '# All of GEUVADIS manifest file'
          print >>geuvadis_stream, \
              '# Generated by ' \
              'geuvadis_lab_ethnicity_sex_metadata_all_revised.py from: '
          print >>geuvadis_stream, ('# 1. ftp://ftp.1000genomes.ebi.ac.uk/'
                  'vol1/ftp/technical/working/'
                  '20130606_sample_info/20130606_sample_info.xlsx, which '
                  'provides metadata on 1000 Genomes samples')
          print >>geuvadis_stream, ('# 2. http://www.ebi.ac.uk/arrayexpress/'
                  'files/E-GEUV-3/E-GEUV-3.sdrf.txt, which provides metadata '
                  'about each GEUVADIS fastq'
              )
          print >>geuvadis_stream, ('# 3. GEUVADIS_all_samples.manifest, a '
                 'Myrna-style manifest file listing all GEUVADIS samples.\n')
          for line in manifest_line_to_metadata:
              tokens = line.strip().split('\t')
              replicate_numbers[manifest_line_to_metadata[line][:-2]] += 1
              tokens[-1] = '_'.join(manifest_line_to_metadata[line]) \
                  + '-1-%d' % (
                      replicate_numbers[manifest_line_to_metadata[line][:-2]]
                  )
              print >>geuvadis_stream, '\t'.join(tokens)
  
  write_output(args.out + '.manifest')