content: 
  import numpy as np
  
  import unicodecsv
  import codecs
  import goslate
  import sqlite3
  
  from sklearn.feature_extraction.text import TfidfVectorizer
  from sklearn.metrics.pairwise import linear_kernel
  
  
  def imp_load(filename):
  
      texts = []
      books = []
      chapters = []
      verses = []
  
      # Read in a whole bible
      with codecs.open(filename,encoding='utf-8') as f:
          bibletext = f.read()
  
      # Split by verse
      bible_verses = bibletext.split('$$$')
  
      # Process verses
      for verse in bible_verses:
          try:
              verse = verse.split('\n',1)
              ref = verse[0].strip()
              text = verse[1].strip()
              ref = ref.split('.')
              book = ref[0].strip()
              cnum = ref[1].strip()
              vnum = ref[2].strip()
  
              texts.append(text)
              books.append(book)
              chapters.append(cnum)
              verses.append(vnum)
  
          except IndexError:
              pass
  
      return books, chapters, verses, texts
  
  
  def calculate_similarity(texts, translations):
  
      # Train the tf-idf thingy on the translated texts
      tfidf = TfidfVectorizer().fit_transform(texts)
  
      # Build a matrix representation of the similarities between verses
      # This will yield a simmetrical matrix
      # TODO: For performance and logical reasons: Only calculate similarity for nearby verses, assume others 0 ?
      M = np.array([linear_kernel(tfidf[j:j+1], tfidf).flatten() for j in range(len(texts))])
  
      # Hack(ish): Set similarity with verses of same translation to 0
      for i in range(len(M)):
          for j in range(i+1):
              if translations[i] == translations[j]:
                  M[i][j] = M[j][i] = 0
  
      # print np.round(M*100,0)
  
      return M
  
  
  def find_best_couple(M,t):
      """
      find best couple in similarity matrix M
      the translation(s) of each verse is given in t
      """
  
      # assume values are 0 for verses in same translation
      i_max, j_max = np.unravel_index(M.argmax(), M.shape)
      P_max = M[i_max, j_max]
  
      return i_max, j_max, P_max
  
  
  def merge_nodes(M,a,b):
      """
      merge indices a and b in similarity matrix M into one supernode,
      averaging similarity values between the supernode and other verses
      """
      
      N = len(M)
      
      # calculate a new row (and column) for the supernode
      supernode_similarity = [np.average([M[k][a],M[k][b]]) for k in range(N)]
      
      # append the row (this will jumble the verse order...)
      newM = np.append(M, np.array(supernode_similarity)[None,:], axis=0)
      
      # append 0 (supernode's similarity with itself) to the row and add it as a column
      supernode_similarity.append(0.)
      newM = np.append(newM, np.array(supernode_similarity)[:,None], axis=1)
      
      # to preserve verse indices, don't delete
      # newM = np.delete(newM,[a,b],axis=0)
      # rather make rows a and b 0
      # to preserve verse indices, don't delete
      # newM = np.delete(newM,[a,b],axis=1)
      # rather make columns a and b 0
      
      newM[:,a] = np.zeros_like(newM[:,a])
      newM[:,b] = np.zeros_like(newM[:,b])
      newM[a,:] = np.zeros_like(newM[a,:])
      newM[b,:] = np.zeros_like(newM[b,:])
      
      return newM
  
  
  def group_verses(M, t, numT, P_min = 0.1):
      """
      Automatically group verses
      t = the translation of each verse
      numT = max number of verses in a group = number of translations
      """
  
      t = [[val] for val in t]
      N = len(M)
      groups = {} # keyed by supernode index
      iteration = 0
      max_iteration = N
      
      while iteration < max_iteration:
          iteration += 1
          #print "\t\tGrouping: iteration ",iteration
  
          i,j,P = find_best_couple(M, t)
          #print "\t\tbest couple: ",i,j,P
  
          # Stop iterating if similarity gets too low...
          if P < P_min:
              break;
          
          group = []
          
          # merge supernodes if they exist, else merge nodes:
          
          if i in groups:
              group.extend(groups[i])
          else:
              group.append(i)
          
          if j in groups:
              group.extend(groups[j])
          else:
              group.append(j)
          
          # group now contains all of the verses for the new supernode
  
          if len(group) > numT:
              # this grouping is invalid
              # prevent it from happening again by making P 0
              M[i][j] = 0
          else:
              # valid grouping. save it.
  
              # Remove the previous supernode groups
              if i in groups:
                  del groups[i]
  
              if j in groups:
                  del groups[j]
  
              # Create the supernode
              M = merge_nodes(M,i,j)
              t.append(t[i] + t[j])
  
              # Save the index of the new supernode
              supernode_index = len(M)-1
              groups[supernode_index] = group
  
          print "\r\t\t",len(groups),
  
      print
  
      return groups
  
  
  def align(input_translations, input_filenames, output_filename):
      """
      Load one csv file for each translation
      Group, align and sort the verses
      Export a csv file containing a column for each translation
      """
  
      if len(input_translations) != len(input_filenames):
          raise ValueError("Number of translations and number of files must be the same")
  
      M = len(input_translations)
  
      # Load pre-translated data
      print "\tLoading data from files..."
      #translations,books,chapters,verses,texts_original,texts_en = load_translated_verses(input_translations, input_filenames)
      translations,chapters,verses,texts_original,texts_en = csv_import_translated_books(input_filenames, input_translations)
  
      # Calculate similarity between verses
      print "\tCalculating similarity matrix..."
      similarity = calculate_similarity(texts_en, translations)
  
      def canonical_group_cmp(a, b):
          """
          Define sort order for groups of verses
          """
  
          # find two verses from the same translation to compare their canonical order
          for i in a:
              for j in b:
                  if translations[i] == translations[j]:
                      return i - j
  
      # Group the verses
      print "\tGrouping verses..."
      groups = group_verses(similarity, translations, 3).values()
      # print groups
  
      # Put groups back into canonical order
      print "\tSorting verses..."
      groups.sort(canonical_group_cmp)
  
      # prepare data for csv export
      print "\tPreparing csv data..."
      csv_rows = []
      csv_rows.append(input_translations) # headers
  
      for group in groups:
  
          # create a row in the csv file for every group
          if len(group) == M:
              # rows where all translations are present, are quick:
              group.sort()
              row = [u"{0}:{1}:{2}".format(chapters[verse],verses[verse],texts_original[verse]) for verse in group]
          else:
              # for other rows, we have to find the missing translation, and substitute it with a blank
              row = []
              for translation in input_translations:
                  found = False
                  for verse in group:
                      if translation == translations[verse]:
                          # verse found for this translation
                          row.append(u"{0}:{1}:{2}".format(chapters[verse],verses[verse],texts_original[verse]))
                          found = True
                          break
                  if not found:
                      # fill in a blank
                      row.append("")
  
          csv_rows.append(row)
  
      # print csv_rows
  
      # Export to csv file
      print "\tWriting csv file..."
      with open(output_filename,'wb') as f:
          cw = unicodecsv.writer(f, encoding='utf-8')
          cw.writerows(csv_rows)
  
      print "\tDone!"
  
  
  def translate_csv(in_filename, language, out_filename):
      """
      Load a bible book from csv file
      translate it
      save it as a new file
      """
  
      # Create a translator object
      gs = goslate.Goslate(retry_times=100, timeout=100)
  
      # Load the bible book to be translated
      chapters,verses,texts_original = csv_import_book(in_filename)
  
      # Batch translate the verses if necessary
      if language != 'en':
          print "Batch translating {0} verses from '{1}' to 'en'".format(len(texts_original), language)
          texts_translated = gs.translate(texts_original, 'en', language)
      else:
          print "Not translating {0} verses already in 'en'".format(len(texts_original))
          texts_translated = texts_original
  
      # Write to CSV file
      rows = zip(chapters, verses, texts_original, texts_translated)
      with open(out_filename,'wb') as f:
          cw = unicodecsv.writer(f, encoding='utf-8')
          cw.writerow(['chapter','verse','text_original','text_english'])
          cw.writerows(rows)
  
  
  def csv_import_book(filename):
      """
      load bible book from csv file
      """
  
      texts = []
      chapters = []
      verses = []
  
      # Read in a whole file of verses
      with open(filename,'rb') as f:
          cr = unicodecsv.reader(f, encoding='utf-8')
          header = cr.next() # skip header
  
          # Process verses
          for cnum,vnum,text in cr:
              chapters.append(int(cnum))  # parse integer
              verses.append(int(vnum))    # parse integer
              texts.append(text.strip())  # remove surrounding whitespace
  
      # return results
      return chapters,verses,texts
  
  
  def csv_export_book(filename, rows=[], chapters=[], verses=[], texts=[]):
  
      if not len(rows) > 0:
          rows = zip(chapters, verses, texts)
  
      with open(filename,'wb') as f:
          cw = unicodecsv.writer(f,encoding='utf-8')
          cw.writerow(['chapter','verse','text'])
          cw.writerows(rows)
  
  
  def csv_import_translated_book(input_file):
      """
      import a single translated book from a single translation from single csv file
      """
  
      texts_en = []
      texts_original = []
      chapters = []
      verses = []
  
      # Read in a whole (Google translated) file of verses
      with open(input_file, 'rb') as f:
          cr = unicodecsv.reader(f, encoding='utf-8')
          header = cr.next() # skip header
  
          # Process verses
          for cnum,vnum,text_original,text_en in cr:
              chapters.append(int(cnum))
              verses.append(int(vnum))
              texts_original.append(text_original.strip())
              texts_en.append(text_en.strip())
  
      # return results
      return chapters,verses,texts_original,texts_en
  
  
  def csv_import_translated_books(input_files, input_translations):
      """
      import a single book from M translations from M csv files
      """
  
      if len(input_files) != len(input_translations):
          raise ValueError("Number of input files and translations are not the same")
  
      translations = []
      chapters = []
      verses = []
      texts_original = []
      texts_en = []
  
      for in_file,translation in zip(input_files,input_translations):
          c,v,o,e = csv_import_translated_book(in_file)
          chapters.extend(c)
          verses.extend(v)
          texts_original.extend(o)
          texts_en.extend(e)
          translations.extend([translation]*len(e))
  
      return translations,chapters,verses,texts_original,texts_en
  
  
  def csv_import_aligned_book(input_file):
      """
      Import a single aligned book (e.g. after it is checked by humans)
      """
  
      groups = []
  
      with open(input_file, 'rb') as f:
          cr = unicodecsv.reader(f, encoding='utf-8')
  
          translations = cr.next() # header contains translation names
  
          for row in cr:
              group = {}
              for i in range(len(translations)):
                  verse = row[i].split(':',3)
                  group[translations[i]] = {
                      'chapternum':int(verse[0]),
                      'versenum':int(verse[1]),
                      'text':verse[2].strip()
                  }
              groups.append(group)
  
      return groups