#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""
    Copyright (c) 2015, Elham Abbasian <e_abbasian@yahoo.com>, Kersten Doering <kersten.doering@gmail.com>

    This parser reads annotated sentences (output from get_relations.py) in a tab-separated format to generate a unified XML format (Tikk et al., 2010. A comprehensive benchmark of kernel methods to extract protein-protein interactions from literature. PLoS Comput. Biol).
"""

# module to make use of regular expressions
import re

# set the default encoding to utf8 and ignore all decoding/encoding steps.
# (ToDo: check whether the encoding command is needed - debug)
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

# optparse - Parser for command-line options
from optparse import OptionParser

# import this function to add quotation arround the input text and ignore the extra quotations inside the sentence text
#from xml.sax.saxutils import escape # (ToDo: not needed - debug)
from xml.sax.saxutils import quoteattr

### MAIN PART OF THE SCRIPT ###
if __name__=="__main__":
    # configure parsing of command-line arguments
    parser= OptionParser()
    parser.add_option("-i", "--input", dest="i", help='name of the input file',default="training_dataset_sorted.csv")
    parser.add_option("-o", "--output", dest="o", help='name of the output file',default="DS1.xml")
    (options,args)=parser.parse_args()
    # save parameters in an extra variable
    input_file= options.i
    output_file = options.o
    # open input file with annotated sentences
    infile = open(input_file,"r")
    # open output file
    outfile = open(output_file,"w")

    #example for the input format:
    #18227838-359	The mood stabilizers <compound-id="28486,3028194">lithium</compound-id> and <compound-id="3121">valproate</compound-id>  activate the <protein-id="P29323">ERK</protein-id> pathway in prefrontal cortex and hippocampus and potentiate <protein-id="P29323">ERK</protein-id> pathway-mediated neurite growth, neuronal survival and hippocampal neurogenesis.	lithium__ERK__no_interaction	valproate__ERK__interaction

    #example for the output format
    """
    <?xml version="1.0" encoding="UTF-8">
    <corpus source="DS1">
      <document id="DS1.d0" origId="18227838">
        <sentence id="DS1.d0.s0" origId="18227838-359" text="The mood stabilizers lithium and valproate activate the ERK pathway in prefrontal cortex and hippocampus and potentiate ERK pathway-mediated neurite growth, neuronal survival and hippocampal neurogenesis."/>
            <entity id="DS1.d0.s0.e0" origId="28486,3028194" charOffset="x1-y1" type="compound" text="lithium"/>
            <entity id="DS1.d0.s0.e1" origId="3121" charOffset="x2-y2" type="compound" text="valproate"/>
            <entity id="DS1.d0.s0.e2" origId="P29323" charOffset="x3-y3" type="protein" text="ERK"/>
            <interaction id="DS1.d0.s0.i0" e1="DS1.do.s0.e0" e2="DS1.do.s0.e2" type="no_interaction" directed="False" />
            <interaction id="DS1.d0.s0.i1" e1="DS1.do.s0.e1" e2="DS1.do.s0.e2" type="interaction" directed="False" />
        </sentence>
        [...]
      </document>
      [...]
    </corpus>
    """

    # add XML header and define corpus source
    outfile.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"+"\n")
    outfile.write("<corpus source=\"DS1\">"+"\n")

    # variable to store and compare the last read PubMed ID to notice whether there are multiple sentences with the same PubMed ID or not
    # the document ID refers to the PubMed ID (origID)
    pre_pmid=""

    # doc_num counts the number of created documents
    doc_num =0

    # read lines in CSV file
    for line in infile :
        # tab-separated format
        temp = line.strip().split("\t")
        # get PubMed ID, sentences ID, and the sentence itself
        # (ToDo: use a split command instead of this regular expression - debug)
        curr_pmid = re.match('(\d{8})',temp[0]).group(0)
        pmid_sent_num = temp[0]
        sentence_text = temp[1]
        # find all annotated proteins and compounds by matching their tags
        pro_positions=  [(a.start(), a.end()) for a in list(re.finditer('<protein-id="(.*?)">(.*?)</protein-id>',sentence_text))]
        cmp_positions =  [(a.start(), a.end()) for a in list(re.finditer('<compound-id="(.*?)">(.*?)</compound-id>',sentence_text))]
        # join the two lists
        positions = pro_positions + cmp_positions
        positions.sort()
        #Initialize the list with the number of identified tags
        entity_list =[]
        entity_list=[0]*len(positions)
        # iterate over all identified positions of the identified tags
        for i in range(len(positions)):
            # initialze the second dimension of the list with a length of four (entity_type,entity_id,entity_text,entity_charoffset)
            entity_list[i]=[0]*4
            # store these four elements with grouping in the regular expression
            obj = re.match('<(protein|compound)-id="(.*?)">(.*?)</(protein-id|compound-id)>',sentence_text[positions[i][0]:positions[i][1]])
            entity_list[i][0]=obj.group(1) #entity_type
            entity_list[i][1]=obj.group(2) #entity_id
            entity_list[i][2]=obj.group(3) #entity_text
            entity_list[i][2]=entity_list[i][2].replace("[","(").replace("]",")")
            # the entity_charoffset will be assign later after having the pure sentence text generated (without any tags)

        # the sentence without any tags will be generated by deleting all tags via text concatenation
        # initialize (ToDo: initialization like this not needed - debug)
        pur_sent_text = sentence_text
        # enumerate over the list of positions (index, value)
        for i,e in reversed(list(enumerate(positions))):
            pur_sent_text = pur_sent_text[0:positions[i][0]]+entity_list[i][2]+pur_sent_text[positions[i][1]:]

        # get the character offset of all identified synonyms
        # decode the sentences to UTF8 to prevent the usage of more than one character for special letters, symbols, etc.
        # make use of a list of repeated synonyms and synonym positions
        repeated_syn_pos =[]
        rep_syn =[]
        for i in range(len(entity_list)) :
            # check whether this is the fist occurrence of the current synonym
            if not entity_list[i][2] in rep_syn :
                # get the list of positions of all occurences of the current synonym
                u_pur_sent_text = pur_sent_text.decode("utf8")
                charoffset_value = [(a.start(), a.end()) for a in list(re.finditer(re.escape(entity_list[i][2]),u_pur_sent_text))]
                # check whether it occures only once such that the charoffsetone directly be assigned
                if len(charoffset_value) == 1 :
                    entity_list[i][3] = str(charoffset_value[0][0])+"-"+str(charoffset_value[0][1])
                else:
                    # if it occures more than one time, the charoffset has to be assigned according to the first pair of positions
                    entity_list[i][3] = str(charoffset_value[0][0])+"-"+str(charoffset_value[0][1])
                    # append this synonym to the rep_syn list to store all repeated synonyms in this sentence
                    rep_syn.append(entity_list[i][2])
                    # delete the fist pair of positions from the list
                    charoffset_value = charoffset_value[1:]
                    # add the rest of positions pairs for the current synonym to another list
                    for j in range(len(charoffset_value)):
                        repeated_syn_pos.append([entity_list[i][2],charoffset_value[j][0],charoffset_value[j][1]]) 
            else:
                # this case refers to at least the second occurrence of the synonym
                # for each repeated synonym, assign the first position pair from the repeated_syn_pos list
                for k in range(len(repeated_syn_pos)):
                    if repeated_syn_pos[k][0] == entity_list[i][2]:
                        break
                entity_list[i][3] = str(repeated_syn_pos[k][1])+"-"+str(repeated_syn_pos[k][2])

        # get pairs and their interaction status (separated by a double underscore)
        listof_int_noint = temp[2:]
        interaction_list=[0]*len(listof_int_noint)
        for i in range(len(listof_int_noint)):
            interaction_list[i]=listof_int_noint[i].split('__')
        # interaction/no_interaction corresponds to True/False
        TF_int_list=[0]*len(interaction_list)
        for intid in range(len(interaction_list)) :
            if interaction_list[intid][2]=="interaction" :
                TF_int_list[intid]="True"
            else :
                TF_int_list[intid]="False"
        # debug:
#        print TF_int_list

        # build XML structure
        # check whether the PubMed ID changed in comparision to the last parsed sentence
        if curr_pmid == pre_pmid :
            # if this is the case, only the sentence ID has to be increased
            sent_num +=1
            
            # add sentence ID using the current document number 
            # (doc_num has to be decreased by one, because this index is automatically increased after each sentence)
            # all openning and closing squared brackets ([,]) should be replaced with round brackets, because they will make problems in the tokenization step of the (preprocessing) pipeline
            pur_sent_text = pur_sent_text.replace("[","(").replace("]",")")

            outfile.write("        <sentence id=\"DS1.d"+str(doc_num-1)+".s"+str(sent_num)+"\" origId=\""+str(pmid_sent_num)+"\" text="+quoteattr(pur_sent_text)+">"+"\n")

            # build entity tags according to the list identified tags from the CSV file (entity_list)
            for i in range(0,len(entity_list)) :
                outfile.write("            <entity id=\"DS1.d"+str(doc_num-1)+".s"+str(sent_num)+".e"+str(i)+"\" origId=\""+entity_list[i][1]+"\" charOffset=\""+entity_list[i][3]+"\" type=\""+entity_list[i][0]+"\" text=\""+entity_list[i][2]+"\"/>"+"\n")

            # insert types of interaction for each pair of entities
            # get the index of the synonym interactions in entity_list
            origId = "DS1.d"+str(doc_num-1)+".s"+str(sent_num)
                
            for int_id in range(len(interaction_list)) :
                for ent_id in range(len(entity_list)):
                    if interaction_list[int_id][0] in entity_list[ent_id]:
                        break
                first_entity=ent_id
                for k in range(len(entity_list)):
                    if interaction_list[int_id][1] in entity_list[k]:
                        break
                second_entity=k
                outfile.write("            <pair e1=\""+origId+".e"+str(first_entity)+"\" e2=\""+origId+".e"+str(second_entity)+"\" id=\""+origId+".i"+str(int_id)+"\" interaction=\""+TF_int_list[int_id]+"\" />"+"\n")                
            # close sentence tag
            outfile.write("        </sentence>\n")
        # if the current PubMed ID changed in comparison to the last parsed sentences
        else :
            if not doc_num == 0 :
                outfile.write("    </document>\n")
            sent_num =0
            # a new document tag has to be opened and the sentences can be added
            outfile.write("    <document id=\"DS1.d"+str(doc_num)+"\" origId=\""+str(curr_pmid)+"\">"+"\n")
            # replace squared brackets ([,]) with round brackets
            pur_sent_text = pur_sent_text.replace("[","(").replace("]",")")
            outfile.write("        <sentence id=\"DS1.d"+str(doc_num)+".s"+str(sent_num)+"\" origId=\""+str(pmid_sent_num)+"\" text="+quoteattr(pur_sent_text)+">"+"\n")
            # now have to make entity tags according to entity_list data.
            for i in range(0,len(entity_list)) :
                outfile.write("            <entity id=\"DS1.d"+str(doc_num)+".s"+str(sent_num)+".e"+str(i)+"\" origId=\""+entity_list[i][1]+"\" charOffset=\""+entity_list[i][3]+"\" type=\""+entity_list[i][0]+"\" text=\""+entity_list[i][2]+"\"/>"+"\n")
            # build entity tags
            origId = "DS1.d"+str(doc_num)+".s"+str(sent_num)
            for int_id in range(len(interaction_list)) :
                for ent_id in range(len(entity_list)):
                    if interaction_list[int_id][0] in entity_list[ent_id]:
                        break
                first_entity=ent_id
                for k in range(len(entity_list)):
                    if interaction_list[int_id][1] in entity_list[k]:
                        break
                second_entity=k
                outfile.write("            <pair e1=\""+origId+".e"+str(first_entity)+"\" e2=\""+origId+".e"+str(second_entity)+"\" id=\""+origId+".i"+str(int_id)+"\" interaction=\""+TF_int_list[int_id]+"\" />"+"\n")
            # close sentence tag
            outfile.write("        </sentence>\n")
            # set new PubMed ID as the last parsed document ID and increase document index
            pre_pmid = curr_pmid
            doc_num+=1

    # close document tag
    outfile.write("</document>\n")
    # close corpus tag
    outfile.write("</corpus>\n")
    # close files
    infile.close()
    outfile.close()

[((17, 0, 17, 31), 'sys.setdefaultencoding', 'sys.setdefaultencoding', ({(17, 23, 17, 30): '"""utf-8"""'}, {}), "('utf-8')", False, 'import sys\n'), ((29, 12, 29, 26), 'optparse.OptionParser', 'OptionParser', ({}, {}), '()', False, 'from optparse import OptionParser\n'), ((96, 18, 96, 140), 're.match', 're.match', ({(96, 27, 96, 92): '"""<(protein|compound)-id="(.*?)">(.*?)</(protein-id|compound-id)>"""', (96, 93, 96, 139): 'sentence_text[positions[i][0]:positions[i][1]]'}, {}), '(\'<(protein|compound)-id="(.*?)">(.*?)</(protein-id|compound-id)>\',\n    sentence_text[positions[i][0]:positions[i][1]])', False, 'import re\n'), ((79, 20, 79, 47), 're.match', 're.match', ({(79, 29, 79, 38): '"""(\\\\d{8})"""', (79, 39, 79, 46): 'temp[0]'}, {}), "('(\\\\d{8})', temp[0])", False, 'import re\n'), ((83, 60, 83, 127), 're.finditer', 're.finditer', ({(83, 72, 83, 112): '"""<protein-id="(.*?)">(.*?)</protein-id>"""', (83, 113, 83, 126): 'sentence_text'}, {}), '(\'<protein-id="(.*?)">(.*?)</protein-id>\', sentence_text)', False, 'import re\n'), ((84, 61, 84, 130), 're.finditer', 're.finditer', ({(84, 73, 84, 115): '"""<compound-id="(.*?)">(.*?)</compound-id>"""', (84, 116, 84, 129): 'sentence_text'}, {}), '(\'<compound-id="(.*?)">(.*?)</compound-id>\', sentence_text)', False, 'import re\n'), ((168, 136, 168, 160), 'xml.sax.saxutils.quoteattr', 'quoteattr', ({(168, 146, 168, 159): 'pur_sent_text'}, {}), '(pur_sent_text)', False, 'from xml.sax.saxutils import quoteattr\n'), ((199, 134, 199, 158), 'xml.sax.saxutils.quoteattr', 'quoteattr', ({(199, 144, 199, 157): 'pur_sent_text'}, {}), '(pur_sent_text)', False, 'from xml.sax.saxutils import quoteattr\n'), ((120, 83, 120, 111), 're.escape', 're.escape', ({(120, 93, 120, 110): 'entity_list[i][2]'}, {}), '(entity_list[i][2])', False, 'import re\n')]