#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""
    Copyright (c) 2015, Elham Abbasian <e_abbasian@yahoo.com>, Kersten Doering <kersten.doering@gmail.com>

    This parser reads annotated sentences (output from get_relations.py) in a tab-separated format to generate a unified XML format (Tikk et al., 2010. A comprehensive benchmark of kernel methods to extract protein-protein interactions from literature. PLoS Comput. Biol).
"""

# module to make use of regular expressions
import re

# set the default encoding to utf8 and ignore all decoding/encoding steps.
# (ToDo: check whether the encoding command is needed - debug)
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

# optparse - Parser for command-line options
from optparse import OptionParser

# import this function to add quotation arround the input text and ignore the extra quotations inside the sentence text
#from xml.sax.saxutils import escape # (ToDo: not needed - debug)
from xml.sax.saxutils import quoteattr

### MAIN PART OF THE SCRIPT ###
if __name__=="__main__":
    # configure parsing of command-line arguments
    parser= OptionParser()
    parser.add_option("-i", "--input", dest="i", help='name of the input file',default="training_dataset_sorted.csv")
    parser.add_option("-o", "--output", dest="o", help='name of the output file',default="DS1.xml")
    (options,args)=parser.parse_args()
    # save parameters in an extra variable
    input_file= options.i
    output_file = options.o
    # open input file with annotated sentences
    infile = open(input_file,"r")
    # open output file
    outfile = open(output_file,"w")

    #example for the input format:
    #18227838-359	The mood stabilizers <compound-id="28486,3028194">lithium</compound-id> and <compound-id="3121">valproate</compound-id>  activate the <protein-id="P29323">ERK</protein-id> pathway in prefrontal cortex and hippocampus and potentiate <protein-id="P29323">ERK</protein-id> pathway-mediated neurite growth, neuronal survival and hippocampal neurogenesis.	lithium__ERK__no_interaction	valproate__ERK__interaction

    #example for the output format
    """
    <?xml version="1.0" encoding="UTF-8">
    <corpus source="DS1">
      <document id="DS1.d0" origId="18227838">
        <sentence id="DS1.d0.s0" origId="18227838-359" text="The mood stabilizers lithium and valproate activate the ERK pathway in prefrontal cortex and hippocampus and potentiate ERK pathway-mediated neurite growth, neuronal survival and hippocampal neurogenesis."/>
            <entity id="DS1.d0.s0.e0" origId="28486,3028194" charOffset="x1-y1" type="compound" text="lithium"/>
            <entity id="DS1.d0.s0.e1" origId="3121" charOffset="x2-y2" type="compound" text="valproate"/>
            <entity id="DS1.d0.s0.e2" origId="P29323" charOffset="x3-y3" type="protein" text="ERK"/>
            <interaction id="DS1.d0.s0.i0" e1="DS1.do.s0.e0" e2="DS1.do.s0.e2" type="no_interaction" directed="False" />
            <interaction id="DS1.d0.s0.i1" e1="DS1.do.s0.e1" e2="DS1.do.s0.e2" type="interaction" directed="False" />
        </sentence>
        [...]
      </document>
      [...]
    </corpus>
    """

    # add XML header and define corpus source
    outfile.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"+"\n")
    outfile.write("<corpus source=\"DS1\">"+"\n")

    # variable to store and compare the last read PubMed ID to notice whether there are multiple sentences with the same PubMed ID or not
    # the document ID refers to the PubMed ID (origID)
    pre_pmid=""

    # doc_num counts the number of created documents
    doc_num =0

    # read lines in CSV file
    for line in infile :
        # tab-separated format
        temp = line.strip().split("\t")
        # get PubMed ID, sentences ID, and the sentence itself
        # (ToDo: use a split command instead of this regular expression - debug)
        curr_pmid = re.match('(\d{8})',temp[0]).group(0)
        pmid_sent_num = temp[0]
        sentence_text = temp[1]
        # find all annotated proteins and compounds by matching their tags
        pro_positions=  [(a.start(), a.end()) for a in list(re.finditer('<protein-id="(.*?)">(.*?)</protein-id>',sentence_text))]
        cmp_positions =  [(a.start(), a.end()) for a in list(re.finditer('<compound-id="(.*?)">(.*?)</compound-id>',sentence_text))]
        # join the two lists
        positions = pro_positions + cmp_positions
        positions.sort()
        #Initialize the list with the number of identified tags
        entity_list =[]
        entity_list=[0]*len(positions)
        # iterate over all identified positions of the identified tags
        for i in range(len(positions)):
            # initialze the second dimension of the list with a length of four (entity_type,entity_id,entity_text,entity_charoffset)
            entity_list[i]=[0]*4
            # store these four elements with grouping in the regular expression
            obj = re.match('<(protein|compound)-id="(.*?)">(.*?)</(protein-id|compound-id)>',sentence_text[positions[i][0]:positions[i][1]])
            entity_list[i][0]=obj.group(1) #entity_type
            entity_list[i][1]=obj.group(2) #entity_id
            entity_list[i][2]=obj.group(3) #entity_text
            entity_list[i][2]=entity_list[i][2].replace("[","(").replace("]",")")
            # the entity_charoffset will be assign later after having the pure sentence text generated (without any tags)

        # the sentence without any tags will be generated by deleting all tags via text concatenation
        # initialize (ToDo: initialization like this not needed - debug)
        pur_sent_text = sentence_text
        # enumerate over the list of positions (index, value)
        for i,e in reversed(list(enumerate(positions))):
            pur_sent_text = pur_sent_text[0:positions[i][0]]+entity_list[i][2]+pur_sent_text[positions[i][1]:]

        # get the character offset of all identified synonyms
        # decode the sentences to UTF8 to prevent the usage of more than one character for special letters, symbols, etc.
        # make use of a list of repeated synonyms and synonym positions
        repeated_syn_pos =[]
        rep_syn =[]
        for i in range(len(entity_list)) :
            # check whether this is the fist occurrence of the current synonym
            if not entity_list[i][2] in rep_syn :
                # get the list of positions of all occurences of the current synonym
                u_pur_sent_text = pur_sent_text.decode("utf8")
                charoffset_value = [(a.start(), a.end()) for a in list(re.finditer(re.escape(entity_list[i][2]),u_pur_sent_text))]
                # check whether it occures only once such that the charoffsetone directly be assigned
                if len(charoffset_value) == 1 :
                    entity_list[i][3] = str(charoffset_value[0][0])+"-"+str(charoffset_value[0][1])
                else:
                    # if it occures more than one time, the charoffset has to be assigned according to the first pair of positions
                    entity_list[i][3] = str(charoffset_value[0][0])+"-"+str(charoffset_value[0][1])
                    # append this synonym to the rep_syn list to store all repeated synonyms in this sentence
                    rep_syn.append(entity_list[i][2])
                    # delete the fist pair of positions from the list
                    charoffset_value = charoffset_value[1:]
                    # add the rest of positions pairs for the current synonym to another list
                    for j in range(len(charoffset_value)):
                        repeated_syn_pos.append([entity_list[i][2],charoffset_value[j][0],charoffset_value[j][1]]) 
            else:
                # this case refers to at least the second occurrence of the synonym
                # for each repeated synonym, assign the first position pair from the repeated_syn_pos list
                for k in range(len(repeated_syn_pos)):
                    if repeated_syn_pos[k][0] == entity_list[i][2]:
                        break
                entity_list[i][3] = str(repeated_syn_pos[k][1])+"-"+str(repeated_syn_pos[k][2])

        # get pairs and their interaction status (separated by a double underscore)
        listof_int_noint = temp[2:]
        interaction_list=[0]*len(listof_int_noint)
        for i in range(len(listof_int_noint)):
            interaction_list[i]=listof_int_noint[i].split('__')
        # interaction/no_interaction corresponds to True/False
        TF_int_list=[0]*len(interaction_list)
        for intid in range(len(interaction_list)) :
            if interaction_list[intid][2]=="interaction" :
                TF_int_list[intid]="True"
            else :
                TF_int_list[intid]="False"
        # debug:
#        print TF_int_list

        # build XML structure
        # check whether the PubMed ID changed in comparision to the last parsed sentence
        if curr_pmid == pre_pmid :
            # if this is the case, only the sentence ID has to be increased
            sent_num +=1
            
            # add sentence ID using the current document number 
            # (doc_num has to be decreased by one, because this index is automatically increased after each sentence)
            # all openning and closing squared brackets ([,]) should be replaced with round brackets, because they will make problems in the tokenization step of the (preprocessing) pipeline
            pur_sent_text = pur_sent_text.replace("[","(").replace("]",")")

            outfile.write("        <sentence id=\"DS1.d"+str(doc_num-1)+".s"+str(sent_num)+"\" origId=\""+str(pmid_sent_num)+"\" text="+quoteattr(pur_sent_text)+">"+"\n")

            # build entity tags according to the list identified tags from the CSV file (entity_list)
            for i in range(0,len(entity_list)) :
                outfile.write("            <entity id=\"DS1.d"+str(doc_num-1)+".s"+str(sent_num)+".e"+str(i)+"\" origId=\""+entity_list[i][1]+"\" charOffset=\""+entity_list[i][3]+"\" type=\""+entity_list[i][0]+"\" text=\""+entity_list[i][2]+"\"/>"+"\n")

            # insert types of interaction for each pair of entities
            # get the index of the synonym interactions in entity_list
            origId = "DS1.d"+str(doc_num-1)+".s"+str(sent_num)
                
            for int_id in range(len(interaction_list)) :
                for ent_id in range(len(entity_list)):
                    if interaction_list[int_id][0] in entity_list[ent_id]:
                        break
                first_entity=ent_id
                for k in range(len(entity_list)):
                    if interaction_list[int_id][1] in entity_list[k]:
                        break
                second_entity=k
                outfile.write("            <pair e1=\""+origId+".e"+str(first_entity)+"\" e2=\""+origId+".e"+str(second_entity)+"\" id=\""+origId+".i"+str(int_id)+"\" interaction=\""+TF_int_list[int_id]+"\" />"+"\n")                
            # close sentence tag
            outfile.write("        </sentence>\n")
        # if the current PubMed ID changed in comparison to the last parsed sentences
        else :
            if not doc_num == 0 :
                outfile.write("    </document>\n")
            sent_num =0
            # a new document tag has to be opened and the sentences can be added
            outfile.write("    <document id=\"DS1.d"+str(doc_num)+"\" origId=\""+str(curr_pmid)+"\">"+"\n")
            # replace squared brackets ([,]) with round brackets
            pur_sent_text = pur_sent_text.replace("[","(").replace("]",")")
            outfile.write("        <sentence id=\"DS1.d"+str(doc_num)+".s"+str(sent_num)+"\" origId=\""+str(pmid_sent_num)+"\" text="+quoteattr(pur_sent_text)+">"+"\n")
            # now have to make entity tags according to entity_list data.
            for i in range(0,len(entity_list)) :
                outfile.write("            <entity id=\"DS1.d"+str(doc_num)+".s"+str(sent_num)+".e"+str(i)+"\" origId=\""+entity_list[i][1]+"\" charOffset=\""+entity_list[i][3]+"\" type=\""+entity_list[i][0]+"\" text=\""+entity_list[i][2]+"\"/>"+"\n")
            # build entity tags
            origId = "DS1.d"+str(doc_num)+".s"+str(sent_num)
            for int_id in range(len(interaction_list)) :
                for ent_id in range(len(entity_list)):
                    if interaction_list[int_id][0] in entity_list[ent_id]:
                        break
                first_entity=ent_id
                for k in range(len(entity_list)):
                    if interaction_list[int_id][1] in entity_list[k]:
                        break
                second_entity=k
                outfile.write("            <pair e1=\""+origId+".e"+str(first_entity)+"\" e2=\""+origId+".e"+str(second_entity)+"\" id=\""+origId+".i"+str(int_id)+"\" interaction=\""+TF_int_list[int_id]+"\" />"+"\n")
            # close sentence tag
            outfile.write("        </sentence>\n")
            # set new PubMed ID as the last parsed document ID and increase document index
            pre_pmid = curr_pmid
            doc_num+=1

    # close document tag
    outfile.write("</document>\n")
    # close corpus tag
    outfile.write("</corpus>\n")
    # close files
    infile.close()
    outfile.close()

[(17, 'sys.setdefaultencoding', 'sys.setdefaultencoding', False, 'import sys\n'), (29, 'optparse.OptionParser', 'OptionParser', False, 'from optparse import OptionParser\n'), (96, 're.match', 're.match', False, 'import re\n'), (79, 're.match', 're.match', False, 'import re\n'), (83, 're.finditer', 're.finditer', False, 'import re\n'), (84, 're.finditer', 're.finditer', False, 'import re\n'), (168, 'xml.sax.saxutils.quoteattr', 'quoteattr', False, 'from xml.sax.saxutils import quoteattr\n'), (199, 'xml.sax.saxutils.quoteattr', 'quoteattr', False, 'from xml.sax.saxutils import quoteattr\n'), (120, 're.escape', 're.escape', False, 'import re\n')]