utils.py
def generate_scu_oie_multiSent(sentences, doc_summ='summ'):
    """ Given a scu sentence retrieve SCUs"""

    if doc_summ=='summ':
        KEY_sent = 'scuSentence'
        KEY_sent_char_idx = 'scuSentCharIdx'
        KEY_scu_text = 'scuText'
        KEY_scu_offset = 'scuOffsets'
    else:
        KEY_sent = 'docSentText'
        KEY_sent_char_idx = 'docSentCharIdx'
        KEY_scu_text = 'docScuText'
        KEY_scu_offset = 'docScuOffsets'

    _, oies = run_oie([sentence[KEY_sent] for sentence in sentences], cuda_device = 0)
    #adaptation for srl
    # oies = []
    # for sentence in sentences:
    #     oies.append(predictor.predict(sentence = sentence[KEY_sent] ))


    scu_list = []
    assert(len(sentences) == len(oies))
    for sentence ,oie in zip(sentences,oies):
        sentence[KEY_sent] = sentence[KEY_sent].replace(u'\u00a0', ' ')
        # ipdb.set_trace()
        if not oie:  # if list is empty
            continue

        # if  sentence[KEY_sent] =='Johnson\'s new TV show, ``The Magic Hour,\'\' is just one aspect of a busy life:  -- HIS HEALTH: While by no means cured, he owes the appearance of remarkable health to a Spartan lifestyle and modern medicine.':
        #     print('here')
        scus = oie['verbs']
        in_sentence_scu_dict = {}
        tokens = oie['words']
        for scu in scus:
            tags = scu['tags']
            words = []
            if not ("B-ARG1" in tags or "B-ARG2" in tags or "B-ARG0" in tags):
                continue
            sub_scu_offsets = []
            scu_start_offset = None
            offset = 0
            initialSpace = 0
            while sentence[KEY_sent][offset + initialSpace] == ' ':
                initialSpace += 1  ## add space if exists, so 'offset' would start from next token and not from space
            offset += initialSpace
            for ind, tag in enumerate(tags):
                # if "ARG0" in tag or "ARG1" in tag or "V" in tag:
                assert (sentence[KEY_sent][offset] == tokens[ind][0])
                if "O" not in tag:
                    if scu_start_offset is None:
                        scu_start_offset = sentence[KEY_sent_char_idx] + offset

                        assert(sentence[KEY_sent][offset] == tokens[ind][0])

                    words.append(tokens[ind])
                else:
                    if scu_start_offset is not None:
                        spaceBeforeToken = 0
                        while sentence[KEY_sent][offset-1-spaceBeforeToken] == ' ':
                            spaceBeforeToken += 1## add space if exists
                        if sentence[KEY_sent][offset] == '.' or sentence[KEY_sent][offset] == '?':
                            dotAfter = 1 + spaceAfterToken
                            dotTest = 1
                        else:
                            dotAfter = 0
                            dotTest = 0
                        scu_end_offset = sentence[KEY_sent_char_idx] + offset - spaceBeforeToken + dotAfter

                        if dotTest:
                            assert (sentence[KEY_sent][offset - spaceBeforeToken + dotAfter -1] == tokens[ind-1+ dotTest][0]) #check only the dot, the start of the token
                        else:
                            assert (sentence[KEY_sent][offset - spaceBeforeToken + dotAfter - 1] == tokens[ind - 1 + dotTest][-1])  #check end of token
                        sub_scu_offsets.append([scu_start_offset, scu_end_offset])
                        scu_start_offset = None


                ## update offset

                offset += len(tokens[ind])
                if ind < len(tags) - 1: #if not last token
                    spaceAfterToken = 0
                    while sentence[KEY_sent][offset + spaceAfterToken] == ' ':
                        spaceAfterToken += 1## add space after token if exists, so 'offset' would start from next token and not from space
                    offset += spaceAfterToken

            if scu_start_offset is not None: #end of sentence
                scu_end_offset = sentence[KEY_sent_char_idx] + offset
                sub_scu_offsets.append([scu_start_offset, scu_end_offset])
                scu_start_offset = None



            # if len(words) <= 3:
            #     continue
            scuText = "...".join([sentence[KEY_sent][strt_end_indx[0] - sentence[KEY_sent_char_idx]:strt_end_indx[1] - sentence[KEY_sent_char_idx]] for strt_end_indx in sub_scu_offsets])
            #assert(scuText==" ".join([sentence[KEY_sent][strt_end_indx[0]:strt_end_indx[1]] for strt_end_indx in sub_scu_offsets]))
            in_sentence_scu_dict[scuText] = sub_scu_offsets

        notContainedDict = checkContained(in_sentence_scu_dict, sentence[KEY_sent], sentence[KEY_sent_char_idx])


        for scuText, binaryNotContained in notContainedDict.items():
            scu_offsets = in_sentence_scu_dict[scuText]
            if binaryNotContained:
                tmp = copy.deepcopy(sentence)
                tmp[KEY_scu_text] = scuText
                tmp[KEY_scu_offset] = scu_offsets
                scu_list.append(tmp)
    # select the best SCU
    # sort SCUs based on their length and select middle one
    # scu_list = sorted(scu_list, key=lambda x: len(x[KEY_scu_text].split()), reverse=True)
    # print(f"Best SCU:::{scu_list[int(len(scu_list)/2)]}")
    # return scu_list[int(len(scu_list)/2)]
    return scu_list
 Given a scu sentence retrieve SCUs
Given a scu sentence retrieve SCUs
Given a scu sentence retrieve SCUs
def generate_scu_oie_multiSent(sentences, doc_summ='summ'):
    if doc_summ=='summ':
        KEY_sent = 'scuSentence'
        KEY_sent_char_idx = 'scuSentCharIdx'
        KEY_scu_text = 'scuText'
        KEY_scu_offset = 'scuOffsets'
    else:
        KEY_sent = 'docSentText'
        KEY_sent_char_idx = 'docSentCharIdx'
        KEY_scu_text = 'docScuText'
        KEY_scu_offset = 'docScuOffsets'
    _, oies = run_oie([sentence[KEY_sent] for sentence in sentences], cuda_device = 0)
    scu_list = []
    assert(len(sentences) == len(oies))
    for sentence ,oie in zip(sentences,oies):
        sentence[KEY_sent] = sentence[KEY_sent].replace(u'\u00a0', ' ')
        if not oie:  
            continue
        scus = oie['verbs']
        in_sentence_scu_dict = {}
        tokens = oie['words']
        for scu in scus:
            tags = scu['tags']
            words = []
            if not ("B-ARG1" in tags or "B-ARG2" in tags or "B-ARG0" in tags):
                continue
            sub_scu_offsets = []
            scu_start_offset = None
            offset = 0
            initialSpace = 0
            while sentence[KEY_sent][offset + initialSpace] == ' ':
                initialSpace += 1  
            offset += initialSpace
            for ind, tag in enumerate(tags):
                assert (sentence[KEY_sent][offset] == tokens[ind][0])
                if "O" not in tag:
                    if scu_start_offset is None:
                        scu_start_offset = sentence[KEY_sent_char_idx] + offset
                        assert(sentence[KEY_sent][offset] == tokens[ind][0])
                    words.append(tokens[ind])
                else:
                    if scu_start_offset is not None:
                        spaceBeforeToken = 0
                        while sentence[KEY_sent][offset-1-spaceBeforeToken] == ' ':
                            spaceBeforeToken += 1
                        if sentence[KEY_sent][offset] == '.' or sentence[KEY_sent][offset] == '?':
                            dotAfter = 1 + spaceAfterToken
                            dotTest = 1
                        else:
                            dotAfter = 0
                            dotTest = 0
                        scu_end_offset = sentence[KEY_sent_char_idx] + offset - spaceBeforeToken + dotAfter
                        if dotTest:
                            assert (sentence[KEY_sent][offset - spaceBeforeToken + dotAfter -1] == tokens[ind-1+ dotTest][0]) 
                        else:
                            assert (sentence[KEY_sent][offset - spaceBeforeToken + dotAfter - 1] == tokens[ind - 1 + dotTest][-1])  
                        sub_scu_offsets.append([scu_start_offset, scu_end_offset])
                        scu_start_offset = None
                offset += len(tokens[ind])
                if ind < len(tags) - 1: 
                    spaceAfterToken = 0
                    while sentence[KEY_sent][offset + spaceAfterToken] == ' ':
                        spaceAfterToken += 1
                    offset += spaceAfterToken
            if scu_start_offset is not None: 
                scu_end_offset = sentence[KEY_sent_char_idx] + offset
                sub_scu_offsets.append([scu_start_offset, scu_end_offset])
                scu_start_offset = None
            scuText = "...".join([sentence[KEY_sent][strt_end_indx[0] - sentence[KEY_sent_char_idx]:strt_end_indx[1] - sentence[KEY_sent_char_idx]] for strt_end_indx in sub_scu_offsets])
            in_sentence_scu_dict[scuText] = sub_scu_offsets
        notContainedDict = checkContained(in_sentence_scu_dict, sentence[KEY_sent], sentence[KEY_sent_char_idx])
        for scuText, binaryNotContained in notContainedDict.items():
            scu_offsets = in_sentence_scu_dict[scuText]
            if binaryNotContained:
                tmp = copy.deepcopy(sentence)
                tmp[KEY_scu_text] = scuText
                tmp[KEY_scu_offset] = scu_offsets
                scu_list.append(tmp)
    return scu_list
def generate_scu_oie_multiSent ( sentences , doc_summ = 'summ' ) : if doc_summ == 'summ' : KEY_sent = 'scuSentence' KEY_sent_char_idx = 'scuSentCharIdx' KEY_scu_text = 'scuText' KEY_scu_offset = 'scuOffsets' else : KEY_sent = 'docSentText' KEY_sent_char_idx = 'docSentCharIdx' KEY_scu_text = 'docScuText' KEY_scu_offset = 'docScuOffsets' _ , oies = run_oie ( [ sentence [ KEY_sent ] for sentence in sentences ] , cuda_device = 0 ) scu_list = [ ] assert ( len ( sentences ) == len ( oies ) ) for sentence , oie in zip ( sentences , oies ) : sentence [ KEY_sent ] = sentence [ KEY_sent ] . replace ( u'\u00a0' , ' ' ) if not oie : continue scus = oie [ 'verbs' ] in_sentence_scu_dict = { } tokens = oie [ 'words' ] for scu in scus : tags = scu [ 'tags' ] words = [ ] if not ( "B-ARG1" in tags or "B-ARG2" in tags or "B-ARG0" in tags ) : continue sub_scu_offsets = [ ] scu_start_offset = None offset = 0 initialSpace = 0 while sentence [ KEY_sent ] [ offset + initialSpace ] == ' ' : initialSpace += 1 offset += initialSpace for ind , tag in enumerate ( tags ) : assert ( sentence [ KEY_sent ] [ offset ] == tokens [ ind ] [ 0 ] ) if "O" not in tag : if scu_start_offset is None : scu_start_offset = sentence [ KEY_sent_char_idx ] + offset assert ( sentence [ KEY_sent ] [ offset ] == tokens [ ind ] [ 0 ] ) words . append ( tokens [ ind ] ) else : if scu_start_offset is not None : spaceBeforeToken = 0 while sentence [ KEY_sent ] [ offset - 1 - spaceBeforeToken ] == ' ' : spaceBeforeToken += 1 if sentence [ KEY_sent ] [ offset ] == '.' or sentence [ KEY_sent ] [ offset ] == '?' : dotAfter = 1 + spaceAfterToken dotTest = 1 else : dotAfter = 0 dotTest = 0 scu_end_offset = sentence [ KEY_sent_char_idx ] + offset - spaceBeforeToken + dotAfter if dotTest : assert ( sentence [ KEY_sent ] [ offset - spaceBeforeToken + dotAfter - 1 ] == tokens [ ind - 1 + dotTest ] [ 0 ] ) else : assert ( sentence [ KEY_sent ] [ offset - spaceBeforeToken + dotAfter - 1 ] == tokens [ ind - 1 + dotTest ] [ - 1 ] ) sub_scu_offsets . append ( [ scu_start_offset , scu_end_offset ] ) scu_start_offset = None offset += len ( tokens [ ind ] ) if ind < len ( tags ) - 1 : spaceAfterToken = 0 while sentence [ KEY_sent ] [ offset + spaceAfterToken ] == ' ' : spaceAfterToken += 1 offset += spaceAfterToken if scu_start_offset is not None : scu_end_offset = sentence [ KEY_sent_char_idx ] + offset sub_scu_offsets . append ( [ scu_start_offset , scu_end_offset ] ) scu_start_offset = None scuText = "..." . join ( [ sentence [ KEY_sent ] [ strt_end_indx [ 0 ] - sentence [ KEY_sent_char_idx ] : strt_end_indx [ 1 ] - sentence [ KEY_sent_char_idx ] ] for strt_end_indx in sub_scu_offsets ] ) in_sentence_scu_dict [ scuText ] = sub_scu_offsets notContainedDict = checkContained ( in_sentence_scu_dict , sentence [ KEY_sent ] , sentence [ KEY_sent_char_idx ] ) for scuText , binaryNotContained in notContainedDict . items ( ) : scu_offsets = in_sentence_scu_dict [ scuText ] if binaryNotContained : tmp = copy . deepcopy ( sentence ) tmp [ KEY_scu_text ] = scuText tmp [ KEY_scu_offset ] = scu_offsets scu_list . append ( tmp ) return scu_list
Given a scu sentence retrieve SCUs
Given a scu sentence retrieve SCUs
""" Given a scu sentence retrieve SCUs""" #adaptation for srl # oies = [] # for sentence in sentences: #     oies.append(predictor.predict(sentence = sentence[KEY_sent] )) # ipdb.set_trace() # if list is empty # if  sentence[KEY_sent] =='Johnson\'s new TV show, ``The Magic Hour,\'\' is just one aspect of a busy life:  -- HIS HEALTH: While by no means cured, he owes the appearance of remarkable health to a Spartan lifestyle and modern medicine.': #     print('here') ## add space if exists, so 'offset' would start from next token and not from space # if "ARG0" in tag or "ARG1" in tag or "V" in tag: ## add space if exists #check only the dot, the start of the token #check end of token ## update offset #if not last token ## add space after token if exists, so 'offset' would start from next token and not from space #end of sentence # if len(words) <= 3: #     continue #assert(scuText==" ".join([sentence[KEY_sent][strt_end_indx[0]:strt_end_indx[1]] for strt_end_indx in sub_scu_offsets])) # select the best SCU # sort SCUs based on their length and select middle one # scu_list = sorted(scu_list, key=lambda x: len(x[KEY_scu_text].split()), reverse=True) # print(f"Best SCU:::{scu_list[int(len(scu_list)/2)]}") # return scu_list[int(len(scu_list)/2)]
sentences
doc_summ