data.py
def load_data(story_path = "data/", question_filename="newsqa-data-v1", size=None):
    """Loads the NewsQA data with the respective [CNN story](https://cs.nyu.edu/~kcho/DMQA/)
    The function makes custom tokenization that creates character to word answer index offset 
    Input: NewsQA dataset file (csv)
    Output: None 
    """
    if path.exists(question_filename+".pkl"):
        df = pd.read_pickle(question_filename+".pkl")
    else:
        df = pd.read_csv(question_filename+".csv")
        if size != None:
            df = df.head(size)
        df = df.dropna(subset=['question'], axis=0)
        df.answer_char_ranges = df.answer_char_ranges.apply(lambda x: get_char_answer_range(x))
        df.validated_answers = df.validated_answers.apply(lambda x: dict(zip(json.loads(x).values(), json.loads(x).keys())) if str(x) != 'nan' else None)
        df['validater_one'] = df.validated_answers.apply(lambda x: x[1] if x != None and 1 in x.keys() else None)
        df['validater_two'] = df.validated_answers.apply(lambda x: x[2] if x != None and 2 in x.keys() else None)
        df['validater_one'] = df.validater_one.apply(lambda x : None if x == None else 'bad_question' if x == 'bad_question' else x.split(":"))
        df['validater_two'] = df.validater_two.apply(lambda x : None if x == None else 'bad_question' if x == 'bad_question' else x.split(":"))
        tokens = []
        ans_word_ranges = []
        validated_ans_word_ranges = []
        all_answers = []
        for i, story_idx in tqdm(enumerate(df.story_id)):
            story = get_story(story_idx, story_path)
            token, char_word_offset = get_doc_tokens(story)
            tokens.append(token)
            ans_word_ranges.append([[char_word_offset[x[0]] , char_word_offset[x[1]]] if x != None else None for x in  df.iloc[i].answer_char_ranges]) 
            v1, v2 = df.iloc[i].validater_one, df.iloc[i].validater_two
            validations = list()
            validations.append('bad_question' if v1 == 'bad_question' else [char_word_offset[int(v1[0])] , char_word_offset[int(v1[1])]] if  v1 != None and v1 != ['none'] else None)
            validations.append('bad_question' if v2 == 'bad_question' else [char_word_offset[int(v2[0])] , char_word_offset[int(v2[1])]] if  v2 != None and v2 != ['none'] else None)
            validated_ans_word_ranges.append(validations)
            answers = Union(validated_ans_word_ranges[-1], ans_word_ranges[-1])            
            all_answers.append(answers)
        df['ans_word_ranges'] = ans_word_ranges
        df['validated_word_answer'] = validated_ans_word_ranges
        df['all_answers'] = all_answers
        df['tokens'] = tokens
        df = df.drop(['answer_char_ranges','validated_answers','validater_one','validater_two'], axis=1)
        # df.to_pickle(question_filename+".pkl")
    return df
Loads the NewsQA data with the respective [CNN story](https://cs.nyu.edu/~kcho/DMQA/)
    The function makes custom tokenization that creates character to word answer index offset 
    Input: NewsQA dataset file (csv)
    Output: None 
    
Loads the NewsQA data with the respective [CNN story]
The function makes custom tokenization that creates character to word answer index offset
Input: NewsQA dataset file (csv)
Output: None
Loads the NewsQA data with the respective [ CNN story ] The function makes custom tokenization that creates character to word answer index offset Input : NewsQA dataset file ( csv ) Output : None
def load_data(story_path = "data/", question_filename="newsqa-data-v1", size=None):
    if path.exists(question_filename+".pkl"):
        df = pd.read_pickle(question_filename+".pkl")
    else:
        df = pd.read_csv(question_filename+".csv")
        if size != None:
            df = df.head(size)
        df = df.dropna(subset=['question'], axis=0)
        df.answer_char_ranges = df.answer_char_ranges.apply(lambda x: get_char_answer_range(x))
        df.validated_answers = df.validated_answers.apply(lambda x: dict(zip(json.loads(x).values(), json.loads(x).keys())) if str(x) != 'nan' else None)
        df['validater_one'] = df.validated_answers.apply(lambda x: x[1] if x != None and 1 in x.keys() else None)
        df['validater_two'] = df.validated_answers.apply(lambda x: x[2] if x != None and 2 in x.keys() else None)
        df['validater_one'] = df.validater_one.apply(lambda x : None if x == None else 'bad_question' if x == 'bad_question' else x.split(":"))
        df['validater_two'] = df.validater_two.apply(lambda x : None if x == None else 'bad_question' if x == 'bad_question' else x.split(":"))
        tokens = []
        ans_word_ranges = []
        validated_ans_word_ranges = []
        all_answers = []
        for i, story_idx in tqdm(enumerate(df.story_id)):
            story = get_story(story_idx, story_path)
            token, char_word_offset = get_doc_tokens(story)
            tokens.append(token)
            ans_word_ranges.append([[char_word_offset[x[0]] , char_word_offset[x[1]]] if x != None else None for x in  df.iloc[i].answer_char_ranges]) 
            v1, v2 = df.iloc[i].validater_one, df.iloc[i].validater_two
            validations = list()
            validations.append('bad_question' if v1 == 'bad_question' else [char_word_offset[int(v1[0])] , char_word_offset[int(v1[1])]] if  v1 != None and v1 != ['none'] else None)
            validations.append('bad_question' if v2 == 'bad_question' else [char_word_offset[int(v2[0])] , char_word_offset[int(v2[1])]] if  v2 != None and v2 != ['none'] else None)
            validated_ans_word_ranges.append(validations)
            answers = Union(validated_ans_word_ranges[-1], ans_word_ranges[-1])            
            all_answers.append(answers)
        df['ans_word_ranges'] = ans_word_ranges
        df['validated_word_answer'] = validated_ans_word_ranges
        df['all_answers'] = all_answers
        df['tokens'] = tokens
        df = df.drop(['answer_char_ranges','validated_answers','validater_one','validater_two'], axis=1)
    return df
def load_data ( story_path = "data/" , question_filename = "newsqa-data-v1" , size = None ) : if path . exists ( question_filename + ".pkl" ) : df = pd . read_pickle ( question_filename + ".pkl" ) else : df = pd . read_csv ( question_filename + ".csv" ) if size != None : df = df . head ( size ) df = df . dropna ( subset = [ 'question' ] , axis = 0 ) df . answer_char_ranges = df . answer_char_ranges . apply ( lambda x : get_char_answer_range ( x ) ) df . validated_answers = df . validated_answers . apply ( lambda x : dict ( zip ( json . loads ( x ) . values ( ) , json . loads ( x ) . keys ( ) ) ) if str ( x ) != 'nan' else None ) df [ 'validater_one' ] = df . validated_answers . apply ( lambda x : x [ 1 ] if x != None and 1 in x . keys ( ) else None ) df [ 'validater_two' ] = df . validated_answers . apply ( lambda x : x [ 2 ] if x != None and 2 in x . keys ( ) else None ) df [ 'validater_one' ] = df . validater_one . apply ( lambda x : None if x == None else 'bad_question' if x == 'bad_question' else x . split ( ":" ) ) df [ 'validater_two' ] = df . validater_two . apply ( lambda x : None if x == None else 'bad_question' if x == 'bad_question' else x . split ( ":" ) ) tokens = [ ] ans_word_ranges = [ ] validated_ans_word_ranges = [ ] all_answers = [ ] for i , story_idx in tqdm ( enumerate ( df . story_id ) ) : story = get_story ( story_idx , story_path ) token , char_word_offset = get_doc_tokens ( story ) tokens . append ( token ) ans_word_ranges . append ( [ [ char_word_offset [ x [ 0 ] ] , char_word_offset [ x [ 1 ] ] ] if x != None else None for x in df . iloc [ i ] . answer_char_ranges ] ) v1 , v2 = df . iloc [ i ] . validater_one , df . iloc [ i ] . validater_two validations = list ( ) validations . append ( 'bad_question' if v1 == 'bad_question' else [ char_word_offset [ int ( v1 [ 0 ] ) ] , char_word_offset [ int ( v1 [ 1 ] ) ] ] if v1 != None and v1 != [ 'none' ] else None ) validations . append ( 'bad_question' if v2 == 'bad_question' else [ char_word_offset [ int ( v2 [ 0 ] ) ] , char_word_offset [ int ( v2 [ 1 ] ) ] ] if v2 != None and v2 != [ 'none' ] else None ) validated_ans_word_ranges . append ( validations ) answers = Union ( validated_ans_word_ranges [ - 1 ] , ans_word_ranges [ - 1 ] ) all_answers . append ( answers ) df [ 'ans_word_ranges' ] = ans_word_ranges df [ 'validated_word_answer' ] = validated_ans_word_ranges df [ 'all_answers' ] = all_answers df [ 'tokens' ] = tokens df = df . drop ( [ 'answer_char_ranges' , 'validated_answers' , 'validater_one' , 'validater_two' ] , axis = 1 ) return df
Loads the NewsQA data with the respective [CNN story](https://cs.nyu.edu/~kcho/DMQA/)
The function makes custom tokenization that creates character to word answer index offset
Input: NewsQA dataset file (csv)
Output: None
Loads the NewsQA data with the respective [ CNN story ] ( https : // cs . nyu . edu / ~kcho / DMQA / ) The function makes custom tokenization that creates character to word answer index offset Input : NewsQA dataset file ( csv ) Output : None
"""Loads the NewsQA data with the respective [CNN story](https://cs.nyu.edu/~kcho/DMQA/)
    The function makes custom tokenization that creates character to word answer index offset 
    Input: NewsQA dataset file (csv)
    Output: None 
    """ # df.to_pickle(question_filename+".pkl")
story_path
question_filename
size