bert_max_doc_str_max_str_pos_clean_doc_noise_doc/run_doc_qa.py
def read_squad_examples(input_file, is_training):
    """Read a SQuAD json file into a list of SquadExample."""
    with tf.gfile.Open(input_file, "r") as reader:
        input_data = json.load(reader)["data"]

    def is_whitespace(c):
        if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F:
            return True
        return False

    examples = []
    stop_processing = False
    for entry in input_data:
        for paragraph in entry["paragraphs"]:
            if FLAGS.debug and len(examples) >= 1000:
                tf.logging.info("[Debugging]: only keeps 1000 examples.")
                stop_processing = True
                break
            paragraph_text = paragraph["context"]
            doc_tokens = []
            char_to_word_offset = []
            prev_is_whitespace = True
            for c in paragraph_text:
                if is_whitespace(c):
                    prev_is_whitespace = True
                else:
                    if prev_is_whitespace:
                        doc_tokens.append(c)
                    else:
                        doc_tokens[-1] += c
                    prev_is_whitespace = False
                char_to_word_offset.append(len(doc_tokens) - 1)

            for qa in paragraph["qas"]:
                qas_id = qa["id"]
                qid = None
                if "qid" in qa:
                    qid = qa["qid"]
                else:
                    # TODO(chenghao): This is debug report for TriviaQA.
                    raise ValueError("No qid in qa")

                question_text = qa["question"]
                start_position = None
                end_position = None
                orig_answer_text = None
                is_impossible = False
                start_position_list = None
                end_position_list = None
                orig_answer_text_list = None
                if is_training:
                    if FLAGS.version_2_with_negative:
                        if not (type(qa["is_impossible"]) is bool):
                            raise ValueError("is_impossible is not bool")

                        is_impossible = qa["is_impossible"]

                    if not is_impossible:
                        answer = qa["answers"][0]
                        orig_answer_text = answer["text"]
                        answer_offset = int(answer["answer_start"])
                        answer_length = len(orig_answer_text)
                        start_position = char_to_word_offset[answer_offset]
                        end_position = char_to_word_offset[
                            answer_offset + answer_length - 1]

                        # For weak-supervision datasets, there might be multiple
                        # answers.
                        (start_position_list, end_position_list,
                         orig_answer_text_list) = get_position_lists(
                             qa["answers"], char_to_word_offset, doc_tokens
                         )

                        if not start_position_list:
                            continue

                        # Only add answers where the text can be exactly
                        # recovered from the document. If this CAN'T happen it's
                        # likely due to weird Unicode stuff so we will just skip
                        # the example.
                        #
                        # Note that this means for training mode, every example
                        # is NOT guaranteed to be preserved.
                        actual_text = " ".join(
                            doc_tokens[start_position:(end_position + 1)])
                        cleaned_answer_text = " ".join(
                            tokenization.whitespace_tokenize(orig_answer_text))
                        if actual_text.find(cleaned_answer_text) == -1:
                            tf.logging.warning(
                                "Could not find answer: '%s' vs. '%s'",
                                actual_text, cleaned_answer_text)
                            continue


                    else:
                        start_position = -1
                        end_position = -1
                        orig_answer_text = ""

                        start_position_list = [-1]
                        end_position_list = [-1]
                        orig_answer_text_list = [""]

                example = SquadExample(
                    qas_id=qas_id,
                    question_text=question_text,
                    doc_tokens=doc_tokens,
                    qid=qid,
                    orig_answer_text=orig_answer_text,
                    start_position=start_position,
                    end_position=end_position,
                    is_impossible=is_impossible,
                    orig_answer_text_list=orig_answer_text_list,
                    start_position_list=start_position_list,
                    end_position_list=end_position_list
                )
                examples.append(example)

        if stop_processing:
            break

    return examples
Read a SQuAD json file into a list of SquadExample.
Read a SQuAD json file into a list of SquadExample.
Read a SQuAD json file into a list of SquadExample .
def read_squad_examples(input_file, is_training):
    with tf.gfile.Open(input_file, "r") as reader:
        input_data = json.load(reader)["data"]
    def is_whitespace(c):
        if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F:
            return True
        return False
    examples = []
    stop_processing = False
    for entry in input_data:
        for paragraph in entry["paragraphs"]:
            if FLAGS.debug and len(examples) >= 1000:
                tf.logging.info("[Debugging]: only keeps 1000 examples.")
                stop_processing = True
                break
            paragraph_text = paragraph["context"]
            doc_tokens = []
            char_to_word_offset = []
            prev_is_whitespace = True
            for c in paragraph_text:
                if is_whitespace(c):
                    prev_is_whitespace = True
                else:
                    if prev_is_whitespace:
                        doc_tokens.append(c)
                    else:
                        doc_tokens[-1] += c
                    prev_is_whitespace = False
                char_to_word_offset.append(len(doc_tokens) - 1)
            for qa in paragraph["qas"]:
                qas_id = qa["id"]
                qid = None
                if "qid" in qa:
                    qid = qa["qid"]
                else:
                    raise ValueError("No qid in qa")
                question_text = qa["question"]
                start_position = None
                end_position = None
                orig_answer_text = None
                is_impossible = False
                start_position_list = None
                end_position_list = None
                orig_answer_text_list = None
                if is_training:
                    if FLAGS.version_2_with_negative:
                        if not (type(qa["is_impossible"]) is bool):
                            raise ValueError("is_impossible is not bool")
                        is_impossible = qa["is_impossible"]
                    if not is_impossible:
                        answer = qa["answers"][0]
                        orig_answer_text = answer["text"]
                        answer_offset = int(answer["answer_start"])
                        answer_length = len(orig_answer_text)
                        start_position = char_to_word_offset[answer_offset]
                        end_position = char_to_word_offset[
                            answer_offset + answer_length - 1]
                        (start_position_list, end_position_list,
                         orig_answer_text_list) = get_position_lists(
                             qa["answers"], char_to_word_offset, doc_tokens
                         )
                        if not start_position_list:
                            continue
                         Note that this means for training mode, every example
                         is NOT guaranteed to be preserved.
                        actual_text = " ".join(
                            doc_tokens[start_position:(end_position + 1)])
                        cleaned_answer_text = " ".join(
                            tokenization.whitespace_tokenize(orig_answer_text))
                        if actual_text.find(cleaned_answer_text) == -1:
                            tf.logging.warning(
                                "Could not find answer: '%s' vs. '%s'",
                                actual_text, cleaned_answer_text)
                            continue
                    else:
                        start_position = -1
                        end_position = -1
                        orig_answer_text = ""
                        start_position_list = [-1]
                        end_position_list = [-1]
                        orig_answer_text_list = [""]
                example = SquadExample(
                    qas_id=qas_id,
                    question_text=question_text,
                    doc_tokens=doc_tokens,
                    qid=qid,
                    orig_answer_text=orig_answer_text,
                    start_position=start_position,
                    end_position=end_position,
                    is_impossible=is_impossible,
                    orig_answer_text_list=orig_answer_text_list,
                    start_position_list=start_position_list,
                    end_position_list=end_position_list
                )
                examples.append(example)
        if stop_processing:
            break
    return examples
def read_squad_examples ( input_file , is_training ) : with tf . gfile . Open ( input_file , "r" ) as reader : input_data = json . load ( reader ) [ "data" ] def is_whitespace ( c ) : if c == " " or c == "\t" or c == "\r" or c == "\n" or ord ( c ) == 0x202F : return True return False examples = [ ] stop_processing = False for entry in input_data : for paragraph in entry [ "paragraphs" ] : if FLAGS . debug and len ( examples ) >= 1000 : tf . logging . info ( "[Debugging]: only keeps 1000 examples." ) stop_processing = True break paragraph_text = paragraph [ "context" ] doc_tokens = [ ] char_to_word_offset = [ ] prev_is_whitespace = True for c in paragraph_text : if is_whitespace ( c ) : prev_is_whitespace = True else : if prev_is_whitespace : doc_tokens . append ( c ) else : doc_tokens [ - 1 ] += c prev_is_whitespace = False char_to_word_offset . append ( len ( doc_tokens ) - 1 ) for qa in paragraph [ "qas" ] : qas_id = qa [ "id" ] qid = None if "qid" in qa : qid = qa [ "qid" ] else : raise ValueError ( "No qid in qa" ) question_text = qa [ "question" ] start_position = None end_position = None orig_answer_text = None is_impossible = False start_position_list = None end_position_list = None orig_answer_text_list = None if is_training : if FLAGS . version_2_with_negative : if not ( type ( qa [ "is_impossible" ] ) is bool ) : raise ValueError ( "is_impossible is not bool" ) is_impossible = qa [ "is_impossible" ] if not is_impossible : answer = qa [ "answers" ] [ 0 ] orig_answer_text = answer [ "text" ] answer_offset = int ( answer [ "answer_start" ] ) answer_length = len ( orig_answer_text ) start_position = char_to_word_offset [ answer_offset ] end_position = char_to_word_offset [ answer_offset + answer_length - 1 ] ( start_position_list , end_position_list , orig_answer_text_list ) = get_position_lists ( qa [ "answers" ] , char_to_word_offset , doc_tokens ) if not start_position_list : continue actual_text = " " . join ( doc_tokens [ start_position : ( end_position + 1 ) ] ) cleaned_answer_text = " " . join ( tokenization . whitespace_tokenize ( orig_answer_text ) ) if actual_text . find ( cleaned_answer_text ) == - 1 : tf . logging . warning ( "Could not find answer: '%s' vs. '%s'" , actual_text , cleaned_answer_text ) continue else : start_position = - 1 end_position = - 1 orig_answer_text = "" start_position_list = [ - 1 ] end_position_list = [ - 1 ] orig_answer_text_list = [ "" ] example = SquadExample ( qas_id = qas_id , question_text = question_text , doc_tokens = doc_tokens , qid = qid , orig_answer_text = orig_answer_text , start_position = start_position , end_position = end_position , is_impossible = is_impossible , orig_answer_text_list = orig_answer_text_list , start_position_list = start_position_list , end_position_list = end_position_list ) examples . append ( example ) if stop_processing : break return examples
Read a SQuAD json file into a list of SquadExample.
Read a SQuAD json file into a list of SquadExample .
"""Read a SQuAD json file into a list of SquadExample.""" # TODO(chenghao): This is debug report for TriviaQA. # For weak-supervision datasets, there might be multiple # answers. # Only add answers where the text can be exactly # recovered from the document. If this CAN'T happen it's # likely due to weird Unicode stuff so we will just skip # the example. # # Note that this means for training mode, every example # is NOT guaranteed to be preserved.
input_file
is_training