run_ner.py
def file_based_convert_examples_to_features(examples, label_list,
                                            max_seq_length, tokenizer, output_file):
    """Convert a set of `InputExample`s to a TFRecord file."""

    writer = tf.python_io.TFRecordWriter(output_file)

    for (ex_index, example) in enumerate(examples):
        if ex_index % 10000 == 0:
            tf.logging.info("Writing example %d of %d" % (ex_index, len(examples)))

        feature = convert_single_example(ex_index, example, label_list,
                                         max_seq_length, tokenizer)

        def create_int_feature(values):
            f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values)))
            return f

        features = collections.OrderedDict()
        features["input_ids"] = create_int_feature(feature.input_ids)
        features["input_mask"] = create_int_feature(feature.input_mask)
        features["segment_ids"] = create_int_feature(feature.segment_ids)
        features["label_ids"] = create_int_feature(feature.label_ids)
        # features["lengths"] = create_int_feature([feature.length])

        tf_example = tf.train.Example(features=tf.train.Features(feature=features))
        writer.write(tf_example.SerializeToString())
    writer.close()
Convert a set of `InputExample`s to a TFRecord file.
Convert a set of `InputExample`s to a TFRecord file.
Convert a set of ` InputExample ` s to a TFRecord file .
def file_based_convert_examples_to_features(examples, label_list,
                                            max_seq_length, tokenizer, output_file):
    writer = tf.python_io.TFRecordWriter(output_file)
    for (ex_index, example) in enumerate(examples):
        if ex_index % 10000 == 0:
            tf.logging.info("Writing example %d of %d" % (ex_index, len(examples)))
        feature = convert_single_example(ex_index, example, label_list,
                                         max_seq_length, tokenizer)
        def create_int_feature(values):
            f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values)))
            return f
        features = collections.OrderedDict()
        features["input_ids"] = create_int_feature(feature.input_ids)
        features["input_mask"] = create_int_feature(feature.input_mask)
        features["segment_ids"] = create_int_feature(feature.segment_ids)
        features["label_ids"] = create_int_feature(feature.label_ids)
        tf_example = tf.train.Example(features=tf.train.Features(feature=features))
        writer.write(tf_example.SerializeToString())
    writer.close()
def file_based_convert_examples_to_features ( examples , label_list , max_seq_length , tokenizer , output_file ) : writer = tf . python_io . TFRecordWriter ( output_file ) for ( ex_index , example ) in enumerate ( examples ) : if ex_index % 10000 == 0 : tf . logging . info ( "Writing example %d of %d" % ( ex_index , len ( examples ) ) ) feature = convert_single_example ( ex_index , example , label_list , max_seq_length , tokenizer ) def create_int_feature ( values ) : f = tf . train . Feature ( int64_list = tf . train . Int64List ( value = list ( values ) ) ) return f features = collections . OrderedDict ( ) features [ "input_ids" ] = create_int_feature ( feature . input_ids ) features [ "input_mask" ] = create_int_feature ( feature . input_mask ) features [ "segment_ids" ] = create_int_feature ( feature . segment_ids ) features [ "label_ids" ] = create_int_feature ( feature . label_ids ) tf_example = tf . train . Example ( features = tf . train . Features ( feature = features ) ) writer . write ( tf_example . SerializeToString ( ) ) writer . close ( )
Convert a set of `InputExample`s to a TFRecord file.
Convert a set of ` InputExample ` s to a TFRecord file .
"""Convert a set of `InputExample`s to a TFRecord file.""" # features["lengths"] = create_int_feature([feature.length])
examples
label_list
max_seq_length
tokenizer
output_file