pointer-generator/my_make_datasets.py
def write_to_bin(out_file, fulltext_docs, summary_docs, makevocab=False):
    """
    Reads the tokenized .text files corresponding to the urls listed in the
    url_file and writes them to a out_file.
    """
    num_docs = len(fulltext_docs)

    if makevocab:
        vocab_counter = collections.Counter()

    with open(out_file, 'wb') as writer:
        for idx, (f_doc, s_doc) in enumerate(zip(fulltext_docs, summary_docs)):
            if idx % 1000 == 0:
                print("Writing story %i of %i; %.2f percent done" %
                      (idx, num_docs, float(idx) * 100.0 / float(num_docs)))

            # Get the strings to write to .bin file
            article, abstract = get_fulltext_summary(f_doc, s_doc)

            # Write to tf.Example
            tf_example = example_pb2.Example()
            tf_example.features.feature['article'].bytes_list.value.extend(
                [article.encode()])
            tf_example.features.feature['abstract'].bytes_list.value.extend(
                [abstract.encode()])
            tf_example_str = tf_example.SerializeToString()
            str_len = len(tf_example_str)
            writer.write(struct.pack('q', str_len))
            writer.write(struct.pack('%ds' % str_len, tf_example_str))

            # Write the vocab to file, if applicable
            if makevocab:
                art_tokens = article.split(' ')
                abs_tokens = abstract.split(' ')
                abs_tokens = [
                    t for t in abs_tokens
                    if t not in [SENTENCE_START, SENTENCE_END]
                ]  # remove these tags from vocab
                tokens = art_tokens + abs_tokens
                tokens = [t.strip() for t in tokens]  # strip
                tokens = [t for t in tokens if t != ""]  # remove empty
                vocab_counter.update(tokens)

    print("Finished writing file %s\n" % out_file)

    # write vocab to file
    if makevocab:
        print("Writing vocab file...")
        with open(os.path.join(FINISHED_FILES_DIR, "vocab"), 'w+', encoding="utf-8") as writer:
            for word, count in vocab_counter.most_common(VOCAB_SIZE):
                writer.write(word + ' ' + str(count) + '\n')
        print("Finished writing vocab file")

    Reads the tokenized .text files corresponding to the urls listed in the
    url_file and writes them to a out_file.
    
Reads the tokenized .text files corresponding to the urls listed in the
url_file and writes them to a out_file.
Reads the tokenized . text files corresponding to the urls listed in the url_file and writes them to a out_file .
def write_to_bin(out_file, fulltext_docs, summary_docs, makevocab=False):
    num_docs = len(fulltext_docs)
    if makevocab:
        vocab_counter = collections.Counter()
    with open(out_file, 'wb') as writer:
        for idx, (f_doc, s_doc) in enumerate(zip(fulltext_docs, summary_docs)):
            if idx % 1000 == 0:
                print("Writing story %i of %i; %.2f percent done" %
                      (idx, num_docs, float(idx) * 100.0 / float(num_docs)))
            article, abstract = get_fulltext_summary(f_doc, s_doc)
            tf_example = example_pb2.Example()
            tf_example.features.feature['article'].bytes_list.value.extend(
                [article.encode()])
            tf_example.features.feature['abstract'].bytes_list.value.extend(
                [abstract.encode()])
            tf_example_str = tf_example.SerializeToString()
            str_len = len(tf_example_str)
            writer.write(struct.pack('q', str_len))
            writer.write(struct.pack('%ds' % str_len, tf_example_str))
            if makevocab:
                art_tokens = article.split(' ')
                abs_tokens = abstract.split(' ')
                abs_tokens = [
                    t for t in abs_tokens
                    if t not in [SENTENCE_START, SENTENCE_END]
                ]  
                tokens = art_tokens + abs_tokens
                tokens = [t.strip() for t in tokens]  
                tokens = [t for t in tokens if t != ""]  
                vocab_counter.update(tokens)
    print("Finished writing file %s\n" % out_file)
    if makevocab:
        print("Writing vocab file...")
        with open(os.path.join(FINISHED_FILES_DIR, "vocab"), 'w+', encoding="utf-8") as writer:
            for word, count in vocab_counter.most_common(VOCAB_SIZE):
                writer.write(word + ' ' + str(count) + '\n')
        print("Finished writing vocab file")
def write_to_bin ( out_file , fulltext_docs , summary_docs , makevocab = False ) : num_docs = len ( fulltext_docs ) if makevocab : vocab_counter = collections . Counter ( ) with open ( out_file , 'wb' ) as writer : for idx , ( f_doc , s_doc ) in enumerate ( zip ( fulltext_docs , summary_docs ) ) : if idx % 1000 == 0 : print ( "Writing story %i of %i; %.2f percent done" % ( idx , num_docs , float ( idx ) * 100.0 / float ( num_docs ) ) ) article , abstract = get_fulltext_summary ( f_doc , s_doc ) tf_example = example_pb2 . Example ( ) tf_example . features . feature [ 'article' ] . bytes_list . value . extend ( [ article . encode ( ) ] ) tf_example . features . feature [ 'abstract' ] . bytes_list . value . extend ( [ abstract . encode ( ) ] ) tf_example_str = tf_example . SerializeToString ( ) str_len = len ( tf_example_str ) writer . write ( struct . pack ( 'q' , str_len ) ) writer . write ( struct . pack ( '%ds' % str_len , tf_example_str ) ) if makevocab : art_tokens = article . split ( ' ' ) abs_tokens = abstract . split ( ' ' ) abs_tokens = [ t for t in abs_tokens if t not in [ SENTENCE_START , SENTENCE_END ] ] tokens = art_tokens + abs_tokens tokens = [ t . strip ( ) for t in tokens ] tokens = [ t for t in tokens if t != "" ] vocab_counter . update ( tokens ) print ( "Finished writing file %s\n" % out_file ) if makevocab : print ( "Writing vocab file..." ) with open ( os . path . join ( FINISHED_FILES_DIR , "vocab" ) , 'w+' , encoding = "utf-8" ) as writer : for word , count in vocab_counter . most_common ( VOCAB_SIZE ) : writer . write ( word + ' ' + str ( count ) + '\n' ) print ( "Finished writing vocab file" )
Reads the tokenized .text files corresponding to the urls listed in the
url_file and writes them to a out_file.
Reads the tokenized . text files corresponding to the urls listed in the url_file and writes them to a out_file .
"""
    Reads the tokenized .text files corresponding to the urls listed in the
    url_file and writes them to a out_file.
    """ # Get the strings to write to .bin file # Write to tf.Example # Write the vocab to file, if applicable # remove these tags from vocab # strip # remove empty # write vocab to file
out_file
fulltext_docs
summary_docs
makevocab