pointer-generator/my_make_datasets.py
def tokenize_documents(fulltext_dir, summary_dir, tokenized_documents_dir,
                       tokenized_summary_dir, tokenize_docs,
                       tokenize_summaries):
    """
    Maps a whole directory of .story files to a tokenized version using
    Stanford CoreNLP Tokenizer
    """
    print("Preparing to tokenize %s, %s to %s..." % (fulltext_dir, summary_dir,
                                                     tokenized_documents_dir))
    documents = get_documents(fulltext_dir, "*.text")
    summaries = get_documents(summary_dir, "*.summary")
    print("Making list of files to tokenize...")

    with open("mapping_fulltext.txt", "w") as f:
        for s in documents:
            f.write("%s \t %s\n" % (os.path.join(fulltext_dir, s),
                                    os.path.join(tokenized_documents_dir, s)))

    with open("mapping_summary.txt", "w") as f:
        for s in summaries:
            f.write("%s \t %s\n" % (os.path.join(summary_dir, s),
                                    os.path.join(tokenized_summary_dir, s)))

    command_fulltext = [
        'java', 'edu.stanford.nlp.process.PTBTokenizer', '-ioFileList',
        '-preserveLines', 'mapping_fulltext.txt'
    ]
    command_summary = [
        'java', 'edu.stanford.nlp.process.PTBTokenizer', '-ioFileList',
        '-preserveLines', 'mapping_summary.txt'
    ]

    if tokenize_docs:
        print("Tokenizing %i fulltext documents in %s and saving in %s..." %
              (len(documents), fulltext_dir, tokenized_documents_dir))
        subprocess.call(command_fulltext)

    if tokenize_summaries:
        print("Tokenizing %i summary documents in %s and saving in %s..." %
              (len(documents), summary_dir, tokenized_summary_dir))
        subprocess.call(command_summary)

    print("Stanford CoreNLP Tokenizer has finished.")

    # Remove temporary files
    os.remove("mapping_fulltext.txt")
    os.remove("mapping_summary.txt")

    # Check that the tokenized stories directory contains the same number of
    # files as the original directory
    num_orig = len(documents)
    num_tokenized = len(get_documents(tokenized_documents_dir, "*.text"))
    if num_orig != num_tokenized:
        raise Exception(
            ("The tokenized documents directory %s contains %i files, but it "
             + "should contain the same number as %s (which has %i files). " +
             "Was there an error during tokenization?") %
            (tokenized_documents_dir, num_tokenized, fulltext_dir, num_orig))
    print("Successfully finished tokenizing %s to %s.\n" %
          (fulltext_dir, tokenized_documents_dir))

    num_orig = len(summaries)
    num_tokenized = len(get_documents(tokenized_summary_dir, "*.summary"))
    if num_orig != num_tokenized:
        raise Exception(
            ("The tokenized summaries directory %s contains %i files, but it "
             + "should contain the same number as %s (which has %i files). " +
             "Was there an error during tokenization?") %
            (tokenized_summary_dir, num_tokenized, summary_dir, num_orig))
    print("Successfully finished tokenizing %s to %s.\n" %
          (summary_dir, tokenized_summary_dir))

    Maps a whole directory of .story files to a tokenized version using
    Stanford CoreNLP Tokenizer
    
Maps a whole directory of .story files to a tokenized version using
Stanford CoreNLP Tokenizer
Maps a whole directory of . story files to a tokenized version using Stanford CoreNLP Tokenizer
def tokenize_documents(fulltext_dir, summary_dir, tokenized_documents_dir,
                       tokenized_summary_dir, tokenize_docs,
                       tokenize_summaries):
    print("Preparing to tokenize %s, %s to %s..." % (fulltext_dir, summary_dir,
                                                     tokenized_documents_dir))
    documents = get_documents(fulltext_dir, "*.text")
    summaries = get_documents(summary_dir, "*.summary")
    print("Making list of files to tokenize...")
    with open("mapping_fulltext.txt", "w") as f:
        for s in documents:
            f.write("%s \t %s\n" % (os.path.join(fulltext_dir, s),
                                    os.path.join(tokenized_documents_dir, s)))
    with open("mapping_summary.txt", "w") as f:
        for s in summaries:
            f.write("%s \t %s\n" % (os.path.join(summary_dir, s),
                                    os.path.join(tokenized_summary_dir, s)))
    command_fulltext = [
        'java', 'edu.stanford.nlp.process.PTBTokenizer', '-ioFileList',
        '-preserveLines', 'mapping_fulltext.txt'
    ]
    command_summary = [
        'java', 'edu.stanford.nlp.process.PTBTokenizer', '-ioFileList',
        '-preserveLines', 'mapping_summary.txt'
    ]
    if tokenize_docs:
        print("Tokenizing %i fulltext documents in %s and saving in %s..." %
              (len(documents), fulltext_dir, tokenized_documents_dir))
        subprocess.call(command_fulltext)
    if tokenize_summaries:
        print("Tokenizing %i summary documents in %s and saving in %s..." %
              (len(documents), summary_dir, tokenized_summary_dir))
        subprocess.call(command_summary)
    print("Stanford CoreNLP Tokenizer has finished.")
    os.remove("mapping_fulltext.txt")
    os.remove("mapping_summary.txt")
    num_orig = len(documents)
    num_tokenized = len(get_documents(tokenized_documents_dir, "*.text"))
    if num_orig != num_tokenized:
        raise Exception(
            ("The tokenized documents directory %s contains %i files, but it "
             + "should contain the same number as %s (which has %i files). " +
             "Was there an error during tokenization?") %
            (tokenized_documents_dir, num_tokenized, fulltext_dir, num_orig))
    print("Successfully finished tokenizing %s to %s.\n" %
          (fulltext_dir, tokenized_documents_dir))
    num_orig = len(summaries)
    num_tokenized = len(get_documents(tokenized_summary_dir, "*.summary"))
    if num_orig != num_tokenized:
        raise Exception(
            ("The tokenized summaries directory %s contains %i files, but it "
             + "should contain the same number as %s (which has %i files). " +
             "Was there an error during tokenization?") %
            (tokenized_summary_dir, num_tokenized, summary_dir, num_orig))
    print("Successfully finished tokenizing %s to %s.\n" %
          (summary_dir, tokenized_summary_dir))
def tokenize_documents ( fulltext_dir , summary_dir , tokenized_documents_dir , tokenized_summary_dir , tokenize_docs , tokenize_summaries ) : print ( "Preparing to tokenize %s, %s to %s..." % ( fulltext_dir , summary_dir , tokenized_documents_dir ) ) documents = get_documents ( fulltext_dir , "*.text" ) summaries = get_documents ( summary_dir , "*.summary" ) print ( "Making list of files to tokenize..." ) with open ( "mapping_fulltext.txt" , "w" ) as f : for s in documents : f . write ( "%s \t %s\n" % ( os . path . join ( fulltext_dir , s ) , os . path . join ( tokenized_documents_dir , s ) ) ) with open ( "mapping_summary.txt" , "w" ) as f : for s in summaries : f . write ( "%s \t %s\n" % ( os . path . join ( summary_dir , s ) , os . path . join ( tokenized_summary_dir , s ) ) ) command_fulltext = [ 'java' , 'edu.stanford.nlp.process.PTBTokenizer' , '-ioFileList' , '-preserveLines' , 'mapping_fulltext.txt' ] command_summary = [ 'java' , 'edu.stanford.nlp.process.PTBTokenizer' , '-ioFileList' , '-preserveLines' , 'mapping_summary.txt' ] if tokenize_docs : print ( "Tokenizing %i fulltext documents in %s and saving in %s..." % ( len ( documents ) , fulltext_dir , tokenized_documents_dir ) ) subprocess . call ( command_fulltext ) if tokenize_summaries : print ( "Tokenizing %i summary documents in %s and saving in %s..." % ( len ( documents ) , summary_dir , tokenized_summary_dir ) ) subprocess . call ( command_summary ) print ( "Stanford CoreNLP Tokenizer has finished." ) os . remove ( "mapping_fulltext.txt" ) os . remove ( "mapping_summary.txt" ) num_orig = len ( documents ) num_tokenized = len ( get_documents ( tokenized_documents_dir , "*.text" ) ) if num_orig != num_tokenized : raise Exception ( ( "The tokenized documents directory %s contains %i files, but it " + "should contain the same number as %s (which has %i files). " + "Was there an error during tokenization?" ) % ( tokenized_documents_dir , num_tokenized , fulltext_dir , num_orig ) ) print ( "Successfully finished tokenizing %s to %s.\n" % ( fulltext_dir , tokenized_documents_dir ) ) num_orig = len ( summaries ) num_tokenized = len ( get_documents ( tokenized_summary_dir , "*.summary" ) ) if num_orig != num_tokenized : raise Exception ( ( "The tokenized summaries directory %s contains %i files, but it " + "should contain the same number as %s (which has %i files). " + "Was there an error during tokenization?" ) % ( tokenized_summary_dir , num_tokenized , summary_dir , num_orig ) ) print ( "Successfully finished tokenizing %s to %s.\n" % ( summary_dir , tokenized_summary_dir ) )
Maps a whole directory of .story files to a tokenized version using
Stanford CoreNLP Tokenizer
Maps a whole directory of . story files to a tokenized version using Stanford CoreNLP Tokenizer
"""
    Maps a whole directory of .story files to a tokenized version using
    Stanford CoreNLP Tokenizer
    """ # Remove temporary files # Check that the tokenized stories directory contains the same number of # files as the original directory
fulltext_dir
summary_dir
tokenized_documents_dir
tokenized_summary_dir
tokenize_docs
tokenize_summaries