bin/tokenize_data.py
def dump_processed_shard_load_shard(vocab, shard_name):
        """Read one file backwards and forwards, get a ids lookup and drop output.

        Args:
            shard_name: file path.

        Returns:
            Nothing
        """
        print('Loading data from: %s' % shard_name)
        with open(shard_name) as f:
            sentences_raw = f.readlines()
        
        sentences_reverse = []
        for sentence in sentences_raw:
            splitted = sentence.split()
            splitted.reverse()
            sentences_reverse.append(' '.join(splitted))
        sentences = sentences_raw
        random.shuffle(sentences)
        random.shuffle(sentences_reverse)
        ids = [vocab.encode(sentence, False) for sentence in sentences]
        ids_reverse = [vocab.encode(sentence, True) for sentence in sentences]
        sentence_count = len(sentences)
        chars_ids = [None] * len(ids)
        print("There are {} sentences".format(sentence_count))
        with open(shard_name+"processed",'w') as w:
            w.write("sentence\treversed_sentence\tids\treversed_ids\tchar_is\n")
            for i in range(sentence_count):
                w.write("{}\t{}\t{}\t{}\t{}\n".format(sentences[i].strip(),sentences_reverse[i].strip(), str(ids[i]), list(ids_reverse[i]), chars_ids[i]))
        print("Done printing tokenized sentences")
Read one file backwards and forwards, get a ids lookup and drop output.

        Args:
            shard_name: file path.

        Returns:
            Nothing
        
Read one file backwards and forwards, get a ids lookup and drop output.
Read one file backwards and forwards get a ids lookup and drop output .
def dump_processed_shard_load_shard(vocab, shard_name):
        print('Loading data from: %s' % shard_name)
        with open(shard_name) as f:
            sentences_raw = f.readlines()
        sentences_reverse = []
        for sentence in sentences_raw:
            splitted = sentence.split()
            splitted.reverse()
            sentences_reverse.append(' '.join(splitted))
        sentences = sentences_raw
        random.shuffle(sentences)
        random.shuffle(sentences_reverse)
        ids = [vocab.encode(sentence, False) for sentence in sentences]
        ids_reverse = [vocab.encode(sentence, True) for sentence in sentences]
        sentence_count = len(sentences)
        chars_ids = [None] * len(ids)
        print("There are {} sentences".format(sentence_count))
        with open(shard_name+"processed",'w') as w:
            w.write("sentence\treversed_sentence\tids\treversed_ids\tchar_is\n")
            for i in range(sentence_count):
                w.write("{}\t{}\t{}\t{}\t{}\n".format(sentences[i].strip(),sentences_reverse[i].strip(), str(ids[i]), list(ids_reverse[i]), chars_ids[i]))
        print("Done printing tokenized sentences")
def dump_processed_shard_load_shard ( vocab , shard_name ) : print ( 'Loading data from: %s' % shard_name ) with open ( shard_name ) as f : sentences_raw = f . readlines ( ) sentences_reverse = [ ] for sentence in sentences_raw : splitted = sentence . split ( ) splitted . reverse ( ) sentences_reverse . append ( ' ' . join ( splitted ) ) sentences = sentences_raw random . shuffle ( sentences ) random . shuffle ( sentences_reverse ) ids = [ vocab . encode ( sentence , False ) for sentence in sentences ] ids_reverse = [ vocab . encode ( sentence , True ) for sentence in sentences ] sentence_count = len ( sentences ) chars_ids = [ None ] * len ( ids ) print ( "There are {} sentences" . format ( sentence_count ) ) with open ( shard_name + "processed" , 'w' ) as w : w . write ( "sentence\treversed_sentence\tids\treversed_ids\tchar_is\n" ) for i in range ( sentence_count ) : w . write ( "{}\t{}\t{}\t{}\t{}\n" . format ( sentences [ i ] . strip ( ) , sentences_reverse [ i ] . strip ( ) , str ( ids [ i ] ) , list ( ids_reverse [ i ] ) , chars_ids [ i ] ) ) print ( "Done printing tokenized sentences" )
Read one file backwards and forwards, get a ids lookup and drop output.
Read one file backwards and forwards get a ids lookup and drop output .
"""Read one file backwards and forwards, get a ids lookup and drop output.

        Args:
            shard_name: file path.

        Returns:
            Nothing
        """
None
vocab
shard_name
None