text_model/example/simple_text_clf.py
def _init_tfrec_dataset(self, data_file, need_shuffle):
        """
        Get dataset from tf record file.
        """
        dataset = tf.data.TFRecordDataset(data_file)
        def parse_func(example_proto):
            proto_dict = { "label"   : tf.FixedLenFeature(shape=[self.class_num], dtype=tf.int64),
                           "txt_ids" : tf.FixedLenFeature(shape=[self.max_seq_len], dtype=tf.int64),
                           "txt_len" : tf.FixedLenFeature(shape=[], dtype=tf.int64) }
            parsed_features = tf.parse_single_example(example_proto, proto_dict)
            return ( parsed_features["label"], parsed_features["txt_ids"], parsed_features["txt_len"] )
        dataset = dataset.prefetch(buffer_size=self.batch_size * self.gpu_num)
        if need_shuffle:
            dataset = dataset.shuffle(buffer_size=self.batch_size * self.gpu_num * 2)
        dataset = dataset.apply(tf.contrib.data.map_and_batch(map_func=parse_func,
                                                              batch_size=self.batch_size,
                                                              num_parallel_batches=self.gpu_num,
                                                              drop_remainder=(self.gpu_num > 1)))
        return dataset

        Get dataset from tf record file.
        
Get dataset from tf record file.
Get dataset from tf record file .
def _init_tfrec_dataset(self, data_file, need_shuffle):
        dataset = tf.data.TFRecordDataset(data_file)
        def parse_func(example_proto):
            proto_dict = { "label"   : tf.FixedLenFeature(shape=[self.class_num], dtype=tf.int64),
                           "txt_ids" : tf.FixedLenFeature(shape=[self.max_seq_len], dtype=tf.int64),
                           "txt_len" : tf.FixedLenFeature(shape=[], dtype=tf.int64) }
            parsed_features = tf.parse_single_example(example_proto, proto_dict)
            return ( parsed_features["label"], parsed_features["txt_ids"], parsed_features["txt_len"] )
        dataset = dataset.prefetch(buffer_size=self.batch_size * self.gpu_num)
        if need_shuffle:
            dataset = dataset.shuffle(buffer_size=self.batch_size * self.gpu_num * 2)
        dataset = dataset.apply(tf.contrib.data.map_and_batch(map_func=parse_func,
                                                              batch_size=self.batch_size,
                                                              num_parallel_batches=self.gpu_num,
                                                              drop_remainder=(self.gpu_num > 1)))
        return dataset
def _init_tfrec_dataset ( self , data_file , need_shuffle ) : dataset = tf . data . TFRecordDataset ( data_file ) def parse_func ( example_proto ) : proto_dict = { "label" : tf . FixedLenFeature ( shape = [ self . class_num ] , dtype = tf . int64 ) , "txt_ids" : tf . FixedLenFeature ( shape = [ self . max_seq_len ] , dtype = tf . int64 ) , "txt_len" : tf . FixedLenFeature ( shape = [ ] , dtype = tf . int64 ) } parsed_features = tf . parse_single_example ( example_proto , proto_dict ) return ( parsed_features [ "label" ] , parsed_features [ "txt_ids" ] , parsed_features [ "txt_len" ] ) dataset = dataset . prefetch ( buffer_size = self . batch_size * self . gpu_num ) if need_shuffle : dataset = dataset . shuffle ( buffer_size = self . batch_size * self . gpu_num * 2 ) dataset = dataset . apply ( tf . contrib . data . map_and_batch ( map_func = parse_func , batch_size = self . batch_size , num_parallel_batches = self . gpu_num , drop_remainder = ( self . gpu_num > 1 ) ) ) return dataset
Get dataset from tf record file.
Get dataset from tf record file .
"""
        Get dataset from tf record file.
        """
self
data_file
need_shuffle