'''simple docstring'''


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


snake_case__					:      Tuple				      =  logging.getLogger(__name__)


def  _lowerCamelCase   (				):



     """simple docstring"""





     UpperCAmelCase_							:       int       =				argparse.ArgumentParser(
         description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'       )
     parser.add_argument(
         '--dataset_name'     ,					type=lowerCamelCase_     ,					default='wikitext'     ,					help='Name of the training. Explore datasets at: hf.co/datasets.'     ,					)
     parser.add_argument(
         '--dataset_config'     ,					type=lowerCamelCase_     ,					default='wikitext-103-raw-v1'     ,					help='Configuration name of the dataset.'       )
     parser.add_argument(
         '--tokenizer_name_or_path'     ,					type=lowerCamelCase_     ,					default='sayakpaul/unigram-tokenizer-wikitext'     ,					help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'     ,					)
     parser.add_argument(
         '--shard_size'     ,					type=lowerCamelCase_     ,					default=1000     ,					help='Number of entries to go in a single shard.'     ,					)
     parser.add_argument('--split'     ,					type=lowerCamelCase_     ,					default='train'     ,					choices=['train', 'test', 'validation']       )
     parser.add_argument(
         '--limit'     ,					default=lowerCamelCase_     ,					type=lowerCamelCase_     ,					help='Limit the number of shards (used for debugging).'     ,					)
     parser.add_argument(
         '--max_length'     ,					type=lowerCamelCase_     ,					default=512     ,					help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
         ' sequence length that is a multiple of 8.'     ,					)
     parser.add_argument(
         '--output_dir'     ,					default='tf-tpu'     ,					type=lowerCamelCase_     ,					help='Output directory where the TFRecord shards will be saved. If the'
         ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
         ' shards will be directly saved to a Google Cloud Storage bucket.'     ,					)

     UpperCAmelCase_							:       str       =				parser.parse_args()
     return args


def  _lowerCamelCase   (				lowerCamelCase_ :							List[str]       ):



     """simple docstring"""





     def fn(lowerCamelCase_ :							List[str]       ):
          return tokenizer(examples['text']       )

     return fn


def  _lowerCamelCase   (				lowerCamelCase_ :							Optional[Any]       ):



     """simple docstring"""





     UpperCAmelCase_							:       Any       =				[]
     for i in range(len(tokenized_data['input_ids']       )       ):
          UpperCAmelCase_							:       Tuple       =				{
              'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]       )       ),
              'attention_mask': tf.train.Feature(
                  intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]       )       ),
          }
          UpperCAmelCase_							:       Optional[Any]       =				tf.train.Features(feature=lowerCamelCase_       )
          UpperCAmelCase_							:       Optional[int]       =				tf.train.Example(features=lowerCamelCase_       )
          UpperCAmelCase_							:       int       =				example.SerializeToString()
          records.append(lowerCamelCase_       )
     return records


def  _lowerCamelCase   (				lowerCamelCase_ :							Any       ):



     """simple docstring"""





     UpperCAmelCase_							:       Optional[int]       =				datasets.load_dataset(args.dataset_name     ,					args.dataset_config     ,					split=args.split       )

     if args.limit is not None:
          UpperCAmelCase_							:       Tuple       =				min(len(lowerCamelCase_       )     ,					args.limit       )
          UpperCAmelCase_							:       Any       =				dataset.select(range(lowerCamelCase_       )       )
          print(F'''Limiting the dataset to {args.limit} entries.'''       )

     UpperCAmelCase_							:       Optional[int]       =				AutoTokenizer.from_pretrained(args.tokenizer_name_or_path       )

     # Handle output directory creation.
     # For serializing into a Google Cloud Storage Bucket, one needs to first
     # create a bucket.
     if "gs" not in args.output_dir:
          if not os.path.exists(args.output_dir       ):
               os.makedirs(args.output_dir       )
          UpperCAmelCase_							:       Tuple       =				os.path.join(args.output_dir     ,					args.split       )
          if not os.path.exists(lowerCamelCase_       ):
               os.makedirs(lowerCamelCase_       )
     else:
          UpperCAmelCase_							:       Union[str, Any]       =				os.path.join(args.output_dir     ,					args.split       )

     # Tokenize the whole dataset at once.
     UpperCAmelCase_							:       Union[str, Any]       =				tokenize_function(lowerCamelCase_       )
     UpperCAmelCase_							:       str       =				dataset.map(lowerCamelCase_     ,					batched=lowerCamelCase_     ,					num_proc=4     ,					remove_columns=['text']       )

     # We need to concatenate all our texts together, and then split the result
     # into chunks of a fixed size, which we will call block_size. To do this, we
     # will use the map method again, with the option batched=True. When we use batched=True,
     # the function we pass to map() will be passed multiple inputs at once, allowing us
     # to group them into more or fewer examples than we had in the input.
     # This allows us to create our new fixed-length samples. The advantage of this
     # method is that we don't lose a whole lot of content from the dataset compared to the
     # case where we simply tokenize with a pre-defined max_length.

     def group_texts(lowerCamelCase_ :							Tuple       ):
          # Concatenate all texts.
          UpperCAmelCase_							:       int       =				{k: sum(examples[k]     ,					[]       ) for k in examples.keys()}
          UpperCAmelCase_							:       Dict       =				len(concatenated_examples[list(examples.keys()       )[0]]       )
          # We drop the small remainder, though you could add padding instead if the model supports it
          # In this, as in all things, we advise you to follow your heart 🫀
          UpperCAmelCase_							:       Union[str, Any]       =				(total_length // args.max_length) * args.max_length
          # Split by chunks of max_len.
          UpperCAmelCase_							:       Tuple       =				{
              k: [t[i : i + args.max_length] for i in range(0     ,					lowerCamelCase_     ,					args.max_length       )]
              for k, t in concatenated_examples.items()
          }
          return result

     UpperCAmelCase_							:       Optional[Any]       =				dataset_tokenized.map(lowerCamelCase_     ,					batched=lowerCamelCase_     ,					batch_size=1000     ,					num_proc=4       )

     UpperCAmelCase_							:       List[str]       =				0
     UpperCAmelCase_							:       Optional[int]       =				0
     for shard in range(0     ,					len(lowerCamelCase_       )     ,					args.shard_size       ):
          UpperCAmelCase_							:       Union[str, Any]       =				grouped_dataset[shard : shard + args.shard_size]
          UpperCAmelCase_							:       Optional[Any]       =				len(dataset_snapshot['input_ids']       )
          UpperCAmelCase_							:       Any       =				os.path.join(lowerCamelCase_     ,					F'''dataset-{shard_count}-{records_containing}.tfrecord'''       )
          UpperCAmelCase_							:       Optional[Any]       =				get_serialized_examples(lowerCamelCase_       )

          with tf.io.TFRecordWriter(lowerCamelCase_       ) as out_file:
               for i in range(len(lowerCamelCase_       )       ):
                    UpperCAmelCase_							:       List[Any]       =				serialized_examples[i]
                    out_file.write(lowerCamelCase_       )
               print('Wrote file {} containing {} records'.format(lowerCamelCase_     ,					lowerCamelCase_       )       )

          shard_count += 1
          total_records += records_containing

     with open(F'''split-{args.split}-records-count.txt'''     ,					'w'       ) as f:
          print(F'''Total {args.split} records: {total_records}'''     ,					file=lowerCamelCase_       )


if __name__ == "__main__":
 snake_case__					:      Union[str, Any]				      =  parse_args()
 main(args)







'''simple docstring'''


from typing import List, Optional

from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case__					:      List[Any]				      =  logging.get_logger(__name__)

snake_case__					:      Optional[Any]				      =  {
    '''huggingface/autoformer-tourism-monthly''': '''https://huggingface.co/huggingface/autoformer-tourism-monthly/resolve/main/config.json''',
}






class 					__SCREAMING_SNAKE_CASE						(	lowerCamelCase_	):







  '''simple docstring'''






  lowerCamelCase_		:List[str]						   =		'''autoformer'''
  lowerCamelCase_		:Optional[Any]						   =		{
      '''hidden_size''': '''d_model''',
      '''num_attention_heads''': '''encoder_attention_heads''',
      '''num_hidden_layers''': '''encoder_layers''',
  }


  def __init__(			self				,		snake_case_ = None				,		snake_case_ = None				,		snake_case_ = "student_t"				,		snake_case_ = "nll"				,		snake_case_ = 1				,		snake_case_ = [1, 2, 3, 4, 5, 6, 7]				,		snake_case_ = True				,		snake_case_ = 0				,		snake_case_ = 0				,		snake_case_ = 0				,		snake_case_ = 0				,		snake_case_ = None				,		snake_case_ = None				,		snake_case_ = 6_4				,		snake_case_ = 2				,		snake_case_ = 2				,		snake_case_ = 2				,		snake_case_ = 2				,		snake_case_ = 3_2				,		snake_case_ = 3_2				,		snake_case_ = "gelu"				,		snake_case_ = 0.1				,		snake_case_ = 0.1				,		snake_case_ = 0.1				,		snake_case_ = 0.1				,		snake_case_ = 0.1				,		snake_case_ = 1_0_0				,		snake_case_ = 0.02				,		snake_case_ = True				,		snake_case_=True				,		snake_case_ = 1_0				,		snake_case_ = 2_5				,		snake_case_ = 3				,		**snake_case_				,		):




       '''simple docstring'''

       UpperCAmelCase_							:       List[Any]       =				prediction_length
       UpperCAmelCase_							:       List[str]       =				context_length if context_length is not None else prediction_length
       UpperCAmelCase_							:       Optional[int]       =				distribution_output
       UpperCAmelCase_							:       Optional[int]       =				loss
       UpperCAmelCase_							:       Union[str, Any]       =				input_size
       UpperCAmelCase_							:       int       =				num_time_features
       UpperCAmelCase_							:       List[str]       =				lags_sequence
       UpperCAmelCase_							:       Any       =				scaling
       UpperCAmelCase_							:       Any       =				num_dynamic_real_features
       UpperCAmelCase_							:       int       =				num_static_real_features
       UpperCAmelCase_							:       Optional[Any]       =				num_static_categorical_features
       if cardinality is not None and num_static_categorical_features > 0:
            if len(snake_case_						) != num_static_categorical_features:
                 raise ValueError(
                     'The cardinality should be a list of the same length as `num_static_categorical_features`'						)
            UpperCAmelCase_							:       List[Any]       =				cardinality
       else:
            UpperCAmelCase_							:       Optional[int]       =				[0]
       if embedding_dimension is not None and num_static_categorical_features > 0:
            if len(snake_case_						) != num_static_categorical_features:
                 raise ValueError(
                     'The embedding dimension should be a list of the same length as `num_static_categorical_features`'						)
            UpperCAmelCase_							:       List[str]       =				embedding_dimension
       else:
            UpperCAmelCase_							:       List[Any]       =				[min(5_0				,		(cat + 1) // 2						) for cat in self.cardinality]
       UpperCAmelCase_							:       List[str]       =				num_parallel_samples

       # Transformer architecture configuration
       UpperCAmelCase_							:       Union[str, Any]       =				input_size * len(self.lags_sequence						) + self._number_of_features
       UpperCAmelCase_							:       str       =				d_model
       UpperCAmelCase_							:       str       =				encoder_attention_heads
       UpperCAmelCase_							:       str       =				decoder_attention_heads
       UpperCAmelCase_							:       str       =				encoder_ffn_dim
       UpperCAmelCase_							:       str       =				decoder_ffn_dim
       UpperCAmelCase_							:       str       =				encoder_layers
       UpperCAmelCase_							:       str       =				decoder_layers

       UpperCAmelCase_							:       str       =				dropout
       UpperCAmelCase_							:       Optional[int]       =				attention_dropout
       UpperCAmelCase_							:       Tuple       =				activation_dropout
       UpperCAmelCase_							:       Any       =				encoder_layerdrop
       UpperCAmelCase_							:       Tuple       =				decoder_layerdrop

       UpperCAmelCase_							:       List[str]       =				activation_function
       UpperCAmelCase_							:       Tuple       =				init_std

       UpperCAmelCase_							:       Union[str, Any]       =				use_cache

       # Autoformer
       UpperCAmelCase_							:       Any       =				label_length
       UpperCAmelCase_							:       Union[str, Any]       =				moving_average
       UpperCAmelCase_							:       Tuple       =				autocorrelation_factor

       super().__init__(is_encoder_decoder=snake_case_				,		**snake_case_						)






  @property
  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       return (
           sum(self.embedding_dimension						)
           + self.num_dynamic_real_features
           + self.num_time_features
           + self.num_static_real_features
           + self.input_size * 2  # the log1p(abs(loc)) and log(scale) features
       )






