


'''simple docstring'''






import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


a_					:    Optional[Any]          =    logging.getLogger(__name__)

a_					:    Dict          =    tf.data.AUTOTUNE






def 	_A				()   ->  Any:




     '''simple docstring'''



     _a			      =							argparse.ArgumentParser(description='Train a masked language model on TPU.'		)
     parser.add_argument(
         '--pretrained_model_config'				,						type=lowerCAmelCase__				,						default='roberta-base'				,						help='The model config to use. Note that we don\'t copy the model\'s weights, only the config!'				,						)
     parser.add_argument(
         '--tokenizer'				,						type=lowerCAmelCase__				,						default='unigram-tokenizer-wikitext'				,						help='The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model\'s vocab size.'				,						)

     parser.add_argument(
         '--per_replica_batch_size'				,						type=lowerCAmelCase__				,						default=8				,						help='Batch size per TPU core.'				,						)

     parser.add_argument(
         '--no_tpu'				,						action='store_true'				,						help='If set, run on CPU and don\'t try to initialize a TPU. Useful for debugging on non-TPU instances.'				,						)

     parser.add_argument(
         '--tpu_name'				,						type=lowerCAmelCase__				,						help='Name of TPU resource to initialize. Should be blank on Colab, and \'local\' on TPU VMs.'				,						default='local'				,						)

     parser.add_argument(
         '--tpu_zone'				,						type=lowerCAmelCase__				,						help='Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes.'				,						)

     parser.add_argument(
         '--gcp_project'				,						type=lowerCAmelCase__				,						help='Google cloud project name. Only used for non-Colab TPU nodes.'		)

     parser.add_argument(
         '--bfloat16'				,						action='store_true'				,						help='Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU.'				,						)

     parser.add_argument(
         '--train_dataset'				,						type=lowerCAmelCase__				,						help='Path to training dataset to load. If the path begins with `gs://`'
         ' then the dataset will be loaded from a Google Cloud Storage bucket.'				,						)

     parser.add_argument(
         '--shuffle_buffer_size'				,						type=lowerCAmelCase__				,						default=2**18				,						help='Size of the shuffle buffer (in samples)'				,						)

     parser.add_argument(
         '--eval_dataset'				,						type=lowerCAmelCase__				,						help='Path to evaluation dataset to load. If the path begins with `gs://`'
         ' then the dataset will be loaded from a Google Cloud Storage bucket.'				,						)

     parser.add_argument(
         '--num_epochs'				,						type=lowerCAmelCase__				,						default=1				,						help='Number of epochs to train for.'				,						)

     parser.add_argument(
         '--learning_rate'				,						type=lowerCAmelCase__				,						default=1E-4				,						help='Learning rate to use for training.'				,						)

     parser.add_argument(
         '--weight_decay_rate'				,						type=lowerCAmelCase__				,						default=1E-3				,						help='Weight decay rate to use for training.'				,						)

     parser.add_argument(
         '--max_length'				,						type=lowerCAmelCase__				,						default=5_12				,						help='Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py'				,						)

     parser.add_argument(
         '--mlm_probability'				,						type=lowerCAmelCase__				,						default=0.1_5				,						help='Fraction of tokens to mask during training.'				,						)

     parser.add_argument('--output_dir'				,						type=lowerCAmelCase__				,						required=lowerCAmelCase__				,						help='Path to save model checkpoints to.'		)
     parser.add_argument('--hub_model_id'				,						type=lowerCAmelCase__				,						help='Model ID to upload to on the Hugging Face Hub.'		)

     _a			      =							parser.parse_args()
     return args






def 	_A				(lowerCAmelCase__  :str		)   ->  Any:




     '''simple docstring'''



     try:
          if args.tpu_name:
               _a			      =							tf.distribute.cluster_resolver.TPUClusterResolver(
                   args.tpu_name				,						zone=args.tpu_zone				,						project=args.gcp_project		)
          else:
               _a			      =							tf.distribute.cluster_resolver.TPUClusterResolver()
     except ValueError:
          raise RuntimeError(
              'Couldn\'t connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or '
              '--gcp_project. When running on a TPU VM, use --tpu_name local.'		)

     tf.config.experimental_connect_to_cluster(lowerCAmelCase__		)
     tf.tpu.experimental.initialize_tpu_system(lowerCAmelCase__		)

     return tpu






def 	_A				(lowerCAmelCase__  :List[Any]		)   ->  int:




     '''simple docstring'''



     _a			      =							0
     for file in file_list:
          _a			      =							file.split('/'		)[-1]
          _a			      =							re.search(r'-\d+-(\d+)\.tfrecord'				,						lowerCAmelCase__		).group(1		)
          _a			      =							int(lowerCAmelCase__		)
          num_samples += sample_count

     return num_samples






def 	_A				(lowerCAmelCase__  :Optional[int]				,						lowerCAmelCase__  :int				,						lowerCAmelCase__  :List[Any]				,						lowerCAmelCase__  :List[Any]				,						lowerCAmelCase__  :Tuple				,						lowerCAmelCase__  :Optional[Any]=None		)   ->  Any:




     '''simple docstring'''



     _a			      =							count_samples(lowerCAmelCase__		)
     _a			      =							tf.data.Dataset.from_tensor_slices(lowerCAmelCase__		)
     if shuffle:
          _a			      =							dataset.shuffle(len(lowerCAmelCase__		)		)
     _a			      =							tf.data.TFRecordDataset(lowerCAmelCase__				,						num_parallel_reads=lowerCAmelCase__		)
     # TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
     _a			      =							dataset.apply(tf.data.experimental.assert_cardinality(lowerCAmelCase__		)		)
     _a			      =							dataset.map(lowerCAmelCase__				,						num_parallel_calls=lowerCAmelCase__		)
     if shuffle:
          assert shuffle_buffer_size is not None
          _a			      =							dataset.shuffle(args.shuffle_buffer_size		)
     _a			      =							dataset.batch(lowerCAmelCase__				,						drop_remainder=lowerCAmelCase__		)
     _a			      =							dataset.map(lowerCAmelCase__				,						num_parallel_calls=lowerCAmelCase__		)
     _a			      =							dataset.prefetch(lowerCAmelCase__		)
     return dataset






def 	_A				(lowerCAmelCase__  :List[Any]		)   ->  str:




     '''simple docstring'''



     if not args.no_tpu:
          _a			      =							initialize_tpu(lowerCAmelCase__		)
          _a			      =							tf.distribute.TPUStrategy(lowerCAmelCase__		)
     else:
          _a			      =							tf.distribute.OneDeviceStrategy(device='/gpu:0'		)

     if args.bfloataa:
          tf.keras.mixed_precision.set_global_policy('mixed_bfloat16'		)

     _a			      =							AutoTokenizer.from_pretrained(args.tokenizer		)
     _a			      =							AutoConfig.from_pretrained(args.pretrained_model_config		)
     _a			      =							tokenizer.vocab_size

     _a			      =							tf.io.gfile.glob(os.path.join(args.train_dataset				,						'*.tfrecord'		)		)
     if not training_records:
          raise ValueError(f'No .tfrecord files found in {args.train_dataset}.'		)
     _a			      =							tf.io.gfile.glob(os.path.join(args.eval_dataset				,						'*.tfrecord'		)		)
     if not eval_records:
          raise ValueError(f'No .tfrecord files found in {args.eval_dataset}.'		)

     _a			      =							count_samples(lowerCAmelCase__		)

     _a			      =							num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
     _a			      =							steps_per_epoch * args.num_epochs

     with strategy.scope():
          _a			      =							TFAutoModelForMaskedLM.from_config(lowerCAmelCase__		)
          model(model.dummy_inputs		)  # Pass some dummy inputs through the model to ensure all the weights are built
          _a      ,   _a			      =							create_optimizer(
              num_train_steps=lowerCAmelCase__				,						num_warmup_steps=total_train_steps // 20				,						init_lr=args.learning_rate				,						weight_decay_rate=args.weight_decay_rate				,						)

          # Transformers models compute the right loss for their task by default when labels are passed, and will
          # use this for training unless you specify your own loss function in compile().
          model.compile(optimizer=lowerCAmelCase__				,						metrics=['accuracy']		)

     def decode_fn(lowerCAmelCase__  :Any		):
          _a			      =							{
              'input_ids': tf.io.FixedLenFeature(dtype=tf.intaa				,						shape=(args.max_length,)		),
              'attention_mask': tf.io.FixedLenFeature(dtype=tf.intaa				,						shape=(args.max_length,)		),
          }
          return tf.io.parse_single_example(lowerCAmelCase__				,						lowerCAmelCase__		)

     # Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
     # use their methods in our data pipeline.
     _a			      =							DataCollatorForLanguageModeling(
         tokenizer=lowerCAmelCase__				,						mlm_probability=args.mlm_probability				,						mlm=lowerCAmelCase__				,						return_tensors='tf'		)

     def mask_with_collator(lowerCAmelCase__  :Optional[int]		):
          # TF really needs an isin() function
          _a			      =							(
              ~tf.cast(batch['attention_mask']				,						tf.bool		)
              | (batch['input_ids'] == tokenizer.cls_token_id)
              | (batch['input_ids'] == tokenizer.sep_token_id)
          )
          _a      ,   _a			      =							data_collator.tf_mask_tokens(
              batch['input_ids']				,						vocab_size=len(lowerCAmelCase__		)				,						mask_token_id=tokenizer.mask_token_id				,						special_tokens_mask=lowerCAmelCase__				,						)
          return batch

     _a			      =							args.per_replica_batch_size * strategy.num_replicas_in_sync

     _a			      =							prepare_dataset(
         lowerCAmelCase__				,						decode_fn=lowerCAmelCase__				,						mask_fn=lowerCAmelCase__				,						batch_size=lowerCAmelCase__				,						shuffle=lowerCAmelCase__				,						shuffle_buffer_size=args.shuffle_buffer_size				,						)

     _a			      =							prepare_dataset(
         lowerCAmelCase__				,						decode_fn=lowerCAmelCase__				,						mask_fn=lowerCAmelCase__				,						batch_size=lowerCAmelCase__				,						shuffle=lowerCAmelCase__				,						)

     _a			      =							[]
     if args.hub_model_id:
          callbacks.append(
              PushToHubCallback(output_dir=args.output_dir				,						hub_model_id=args.hub_model_id				,						tokenizer=lowerCAmelCase__		)		)

     model.fit(
         lowerCAmelCase__				,						validation_data=lowerCAmelCase__				,						epochs=args.num_epochs				,						callbacks=lowerCAmelCase__				,						)

     model.save_pretrained(args.output_dir		)


if __name__ == "__main__":
 a_					:    Dict          =    parse_args()
 main(args)





'''simple docstring'''






import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


a_					:    Optional[Any]          =    logging.getLogger(__name__)




@dataclass(frozen=_SCREAMING_SNAKE_CASE       )
class  a     :
 _lowerCAmelCase 					=	42
 _lowerCAmelCase 					=	42
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None





@dataclass(frozen=_SCREAMING_SNAKE_CASE       )
class  a     :
 _lowerCAmelCase 					=	42
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None


if is_torch_available():
 import torch
 from torch.utils.data import Dataset




 class  a     (     _SCREAMING_SNAKE_CASE       ):
  _lowerCAmelCase 					=	42
  def __init__( self      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__ = None      ,				__magic_name__=False      ,				__magic_name__ = False      ,				)					->							Optional[int]:
       _a			      =							hans_processors[task]()

       _a			      =							os.path.join(
           __magic_name__      ,				'cached_{}_{}_{}_{}'.format(
               'dev' if evaluate else 'train'      ,				tokenizer.__class__.__name__      ,				str(__magic_name__						)      ,				__magic_name__      ,				)      ,				)
       _a			      =							processor.get_labels()
       if tokenizer.__class__ in (
           RobertaTokenizer,
           RobertaTokenizerFast,
           XLMRobertaTokenizer,
           BartTokenizer,
           BartTokenizerFast,
       ):
            # HACK(label indices are swapped in RoBERTa pretrained model)
            _a      ,   _a			      =							label_list[2], label_list[1]
       _a			      =							label_list

       # Make sure only the first process in distributed training processes the dataset,
       # and the others will use the cache.
       _a			      =							cached_features_file + '.lock'
       with FileLock(__magic_name__						):
            if os.path.exists(__magic_name__						) and not overwrite_cache:
                 logger.info(f'Loading features from cached file {cached_features_file}'						)
                 _a			      =							torch.load(__magic_name__						)
            else:
                 logger.info(f'Creating features from dataset file at {data_dir}'						)

                 _a			      =							(
                     processor.get_dev_examples(__magic_name__						) if evaluate else processor.get_train_examples(__magic_name__						)
                 )

                 logger.info('Training examples: %s'      ,				len(__magic_name__						)						)
                 _a			      =							hans_convert_examples_to_features(__magic_name__      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__						)
                 logger.info('Saving features into cached file %s'      ,				__magic_name__						)
                 torch.save(self.features      ,				__magic_name__						)
  def __len__( self						)					->							List[Any]:
       return len(self.features						)
  def __getitem__( self      ,				__magic_name__						)					->							InputFeatures:
       return self.features[i]




  def 							__UpperCAmelCase		( self						)					->							Optional[Any]:
       return self.label_list


if is_tf_available():
 import tensorflow as tf




 class  a     :
  _lowerCAmelCase 					=	42
  def __init__( self      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__ = 1_28      ,				__magic_name__=False      ,				__magic_name__ = False      ,				)					->							str:
       _a			      =							hans_processors[task]()
       _a			      =							processor.get_labels()
       if tokenizer.__class__ in (
           RobertaTokenizer,
           RobertaTokenizerFast,
           XLMRobertaTokenizer,
           BartTokenizer,
           BartTokenizerFast,
       ):
            # HACK(label indices are swapped in RoBERTa pretrained model)
            _a      ,   _a			      =							label_list[2], label_list[1]
       _a			      =							label_list

       _a			      =							processor.get_dev_examples(__magic_name__						) if evaluate else processor.get_train_examples(__magic_name__						)
       _a			      =							hans_convert_examples_to_features(__magic_name__      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__						)

       def gen():
            for ex_index, ex in tqdm.tqdm(enumerate(self.features						)      ,				desc='convert examples to features'						):
                 if ex_index % 1_00_00 == 0:
                      logger.info('Writing example %d of %d' % (ex_index, len(__magic_name__						))						)

                 yield (
                     {
                         "example_id": 0,
                         "input_ids": ex.input_ids,
                         "attention_mask": ex.attention_mask,
                         "token_type_ids": ex.token_type_ids,
                     },
                     ex.label,
                 )

       _a			      =							tf.data.Dataset.from_generator(
           __magic_name__      ,				(
               {
                   'example_id': tf.intaa,
                   'input_ids': tf.intaa,
                   'attention_mask': tf.intaa,
                   'token_type_ids': tf.intaa,
               },
               tf.intaa,
           )      ,				(
               {
                   'example_id': tf.TensorShape([]						),
                   'input_ids': tf.TensorShape([None, None]						),
                   'attention_mask': tf.TensorShape([None, None]						),
                   'token_type_ids': tf.TensorShape([None, None]						),
               },
               tf.TensorShape([]						),
           )      ,				)
  def 							__UpperCAmelCase		( self						)					->							Tuple:
       return self.dataset
  def __len__( self						)					->							Optional[Any]:
       return len(self.features						)
  def __getitem__( self      ,				__magic_name__						)					->							InputFeatures:
       return self.features[i]




  def 							__UpperCAmelCase		( self						)					->							List[Any]:
       return self.label_list




class  a     (     _SCREAMING_SNAKE_CASE       ):
 def 							__UpperCAmelCase		( self      ,				__magic_name__						)					->							Tuple:
      return self._create_examples(self._read_tsv(os.path.join(__magic_name__      ,				'heuristics_train_set.txt'						)						)      ,				'train'						)
 def 							__UpperCAmelCase		( self      ,				__magic_name__						)					->							List[Any]:
      return self._create_examples(self._read_tsv(os.path.join(__magic_name__      ,				'heuristics_evaluation_set.txt'						)						)      ,				'dev'						)
 def 							__UpperCAmelCase		( self						)					->							Tuple:
      return ["contradiction", "entailment", "neutral"]




 def 							__UpperCAmelCase		( self      ,				__magic_name__      ,				__magic_name__						)					->							Optional[Any]:
      _a			      =							[]
      for i, line in enumerate(__magic_name__						):
           if i == 0:
                continue
           _a			      =							'%s-%s' % (set_type, line[0])
           _a			      =							line[5]
           _a			      =							line[6]
           _a			      =							line[7][2:] if line[7].startswith('ex'						) else line[7]
           _a			      =							line[0]
           examples.append(InputExample(guid=__magic_name__      ,				text_a=__magic_name__      ,				text_b=__magic_name__      ,				label=__magic_name__      ,				pairID=__magic_name__						)						)
      return examples






def 	_A				(lowerCAmelCase__  :List[InputExample]				,						lowerCAmelCase__  :List[str]				,						lowerCAmelCase__  :int				,						lowerCAmelCase__  :PreTrainedTokenizer				,						)   ->  Tuple:




     '''simple docstring'''



     _a			      =							{label: i for i, label in enumerate(lowerCAmelCase__		)}

     _a			      =							[]
     for ex_index, example in tqdm.tqdm(enumerate(lowerCAmelCase__		)				,						desc='convert examples to features'		):
          if ex_index % 1_00_00 == 0:
               logger.info('Writing example %d' % (ex_index)		)

          _a			      =							tokenizer(
              example.text_a				,						example.text_b				,						add_special_tokens=lowerCAmelCase__				,						max_length=lowerCAmelCase__				,						padding='max_length'				,						truncation=lowerCAmelCase__				,						return_overflowing_tokens=lowerCAmelCase__				,						)

          _a			      =							label_map[example.label] if example.label in label_map else 0

          _a			      =							int(example.pairID		)

          features.append(InputFeatures(**lowerCAmelCase__				,						label=lowerCAmelCase__				,						pairID=lowerCAmelCase__		)		)

     for i, example in enumerate(examples[:5]		):
          logger.info('*** Example ***'		)
          logger.info(f'guid: {example}'		)
          logger.info(f'features: {features[i]}'		)

     return features


a_					:    Optional[int]          =    {
    "hans": 3,
}






a_					:    Optional[Any]          =    {
    "hans": HansProcessor,
}

