




from transformers import BertTokenizer, EncoderDecoderModel, SeqaSeqTrainer, SeqaSeqTrainingArguments
from transformers.testing_utils import TestCasePlus, require_torch, slow
from transformers.utils import is_datasets_available


if is_datasets_available():
						import datasets







class 				lowerCAmelCase_       (					lowerCamelCase_							):






				@slow
				@require_torch
				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : List[str]					  =      EncoderDecoderModel.from_encoder_decoder_pretrained('prajjwal1/bert-tiny'      ,'prajjwal1/bert-tiny' )
							SCREAMING_SNAKE_CASE_   : Tuple					  =      BertTokenizer.from_pretrained('bert-base-uncased' )

							SCREAMING_SNAKE_CASE_   : Dict					  =      bertabert.config.encoder.vocab_size
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tokenizer.sep_token_id
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      tokenizer.cls_token_id
							SCREAMING_SNAKE_CASE_   : Any					  =      128

							SCREAMING_SNAKE_CASE_   : List[str]					  =      datasets.load_dataset('cnn_dailymail'      ,'3.0.0'      ,split='train[:1%]' )
							SCREAMING_SNAKE_CASE_   : Dict					  =      datasets.load_dataset('cnn_dailymail'      ,'3.0.0'      ,split='validation[:1%]' )

							SCREAMING_SNAKE_CASE_   : List[Any]					  =      train_dataset.select(range(32 ) )
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      val_dataset.select(range(16 ) )

							SCREAMING_SNAKE_CASE_   : str					  =      4

							def _map_to_encoder_decoder_inputs(snake_case__ ):
										# Tokenizer will automatically set [BOS] <text> [EOS]
										SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tokenizer(batch['article']      ,padding='max_length'      ,truncation=snake_case__      ,max_length=512 )
										SCREAMING_SNAKE_CASE_   : int					  =      tokenizer(batch['highlights']      ,padding='max_length'      ,truncation=snake_case__      ,max_length=128 )
										SCREAMING_SNAKE_CASE_   : Any					  =      inputs.input_ids
										SCREAMING_SNAKE_CASE_   : Optional[int]					  =      inputs.attention_mask

										SCREAMING_SNAKE_CASE_   : List[str]					  =      outputs.input_ids
										SCREAMING_SNAKE_CASE_   : int					  =      outputs.input_ids.copy()
										SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      [
										    [-100 if token == tokenizer.pad_token_id else token for token in labels] for labels in batch['labels']
										]
										SCREAMING_SNAKE_CASE_   : int					  =      outputs.attention_mask

										assert all(len(snake_case__ ) == 512 for x in inputs.input_ids )
										assert all(len(snake_case__ ) == 128 for x in outputs.input_ids )

										return batch

							def _compute_metrics(snake_case__ ):
										SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      pred.label_ids
										SCREAMING_SNAKE_CASE_   : Any					  =      pred.predictions

										# all unnecessary tokens are removed
										SCREAMING_SNAKE_CASE_   : Any					  =      tokenizer.batch_decode(snake_case__      ,skip_special_tokens=snake_case__ )
										SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      tokenizer.batch_decode(snake_case__      ,skip_special_tokens=snake_case__ )

										SCREAMING_SNAKE_CASE_   : List[Any]					  =      sum([int(pred_str[i] == label_str[i] ) for i in range(len(snake_case__ ) )] ) / len(snake_case__ )

										return {"accuracy": accuracy}

							# map train dataset
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      train_dataset.map(
							    _map_to_encoder_decoder_inputs      ,batched=snake_case__      ,batch_size=snake_case__      ,remove_columns=['article', 'highlights']      ,)
							train_dataset.set_format(
							    type='torch'      ,columns=['input_ids', 'attention_mask', 'decoder_input_ids', 'decoder_attention_mask', 'labels']      ,)

							# same for validation dataset
							SCREAMING_SNAKE_CASE_   : Dict					  =      val_dataset.map(
							    _map_to_encoder_decoder_inputs      ,batched=snake_case__      ,batch_size=snake_case__      ,remove_columns=['article', 'highlights']      ,)
							val_dataset.set_format(
							    type='torch'      ,columns=['input_ids', 'attention_mask', 'decoder_input_ids', 'decoder_attention_mask', 'labels']      ,)

							SCREAMING_SNAKE_CASE_   : List[Any]					  =      self.get_auto_remove_tmp_dir()

							SCREAMING_SNAKE_CASE_   : List[str]					  =      SeqaSeqTrainingArguments(
							    output_dir=snake_case__      ,per_device_train_batch_size=snake_case__      ,per_device_eval_batch_size=snake_case__      ,predict_with_generate=snake_case__      ,evaluation_strategy='steps'      ,do_train=snake_case__      ,do_eval=snake_case__      ,warmup_steps=0      ,eval_steps=2      ,logging_steps=2      ,)

							# instantiate trainer
							SCREAMING_SNAKE_CASE_   : str					  =      SeqaSeqTrainer(
							    model=snake_case__      ,args=snake_case__      ,compute_metrics=_compute_metrics      ,train_dataset=snake_case__      ,eval_dataset=snake_case__      ,tokenizer=snake_case__      ,)

							# start training
							trainer.train()









import argparse

import torch

from transformers import FunnelBaseModel, FunnelConfig, FunnelModel, load_tf_weights_in_funnel
from transformers.utils import logging


logging.set_verbosity_info()





def        __UpperCAmelCase						(							lowerCamelCase_ :     Dict  ,							lowerCamelCase_ :     Any  ,							lowerCamelCase_ :     int  ,							lowerCamelCase_ :     List[Any]  )       ->  int:







			"""simple docstring"""





			SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      FunnelConfig.from_json_file(lowerCamelCase_  )
			print(F'Building PyTorch model from configuration: {config}'  )
			SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      FunnelBaseModel(lowerCamelCase_  ) if base_model else FunnelModel(lowerCamelCase_  )

			# Load weights from tf checkpoint
			load_tf_weights_in_funnel(lowerCamelCase_  ,							lowerCamelCase_  ,							lowerCamelCase_  )

			# Save pytorch-model
			print(F'Save PyTorch model to {pytorch_dump_path}'  )
			torch.save(model.state_dict()  ,							lowerCamelCase_  )


if __name__ == "__main__":
						UpperCamelCase__ :   Optional[Any]						=							argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    '''--tf_checkpoint_path''', default=None, type=str, required=True, help='''Path to the TensorFlow checkpoint path.'''
						)
						parser.add_argument(
						    '''--config_file''',
						    default=None,
						    type=str,
						    required=True,
						    help='''The config json file corresponding to the pre-trained model. \nThis specifies the model architecture.''',
						)
						parser.add_argument(
						    '''--pytorch_dump_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
						)
						parser.add_argument(
						    '''--base_model''', action='''store_true''', help='''Whether you want just the base model (no decoder) or not.'''
						)
						UpperCamelCase__ :   Tuple						=							parser.parse_args()
						convert_tf_checkpoint_to_pytorch(
						    args.tf_checkpoint_path, args.config_file, args.pytorch_dump_path, args.base_model
						)



