





'''simple docstring'''




import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
lowercase__       : Optional[int]         =       data_utils.TransfoXLTokenizer
lowercase__       : str         =       data_utils.TransfoXLCorpus
lowercase__       : Union[str, Any]         =       data_utils
lowercase__       : int         =       data_utils






def 			_lowerCAmelCase     (    __snake_case    :					str					,				__snake_case    :					Union[str, Any]					,				__snake_case    :					Dict					,				__snake_case    :					List[Any]     )	->	Optional[Any]:
     if transfo_xl_dataset_file:
          # Convert a pre-processed corpus (see original TensorFlow repo)
          with open(__snake_case					,				'rb'     ) as fp:
               __A		:					Union[str, Any]						=			pickle.load(__snake_case					,				encoding='latin1'     )
          # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
          __A		:					Any						=			pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['pretrained_vocab_file']
          print(f'Save vocabulary to {pytorch_vocab_dump_path}'     )
          __A		:					Optional[Any]						=			corpus.vocab.__dict__
          torch.save(__snake_case					,				__snake_case     )

          __A		:					List[str]						=			corpus.__dict__
          corpus_dict_no_vocab.pop('vocab'					,				__snake_case     )
          __A		:					Dict						=			pytorch_dump_folder_path + '/' + CORPUS_NAME
          print(f'Save dataset to {pytorch_dataset_dump_path}'     )
          torch.save(__snake_case					,				__snake_case     )

     if tf_checkpoint_path:
          # Convert a pre-trained TensorFlow model
          __A		:					Any						=			os.path.abspath(__snake_case     )
          __A		:					List[Any]						=			os.path.abspath(__snake_case     )

          print(f'Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.'     )
          # Initialise PyTorch model
          if transfo_xl_config_file == "":
               __A		:					List[str]						=			TransfoXLConfig()
          else:
               __A		:					str						=			TransfoXLConfig.from_json_file(__snake_case     )
          print(f'Building PyTorch model from configuration: {config}'     )
          __A		:					Any						=			TransfoXLLMHeadModel(__snake_case     )

          __A		:					Optional[Any]						=			load_tf_weights_in_transfo_xl(__snake_case					,				__snake_case					,				__snake_case     )
          # Save pytorch-model
          __A		:					Dict						=			os.path.join(__snake_case					,				__snake_case     )
          __A		:					Optional[int]						=			os.path.join(__snake_case					,				__snake_case     )
          print(f'Save PyTorch model to {os.path.abspath(__snake_case     )}'     )
          torch.save(model.state_dict()					,				__snake_case     )
          print(f'Save configuration file to {os.path.abspath(__snake_case     )}'     )
          with open(__snake_case					,				'w'					,				encoding='utf-8'     ) as f:
               f.write(config.to_json_string()     )


if __name__ == "__main__":
  lowercase__       : List[Any]         =       argparse.ArgumentParser()
  parser.add_argument(
      '''--pytorch_dump_folder_path''',
      default=None,
      type=str,
      required=True,
      help='''Path to the folder to store the PyTorch model or dataset/vocab.''',
  )
  parser.add_argument(
      '''--tf_checkpoint_path''',
      default='''''',
      type=str,
      help='''An optional path to a TensorFlow checkpoint path to be converted.''',
  )
  parser.add_argument(
      '''--transfo_xl_config_file''',
      default='''''',
      type=str,
      help=(
          '''An optional config json file corresponding to the pre-trained BERT model. \n'''
          '''This specifies the model architecture.'''
      ),
  )
  parser.add_argument(
      '''--transfo_xl_dataset_file''',
      default='''''',
      type=str,
      help='''An optional dataset file to be converted in a vocabulary.''',
  )
  lowercase__       : int         =       parser.parse_args()
  convert_transfo_xl_checkpoint_to_pytorch(
      args.tf_checkpoint_path,
      args.transfo_xl_config_file,
      args.pytorch_dump_folder_path,
      args.transfo_xl_dataset_file,
  )


'''simple docstring'''





import math

import flax.linen as nn
import jax.numpy as jnp






def 				_lowerCAmelCase			(   __magic_name__						:     jnp.ndarray   , __magic_name__						:     int   , __magic_name__						:     float = 1   , __magic_name__						:     float = 1   , __magic_name__						:     float = 1.0E4   , __magic_name__						:     bool = False   , __magic_name__						:     float = 1.0   , )				->  jnp.ndarray:
						assert timesteps.ndim == 1, "Timesteps should be a 1d-array"
						assert embedding_dim % 2 == 0, f'''Embedding dimension {embedding_dim} should be even'''
						lowercase :							int    		=float(embedding_dim // 2						)
						lowercase :							Optional[int]    		=math.log(max_timescale / min_timescale						) / (num_timescales - freq_shift)
						lowercase :							Any    		=min_timescale * jnp.exp(jnp.arange(__magic_name__   , dtype=jnp.floataa						) * -log_timescale_increment						)
						lowercase :							List[Any]    		=jnp.expand_dims(__magic_name__   , 1						) * jnp.expand_dims(__magic_name__   , 0						)

						# scale embeddings
						lowercase :							Tuple    		=scale * emb

						if flip_sin_to_cos:
												lowercase :							Dict    		=jnp.concatenate([jnp.cos(__magic_name__						), jnp.sin(__magic_name__						)]   , axis=1						)
						else:
												lowercase :							Any    		=jnp.concatenate([jnp.sin(__magic_name__						), jnp.cos(__magic_name__						)]   , axis=1						)
						lowercase :							List[str]    		=jnp.reshape(__magic_name__   , [jnp.shape(__magic_name__						)[0], embedding_dim]						)
						return signal





class 		__SCREAMING_SNAKE_CASE							(			nn.Module      ):
				lowerCamelCase_		       =    32
				lowerCamelCase_		       =    jnp.floataa







				@nn.compact
				def __call__(       self     :		Tuple		,					UpperCAmelCase__     :		int   ):
										'''simple docstring'''
										lowercase :							List[Any]    		=nn.Dense(self.time_embed_dim		,					dtype=self.dtype		,					name='''linear_1'''   )(UpperCAmelCase__   )
										lowercase :							Any    		=nn.silu(UpperCAmelCase__   )
										lowercase :							int    		=nn.Dense(self.time_embed_dim		,					dtype=self.dtype		,					name='''linear_2'''   )(UpperCAmelCase__   )
										return temb





class 		__SCREAMING_SNAKE_CASE							(			nn.Module      ):
				lowerCamelCase_		       =    32
				lowerCamelCase_		       =    False
				lowerCamelCase_		       =    1







				@nn.compact
				def __call__(       self     :		int		,					UpperCAmelCase__     :		str   ):
										'''simple docstring'''
										return get_sinusoidal_embeddings(
										    UpperCAmelCase__		,					embedding_dim=self.dim		,					flip_sin_to_cos=self.flip_sin_to_cos		,					freq_shift=self.freq_shift   )





