
"""simple docstring"""






import os
from pickle import UnpicklingError
from typing import Dict, Tuple

import jax
import jax.numpy as jnp
import numpy as np
from flax.serialization import from_bytes
from flax.traverse_util import flatten_dict, unflatten_dict

import transformers

from .utils import logging


__SCREAMING_SNAKE_CASE							=     logging.get_logger(__name__)




def       A_  (				__lowercase						,	__lowercase						,	__lowercase						,	__lowercase=False	):
 try:
  import torch  # noqa: F401
 except ImportError:
  logger.error(
      'Loading a PyTorch model in Flax, requires both PyTorch and Flax to be installed. Please see'
      ' https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation'
      ' instructions.'	)
  raise

 if not is_sharded:
  UpperCamelCase_     :					Dict   	=os.path.abspath(__lowercase	)
  logger.info(F'''Loading PyTorch weights from {pt_path}'''	)

  UpperCamelCase_     :					Dict   	=torch.load(__lowercase						,	map_location='cpu'	)
  logger.info(F'''PyTorch checkpoint contains {sum(t.numel() for t in pt_state_dict.values()	):,} parameters.'''	)

  UpperCamelCase_     :					Optional[int]   	=convert_pytorch_state_dict_to_flax(__lowercase						,	__lowercase	)
 else:
  # model is sharded and pytorch_checkpoint_path already contains the list of .pt shard files
  UpperCamelCase_     :					List[Any]   	=convert_pytorch_sharded_state_dict_to_flax(__lowercase						,	__lowercase	)
 return flax_state_dict




def       A_  (				__lowercase						,	__lowercase						,	__lowercase						,	__lowercase						,	):

 def is_key_or_prefix_key_in_dict(__lowercase	) -> bool:
  return len(set(__lowercase	) & {key, (model_prefix,) + key}	) > 0

 # layer norm
 UpperCamelCase_     :					Dict   	=pt_tuple_key[:-1] + ('scale',)
 if pt_tuple_key[-1] in ["weight", "gamma"] and is_key_or_prefix_key_in_dict(__lowercase	):
  return renamed_pt_tuple_key, pt_tensor

 # batch norm layer mean
 UpperCamelCase_     :					Optional[Any]   	=pt_tuple_key[:-1] + ('mean',)
 if pt_tuple_key[-1] == "running_mean" and not is_key_or_prefix_key_in_dict(__lowercase	):
  return renamed_pt_tuple_key, pt_tensor

 # batch norm layer var
 UpperCamelCase_     :					Tuple   	=pt_tuple_key[:-1] + ('var',)
 if pt_tuple_key[-1] == "running_var" and not is_key_or_prefix_key_in_dict(__lowercase	):
  return renamed_pt_tuple_key, pt_tensor

 # embedding
 UpperCamelCase_     :					List[str]   	=pt_tuple_key[:-1] + ('embedding',)
 if pt_tuple_key[-1] == "weight" and is_key_or_prefix_key_in_dict(__lowercase	):
  return renamed_pt_tuple_key, pt_tensor

 # conv layer
 UpperCamelCase_     :					int   	=pt_tuple_key[:-1] + ('kernel',)
 if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4 and not is_key_or_prefix_key_in_dict(__lowercase	):
  UpperCamelCase_     :					Union[str, Any]   	=pt_tensor.transpose(2						,	3						,	1						,	0	)
  return renamed_pt_tuple_key, pt_tensor

 # linear layer
 UpperCamelCase_     :					Optional[int]   	=pt_tuple_key[:-1] + ('kernel',)
 if pt_tuple_key[-1] == "weight" and not is_key_or_prefix_key_in_dict(__lowercase	):
  UpperCamelCase_     :					List[Any]   	=pt_tensor.T
  return renamed_pt_tuple_key, pt_tensor

 # old PyTorch layer norm weight
 UpperCamelCase_     :					Union[str, Any]   	=pt_tuple_key[:-1] + ('weight',)
 if pt_tuple_key[-1] == "gamma":
  return renamed_pt_tuple_key, pt_tensor

 # old PyTorch layer norm bias
 UpperCamelCase_     :					Dict   	=pt_tuple_key[:-1] + ('bias',)
 if pt_tuple_key[-1] == "beta":
  return renamed_pt_tuple_key, pt_tensor

 # New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
 UpperCamelCase_     :					List[str]   	=None
 if pt_tuple_key[-3::2] == ("parametrizations", "original0"):
  UpperCamelCase_     :					Optional[int]   	=pt_tuple_key[-2] + '_g'
 elif pt_tuple_key[-3::2] == ("parametrizations", "original1"):
  UpperCamelCase_     :					Optional[int]   	=pt_tuple_key[-2] + '_v'
 if name is not None:
  UpperCamelCase_     :					Tuple   	=pt_tuple_key[:-3] + (name,)
  return renamed_pt_tuple_key, pt_tensor

 return pt_tuple_key, pt_tensor




def       A_  (				__lowercase						,	__lowercase	):
 # convert pytorch tensor to numpy
 UpperCamelCase_     :					str   	={k: v.numpy() for k, v in pt_state_dict.items()}

 UpperCamelCase_     :					Tuple   	=flax_model.base_model_prefix

 # use params dict if the model contains batch norm layers
 if "params" in flax_model.params:
  UpperCamelCase_     :					List[Any]   	=flax_model.params['params']
 else:
  UpperCamelCase_     :					Any   	=flax_model.params
 UpperCamelCase_     :					Union[str, Any]   	=flatten_dict(__lowercase	)

 # add batch_stats keys,values to dict
 if "batch_stats" in flax_model.params:
  UpperCamelCase_     :					Tuple   	=flatten_dict(flax_model.params['batch_stats']	)
  random_flax_state_dict.update(__lowercase	)

 UpperCamelCase_     :					Any   	={}

 UpperCamelCase_     :					int   	=(model_prefix not in flax_model_params) and (
     model_prefix in {k.split('.'	)[0] for k in pt_state_dict.keys()}
 )
 UpperCamelCase_     :					Dict   	=(model_prefix in flax_model_params) and (
     model_prefix not in {k.split('.'	)[0] for k in pt_state_dict.keys()}
 )

 # Need to change some parameters name to match Flax names
 for pt_key, pt_tensor in pt_state_dict.items():
  UpperCamelCase_     :					Tuple   	=tuple(pt_key.split('.'	)	)

  # remove base model prefix if necessary
  UpperCamelCase_     :					List[str]   	=pt_tuple_key[0] == model_prefix
  if load_model_with_head_into_base_model and has_base_model_prefix:
   UpperCamelCase_     :					Any   	=pt_tuple_key[1:]

  # Correctly rename weight parameters
  UpperCamelCase_	,     UpperCamelCase_     :					Dict   	=rename_key_and_reshape_tensor(
      __lowercase						,	__lowercase						,	__lowercase						,	__lowercase	)

  # add model prefix if necessary
  UpperCamelCase_     :					List[Any]   	=(model_prefix,) + flax_key in random_flax_state_dict
  if load_base_model_into_model_with_head and require_base_model_prefix:
   UpperCamelCase_     :					str   	=(model_prefix,) + flax_key

  if flax_key in random_flax_state_dict:
   if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
    raise ValueError(
        F'''PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '''
        F'''{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'''	)

        # add batch stats if the model contains batchnorm layers
  if "batch_stats" in flax_model.params:
   if "mean" in flax_key[-1] or "var" in flax_key[-1]:
    UpperCamelCase_     :					Dict   	=jnp.asarray(__lowercase	)
    continue
   # remove num_batches_tracked key
   if "num_batches_tracked" in flax_key[-1]:
    flax_state_dict.pop(__lowercase						,	__lowercase	)
    continue

   # also add unexpected weight so that warning is thrown
   UpperCamelCase_     :					Optional[int]   	=jnp.asarray(__lowercase	)

  else:
   # also add unexpected weight so that warning is thrown
   UpperCamelCase_     :					List[Any]   	=jnp.asarray(__lowercase	)

 return unflatten_dict(__lowercase	)




def       A_  (				__lowercase						,	__lowercase	):
 import torch

 # Load the index
 UpperCamelCase_     :					List[str]   	={}
 for shard_file in shard_filenames:
  # load using msgpack utils
  UpperCamelCase_     :					Optional[Any]   	=torch.load(__lowercase	)
  UpperCamelCase_     :					List[str]   	={k: v.numpy() for k, v in pt_state_dict.items()}

  UpperCamelCase_     :					List[str]   	=flax_model.base_model_prefix

  # use params dict if the model contains batch norm layers and then add batch_stats keys,values to dict
  if "batch_stats" in flax_model.params:
   UpperCamelCase_     :					Any   	=flax_model.params['params']

   UpperCamelCase_     :					Dict   	=flatten_dict(__lowercase	)
   random_flax_state_dict.update(flatten_dict(flax_model.params['batch_stats']	)	)
  else:
   UpperCamelCase_     :					Optional[int]   	=flax_model.params
   UpperCamelCase_     :					Dict   	=flatten_dict(__lowercase	)

  UpperCamelCase_     :					Optional[int]   	=(model_prefix not in flax_model_params) and (
      model_prefix in {k.split('.'	)[0] for k in pt_state_dict.keys()}
  )
  UpperCamelCase_     :					List[Any]   	=(model_prefix in flax_model_params) and (
      model_prefix not in {k.split('.'	)[0] for k in pt_state_dict.keys()}
  )
  # Need to change some parameters name to match Flax names
  for pt_key, pt_tensor in pt_state_dict.items():
   UpperCamelCase_     :					List[str]   	=tuple(pt_key.split('.'	)	)

   # remove base model prefix if necessary
   UpperCamelCase_     :					int   	=pt_tuple_key[0] == model_prefix
   if load_model_with_head_into_base_model and has_base_model_prefix:
    UpperCamelCase_     :					List[Any]   	=pt_tuple_key[1:]

   # Correctly rename weight parameters
   UpperCamelCase_	,     UpperCamelCase_     :					str   	=rename_key_and_reshape_tensor(
       __lowercase						,	__lowercase						,	__lowercase						,	__lowercase	)
   # add model prefix if necessary
   UpperCamelCase_     :					Dict   	=(model_prefix,) + flax_key in random_flax_state_dict
   if load_base_model_into_model_with_head and require_base_model_prefix:
    UpperCamelCase_     :					Optional[Any]   	=(model_prefix,) + flax_key

   if flax_key in random_flax_state_dict:
    if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
     raise ValueError(
         F'''PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '''
         F'''{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'''	)

            # add batch stats if the model contains batchnorm layers
   if "batch_stats" in flax_model.params:
    if "mean" in flax_key[-1]:
     UpperCamelCase_     :					int   	=jnp.asarray(__lowercase	)
     continue
    if "var" in flax_key[-1]:
     UpperCamelCase_     :					Tuple   	=jnp.asarray(__lowercase	)
     continue
    # remove num_batches_tracked key
    if "num_batches_tracked" in flax_key[-1]:
     flax_state_dict.pop(__lowercase						,	__lowercase	)
     continue

    # also add unexpected weight so that warning is thrown
    UpperCamelCase_     :					Dict   	=jnp.asarray(__lowercase	)

   else:
    # also add unexpected weight so that warning is thrown
    UpperCamelCase_     :					int   	=jnp.asarray(__lowercase	)
 return unflatten_dict(__lowercase	)




def       A_  (				__lowercase						,	__lowercase	):
 UpperCamelCase_     :					Any   	=os.path.abspath(__lowercase	)
 logger.info(F'''Loading Flax weights from {flax_checkpoint_path}'''	)

 # import correct flax class
 UpperCamelCase_     :					Any   	=getattr(__lowercase						,	'Flax' + model.__class__.__name__	)

 # load flax weight dict
 with open(__lowercase						,	'rb'	) as state_f:
  try:
   UpperCamelCase_     :					Dict   	=from_bytes(__lowercase						,	state_f.read()	)
  except UnpicklingError:
   raise EnvironmentError(F'''Unable to convert {flax_checkpoint_path} to Flax deserializable object. '''	)

 return load_flax_weights_in_pytorch_model(__lowercase						,	__lowercase	)





def       A_  (				__lowercase						,	__lowercase	):

 try:
  import torch  # noqa: F401
 except ImportError:
  logger.error(
      'Loading a Flax weights in PyTorch, requires both PyTorch and Flax to be installed. Please see'
      ' https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation'
      ' instructions.'	)
  raise

 # check if we have bf16 weights
 UpperCamelCase_     :					List[str]   	=flatten_dict(jax.tree_util.tree_map(lambda __lowercase	: x.dtype == jnp.bfloataa						,	__lowercase	)	).values()
 if any(__lowercase	):
  # convert all weights to fp32 if the are bf16 since torch.from_numpy can-not handle bf16
  # and bf16 is not fully supported in PT yet.
  logger.warning(
      'Found ``bfloat16`` weights in Flax model. Casting all ``bfloat16`` weights to ``float32`` '
      'before loading those in PyTorch model.'	)
  UpperCamelCase_     :					int   	=jax.tree_util.tree_map(
      lambda __lowercase	: params.astype(np.floataa	) if params.dtype == jnp.bfloataa else params						,	__lowercase	)

 UpperCamelCase_     :					int   	=flatten_dict(__lowercase	)
 UpperCamelCase_     :					Optional[Any]   	=pt_model.state_dict()

 UpperCamelCase_     :					List[Any]   	=(pt_model.base_model_prefix in flax_state) and (
     pt_model.base_model_prefix not in {k.split('.'	)[0] for k in pt_model_dict.keys()}
 )
 UpperCamelCase_     :					List[str]   	=(pt_model.base_model_prefix not in flax_state) and (
     pt_model.base_model_prefix in {k.split('.'	)[0] for k in pt_model_dict.keys()}
 )

 # keep track of unexpected & missing keys
 UpperCamelCase_     :					Dict   	=[]
 UpperCamelCase_     :					Union[str, Any]   	=set(pt_model_dict.keys()	)

 for flax_key_tuple, flax_tensor in flax_state_dict.items():
  UpperCamelCase_     :					Optional[int]   	=flax_key_tuple[0] == pt_model.base_model_prefix
  UpperCamelCase_     :					List[Any]   	='.'.join((pt_model.base_model_prefix,) + flax_key_tuple	) in pt_model_dict

  # adapt flax_key to prepare for loading from/to base model only
  if load_model_with_head_into_base_model and has_base_model_prefix:
   UpperCamelCase_     :					Any   	=flax_key_tuple[1:]
  elif load_base_model_into_model_with_head and require_base_model_prefix:
   UpperCamelCase_     :					Optional[int]   	=(pt_model.base_model_prefix,) + flax_key_tuple

  # rename flax weights to PyTorch format
  if flax_key_tuple[-1] == "kernel" and flax_tensor.ndim == 4 and ".".join(__lowercase	) not in pt_model_dict:
   # conv layer
   UpperCamelCase_     :					Optional[Any]   	=flax_key_tuple[:-1] + ('weight',)
   UpperCamelCase_     :					Union[str, Any]   	=jnp.transpose(__lowercase						,	(3, 2, 0, 1)	)
  elif flax_key_tuple[-1] == "kernel" and ".".join(__lowercase	) not in pt_model_dict:
   # linear layer
   UpperCamelCase_     :					Optional[Any]   	=flax_key_tuple[:-1] + ('weight',)
   UpperCamelCase_     :					int   	=flax_tensor.T
  elif flax_key_tuple[-1] in ["scale", "embedding"]:
   UpperCamelCase_     :					Optional[Any]   	=flax_key_tuple[:-1] + ('weight',)

  # adding batch stats from flax batch norm to pt
  elif "mean" in flax_key_tuple[-1]:
   UpperCamelCase_     :					List[Any]   	=flax_key_tuple[:-1] + ('running_mean',)
  elif "var" in flax_key_tuple[-1]:
   UpperCamelCase_     :					Union[str, Any]   	=flax_key_tuple[:-1] + ('running_var',)

  if "batch_stats" in flax_state:
   UpperCamelCase_     :					Optional[Any]   	='.'.join(flax_key_tuple[1:]	)  # Remove the params/batch_stats header
  else:
   UpperCamelCase_     :					Union[str, Any]   	='.'.join(__lowercase	)

  # We also need to look at `pt_model_dict` and see if there are keys requiring further transformation.
  UpperCamelCase_     :					List[Any]   	={}
  # New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
  for key in pt_model_dict:
   UpperCamelCase_     :					List[str]   	=key.split('.'	)
   UpperCamelCase_     :					Union[str, Any]   	=None
   if key_components[-3::2] == ["parametrizations", "original0"]:
    UpperCamelCase_     :					List[str]   	=key_components[-2] + '_g'
   elif key_components[-3::2] == ["parametrizations", "original1"]:
    UpperCamelCase_     :					Any   	=key_components[-2] + '_v'
   if name is not None:
    UpperCamelCase_     :					List[Any]   	=key_components[:-3] + [name]
    UpperCamelCase_     :					Union[str, Any]   	='.'.join(__lowercase	)
    UpperCamelCase_     :					List[Any]   	=key

  if flax_key in special_pt_names:
   UpperCamelCase_     :					str   	=special_pt_names[flax_key]

  if flax_key in pt_model_dict:
   if flax_tensor.shape != pt_model_dict[flax_key].shape:
    raise ValueError(
        F'''Flax checkpoint seems to be incorrect. Weight {flax_key_tuple} was expected '''
        F'''to be of shape {pt_model_dict[flax_key].shape}, but is {flax_tensor.shape}.'''	)
   else:
    # add weight to pytorch dict
    UpperCamelCase_     :					Optional[Any]   	=np.asarray(__lowercase	) if not isinstance(__lowercase						,	np.ndarray	) else flax_tensor
    UpperCamelCase_     :					List[Any]   	=torch.from_numpy(__lowercase	)
    # remove from missing keys
    missing_keys.remove(__lowercase	)
  else:
   # weight is not expected by PyTorch model
   unexpected_keys.append(__lowercase	)

 pt_model.load_state_dict(__lowercase	)

 # re-transform missing_keys to list
 UpperCamelCase_     :					int   	=list(__lowercase	)

 if len(__lowercase	) > 0:
  logger.warning(
      'Some weights of the Flax model were not used when initializing the PyTorch model'
      F''' {pt_model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are initializing'''
      F''' {pt_model.__class__.__name__} from a Flax model trained on another task or with another architecture'''
      ' (e.g. initializing a BertForSequenceClassification model from a FlaxBertForPreTraining model).\n- This'
      F''' IS NOT expected if you are initializing {pt_model.__class__.__name__} from a Flax model that you expect'''
      ' to be exactly identical (e.g. initializing a BertForSequenceClassification model from a'
      ' FlaxBertForSequenceClassification model).'	)
 else:
  logger.warning(F'''All Flax model weights were used when initializing {pt_model.__class__.__name__}.\n'''	)
 if len(__lowercase	) > 0:
  logger.warning(
      F'''Some weights of {pt_model.__class__.__name__} were not initialized from the Flax model and are newly'''
      F''' initialized: {missing_keys}\nYou should probably TRAIN this model on a down-stream task to be able to'''
      ' use it for predictions and inference.'	)
 else:
  logger.warning(
      F'''All the weights of {pt_model.__class__.__name__} were initialized from the Flax model.\n'''
      'If your task is similar to the task the model of the checkpoint was trained on, '
      F'''you can already use {pt_model.__class__.__name__} for predictions without further training.'''	)

 return pt_model


"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__SCREAMING_SNAKE_CASE							=     logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							=     {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__SCREAMING_SNAKE_CASE							=     [
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

__SCREAMING_SNAKE_CASE							=     {
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
__SCREAMING_SNAKE_CASE							=     {F"""funnel-transformer/{name}""": 512 for name in _model_names}
__SCREAMING_SNAKE_CASE							=     {F"""funnel-transformer/{name}""": {'do_lower_case': True} for name in _model_names}





class 				a__					(						A__       ):
     UpperCAmelCase__         =      VOCAB_FILES_NAMES
     UpperCAmelCase__         =      PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__         =      PRETRAINED_INIT_CONFIGURATION
     UpperCAmelCase__         =      FunnelTokenizer
     UpperCAmelCase__         =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__         =      2



     def __init__(							self :str      ,  _lowerCamelCase :int=None      ,  _lowerCamelCase :List[Any]=None      ,  _lowerCamelCase :Dict=True      ,  _lowerCamelCase :Optional[Any]="<unk>"      ,  _lowerCamelCase :Optional[Any]="<sep>"      ,  _lowerCamelCase :List[Any]="<pad>"      ,  _lowerCamelCase :List[Any]="<cls>"      ,  _lowerCamelCase :Union[str, Any]="<mask>"      ,  _lowerCamelCase :Any="<s>"      ,  _lowerCamelCase :Union[str, Any]="</s>"      ,  _lowerCamelCase :str=True      ,  _lowerCamelCase :Dict=True      ,  _lowerCamelCase :Tuple=None      ,  _lowerCamelCase :Tuple="##"      ,  **_lowerCamelCase :int      ,  ):







      '''simple docstring'''



      super().__init__(
          _lowerCamelCase      ,  tokenizer_file=_lowerCamelCase      ,  do_lower_case=_lowerCamelCase      ,  unk_token=_lowerCamelCase      ,  sep_token=_lowerCamelCase      ,  pad_token=_lowerCamelCase      ,  cls_token=_lowerCamelCase      ,  mask_token=_lowerCamelCase      ,  bos_token=_lowerCamelCase      ,  eos_token=_lowerCamelCase      ,  clean_text=_lowerCamelCase      ,  tokenize_chinese_chars=_lowerCamelCase      ,  strip_accents=_lowerCamelCase      ,  wordpieces_prefix=_lowerCamelCase      ,  **_lowerCamelCase      ,  )

      UpperCamelCase_     :					str   	=json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
      if (
          normalizer_state.get('lowercase'      ,  _lowerCamelCase   ) != do_lower_case
          or normalizer_state.get('strip_accents'      ,  _lowerCamelCase   ) != strip_accents
          or normalizer_state.get('handle_chinese_chars'      ,  _lowerCamelCase   ) != tokenize_chinese_chars
      ):
       UpperCamelCase_     :					int   	=getattr(_lowerCamelCase      ,  normalizer_state.pop('type'   )   )
       UpperCamelCase_     :					Dict   	=do_lower_case
       UpperCamelCase_     :					Tuple   	=strip_accents
       UpperCamelCase_     :					int   	=tokenize_chinese_chars
       UpperCamelCase_     :					Tuple   	=normalizer_class(**_lowerCamelCase   )

      UpperCamelCase_     :					Dict   	=do_lower_case



     def  lowerCamelCase_						(							self :Optional[int]      ,  _lowerCamelCase :str      ,  _lowerCamelCase :Optional[int]=None   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

      return output



     def  lowerCamelCase_						(							self :List[Any]      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[int]   	=[self.sep_token_id]
      UpperCamelCase_     :					List[Any]   	=[self.cls_token_id]
      if token_ids_a is None:
       return len(cls   ) * [self.cls_token_type_id] + len(token_ids_a + sep   ) * [0]
      return len(cls   ) * [self.cls_token_type_id] + len(token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]




     def  lowerCamelCase_						(							self :Optional[Any]      ,  _lowerCamelCase :str      ,  _lowerCamelCase :Optional[str] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=self._tokenizer.model.save(_lowerCamelCase      ,  name=_lowerCamelCase   )
      return tuple(_lowerCamelCase   )
