

"""simple docstring"""






import argparse

import torch

from transformers import (
    SpeechTaConfig,
    SpeechTaFeatureExtractor,
    SpeechTaForSpeechToSpeech,
    SpeechTaForSpeechToText,
    SpeechTaForTextToSpeech,
    SpeechTaProcessor,
    SpeechTaTokenizer,
    logging,
)
from transformers.tokenization_utils import AddedToken


logging.set_verbosity_info()
__lowerCamelCase										=		logging.get_logger("transformers.models.speecht5")

__lowerCamelCase										=		{
    "speech_encoder_prenet.layer_norm": "speecht5.encoder.prenet.feature_projection.layer_norm",
    "speech_encoder_prenet.post_extract_proj": "speecht5.encoder.prenet.feature_projection.projection",
    "speech_encoder_prenet.pos_conv.0": "speecht5.encoder.prenet.pos_conv_embed.conv",
    "speech_encoder_prenet.mask_emb": "speecht5.encoder.prenet.masked_spec_embed",
}
__lowerCamelCase										=		{
    "text_encoder_prenet.encoder_prenet.0": "speecht5.encoder.prenet.embed_tokens",
    "text_encoder_prenet.encoder_prenet.1.alpha": "speecht5.encoder.prenet.encode_positions.alpha",
}
__lowerCamelCase										=		{
    "speech_decoder_prenet.decoder_prenet.0.0.prenet.0.0": "speecht5.decoder.prenet.layers.0",
    "speech_decoder_prenet.decoder_prenet.0.0.prenet.1.0": "speecht5.decoder.prenet.layers.1",
    "speech_decoder_prenet.decoder_prenet.0.1": "speecht5.decoder.prenet.final_layer",
    "speech_decoder_prenet.decoder_prenet.1.alpha": "speecht5.decoder.prenet.encode_positions.alpha",
    "speech_decoder_prenet.spkembs_layer.0": "speecht5.decoder.prenet.speaker_embeds_layer",
}
__lowerCamelCase										=		{
    "speech_decoder_postnet.feat_out": "speech_decoder_postnet.feat_out",
    "speech_decoder_postnet.prob_out": "speech_decoder_postnet.prob_out",
    "speech_decoder_postnet.postnet.postnet.0.0": "speech_decoder_postnet.layers.0.conv",
    "speech_decoder_postnet.postnet.postnet.0.1": "speech_decoder_postnet.layers.0.batch_norm",
    "speech_decoder_postnet.postnet.postnet.1.0": "speech_decoder_postnet.layers.1.conv",
    "speech_decoder_postnet.postnet.postnet.1.1": "speech_decoder_postnet.layers.1.batch_norm",
    "speech_decoder_postnet.postnet.postnet.2.0": "speech_decoder_postnet.layers.2.conv",
    "speech_decoder_postnet.postnet.postnet.2.1": "speech_decoder_postnet.layers.2.batch_norm",
    "speech_decoder_postnet.postnet.postnet.3.0": "speech_decoder_postnet.layers.3.conv",
    "speech_decoder_postnet.postnet.postnet.3.1": "speech_decoder_postnet.layers.3.batch_norm",
    "speech_decoder_postnet.postnet.postnet.4.0": "speech_decoder_postnet.layers.4.conv",
    "speech_decoder_postnet.postnet.postnet.4.1": "speech_decoder_postnet.layers.4.batch_norm",
}
__lowerCamelCase										=		{
    "text_decoder_prenet.embed_tokens": "speecht5.decoder.prenet.embed_tokens",
}
__lowerCamelCase										=		{
    "text_decoder_postnet.output_projection": "text_decoder_postnet.lm_head",
}
__lowerCamelCase										=		{
    "encoder.layers.*.self_attn.k_proj": "speecht5.encoder.wrapped_encoder.layers.*.attention.k_proj",
    "encoder.layers.*.self_attn.v_proj": "speecht5.encoder.wrapped_encoder.layers.*.attention.v_proj",
    "encoder.layers.*.self_attn.q_proj": "speecht5.encoder.wrapped_encoder.layers.*.attention.q_proj",
    "encoder.layers.*.self_attn.out_proj": "speecht5.encoder.wrapped_encoder.layers.*.attention.out_proj",
    "encoder.layers.*.self_attn_layer_norm": "speecht5.encoder.wrapped_encoder.layers.*.layer_norm",
    "encoder.layers.*.fc1": "speecht5.encoder.wrapped_encoder.layers.*.feed_forward.intermediate_dense",
    "encoder.layers.*.fc2": "speecht5.encoder.wrapped_encoder.layers.*.feed_forward.output_dense",
    "encoder.layers.*.final_layer_norm": "speecht5.encoder.wrapped_encoder.layers.*.final_layer_norm",
    "encoder.layer_norm": "speecht5.encoder.wrapped_encoder.layer_norm",
    "encoder.pos_emb.pe_k": "speecht5.encoder.wrapped_encoder.embed_positions.pe_k",
}
__lowerCamelCase										=		{
    "decoder.layers.*.self_attn.k_proj": "speecht5.decoder.wrapped_decoder.layers.*.self_attn.k_proj",
    "decoder.layers.*.self_attn.v_proj": "speecht5.decoder.wrapped_decoder.layers.*.self_attn.v_proj",
    "decoder.layers.*.self_attn.q_proj": "speecht5.decoder.wrapped_decoder.layers.*.self_attn.q_proj",
    "decoder.layers.*.self_attn.out_proj": "speecht5.decoder.wrapped_decoder.layers.*.self_attn.out_proj",
    "decoder.layers.*.self_attn_layer_norm": "speecht5.decoder.wrapped_decoder.layers.*.self_attn_layer_norm",
    "decoder.layers.*.encoder_attn.k_proj": "speecht5.decoder.wrapped_decoder.layers.*.encoder_attn.k_proj",
    "decoder.layers.*.encoder_attn.v_proj": "speecht5.decoder.wrapped_decoder.layers.*.encoder_attn.v_proj",
    "decoder.layers.*.encoder_attn.q_proj": "speecht5.decoder.wrapped_decoder.layers.*.encoder_attn.q_proj",
    "decoder.layers.*.encoder_attn.out_proj": "speecht5.decoder.wrapped_decoder.layers.*.encoder_attn.out_proj",
    "decoder.layers.*.encoder_attn_layer_norm": "speecht5.decoder.wrapped_decoder.layers.*.encoder_attn_layer_norm",
    "decoder.layers.*.fc1": "speecht5.decoder.wrapped_decoder.layers.*.feed_forward.intermediate_dense",
    "decoder.layers.*.fc2": "speecht5.decoder.wrapped_decoder.layers.*.feed_forward.output_dense",
    "decoder.layers.*.final_layer_norm": "speecht5.decoder.wrapped_decoder.layers.*.final_layer_norm",
}
__lowerCamelCase										=		{
    **MAPPING_SPEECH_ENCODER_PRENET,
    **MAPPING_ENCODER,
    **MAPPING_DECODER,
    **MAPPING_TEXT_DECODER_PRENET,
    **MAPPING_TEXT_DECODER_POSTNET,
}
__lowerCamelCase										=		{
    **MAPPING_TEXT_ENCODER_PRENET,
    **MAPPING_ENCODER,
    **MAPPING_DECODER,
    **MAPPING_SPEECH_DECODER_PRENET,
    **MAPPING_SPEECH_DECODER_POSTNET,
}
__lowerCamelCase										=		{
    **MAPPING_SPEECH_ENCODER_PRENET,
    **MAPPING_ENCODER,
    **MAPPING_DECODER,
    **MAPPING_SPEECH_DECODER_PRENET,
    **MAPPING_SPEECH_DECODER_POSTNET,
}
__lowerCamelCase										=		[]
__lowerCamelCase										=		[
    "encoder.version",
    "encoder.layers.*.norm_k.weight",
    "encoder.layers.*.norm_k.bias",
    "decoder.version",
    "decoder.layers.*.norm_k.weight",
    "decoder.layers.*.norm_k.bias",
    "decoder.pos_emb.pe_k",
    "speech_encoder_prenet.embed_positions._float_tensor",
    "text_decoder_prenet.embed_positions._float_tensor",
]
__lowerCamelCase										=		IGNORE_KEYS + [
    "encoder.proj",
    "text_encoder_prenet.*",
    "speech_decoder_prenet.*",
    "speech_decoder_postnet.*",
]
__lowerCamelCase										=		IGNORE_KEYS + [
    "encoder.proj",
    "speech_encoder_prenet.*",
    "text_decoder_prenet.*",
    "text_decoder_postnet.*",
]
__lowerCamelCase										=		IGNORE_KEYS + [
    "encoder.proj",
    "text_encoder_prenet.*",
    "text_decoder_prenet.*",
    "text_decoder_postnet.*",
]





def 	lowercase      (   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase						) ->      List[Any]:
 for attribute in key.split('''.'''						):
  __magic_name__     =      getattr(__UpperCamelCase      ,   __UpperCamelCase						)

 if weight_type is not None:
  __magic_name__     =      getattr(__UpperCamelCase      ,   __UpperCamelCase						).shape
 else:
  __magic_name__     =      hf_pointer.shape

 if hf_shape != value.shape:
  raise ValueError(
      f'''Shape of hf {key + "." + weight_type if weight_type is not None else ""} is {hf_shape}, but should be'''
      f''' {value.shape} for {full_name}'''						)

 if weight_type == "weight":
  __magic_name__     =      value
 elif weight_type == "weight_g":
  __magic_name__     =      value
 elif weight_type == "weight_v":
  __magic_name__     =      value
 elif weight_type == "bias":
  __magic_name__     =      value
 elif weight_type == "running_mean":
  __magic_name__     =      value
 elif weight_type == "running_var":
  __magic_name__     =      value
 elif weight_type == "num_batches_tracked":
  __magic_name__     =      value
 else:
  __magic_name__     =      value

 logger.info(f'''{key + ("." + weight_type if weight_type is not None else "")} was initialized from {full_name}.'''						)





def 	lowercase      (   __UpperCamelCase      ,   __UpperCamelCase						) ->      Dict:
 for key in ignore_keys:
  if key.endswith('''.*'''						):
   if name.startswith(key[:-1]						):
    return True
  elif ".*." in key:
   __magic_name__   ,						__magic_name__     =      key.split('''.*.'''						)
   if prefix in name and suffix in name:
    return True
  elif key in name:
   return True
 return False





def 	lowercase      (   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase						) ->      Any:
 __magic_name__     =      []

 if task == "s2t":
  __magic_name__     =      hf_model.speechta.encoder.prenet.feature_encoder
  __magic_name__     =      MAPPING_S2T
  __magic_name__     =      IGNORE_KEYS_S2T
 elif task == "t2s":
  __magic_name__     =      None
  __magic_name__     =      MAPPING_T2S
  __magic_name__     =      IGNORE_KEYS_T2S
 elif task == "s2s":
  __magic_name__     =      hf_model.speechta.encoder.prenet.feature_encoder
  __magic_name__     =      MAPPING_S2S
  __magic_name__     =      IGNORE_KEYS_S2S
 else:
  raise ValueError(f'''Unsupported task: {task}'''						)

 for name, value in fairseq_dict.items():
  if should_ignore(__UpperCamelCase      ,   __UpperCamelCase						):
   logger.info(f'''{name} was ignored'''						)
   continue

  __magic_name__     =      False
  if "conv_layers" in name:
   load_conv_layer(
       __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   hf_model.config.feat_extract_norm == '''group'''      ,   )
   __magic_name__     =      True
  else:
   for key, mapped_key in MAPPING.items():
    # mapped_key = "speecht5." + mapped_key if mapped_key not in TOP_LEVEL_KEYS else mapped_key

    if "*" in key:
     __magic_name__   ,						__magic_name__     =      key.split('''.*.'''						)
     if prefix in name and suffix in name:
      __magic_name__     =      suffix

                # if key in name or key.split("w2v_model.")[-1] == name.split(".")[0]:
    if key in name:
     __magic_name__     =      True
     if "*" in mapped_key:
      __magic_name__     =      name.split(__UpperCamelCase						)[0].split('''.'''						)[-2]
      __magic_name__     =      mapped_key.replace('''*'''      ,   __UpperCamelCase						)
     if "weight_g" in name:
      __magic_name__     =      '''weight_g'''
     elif "weight_v" in name:
      __magic_name__     =      '''weight_v'''
     elif "bias" in name:
      __magic_name__     =      '''bias'''
     elif "weight" in name:
      __magic_name__     =      '''weight'''
     elif "running_mean" in name:
      __magic_name__     =      '''running_mean'''
     elif "running_var" in name:
      __magic_name__     =      '''running_var'''
     elif "num_batches_tracked" in name:
      __magic_name__     =      '''num_batches_tracked'''
     else:
      __magic_name__     =      None
     set_recursively(__UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase						)
    continue
  if not is_used:
   unused_weights.append(__UpperCamelCase						)

 logger.warning(f'''Unused weights: {unused_weights}'''						)





def 	lowercase      (   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase						) ->      Optional[int]:
 __magic_name__     =      full_name.split('''conv_layers.'''						)[-1]
 __magic_name__     =      name.split('''.'''						)
 __magic_name__     =      int(items[0]						)
 __magic_name__     =      int(items[1]						)

 if type_id == 0:
  if "bias" in name:
   if value.shape != feature_extractor.conv_layers[layer_id].conv.bias.data.shape:
    raise ValueError(
        f'''{full_name} has size {value.shape}, but'''
        f''' {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.'''						)
   __magic_name__     =      value
   logger.info(f'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''						)
  elif "weight" in name:
   if value.shape != feature_extractor.conv_layers[layer_id].conv.weight.data.shape:
    raise ValueError(
        f'''{full_name} has size {value.shape}, but'''
        f''' {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.'''						)
   __magic_name__     =      value
   logger.info(f'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''						)
 elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
  if "bias" in name:
   if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape:
    raise ValueError(
        f'''{full_name} has size {value.shape}, but'''
        f''' {feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape} was found.'''						)
   __magic_name__     =      value
   logger.info(f'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''						)
  elif "weight" in name:
   if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape:
    raise ValueError(
        f'''{full_name} has size {value.shape}, but'''
        f''' {feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape} was found.'''						)
   __magic_name__     =      value
   logger.info(f'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''						)
 else:
  unused_weights.append(__UpperCamelCase						)





@torch.no_grad()
def 	lowercase      (   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase      ,   __UpperCamelCase=None      ,   __UpperCamelCase=None      ,   __UpperCamelCase=None      ,   ) ->      Optional[Any]:
 if config_path is not None:
  __magic_name__     =      SpeechTaConfig.from_pretrained(__UpperCamelCase						)
 else:
  __magic_name__     =      SpeechTaConfig()

 if task == "s2t":
  __magic_name__     =      config.max_text_positions
  __magic_name__     =      SpeechTaForSpeechToText(__UpperCamelCase						)
 elif task == "t2s":
  __magic_name__     =      1876
  __magic_name__     =      600
  __magic_name__     =      config.max_speech_positions
  __magic_name__     =      SpeechTaForTextToSpeech(__UpperCamelCase						)
 elif task == "s2s":
  __magic_name__     =      1876
  __magic_name__     =      config.max_speech_positions
  __magic_name__     =      SpeechTaForSpeechToSpeech(__UpperCamelCase						)
 else:
  raise ValueError(f'''Unknown task name: {task}'''						)

 if vocab_path:
  __magic_name__     =      SpeechTaTokenizer(__UpperCamelCase      ,   model_max_length=config.max_text_positions						)

  # Mask token behaves like a normal word, i.e. include the space before it
  __magic_name__     =      AddedToken('''<mask>'''      ,   lstrip=__UpperCamelCase      ,   rstrip=__UpperCamelCase						)
  __magic_name__     =      mask_token
  tokenizer.add_special_tokens({'''mask_token''': mask_token}						)
  tokenizer.add_tokens(['''<ctc_blank>''']						)

 __magic_name__     =      SpeechTaFeatureExtractor()
 __magic_name__     =      SpeechTaProcessor(tokenizer=__UpperCamelCase      ,   feature_extractor=__UpperCamelCase						)
 processor.save_pretrained(__UpperCamelCase						)

 __magic_name__     =      torch.load(__UpperCamelCase						)
 recursively_load_weights(fairseq_checkpoint['''model''']      ,   __UpperCamelCase      ,   __UpperCamelCase						)

 model.save_pretrained(__UpperCamelCase						)

 if repo_id:
  print('''Pushing to the hub...'''						)
  processor.push_to_hub(__UpperCamelCase						)
  model.push_to_hub(__UpperCamelCase						)


if __name__ == "__main__":
 __lowerCamelCase										=		argparse.ArgumentParser()
 parser.add_argument(
     "--task",
     default="s2t",
     type=str,
     help="Type of the SpeechT5 model you'd like to convert. Should be one of 's2t', 't2s', 's2s'.",
 )
 parser.add_argument("--checkpoint_path", required=True, default=None, type=str, help="Path to fairseq checkpoint")
 parser.add_argument("--vocab_path", default=None, type=str, help="Path to SentencePiece model")
 parser.add_argument("--config_path", default=None, type=str, help="Path to hf config.json of model to convert")
 parser.add_argument(
     "--pytorch_dump_folder_path", required=True, default=None, type=str, help="Path to the output PyTorch model."
 )
 parser.add_argument(
     "--push_to_hub", default=None, type=str, help="Where to upload the converted model on the 🤗 hub."
 )

 __lowerCamelCase										=		parser.parse_args()
 convert_speechta_checkpoint(
     args.task,
     args.checkpoint_path,
     args.pytorch_dump_folder_path,
     args.config_path,
     args.vocab_path,
     args.push_to_hub,
 )




"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


__lowerCamelCase										=		logging.get_logger(__name__)

__lowerCamelCase										=		{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__lowerCamelCase										=		{
    "vocab_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"
        ),
        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"
        ),
        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"
        ),
        "bert-base-multilingual-cased": (
            "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-cased": (
            "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"
        ),
    },
}

__lowerCamelCase										=		{
    "bert-base-uncased": 5_12,
    "bert-large-uncased": 5_12,
    "bert-base-cased": 5_12,
    "bert-large-cased": 5_12,
    "bert-base-multilingual-uncased": 5_12,
    "bert-base-multilingual-cased": 5_12,
    "bert-base-chinese": 5_12,
    "bert-base-german-cased": 5_12,
    "bert-large-uncased-whole-word-masking": 5_12,
    "bert-large-cased-whole-word-masking": 5_12,
    "bert-large-uncased-whole-word-masking-finetuned-squad": 5_12,
    "bert-large-cased-whole-word-masking-finetuned-squad": 5_12,
    "bert-base-cased-finetuned-mrpc": 5_12,
    "bert-base-german-dbmdz-cased": 5_12,
    "bert-base-german-dbmdz-uncased": 5_12,
    "TurkuNLP/bert-base-finnish-cased-v1": 5_12,
    "TurkuNLP/bert-base-finnish-uncased-v1": 5_12,
    "wietsedv/bert-base-dutch-cased": 5_12,
}

__lowerCamelCase										=		{
    "bert-base-uncased": {"do_lower_case": True},
    "bert-large-uncased": {"do_lower_case": True},
    "bert-base-cased": {"do_lower_case": False},
    "bert-large-cased": {"do_lower_case": False},
    "bert-base-multilingual-uncased": {"do_lower_case": True},
    "bert-base-multilingual-cased": {"do_lower_case": False},
    "bert-base-chinese": {"do_lower_case": False},
    "bert-base-german-cased": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
}
class        _lowercase					(     __UpperCAmelCase     ):
     _lowerCamelCase     =       VOCAB_FILES_NAMES
     _lowerCamelCase     =       PRETRAINED_VOCAB_FILES_MAP
     _lowerCamelCase     =       PRETRAINED_INIT_CONFIGURATION
     _lowerCamelCase     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _lowerCamelCase     =       BertTokenizer







     def __init__(      self ,						UpperCamelCase_=None ,						UpperCamelCase_=None ,						UpperCamelCase_=True ,						UpperCamelCase_="[UNK]" ,						UpperCamelCase_="[SEP]" ,						UpperCamelCase_="[PAD]" ,						UpperCamelCase_="[CLS]" ,						UpperCamelCase_="[MASK]" ,						UpperCamelCase_=True ,						UpperCamelCase_=None ,						**UpperCamelCase_ ,						):
      super().__init__(
          UpperCamelCase_ ,						tokenizer_file=UpperCamelCase_ ,						do_lower_case=UpperCamelCase_ ,						unk_token=UpperCamelCase_ ,						sep_token=UpperCamelCase_ ,						pad_token=UpperCamelCase_ ,						cls_token=UpperCamelCase_ ,						mask_token=UpperCamelCase_ ,						tokenize_chinese_chars=UpperCamelCase_ ,						strip_accents=UpperCamelCase_ ,						**UpperCamelCase_ ,						)

      __magic_name__     =      json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
      if (
          normalizer_state.get('''lowercase''' ,						UpperCamelCase_       ) != do_lower_case
          or normalizer_state.get('''strip_accents''' ,						UpperCamelCase_       ) != strip_accents
          or normalizer_state.get('''handle_chinese_chars''' ,						UpperCamelCase_       ) != tokenize_chinese_chars
      ):
       __magic_name__     =      getattr(UpperCamelCase_ ,						normalizer_state.pop('''type'''       )       )
       __magic_name__     =      do_lower_case
       __magic_name__     =      strip_accents
       __magic_name__     =      tokenize_chinese_chars
       __magic_name__     =      normalizer_class(**UpperCamelCase_       )

      __magic_name__     =      do_lower_case







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_=None       ):
      __magic_name__     =      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

      return output







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_ = None       ):
      __magic_name__     =      [self.sep_token_id]
      __magic_name__     =      [self.cls_token_id]
      if token_ids_a is None:
       return len(cls + token_ids_a + sep       ) * [0]
      return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_ = None       ):
      __magic_name__     =      self._tokenizer.model.save(UpperCamelCase_ ,						name=UpperCamelCase_       )
      return tuple(UpperCamelCase_       )

