

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


UpperCamelCase__		 =     logging.get_logger(__name__)

UpperCamelCase__		 =     {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCamelCase__		 =     {
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

UpperCamelCase__		 =     {
    'junnyu/roformer_chinese_small': 1536,
    'junnyu/roformer_chinese_base': 1536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


UpperCamelCase__		 =     {
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}

class     __snake_case    (				UpperCamelCase__  ):
       """simple docstring"""







       __SCREAMING_SNAKE_CASE				    =							VOCAB_FILES_NAMES
       __SCREAMING_SNAKE_CASE				    =							PRETRAINED_VOCAB_FILES_MAP
       __SCREAMING_SNAKE_CASE				    =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __SCREAMING_SNAKE_CASE				    =							PRETRAINED_INIT_CONFIGURATION
       __SCREAMING_SNAKE_CASE				    =							RoFormerTokenizer


       def __init__(		self      , _UpperCamelCase=None      , _UpperCamelCase=None      , _UpperCamelCase=True      , _UpperCamelCase="[UNK]"      , _UpperCamelCase="[SEP]"      , _UpperCamelCase="[PAD]"      , _UpperCamelCase="[CLS]"      , _UpperCamelCase="[MASK]"      , _UpperCamelCase=True      , _UpperCamelCase=None      , **_UpperCamelCase      , )      ->							List[Any]:






              """simple docstring"""


              super().__init__(
                  __lowerCamelCase      , tokenizer_file=__lowerCamelCase      , do_lower_case=__lowerCamelCase      , unk_token=__lowerCamelCase      , sep_token=__lowerCamelCase      , pad_token=__lowerCamelCase      , cls_token=__lowerCamelCase      , mask_token=__lowerCamelCase      , tokenize_chinese_chars=__lowerCamelCase      , strip_accents=__lowerCamelCase      , **__lowerCamelCase      , )

              __snake_case       	=							json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
              if (
                  pre_tok_state.get("""lowercase"""      , __lowerCamelCase   ) != do_lower_case
                  or pre_tok_state.get("""strip_accents"""      , __lowerCamelCase   ) != strip_accents
              ):
                     __snake_case       	=							getattr(__lowerCamelCase      , pre_tok_state.pop("""type"""   )   )
                     __snake_case       	=							do_lower_case
                     __snake_case       	=							strip_accents
                     __snake_case       	=							pre_tok_class(**__lowerCamelCase   )

              __snake_case       	=							do_lower_case


       def __getstate__(		self   )      ->							Union[str, Any]:






              """simple docstring"""


              __snake_case       	=							self.__dict__.copy()
              __snake_case       	=							BertPreTokenizer()
              return state


       def __setstate__(		self      , _UpperCamelCase   )      ->							int:






              """simple docstring"""


              __snake_case       	=							d
              __snake_case       	=							self.__dict__["""_tokenizer"""].get_vocab()
              __snake_case       	=							PreTokenizer.custom(JiebaPreTokenizer(__lowerCamelCase   )   )


       def 			a		(		self      , _UpperCamelCase      , _UpperCamelCase=None   )      ->							str:






              """simple docstring"""


              __snake_case       	=							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

              if token_ids_a:
                     output += token_ids_a + [self.sep_token_id]

              return output


       def 			a		(		self      , _UpperCamelCase      , _UpperCamelCase = None   )      ->							Dict:






              """simple docstring"""


              __snake_case       	=							[self.sep_token_id]
              __snake_case       	=							[self.cls_token_id]
              if token_ids_a is None:
                     return len(cls + token_ids_a + sep   ) * [0]
              return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]


       def 			a		(		self      , _UpperCamelCase      , _UpperCamelCase = None   )      ->							Tuple:






              """simple docstring"""


              __snake_case       	=							self._tokenizer.model.save(__lowerCamelCase      , name=__lowerCamelCase   )
              return tuple(__lowerCamelCase   )






       def 			a		(		self      , _UpperCamelCase      , _UpperCamelCase=None      , _UpperCamelCase=None      , _UpperCamelCase=False      , **_UpperCamelCase      , )      ->							Optional[int]:






              """simple docstring"""


              __snake_case       	=							BertPreTokenizer()
              return super().save_pretrained(__lowerCamelCase      , __lowerCamelCase      , __lowerCamelCase      , __lowerCamelCase      , **__lowerCamelCase   )








import argparse

from transformers import BigBirdConfig, BigBirdForPreTraining, BigBirdForQuestionAnswering, load_tf_weights_in_big_bird
from transformers.utils import logging


logging.set_verbosity_info()


def       UpperCamelCase__(			UpperCamelCase__     :			Any					,	UpperCamelCase__     :			Optional[Any]					,	UpperCamelCase__     :			Dict					,	UpperCamelCase__     :			int    )->Union[str, Any]:
       # Initialise PyTorch model
       A__		   =		BigBirdConfig.from_json_file(UpperCamelCase__    )
       print(f"Building PyTorch model from configuration: {config}"    )

       if is_trivia_qa:
              A__		   =		BigBirdForQuestionAnswering(UpperCamelCase__    )
       else:
              A__		   =		BigBirdForPreTraining(UpperCamelCase__    )

       # Load weights from tf checkpoint
       load_tf_weights_in_big_bird(UpperCamelCase__					,	UpperCamelCase__					,	is_trivia_qa=UpperCamelCase__    )

       # Save pytorch-model
       print(f"Save PyTorch model to {pytorch_dump_path}"    )
       model.save_pretrained(UpperCamelCase__    )


if __name__ == "__main__":
       a__: Optional[int]							   =						argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
       )
       parser.add_argument(
           '--big_bird_config_file',
           default=None,
           type=str,
           required=True,
           help=(
               'The config json file corresponding to the pre-trained BERT model. \n'
               'This specifies the model architecture.'
           ),
       )
       parser.add_argument(
           '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
       )
       parser.add_argument(
           '--is_trivia_qa', action='store_true', help='Whether to convert a model with a trivia_qa head.'
       )
       a__: Tuple							   =						parser.parse_args()
       convert_tf_checkpoint_to_pytorch(
           args.tf_checkpoint_path, args.big_bird_config_file, args.pytorch_dump_path, args.is_trivia_qa
       )





