


'''simple docstring'''


import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase 							=  logging.get_logger(__name__)


_UpperCamelCase 							=  {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

_UpperCamelCase 							=  {
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
}

_UpperCamelCase 							=  {
    """allenai/longformer-base-4096""": 4096,
    """allenai/longformer-large-4096""": 4096,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4096,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4096,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4096,
}


@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def 		_lowercase ():




     '''simple docstring'''





     __A	:       List[str]      =							(
         list(range(ord("!"							)       ,					ord("~"							) + 1							)							) + list(range(ord("¡"							)       ,					ord("¬"							) + 1							)							) + list(range(ord("®"							)       ,					ord("ÿ"							) + 1							)							)
     )
     __A	:       List[Any]      =							bs[:]
     __A	:       Optional[int]      =							0
     for b in range(2**8							):
          if b not in bs:
               bs.append(SCREAMING_SNAKE_CASE							)
               cs.append(2**8 + n							)
               n += 1
     __A	:       Optional[Any]      =							[chr(SCREAMING_SNAKE_CASE							) for n in cs]
     return dict(zip(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							)							)


def 		_lowercase (SCREAMING_SNAKE_CASE							):




     '''simple docstring'''





     __A	:       Any      =							set()
     __A	:       List[str]      =							word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)							)
          __A	:       Optional[Any]      =							char
     return pairs







class 							__magic_name__	(       lowerCAmelCase ):

     """simple docstring"""






     lowerCamelCase__       						=				VOCAB_FILES_NAMES
     lowerCamelCase__       						=				PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase__       						=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCamelCase__       						=				['input_ids', 'attention_mask']


     def __init__(   self			,     lowerCamelCase			,     lowerCamelCase			,     lowerCamelCase="replace"			,     lowerCamelCase="<s>"			,     lowerCamelCase="</s>"			,     lowerCamelCase="</s>"			,     lowerCamelCase="<s>"			,     lowerCamelCase="<unk>"			,     lowerCamelCase="<pad>"			,     lowerCamelCase="<mask>"			,     lowerCamelCase=False			,     **lowerCamelCase			,     ):


          '''simple docstring'''





          __A	:       Any      =							AddedToken(lowerCamelCase			,     lstrip=lowerCamelCase			,     rstrip=lowerCamelCase		) if isinstance(lowerCamelCase			,     lowerCamelCase		) else bos_token
          __A	:       int      =							AddedToken(lowerCamelCase			,     lstrip=lowerCamelCase			,     rstrip=lowerCamelCase		) if isinstance(lowerCamelCase			,     lowerCamelCase		) else eos_token
          __A	:       Optional[Any]      =							AddedToken(lowerCamelCase			,     lstrip=lowerCamelCase			,     rstrip=lowerCamelCase		) if isinstance(lowerCamelCase			,     lowerCamelCase		) else sep_token
          __A	:       str      =							AddedToken(lowerCamelCase			,     lstrip=lowerCamelCase			,     rstrip=lowerCamelCase		) if isinstance(lowerCamelCase			,     lowerCamelCase		) else cls_token
          __A	:       List[str]      =							AddedToken(lowerCamelCase			,     lstrip=lowerCamelCase			,     rstrip=lowerCamelCase		) if isinstance(lowerCamelCase			,     lowerCamelCase		) else unk_token
          __A	:       Optional[Any]      =							AddedToken(lowerCamelCase			,     lstrip=lowerCamelCase			,     rstrip=lowerCamelCase		) if isinstance(lowerCamelCase			,     lowerCamelCase		) else pad_token

          # Mask token behave like a normal word, i.e. include the space before it
          __A	:       str      =							AddedToken(lowerCamelCase			,     lstrip=lowerCamelCase			,     rstrip=lowerCamelCase		) if isinstance(lowerCamelCase			,     lowerCamelCase		) else mask_token

          super().__init__(
              errors=lowerCamelCase			,     bos_token=lowerCamelCase			,     eos_token=lowerCamelCase			,     unk_token=lowerCamelCase			,     sep_token=lowerCamelCase			,     cls_token=lowerCamelCase			,     pad_token=lowerCamelCase			,     mask_token=lowerCamelCase			,     add_prefix_space=lowerCamelCase			,     **lowerCamelCase			,     )

          with open(lowerCamelCase			,     encoding="utf-8"		) as vocab_handle:
               __A	:       Tuple      =							json.load(lowerCamelCase		)
          __A	:       Any      =							{v: k for k, v in self.encoder.items()}
          __A	:       Dict      =							errors  # how to handle errors in decoding
          __A	:       int      =							bytes_to_unicode()
          __A	:       int      =							{v: k for k, v in self.byte_encoder.items()}
          with open(lowerCamelCase			,     encoding="utf-8"		) as merges_handle:
               __A	:       List[str]      =							merges_handle.read().split("\n"		)[1:-1]
          __A	:       Any      =							[tuple(merge.split()		) for merge in bpe_merges]
          __A	:       int      =							dict(zip(lowerCamelCase			,     range(len(lowerCamelCase		)		)		)		)
          __A	:       int      =							{}
          __A	:       Optional[int]      =							add_prefix_space

          # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
          __A	:       Optional[int]      =							re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"		)


     @property
     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          return len(self.encoder		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          return dict(self.encoder			,     **self.added_tokens_encoder		)


     def lowerCAmelCase__  (   self			,     lowerCamelCase		):


          '''simple docstring'''





          if token in self.cache:
               return self.cache[token]
          __A	:       List[Any]      =							tuple(lowerCamelCase		)
          __A	:       Tuple      =							get_pairs(lowerCamelCase		)

          if not pairs:
               return token

          while True:
               __A	:       Union[str, Any]      =							min(lowerCamelCase			,     key=lambda lowerCamelCase		: self.bpe_ranks.get(lowerCamelCase			,     float("inf"		)		)		)
               if bigram not in self.bpe_ranks:
                    break
               __A  ,__A	:       Any      =							bigram
               __A	:       List[str]      =							[]
               __A	:       Optional[Any]      =							0
               while i < len(lowerCamelCase		):
                    try:
                         __A	:       Any      =							word.index(lowerCamelCase			,     lowerCamelCase		)
                    except ValueError:
                         new_word.extend(word[i:]		)
                         break
                    else:
                         new_word.extend(word[i:j]		)
                         __A	:       Union[str, Any]      =							j

                    if word[i] == first and i < len(lowerCamelCase		) - 1 and word[i + 1] == second:
                         new_word.append(first + second		)
                         i += 2
                    else:
                         new_word.append(word[i]		)
                         i += 1
               __A	:       Optional[int]      =							tuple(lowerCamelCase		)
               __A	:       List[Any]      =							new_word
               if len(lowerCamelCase		) == 1:
                    break
               else:
                    __A	:       Dict      =							get_pairs(lowerCamelCase		)
          __A	:       str      =							" ".join(lowerCamelCase		)
          __A	:       Dict      =							word
          return word


     def lowerCAmelCase__  (   self			,     lowerCamelCase		):


          '''simple docstring'''





          __A	:       Tuple      =							[]
          for token in re.findall(self.pat			,     lowerCamelCase		):
               __A	:       Any      =							"".join(
                   self.byte_encoder[b] for b in token.encode("utf-8"		)		)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCamelCase		).split(" "		)		)
          return bpe_tokens


     def lowerCAmelCase__  (   self			,     lowerCamelCase		):


          '''simple docstring'''





          return self.encoder.get(lowerCamelCase			,     self.encoder.get(self.unk_token		)		)


     def lowerCAmelCase__  (   self			,     lowerCamelCase		):


          '''simple docstring'''





          return self.decoder.get(lowerCamelCase		)


     def lowerCAmelCase__  (   self			,     lowerCamelCase		):


          '''simple docstring'''





          __A	:       Union[str, Any]      =							"".join(lowerCamelCase		)
          __A	:       str      =							bytearray([self.byte_decoder[c] for c in text]		).decode("utf-8"			,     errors=self.errors		)
          return text


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase = None		):


          '''simple docstring'''





          if not os.path.isdir(lowerCamelCase		):
               logger.error(f"Vocabulary path ({save_directory}) should be a directory"		)
               return
          __A	:       List[str]      =							os.path.join(
              lowerCamelCase			,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
          __A	:       List[Any]      =							os.path.join(
              lowerCamelCase			,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]		)

          with open(lowerCamelCase			,     "w"			,     encoding="utf-8"		) as f:
               f.write(json.dumps(self.encoder			,     indent=2			,     sort_keys=lowerCamelCase			,     ensure_ascii=lowerCamelCase		) + "\n"		)

          __A	:       List[Any]      =							0
          with open(lowerCamelCase			,     "w"			,     encoding="utf-8"		) as writer:
               writer.write("#version: 0.2\n"		)
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,     key=lambda lowerCamelCase		: kv[1]		):
                    if index != token_index:
                         logger.warning(
                             f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                             " Please check that the tokenizer is not corrupted!"		)
                         __A	:       int      =							token_index
                    writer.write(" ".join(lowerCamelCase		) + "\n"		)
                    index += 1

          return vocab_file, merge_file


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase = None		):


          '''simple docstring'''





          if token_ids_a is None:
               return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
          __A	:       Any      =							[self.cls_token_id]
          __A	:       List[Any]      =							[self.sep_token_id]
          return cls + token_ids_a + sep + sep + token_ids_a + sep


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase = None			,     lowerCamelCase = False		):


          '''simple docstring'''





          if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=lowerCamelCase			,     token_ids_a=lowerCamelCase			,     already_has_special_tokens=lowerCamelCase		)

          if token_ids_a is None:
               return [1] + ([0] * len(lowerCamelCase		)) + [1]
          return [1] + ([0] * len(lowerCamelCase		)) + [1, 1] + ([0] * len(lowerCamelCase		)) + [1]


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase = None		):


          '''simple docstring'''





          __A	:       List[Any]      =							[self.sep_token_id]
          __A	:       Dict      =							[self.cls_token_id]

          if token_ids_a is None:
               return len(cls + token_ids_a + sep		) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase=False			,     **lowerCamelCase		):


          '''simple docstring'''





          __A	:       Optional[Any]      =							kwargs.pop("add_prefix_space"			,     self.add_prefix_space		)
          if (is_split_into_words or add_prefix_space) and (len(lowerCamelCase		) > 0 and not text[0].isspace()):
               __A	:       int      =							" " + text
          return (text, kwargs)





'''simple docstring'''


import argparse

from transformers import BigBirdConfig, BigBirdForPreTraining, BigBirdForQuestionAnswering, load_tf_weights_in_big_bird
from transformers.utils import logging


logging.set_verbosity_info()


def 		_lowercase (SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE							):




     '''simple docstring'''





     __A	:       List[str]      =							BigBirdConfig.from_json_file(SCREAMING_SNAKE_CASE							)
     print(f"Building PyTorch model from configuration: {config}"							)

     if is_trivia_qa:
          __A	:       List[str]      =							BigBirdForQuestionAnswering(SCREAMING_SNAKE_CASE							)
     else:
          __A	:       List[Any]      =							BigBirdForPreTraining(SCREAMING_SNAKE_CASE							)

     # Load weights from tf checkpoint
     load_tf_weights_in_big_bird(SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					is_trivia_qa=SCREAMING_SNAKE_CASE							)

     # Save pytorch-model
     print(f"Save PyTorch model to {pytorch_dump_path}"							)
     model.save_pretrained(SCREAMING_SNAKE_CASE							)


if __name__ == "__main__":
      _UpperCamelCase 							=  argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """--tf_checkpoint_path""", default=None, type=str, required=True, help="""Path to the TensorFlow checkpoint path."""
      )
      parser.add_argument(
          """--big_bird_config_file""",
          default=None,
          type=str,
          required=True,
          help=(
              """The config json file corresponding to the pre-trained BERT model. \n"""
              """This specifies the model architecture."""
          ),
      )
      parser.add_argument(
          """--pytorch_dump_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
      )
      parser.add_argument(
          """--is_trivia_qa""", action="""store_true""", help="""Whether to convert a model with a trivia_qa head."""
      )
      _UpperCamelCase 							=  parser.parse_args()
      convert_tf_checkpoint_to_pytorch(
          args.tf_checkpoint_path, args.big_bird_config_file, args.pytorch_dump_path, args.is_trivia_qa
      )

