






"""simple docstring"""






import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_  :					List[str]       =					logging.get_logger(__name__)


UpperCAmelCase_  :					Dict       =					{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

UpperCAmelCase_  :					Tuple       =					{
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
}

UpperCAmelCase_  :					Dict       =					{
    """allenai/longformer-base-4096""": 4096,
    """allenai/longformer-large-4096""": 4096,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4096,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4096,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4096,
}



@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def    _A     ()					->			Dict:
    """simple docstring"""






    SCREAMING_SNAKE_CASE_ :    Optional[Any]       		=						(
        list(range(ord('''!'''				)       ,							ord('''~'''				) + 1				)				) + list(range(ord('''¡'''				)       ,							ord('''¬'''				) + 1				)				) + list(range(ord('''®'''				)       ,							ord('''ÿ'''				) + 1				)				)
    )
    SCREAMING_SNAKE_CASE_ :    Any       		=						bs[:]
    SCREAMING_SNAKE_CASE_ :    Optional[Any]       		=						0
    for b in range(2**8				):
        if b not in bs:
            bs.append(_lowerCamelCase				)
            cs.append(2**8 + n				)
            n += 1
    SCREAMING_SNAKE_CASE_ :    str       		=						[chr(_lowerCamelCase				) for n in cs]
    return dict(zip(_lowerCamelCase       ,							_lowerCamelCase				)				)



def    _A     (__a				)					->			List[Any]:
    """simple docstring"""






    SCREAMING_SNAKE_CASE_ :    Dict       		=						set()
    SCREAMING_SNAKE_CASE_ :    Dict       		=						word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)				)
        SCREAMING_SNAKE_CASE_ :    Tuple       		=						char
    return pairs





class 				lowerCAmelCase__							(     lowercase__					):






      '''simple docstring'''

      __UpperCamelCase      					=    VOCAB_FILES_NAMES
      __UpperCamelCase      					=    PRETRAINED_VOCAB_FILES_MAP
      __UpperCamelCase      					=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __UpperCamelCase      					=    ["input_ids", "attention_mask"]
      def __init__(		self	:    Optional[int]		,      lowercase_	:    List[Any]		,      lowercase_	:    str		,      lowercase_	:    Any="replace"		,      lowercase_	:    Optional[Any]="<s>"		,      lowercase_	:    str="</s>"		,      lowercase_	:    Optional[int]="</s>"		,      lowercase_	:    List[Any]="<s>"		,      lowercase_	:    Optional[Any]="<unk>"		,      lowercase_	:    List[str]="<pad>"		,      lowercase_	:    List[Any]="<mask>"		,      lowercase_	:    str=False		,      **lowercase_	:    Optional[Any]		,      ):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_ :    Union[str, Any]       		=						AddedToken(SCREAMING_SNAKE_CASE_		,      lstrip=SCREAMING_SNAKE_CASE_		,      rstrip=SCREAMING_SNAKE_CASE_) if isinstance(SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_) else bos_token
          SCREAMING_SNAKE_CASE_ :    Optional[Any]       		=						AddedToken(SCREAMING_SNAKE_CASE_		,      lstrip=SCREAMING_SNAKE_CASE_		,      rstrip=SCREAMING_SNAKE_CASE_) if isinstance(SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_) else eos_token
          SCREAMING_SNAKE_CASE_ :    Optional[Any]       		=						AddedToken(SCREAMING_SNAKE_CASE_		,      lstrip=SCREAMING_SNAKE_CASE_		,      rstrip=SCREAMING_SNAKE_CASE_) if isinstance(SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_) else sep_token
          SCREAMING_SNAKE_CASE_ :    Dict       		=						AddedToken(SCREAMING_SNAKE_CASE_		,      lstrip=SCREAMING_SNAKE_CASE_		,      rstrip=SCREAMING_SNAKE_CASE_) if isinstance(SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_) else cls_token
          SCREAMING_SNAKE_CASE_ :    List[str]       		=						AddedToken(SCREAMING_SNAKE_CASE_		,      lstrip=SCREAMING_SNAKE_CASE_		,      rstrip=SCREAMING_SNAKE_CASE_) if isinstance(SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_) else unk_token
          SCREAMING_SNAKE_CASE_ :    str       		=						AddedToken(SCREAMING_SNAKE_CASE_		,      lstrip=SCREAMING_SNAKE_CASE_		,      rstrip=SCREAMING_SNAKE_CASE_) if isinstance(SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_) else pad_token

          # Mask token behave like a normal word, i.e. include the space before it
          SCREAMING_SNAKE_CASE_ :    Optional[Any]       		=						AddedToken(SCREAMING_SNAKE_CASE_		,      lstrip=SCREAMING_SNAKE_CASE_		,      rstrip=SCREAMING_SNAKE_CASE_) if isinstance(SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_) else mask_token

          super().__init__(
              errors=SCREAMING_SNAKE_CASE_		,      bos_token=SCREAMING_SNAKE_CASE_		,      eos_token=SCREAMING_SNAKE_CASE_		,      unk_token=SCREAMING_SNAKE_CASE_		,      sep_token=SCREAMING_SNAKE_CASE_		,      cls_token=SCREAMING_SNAKE_CASE_		,      pad_token=SCREAMING_SNAKE_CASE_		,      mask_token=SCREAMING_SNAKE_CASE_		,      add_prefix_space=SCREAMING_SNAKE_CASE_		,      **SCREAMING_SNAKE_CASE_		,      )

          with open(SCREAMING_SNAKE_CASE_		,      encoding='''utf-8''') as vocab_handle:
              SCREAMING_SNAKE_CASE_ :    str       		=						json.load(SCREAMING_SNAKE_CASE_)
          SCREAMING_SNAKE_CASE_ :    Optional[Any]       		=						{v: k for k, v in self.encoder.items()}
          SCREAMING_SNAKE_CASE_ :    str       		=						errors  # how to handle errors in decoding
          SCREAMING_SNAKE_CASE_ :    str       		=						bytes_to_unicode()
          SCREAMING_SNAKE_CASE_ :    List[Any]       		=						{v: k for k, v in self.byte_encoder.items()}
          with open(SCREAMING_SNAKE_CASE_		,      encoding='''utf-8''') as merges_handle:
              SCREAMING_SNAKE_CASE_ :    int       		=						merges_handle.read().split('''\n''')[1:-1]
          SCREAMING_SNAKE_CASE_ :    Any       		=						[tuple(merge.split()) for merge in bpe_merges]
          SCREAMING_SNAKE_CASE_ :    Optional[int]       		=						dict(zip(SCREAMING_SNAKE_CASE_		,      range(len(SCREAMING_SNAKE_CASE_))))
          SCREAMING_SNAKE_CASE_ :    Tuple       		=						{}
          SCREAMING_SNAKE_CASE_ :    Any       		=						add_prefix_space

          # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
          SCREAMING_SNAKE_CASE_ :    Optional[Any]       		=						re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+''')
      @property
      def _SCREAMING_SNAKE_CASE					(		self	:    int):




          '''simple docstring'''


          return len(self.encoder)
      def _SCREAMING_SNAKE_CASE					(		self	:    Tuple):




          '''simple docstring'''


          return dict(self.encoder		,      **self.added_tokens_encoder)
      def _SCREAMING_SNAKE_CASE					(		self	:    Union[str, Any]		,      lowercase_	:    Union[str, Any]):




          '''simple docstring'''


          if token in self.cache:
              return self.cache[token]
          SCREAMING_SNAKE_CASE_ :    Optional[int]       		=						tuple(SCREAMING_SNAKE_CASE_)
          SCREAMING_SNAKE_CASE_ :    Dict       		=						get_pairs(SCREAMING_SNAKE_CASE_)

          if not pairs:
              return token

          while True:
              SCREAMING_SNAKE_CASE_ :    Dict       		=						min(SCREAMING_SNAKE_CASE_		,      key=lambda lowercase_: self.bpe_ranks.get(SCREAMING_SNAKE_CASE_		,      float('''inf''')))
              if bigram not in self.bpe_ranks:
                  break
              SCREAMING_SNAKE_CASE_		,     SCREAMING_SNAKE_CASE_ :    str       		=						bigram
              SCREAMING_SNAKE_CASE_ :    List[str]       		=						[]
              SCREAMING_SNAKE_CASE_ :    Tuple       		=						0
              while i < len(SCREAMING_SNAKE_CASE_):
                  try:
                      SCREAMING_SNAKE_CASE_ :    str       		=						word.index(SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_)
                  except ValueError:
                      new_word.extend(word[i:])
                      break
                  else:
                      new_word.extend(word[i:j])
                      SCREAMING_SNAKE_CASE_ :    Tuple       		=						j

                  if word[i] == first and i < len(SCREAMING_SNAKE_CASE_) - 1 and word[i + 1] == second:
                      new_word.append(first + second)
                      i += 2
                  else:
                      new_word.append(word[i])
                      i += 1
              SCREAMING_SNAKE_CASE_ :    Optional[Any]       		=						tuple(SCREAMING_SNAKE_CASE_)
              SCREAMING_SNAKE_CASE_ :    Any       		=						new_word
              if len(SCREAMING_SNAKE_CASE_) == 1:
                  break
              else:
                  SCREAMING_SNAKE_CASE_ :    Dict       		=						get_pairs(SCREAMING_SNAKE_CASE_)
          SCREAMING_SNAKE_CASE_ :    Union[str, Any]       		=						''' '''.join(SCREAMING_SNAKE_CASE_)
          SCREAMING_SNAKE_CASE_ :    List[Any]       		=						word
          return word
      def _SCREAMING_SNAKE_CASE					(		self	:    Optional[int]		,      lowercase_	:    Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_ :    Optional[Any]       		=						[]
          for token in re.findall(self.pat		,      SCREAMING_SNAKE_CASE_):
              SCREAMING_SNAKE_CASE_ :    Dict       		=						''''''.join(
                  self.byte_encoder[b] for b in token.encode('''utf-8'''))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
              bpe_tokens.extend(bpe_token for bpe_token in self.bpe(SCREAMING_SNAKE_CASE_).split(''' '''))
          return bpe_tokens
      def _SCREAMING_SNAKE_CASE					(		self	:    Optional[int]		,      lowercase_	:    Dict):




          '''simple docstring'''


          return self.encoder.get(SCREAMING_SNAKE_CASE_		,      self.encoder.get(self.unk_token))
      def _SCREAMING_SNAKE_CASE					(		self	:    Optional[int]		,      lowercase_	:    Optional[int]):




          '''simple docstring'''


          return self.decoder.get(SCREAMING_SNAKE_CASE_)
      def _SCREAMING_SNAKE_CASE					(		self	:    List[str]		,      lowercase_	:    Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_ :    str       		=						''''''.join(SCREAMING_SNAKE_CASE_)
          SCREAMING_SNAKE_CASE_ :    List[str]       		=						bytearray([self.byte_decoder[c] for c in text]).decode('''utf-8'''		,      errors=self.errors)
          return text
      def _SCREAMING_SNAKE_CASE					(		self	:    List[str]		,      lowercase_	:    str		,      lowercase_	:    Optional[str] = None):




          '''simple docstring'''


          if not os.path.isdir(SCREAMING_SNAKE_CASE_):
              logger.error(F'Vocabulary path ({save_directory}) should be a directory')
              return
          SCREAMING_SNAKE_CASE_ :    Optional[Any]       		=						os.path.join(
              SCREAMING_SNAKE_CASE_		,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
          SCREAMING_SNAKE_CASE_ :    Optional[Any]       		=						os.path.join(
              SCREAMING_SNAKE_CASE_		,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''])

          with open(SCREAMING_SNAKE_CASE_		,      '''w'''		,      encoding='''utf-8''') as f:
              f.write(json.dumps(self.encoder		,      indent=2		,      sort_keys=SCREAMING_SNAKE_CASE_		,      ensure_ascii=SCREAMING_SNAKE_CASE_) + '''\n''')

          SCREAMING_SNAKE_CASE_ :    Any       		=						0
          with open(SCREAMING_SNAKE_CASE_		,      '''w'''		,      encoding='''utf-8''') as writer:
              writer.write('''#version: 0.2\n''')
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,      key=lambda lowercase_: kv[1]):
                  if index != token_index:
                      logger.warning(
                          F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                          ''' Please check that the tokenizer is not corrupted!''')
                      SCREAMING_SNAKE_CASE_ :    Tuple       		=						token_index
                  writer.write(''' '''.join(SCREAMING_SNAKE_CASE_) + '''\n''')
                  index += 1

          return vocab_file, merge_file
      def _SCREAMING_SNAKE_CASE					(		self	:    List[str]		,      lowercase_	:    List[int]		,      lowercase_	:    Optional[List[int]] = None):




          '''simple docstring'''


          if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
          SCREAMING_SNAKE_CASE_ :    Any       		=						[self.cls_token_id]
          SCREAMING_SNAKE_CASE_ :    List[str]       		=						[self.sep_token_id]
          return cls + token_ids_a + sep + sep + token_ids_a + sep
      def _SCREAMING_SNAKE_CASE					(		self	:    List[str]		,      lowercase_	:    List[int]		,      lowercase_	:    Optional[List[int]] = None		,      lowercase_	:    bool = False):




          '''simple docstring'''


          if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=SCREAMING_SNAKE_CASE_		,      token_ids_a=SCREAMING_SNAKE_CASE_		,      already_has_special_tokens=SCREAMING_SNAKE_CASE_)

          if token_ids_a is None:
              return [1] + ([0] * len(SCREAMING_SNAKE_CASE_)) + [1]
          return [1] + ([0] * len(SCREAMING_SNAKE_CASE_)) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE_)) + [1]
      def _SCREAMING_SNAKE_CASE					(		self	:    Optional[int]		,      lowercase_	:    List[int]		,      lowercase_	:    Optional[List[int]] = None):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_ :    Optional[Any]       		=						[self.sep_token_id]
          SCREAMING_SNAKE_CASE_ :    Any       		=						[self.cls_token_id]

          if token_ids_a is None:
              return len(cls + token_ids_a + sep) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]



      def _SCREAMING_SNAKE_CASE					(		self	:    str		,      lowercase_	:    Any		,      lowercase_	:    Optional[int]=False		,      **lowercase_	:    List[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_ :    int       		=						kwargs.pop('''add_prefix_space'''		,      self.add_prefix_space)
          if (is_split_into_words or add_prefix_space) and (len(SCREAMING_SNAKE_CASE_) > 0 and not text[0].isspace()):
              SCREAMING_SNAKE_CASE_ :    List[Any]       		=						''' ''' + text
          return (text, kwargs)




"""simple docstring"""






def __magic_name__	(				_lowerCamelCase:		Tuple,					_lowerCamelCase:		Any			)					->						Optional[Any]:

	'''simple docstring'''




	lowerCAmelCase												=  [1]
	for i in range(2,					_lowerCamelCase			):
		factorials.append(factorials[-1] * i			)
	assert 0 <= k < factorials[-1] * n, "k out of bounds"

	lowerCAmelCase												=  []
	lowerCAmelCase												=  list(range(_lowerCamelCase			)			)

	# Find permutation
	while factorials:
		lowerCAmelCase												=  factorials.pop()
		lowerCAmelCase							,       lowerCAmelCase												=  divmod(_lowerCamelCase,					_lowerCamelCase			)
		permutation.append(elements[number]			)
		elements.remove(elements[number]			)
	permutation.append(elements[0]			)

	return permutation


if __name__ == "__main__":
					import doctest

					doctest.testmod()


