


'''simple docstring'''



import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A__						:   Dict				=    logging.get_logger(__name__)


A__						:   int				=    {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

A__						:   Any				=    {
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
}

A__						:   Tuple				=    {
    """allenai/longformer-base-4096""": 4096,
    """allenai/longformer-large-4096""": 4096,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4096,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4096,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4096,
}



@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def      UpperCAmelCase__   (      )	->						Optional[int]:
     __lowerCamelCase  :      Tuple														=						(
         list(range(ord('!'				) ,       ord('~'				) + 1				)				) + list(range(ord('¡'				) ,       ord('¬'				) + 1				)				) + list(range(ord('®'				) ,       ord('ÿ'				) + 1				)				)
     )
     __lowerCamelCase  :      str														=						bs[:]
     __lowerCamelCase  :      Tuple														=						0
     for b in range(2**8				):
          if b not in bs:
               bs.append(UpperCAmelCase_				)
               cs.append(2**8 + n				)
               n += 1
     __lowerCamelCase  :      Union[str, Any]														=						[chr(UpperCAmelCase_				) for n in cs]
     return dict(zip(UpperCAmelCase_ ,       UpperCAmelCase_				)				)



def      UpperCAmelCase__   (      UpperCAmelCase_       :							Union[str, Any]				)	->						Union[str, Any]:
     __lowerCamelCase  :      Union[str, Any]														=						set()
     __lowerCamelCase  :      Union[str, Any]														=						word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)				)
          __lowerCamelCase  :      List[str]														=						char
     return pairs





class   UpperCAmelCase_    (_UpperCAmelCase    ):





    """simple docstring"""






    lowerCamelCase    :   List[Any]					=     VOCAB_FILES_NAMES
    lowerCamelCase    :   Tuple					=     PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase    :   List[str]					=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase    :   str					=     ['input_ids', 'attention_mask']







    def __init__(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_="replace"     ,	SCREAMING_SNAKE_CASE_="<s>"     ,	SCREAMING_SNAKE_CASE_="</s>"     ,	SCREAMING_SNAKE_CASE_="</s>"     ,	SCREAMING_SNAKE_CASE_="<s>"     ,	SCREAMING_SNAKE_CASE_="<unk>"     ,	SCREAMING_SNAKE_CASE_="<pad>"     ,	SCREAMING_SNAKE_CASE_="<mask>"     ,	SCREAMING_SNAKE_CASE_=False     ,	**SCREAMING_SNAKE_CASE_     ,	)							->	Union[str, Any]:
         __lowerCamelCase  :      List[Any]														=						AddedToken(SCREAMING_SNAKE_CASE_     ,	lstrip=SCREAMING_SNAKE_CASE_     ,	rstrip=SCREAMING_SNAKE_CASE_      ) if isinstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      ) else bos_token
         __lowerCamelCase  :      str														=						AddedToken(SCREAMING_SNAKE_CASE_     ,	lstrip=SCREAMING_SNAKE_CASE_     ,	rstrip=SCREAMING_SNAKE_CASE_      ) if isinstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      ) else eos_token
         __lowerCamelCase  :      Dict														=						AddedToken(SCREAMING_SNAKE_CASE_     ,	lstrip=SCREAMING_SNAKE_CASE_     ,	rstrip=SCREAMING_SNAKE_CASE_      ) if isinstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      ) else sep_token
         __lowerCamelCase  :      str														=						AddedToken(SCREAMING_SNAKE_CASE_     ,	lstrip=SCREAMING_SNAKE_CASE_     ,	rstrip=SCREAMING_SNAKE_CASE_      ) if isinstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      ) else cls_token
         __lowerCamelCase  :      Optional[int]														=						AddedToken(SCREAMING_SNAKE_CASE_     ,	lstrip=SCREAMING_SNAKE_CASE_     ,	rstrip=SCREAMING_SNAKE_CASE_      ) if isinstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      ) else unk_token
         __lowerCamelCase  :      Dict														=						AddedToken(SCREAMING_SNAKE_CASE_     ,	lstrip=SCREAMING_SNAKE_CASE_     ,	rstrip=SCREAMING_SNAKE_CASE_      ) if isinstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      ) else pad_token

         # Mask token behave like a normal word, i.e. include the space before it
         __lowerCamelCase  :      Optional[Any]														=						AddedToken(SCREAMING_SNAKE_CASE_     ,	lstrip=SCREAMING_SNAKE_CASE_     ,	rstrip=SCREAMING_SNAKE_CASE_      ) if isinstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      ) else mask_token

         super().__init__(
             errors=SCREAMING_SNAKE_CASE_     ,	bos_token=SCREAMING_SNAKE_CASE_     ,	eos_token=SCREAMING_SNAKE_CASE_     ,	unk_token=SCREAMING_SNAKE_CASE_     ,	sep_token=SCREAMING_SNAKE_CASE_     ,	cls_token=SCREAMING_SNAKE_CASE_     ,	pad_token=SCREAMING_SNAKE_CASE_     ,	mask_token=SCREAMING_SNAKE_CASE_     ,	add_prefix_space=SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_     ,	)

         with open(SCREAMING_SNAKE_CASE_     ,	encoding='utf-8'      ) as vocab_handle:
              __lowerCamelCase  :      Optional[Any]														=						json.load(SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      Dict														=						{v: k for k, v in self.encoder.items()}
         __lowerCamelCase  :      List[str]														=						errors  # how to handle errors in decoding
         __lowerCamelCase  :      Dict														=						bytes_to_unicode()
         __lowerCamelCase  :      Optional[int]														=						{v: k for k, v in self.byte_encoder.items()}
         with open(SCREAMING_SNAKE_CASE_     ,	encoding='utf-8'      ) as merges_handle:
              __lowerCamelCase  :      Union[str, Any]														=						merges_handle.read().split('\n'      )[1:-1]
         __lowerCamelCase  :      Tuple														=						[tuple(merge.split()      ) for merge in bpe_merges]
         __lowerCamelCase  :      List[Any]														=						dict(zip(SCREAMING_SNAKE_CASE_     ,	range(len(SCREAMING_SNAKE_CASE_      )      )      )      )
         __lowerCamelCase  :      Dict														=						{}
         __lowerCamelCase  :      Dict														=						add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         __lowerCamelCase  :      Tuple														=						re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'      )







    @property
    def  lowercase_							(				self      )							->	Dict:
         return len(self.encoder      )







    def  lowercase_							(				self      )							->	Union[str, Any]:
         return dict(self.encoder     ,	**self.added_tokens_encoder      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_      )							->	Dict:
         if token in self.cache:
              return self.cache[token]
         __lowerCamelCase  :      str														=						tuple(SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      Optional[Any]														=						get_pairs(SCREAMING_SNAKE_CASE_      )

         if not pairs:
              return token

         while True:
              __lowerCamelCase  :      str														=						min(SCREAMING_SNAKE_CASE_     ,	key=lambda SCREAMING_SNAKE_CASE_      : self.bpe_ranks.get(SCREAMING_SNAKE_CASE_     ,	float('inf'      )      )      )
              if bigram not in self.bpe_ranks:
                   break
              __lowerCamelCase							,			__lowerCamelCase  :      Dict														=						bigram
              __lowerCamelCase  :      List[str]														=						[]
              __lowerCamelCase  :      int														=						0
              while i < len(SCREAMING_SNAKE_CASE_      ):
                   try:
                        __lowerCamelCase  :      Optional[int]														=						word.index(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )
                   except ValueError:
                        new_word.extend(word[i:]      )
                        break
                   else:
                        new_word.extend(word[i:j]      )
                        __lowerCamelCase  :      int														=						j

                   if word[i] == first and i < len(SCREAMING_SNAKE_CASE_      ) - 1 and word[i + 1] == second:
                        new_word.append(first + second      )
                        i += 2
                   else:
                        new_word.append(word[i]      )
                        i += 1
              __lowerCamelCase  :      Tuple														=						tuple(SCREAMING_SNAKE_CASE_      )
              __lowerCamelCase  :      List[str]														=						new_word
              if len(SCREAMING_SNAKE_CASE_      ) == 1:
                   break
              else:
                   __lowerCamelCase  :      str														=						get_pairs(SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      Any														=						' '.join(SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      List[Any]														=						word
         return word







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_      )							->	List[str]:
         __lowerCamelCase  :      str														=						[]
         for token in re.findall(self.pat     ,	SCREAMING_SNAKE_CASE_      ):
              __lowerCamelCase  :      Optional[int]														=						''.join(
                  self.byte_encoder[b] for b in token.encode('utf-8'      )      )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
              bpe_tokens.extend(bpe_token for bpe_token in self.bpe(SCREAMING_SNAKE_CASE_      ).split(' '      )      )
         return bpe_tokens







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_      )							->	List[Any]:
         return self.encoder.get(SCREAMING_SNAKE_CASE_     ,	self.encoder.get(self.unk_token      )      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_      )							->	Union[str, Any]:
         return self.decoder.get(SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_      )							->	Union[str, Any]:
         __lowerCamelCase  :      Dict														=						''.join(SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      Tuple														=						bytearray([self.byte_decoder[c] for c in text]      ).decode('utf-8'     ,	errors=self.errors      )
         return text







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_ = None      )							->	Tuple[str]:
         if not os.path.isdir(SCREAMING_SNAKE_CASE_      ):
              logger.error(f'Vocabulary path ({save_directory}) should be a directory'      )
              return
         __lowerCamelCase  :      Any														=						os.path.join(
             SCREAMING_SNAKE_CASE_     ,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )
         __lowerCamelCase  :      Optional[Any]														=						os.path.join(
             SCREAMING_SNAKE_CASE_     ,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']      )

         with open(SCREAMING_SNAKE_CASE_     ,	'w'     ,	encoding='utf-8'      ) as f:
              f.write(json.dumps(self.encoder     ,	indent=2     ,	sort_keys=SCREAMING_SNAKE_CASE_     ,	ensure_ascii=SCREAMING_SNAKE_CASE_      ) + '\n'      )

         __lowerCamelCase  :      Dict														=						0
         with open(SCREAMING_SNAKE_CASE_     ,	'w'     ,	encoding='utf-8'      ) as writer:
              writer.write('#version: 0.2\n'      )
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,	key=lambda SCREAMING_SNAKE_CASE_      : kv[1]      ):
                   if index != token_index:
                        logger.warning(
                            f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                            ' Please check that the tokenizer is not corrupted!'      )
                        __lowerCamelCase  :      List[Any]														=						token_index
                   writer.write(' '.join(SCREAMING_SNAKE_CASE_      ) + '\n'      )
                   index += 1

         return vocab_file, merge_file







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_ = None      )							->	List[int]:
         if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         __lowerCamelCase  :      List[str]														=						[self.cls_token_id]
         __lowerCamelCase  :      Any														=						[self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_ = None     ,	SCREAMING_SNAKE_CASE_ = False      )							->	List[int]:
         if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=SCREAMING_SNAKE_CASE_     ,	token_ids_a=SCREAMING_SNAKE_CASE_     ,	already_has_special_tokens=SCREAMING_SNAKE_CASE_      )

         if token_ids_a is None:
              return [1] + ([0] * len(SCREAMING_SNAKE_CASE_      )) + [1]
         return [1] + ([0] * len(SCREAMING_SNAKE_CASE_      )) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE_      )) + [1]







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_ = None      )							->	List[int]:
         __lowerCamelCase  :      List[str]														=						[self.sep_token_id]
         __lowerCamelCase  :      Dict														=						[self.cls_token_id]

         if token_ids_a is None:
              return len(cls + token_ids_a + sep      ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_=False     ,	**SCREAMING_SNAKE_CASE_      )							->	List[Any]:
         __lowerCamelCase  :      Dict														=						kwargs.pop('add_prefix_space'     ,	self.add_prefix_space      )
         if (is_split_into_words or add_prefix_space) and (len(SCREAMING_SNAKE_CASE_      ) > 0 and not text[0].isspace()):
              __lowerCamelCase  :      List[str]														=						' ' + text
         return (text, kwargs)










'''simple docstring'''



# Lint as: python3
import dataclasses
import re
from dataclasses import dataclass
from functools import total_ordering
from typing import Optional, Union


A__						:   Any				=    re.compile(R"""^(?P<major>\d+)""" R"""\.(?P<minor>\d+)""" R"""\.(?P<patch>\d+)$""")





@total_ordering
@dataclass
class   UpperCAmelCase_    :





    """simple docstring"""






    lowerCamelCase    :   str
    lowerCamelCase    :   Optional[str]					=     None
    lowerCamelCase    :   Optional[Union[str, int]]					=     None
    lowerCamelCase    :   Optional[Union[str, int]]					=     None
    lowerCamelCase    :   Optional[Union[str, int]]					=     None







    def  lowercase_							(				self      )							->	List[str]:
         __lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase  :      List[Any]														=						_str_to_version_tuple(self.version_str      )







    def __repr__(				self      )							->	Any:
         return f'{self.tuple[0]}.{self.tuple[1]}.{self.tuple[2]}'







    @property
    def  lowercase_							(				self      )							->	int:
         return self.major, self.minor, self.patch







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_      )							->	List[Any]:
         if isinstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      ):
              return Version(SCREAMING_SNAKE_CASE_      )
         elif isinstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      ):
              return other
         raise TypeError(f'{other} (type {type(SCREAMING_SNAKE_CASE_      )}) cannot be compared to version.'      )







    def __eq__(				self     ,	SCREAMING_SNAKE_CASE_      )							->	Optional[Any]:
         try:
              __lowerCamelCase  :      Union[str, Any]														=						self._validate_operand(SCREAMING_SNAKE_CASE_      )
         except (TypeError, ValueError):
              return False
         else:
              return self.tuple == other.tuple







    def __lt__(				self     ,	SCREAMING_SNAKE_CASE_      )							->	List[Any]:
         __lowerCamelCase  :      List[Any]														=						self._validate_operand(SCREAMING_SNAKE_CASE_      )
         return self.tuple < other.tuple







    def __hash__(				self      )							->	List[str]:
         return hash(_version_tuple_to_str(self.tuple      )      )







    @classmethod
    def  lowercase_							(				cls     ,	SCREAMING_SNAKE_CASE_      )							->	List[str]:
         __lowerCamelCase  :      str														=						{f.name for f in dataclasses.fields(cls      )}
         return cls(**{k: v for k, v in dic.items() if k in field_names}      )







    def  lowercase_							(				self      )							->	str:
         return self.version_str







def      UpperCAmelCase__   (      UpperCAmelCase_       :							Union[str, Any]				)	->						str:
     __lowerCamelCase  :      str														=						_VERSION_REG.match(UpperCAmelCase_				)
     if not res:
          raise ValueError(F'Invalid version \'{version_str}\'. Format should be x.y.z with {{x,y,z}} being digits.'				)
     return tuple(int(UpperCAmelCase_				) for v in [res.group('major'				), res.group('minor'				), res.group('patch'				)]				)



def      UpperCAmelCase__   (      UpperCAmelCase_       :							List[str]				)	->						Dict:
     return ".".join(str(UpperCAmelCase_				) for v in version_tuple				)






