

'''simple docstring'''





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A_	= logging.get_logger(__name__)


A_	= {"vocab_file": "vocab.json", "merges_file": "merges.txt"}

A_	= {
    "vocab_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
    },
}

A_	= {
    "allenai/longformer-base-4096": 4_096,
    "allenai/longformer-large-4096": 4_096,
    "allenai/longformer-large-4096-finetuned-triviaqa": 4_096,
    "allenai/longformer-base-4096-extra.pos.embd.only": 4_096,
    "allenai/longformer-large-4096-extra.pos.embd.only": 4_096,
}







@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def   _UpperCamelCase	(		)       ->			Any:
					lowerCamelCase_  				=      (
					    list(range(ord('!'	)    ,ord('~'	) + 1	)	) + list(range(ord('¡'	)    ,ord('¬'	) + 1	)	) + list(range(ord('®'	)    ,ord('ÿ'	) + 1	)	)
					)
					lowerCamelCase_  				=      bs[:]
					lowerCamelCase_  				=      0
					for b in range(2**8	):
										if b not in bs:
															bs.append(__UpperCamelCase	)
															cs.append(2**8 + n	)
															n += 1
					lowerCamelCase_  				=      [chr(__UpperCamelCase	) for n in cs]
					return dict(zip(__UpperCamelCase    ,__UpperCamelCase	)	)







def   _UpperCamelCase	(		__UpperCamelCase	)       ->			List[str]:
					lowerCamelCase_  				=      set()
					lowerCamelCase_  				=      word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)	)
										lowerCamelCase_  				=      char
					return pairs







class  UpperCAmelCase (							UpperCAmelCase__    ):

		'''simple docstring'''






		SCREAMING_SNAKE_CASE_           =					VOCAB_FILES_NAMES
		SCREAMING_SNAKE_CASE_           =					PRETRAINED_VOCAB_FILES_MAP
		SCREAMING_SNAKE_CASE_           =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		SCREAMING_SNAKE_CASE_           =					['input_ids', 'attention_mask']
		def __init__(					self   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_="replace"   ,			SCREAMING_SNAKE_CASE_="<s>"   ,			SCREAMING_SNAKE_CASE_="</s>"   ,			SCREAMING_SNAKE_CASE_="</s>"   ,			SCREAMING_SNAKE_CASE_="<s>"   ,			SCREAMING_SNAKE_CASE_="<unk>"   ,			SCREAMING_SNAKE_CASE_="<pad>"   ,			SCREAMING_SNAKE_CASE_="<mask>"   ,			SCREAMING_SNAKE_CASE_=False   ,			**SCREAMING_SNAKE_CASE_   ,			)  ->     Optional[Any]:






							'''simple docstring'''



							lowerCamelCase_  				=      AddedToken(SCREAMING_SNAKE_CASE_   ,			lstrip=SCREAMING_SNAKE_CASE_   ,			rstrip=SCREAMING_SNAKE_CASE_		) if isinstance(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		) else bos_token
							lowerCamelCase_  				=      AddedToken(SCREAMING_SNAKE_CASE_   ,			lstrip=SCREAMING_SNAKE_CASE_   ,			rstrip=SCREAMING_SNAKE_CASE_		) if isinstance(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		) else eos_token
							lowerCamelCase_  				=      AddedToken(SCREAMING_SNAKE_CASE_   ,			lstrip=SCREAMING_SNAKE_CASE_   ,			rstrip=SCREAMING_SNAKE_CASE_		) if isinstance(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		) else sep_token
							lowerCamelCase_  				=      AddedToken(SCREAMING_SNAKE_CASE_   ,			lstrip=SCREAMING_SNAKE_CASE_   ,			rstrip=SCREAMING_SNAKE_CASE_		) if isinstance(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		) else cls_token
							lowerCamelCase_  				=      AddedToken(SCREAMING_SNAKE_CASE_   ,			lstrip=SCREAMING_SNAKE_CASE_   ,			rstrip=SCREAMING_SNAKE_CASE_		) if isinstance(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		) else unk_token
							lowerCamelCase_  				=      AddedToken(SCREAMING_SNAKE_CASE_   ,			lstrip=SCREAMING_SNAKE_CASE_   ,			rstrip=SCREAMING_SNAKE_CASE_		) if isinstance(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		) else pad_token

							# Mask token behave like a normal word, i.e. include the space before it
							lowerCamelCase_  				=      AddedToken(SCREAMING_SNAKE_CASE_   ,			lstrip=SCREAMING_SNAKE_CASE_   ,			rstrip=SCREAMING_SNAKE_CASE_		) if isinstance(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		) else mask_token

							super().__init__(
							    errors=SCREAMING_SNAKE_CASE_   ,			bos_token=SCREAMING_SNAKE_CASE_   ,			eos_token=SCREAMING_SNAKE_CASE_   ,			unk_token=SCREAMING_SNAKE_CASE_   ,			sep_token=SCREAMING_SNAKE_CASE_   ,			cls_token=SCREAMING_SNAKE_CASE_   ,			pad_token=SCREAMING_SNAKE_CASE_   ,			mask_token=SCREAMING_SNAKE_CASE_   ,			add_prefix_space=SCREAMING_SNAKE_CASE_   ,			**SCREAMING_SNAKE_CASE_   ,			)

							with open(SCREAMING_SNAKE_CASE_   ,			encoding='utf-8'		) as vocab_handle:
												lowerCamelCase_  				=      json.load(SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      {v: k for k, v in self.encoder.items()}
							lowerCamelCase_  				=      errors  # how to handle errors in decoding
							lowerCamelCase_  				=      bytes_to_unicode()
							lowerCamelCase_  				=      {v: k for k, v in self.byte_encoder.items()}
							with open(SCREAMING_SNAKE_CASE_   ,			encoding='utf-8'		) as merges_handle:
												lowerCamelCase_  				=      merges_handle.read().split('\n'		)[1:-1]
							lowerCamelCase_  				=      [tuple(merge.split()		) for merge in bpe_merges]
							lowerCamelCase_  				=      dict(zip(SCREAMING_SNAKE_CASE_   ,			range(len(SCREAMING_SNAKE_CASE_		)		)		)		)
							lowerCamelCase_  				=      {}
							lowerCamelCase_  				=      add_prefix_space

							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
							lowerCamelCase_  				=      re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'		)
		@property
		def 			UpperCamelCase(					self		)  ->     Union[str, Any]:






							'''simple docstring'''



							return len(self.encoder		)
		def 			UpperCamelCase(					self		)  ->     Optional[Any]:






							'''simple docstring'''



							return dict(self.encoder   ,			**self.added_tokens_encoder		)
		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_		)  ->     Any:






							'''simple docstring'''



							if token in self.cache:
												return self.cache[token]
							lowerCamelCase_  				=      tuple(SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      get_pairs(SCREAMING_SNAKE_CASE_		)

							if not pairs:
												return token

							while True:
												lowerCamelCase_  				=      min(SCREAMING_SNAKE_CASE_   ,			key=lambda SCREAMING_SNAKE_CASE_		: self.bpe_ranks.get(SCREAMING_SNAKE_CASE_   ,			float('inf'		)		)		)
												if bigram not in self.bpe_ranks:
																	break
												lowerCamelCase_				,lowerCamelCase_  				=      bigram
												lowerCamelCase_  				=      []
												lowerCamelCase_  				=      0
												while i < len(SCREAMING_SNAKE_CASE_		):
																	try:
																						lowerCamelCase_  				=      word.index(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)
																	except ValueError:
																						new_word.extend(word[i:]		)
																						break
																	else:
																						new_word.extend(word[i:j]		)
																						lowerCamelCase_  				=      j

																	if word[i] == first and i < len(SCREAMING_SNAKE_CASE_		) - 1 and word[i + 1] == second:
																						new_word.append(first + second		)
																						i += 2
																	else:
																						new_word.append(word[i]		)
																						i += 1
												lowerCamelCase_  				=      tuple(SCREAMING_SNAKE_CASE_		)
												lowerCamelCase_  				=      new_word
												if len(SCREAMING_SNAKE_CASE_		) == 1:
																	break
												else:
																	lowerCamelCase_  				=      get_pairs(SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      ' '.join(SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      word
							return word
		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_		)  ->     str:






							'''simple docstring'''



							lowerCamelCase_  				=      []
							for token in re.findall(self.pat   ,			SCREAMING_SNAKE_CASE_		):
												lowerCamelCase_  				=      ''.join(
												    self.byte_encoder[b] for b in token.encode('utf-8'		)		)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
												bpe_tokens.extend(bpe_token for bpe_token in self.bpe(SCREAMING_SNAKE_CASE_		).split(' '		)		)
							return bpe_tokens
		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_		)  ->     Dict:






							'''simple docstring'''



							return self.encoder.get(SCREAMING_SNAKE_CASE_   ,			self.encoder.get(self.unk_token		)		)
		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_		)  ->     Union[str, Any]:






							'''simple docstring'''



							return self.decoder.get(SCREAMING_SNAKE_CASE_		)
		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_		)  ->     Tuple:






							'''simple docstring'''



							lowerCamelCase_  				=      ''.join(SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      bytearray([self.byte_decoder[c] for c in text]		).decode('utf-8'   ,			errors=self.errors		)
							return text
		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_ = None		)  ->     Tuple[str]:






							'''simple docstring'''



							if not os.path.isdir(SCREAMING_SNAKE_CASE_		):
												logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''		)
												return
							lowerCamelCase_  				=      os.path.join(
							    SCREAMING_SNAKE_CASE_   ,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']		)
							lowerCamelCase_  				=      os.path.join(
							    SCREAMING_SNAKE_CASE_   ,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']		)

							with open(SCREAMING_SNAKE_CASE_   ,			'w'   ,			encoding='utf-8'		) as f:
												f.write(json.dumps(self.encoder   ,			indent=2   ,			sort_keys=SCREAMING_SNAKE_CASE_   ,			ensure_ascii=SCREAMING_SNAKE_CASE_		) + '\n'		)

							lowerCamelCase_  				=      0
							with open(SCREAMING_SNAKE_CASE_   ,			'w'   ,			encoding='utf-8'		) as writer:
												writer.write('#version: 0.2\n'		)
												for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,			key=lambda SCREAMING_SNAKE_CASE_		: kv[1]		):
																	if index != token_index:
																						logger.warning(
																						    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																						    ' Please check that the tokenizer is not corrupted!'		)
																						lowerCamelCase_  				=      token_index
																	writer.write(' '.join(SCREAMING_SNAKE_CASE_		) + '\n'		)
																	index += 1

							return vocab_file, merge_file
		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_ = None		)  ->     List[int]:






							'''simple docstring'''



							if token_ids_a is None:
												return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							lowerCamelCase_  				=      [self.cls_token_id]
							lowerCamelCase_  				=      [self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep
		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_ = None   ,			SCREAMING_SNAKE_CASE_ = False		)  ->     List[int]:






							'''simple docstring'''



							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=SCREAMING_SNAKE_CASE_   ,			token_ids_a=SCREAMING_SNAKE_CASE_   ,			already_has_special_tokens=SCREAMING_SNAKE_CASE_		)

							if token_ids_a is None:
												return [1] + ([0] * len(SCREAMING_SNAKE_CASE_		)) + [1]
							return [1] + ([0] * len(SCREAMING_SNAKE_CASE_		)) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE_		)) + [1]
		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_ = None		)  ->     List[int]:






							'''simple docstring'''



							lowerCamelCase_  				=      [self.sep_token_id]
							lowerCamelCase_  				=      [self.cls_token_id]

							if token_ids_a is None:
												return len(cls + token_ids_a + sep		) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]






		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_=False   ,			**SCREAMING_SNAKE_CASE_		)  ->     List[str]:






							'''simple docstring'''



							lowerCamelCase_  				=      kwargs.pop('add_prefix_space'   ,			self.add_prefix_space		)
							if (is_split_into_words or add_prefix_space) and (len(SCREAMING_SNAKE_CASE_		) > 0 and not text[0].isspace()):
												lowerCamelCase_  				=      ' ' + text
							return (text, kwargs)






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


UpperCamelCase__		 =     {
    '''configuration_convnext''': ['''CONVNEXT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ConvNextConfig''', '''ConvNextOnnxConfig''']
}

try:
    if not is_vision_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    UpperCamelCase__		 =     ['''ConvNextFeatureExtractor''']
    UpperCamelCase__		 =     ['''ConvNextImageProcessor''']

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    UpperCamelCase__		 =     [
        '''CONVNEXT_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''ConvNextForImageClassification''',
        '''ConvNextModel''',
        '''ConvNextPreTrainedModel''',
        '''ConvNextBackbone''',
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    UpperCamelCase__		 =     [
        '''TFConvNextForImageClassification''',
        '''TFConvNextModel''',
        '''TFConvNextPreTrainedModel''',
    ]

if TYPE_CHECKING:
    from .configuration_convnext import CONVNEXT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvNextConfig, ConvNextOnnxConfig

    try:
        if not is_vision_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .feature_extraction_convnext import ConvNextFeatureExtractor
        from .image_processing_convnext import ConvNextImageProcessor

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_convnext import (
            CONVNEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
            ConvNextBackbone,
            ConvNextForImageClassification,
            ConvNextModel,
            ConvNextPreTrainedModel,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_convnext import TFConvNextForImageClassification, TFConvNextModel, TFConvNextPreTrainedModel


else:
    import sys

    UpperCamelCase__		 =     _LazyModule(__name__, globals()['''__file__'''], _import_structure)

