



'''simple docstring'''

import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__     :						Dict	=  logging.get_logger(__name__)


SCREAMING_SNAKE_CASE__     :						List[str]	=  {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all BART models at https://huggingface.co/models?filter=bart
SCREAMING_SNAKE_CASE__     :						Tuple	=  {
    '''vocab_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/vocab.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/vocab.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/merges.txt''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/merges.txt''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt''',
    },
}

SCREAMING_SNAKE_CASE__     :						Dict	=  {
    '''facebook/bart-base''': 1_0_2_4,
    '''facebook/bart-large''': 1_0_2_4,
    '''facebook/bart-large-mnli''': 1_0_2_4,
    '''facebook/bart-large-cnn''': 1_0_2_4,
    '''facebook/bart-large-xsum''': 1_0_2_4,
    '''yjernite/bart_eli5''': 1_0_2_4,
}






@lru_cache()
def 				a  (     )       ->	Union[str, Any]:
		snake_case__	     =(
		    list(range(ord('!'			)      ,					ord('~'			) + 1			)			) + list(range(ord('¡'			)      ,					ord('¬'			) + 1			)			) + list(range(ord('®'			)      ,					ord('ÿ'			) + 1			)			)
		)
		snake_case__	     =bs[:]
		snake_case__	     =0
		for b in range(2**8			):
				if b not in bs:
						bs.append(UpperCamelCase_			)
						cs.append(2**8 + n			)
						n += 1
		snake_case__	     =[chr(UpperCamelCase_			) for n in cs]
		return dict(zip(UpperCamelCase_      ,					UpperCamelCase_			)			)






def 				a  (     UpperCamelCase_    :    str			)       ->	List[Any]:
		snake_case__	     =set()
		snake_case__	     =word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)			)
				snake_case__	     =char
		return pairs


class 							a__( snake_case__     ):
			a_	: List[Any]									=      VOCAB_FILES_NAMES
			a_	: List[str]									=      PRETRAINED_VOCAB_FILES_MAP
			a_	: Any									=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			a_	: Union[str, Any]									=      ['''input_ids''', '''attention_mask''']






			def __init__(     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase="replace"					,   _UpperCAmelCase="<s>"					,   _UpperCAmelCase="</s>"					,   _UpperCAmelCase="</s>"					,   _UpperCAmelCase="<s>"					,   _UpperCAmelCase="<unk>"					,   _UpperCAmelCase="<pad>"					,   _UpperCAmelCase="<mask>"					,   _UpperCAmelCase=False					,   **_UpperCAmelCase					,   )							->							Union[str, Any]:
					snake_case__	     =AddedToken(_UpperCAmelCase					,   lstrip=_UpperCAmelCase					,   rstrip=_UpperCAmelCase						) if isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) else bos_token
					snake_case__	     =AddedToken(_UpperCAmelCase					,   lstrip=_UpperCAmelCase					,   rstrip=_UpperCAmelCase						) if isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) else eos_token
					snake_case__	     =AddedToken(_UpperCAmelCase					,   lstrip=_UpperCAmelCase					,   rstrip=_UpperCAmelCase						) if isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) else sep_token
					snake_case__	     =AddedToken(_UpperCAmelCase					,   lstrip=_UpperCAmelCase					,   rstrip=_UpperCAmelCase						) if isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) else cls_token
					snake_case__	     =AddedToken(_UpperCAmelCase					,   lstrip=_UpperCAmelCase					,   rstrip=_UpperCAmelCase						) if isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) else unk_token
					snake_case__	     =AddedToken(_UpperCAmelCase					,   lstrip=_UpperCAmelCase					,   rstrip=_UpperCAmelCase						) if isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) else pad_token

					# Mask token behave like a normal word, i.e. include the space before it
					snake_case__	     =AddedToken(_UpperCAmelCase					,   lstrip=_UpperCAmelCase					,   rstrip=_UpperCAmelCase						) if isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) else mask_token

					super().__init__(
					    errors=_UpperCAmelCase					,   bos_token=_UpperCAmelCase					,   eos_token=_UpperCAmelCase					,   unk_token=_UpperCAmelCase					,   sep_token=_UpperCAmelCase					,   cls_token=_UpperCAmelCase					,   pad_token=_UpperCAmelCase					,   mask_token=_UpperCAmelCase					,   add_prefix_space=_UpperCAmelCase					,   **_UpperCAmelCase					,   )

					with open(_UpperCAmelCase					,   encoding='utf-8'						) as vocab_handle:
							snake_case__	     =json.load(_UpperCAmelCase						)
					snake_case__	     ={v: k for k, v in self.encoder.items()}
					snake_case__	     =errors  # how to handle errors in decoding
					snake_case__	     =bytes_to_unicode()
					snake_case__	     ={v: k for k, v in self.byte_encoder.items()}
					with open(_UpperCAmelCase					,   encoding='utf-8'						) as merges_handle:
							snake_case__	     =merges_handle.read().split('\n'						)[1:-1]
					snake_case__	     =[tuple(merge.split()						) for merge in bpe_merges]
					snake_case__	     =dict(zip(_UpperCAmelCase					,   range(len(_UpperCAmelCase						)						)						)						)
					snake_case__	     ={}
					snake_case__	     =add_prefix_space

					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
					snake_case__	     =re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'						)






			@property
			def 						_lowercase  (     self						)							->							Any:
					return len(self.encoder						)






			def 						_lowercase  (     self						)							->							Dict:
					return dict(self.encoder					,   **self.added_tokens_encoder						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							int:
					if token in self.cache:
							return self.cache[token]
					snake_case__	     =tuple(_UpperCAmelCase						)
					snake_case__	     =get_pairs(_UpperCAmelCase						)

					if not pairs:
							return token

					while True:
							snake_case__	     =min(_UpperCAmelCase					,   key=lambda _UpperCAmelCase						: self.bpe_ranks.get(_UpperCAmelCase					,   float('inf'						)						)						)
							if bigram not in self.bpe_ranks:
									break
							snake_case__     ,	snake_case__	     =bigram
							snake_case__	     =[]
							snake_case__	     =0
							while i < len(_UpperCAmelCase						):
									try:
											snake_case__	     =word.index(_UpperCAmelCase					,   _UpperCAmelCase						)
									except ValueError:
											new_word.extend(word[i:]						)
											break
									else:
											new_word.extend(word[i:j]						)
											snake_case__	     =j

									if word[i] == first and i < len(_UpperCAmelCase						) - 1 and word[i + 1] == second:
											new_word.append(first + second						)
											i += 2
									else:
											new_word.append(word[i]						)
											i += 1
							snake_case__	     =tuple(_UpperCAmelCase						)
							snake_case__	     =new_word
							if len(_UpperCAmelCase						) == 1:
									break
							else:
									snake_case__	     =get_pairs(_UpperCAmelCase						)
					snake_case__	     =' '.join(_UpperCAmelCase						)
					snake_case__	     =word
					return word






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							List[str]:
					snake_case__	     =[]
					for token in re.findall(self.pat					,   _UpperCAmelCase						):
							snake_case__	     =''.join(
							    self.byte_encoder[b] for b in token.encode('utf-8'						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
							bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_UpperCAmelCase						).split(' '						)						)
					return bpe_tokens






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							List[Any]:
					return self.encoder.get(_UpperCAmelCase					,   self.encoder.get(self.unk_token						)						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							int:
					return self.decoder.get(_UpperCAmelCase						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							List[Any]:
					snake_case__	     =''.join(_UpperCAmelCase						)
					snake_case__	     =bytearray([self.byte_decoder[c] for c in text]						).decode('utf-8'					,   errors=self.errors						)
					return text






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase = None						)							->							Tuple[str]:
					if not os.path.isdir(_UpperCAmelCase						):
							logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
							return
					snake_case__	     =os.path.join(
					    _UpperCAmelCase					,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)
					snake_case__	     =os.path.join(
					    _UpperCAmelCase					,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']						)

					with open(_UpperCAmelCase					,   'w'					,   encoding='utf-8'						) as f:
							f.write(json.dumps(self.encoder					,   indent=2					,   sort_keys=_UpperCAmelCase					,   ensure_ascii=_UpperCAmelCase						) + '\n'						)

					snake_case__	     =0
					with open(_UpperCAmelCase					,   'w'					,   encoding='utf-8'						) as writer:
							writer.write('#version: 0.2\n'						)
							for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,   key=lambda _UpperCAmelCase						: kv[1]						):
									if index != token_index:
											logger.warning(
											    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
											    ' Please check that the tokenizer is not corrupted!'						)
											snake_case__	     =token_index
									writer.write(' '.join(_UpperCAmelCase						) + '\n'						)
									index += 1

					return vocab_file, merge_file






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase = None						)							->							List[int]:
					if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
					snake_case__	     =[self.cls_token_id]
					snake_case__	     =[self.sep_token_id]
					return cls + token_ids_a + sep + sep + token_ids_a + sep






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase = None					,   _UpperCAmelCase = False						)							->							List[int]:
					if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=_UpperCAmelCase					,   token_ids_a=_UpperCAmelCase					,   already_has_special_tokens=_UpperCAmelCase						)

					if token_ids_a is None:
							return [1] + ([0] * len(_UpperCAmelCase						)) + [1]
					return [1] + ([0] * len(_UpperCAmelCase						)) + [1, 1] + ([0] * len(_UpperCAmelCase						)) + [1]






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase = None						)							->							List[int]:
					snake_case__	     =[self.sep_token_id]
					snake_case__	     =[self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep						) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase=False					,   **_UpperCAmelCase						)							->							str:
					snake_case__	     =kwargs.pop('add_prefix_space'					,   self.add_prefix_space						)
					if (is_split_into_words or add_prefix_space) and (len(_UpperCAmelCase						) > 0 and not text[0].isspace()):
							snake_case__	     =' ' + text
					return (text, kwargs)









'''simple docstring'''

from timeit import timeit






def 				a  (     UpperCamelCase_    :    int			)       ->	int:
		if number < 0:
				raise ValueError('the value of input must not be negative'			)
		snake_case__	     =0
		while number:
				number &= number - 1
				result += 1
		return result






def 				a  (     UpperCamelCase_    :    int			)       ->	int:
		if number < 0:
				raise ValueError('the value of input must not be negative'			)
		snake_case__	     =0
		while number:
				if number % 2 == 1:
						result += 1
				number >>= 1
		return result






def 				a  (     )       ->	None:

		def do_benchmark(UpperCamelCase_    :    int			) -> None:
				snake_case__	     ='import __main__ as z'
				print(f"""Benchmark when {number = }:"""			)
				print(f"""{get_set_bits_count_using_modulo_operator(UpperCamelCase_			) = }"""			)
				snake_case__	     =timeit('z.get_set_bits_count_using_modulo_operator(25)'      ,					setup=UpperCamelCase_			)
				print(f"""timeit() runs in {timing} seconds"""			)
				print(f"""{get_set_bits_count_using_brian_kernighans_algorithm(UpperCamelCase_			) = }"""			)
				snake_case__	     =timeit(
				    'z.get_set_bits_count_using_brian_kernighans_algorithm(25)'      ,					setup=UpperCamelCase_      ,					)
				print(f"""timeit() runs in {timing} seconds"""			)

		for number in (25, 37, 58, 0):
				do_benchmark(UpperCamelCase_			)
				print()


if __name__ == "__main__":
			import doctest

			doctest.testmod()
			benchmark()




