





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_      							=     logging.get_logger(__name__)


UpperCAmelCase_      							=     {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all BART models at https://huggingface.co/models?filter=bart
UpperCAmelCase_      							=     {
    'vocab_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/vocab.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/vocab.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json',
    },
    'merges_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/merges.txt',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/merges.txt',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt',
    },
}

UpperCAmelCase_      							=     {
    'facebook/bart-base': 1024,
    'facebook/bart-large': 1024,
    'facebook/bart-large-mnli': 1024,
    'facebook/bart-large-cnn': 1024,
    'facebook/bart-large-xsum': 1024,
    'yjernite/bart_eli5': 1024,
}





@lru_cache()
def 						lowerCamelCase__     (     )			-> Tuple:



					'''simple docstring'''




					_snake_case									=				(
					    list(range(ord('!'						)       ,    ord('~'						) + 1						)						) + list(range(ord('¡'						)       ,    ord('¬'						) + 1						)						) + list(range(ord('®'						)       ,    ord('ÿ'						) + 1						)						)
					)
					_snake_case									=				bs[:]
					_snake_case									=				0
					for b in range(2**8						):
										if b not in bs:
															bs.append(_lowercase						)
															cs.append(2**8 + n						)
															n += 1
					_snake_case									=				[chr(_lowercase						) for n in cs]
					return dict(zip(_lowercase       ,    _lowercase						)						)





def 						lowerCamelCase__     (     UpperCamelCase__			:       Optional[Any]						)			-> Tuple:



					'''simple docstring'''




					_snake_case									=				set()
					_snake_case									=				word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)						)
										_snake_case									=				char
					return pairs




class 			UpperCamelCase_		(  UpperCamelCase_					):
					lowerCAmelCase_							=	VOCAB_FILES_NAMES
					lowerCAmelCase_							=	PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_							=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_							=	['''input_ids''', '''attention_mask''']




					def __init__(		self				,						lowerCAmelCase_				,						lowerCAmelCase_				,						lowerCAmelCase_="replace"				,						lowerCAmelCase_="<s>"				,						lowerCAmelCase_="</s>"				,						lowerCAmelCase_="</s>"				,						lowerCAmelCase_="<s>"				,						lowerCAmelCase_="<unk>"				,						lowerCAmelCase_="<pad>"				,						lowerCAmelCase_="<mask>"				,						lowerCAmelCase_=False				,						**lowerCAmelCase_				,						)		->							List[Any]:
										_snake_case									=				AddedToken(UpperCamelCase__				,						lstrip=UpperCamelCase__				,						rstrip=UpperCamelCase__  ) if isinstance(UpperCamelCase__				,						UpperCamelCase__  ) else bos_token
										_snake_case									=				AddedToken(UpperCamelCase__				,						lstrip=UpperCamelCase__				,						rstrip=UpperCamelCase__  ) if isinstance(UpperCamelCase__				,						UpperCamelCase__  ) else eos_token
										_snake_case									=				AddedToken(UpperCamelCase__				,						lstrip=UpperCamelCase__				,						rstrip=UpperCamelCase__  ) if isinstance(UpperCamelCase__				,						UpperCamelCase__  ) else sep_token
										_snake_case									=				AddedToken(UpperCamelCase__				,						lstrip=UpperCamelCase__				,						rstrip=UpperCamelCase__  ) if isinstance(UpperCamelCase__				,						UpperCamelCase__  ) else cls_token
										_snake_case									=				AddedToken(UpperCamelCase__				,						lstrip=UpperCamelCase__				,						rstrip=UpperCamelCase__  ) if isinstance(UpperCamelCase__				,						UpperCamelCase__  ) else unk_token
										_snake_case									=				AddedToken(UpperCamelCase__				,						lstrip=UpperCamelCase__				,						rstrip=UpperCamelCase__  ) if isinstance(UpperCamelCase__				,						UpperCamelCase__  ) else pad_token

										# Mask token behave like a normal word, i.e. include the space before it
										_snake_case									=				AddedToken(UpperCamelCase__				,						lstrip=UpperCamelCase__				,						rstrip=UpperCamelCase__  ) if isinstance(UpperCamelCase__				,						UpperCamelCase__  ) else mask_token

										super().__init__(
										    errors=UpperCamelCase__				,						bos_token=UpperCamelCase__				,						eos_token=UpperCamelCase__				,						unk_token=UpperCamelCase__				,						sep_token=UpperCamelCase__				,						cls_token=UpperCamelCase__				,						pad_token=UpperCamelCase__				,						mask_token=UpperCamelCase__				,						add_prefix_space=UpperCamelCase__				,						**UpperCamelCase__				,						)

										with open(UpperCamelCase__				,						encoding='utf-8'  ) as vocab_handle:
															_snake_case									=				json.load(UpperCamelCase__  )
										_snake_case									=				{v: k for k, v in self.encoder.items()}
										_snake_case									=				errors  # how to handle errors in decoding
										_snake_case									=				bytes_to_unicode()
										_snake_case									=				{v: k for k, v in self.byte_encoder.items()}
										with open(UpperCamelCase__				,						encoding='utf-8'  ) as merges_handle:
															_snake_case									=				merges_handle.read().split('\n'  )[1:-1]
										_snake_case									=				[tuple(merge.split()  ) for merge in bpe_merges]
										_snake_case									=				dict(zip(UpperCamelCase__				,						range(len(UpperCamelCase__  )  )  )  )
										_snake_case									=				{}
										_snake_case									=				add_prefix_space

										# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
										_snake_case									=				re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'  )




					@property
					def 	lowerCAmelCase			(		self  )		->							int:
										return len(self.encoder  )




					def 	lowerCAmelCase			(		self  )		->							str:
										return dict(self.encoder				,						**self.added_tokens_encoder  )




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_  )		->							List[str]:





										if token in self.cache:
															return self.cache[token]
										_snake_case									=				tuple(UpperCamelCase__  )
										_snake_case									=				get_pairs(UpperCamelCase__  )

										if not pairs:
															return token

										while True:
															_snake_case									=				min(UpperCamelCase__				,						key=lambda lowerCAmelCase_  : self.bpe_ranks.get(UpperCamelCase__				,						float('inf'  )  )  )
															if bigram not in self.bpe_ranks:
																				break
															_snake_case									=				bigram
															_snake_case									=				[]
															_snake_case									=				0
															while i < len(UpperCamelCase__  ):
																				try:
																									_snake_case									=				word.index(UpperCamelCase__				,						UpperCamelCase__  )
																				except ValueError:
																									new_word.extend(word[i:]  )
																									break
																				else:
																									new_word.extend(word[i:j]  )
																									_snake_case									=				j

																				if word[i] == first and i < len(UpperCamelCase__  ) - 1 and word[i + 1] == second:
																									new_word.append(first + second  )
																									i += 2
																				else:
																									new_word.append(word[i]  )
																									i += 1
															_snake_case									=				tuple(UpperCamelCase__  )
															_snake_case									=				new_word
															if len(UpperCamelCase__  ) == 1:
																				break
															else:
																				_snake_case									=				get_pairs(UpperCamelCase__  )
										_snake_case									=				''' '''.join(UpperCamelCase__  )
										_snake_case									=				word
										return word




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_  )		->							Optional[int]:
										_snake_case									=				[]
										for token in re.findall(self.pat				,						UpperCamelCase__  ):
															_snake_case									=				''''''.join(
															    self.byte_encoder[b] for b in token.encode('utf-8'  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
															bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase__  ).split(' '  )  )
										return bpe_tokens




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_  )		->							Any:
										return self.encoder.get(UpperCamelCase__				,						self.encoder.get(self.unk_token  )  )




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_  )		->							Tuple:
										return self.decoder.get(UpperCamelCase__  )




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_  )		->							Union[str, Any]:
										_snake_case									=				''''''.join(UpperCamelCase__  )
										_snake_case									=				bytearray([self.byte_decoder[c] for c in text]  ).decode('utf-8'				,						errors=self.errors  )
										return text




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = None  )		->							Tuple:





										if not os.path.isdir(UpperCamelCase__  ):
															logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
															return
										_snake_case									=				os.path.join(
										    UpperCamelCase__				,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )
										_snake_case									=				os.path.join(
										    UpperCamelCase__				,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']  )

										with open(UpperCamelCase__				,						'w'				,						encoding='utf-8'  ) as f:
															f.write(json.dumps(self.encoder				,						indent=2				,						sort_keys=UpperCamelCase__				,						ensure_ascii=UpperCamelCase__  ) + '\n'  )

										_snake_case									=				0
										with open(UpperCamelCase__				,						'w'				,						encoding='utf-8'  ) as writer:
															writer.write('#version: 0.2\n'  )
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,						key=lambda lowerCAmelCase_  : kv[1]  ):
																				if index != token_index:
																									logger.warning(
																									    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																									    ' Please check that the tokenizer is not corrupted!'  )
																									_snake_case									=				token_index
																				writer.write(' '.join(UpperCamelCase__  ) + '\n'  )
																				index += 1

										return vocab_file, merge_file




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = None  )		->							Tuple:





										if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										_snake_case									=				[self.cls_token_id]
										_snake_case									=				[self.sep_token_id]
										return cls + token_ids_a + sep + sep + token_ids_a + sep




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = None				,						lowerCAmelCase_ = False  )		->							Union[str, Any]:





										if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=UpperCamelCase__				,						token_ids_a=UpperCamelCase__				,						already_has_special_tokens=UpperCamelCase__  )

										if token_ids_a is None:
															return [1] + ([0] * len(UpperCamelCase__  )) + [1]
										return [1] + ([0] * len(UpperCamelCase__  )) + [1, 1] + ([0] * len(UpperCamelCase__  )) + [1]




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = None  )		->							Tuple:
										_snake_case									=				[self.sep_token_id]
										_snake_case									=				[self.cls_token_id]

										if token_ids_a is None:
															return len(cls + token_ids_a + sep  ) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]






					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_=False				,						**lowerCAmelCase_  )		->							int:
										_snake_case									=				kwargs.pop('add_prefix_space'				,						self.add_prefix_space  )
										if (is_split_into_words or add_prefix_space) and (len(UpperCamelCase__  ) > 0 and not text[0].isspace()):
															_snake_case									=				''' ''' + text
										return (text, kwargs)










from collections import deque

from .hash_table import HashTable
class 			UpperCamelCase_		(  _lowerCamelCase					):




					def __init__(		self				,						*lowerCAmelCase_				,						**lowerCAmelCase_  )		->							Any:
										super().__init__(*lowerCAmelCase_				,						**lowerCAmelCase_  )




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_  )		->							Optional[int]:
										_snake_case									=				deque([]  ) if self.values[key] is None else self.values[key]
										self.values[key].appendleft(lowerCAmelCase_  )
										_snake_case									=				self.values[key]




					def 	lowerCAmelCase			(		self  )		->							Optional[int]:
										return (
										    sum(self.charge_factor - len(lowerCAmelCase_  ) for slot in self.values  )
										    / self.size_table
										    * self.charge_factor
										)






					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_=None  )		->							str:
										if not (
										    len(self.values[key]  ) == self.charge_factor and self.values.count(lowerCAmelCase_  ) == 0
										):
															return key
										return super()._collision_resolution(lowerCAmelCase_				,						lowerCAmelCase_  )



