


import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation

lowerCAmelCase__       :	Union[str, Any]     				=logging.get_logger(__name__)

lowerCAmelCase__       :	Optional[int]     				={
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

lowerCAmelCase__       :	Tuple     				={
    'vocab_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/vocab.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/vocab.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/vocab.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/vocab.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/vocab.json',
    },
    'merges_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/merges.txt',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/merges.txt',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/merges.txt',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/merges.txt',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/merges.txt',
    },
}

lowerCAmelCase__       :	Dict     				={
    'gpt2': 10_24,
    'gpt2-medium': 10_24,
    'gpt2-large': 10_24,
    'gpt2-xl': 10_24,
    'distilgpt2': 10_24,
}






@lru_cache()
def 		a__       (							):
			SCREAMING_SNAKE_CASE_						:			Dict       						= (
			    list(range(ord('!'							),	ord('~'							) + 1							)							) + list(range(ord('¡'							),	ord('¬'							) + 1							)							) + list(range(ord('®'							),	ord('ÿ'							) + 1							)							)
			)
			SCREAMING_SNAKE_CASE_						:			str       						= bs[:]
			SCREAMING_SNAKE_CASE_						:			Any       						= 0
			for b in range(2**8							):
						if b not in bs:
									bs.append(A__							)
									cs.append(2**8 + n							)
									n += 1
			SCREAMING_SNAKE_CASE_						:			List[Any]       						= [chr(A__							) for n in cs]
			return dict(zip(A__,	A__							)							)






def 		a__       (							A__							):
			SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= set()
			SCREAMING_SNAKE_CASE_						:			Tuple       						= word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)							)
						SCREAMING_SNAKE_CASE_						:			Tuple       						= char
			return pairs







class      __lowercase  (__SCREAMING_SNAKE_CASE   ):





							"""simple docstring"""







							_UpperCAmelCase     =		VOCAB_FILES_NAMES
							_UpperCAmelCase     =		PRETRAINED_VOCAB_FILES_MAP
							_UpperCAmelCase     =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_UpperCAmelCase     =		["""input_ids""", """attention_mask"""]






							def __init__(	self       ,		lowerCAmelCase__       ,		lowerCAmelCase__       ,		lowerCAmelCase__="replace"       ,		lowerCAmelCase__="<|endoftext|>"       ,		lowerCAmelCase__="<|endoftext|>"       ,		lowerCAmelCase__="<|endoftext|>"       ,		lowerCAmelCase__=None       ,		lowerCAmelCase__=False       ,		lowerCAmelCase__=False       ,		**lowerCAmelCase__       ,		):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Dict       						= AddedToken(lowerCAmelCase__       ,		lstrip=lowerCAmelCase__       ,		rstrip=lowerCAmelCase__    ) if isinstance(lowerCAmelCase__       ,		lowerCAmelCase__    ) else bos_token
										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= AddedToken(lowerCAmelCase__       ,		lstrip=lowerCAmelCase__       ,		rstrip=lowerCAmelCase__    ) if isinstance(lowerCAmelCase__       ,		lowerCAmelCase__    ) else eos_token
										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= AddedToken(lowerCAmelCase__       ,		lstrip=lowerCAmelCase__       ,		rstrip=lowerCAmelCase__    ) if isinstance(lowerCAmelCase__       ,		lowerCAmelCase__    ) else unk_token
										SCREAMING_SNAKE_CASE_						:			Tuple       						= AddedToken(lowerCAmelCase__       ,		lstrip=lowerCAmelCase__       ,		rstrip=lowerCAmelCase__    ) if isinstance(lowerCAmelCase__       ,		lowerCAmelCase__    ) else pad_token
										super().__init__(
										    errors=lowerCAmelCase__       ,		unk_token=lowerCAmelCase__       ,		bos_token=lowerCAmelCase__       ,		eos_token=lowerCAmelCase__       ,		pad_token=lowerCAmelCase__       ,		add_prefix_space=lowerCAmelCase__       ,		add_bos_token=lowerCAmelCase__       ,		**lowerCAmelCase__       ,		)
										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= add_bos_token

										with open(lowerCAmelCase__       ,		encoding='utf-8'    ) as vocab_handle:
													SCREAMING_SNAKE_CASE_						:			str       						= json.load(lowerCAmelCase__    )
										SCREAMING_SNAKE_CASE_						:			List[str]       						= {v: k for k, v in self.encoder.items()}
										SCREAMING_SNAKE_CASE_						:			Dict       						= errors  # how to handle errors in decoding
										SCREAMING_SNAKE_CASE_						:			List[Any]       						= bytes_to_unicode()
										SCREAMING_SNAKE_CASE_						:			Any       						= {v: k for k, v in self.byte_encoder.items()}
										with open(lowerCAmelCase__       ,		encoding='utf-8'    ) as merges_handle:
													SCREAMING_SNAKE_CASE_						:			Any       						= merges_handle.read().split('\n'    )[1:-1]
										SCREAMING_SNAKE_CASE_						:			List[Any]       						= [tuple(merge.split()    ) for merge in bpe_merges]
										SCREAMING_SNAKE_CASE_						:			Any       						= dict(zip(lowerCAmelCase__       ,		range(len(lowerCAmelCase__    )    )    )    )
										SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= {}
										SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= add_prefix_space

										# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
										SCREAMING_SNAKE_CASE_						:			Dict       						= re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'    )






							@property
							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										return len(self.encoder    )






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										return dict(self.encoder       ,		**self.added_tokens_encoder    )






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										if token in self.cache:
													return self.cache[token]
										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= tuple(lowerCAmelCase__    )
										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= get_pairs(lowerCAmelCase__    )

										if not pairs:
													return token

										while True:
													SCREAMING_SNAKE_CASE_						:			Tuple       						= min(lowerCAmelCase__       ,		key=lambda lowerCAmelCase__    : self.bpe_ranks.get(lowerCAmelCase__       ,		float('inf'    )    )    )
													if bigram not in self.bpe_ranks:
																break
													SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_						:			str       						= bigram
													SCREAMING_SNAKE_CASE_						:			str       						= []
													SCREAMING_SNAKE_CASE_						:			str       						= 0
													while i < len(lowerCAmelCase__    ):
																try:
																			SCREAMING_SNAKE_CASE_						:			List[Any]       						= word.index(lowerCAmelCase__       ,		lowerCAmelCase__    )
																except ValueError:
																			new_word.extend(word[i:]    )
																			break
																else:
																			new_word.extend(word[i:j]    )
																			SCREAMING_SNAKE_CASE_						:			Tuple       						= j

																if word[i] == first and i < len(lowerCAmelCase__    ) - 1 and word[i + 1] == second:
																			new_word.append(first + second    )
																			i += 2
																else:
																			new_word.append(word[i]    )
																			i += 1
													SCREAMING_SNAKE_CASE_						:			List[Any]       						= tuple(lowerCAmelCase__    )
													SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= new_word
													if len(lowerCAmelCase__    ) == 1:
																break
													else:
																SCREAMING_SNAKE_CASE_						:			str       						= get_pairs(lowerCAmelCase__    )
										SCREAMING_SNAKE_CASE_						:			Dict       						= ' '.join(lowerCAmelCase__    )
										SCREAMING_SNAKE_CASE_						:			List[str]       						= word
										return word






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__       ,		lowerCAmelCase__=None    ):





										"""simple docstring"""




										if self.add_bos_token:
													SCREAMING_SNAKE_CASE_						:			str       						= [self.bos_token_id]
										else:
													SCREAMING_SNAKE_CASE_						:			List[str]       						= []

										SCREAMING_SNAKE_CASE_						:			int       						= bos_token_ids + token_ids_a

										if token_ids_a is None:
													return output

										return output + bos_token_ids + token_ids_a






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__       ,		lowerCAmelCase__ = None       ,		lowerCAmelCase__ = False    ):





										"""simple docstring"""




										if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=lowerCAmelCase__       ,		token_ids_a=lowerCAmelCase__       ,		already_has_special_tokens=lowerCAmelCase__    )

										if not self.add_bos_token:
													return super().get_special_tokens_mask(
													    token_ids_a=lowerCAmelCase__       ,		token_ids_a=lowerCAmelCase__       ,		already_has_special_tokens=lowerCAmelCase__    )

										if token_ids_a is None:
													return [1] + ([0] * len(lowerCAmelCase__    ))
										return [1] + ([0] * len(lowerCAmelCase__    )) + [1] + ([0] * len(lowerCAmelCase__    ))






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= []
										for token in re.findall(self.pat       ,		lowerCAmelCase__    ):
													SCREAMING_SNAKE_CASE_						:			Dict       						= ''.join(
													    self.byte_encoder[b] for b in token.encode('utf-8'    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
													bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase__    ).split(' '    )    )
										return bpe_tokens






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										return self.encoder.get(lowerCAmelCase__       ,		self.encoder.get(self.unk_token    )    )






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										return self.decoder.get(lowerCAmelCase__    )






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			List[str]       						= ''.join(lowerCAmelCase__    )
										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= bytearray([self.byte_decoder[c] for c in text]    ).decode('utf-8'       ,		errors=self.errors    )
										return text






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__       ,		lowerCAmelCase__ = None    ):





										"""simple docstring"""




										if not os.path.isdir(lowerCAmelCase__    ):
													logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
													return
										SCREAMING_SNAKE_CASE_						:			str       						= os.path.join(
										    lowerCAmelCase__       ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )
										SCREAMING_SNAKE_CASE_						:			Tuple       						= os.path.join(
										    lowerCAmelCase__       ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']    )

										with open(lowerCAmelCase__       ,		'w'       ,		encoding='utf-8'    ) as f:
													f.write(json.dumps(self.encoder       ,		indent=2       ,		sort_keys=lowerCAmelCase__       ,		ensure_ascii=lowerCAmelCase__    ) + '\n'    )

										SCREAMING_SNAKE_CASE_						:			int       						= 0
										with open(lowerCAmelCase__       ,		'w'       ,		encoding='utf-8'    ) as writer:
													writer.write('#version: 0.2\n'    )
													for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,		key=lambda lowerCAmelCase__    : kv[1]    ):
																if index != token_index:
																			logger.warning(
																			    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																			    ' Please check that the tokenizer is not corrupted!'    )
																			SCREAMING_SNAKE_CASE_						:			Any       						= token_index
																writer.write(' '.join(lowerCAmelCase__    ) + '\n'    )
																index += 1

										return vocab_file, merge_file






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__       ,		lowerCAmelCase__=False       ,		**lowerCAmelCase__    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			int       						= kwargs.pop('add_prefix_space'       ,		self.add_prefix_space    )
										if is_split_into_words or add_prefix_space:
													SCREAMING_SNAKE_CASE_						:			Any       						= ' ' + text
										return (text, kwargs)






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Tuple       						= []
										for is_user, text in conversation.iter_texts():
													input_ids.extend(self.encode(lowerCAmelCase__       ,		add_special_tokens=lowerCAmelCase__    ) + [self.eos_token_id]    )
										if len(lowerCAmelCase__    ) > self.model_max_length:
													SCREAMING_SNAKE_CASE_						:			Optional[int]       						= input_ids[-self.model_max_length :]
										return input_ids









from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_tokenizers_available


lowerCAmelCase__       :	List[Any]     				={}

try:
				if not is_sentencepiece_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowerCAmelCase__       :	Optional[Any]     				=['MBart50Tokenizer']

try:
				if not is_tokenizers_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowerCAmelCase__       :	Any     				=['MBart50TokenizerFast']


if TYPE_CHECKING:
				try:
								if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_mbartaa import MBartaaTokenizer

				try:
								if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_mbartaa_fast import MBartaaTokenizerFast

else:
				import sys

				lowerCAmelCase__       :	Optional[int]     				=_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





