


"""simple docstring"""


import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


a_											=			logging.get_logger(__name__)

a_											=			{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

a_											=			{
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

a_											=			{
    "openai-gpt": 5_12,
}






def    a__       ( __lowercase      )		->		Dict:
				_A         =	text.strip()
				if not text:
								return []
				_A         =	text.split()
				return tokens
class snake_case					(     _UpperCamelCase):







		def __init__(    self					:				Optional[Any]			,      a__					:				Optional[Any]=True			,      a__					:				Tuple=None			,      a__					:				Optional[int]=True			,      a__					:				Dict=None			,      a__					:				Dict=True			,      )				->  str:





						'''simple docstring'''







						if never_split is None:
										_A         =	[]
						_A         =	do_lower_case
						_A         =	set(a__						)
						_A         =	tokenize_chinese_chars
						_A         =	strip_accents
						_A         =	do_split_on_punc







		def        a_       (    self					:				int			,      a__					:				str			,      a__					:				Optional[int]=None						)				->  int:





						'''simple docstring'''







						_A         =	self.never_split.union(set(a__						)						) if never_split else self.never_split
						_A         =	self._clean_text(a__						)

						# This was added on November 1st, 2018 for the multilingual and Chinese
						# models. This is also applied to the English models now, but it doesn't
						# matter since the English models were not trained on any Chinese data
						# and generally don't have any Chinese data in them (there are Chinese
						# characters in the vocabulary because Wikipedia does have some Chinese
						# words in the English Wikipedia.).
						if self.tokenize_chinese_chars:
										_A         =	self._tokenize_chinese_chars(a__						)
						# prevents treating the same character with different unicode codepoints as different characters
						_A         =	unicodedata.normalize("NFC"			,      a__						)
						_A         =	whitespace_tokenize(a__						)
						_A         =	[]
						for token in orig_tokens:
										if token not in never_split:
														if self.do_lower_case:
																		_A         =	token.lower()
																		if self.strip_accents is not False:
																						_A         =	self._run_strip_accents(a__						)
														elif self.strip_accents:
																		_A         =	self._run_strip_accents(a__						)
										split_tokens.extend(self._run_split_on_punc(a__			,      a__						)						)

						_A         =	whitespace_tokenize(" ".join(a__						)						)
						return output_tokens







		def        a_       (    self					:				Any			,      a__					:				Any						)				->  Tuple:





						'''simple docstring'''







						_A         =	unicodedata.normalize("NFD"			,      a__						)
						_A         =	[]
						for char in text:
										_A         =	unicodedata.category(a__						)
										if cat == "Mn":
														continue
										output.append(a__						)
						return "".join(a__						)







		def        a_       (    self					:				List[str]			,      a__					:				Union[str, Any]			,      a__					:				Optional[Any]=None						)				->  List[str]:





						'''simple docstring'''







						if not self.do_split_on_punc or (never_split is not None and text in never_split):
										return [text]
						_A         =	list(a__						)
						_A         =	0
						_A         =	True
						_A         =	[]
						while i < len(a__						):
										_A         =	chars[i]
										if _is_punctuation(a__						):
														output.append([char]						)
														_A         =	True
										else:
														if start_new_word:
																		output.append([]						)
														_A         =	False
														output[-1].append(a__						)
										i += 1

						return ["".join(a__						) for x in output]







		def        a_       (    self					:				Optional[int]			,      a__					:				int						)				->  List[Any]:





						'''simple docstring'''







						_A         =	[]
						for char in text:
										_A         =	ord(a__						)
										if self._is_chinese_char(a__						):
														output.append(" "						)
														output.append(a__						)
														output.append(" "						)
										else:
														output.append(a__						)
						return "".join(a__						)







		def        a_       (    self					:				Dict			,      a__					:				Optional[Any]						)				->  Optional[int]:





						'''simple docstring'''







						if (
						    (cp >= 0x4_e00 and cp <= 0x9_fff)
						    or (cp >= 0x3_400 and cp <= 0x4_dbf)  #
						    or (cp >= 0x20_000 and cp <= 0x2a_6df)  #
						    or (cp >= 0x2a_700 and cp <= 0x2b_73f)  #
						    or (cp >= 0x2b_740 and cp <= 0x2b_81f)  #
						    or (cp >= 0x2b_820 and cp <= 0x2c_eaf)  #
						    or (cp >= 0xf_900 and cp <= 0xf_aff)
						    or (cp >= 0x2f_800 and cp <= 0x2f_a1f)  #
						):  #
										return True

						return False







		def        a_       (    self					:				List[Any]			,      a__					:				List[str]						)				->  str:





						'''simple docstring'''







						_A         =	[]
						for char in text:
										_A         =	ord(a__						)
										if cp == 0 or cp == 0xf_ffd or _is_control(a__						):
														continue
										if _is_whitespace(a__						):
														output.append(" "						)
										else:
														output.append(a__						)
						return "".join(a__						)






def    a__       ( __lowercase      )		->		Optional[int]:
				_A         =	set()
				_A         =	word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)      )
								_A         =	char
				return pairs






def    a__       ( __lowercase      )		->		Dict:
				_A         =	text.replace("—"   ,	"-"      )
				_A         =	text.replace("–"   ,	"-"      )
				_A         =	text.replace("―"   ,	"-"      )
				_A         =	text.replace("…"   ,	"..."      )
				_A         =	text.replace("´"   ,	"'"      )
				_A         =	re.sub(R"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"   ,	R" \1 "   ,	__lowercase      )
				_A         =	re.sub(R"\s*\n\s*"   ,	" \n "   ,	__lowercase      )
				_A         =	re.sub(R"[^\S\n]+"   ,	" "   ,	__lowercase      )
				return text.strip()




class snake_case					(     _UpperCamelCase):
		__UpperCamelCase							=	VOCAB_FILES_NAMES
		__UpperCamelCase							=	PRETRAINED_VOCAB_FILES_MAP
		__UpperCamelCase							=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__UpperCamelCase							=	['input_ids', 'attention_mask']







		def __init__(    self					:				Optional[int]			,      a__					:				Optional[Any]			,      a__					:				Optional[int]			,      a__					:				List[Any]="<unk>"			,      **a__					:				Optional[int]						)				->  Optional[int]:





						'''simple docstring'''







						super().__init__(unk_token=a__			,      **a__						)

						try:
										import ftfy
										from spacy.lang.en import English

										_A         =	English()
										_A         =	_nlp.tokenizer
										_A         =	ftfy.fix_text
						except ImportError:
										logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."						)
										_A         =	BasicTokenizer(do_lower_case=a__						)
										_A         =	None

						with open(a__			,      encoding="utf-8"						) as vocab_handle:
										_A         =	json.load(a__						)
						_A         =	{v: k for k, v in self.encoder.items()}
						with open(a__			,      encoding="utf-8"						) as merges_handle:
										_A         =	merges_handle.read().split("\n"						)[1:-1]
						_A         =	[tuple(merge.split()						) for merge in merges]
						_A         =	dict(zip(a__			,      range(len(a__						)						)						)						)
						_A         =	{}







		@property
		def        a_       (    self					:				str						)				->  str:





						'''simple docstring'''







						return True







		@property
		def        a_       (    self					:				List[str]						)				->  Tuple:





						'''simple docstring'''







						return len(self.encoder						)







		def        a_       (    self					:				List[Any]						)				->  int:





						'''simple docstring'''







						return dict(self.encoder			,      **self.added_tokens_encoder						)







		def        a_       (    self					:				List[Any]			,      a__					:				Any						)				->  Optional[int]:





						'''simple docstring'''







						_A         =	tuple(token[:-1]						) + (token[-1] + "</w>",)
						if token in self.cache:
										return self.cache[token]
						_A         =	get_pairs(a__						)

						if not pairs:
										return token + "</w>"

						while True:
										_A         =	min(a__			,      key=lambda a__						: self.bpe_ranks.get(a__			,      float("inf"						)						)						)
										if bigram not in self.bpe_ranks:
														break
										_A     ,						_A         =	bigram
										_A         =	[]
										_A         =	0
										while i < len(a__						):
														try:
																		_A         =	word.index(a__			,      a__						)
														except ValueError:
																		new_word.extend(word[i:]						)
																		break
														else:
																		new_word.extend(word[i:j]						)
																		_A         =	j

														if word[i] == first and i < len(a__						) - 1 and word[i + 1] == second:
																		new_word.append(first + second						)
																		i += 2
														else:
																		new_word.append(word[i]						)
																		i += 1
										_A         =	tuple(a__						)
										_A         =	new_word
										if len(a__						) == 1:
														break
										else:
														_A         =	get_pairs(a__						)
						_A         =	" ".join(a__						)
						if word == "\n  </w>":
										_A         =	"\n</w>"
						_A         =	word
						return word







		def        a_       (    self					:				Optional[Any]			,      a__					:				List[str]						)				->  int:





						'''simple docstring'''







						_A         =	[]
						if self.fix_text is None:
										# Using BERT's BasicTokenizer
										_A         =	self.nlp.tokenize(a__						)
										for token in text:
														split_tokens.extend(list(self.bpe(a__						).split(" "						)						)						)
						else:
										# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
										_A         =	self.nlp(text_standardize(self.fix_text(a__						)						)						)
										for token in text:
														split_tokens.extend(list(self.bpe(token.text.lower()						).split(" "						)						)						)
						return split_tokens







		def        a_       (    self					:				List[Any]			,      a__					:				Optional[int]						)				->  List[str]:





						'''simple docstring'''







						return self.encoder.get(a__			,      self.encoder.get(self.unk_token						)						)







		def        a_       (    self					:				Optional[int]			,      a__					:				Optional[Any]						)				->  Optional[int]:





						'''simple docstring'''







						return self.decoder.get(a__			,      self.unk_token						)







		def        a_       (    self					:				str			,      a__					:				List[str]						)				->  List[Any]:





						'''simple docstring'''







						_A         =	"".join(a__						).replace("</w>"			,      " "						).strip()
						return out_string







		def        a_       (    self					:				int			,      a__					:				str			,      a__					:				Optional[str] = None						)				->  Tuple[str]:





						'''simple docstring'''







						if not os.path.isdir(a__						):
										logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""						)
										return
						_A         =	os.path.join(
						    a__			,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)
						_A         =	os.path.join(
						    a__			,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]						)

						with open(a__			,      "w"			,      encoding="utf-8"						) as f:
										f.write(json.dumps(self.encoder			,      indent=2			,      sort_keys=a__			,      ensure_ascii=a__						) + "\n"						)

						_A         =	0
						with open(a__			,      "w"			,      encoding="utf-8"						) as writer:
										writer.write("#version: 0.2\n"						)
										for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,      key=lambda a__						: kv[1]						):
														if index != token_index:
																		logger.warning(
																		    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																		    " Please check that the tokenizer is not corrupted!"						)
																		_A         =	token_index
														writer.write(" ".join(a__						) + "\n"						)
														index += 1

						return vocab_file, merge_file



"""simple docstring"""


import warnings

from ...utils import logging
from .image_processing_mobilenet_va import MobileNetVaImageProcessor


a_											=			logging.get_logger(__name__)
class snake_case					(     _UpperCamelCase):







		def __init__(    self					:				str			,      *a__					:				Union[str, Any]			,      **a__					:				Optional[Any]						)				->  None:





						'''simple docstring'''







						warnings.warn(
						    "The class MobileNetV2FeatureExtractor is deprecated and will be removed in version 5 of Transformers."
						    " Please use MobileNetV2ImageProcessor instead."			,      a__			,      )
						super().__init__(*a__			,      **a__						)