
import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase						:      List[Any]									= logging.get_logger(__name__)

lowercase						:      Union[str, Any]									= '▁'

lowercase						:      Dict									= {'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

lowercase						:      Optional[Any]									= {
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

lowercase						:      Any									= {
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

lowercase						:      Dict									= {
    'ernie-m-base': 514,
    'ernie-m-large': 514,
}

lowercase						:      List[str]									= {
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}




class 				lowerCamelCase__							(       a__):




	'''simple docstring'''




	_A			=  ["input_ids"]

	_A			=  VOCAB_FILES_NAMES
	_A			=  PRETRAINED_INIT_CONFIGURATION
	_A			=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_A			=  PRETRAINED_VOCAB_FILES_MAP
	_A			=  RESOURCE_FILES_NAMES







	def __init__(      self	:int  ,     a	:List[str]  ,     a	:Optional[int]=None  ,     a	:Tuple=False  ,     a	:Union[str, Any]="utf8"  ,     a	:List[str]="[UNK]"  ,     a	:Optional[int]="[SEP]"  ,     a	:Dict="[PAD]"  ,     a	:Optional[Any]="[CLS]"  ,     a	:Optional[Any]="[MASK]"  ,     a	:Any = None  ,     **a	:Tuple  ,     )	->				None:
								__UpperCamelCase			:     List[str]         =      {} if sp_model_kwargs is None else sp_model_kwargs
								super().__init__(
								    do_lower_case=lowercase__  ,     unk_token=lowercase__  ,     sep_token=lowercase__  ,     pad_token=lowercase__  ,     cls_token=lowercase__  ,     mask_token=lowercase__  ,     vocab_file=lowercase__  ,     encoding=lowercase__  ,     sp_model_kwargs=self.sp_model_kwargs  ,     **lowercase__  ,     )
								__UpperCamelCase			:     Union[str, Any]         =      do_lower_case
								__UpperCamelCase			:     List[Any]         =      sentencepiece_model_ckpt
								__UpperCamelCase			:     Tuple         =      spm.SentencePieceProcessor(**self.sp_model_kwargs		)
								self.sp_model.Load(lowercase__		)

								# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
								if vocab_file is not None:
															__UpperCamelCase			:     Union[str, Any]         =      self.load_vocab(filepath=lowercase__		)
								else:
															__UpperCamelCase			:     int         =      {self.sp_model.id_to_piece(lowercase__		): id for id in range(self.sp_model.get_piece_size()		)}
								__UpperCamelCase			:     Any         =      {v: k for k, v in self.vocab.items()}







	def      _lowerCamelCase       (      self	:Dict  ,     a	:Union[str, Any]		)	->				Optional[Any]:





								if text is None:
															return None

								__UpperCamelCase			:     List[str]         =      self.tokenize(lowercase__		)
								__UpperCamelCase			:     Union[str, Any]         =      '''''', []

								for i, ch in enumerate(lowercase__		):
															if ch in self.SP_CHAR_MAPPING:
																						__UpperCamelCase			:     Optional[Any]         =      self.SP_CHAR_MAPPING.get(lowercase__		)
															else:
																						__UpperCamelCase			:     Tuple         =      unicodedata.normalize("NFKC"  ,     lowercase__		)
															if self.is_whitespace(lowercase__		):
																						continue
															normalized_text += ch
															char_mapping.extend([i] * len(lowercase__		)		)

								__UpperCamelCase			:     Dict         =      normalized_text, [], 0

								if self.do_lower_case:
															__UpperCamelCase			:     Optional[int]         =      text.lower()

								for token in split_tokens:
															if token[:1] == "▁":
																						__UpperCamelCase			:     List[Any]         =      token[1:]
															__UpperCamelCase			:     int         =      text[offset:].index(lowercase__		) + offset
															__UpperCamelCase			:     Optional[Any]         =      start + len(lowercase__		)

															token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)		)
															__UpperCamelCase			:     str         =      end
								return token_mapping







	@property
	def      _lowerCamelCase       (      self	:Any		)	->				Union[str, Any]:
								return len(self.vocab		)







	def      _lowerCamelCase       (      self	:int		)	->				str:
								return dict(self.vocab  ,     **self.added_tokens_encoder		)







	def __getstate__(      self	:int		)	->				Optional[Any]:
								__UpperCamelCase			:     str         =      self.__dict__.copy()
								__UpperCamelCase			:     Optional[Any]         =      None
								return state







	def __setstate__(      self	:Tuple  ,     a	:Any		)	->				Tuple:
								__UpperCamelCase			:     List[Any]         =      d

								# for backward compatibility
								if not hasattr(self  ,     "sp_model_kwargs"		):
															__UpperCamelCase			:     str         =      {}

								__UpperCamelCase			:     Optional[Any]         =      spm.SentencePieceProcessor(**self.sp_model_kwargs		)
								self.sp_model.Load(self.sentencepiece_model_ckpt		)







	def      _lowerCamelCase       (      self	:Optional[int]  ,     a	:Tuple		)	->				Union[str, Any]:
								return "".join((self.SP_CHAR_MAPPING.get(lowercase__  ,     lowercase__		) for c in text)		)







	def      _lowerCamelCase       (      self	:List[str]  ,     a	:int  ,     a	:str=False  ,     a	:int=6_4  ,     a	:Union[str, Any]=0.1		)	->				Any:





								if self.sp_model_kwargs.get("enable_sampling"		) is True:
															__UpperCamelCase			:     int         =      True
								if self.sp_model_kwargs.get("alpha"		) is not None:
															__UpperCamelCase			:     Dict         =      self.sp_model_kwargs.get("alpha"		)
								if self.sp_model_kwargs.get("nbest_size"		) is not None:
															__UpperCamelCase			:     Union[str, Any]         =      self.sp_model_kwargs.get("nbest_size"		)

								if not enable_sampling:
															__UpperCamelCase			:     str         =      self.sp_model.EncodeAsPieces(lowercase__		)
								else:
															__UpperCamelCase			:     Optional[int]         =      self.sp_model.SampleEncodeAsPieces(lowercase__  ,     lowercase__  ,     lowercase__		)
								__UpperCamelCase			:     Tuple         =      []
								for pi, piece in enumerate(lowercase__		):
															if piece == SPIECE_UNDERLINE:
																						if not pieces[pi + 1].startswith(lowercase__		) and pi != 0:
																													new_pieces.append(lowercase__		)
																													continue
																						else:
																													continue
															__UpperCamelCase			:     List[Any]         =      0
															for i, chunk in enumerate(lowercase__		):
																						if chunk == SPIECE_UNDERLINE:
																													continue
																						if self.is_ch_char(lowercase__		) or self.is_punct(lowercase__		):
																													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																				new_pieces.append(piece[lst_i:i]		)
																													new_pieces.append(lowercase__		)
																													__UpperCamelCase			:     List[str]         =      i + 1
																						elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																				new_pieces.append(piece[lst_i:i]		)
																													__UpperCamelCase			:     Optional[int]         =      i
																						elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																				new_pieces.append(piece[lst_i:i]		)
																													__UpperCamelCase			:     Any         =      i
															if len(lowercase__		) > lst_i:
																						new_pieces.append(piece[lst_i:]		)
								return new_pieces







	def      _lowerCamelCase       (      self	:Optional[Any]  ,     a	:int		)	->				int:
								__UpperCamelCase			:     Union[str, Any]         =      ''''''.join(lowercase__		).replace(lowercase__  ,     " "		).strip()
								return out_string







	def      _lowerCamelCase       (      self	:Optional[int]  ,     a	:Tuple		)	->				str:
								__UpperCamelCase			:     List[Any]         =      self.convert_ids_to_tokens(lowercase__		)
								__UpperCamelCase			:     Dict         =      ''''''.join(lowercase__		).replace(lowercase__  ,     " "		).strip()
								return out_string







	def      _lowerCamelCase       (      self	:Tuple  ,     a	:Optional[Any]		)	->				Union[str, Any]:
								return self.vocab.get(lowercase__  ,     self.vocab.get(self.unk_token		)		)







	def      _lowerCamelCase       (      self	:Dict  ,     a	:Any		)	->				Optional[Any]:
								return self.reverse_vocab.get(lowercase__  ,     self.unk_token		)







	def      _lowerCamelCase       (      self	:int  ,     a	:Any  ,     a	:int=None		)	->				str:





								if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								__UpperCamelCase			:     List[str]         =      [self.cls_token_id]
								__UpperCamelCase			:     Tuple         =      [self.sep_token_id]
								return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep







	def      _lowerCamelCase       (      self	:Tuple  ,     a	:Union[str, Any]  ,     a	:Optional[Any]=None		)	->				Union[str, Any]:





								if offset_mapping_a is None:
															return [(0, 0)] + offset_mapping_a + [(0, 0)]

								return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]







	def      _lowerCamelCase       (      self	:List[str]  ,     a	:Dict  ,     a	:str=None  ,     a	:int=False		)	->				Optional[int]:





								if already_has_special_tokens:
															if token_ids_a is not None:
																						raise ValueError(
																						    "You should not supply a second sequence if the provided sequence of "
																						    "ids is already formatted with special tokens for the model."		)
															return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

								if token_ids_a is not None:
															return [1] + ([0] * len(lowercase__		)) + [1, 1] + ([0] * len(lowercase__		)) + [1]
								return [1] + ([0] * len(lowercase__		)) + [1]







	def      _lowerCamelCase       (      self	:Optional[Any]  ,     a	:List[str]  ,     a	:str = None		)	->				List[int]:





								if token_ids_a is None:
															# [CLS] X [SEP]
															return (len(lowercase__		) + 2) * [0]

								# [CLS] A [SEP] [SEP] B [SEP]
								return [0] * (len(lowercase__		) + 1) + [1] * (len(lowercase__		) + 3)







	def      _lowerCamelCase       (      self	:Dict  ,     a	:List[Any]		)	->				Dict:





								if "\u4e00" <= char <= "\u9fff":
															return True
								return False







	def      _lowerCamelCase       (      self	:List[str]  ,     a	:Optional[int]		)	->				Any:





								if ("a" <= char <= "z") or ("A" <= char <= "Z"):
															return True
								return False







	def      _lowerCamelCase       (      self	:str  ,     a	:int		)	->				str:





								if char in ",;:.?!~，；：。？！《》【】":
															return True
								return False







	def      _lowerCamelCase       (      self	:Optional[Any]  ,     a	:Any		)	->				Tuple:





								if char == " " or char == "\t" or char == "\n" or char == "\r":
															return True
								if len(lowercase__		) == 1:
															__UpperCamelCase			:     Union[str, Any]         =      unicodedata.category(lowercase__		)
															if cat == "Zs":
																						return True
								return False







	def      _lowerCamelCase       (      self	:Dict  ,     a	:Tuple		)	->				Any:
								__UpperCamelCase			:     Union[str, Any]         =      {}
								with io.open(lowercase__  ,     "r"  ,     encoding="utf-8"		) as f:
															for index, line in enumerate(lowercase__		):
																						__UpperCamelCase			:     Optional[int]         =      line.rstrip("\n"		)
																						__UpperCamelCase			:     List[Any]         =      int(lowercase__		)

								return token_to_idx







	def      _lowerCamelCase       (      self	:Optional[Any]  ,     a	:Optional[int]  ,     a	:List[Any] = None		)	->				Tuple[str]:
								__UpperCamelCase			:     Dict         =      0
								if os.path.isdir(lowercase__		):
															__UpperCamelCase			:     Tuple         =      os.path.join(
															    lowercase__  ,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
								else:
															__UpperCamelCase			:     Union[str, Any]         =      (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
								with open(lowercase__  ,     "w"  ,     encoding="utf-8"		) as writer:
															for token, token_index in sorted(self.vocab.items()  ,     key=lambda a		: kv[1]		):
																						if index != token_index:
																													logger.warning(
																													    f'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
																													    " Please check that the vocabulary is not corrupted!"		)
																													__UpperCamelCase			:     Optional[int]         =      token_index
																						writer.write(token + "\n"		)
																						index += 1

								__UpperCamelCase			:     Dict         =      os.path.join(lowercase__  ,     "sentencepiece.bpe.model"		)
								with open(lowercase__  ,     "wb"		) as fi:
															__UpperCamelCase			:     Dict         =      self.sp_model.serialized_model_proto()
															fi.write(lowercase__		)

								return (vocab_file,)

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


lowercase						:      str									= {
    'configuration_vision_encoder_decoder': ['VisionEncoderDecoderConfig', 'VisionEncoderDecoderOnnxConfig']
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowercase						:      str									= ['VisionEncoderDecoderModel']

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowercase						:      List[Any]									= ['TFVisionEncoderDecoderModel']

try:
		if not is_flax_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowercase						:      int									= ['FlaxVisionEncoderDecoderModel']

if TYPE_CHECKING:
		from .configuration_vision_encoder_decoder import VisionEncoderDecoderConfig, VisionEncoderDecoderOnnxConfig

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_vision_encoder_decoder import VisionEncoderDecoderModel

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_vision_encoder_decoder import TFVisionEncoderDecoderModel

		try:
				if not is_flax_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_flax_vision_encoder_decoder import FlaxVisionEncoderDecoderModel

else:
		import sys

		lowercase						:      Dict									= _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)