"""simple docstring"""


import warnings
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a				=logging.get_logger(__name__)






class   __UpperCAmelCase		(      __lowerCAmelCase      ):
				A__ :  Any    				=			['''input_ids''', '''attention_mask''']

				def __init__( self      ,					_lowerCamelCase="</s>"      ,					_lowerCamelCase="<unk>"      ,					_lowerCamelCase="<pad>"      ,					_lowerCamelCase=125      ,					_lowerCamelCase=None      ,					**_lowerCamelCase      ,					):
										# Add extra_ids to the special token list
										if extra_ids > 0 and additional_special_tokens is None:
																lowerCamelCase__   =[F'''<extra_id_{i}>''' for i in range(_lowerCamelCase )]
										elif extra_ids > 0 and additional_special_tokens is not None:
																# Check that we have the right number of extra_id special tokens
																lowerCamelCase__   =len(set(filter(lambda _lowerCamelCase : bool("extra_id" in str(_lowerCamelCase ) )      ,					_lowerCamelCase ) ) )
																if extra_tokens != extra_ids:
																						raise ValueError(
																						    F'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
																						    " provided to ByT5Tokenizer. In this case the additional_special_tokens must include the"
																						    " extra_ids tokens" )

										lowerCamelCase__   =AddedToken(_lowerCamelCase      ,					lstrip=_lowerCamelCase      ,					rstrip=_lowerCamelCase ) if isinstance(_lowerCamelCase      ,					_lowerCamelCase ) else pad_token
										lowerCamelCase__   =AddedToken(_lowerCamelCase      ,					lstrip=_lowerCamelCase      ,					rstrip=_lowerCamelCase ) if isinstance(_lowerCamelCase      ,					_lowerCamelCase ) else eos_token
										lowerCamelCase__   =AddedToken(_lowerCamelCase      ,					lstrip=_lowerCamelCase      ,					rstrip=_lowerCamelCase ) if isinstance(_lowerCamelCase      ,					_lowerCamelCase ) else unk_token

										super().__init__(
										    eos_token=_lowerCamelCase      ,					unk_token=_lowerCamelCase      ,					pad_token=_lowerCamelCase      ,					extra_ids=_lowerCamelCase      ,					additional_special_tokens=_lowerCamelCase      ,					**_lowerCamelCase      ,					)

										lowerCamelCase__   =extra_ids

										lowerCamelCase__   =2**8  # utf is 8 bits

										# define special tokens dict
										lowerCamelCase__   ={
										    self.pad_token: 0,
										    self.eos_token: 1,
										    self.unk_token: 2,
										}
										lowerCamelCase__   =len(self.special_tokens_encoder )
										lowerCamelCase__   =len(_lowerCamelCase )
										for i, token in enumerate(_lowerCamelCase ):
																lowerCamelCase__   =self.vocab_size + i - n
										lowerCamelCase__   ={v: k for k, v in self.special_tokens_encoder.items()}

				@property
				def 						_a      ( self ):
										return self._utf_vocab_size + self._num_special_tokens + self._extra_ids

				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None      ,					_lowerCamelCase = False ):
										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=_lowerCamelCase      ,					token_ids_a=_lowerCamelCase      ,					already_has_special_tokens=_lowerCamelCase )

										# normal case: some special tokens
										if token_ids_a is None:
																return ([0] * len(_lowerCamelCase )) + [1]
										return ([0] * len(_lowerCamelCase )) + [1] + ([0] * len(_lowerCamelCase )) + [1]

				def 						_a      ( self      ,					_lowerCamelCase ):
										if len(_lowerCamelCase ) > 0 and token_ids[-1] == self.eos_token_id:
																warnings.warn(
																    F'''This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'''
																    " eos tokens being added." )
																return token_ids
										else:
																return token_ids + [self.eos_token_id]

				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None ):
										lowerCamelCase__   =[self.eos_token_id]

										if token_ids_a is None:
																return len(token_ids_a + eos ) * [0]
										return len(token_ids_a + eos + token_ids_a + eos ) * [0]

				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None ):
										lowerCamelCase__   =self._add_eos_if_not_present(_lowerCamelCase )
										if token_ids_a is None:
																return token_ids_a
										else:
																lowerCamelCase__   =self._add_eos_if_not_present(_lowerCamelCase )
																return token_ids_a + token_ids_a

				def 						_a      ( self      ,					_lowerCamelCase ):
										lowerCamelCase__   =[chr(_lowerCamelCase ) for i in text.encode("utf-8" )]
										return tokens

				def 						_a      ( self      ,					_lowerCamelCase ):
										if token in self.special_tokens_encoder:
																lowerCamelCase__   =self.special_tokens_encoder[token]
										elif token in self.added_tokens_encoder:
																lowerCamelCase__   =self.added_tokens_encoder[token]
										elif len(_lowerCamelCase ) != 1:
																lowerCamelCase__   =self.unk_token_id
										else:
																lowerCamelCase__   =ord(_lowerCamelCase ) + self._num_special_tokens
										return token_id

				def 						_a      ( self      ,					_lowerCamelCase ):
										if index in self.special_tokens_decoder:
																lowerCamelCase__   =self.special_tokens_decoder[index]
										else:
																lowerCamelCase__   =chr(index - self._num_special_tokens )
										return token

				def 						_a      ( self      ,					_lowerCamelCase ):
										lowerCamelCase__   =b""
										for token in tokens:
																if token in self.special_tokens_decoder:
																						lowerCamelCase__   =self.special_tokens_decoder[token].encode("utf-8" )
																elif token in self.added_tokens_decoder:
																						lowerCamelCase__   =self.special_tokens_decoder[token].encode("utf-8" )
																elif token in self.special_tokens_encoder:
																						lowerCamelCase__   =token.encode("utf-8" )
																elif token in self.added_tokens_encoder:
																						lowerCamelCase__   =token.encode("utf-8" )
																else:
																						lowerCamelCase__   =bytes([ord(_lowerCamelCase )] )
																bstring += tok_string
										lowerCamelCase__   =bstring.decode("utf-8"      ,					errors="ignore" )
										return string






				def 						_a      ( self      ,					_lowerCamelCase      ,					_lowerCamelCase = None ):
										return ()



"""simple docstring"""


# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


a				={
    'configuration_mgp_str': ['MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MgpstrConfig'],
    'processing_mgp_str': ['MgpstrProcessor'],
    'tokenization_mgp_str': ['MgpstrTokenizer'],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	a				=[
	    'MGP_STR_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'MgpstrModel',
	    'MgpstrPreTrainedModel',
	    'MgpstrForSceneTextRecognition',
	]

if TYPE_CHECKING:
	from .configuration_mgp_str import MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP, MgpstrConfig
	from .processing_mgp_str import MgpstrProcessor
	from .tokenization_mgp_str import MgpstrTokenizer

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_mgp_str import (
		    MGP_STR_PRETRAINED_MODEL_ARCHIVE_LIST,
		    MgpstrForSceneTextRecognition,
		    MgpstrModel,
		    MgpstrPreTrainedModel,
		)
else:
	import sys

	a				=_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


