





import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


_lowerCamelCase 					=      logging.get_logger(__name__)

_lowerCamelCase 					=      {'tokenizer_file': 'tokenizer.json'}

_lowerCamelCase 					=      {
    'tokenizer_file': {
        'bigscience/tokenizer': 'https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json',
        'bigscience/bloom-560m': 'https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json',
        'bigscience/bloom-1b1': 'https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json',
        'bigscience/bloom-1b7': 'https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json',
        'bigscience/bloom-3b': 'https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json',
        'bigscience/bloom-7b1': 'https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json',
        'bigscience/bloom': 'https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json',
    },
}




class 	UpperCamelCase_							(  UpperCamelCase__     ):
							lowerCamelCase_							     =       VOCAB_FILES_NAMES
							lowerCamelCase_							     =       PRETRAINED_VOCAB_FILES_MAP
							lowerCamelCase_							     =       ["input_ids", "attention_mask"]
							lowerCamelCase_							     =       None


							def __init__(				self     :Any				,					__A     :List[str]=None				,					__A     :Tuple=None				,					__A     :Union[str, Any]=None				,					__A     :Tuple="<unk>"				,					__A     :Union[str, Any]="<s>"				,					__A     :Optional[Any]="</s>"				,					__A     :Tuple="<pad>"				,					__A     :str=False				,					__A     :str=False				,					**__A     :Dict				,					) -> Tuple:



									"""simple docstring"""



									super().__init__(
									    __A				,					__A				,					tokenizer_file=__A				,					unk_token=__A				,					bos_token=__A				,					eos_token=__A				,					pad_token=__A				,					add_prefix_space=__A				,					clean_up_tokenization_spaces=__A				,					**__A				,					)
									SCREAMING_SNAKE_CASE__								=   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
									if pre_tok_state.get("""add_prefix_space"""				,					__A		) != add_prefix_space:
											SCREAMING_SNAKE_CASE__								=   getattr(__A				,					pre_tok_state.pop("""type"""		)		)
											SCREAMING_SNAKE_CASE__								=   add_prefix_space
											SCREAMING_SNAKE_CASE__								=   pre_tok_class(**__A		)

									SCREAMING_SNAKE_CASE__								=   add_prefix_space


							def   _snake_case	(				self     :Tuple				,					*__A     :List[str]				,					**__A     :List[Any]		) -> BatchEncoding:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   kwargs.get("""is_split_into_words"""				,					__A		)
									if not (self.add_prefix_space or not is_split_into_words):
											raise Exception(
											    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
											    """ pretokenized inputs."""		)

									return super()._batch_encode_plus(*__A				,					**__A		)


							def   _snake_case	(				self     :Union[str, Any]				,					*__A     :List[str]				,					**__A     :List[str]		) -> BatchEncoding:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   kwargs.get("""is_split_into_words"""				,					__A		)

									if not (self.add_prefix_space or not is_split_into_words):
											raise Exception(
											    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
											    """ pretokenized inputs."""		)

									return super()._encode_plus(*__A				,					**__A		)


							def   _snake_case	(				self     :Union[str, Any]				,					__A     :str				,					__A     :Optional[str] = None		) -> Tuple[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   self._tokenizer.model.save(__A				,					name=__A		)
									return tuple(__A		)


							def   _snake_case	(				self     :str				,					__A     :"Conversation"		) -> List[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   []
									for is_user, text in conversation.iter_texts():
											input_ids.extend(self.encode(__A				,					add_special_tokens=__A		) + [self.eos_token_id]		)

									if len(__A		) > self.model_max_length:
											SCREAMING_SNAKE_CASE__								=   input_ids[-self.model_max_length :]
									return input_ids






from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding




class 	UpperCamelCase_							(  UpperCamelCase__     ):
							lowerCamelCase_							     =       ["image_processor", "tokenizer"]
							lowerCamelCase_							     =       "AutoImageProcessor"
							lowerCamelCase_							     =       "AutoTokenizer"


							def __init__(				self     :Optional[int]				,					__A     :Optional[Any]				,					__A     :Dict		) -> Dict:



									"""simple docstring"""



									super().__init__(__A				,					__A		)
									SCREAMING_SNAKE_CASE__								=   self.image_processor


							def __call__(				self     :int				,					__A     :str=None				,					__A     :int=None				,					__A     :Union[str, Any]=None				,					**__A     :str		) -> Optional[Any]:



									"""simple docstring"""



									if text is None and images is None:
											raise ValueError("""You have to specify either text or images. Both cannot be none."""		)

									if text is not None:
											SCREAMING_SNAKE_CASE__								=   self.tokenizer(__A				,					return_tensors=__A				,					**__A		)

									if images is not None:
											SCREAMING_SNAKE_CASE__								=   self.image_processor(__A				,					return_tensors=__A				,					**__A		)

									if text is not None and images is not None:
											SCREAMING_SNAKE_CASE__								=   image_features.pixel_values
											return encoding
									elif text is not None:
											return encoding
									else:
											return BatchEncoding(data=dict(**__A		)				,					tensor_type=__A		)


							def   _snake_case	(				self     :str				,					*__A     :List[str]				,					**__A     :List[str]		) -> List[Any]:



									"""simple docstring"""



									return self.tokenizer.batch_decode(*__A				,					**__A		)


							def   _snake_case	(				self     :List[str]				,					*__A     :Any				,					**__A     :Any		) -> Tuple:



									"""simple docstring"""



									return self.tokenizer.decode(*__A				,					**__A		)


							@property
							def   _snake_case	(				self     :Dict		) -> List[Any]:



									"""simple docstring"""



									return ["input_ids", "attention_mask", "pixel_values"]