


import enum
import warnings

from ..tokenization_utils import TruncationStrategy
from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_tf_available():
			import tensorflow as tf

			from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

if is_torch_available():
			from ..models.auto.modeling_auto import MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

lowerCAmelCase			=				logging.get_logger(__name__)


class 			_a				(     enum.Enum     ):
							_lowercase      :     Union[str, Any]     =					0
							_lowercase      :     List[str]     =					1


@add_end_docstrings(UpperCamelCase__     )
class 			_a				(     UpperCamelCase__     ):
							_lowercase      :     Any     =					'''generated'''

							def __init__(				self:      int       ,		*UpperCamelCase_:      Optional[int]       ,		**UpperCamelCase_:      Union[str, Any]				)     ->  Tuple:





									"""simple docstring"""







									super().__init__(*UpperCamelCase_       ,		**UpperCamelCase_				)

									self.check_model_type(
									    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
									    if self.framework == '''tf'''
									    else MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING				)

							def  lowerCamelCase_					(				self:      Union[str, Any]       ,		UpperCamelCase_:      int=None       ,		UpperCamelCase_:      Optional[int]=None       ,		UpperCamelCase_:      List[str]=None       ,		UpperCamelCase_:      int=None       ,		UpperCamelCase_:      Tuple=None       ,		UpperCamelCase_:      Any=None       ,		**UpperCamelCase_:      Optional[int]       ,		)     ->  Union[str, Any]:





									"""simple docstring"""







									lowercase__				= {}
									if truncation is not None:
											lowercase__				= truncation

									lowercase__				= generate_kwargs

									lowercase__				= {}
									if return_tensors is not None and return_type is None:
											lowercase__				= ReturnType.TENSORS if return_tensors else ReturnType.TEXT
									if return_type is not None:
											lowercase__				= return_type

									if clean_up_tokenization_spaces is not None:
											lowercase__				= clean_up_tokenization_spaces

									if stop_sequence is not None:
											lowercase__				= self.tokenizer.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)
											if len(UpperCamelCase_				) > 1:
													warnings.warn(
													    '''Stopping on a multiple token sequence is not yet supported on transformers. The first token of'''
													    ''' the stop sequence will be used as the stop sequence string in the interim.'''				)
											lowercase__				= stop_sequence_ids[0]

									return preprocess_params, forward_params, postprocess_params

							def  lowerCamelCase_					(				self:      Dict       ,		UpperCamelCase_:      int       ,		UpperCamelCase_:      int       ,		UpperCamelCase_:      int				)     ->  Optional[int]:





									"""simple docstring"""







									return True

							def  lowerCamelCase_					(				self:      List[Any]       ,		*UpperCamelCase_:      Optional[Any]       ,		UpperCamelCase_:      Any				)     ->  Tuple:





									"""simple docstring"""







									lowercase__				= self.model.config.prefix if self.model.config.prefix is not None else ''''''
									if isinstance(args[0]       ,		UpperCamelCase_				):
											if self.tokenizer.pad_token_id is None:
													raise ValueError('''Please make sure that the tokenizer has a pad_token_id when using a batch input'''				)
											lowercase__				= ([prefix + arg for arg in args[0]],)
											lowercase__				= True

									elif isinstance(args[0]       ,		UpperCamelCase_				):
											lowercase__				= (prefix + args[0],)
											lowercase__				= False
									else:
											raise ValueError(
											    f' `args[0]`: {args[0]} have the wrong format. The should be either of type `str` or type `list`'				)
									lowercase__				= self.tokenizer(*UpperCamelCase_       ,		padding=UpperCamelCase_       ,		truncation=UpperCamelCase_       ,		return_tensors=self.framework				)
									# This is produced by tokenizers but is an invalid generate kwargs
									if "token_type_ids" in inputs:
											del inputs["token_type_ids"]
									return inputs

							def __call__(				self:      int       ,		*UpperCamelCase_:      List[str]       ,		**UpperCamelCase_:      Optional[Any]				)     ->  Optional[Any]:





									"""simple docstring"""







									lowercase__				= super().__call__(*UpperCamelCase_       ,		**UpperCamelCase_				)
									if (
									    isinstance(args[0]       ,		UpperCamelCase_				)
									    and all(isinstance(UpperCamelCase_       ,		UpperCamelCase_				) for el in args[0]				)
									    and all(len(UpperCamelCase_				) == 1 for res in result				)
									):
											return [res[0] for res in result]
									return result

							def  lowerCamelCase_					(				self:      Optional[Any]       ,		UpperCamelCase_:      Dict       ,		UpperCamelCase_:      Any=TruncationStrategy.DO_NOT_TRUNCATE       ,		**UpperCamelCase_:      Dict				)     ->  Union[str, Any]:





									"""simple docstring"""







									lowercase__				= self._parse_and_tokenize(UpperCamelCase_       ,		truncation=UpperCamelCase_       ,		**UpperCamelCase_				)
									return inputs

							def  lowerCamelCase_					(				self:      Union[str, Any]       ,		UpperCamelCase_:      Optional[int]       ,		**UpperCamelCase_:      int				)     ->  List[Any]:





									"""simple docstring"""







									if self.framework == "pt":
											lowercase__	,						lowercase__				= model_inputs['''input_ids'''].shape
									elif self.framework == "tf":
											lowercase__	,						lowercase__				= tf.shape(model_inputs['''input_ids''']				).numpy()

									lowercase__				= generate_kwargs.get('''min_length'''       ,		self.model.config.min_length				)
									lowercase__				= generate_kwargs.get('''max_length'''       ,		self.model.config.max_length				)
									self.check_inputs(UpperCamelCase_       ,		generate_kwargs['''min_length''']       ,		generate_kwargs['''max_length''']				)
									lowercase__				= self.model.generate(**UpperCamelCase_       ,		**UpperCamelCase_				)
									lowercase__				= output_ids.shape[0]
									if self.framework == "pt":
											lowercase__				= output_ids.reshape(UpperCamelCase_       ,		out_b // in_b       ,		*output_ids.shape[1:]				)
									elif self.framework == "tf":
											lowercase__				= tf.reshape(UpperCamelCase_       ,		(in_b, out_b // in_b, *output_ids.shape[1:])				)
									return {"output_ids": output_ids}


							def  lowerCamelCase_					(				self:      str       ,		UpperCamelCase_:      Union[str, Any]       ,		UpperCamelCase_:      int=ReturnType.TEXT       ,		UpperCamelCase_:      Union[str, Any]=False				)     ->  Union[str, Any]:





									"""simple docstring"""







									lowercase__				= []
									for output_ids in model_outputs["output_ids"][0]:
											if return_type == ReturnType.TENSORS:
													lowercase__				= {f'{self.return_name}_token_ids': output_ids}
											elif return_type == ReturnType.TEXT:
													lowercase__				= {
													    f'{self.return_name}_text': self.tokenizer.decode(
													        UpperCamelCase_       ,		skip_special_tokens=UpperCamelCase_       ,		clean_up_tokenization_spaces=UpperCamelCase_       ,		)
													}
											records.append(UpperCamelCase_				)
									return records


@add_end_docstrings(UpperCamelCase__     )
class 			_a				(     UpperCamelCase__     ):
							_lowercase      :     int     =					'''summary'''

							def __call__(				self:      Any       ,		*UpperCamelCase_:      Tuple       ,		**UpperCamelCase_:      str				)     ->  Tuple:





									"""simple docstring"""







									return super().__call__(*UpperCamelCase_       ,		**UpperCamelCase_				)


							def  lowerCamelCase_					(				self:      Optional[Any]       ,		UpperCamelCase_:      int       ,		UpperCamelCase_:      int       ,		UpperCamelCase_:      int				)     ->  bool:





									"""simple docstring"""







									if max_length < min_length:
											logger.warning(f'Your min_length={min_length} must be inferior than your max_length={max_length}.'				)

									if input_length < max_length:
											logger.warning(
											    f'Your max_length is set to {max_length}, but your input_length is only {input_length}. Since this is '
											    '''a summarization task, where outputs shorter than the input are typically wanted, you might '''
											    f'consider decreasing max_length manually, e.g. summarizer(\'...\', max_length={input_length//2})'				)


@add_end_docstrings(UpperCamelCase__     )
class 			_a				(     UpperCamelCase__     ):
							_lowercase      :     Union[str, Any]     =					'''translation'''

							def  lowerCamelCase_					(				self:      Optional[Any]       ,		UpperCamelCase_:      int       ,		UpperCamelCase_:      int       ,		UpperCamelCase_:      int				)     ->  Dict:





									"""simple docstring"""







									if input_length > 0.9 * max_length:
											logger.warning(
											    f'Your input_length: {input_length} is bigger than 0.9 * max_length: {max_length}. You might consider '
											    '''increasing your max_length manually, e.g. translator(\'...\', max_length=400)'''				)
									return True

							def  lowerCamelCase_					(				self:      List[Any]       ,		*UpperCamelCase_:      Optional[int]       ,		UpperCamelCase_:      int=TruncationStrategy.DO_NOT_TRUNCATE       ,		UpperCamelCase_:      Any=None       ,		UpperCamelCase_:      List[str]=None				)     ->  Any:





									"""simple docstring"""







									if getattr(self.tokenizer       ,		'''_build_translation_inputs'''       ,		UpperCamelCase_				):
											return self.tokenizer._build_translation_inputs(
											    *UpperCamelCase_       ,		return_tensors=self.framework       ,		truncation=UpperCamelCase_       ,		src_lang=UpperCamelCase_       ,		tgt_lang=UpperCamelCase_				)
									else:
											return super()._parse_and_tokenize(*UpperCamelCase_       ,		truncation=UpperCamelCase_				)

							def  lowerCamelCase_					(				self:      str       ,		UpperCamelCase_:      Union[str, Any]=None       ,		UpperCamelCase_:      str=None       ,		**UpperCamelCase_:      Dict				)     ->  Dict:





									"""simple docstring"""







									lowercase__	,						lowercase__	,						lowercase__				= super()._sanitize_parameters(**UpperCamelCase_				)
									if src_lang is not None:
											lowercase__				= src_lang
									if tgt_lang is not None:
											lowercase__				= tgt_lang
									if src_lang is None and tgt_lang is None:
											# Backward compatibility, direct arguments use is preferred.
											lowercase__				= kwargs.get('''task'''       ,		self.task				)
											lowercase__				= task.split('''_'''				)
											if task and len(UpperCamelCase_				) == 4:
													# translation, XX, to YY
													lowercase__				= items[1]
													lowercase__				= items[3]
									return preprocess_params, forward_params, postprocess_params


							def __call__(				self:      Optional[Any]       ,		*UpperCamelCase_:      Tuple       ,		**UpperCamelCase_:      Dict				)     ->  Tuple:





									"""simple docstring"""







									return super().__call__(*UpperCamelCase_       ,		**UpperCamelCase_				)




import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__A							=get_tests_dir('''fixtures/spiece.model''')




@require_sentencepiece
@require_tokenizers
class        _SCREAMING_SNAKE_CASE	( snake_case_     ,			unittest.TestCase		):
	lowerCAmelCase__   =					AlbertTokenizer
	lowerCAmelCase__   =					AlbertTokenizerFast
	lowerCAmelCase__   =					True
	lowerCAmelCase__   =					True
	lowerCAmelCase__   =					True



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Any:
					super().setUp()

					# We have a SentencePiece fixture for testing
					lowerCamelCase_         =     AlbertTokenizer(lowercase					)
					tokenizer.save_pretrained(self.tmpdirname					)



	def        SCREAMING_SNAKE_CASE_(       self		,    lowercase					)							->			str:
					lowerCamelCase_         =     "this is a test"
					lowerCamelCase_         =     "this is a test"
					return input_text, output_text



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Dict:
					lowerCamelCase_         =     "<pad>"
					lowerCamelCase_         =     0

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase					)		,    lowercase					)
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase					)		,    lowercase					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Any:
					lowerCamelCase_         =     list(self.get_tokenizer().get_vocab().keys()					)

					self.assertEqual(vocab_keys[0]		,    "<pad>"					)
					self.assertEqual(vocab_keys[1]		,    "<unk>"					)
					self.assertEqual(vocab_keys[-1]		,    "▁eloquent"					)
					self.assertEqual(len(lowercase					)		,    30000					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Union[str, Any]:
					self.assertEqual(self.get_tokenizer().vocab_size		,    30000					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Union[str, Any]:
					if not self.test_rust_tokenizer:
									return

					lowerCamelCase_         =     self.get_tokenizer()
					lowerCamelCase_         =     self.get_rust_tokenizer()

					lowerCamelCase_         =     "I was born in 92000, and this is falsé."

					lowerCamelCase_         =     tokenizer.tokenize(lowercase					)
					lowerCamelCase_         =     rust_tokenizer.tokenize(lowercase					)
					self.assertListEqual(lowercase		,    lowercase					)

					lowerCamelCase_         =     tokenizer.encode(lowercase		,    add_special_tokens=lowercase					)
					lowerCamelCase_         =     rust_tokenizer.encode(lowercase		,    add_special_tokens=lowercase					)
					self.assertListEqual(lowercase		,    lowercase					)

					lowerCamelCase_         =     self.get_rust_tokenizer()
					lowerCamelCase_         =     tokenizer.encode(lowercase					)
					lowerCamelCase_         =     rust_tokenizer.encode(lowercase					)
					self.assertListEqual(lowercase		,    lowercase					)



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Optional[Any]:
					lowerCamelCase_         =     AlbertTokenizer(lowercase		,    keep_accents=lowercase					)

					lowerCamelCase_         =     tokenizer.tokenize("This is a test"					)
					self.assertListEqual(lowercase		,    ["▁this", "▁is", "▁a", "▁test"]					)

					self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase					)		,    [48, 25, 21, 1289]					)

					lowerCamelCase_         =     tokenizer.tokenize("I was born in 92000, and this is falsé."					)
					self.assertListEqual(
					    lowercase		,    ["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", "."]					)
					lowerCamelCase_         =     tokenizer.convert_tokens_to_ids(lowercase					)
					self.assertListEqual(lowercase		,    [31, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]					)

					lowerCamelCase_         =     tokenizer.convert_ids_to_tokens(lowercase					)
					self.assertListEqual(
					    lowercase		,    ["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "."]		,    )



	def        SCREAMING_SNAKE_CASE_(       self					)							->			Any:
					lowerCamelCase_         =     AlbertTokenizer(lowercase					)

					lowerCamelCase_         =     tokenizer.encode("sequence builders"					)
					lowerCamelCase_         =     tokenizer.encode("multi-sequence build"					)

					lowerCamelCase_         =     tokenizer.build_inputs_with_special_tokens(lowercase					)
					lowerCamelCase_         =     tokenizer.build_inputs_with_special_tokens(lowercase		,    lowercase					)

					assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
					assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
					    tokenizer.sep_token_id
					]



	@slow
	def        SCREAMING_SNAKE_CASE_(       self					)							->			Optional[Any]:
					# fmt: off
					lowerCamelCase_         =     {"attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "input_ids": [[2, 21970, 13, 5, 6092, 167, 28, 7103, 2153, 673, 8, 7028, 12051, 18, 17, 7103, 2153, 673, 8, 3515, 18684, 8, 4461, 6, 1927, 297, 8, 12060, 2607, 18, 13, 5, 4461, 15, 10538, 38, 8, 135, 15, 822, 58, 15, 993, 10363, 15, 1460, 8005, 4461, 15, 993, 255, 2328, 9, 9, 9, 6, 26, 1112, 816, 3260, 13, 5, 103, 2377, 6, 17, 1112, 816, 2782, 13, 5, 103, 10641, 6, 29, 84, 2512, 2430, 782, 18684, 2761, 19, 808, 2430, 2556, 17, 855, 1480, 9477, 4091, 128, 11712, 15, 7103, 2153, 673, 17, 24883, 9990, 9, 3], [2, 11502, 25, 1006, 20, 782, 8, 11809, 855, 1732, 19393, 18667, 37, 367, 21018, 69, 1854, 34, 11860, 19124, 27, 156, 225, 17, 193, 4141, 19, 65, 9124, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 2231, 886, 2385, 17659, 84, 14, 16792, 1952, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
					# fmt: on

					self.tokenizer_integration_test_util(
					    expected_encoding=lowercase		,    model_name="albert-base-v2"		,    revision="6b6560eaf5ff2e250b00c50f380c5389a9c2d82e"		,    )


