
"""simple docstring"""
import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE_					     =	get_tests_dir('''fixtures/test_sentencepiece.model''')
@require_sentencepiece
class       lowerCAmelCase_		(				A__				,				unittest.TestCase ):



							'''simple docstring'''





							_snake_case						      =		XLMProphetNetTokenizer
							_snake_case						      =		False
							_snake_case						      =		True




							def    A__						( self				)       ->  Optional[Any]:
													super().setUp()

													# We have a SentencePiece fixture for testing
													__lowerCAmelCase     =  XLMProphetNetTokenizer(snake_case_    ,				keep_accents=snake_case_				)
													tokenizer.save_pretrained(self.tmpdirname				)




							def    A__						( self				)       ->  Any:
													__lowerCAmelCase     =  """[PAD]"""
													__lowerCAmelCase     =  0

													self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case_				)    ,				snake_case_				)
													self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case_				)    ,				snake_case_				)




							def    A__						( self				)       ->  List[Any]:
													__lowerCAmelCase     =  list(self.get_tokenizer().get_vocab().keys()				)

													self.assertEqual(vocab_keys[0]    ,				"""[PAD]"""				)
													self.assertEqual(vocab_keys[1]    ,				"""[CLS]"""				)
													self.assertEqual(vocab_keys[-1]    ,				"""j"""				)
													self.assertEqual(len(snake_case_				)    ,				1_012				)




							def    A__						( self				)       ->  Any:
													self.assertEqual(self.get_tokenizer().vocab_size    ,				1_012				)




							def    A__						( self				)       ->  str:
													__lowerCAmelCase     =  XLMProphetNetTokenizer(snake_case_    ,				keep_accents=snake_case_				)

													__lowerCAmelCase     =  tokenizer.tokenize("""This is a test"""				)
													self.assertListEqual(snake_case_    ,				["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]				)

													self.assertListEqual(
													    tokenizer.convert_tokens_to_ids(snake_case_				)    ,				[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]    ,				)

													__lowerCAmelCase     =  tokenizer.tokenize("""I was born in 92000, and this is falsé."""				)
													self.assertListEqual(
													    snake_case_    ,				[
													        SPIECE_UNDERLINE + """I""",
													        SPIECE_UNDERLINE + """was""",
													        SPIECE_UNDERLINE + """b""",
													        """or""",
													        """n""",
													        SPIECE_UNDERLINE + """in""",
													        SPIECE_UNDERLINE + """""",
													        """9""",
													        """2""",
													        """0""",
													        """0""",
													        """0""",
													        """,""",
													        SPIECE_UNDERLINE + """and""",
													        SPIECE_UNDERLINE + """this""",
													        SPIECE_UNDERLINE + """is""",
													        SPIECE_UNDERLINE + """f""",
													        """al""",
													        """s""",
													        """é""",
													        """.""",
													    ]    ,				)
													__lowerCAmelCase     =  tokenizer.convert_tokens_to_ids(snake_case_				)
													self.assertListEqual(
													    snake_case_    ,				[
													        value + tokenizer.fairseq_offset
													        for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
													    ]    ,				)

													__lowerCAmelCase     =  tokenizer.convert_ids_to_tokens(snake_case_				)
													self.assertListEqual(
													    snake_case_    ,				[
													        SPIECE_UNDERLINE + """I""",
													        SPIECE_UNDERLINE + """was""",
													        SPIECE_UNDERLINE + """b""",
													        """or""",
													        """n""",
													        SPIECE_UNDERLINE + """in""",
													        SPIECE_UNDERLINE + """""",
													        """[UNK]""",
													        """2""",
													        """0""",
													        """0""",
													        """0""",
													        """,""",
													        SPIECE_UNDERLINE + """and""",
													        SPIECE_UNDERLINE + """this""",
													        SPIECE_UNDERLINE + """is""",
													        SPIECE_UNDERLINE + """f""",
													        """al""",
													        """s""",
													        """[UNK]""",
													        """.""",
													    ]    ,				)




							@cached_property
							def    A__						( self				)       ->  Tuple:
													return XLMProphetNetTokenizer.from_pretrained("""microsoft/xprophetnet-large-wiki100-cased"""				)




							@slow
							def    A__						( self				)       ->  Optional[int]:
													__lowerCAmelCase     =  """Hello World!"""
													__lowerCAmelCase     =  [35_389, 6_672, 49, 2]
													self.assertListEqual(snake_case_    ,				self.big_tokenizer.encode(snake_case_				)				)




							@slow
							def    A__						( self				)       ->  Union[str, Any]:
													# fmt: off
													__lowerCAmelCase     =  {"""input_ids""": [[11_073, 82_783, 18, 26, 82_783, 549, 51_540, 248, 17_209, 1_301, 217, 20, 215_186, 1_325, 147, 17_209, 1_301, 217, 20, 56_370, 53, 122_020, 20, 16_477, 27, 87_355, 4_548, 20, 4_728, 78_392, 17, 159_969, 18, 26, 24_491, 629, 15, 538, 22_704, 5_439, 15, 2_788, 24_491, 9_885, 15, 43_534, 605, 15, 814, 18_403, 33_200, 29, 15, 43_534, 24_458, 12_410, 111, 24_966, 83_669, 9_637, 144_068, 26, 850, 22_346, 27, 147, 24_966, 83_669, 83_490, 26, 39_113, 735, 27, 689, 656, 2_800, 1_339, 4_600, 53, 122_020, 115_785, 34, 816, 1_339, 46_887, 18, 147, 53_905, 1_951, 42_238, 41_170, 17_732, 834, 436, 15, 27_523, 98_733, 217, 147, 5_542, 4_981, 930, 17_347, 16, 2], [20_091, 629, 94, 82_786, 58, 490, 20, 1_528, 84, 53_905, 344, 80_592, 110_128, 18_822, 5_267, 1_306, 62, 152_537, 308, 7_997, 401, 124_427, 549, 35_442, 225, 109, 15_055, 25_748, 147, 7_119, 43_712, 34, 767, 135_366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63_784, 119_466, 17, 147_808, 88_214, 18, 656, 81, 32, 3_296, 10_280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
													# fmt: on

													self.tokenizer_integration_test_util(
													    expected_encoding=snake_case_    ,				model_name="""microsoft/xprophetnet-large-wiki100-cased"""    ,				revision="""1acad1643ddd54a44df6a1b797ada8373685d90e"""    ,				)


"""simple docstring"""
import enum
import warnings

from ..tokenization_utils import TruncationStrategy
from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_tf_available():
			import tensorflow as tf

			from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

if is_torch_available():
			from ..models.auto.modeling_auto import MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

SCREAMING_SNAKE_CASE_					     =	logging.get_logger(__name__)
class       lowerCAmelCase_		(				enum.Enum ):



							'''simple docstring'''





							_snake_case						      =		0
							_snake_case						      =		1
@add_end_docstrings(A__ )
class       lowerCAmelCase_		(				A__ ):



							'''simple docstring'''





							_snake_case						      =		'''generated'''




							def __init__( self    ,				*snake_case_    ,				**snake_case_				)       ->  Optional[int]:
													super().__init__(*snake_case_    ,				**snake_case_				)

													self.check_model_type(
													    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
													    if self.framework == """tf"""
													    else MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING				)




							def    A__						( self    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				snake_case_=None    ,				**snake_case_    ,				)       ->  Union[str, Any]:
													__lowerCAmelCase     =  {}
													if truncation is not None:
																			__lowerCAmelCase     =  truncation

													__lowerCAmelCase     =  generate_kwargs

													__lowerCAmelCase     =  {}
													if return_tensors is not None and return_type is None:
																			__lowerCAmelCase     =  ReturnType.TENSORS if return_tensors else ReturnType.TEXT
													if return_type is not None:
																			__lowerCAmelCase     =  return_type

													if clean_up_tokenization_spaces is not None:
																			__lowerCAmelCase     =  clean_up_tokenization_spaces

													if stop_sequence is not None:
																			__lowerCAmelCase     =  self.tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)
																			if len(snake_case_				) > 1:
																									warnings.warn(
																									    """Stopping on a multiple token sequence is not yet supported on transformers. The first token of"""
																									    """ the stop sequence will be used as the stop sequence string in the interim."""				)
																			__lowerCAmelCase     =  stop_sequence_ids[0]

													return preprocess_params, forward_params, postprocess_params




							def    A__						( self    ,				snake_case_    ,				snake_case_    ,				snake_case_				)       ->  Any:
													return True




							def    A__						( self    ,				*snake_case_    ,				snake_case_				)       ->  Dict:
													__lowerCAmelCase     =  self.model.config.prefix if self.model.config.prefix is not None else """"""
													if isinstance(args[0]    ,				snake_case_				):
																			if self.tokenizer.pad_token_id is None:
																									raise ValueError("""Please make sure that the tokenizer has a pad_token_id when using a batch input"""				)
																			__lowerCAmelCase     =  ([prefix + arg for arg in args[0]],)
																			__lowerCAmelCase     =  True

													elif isinstance(args[0]    ,				snake_case_				):
																			__lowerCAmelCase     =  (prefix + args[0],)
																			__lowerCAmelCase     =  False
													else:
																			raise ValueError(
																			    f""" `args[0]`: {args[0]} have the wrong format. The should be either of type `str` or type `list`"""				)
													__lowerCAmelCase     =  self.tokenizer(*snake_case_    ,				padding=snake_case_    ,				truncation=snake_case_    ,				return_tensors=self.framework				)
													# This is produced by tokenizers but is an invalid generate kwargs
													if "token_type_ids" in inputs:
																			del inputs["token_type_ids"]
													return inputs




							def __call__( self    ,				*snake_case_    ,				**snake_case_				)       ->  Dict:
													__lowerCAmelCase     =  super().__call__(*snake_case_    ,				**snake_case_				)
													if (
													    isinstance(args[0]    ,				snake_case_				)
													    and all(isinstance(snake_case_    ,				snake_case_				) for el in args[0]				)
													    and all(len(snake_case_				) == 1 for res in result				)
													):
																			return [res[0] for res in result]
													return result




							def    A__						( self    ,				snake_case_    ,				snake_case_=TruncationStrategy.DO_NOT_TRUNCATE    ,				**snake_case_				)       ->  Tuple:
													__lowerCAmelCase     =  self._parse_and_tokenize(snake_case_    ,				truncation=snake_case_    ,				**snake_case_				)
													return inputs




							def    A__						( self    ,				snake_case_    ,				**snake_case_				)       ->  Union[str, Any]:
													if self.framework == "pt":
																			__lowerCAmelCase   ,     __lowerCAmelCase     =  model_inputs["""input_ids"""].shape
													elif self.framework == "tf":
																			__lowerCAmelCase   ,     __lowerCAmelCase     =  tf.shape(model_inputs["""input_ids"""]				).numpy()

													__lowerCAmelCase     =  generate_kwargs.get("""min_length"""    ,				self.model.config.min_length				)
													__lowerCAmelCase     =  generate_kwargs.get("""max_length"""    ,				self.model.config.max_length				)
													self.check_inputs(snake_case_    ,				generate_kwargs["""min_length"""]    ,				generate_kwargs["""max_length"""]				)
													__lowerCAmelCase     =  self.model.generate(**snake_case_    ,				**snake_case_				)
													__lowerCAmelCase     =  output_ids.shape[0]
													if self.framework == "pt":
																			__lowerCAmelCase     =  output_ids.reshape(snake_case_    ,				out_b // in_b    ,				*output_ids.shape[1:]				)
													elif self.framework == "tf":
																			__lowerCAmelCase     =  tf.reshape(snake_case_    ,				(in_b, out_b // in_b, *output_ids.shape[1:])				)
													return {"output_ids": output_ids}




							def    A__						( self    ,				snake_case_    ,				snake_case_=ReturnType.TEXT    ,				snake_case_=False				)       ->  Dict:
													__lowerCAmelCase     =  []
													for output_ids in model_outputs["output_ids"][0]:
																			if return_type == ReturnType.TENSORS:
																									__lowerCAmelCase     =  {f"""{self.return_name}_token_ids""": output_ids}
																			elif return_type == ReturnType.TEXT:
																									__lowerCAmelCase     =  {
																									    f"""{self.return_name}_text""": self.tokenizer.decode(
																									        snake_case_    ,				skip_special_tokens=snake_case_    ,				clean_up_tokenization_spaces=snake_case_    ,				)
																									}
																			records.append(snake_case_				)
													return records
@add_end_docstrings(A__ )
class       lowerCAmelCase_		(				A__ ):



							'''simple docstring'''





							_snake_case						      =		'''summary'''




							def __call__( self    ,				*snake_case_    ,				**snake_case_				)       ->  Tuple:
													return super().__call__(*snake_case_    ,				**snake_case_				)




							def    A__						( self    ,				snake_case_    ,				snake_case_    ,				snake_case_				)       ->  bool:
													if max_length < min_length:
																			logger.warning(f"""Your min_length={min_length} must be inferior than your max_length={max_length}."""				)

													if input_length < max_length:
																			logger.warning(
																			    f"""Your max_length is set to {max_length}, but your input_length is only {input_length}. Since this is """
																			    """a summarization task, where outputs shorter than the input are typically wanted, you might """
																			    f"""consider decreasing max_length manually, e.g. summarizer('...', max_length={input_length//2})"""				)
@add_end_docstrings(A__ )
class       lowerCAmelCase_		(				A__ ):



							'''simple docstring'''





							_snake_case						      =		'''translation'''




							def    A__						( self    ,				snake_case_    ,				snake_case_    ,				snake_case_				)       ->  Optional[int]:
													if input_length > 0.9 * max_length:
																			logger.warning(
																			    f"""Your input_length: {input_length} is bigger than 0.9 * max_length: {max_length}. You might consider """
																			    """increasing your max_length manually, e.g. translator('...', max_length=400)"""				)
													return True




							def    A__						( self    ,				*snake_case_    ,				snake_case_=TruncationStrategy.DO_NOT_TRUNCATE    ,				snake_case_=None    ,				snake_case_=None				)       ->  List[Any]:
													if getattr(self.tokenizer    ,				"""_build_translation_inputs"""    ,				snake_case_				):
																			return self.tokenizer._build_translation_inputs(
																			    *snake_case_    ,				return_tensors=self.framework    ,				truncation=snake_case_    ,				src_lang=snake_case_    ,				tgt_lang=snake_case_				)
													else:
																			return super()._parse_and_tokenize(*snake_case_    ,				truncation=snake_case_				)




							def    A__						( self    ,				snake_case_=None    ,				snake_case_=None    ,				**snake_case_				)       ->  List[Any]:
													__lowerCAmelCase   ,     __lowerCAmelCase   ,     __lowerCAmelCase     =  super()._sanitize_parameters(**snake_case_				)
													if src_lang is not None:
																			__lowerCAmelCase     =  src_lang
													if tgt_lang is not None:
																			__lowerCAmelCase     =  tgt_lang
													if src_lang is None and tgt_lang is None:
																			# Backward compatibility, direct arguments use is preferred.
																			__lowerCAmelCase     =  kwargs.get("""task"""    ,				self.task				)
																			__lowerCAmelCase     =  task.split("""_"""				)
																			if task and len(snake_case_				) == 4:
																									# translation, XX, to YY
																									__lowerCAmelCase     =  items[1]
																									__lowerCAmelCase     =  items[3]
													return preprocess_params, forward_params, postprocess_params




							def __call__( self    ,				*snake_case_    ,				**snake_case_				)       ->  List[Any]:
													return super().__call__(*snake_case_    ,				**snake_case_				)
