





import unittest

from transformers import XLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    XLMForMultipleChoice,
					    XLMForQuestionAnswering,
					    XLMForQuestionAnsweringSimple,
					    XLMForSequenceClassification,
					    XLMForTokenClassification,
					    XLMModel,
					    XLMWithLMHeadModel,
					)
					from transformers.models.xlm.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST


class     A     :
			'''simple docstring'''




			def __init__(     self  :       int				,      __lowerCAmelCase  :       Optional[Any]				,      __lowerCAmelCase  :       List[str]=13				,      __lowerCAmelCase  :       Tuple=7				,      __lowerCAmelCase  :       Optional[Any]=True				,      __lowerCAmelCase  :       Optional[int]=True				,      __lowerCAmelCase  :       Any=True				,      __lowerCAmelCase  :       Tuple=True				,      __lowerCAmelCase  :       Dict=True				,      __lowerCAmelCase  :       Union[str, Any]=False				,      __lowerCAmelCase  :       Tuple=False				,      __lowerCAmelCase  :       Optional[int]=False				,      __lowerCAmelCase  :       Any=2				,      __lowerCAmelCase  :       int=99				,      __lowerCAmelCase  :       List[str]=0				,      __lowerCAmelCase  :       List[str]=32				,      __lowerCAmelCase  :       Union[str, Any]=5				,      __lowerCAmelCase  :       List[Any]=4				,      __lowerCAmelCase  :       Tuple=0.1				,      __lowerCAmelCase  :       List[str]=0.1				,      __lowerCAmelCase  :       Dict=5_12				,      __lowerCAmelCase  :       List[Any]=2				,      __lowerCAmelCase  :       Any=0.0_2				,      __lowerCAmelCase  :       str=2				,      __lowerCAmelCase  :       Tuple=4				,      __lowerCAmelCase  :       Union[str, Any]="last"				,      __lowerCAmelCase  :       str=True				,      __lowerCAmelCase  :       Any=None				,      __lowerCAmelCase  :       List[Any]=0				,      )      ->      Dict:







					"""simple docstring"""






					A__     =  parent
					A__     =  batch_size
					A__     =  seq_length
					A__     =  is_training
					A__     =  use_input_lengths
					A__     =  use_token_type_ids
					A__     =  use_labels
					A__     =  gelu_activation
					A__     =  sinusoidal_embeddings
					A__     =  causal
					A__     =  asm
					A__     =  n_langs
					A__     =  vocab_size
					A__     =  n_special
					A__     =  hidden_size
					A__     =  num_hidden_layers
					A__     =  num_attention_heads
					A__     =  hidden_dropout_prob
					A__     =  attention_probs_dropout_prob
					A__     =  max_position_embeddings
					A__     =  type_sequence_label_size
					A__     =  initializer_range
					A__     =  num_labels
					A__     =  num_choices
					A__     =  summary_type
					A__     =  use_proj
					A__     =  scope
					A__     =  bos_token_id






			def   a_			(     self  :       Dict  )      ->      Optional[Any]:







					"""simple docstring"""






					A__     =  ids_tensor([self.batch_size, self.seq_length]				,      self.vocab_size  )
					A__     =  random_attention_mask([self.batch_size, self.seq_length]  )

					A__     =  None
					if self.use_input_lengths:
							A__     =  (
							    ids_tensor([self.batch_size]				,      vocab_size=2  ) + self.seq_length - 2
							)  # small variation of seq_length

					A__     =  None
					if self.use_token_type_ids:
							A__     =  ids_tensor([self.batch_size, self.seq_length]				,      self.n_langs  )

					A__     =  None
					A__     =  None
					A__     =  None
					if self.use_labels:
							A__     =  ids_tensor([self.batch_size]				,      self.type_sequence_label_size  )
							A__     =  ids_tensor([self.batch_size, self.seq_length]				,      self.num_labels  )
							A__     =  ids_tensor([self.batch_size]				,      2  ).float()
							A__     =  ids_tensor([self.batch_size]				,      self.num_choices  )

					A__     =  self.get_config()

					return (
					    config,
					    input_ids,
					    token_type_ids,
					    input_lengths,
					    sequence_labels,
					    token_labels,
					    is_impossible_labels,
					    choice_labels,
					    input_mask,
					)






			def   a_			(     self  :       Optional[int]  )      ->      Tuple:







					"""simple docstring"""






					return XLMConfig(
					    vocab_size=self.vocab_size				,      n_special=self.n_special				,      emb_dim=self.hidden_size				,      n_layers=self.num_hidden_layers				,      n_heads=self.num_attention_heads				,      dropout=self.hidden_dropout_prob				,      attention_dropout=self.attention_probs_dropout_prob				,      gelu_activation=self.gelu_activation				,      sinusoidal_embeddings=self.sinusoidal_embeddings				,      asm=self.asm				,      causal=self.causal				,      n_langs=self.n_langs				,      max_position_embeddings=self.max_position_embeddings				,      initializer_range=self.initializer_range				,      summary_type=self.summary_type				,      use_proj=self.use_proj				,      num_labels=self.num_labels				,      bos_token_id=self.bos_token_id				,      )






			def   a_			(     self  :       Tuple				,      __lowerCAmelCase  :       Optional[int]				,      __lowerCAmelCase  :       Any				,      __lowerCAmelCase  :       List[str]				,      __lowerCAmelCase  :       Any				,      __lowerCAmelCase  :       Union[str, Any]				,      __lowerCAmelCase  :       List[str]				,      __lowerCAmelCase  :       Optional[Any]				,      __lowerCAmelCase  :       Tuple				,      __lowerCAmelCase  :       int				,      )      ->      Tuple:







					"""simple docstring"""






					A__     =  XLMModel(config=__lowerCAmelCase  )
					model.to(__lowerCAmelCase  )
					model.eval()
					A__     =  model(__lowerCAmelCase				,      lengths=__lowerCAmelCase				,      langs=__lowerCAmelCase  )
					A__     =  model(__lowerCAmelCase				,      langs=__lowerCAmelCase  )
					A__     =  model(__lowerCAmelCase  )
					self.parent.assertEqual(result.last_hidden_state.shape				,      (self.batch_size, self.seq_length, self.hidden_size)  )






			def   a_			(     self  :       Optional[int]				,      __lowerCAmelCase  :       Any				,      __lowerCAmelCase  :       Any				,      __lowerCAmelCase  :       Tuple				,      __lowerCAmelCase  :       Union[str, Any]				,      __lowerCAmelCase  :       Any				,      __lowerCAmelCase  :       Union[str, Any]				,      __lowerCAmelCase  :       Dict				,      __lowerCAmelCase  :       Dict				,      __lowerCAmelCase  :       List[Any]				,      )      ->      int:







					"""simple docstring"""






					A__     =  XLMWithLMHeadModel(__lowerCAmelCase  )
					model.to(__lowerCAmelCase  )
					model.eval()

					A__     =  model(__lowerCAmelCase				,      token_type_ids=__lowerCAmelCase				,      labels=__lowerCAmelCase  )
					self.parent.assertEqual(result.loss.shape				,      ()  )
					self.parent.assertEqual(result.logits.shape				,      (self.batch_size, self.seq_length, self.vocab_size)  )






			def   a_			(     self  :       Optional[int]				,      __lowerCAmelCase  :       Union[str, Any]				,      __lowerCAmelCase  :       Optional[int]				,      __lowerCAmelCase  :       List[Any]				,      __lowerCAmelCase  :       Tuple				,      __lowerCAmelCase  :       str				,      __lowerCAmelCase  :       List[str]				,      __lowerCAmelCase  :       int				,      __lowerCAmelCase  :       List[str]				,      __lowerCAmelCase  :       Tuple				,      )      ->      int:







					"""simple docstring"""






					A__     =  XLMForQuestionAnsweringSimple(__lowerCAmelCase  )
					model.to(__lowerCAmelCase  )
					model.eval()

					A__     =  model(__lowerCAmelCase  )

					A__     =  model(__lowerCAmelCase				,      start_positions=__lowerCAmelCase				,      end_positions=__lowerCAmelCase  )
					A__     =  outputs
					self.parent.assertEqual(result.start_logits.shape				,      (self.batch_size, self.seq_length)  )
					self.parent.assertEqual(result.end_logits.shape				,      (self.batch_size, self.seq_length)  )






			def   a_			(     self  :       List[str]				,      __lowerCAmelCase  :       Dict				,      __lowerCAmelCase  :       Optional[Any]				,      __lowerCAmelCase  :       List[Any]				,      __lowerCAmelCase  :       Dict				,      __lowerCAmelCase  :       Optional[Any]				,      __lowerCAmelCase  :       List[Any]				,      __lowerCAmelCase  :       Dict				,      __lowerCAmelCase  :       Any				,      __lowerCAmelCase  :       Union[str, Any]				,      )      ->      Union[str, Any]:







					"""simple docstring"""






					A__     =  XLMForQuestionAnswering(__lowerCAmelCase  )
					model.to(__lowerCAmelCase  )
					model.eval()

					A__     =  model(__lowerCAmelCase  )

					A__     =  model(
					    __lowerCAmelCase				,      start_positions=__lowerCAmelCase				,      end_positions=__lowerCAmelCase				,      cls_index=__lowerCAmelCase				,      is_impossible=__lowerCAmelCase				,      p_mask=__lowerCAmelCase				,      )

					A__     =  model(
					    __lowerCAmelCase				,      start_positions=__lowerCAmelCase				,      end_positions=__lowerCAmelCase				,      cls_index=__lowerCAmelCase				,      is_impossible=__lowerCAmelCase				,      )

					((A__) ,     )     =  result_with_labels.to_tuple()

					A__     =  model(__lowerCAmelCase				,      start_positions=__lowerCAmelCase				,      end_positions=__lowerCAmelCase  )

					((A__) ,     )     =  result_with_labels.to_tuple()

					self.parent.assertEqual(result_with_labels.loss.shape				,      ()  )
					self.parent.assertEqual(result.start_top_log_probs.shape				,      (self.batch_size, model.config.start_n_top)  )
					self.parent.assertEqual(result.start_top_index.shape				,      (self.batch_size, model.config.start_n_top)  )
					self.parent.assertEqual(
					    result.end_top_log_probs.shape				,      (self.batch_size, model.config.start_n_top * model.config.end_n_top)  )
					self.parent.assertEqual(
					    result.end_top_index.shape				,      (self.batch_size, model.config.start_n_top * model.config.end_n_top)  )
					self.parent.assertEqual(result.cls_logits.shape				,      (self.batch_size,)  )






			def   a_			(     self  :       int				,      __lowerCAmelCase  :       Optional[int]				,      __lowerCAmelCase  :       Union[str, Any]				,      __lowerCAmelCase  :       List[str]				,      __lowerCAmelCase  :       List[str]				,      __lowerCAmelCase  :       str				,      __lowerCAmelCase  :       Dict				,      __lowerCAmelCase  :       Tuple				,      __lowerCAmelCase  :       Optional[Any]				,      __lowerCAmelCase  :       Optional[Any]				,      )      ->      Dict:







					"""simple docstring"""






					A__     =  XLMForSequenceClassification(__lowerCAmelCase  )
					model.to(__lowerCAmelCase  )
					model.eval()

					A__     =  model(__lowerCAmelCase  )
					A__     =  model(__lowerCAmelCase				,      labels=__lowerCAmelCase  )
					self.parent.assertEqual(result.loss.shape				,      ()  )
					self.parent.assertEqual(result.logits.shape				,      (self.batch_size, self.type_sequence_label_size)  )






			def   a_			(     self  :       List[Any]				,      __lowerCAmelCase  :       List[str]				,      __lowerCAmelCase  :       Dict				,      __lowerCAmelCase  :       Any				,      __lowerCAmelCase  :       Optional[int]				,      __lowerCAmelCase  :       List[str]				,      __lowerCAmelCase  :       Optional[Any]				,      __lowerCAmelCase  :       Dict				,      __lowerCAmelCase  :       Dict				,      __lowerCAmelCase  :       str				,      )      ->      Tuple:







					"""simple docstring"""






					A__     =  self.num_labels
					A__     =  XLMForTokenClassification(__lowerCAmelCase  )
					model.to(__lowerCAmelCase  )
					model.eval()

					A__     =  model(__lowerCAmelCase				,      attention_mask=__lowerCAmelCase				,      labels=__lowerCAmelCase  )
					self.parent.assertEqual(result.logits.shape				,      (self.batch_size, self.seq_length, self.num_labels)  )






			def   a_			(     self  :       Optional[Any]				,      __lowerCAmelCase  :       Any				,      __lowerCAmelCase  :       int				,      __lowerCAmelCase  :       int				,      __lowerCAmelCase  :       Union[str, Any]				,      __lowerCAmelCase  :       Any				,      __lowerCAmelCase  :       Optional[Any]				,      __lowerCAmelCase  :       Union[str, Any]				,      __lowerCAmelCase  :       str				,      __lowerCAmelCase  :       List[str]				,      )      ->      List[str]:







					"""simple docstring"""






					A__     =  self.num_choices
					A__     =  XLMForMultipleChoice(config=__lowerCAmelCase  )
					model.to(__lowerCAmelCase  )
					model.eval()
					A__     =  input_ids.unsqueeze(1  ).expand(-1				,      self.num_choices				,      -1  ).contiguous()
					A__     =  token_type_ids.unsqueeze(1  ).expand(-1				,      self.num_choices				,      -1  ).contiguous()
					A__     =  input_mask.unsqueeze(1  ).expand(-1				,      self.num_choices				,      -1  ).contiguous()
					A__     =  model(
					    __lowerCAmelCase				,      attention_mask=__lowerCAmelCase				,      token_type_ids=__lowerCAmelCase				,      labels=__lowerCAmelCase				,      )
					self.parent.assertEqual(result.logits.shape				,      (self.batch_size, self.num_choices)  )







			def   a_			(     self  :       Optional[int]  )      ->      Optional[Any]:







					"""simple docstring"""






					A__     =  self.prepare_config_and_inputs()
					(
					    (
					    A__
					) ,     (
					    A__
					) ,     (
					    A__
					) ,     (
					    A__
					) ,     (
					    A__
					) ,     (
					    A__
					) ,     (
					    A__
					) ,     (
					    A__
					) ,     (
					    A__
					) ,     
					)     =  config_and_inputs
					A__     =  {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """lengths""": input_lengths}
					return config, inputs_dict


@require_torch
class     A     (SCREAMING_SNAKE_CASE  ,	SCREAMING_SNAKE_CASE  ,	SCREAMING_SNAKE_CASE  ,	unittest.TestCase       ):
			'''simple docstring'''




			__lowerCamelCase      :			Tuple							     =       (
			    (
			        XLMModel,
			        XLMWithLMHeadModel,
			        XLMForQuestionAnswering,
			        XLMForSequenceClassification,
			        XLMForQuestionAnsweringSimple,
			        XLMForTokenClassification,
			        XLMForMultipleChoice,
			    )
			    if is_torch_available()
			    else ()
			)
			__lowerCamelCase      :			int							     =       (
			    (XLMWithLMHeadModel,) if is_torch_available() else ()
			)  # TODO (PVP): Check other models whether language generation is also applicable
			__lowerCamelCase      :			Any							     =       (
			    {
			        '''feature-extraction''': XLMModel,
			        '''fill-mask''': XLMWithLMHeadModel,
			        '''question-answering''': XLMForQuestionAnsweringSimple,
			        '''text-classification''': XLMForSequenceClassification,
			        '''text-generation''': XLMWithLMHeadModel,
			        '''token-classification''': XLMForTokenClassification,
			        '''zero-shot''': XLMForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)






			def   a_			(     self  :       Optional[Any]				,      __lowerCAmelCase  :       List[Any]				,      __lowerCAmelCase  :       int				,      __lowerCAmelCase  :       Any				,      __lowerCAmelCase  :       Tuple				,      __lowerCAmelCase  :       Tuple  )      ->      Optional[int]:







					"""simple docstring"""






					if (
					    pipeline_test_casse_name == "QAPipelineTests"
					    and tokenizer_name is not None
					    and not tokenizer_name.endswith("""Fast"""  )
					):
							# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
							# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
							# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
							return True

					return False






			def   a_			(     self  :       Union[str, Any]				,      __lowerCAmelCase  :       Tuple				,      __lowerCAmelCase  :       Union[str, Any]				,      __lowerCAmelCase  :       Optional[Any]=False  )      ->      Union[str, Any]:







					"""simple docstring"""






					A__     =  super()._prepare_for_class(__lowerCAmelCase				,      __lowerCAmelCase				,      return_labels=__lowerCAmelCase  )

					if return_labels:
							if model_class.__name__ == "XLMForQuestionAnswering":
									A__     =  torch.zeros(
									    self.model_tester.batch_size				,      dtype=torch.long				,      device=__lowerCAmelCase  )
									A__     =  torch.zeros(
									    self.model_tester.batch_size				,      dtype=torch.long				,      device=__lowerCAmelCase  )

					return inputs_dict






			def   a_			(     self  :       Dict  )      ->      int:







					"""simple docstring"""






					A__     =  XLMModelTester(self  )
					A__     =  ConfigTester(self				,      config_class=__lowerCAmelCase				,      emb_dim=37  )






			def   a_			(     self  :       Optional[int]  )      ->      Any:







					"""simple docstring"""






					self.config_tester.run_common_tests()






			def   a_			(     self  :       Union[str, Any]  )      ->      Optional[int]:







					"""simple docstring"""






					A__     =  self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_xlm_model(*__lowerCAmelCase  )






			def   a_			(     self  :       List[str]  )      ->      Optional[Any]:







					"""simple docstring"""






					A__     =  self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_xlm_lm_head(*__lowerCAmelCase  )






			def   a_			(     self  :       str  )      ->      Dict:







					"""simple docstring"""






					A__     =  self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_xlm_simple_qa(*__lowerCAmelCase  )






			def   a_			(     self  :       str  )      ->      str:







					"""simple docstring"""






					A__     =  self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_xlm_qa(*__lowerCAmelCase  )






			def   a_			(     self  :       str  )      ->      Union[str, Any]:







					"""simple docstring"""






					A__     =  self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_xlm_sequence_classif(*__lowerCAmelCase  )






			def   a_			(     self  :       Union[str, Any]  )      ->      str:







					"""simple docstring"""






					A__     =  self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_xlm_token_classif(*__lowerCAmelCase  )






			def   a_			(     self  :       Tuple  )      ->      List[Any]:







					"""simple docstring"""






					A__     =  self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_xlm_for_multiple_choice(*__lowerCAmelCase  )






			def   a_			(     self  :       Optional[Any]				,      __lowerCAmelCase  :       Optional[Any]				,      __lowerCAmelCase  :       Dict				,      __lowerCAmelCase  :       Union[str, Any]				,      __lowerCAmelCase  :       str				,      __lowerCAmelCase  :       int				,      __lowerCAmelCase  :       Dict=False				,      __lowerCAmelCase  :       Tuple=1  )      ->      str:







					"""simple docstring"""






					self.assertIsInstance(__lowerCAmelCase				,      __lowerCAmelCase  )
					self.assertListEqual(
					    [isinstance(__lowerCAmelCase				,      __lowerCAmelCase  ) for iter_attentions in attentions]				,      [True] * len(__lowerCAmelCase  )  )
					self.assertEqual(len(__lowerCAmelCase  )				,      (max_length - min_length) * num_beam_groups  )

					for idx, iter_attentions in enumerate(__lowerCAmelCase  ):
							# adds PAD dummy token
							A__     =  min_length + idx + 1
							A__     =  min_length + idx + 1

							A__     =  (
							    batch_size * num_beam_groups,
							    config.num_attention_heads,
							    tgt_len,
							    src_len,
							)
							# check attn size
							self.assertListEqual(
							    [layer_attention.shape for layer_attention in iter_attentions]				,      [expected_shape] * len(__lowerCAmelCase  )  )






			def   a_			(     self  :       Optional[int]				,      __lowerCAmelCase  :       str				,      __lowerCAmelCase  :       Any				,      __lowerCAmelCase  :       Optional[int]				,      __lowerCAmelCase  :       Optional[int]				,      __lowerCAmelCase  :       List[Any]				,      __lowerCAmelCase  :       Optional[Any]=False				,      __lowerCAmelCase  :       Dict=1  )      ->      List[str]:







					"""simple docstring"""






					self.assertIsInstance(__lowerCAmelCase				,      __lowerCAmelCase  )
					self.assertListEqual(
					    [isinstance(__lowerCAmelCase				,      __lowerCAmelCase  ) for iter_hidden_states in hidden_states]				,      [True] * len(__lowerCAmelCase  )				,      )
					self.assertEqual(len(__lowerCAmelCase  )				,      (max_length - min_length) * num_beam_groups  )

					for idx, iter_hidden_states in enumerate(__lowerCAmelCase  ):
							# adds PAD dummy token
							A__     =  min_length + idx + 1
							A__     =  (batch_size * num_beam_groups, seq_len, config.hidden_size)
							# check hidden size
							self.assertListEqual(
							    [layer_hidden_states.shape for layer_hidden_states in iter_hidden_states]				,      [expected_shape] * len(__lowerCAmelCase  )				,      )
					pass







			@slow
			def   a_			(     self  :       Optional[int]  )      ->      List[str]:







					"""simple docstring"""






					for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							A__     =  XLMModel.from_pretrained(__lowerCAmelCase  )
							self.assertIsNotNone(__lowerCAmelCase  )


@require_torch
class     A     (unittest.TestCase       ):
			'''simple docstring'''




			@slow
			def   a_			(     self  :       List[Any]  )      ->      int:







					"""simple docstring"""






					A__     =  XLMWithLMHeadModel.from_pretrained("""xlm-mlm-en-2048"""  )
					model.to(__lowerCAmelCase  )
					A__     =  torch.tensor([[14, 4_47]]				,      dtype=torch.long				,      device=__lowerCAmelCase  )  # the president
					A__     =  [
					    14,
					    4_47,
					    14,
					    4_47,
					    14,
					    4_47,
					    14,
					    4_47,
					    14,
					    4_47,
					    14,
					    4_47,
					    14,
					    4_47,
					    14,
					    4_47,
					    14,
					    4_47,
					    14,
					    4_47,
					]  # the president the president the president the president the president the president the president the president the president the president
					# TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
					A__     =  model.generate(__lowerCAmelCase				,      do_sample=__lowerCAmelCase  )
					self.assertListEqual(output_ids[0].cpu().numpy().tolist()				,      __lowerCAmelCase  )










import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
					A       :   Optional[Any]					= '''pt'''
elif is_tf_available():
					A       :   List[Any]					= '''tf'''
else:
					A       :   Union[str, Any]					= '''jax'''


class     A     (SCREAMING_SNAKE_CASE  ,	unittest.TestCase       ):
			'''simple docstring'''




			__lowerCamelCase      :			str							     =       ByTaTokenizer
			__lowerCamelCase      :			Tuple							     =       False






			def   a_			(     self  :       Dict  )      ->      Dict:







					"""simple docstring"""






					super().setUp()
					A__     =  ByTaTokenizer()
					tokenizer.save_pretrained(self.tmpdirname  )






			@cached_property
			def   a_			(     self  :       Dict  )      ->      List[Any]:







					"""simple docstring"""






					return ByTaTokenizer.from_pretrained("""google/byt5-small"""  )






			def   a_			(     self  :       Union[str, Any]				,      **__lowerCAmelCase  :       int  )      ->      ByTaTokenizer:







					"""simple docstring"""






					return self.tokenizer_class.from_pretrained(self.tmpdirname				,      **__lowerCAmelCase  )






			def   a_			(     self  :       Optional[Any]				,      __lowerCAmelCase  :       Dict				,      __lowerCAmelCase  :       Union[str, Any]=False				,      __lowerCAmelCase  :       Dict=20				,      __lowerCAmelCase  :       List[str]=5  )      ->      Tuple[str, list]:







					"""simple docstring"""






					A__     =  []
					for i in range(len(__lowerCAmelCase  )  ):
							try:
									A__     =  tokenizer.decode([i]				,      clean_up_tokenization_spaces=__lowerCAmelCase  )
							except UnicodeDecodeError:
									pass
							toks.append((i, tok)  )

					A__     =  list(filter(lambda __lowerCAmelCase  : re.match(R"""^[ a-zA-Z]+$"""				,      t[1]  )				,      __lowerCAmelCase  )  )
					A__     =  list(filter(lambda __lowerCAmelCase  : [t[0]] == tokenizer.encode(t[1]				,      add_special_tokens=__lowerCAmelCase  )				,      __lowerCAmelCase  )  )
					if max_length is not None and len(__lowerCAmelCase  ) > max_length:
							A__     =  toks[:max_length]
					if min_length is not None and len(__lowerCAmelCase  ) < min_length and len(__lowerCAmelCase  ) > 0:
							while len(__lowerCAmelCase  ) < min_length:
									A__     =  toks + toks
        # toks_str = [t[1] for t in toks]
					A__     =  [t[0] for t in toks]

					# Ensure consistency
					A__     =  tokenizer.decode(__lowerCAmelCase				,      clean_up_tokenization_spaces=__lowerCAmelCase  )
					if " " not in output_txt and len(__lowerCAmelCase  ) > 1:
							A__     =  (
							    tokenizer.decode([toks_ids[0]]				,      clean_up_tokenization_spaces=__lowerCAmelCase  )
							    + """ """
							    + tokenizer.decode(toks_ids[1:]				,      clean_up_tokenization_spaces=__lowerCAmelCase  )
							)
					if with_prefix_space:
							A__     =  """ """ + output_txt
					A__     =  tokenizer.encode(__lowerCAmelCase				,      add_special_tokens=__lowerCAmelCase  )
					return output_txt, output_ids






			def   a_			(     self  :       List[str]  )      ->      int:







					"""simple docstring"""






					A__     =  self.ta_base_tokenizer
					A__     =  tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]  )
					A__     =  tokenizer(["""hi""", """I went to the gym""", """"""]  )
					self.assertListEqual(batch_with_eos_added["""input_ids"""]				,      batch_without_eos_added["""input_ids"""]  )






			def   a_			(     self  :       Union[str, Any]  )      ->      Tuple:







					"""simple docstring"""






					A__     =  self.ta_base_tokenizer
					A__     =  """Unicode €."""
					A__     =  tokenizer(__lowerCAmelCase  )
					A__     =  [88, 1_13, 1_08, 1_02, 1_14, 1_03, 1_04, 35, 2_29, 1_33, 1_75, 49, 1]
					self.assertEqual(encoded["""input_ids"""]				,      __lowerCAmelCase  )

					# decoding
					A__     =  tokenizer.decode(__lowerCAmelCase  )
					self.assertEqual(__lowerCAmelCase				,      """Unicode €.</s>"""  )

					A__     =  tokenizer("""e è é ê ë"""  )
					A__     =  [1_04, 35, 1_98, 1_71, 35, 1_98, 1_72, 35, 1_98, 1_73, 35, 1_98, 1_74, 1]
					self.assertEqual(encoded["""input_ids"""]				,      __lowerCAmelCase  )
					# decoding
					A__     =  tokenizer.decode(__lowerCAmelCase  )
					self.assertEqual(__lowerCAmelCase				,      """e è é ê ë</s>"""  )

					# encode/decode, but with `encode` instead of `__call__`
					self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""  )  )				,      """e è é ê ë</s>"""  )






			def   a_			(     self  :       Dict  )      ->      Optional[int]:







					"""simple docstring"""






					A__     =  self.ta_base_tokenizer
					A__     =  ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
					# fmt: off
					A__     =  [68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 1, 0]
					# fmt: on
					A__     =  tokenizer(__lowerCAmelCase				,      padding=__lowerCAmelCase				,      return_tensors=__lowerCAmelCase  )
					self.assertIsInstance(__lowerCAmelCase				,      __lowerCAmelCase  )

					if FRAMEWORK != "jax":
							A__     =  list(batch.input_ids.numpy()[0]  )
					else:
							A__     =  list(batch.input_ids.tolist()[0]  )

					self.assertListEqual(__lowerCAmelCase				,      __lowerCAmelCase  )

					self.assertEqual((2, 37)				,      batch.input_ids.shape  )
					self.assertEqual((2, 37)				,      batch.attention_mask.shape  )






			def   a_			(     self  :       Optional[Any]  )      ->      List[str]:







					"""simple docstring"""






					A__     =  self.ta_base_tokenizer
					A__     =  ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
					A__     =  tokenizer(__lowerCAmelCase				,      padding=__lowerCAmelCase				,      return_tensors=__lowerCAmelCase  )
					# check if input_ids are returned and no decoder_input_ids
					self.assertIn("""input_ids"""				,      __lowerCAmelCase  )
					self.assertIn("""attention_mask"""				,      __lowerCAmelCase  )
					self.assertNotIn("""decoder_input_ids"""				,      __lowerCAmelCase  )
					self.assertNotIn("""decoder_attention_mask"""				,      __lowerCAmelCase  )






			def   a_			(     self  :       int  )      ->      Any:







					"""simple docstring"""






					A__     =  self.ta_base_tokenizer
					A__     =  [
					    """Summary of the text.""",
					    """Another summary.""",
					]
					A__     =  tokenizer(
					    text_target=__lowerCAmelCase				,      max_length=32				,      padding="""max_length"""				,      truncation=__lowerCAmelCase				,      return_tensors=__lowerCAmelCase  )
					self.assertEqual(32				,      targets["""input_ids"""].shape[1]  )






			def   a_			(     self  :       List[str]  )      ->      Union[str, Any]:







					"""simple docstring"""






					A__     =  self.ta_base_tokenizer
					A__     =  ["""A long paragraph for summarization. </s>"""]
					A__     =  ["""Summary of the text. </s>"""]
					# fmt: off
					A__     =  [68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 35, 1]
					A__     =  [86, 1_20, 1_12, 1_12, 1_00, 1_17, 1_24, 35, 1_14, 1_05, 35, 1_19, 1_07, 1_04, 35, 1_19, 1_04, 1_23, 1_19, 49, 35, 1]
					# fmt: on

					A__     =  tokenizer(__lowerCAmelCase				,      text_target=__lowerCAmelCase  )

					self.assertEqual(__lowerCAmelCase				,      batch["""input_ids"""][0]  )
					self.assertEqual(__lowerCAmelCase				,      batch["""labels"""][0]  )






			def   a_			(     self  :       str  )      ->      Dict:







					"""simple docstring"""






					A__     =  self.get_tokenizers()
					for tokenizer in tokenizers:
							with self.subTest(f'{tokenizer.__class__.__name__}'  ):
									self.assertNotEqual(tokenizer.model_max_length				,      42  )

        # Now let's start the test
					A__     =  self.get_tokenizers()
					for tokenizer in tokenizers:
							with self.subTest(f'{tokenizer.__class__.__name__}'  ):
									# Isolate this from the other tests because we save additional tokens/etc
									A__     =  tempfile.mkdtemp()

									A__     =  """ He is very happy, UNwant\u00E9d,running"""
									A__     =  tokenizer.encode(__lowerCAmelCase				,      add_special_tokens=__lowerCAmelCase  )
									tokenizer.save_pretrained(__lowerCAmelCase  )

									A__     =  tokenizer.__class__.from_pretrained(__lowerCAmelCase  )
									A__     =  after_tokenizer.encode(__lowerCAmelCase				,      add_special_tokens=__lowerCAmelCase  )
									self.assertListEqual(__lowerCAmelCase				,      __lowerCAmelCase  )

									shutil.rmtree(__lowerCAmelCase  )

					A__     =  self.get_tokenizers(model_max_length=42  )
					for tokenizer in tokenizers:
							with self.subTest(f'{tokenizer.__class__.__name__}'  ):
									# Isolate this from the other tests because we save additional tokens/etc
									A__     =  tempfile.mkdtemp()

									A__     =  """ He is very happy, UNwant\u00E9d,running"""
									tokenizer.add_tokens(["""bim""", """bambam"""]  )
									A__     =  tokenizer.additional_special_tokens
									additional_special_tokens.append("""new_additional_special_token"""  )
									tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}  )
									A__     =  tokenizer.encode(__lowerCAmelCase				,      add_special_tokens=__lowerCAmelCase  )
									tokenizer.save_pretrained(__lowerCAmelCase  )

									A__     =  tokenizer.__class__.from_pretrained(__lowerCAmelCase  )
									A__     =  after_tokenizer.encode(__lowerCAmelCase				,      add_special_tokens=__lowerCAmelCase  )
									self.assertListEqual(__lowerCAmelCase				,      __lowerCAmelCase  )
									self.assertIn("""new_additional_special_token"""				,      after_tokenizer.additional_special_tokens  )
									self.assertEqual(after_tokenizer.model_max_length				,      42  )

									A__     =  tokenizer.__class__.from_pretrained(__lowerCAmelCase				,      model_max_length=43  )
									self.assertEqual(tokenizer.model_max_length				,      43  )

									shutil.rmtree(__lowerCAmelCase  )






			def   a_			(     self  :       Optional[int]  )      ->      str:







					"""simple docstring"""






					A__     =  []
					if self.test_slow_tokenizer:
							tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())  )

					if self.test_rust_tokenizer:
							tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())  )

					for tokenizer_class, tokenizer_utils in tokenizer_list:
							with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer_utils.save_pretrained(__lowerCAmelCase  )

									with open(os.path.join(__lowerCAmelCase				,      """special_tokens_map.json"""  )				,      encoding="""utf-8"""  ) as json_file:
											A__     =  json.load(__lowerCAmelCase  )

									with open(os.path.join(__lowerCAmelCase				,      """tokenizer_config.json"""  )				,      encoding="""utf-8"""  ) as json_file:
											A__     =  json.load(__lowerCAmelCase  )

									A__     =  [f'<extra_id_{i}>' for i in range(1_25  )]

									A__     =  added_tokens_extra_ids + [
									    """an_additional_special_token"""
									]
									A__     =  added_tokens_extra_ids + [
									    """an_additional_special_token"""
									]

									with open(os.path.join(__lowerCAmelCase				,      """special_tokens_map.json"""  )				,      """w"""				,      encoding="""utf-8"""  ) as outfile:
											json.dump(__lowerCAmelCase				,      __lowerCAmelCase  )
									with open(os.path.join(__lowerCAmelCase				,      """tokenizer_config.json"""  )				,      """w"""				,      encoding="""utf-8"""  ) as outfile:
											json.dump(__lowerCAmelCase				,      __lowerCAmelCase  )

									# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
									# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
									# "special_tokens_map.json" files
									A__     =  tokenizer_class.from_pretrained(
									    __lowerCAmelCase				,      )
									self.assertIn(
									    """an_additional_special_token"""				,      tokenizer_without_change_in_init.additional_special_tokens  )
									# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
									self.assertEqual(
									    ["""an_additional_special_token"""]				,      tokenizer_without_change_in_init.convert_ids_to_tokens(
									        tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]  )  )				,      )

									# Now we test that we can change the value of additional_special_tokens in the from_pretrained
									A__     =  added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""				,      lstrip=__lowerCAmelCase  )]
									A__     =  tokenizer_class.from_pretrained(
									    __lowerCAmelCase				,      additional_special_tokens=__lowerCAmelCase				,      )

									self.assertIn("""a_new_additional_special_token"""				,      tokenizer.additional_special_tokens  )
									self.assertEqual(
									    ["""a_new_additional_special_token"""]				,      tokenizer.convert_ids_to_tokens(
									        tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]  )  )				,      )






			def   a_			(     self  :       Tuple  )      ->      Optional[Any]:







					"""simple docstring"""






					A__     =  []
					if self.test_slow_tokenizer:
							tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())  )

					if self.test_rust_tokenizer:
							tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())  )

					for tokenizer_class, tokenizer_utils in tokenizer_list:
							with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer_utils.save_pretrained(__lowerCAmelCase  )

									A__     =  tokenizer_class.from_pretrained(__lowerCAmelCase  )

									self.assertTrue(tokenizer.decode([2_55]  ) == """"""  )






			def   a_			(     self  :       List[Any]  )      ->      Tuple:







					"""simple docstring"""






					pass






			def   a_			(     self  :       Optional[int]  )      ->      Any:







					"""simple docstring"""






					pass






			def   a_			(     self  :       int  )      ->      Optional[Any]:







					"""simple docstring"""






					pass






			def   a_			(     self  :       str  )      ->      Optional[int]:







					"""simple docstring"""






					pass






			def   a_			(     self  :       int  )      ->      Dict:







					"""simple docstring"""






					A__     =  self.get_tokenizers(fast=__lowerCAmelCase				,      do_lower_case=__lowerCAmelCase  )
					for tokenizer in tokenizers:
							with self.subTest(f'{tokenizer.__class__.__name__}'  ):
									A__     =  ["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
									A__     =  tokenizer.convert_tokens_to_string(__lowerCAmelCase  )

									self.assertIsInstance(__lowerCAmelCase				,      __lowerCAmelCase  )







			def   a_			(     self  :       List[Any]  )      ->      int:







					"""simple docstring"""






					A__     =  self.get_tokenizers()
					for tokenizer in tokenizers:
							with self.subTest(f'{tokenizer.__class__.__name__}'  ):
									A__     =  [
									    """bos_token""",
									    """eos_token""",
									    """unk_token""",
									    """sep_token""",
									    """pad_token""",
									    """cls_token""",
									    """mask_token""",
									]

									A__     =  0
									A__     =  tokenizer.convert_ids_to_tokens(
									    __lowerCAmelCase				,      skip_special_tokens=__lowerCAmelCase  )

									for attr in attributes_list:
											setattr(__lowerCAmelCase				,      attr + """_id"""				,      __lowerCAmelCase  )
											self.assertEqual(getattr(__lowerCAmelCase				,      __lowerCAmelCase  )				,      __lowerCAmelCase  )
											self.assertEqual(getattr(__lowerCAmelCase				,      attr + """_id"""  )				,      __lowerCAmelCase  )

											setattr(__lowerCAmelCase				,      attr + """_id"""				,      __lowerCAmelCase  )
											self.assertEqual(getattr(__lowerCAmelCase				,      __lowerCAmelCase  )				,      __lowerCAmelCase  )
											self.assertEqual(getattr(__lowerCAmelCase				,      attr + """_id"""  )				,      __lowerCAmelCase  )

									setattr(__lowerCAmelCase				,      """additional_special_tokens_ids"""				,      []  )
									self.assertListEqual(getattr(__lowerCAmelCase				,      """additional_special_tokens"""  )				,      []  )
									self.assertListEqual(getattr(__lowerCAmelCase				,      """additional_special_tokens_ids"""  )				,      []  )

									setattr(__lowerCAmelCase				,      """additional_special_tokens_ids"""				,      [token_id_to_test_setters]  )
									self.assertListEqual(getattr(__lowerCAmelCase				,      """additional_special_tokens"""  )				,      [token_to_test_setters]  )
									self.assertListEqual(getattr(__lowerCAmelCase				,      """additional_special_tokens_ids"""  )				,      [token_id_to_test_setters]  )



