






"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


UpperCAmelCase_     :				Tuple         =				logging.get_logger(__name__)

UpperCAmelCase_     :				Tuple         =				{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase_     :				Optional[int]         =				{
    """vocab_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/vocab.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/vocab.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/vocab.json""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/merges.txt""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/merges.txt""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/merges.txt""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/tokenizer.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/tokenizer.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json""",
        """roberta-base-openai-detector""": (
            """https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"""
        ),
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCAmelCase_     :				Optional[Any]         =				{
    """roberta-base""": 512,
    """roberta-large""": 512,
    """roberta-large-mnli""": 512,
    """distilroberta-base""": 512,
    """roberta-base-openai-detector""": 512,
    """roberta-large-openai-detector""": 512,
}



class 							lowerCAmelCase__       (    __lowerCamelCase				):






      '''simple docstring'''

      __UpperCamelCase							=						VOCAB_FILES_NAMES
      __UpperCamelCase							=						PRETRAINED_VOCAB_FILES_MAP
      __UpperCamelCase							=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __UpperCamelCase							=						["input_ids", "attention_mask"]
      __UpperCamelCase							=						RobertaTokenizer
      def __init__(		self  :						List[Any]		,			lowercase_  :						Any=None		,			lowercase_  :						Dict=None		,			lowercase_  :						Any=None		,			lowercase_  :						Any="replace"		,			lowercase_  :						Optional[int]="<s>"		,			lowercase_  :						List[str]="</s>"		,			lowercase_  :						int="</s>"		,			lowercase_  :						str="<s>"		,			lowercase_  :						Optional[Any]="<unk>"		,			lowercase_  :						Optional[int]="<pad>"		,			lowercase_  :						Optional[int]="<mask>"		,			lowercase_  :						Any=False		,			lowercase_  :						Any=True		,			**lowercase_  :						Any		,			):




          '''simple docstring'''


          super().__init__(
              SCREAMING_SNAKE_CASE_		,			SCREAMING_SNAKE_CASE_		,			tokenizer_file=SCREAMING_SNAKE_CASE_		,			errors=SCREAMING_SNAKE_CASE_		,			bos_token=SCREAMING_SNAKE_CASE_		,			eos_token=SCREAMING_SNAKE_CASE_		,			sep_token=SCREAMING_SNAKE_CASE_		,			cls_token=SCREAMING_SNAKE_CASE_		,			unk_token=SCREAMING_SNAKE_CASE_		,			pad_token=SCREAMING_SNAKE_CASE_		,			mask_token=SCREAMING_SNAKE_CASE_		,			add_prefix_space=SCREAMING_SNAKE_CASE_		,			trim_offsets=SCREAMING_SNAKE_CASE_		,			**SCREAMING_SNAKE_CASE_		,			)

          SCREAMING_SNAKE_CASE_   :						List[str]							=	json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
          if pre_tok_state.get('''add_prefix_space'''		,			SCREAMING_SNAKE_CASE_) != add_prefix_space:
              SCREAMING_SNAKE_CASE_   :						List[str]							=	getattr(SCREAMING_SNAKE_CASE_		,			pre_tok_state.pop('''type'''))
              SCREAMING_SNAKE_CASE_   :						Optional[int]							=	add_prefix_space
              SCREAMING_SNAKE_CASE_   :						List[Any]							=	pre_tok_class(**SCREAMING_SNAKE_CASE_)

          SCREAMING_SNAKE_CASE_   :						int							=	add_prefix_space

          SCREAMING_SNAKE_CASE_   :						int							=	'''post_processor'''
          SCREAMING_SNAKE_CASE_   :						str							=	getattr(self.backend_tokenizer		,			SCREAMING_SNAKE_CASE_		,			SCREAMING_SNAKE_CASE_)
          if tokenizer_component_instance:
              SCREAMING_SNAKE_CASE_   :						List[Any]							=	json.loads(tokenizer_component_instance.__getstate__())

              # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
              if "sep" in state:
                  SCREAMING_SNAKE_CASE_   :						Any							=	tuple(state['''sep'''])
              if "cls" in state:
                  SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	tuple(state['''cls'''])

              SCREAMING_SNAKE_CASE_   :						Any							=	False

              if state.get('''add_prefix_space'''		,			SCREAMING_SNAKE_CASE_) != add_prefix_space:
                  SCREAMING_SNAKE_CASE_   :						Tuple							=	add_prefix_space
                  SCREAMING_SNAKE_CASE_   :						Tuple							=	True

              if state.get('''trim_offsets'''		,			SCREAMING_SNAKE_CASE_) != trim_offsets:
                  SCREAMING_SNAKE_CASE_   :						List[str]							=	trim_offsets
                  SCREAMING_SNAKE_CASE_   :						int							=	True

              if changes_to_apply:
                  SCREAMING_SNAKE_CASE_   :						str							=	getattr(SCREAMING_SNAKE_CASE_		,			state.pop('''type'''))
                  SCREAMING_SNAKE_CASE_   :						List[str]							=	component_class(**SCREAMING_SNAKE_CASE_)
                  setattr(self.backend_tokenizer		,			SCREAMING_SNAKE_CASE_		,			SCREAMING_SNAKE_CASE_)
      @property
      def _SCREAMING_SNAKE_CASE				(		self  :						int):




          '''simple docstring'''


          if self._mask_token is None:
              if self.verbose:
                  logger.error('''Using mask_token, but it is not set yet.''')
              return None
          return str(self._mask_token)
      @mask_token.setter
      def _SCREAMING_SNAKE_CASE				(		self  :						List[str]		,			lowercase_  :						Tuple):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Dict							=	AddedToken(SCREAMING_SNAKE_CASE_		,			lstrip=SCREAMING_SNAKE_CASE_		,			rstrip=SCREAMING_SNAKE_CASE_) if isinstance(SCREAMING_SNAKE_CASE_		,			SCREAMING_SNAKE_CASE_) else value
          SCREAMING_SNAKE_CASE_   :						str							=	value
      def _SCREAMING_SNAKE_CASE				(		self  :						Dict		,			*lowercase_  :						Union[str, Any]		,			**lowercase_  :						int):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						List[str]							=	kwargs.get('''is_split_into_words'''		,			SCREAMING_SNAKE_CASE_)
          assert self.add_prefix_space or not is_split_into_words, (
              F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
              "to use it with pretokenized inputs."
          )

          return super()._batch_encode_plus(*SCREAMING_SNAKE_CASE_		,			**SCREAMING_SNAKE_CASE_)
      def _SCREAMING_SNAKE_CASE				(		self  :						int		,			*lowercase_  :						Optional[Any]		,			**lowercase_  :						Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Any							=	kwargs.get('''is_split_into_words'''		,			SCREAMING_SNAKE_CASE_)

          assert self.add_prefix_space or not is_split_into_words, (
              F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
              "to use it with pretokenized inputs."
          )

          return super()._encode_plus(*SCREAMING_SNAKE_CASE_		,			**SCREAMING_SNAKE_CASE_)
      def _SCREAMING_SNAKE_CASE				(		self  :						Optional[Any]		,			lowercase_  :						int		,			lowercase_  :						Tuple = None):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						int							=	self._tokenizer.model.save(SCREAMING_SNAKE_CASE_		,			name=SCREAMING_SNAKE_CASE_)
          return tuple(SCREAMING_SNAKE_CASE_)
      def _SCREAMING_SNAKE_CASE				(		self  :						str		,			lowercase_  :						Any		,			lowercase_  :						List[Any]=None):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	[self.bos_token_id] + token_ids_a + [self.eos_token_id]
          if token_ids_a is None:
              return output

          return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



      def _SCREAMING_SNAKE_CASE				(		self  :						str		,			lowercase_  :						int		,			lowercase_  :						Optional[int] = None):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						int							=	[self.sep_token_id]
          SCREAMING_SNAKE_CASE_   :						int							=	[self.cls_token_id]

          if token_ids_a is None:
              return len(cls + token_ids_a + sep) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]










"""simple docstring"""






import unittest

import numpy as np

from transformers.testing_utils import require_flax, require_tf, require_torch
from transformers.utils import (
    expand_dims,
    flatten_dict,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    reshape,
    squeeze,
    transpose,
)


if is_flax_available():
  import jax.numpy as jnp

if is_tf_available():
  import tensorflow as tf

if is_torch_available():
  import torch




class 							lowerCAmelCase__       (    unittest.TestCase				):






      '''simple docstring'''

      def _SCREAMING_SNAKE_CASE				(		self  :						str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						str							=	{
              '''task_specific_params''': {
                  '''summarization''': {'''length_penalty''': 1.0, '''max_length''': 128, '''min_length''': 12, '''num_beams''': 4},
                  '''summarization_cnn''': {'''length_penalty''': 2.0, '''max_length''': 142, '''min_length''': 56, '''num_beams''': 4},
                  '''summarization_xsum''': {'''length_penalty''': 1.0, '''max_length''': 62, '''min_length''': 11, '''num_beams''': 6},
              }
          }
          SCREAMING_SNAKE_CASE_   :						List[str]							=	{
              '''task_specific_params.summarization.length_penalty''': 1.0,
              '''task_specific_params.summarization.max_length''': 128,
              '''task_specific_params.summarization.min_length''': 12,
              '''task_specific_params.summarization.num_beams''': 4,
              '''task_specific_params.summarization_cnn.length_penalty''': 2.0,
              '''task_specific_params.summarization_cnn.max_length''': 142,
              '''task_specific_params.summarization_cnn.min_length''': 56,
              '''task_specific_params.summarization_cnn.num_beams''': 4,
              '''task_specific_params.summarization_xsum.length_penalty''': 1.0,
              '''task_specific_params.summarization_xsum.max_length''': 62,
              '''task_specific_params.summarization_xsum.min_length''': 11,
              '''task_specific_params.summarization_xsum.num_beams''': 6,
          }

          self.assertEqual(flatten_dict(lowercase_)		,			lowercase_)
      def _SCREAMING_SNAKE_CASE				(		self  :						List[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	np.random.randn(3		,			4)
          self.assertTrue(np.allclose(transpose(lowercase_)		,			x.transpose()))

          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	np.random.randn(3		,			4		,			5)
          self.assertTrue(np.allclose(transpose(lowercase_		,			axes=(1, 2, 0))		,			x.transpose((1, 2, 0))))
      @require_torch
      def _SCREAMING_SNAKE_CASE				(		self  :						Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						str							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						str							=	torch.tensor(lowercase_)
          self.assertTrue(np.allclose(transpose(lowercase_)		,			transpose(lowercase_).numpy()))

          SCREAMING_SNAKE_CASE_   :						List[Any]							=	np.random.randn(3		,			4		,			5)
          SCREAMING_SNAKE_CASE_   :						Dict							=	torch.tensor(lowercase_)
          self.assertTrue(np.allclose(transpose(lowercase_		,			axes=(1, 2, 0))		,			transpose(lowercase_		,			axes=(1, 2, 0)).numpy()))
      @require_tf
      def _SCREAMING_SNAKE_CASE				(		self  :						Tuple):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						str							=	tf.constant(lowercase_)
          self.assertTrue(np.allclose(transpose(lowercase_)		,			transpose(lowercase_).numpy()))

          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	np.random.randn(3		,			4		,			5)
          SCREAMING_SNAKE_CASE_   :						int							=	tf.constant(lowercase_)
          self.assertTrue(np.allclose(transpose(lowercase_		,			axes=(1, 2, 0))		,			transpose(lowercase_		,			axes=(1, 2, 0)).numpy()))
      @require_flax
      def _SCREAMING_SNAKE_CASE				(		self  :						Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Tuple							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						Any							=	jnp.array(lowercase_)
          self.assertTrue(np.allclose(transpose(lowercase_)		,			np.asarray(transpose(lowercase_))))

          SCREAMING_SNAKE_CASE_   :						Dict							=	np.random.randn(3		,			4		,			5)
          SCREAMING_SNAKE_CASE_   :						Any							=	jnp.array(lowercase_)
          self.assertTrue(np.allclose(transpose(lowercase_		,			axes=(1, 2, 0))		,			np.asarray(transpose(lowercase_		,			axes=(1, 2, 0)))))
      def _SCREAMING_SNAKE_CASE				(		self  :						Optional[int]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Dict							=	np.random.randn(3		,			4)
          self.assertTrue(np.allclose(reshape(lowercase_		,			(4, 3))		,			np.reshape(lowercase_		,			(4, 3))))

          SCREAMING_SNAKE_CASE_   :						Dict							=	np.random.randn(3		,			4		,			5)
          self.assertTrue(np.allclose(reshape(lowercase_		,			(12, 5))		,			np.reshape(lowercase_		,			(12, 5))))
      @require_torch
      def _SCREAMING_SNAKE_CASE				(		self  :						Union[str, Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						List[str]							=	torch.tensor(lowercase_)
          self.assertTrue(np.allclose(reshape(lowercase_		,			(4, 3))		,			reshape(lowercase_		,			(4, 3)).numpy()))

          SCREAMING_SNAKE_CASE_   :						int							=	np.random.randn(3		,			4		,			5)
          SCREAMING_SNAKE_CASE_   :						int							=	torch.tensor(lowercase_)
          self.assertTrue(np.allclose(reshape(lowercase_		,			(12, 5))		,			reshape(lowercase_		,			(12, 5)).numpy()))
      @require_tf
      def _SCREAMING_SNAKE_CASE				(		self  :						Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						int							=	tf.constant(lowercase_)
          self.assertTrue(np.allclose(reshape(lowercase_		,			(4, 3))		,			reshape(lowercase_		,			(4, 3)).numpy()))

          SCREAMING_SNAKE_CASE_   :						List[Any]							=	np.random.randn(3		,			4		,			5)
          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	tf.constant(lowercase_)
          self.assertTrue(np.allclose(reshape(lowercase_		,			(12, 5))		,			reshape(lowercase_		,			(12, 5)).numpy()))
      @require_flax
      def _SCREAMING_SNAKE_CASE				(		self  :						Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Any							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						List[Any]							=	jnp.array(lowercase_)
          self.assertTrue(np.allclose(reshape(lowercase_		,			(4, 3))		,			np.asarray(reshape(lowercase_		,			(4, 3)))))

          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	np.random.randn(3		,			4		,			5)
          SCREAMING_SNAKE_CASE_   :						List[Any]							=	jnp.array(lowercase_)
          self.assertTrue(np.allclose(reshape(lowercase_		,			(12, 5))		,			np.asarray(reshape(lowercase_		,			(12, 5)))))
      def _SCREAMING_SNAKE_CASE				(		self  :						List[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						List[Any]							=	np.random.randn(1		,			3		,			4)
          self.assertTrue(np.allclose(squeeze(lowercase_)		,			np.squeeze(lowercase_)))

          SCREAMING_SNAKE_CASE_   :						str							=	np.random.randn(1		,			4		,			1		,			5)
          self.assertTrue(np.allclose(squeeze(lowercase_		,			axis=2)		,			np.squeeze(lowercase_		,			axis=2)))
      @require_torch
      def _SCREAMING_SNAKE_CASE				(		self  :						Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	np.random.randn(1		,			3		,			4)
          SCREAMING_SNAKE_CASE_   :						int							=	torch.tensor(lowercase_)
          self.assertTrue(np.allclose(squeeze(lowercase_)		,			squeeze(lowercase_).numpy()))

          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	np.random.randn(1		,			4		,			1		,			5)
          SCREAMING_SNAKE_CASE_   :						str							=	torch.tensor(lowercase_)
          self.assertTrue(np.allclose(squeeze(lowercase_		,			axis=2)		,			squeeze(lowercase_		,			axis=2).numpy()))
      @require_tf
      def _SCREAMING_SNAKE_CASE				(		self  :						List[str]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						int							=	np.random.randn(1		,			3		,			4)
          SCREAMING_SNAKE_CASE_   :						Dict							=	tf.constant(lowercase_)
          self.assertTrue(np.allclose(squeeze(lowercase_)		,			squeeze(lowercase_).numpy()))

          SCREAMING_SNAKE_CASE_   :						List[Any]							=	np.random.randn(1		,			4		,			1		,			5)
          SCREAMING_SNAKE_CASE_   :						Any							=	tf.constant(lowercase_)
          self.assertTrue(np.allclose(squeeze(lowercase_		,			axis=2)		,			squeeze(lowercase_		,			axis=2).numpy()))
      @require_flax
      def _SCREAMING_SNAKE_CASE				(		self  :						int):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	np.random.randn(1		,			3		,			4)
          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	jnp.array(lowercase_)
          self.assertTrue(np.allclose(squeeze(lowercase_)		,			np.asarray(squeeze(lowercase_))))

          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	np.random.randn(1		,			4		,			1		,			5)
          SCREAMING_SNAKE_CASE_   :						str							=	jnp.array(lowercase_)
          self.assertTrue(np.allclose(squeeze(lowercase_		,			axis=2)		,			np.asarray(squeeze(lowercase_		,			axis=2))))
      def _SCREAMING_SNAKE_CASE				(		self  :						Tuple):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						int							=	np.random.randn(3		,			4)
          self.assertTrue(np.allclose(expand_dims(lowercase_		,			axis=1)		,			np.expand_dims(lowercase_		,			axis=1)))
      @require_torch
      def _SCREAMING_SNAKE_CASE				(		self  :						Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						int							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						int							=	torch.tensor(lowercase_)
          self.assertTrue(np.allclose(expand_dims(lowercase_		,			axis=1)		,			expand_dims(lowercase_		,			axis=1).numpy()))
      @require_tf
      def _SCREAMING_SNAKE_CASE				(		self  :						str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						List[str]							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						Any							=	tf.constant(lowercase_)
          self.assertTrue(np.allclose(expand_dims(lowercase_		,			axis=1)		,			expand_dims(lowercase_		,			axis=1).numpy()))



      @require_flax
      def _SCREAMING_SNAKE_CASE				(		self  :						Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Tuple							=	np.random.randn(3		,			4)
          SCREAMING_SNAKE_CASE_   :						int							=	jnp.array(lowercase_)
          self.assertTrue(np.allclose(expand_dims(lowercase_		,			axis=1)		,			np.asarray(expand_dims(lowercase_		,			axis=1))))


