






from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


SCREAMING_SNAKE_CASE__   :      Optional[int]				=		logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__   :      str				=		{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE__   :      int				=		{
    """vocab_file""": {
        """allegro/herbert-base-cased""": """https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json"""
    },
    """merges_file""": {
        """allegro/herbert-base-cased""": """https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt"""
    },
}

SCREAMING_SNAKE_CASE__   :      Optional[int]				=		{"""allegro/herbert-base-cased""": 5_14}
SCREAMING_SNAKE_CASE__   :      Union[str, Any]				=		{}
class     lowerCamelCase_					(   lowerCamelCase			):
				a__       							=	VOCAB_FILES_NAMES
				a__       							=	PRETRAINED_VOCAB_FILES_MAP
				a__       							=	PRETRAINED_INIT_CONFIGURATION
				a__       							=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				a__       							=	HerbertTokenizer







				def __init__(    self							,       __lowerCAmelCase=None							,       __lowerCAmelCase=None							,       __lowerCAmelCase=None							,       __lowerCAmelCase="<s>"							,       __lowerCAmelCase="<unk>"							,       __lowerCAmelCase="<pad>"							,       __lowerCAmelCase="<mask>"							,       __lowerCAmelCase="</s>"							,       **__lowerCAmelCase							,       ):
								"""simple docstring"""
								super().__init__(
								    __lowerCAmelCase							,       __lowerCAmelCase							,       tokenizer_file=__lowerCAmelCase							,       cls_token=__lowerCAmelCase							,       unk_token=__lowerCAmelCase							,       pad_token=__lowerCAmelCase							,       mask_token=__lowerCAmelCase							,       sep_token=__lowerCAmelCase							,       **__lowerCAmelCase							,       )







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase = None  ):
								"""simple docstring"""
								__magic_name__		:str   =   [self.cls_token_id]
								__magic_name__		:Tuple   =   [self.sep_token_id]
								if token_ids_a is None:
												return cls + token_ids_a + sep

								return cls + token_ids_a + sep + token_ids_a + sep







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase = None							,       __lowerCAmelCase = False  ):
								"""simple docstring"""
								if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=__lowerCAmelCase							,       token_ids_a=__lowerCAmelCase							,       already_has_special_tokens=__lowerCAmelCase  )

								if token_ids_a is None:
												return [1] + ([0] * len(__lowerCAmelCase  )) + [1]
								return [1] + ([0] * len(__lowerCAmelCase  )) + [1] + ([0] * len(__lowerCAmelCase  )) + [1]







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase = None  ):
								"""simple docstring"""
								__magic_name__		:Dict   =   [self.sep_token_id]
								__magic_name__		:int   =   [self.cls_token_id]

								if token_ids_a is None:
												return len(cls + token_ids_a + sep  ) * [0]
								return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase = None  ):
								"""simple docstring"""
								__magic_name__		:Optional[int]   =   self._tokenizer.model.save(__lowerCAmelCase							,       name=__lowerCAmelCase  )
								return tuple(__lowerCAmelCase  )














import unittest

import numpy as np

from transformers.testing_utils import require_flax, require_tf, require_torch
from transformers.utils import (
    expand_dims,
    flatten_dict,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    reshape,
    squeeze,
    transpose,
)


if is_flax_available():
					import jax.numpy as jnp

if is_tf_available():
					import tensorflow as tf

if is_torch_available():
					import torch
class     lowerCamelCase_					(   unittest.TestCase			):







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[Any]   =   {
								    '''task_specific_params''': {
								        '''summarization''': {'''length_penalty''': 1.0, '''max_length''': 1_2_8, '''min_length''': 1_2, '''num_beams''': 4},
								        '''summarization_cnn''': {'''length_penalty''': 2.0, '''max_length''': 1_4_2, '''min_length''': 5_6, '''num_beams''': 4},
								        '''summarization_xsum''': {'''length_penalty''': 1.0, '''max_length''': 6_2, '''min_length''': 1_1, '''num_beams''': 6},
								    }
								}
								__magic_name__		:List[str]   =   {
								    '''task_specific_params.summarization.length_penalty''': 1.0,
								    '''task_specific_params.summarization.max_length''': 1_2_8,
								    '''task_specific_params.summarization.min_length''': 1_2,
								    '''task_specific_params.summarization.num_beams''': 4,
								    '''task_specific_params.summarization_cnn.length_penalty''': 2.0,
								    '''task_specific_params.summarization_cnn.max_length''': 1_4_2,
								    '''task_specific_params.summarization_cnn.min_length''': 5_6,
								    '''task_specific_params.summarization_cnn.num_beams''': 4,
								    '''task_specific_params.summarization_xsum.length_penalty''': 1.0,
								    '''task_specific_params.summarization_xsum.max_length''': 6_2,
								    '''task_specific_params.summarization_xsum.min_length''': 1_1,
								    '''task_specific_params.summarization_xsum.num_beams''': 6,
								}

								self.assertEqual(flatten_dict(__lowerCAmelCase  )							,       __lowerCAmelCase  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[Any]   =   np.random.randn(3							,       4  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase  )							,       x.transpose()  )  )

								__magic_name__		:List[Any]   =   np.random.randn(3							,       4							,       5  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  )							,       x.transpose((1, 2, 0)  )  )  )







				@require_torch
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[Any]   =   np.random.randn(3							,       4  )
								__magic_name__		:Tuple   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase  )							,       transpose(__lowerCAmelCase  ).numpy()  )  )

								__magic_name__		:int   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:Union[str, Any]   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  )							,       transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  ).numpy()  )  )







				@require_tf
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:int   =   np.random.randn(3							,       4  )
								__magic_name__		:Optional[Any]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase  )							,       transpose(__lowerCAmelCase  ).numpy()  )  )

								__magic_name__		:List[str]   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:Optional[Any]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  )							,       transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  ).numpy()  )  )







				@require_flax
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:int   =   np.random.randn(3							,       4  )
								__magic_name__		:Dict   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase  )							,       np.asarray(transpose(__lowerCAmelCase  )  )  )  )

								__magic_name__		:Dict   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:Dict   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  )							,       np.asarray(transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  )  )  )  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Any   =   np.random.randn(3							,       4  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (4, 3)  )							,       np.reshape(__lowerCAmelCase							,       (4, 3)  )  )  )

								__magic_name__		:Union[str, Any]   =   np.random.randn(3							,       4							,       5  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (1_2, 5)  )							,       np.reshape(__lowerCAmelCase							,       (1_2, 5)  )  )  )







				@require_torch
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Dict   =   np.random.randn(3							,       4  )
								__magic_name__		:Tuple   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (4, 3)  )							,       reshape(__lowerCAmelCase							,       (4, 3)  ).numpy()  )  )

								__magic_name__		:Union[str, Any]   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:List[str]   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (1_2, 5)  )							,       reshape(__lowerCAmelCase							,       (1_2, 5)  ).numpy()  )  )







				@require_tf
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Dict   =   np.random.randn(3							,       4  )
								__magic_name__		:Union[str, Any]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (4, 3)  )							,       reshape(__lowerCAmelCase							,       (4, 3)  ).numpy()  )  )

								__magic_name__		:List[Any]   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:Optional[int]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (1_2, 5)  )							,       reshape(__lowerCAmelCase							,       (1_2, 5)  ).numpy()  )  )







				@require_flax
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[str]   =   np.random.randn(3							,       4  )
								__magic_name__		:Any   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (4, 3)  )							,       np.asarray(reshape(__lowerCAmelCase							,       (4, 3)  )  )  )  )

								__magic_name__		:List[Any]   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:List[str]   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (1_2, 5)  )							,       np.asarray(reshape(__lowerCAmelCase							,       (1_2, 5)  )  )  )  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[Any]   =   np.random.randn(1							,       3							,       4  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase  )							,       np.squeeze(__lowerCAmelCase  )  )  )

								__magic_name__		:Optional[Any]   =   np.random.randn(1							,       4							,       1							,       5  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase							,       axis=2  )							,       np.squeeze(__lowerCAmelCase							,       axis=2  )  )  )







				@require_torch
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Dict   =   np.random.randn(1							,       3							,       4  )
								__magic_name__		:List[Any]   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase  )							,       squeeze(__lowerCAmelCase  ).numpy()  )  )

								__magic_name__		:List[str]   =   np.random.randn(1							,       4							,       1							,       5  )
								__magic_name__		:str   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase							,       axis=2  )							,       squeeze(__lowerCAmelCase							,       axis=2  ).numpy()  )  )







				@require_tf
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:int   =   np.random.randn(1							,       3							,       4  )
								__magic_name__		:Tuple   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase  )							,       squeeze(__lowerCAmelCase  ).numpy()  )  )

								__magic_name__		:Tuple   =   np.random.randn(1							,       4							,       1							,       5  )
								__magic_name__		:Optional[int]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase							,       axis=2  )							,       squeeze(__lowerCAmelCase							,       axis=2  ).numpy()  )  )







				@require_flax
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Tuple   =   np.random.randn(1							,       3							,       4  )
								__magic_name__		:Optional[Any]   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase  )							,       np.asarray(squeeze(__lowerCAmelCase  )  )  )  )

								__magic_name__		:List[Any]   =   np.random.randn(1							,       4							,       1							,       5  )
								__magic_name__		:Optional[Any]   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase							,       axis=2  )							,       np.asarray(squeeze(__lowerCAmelCase							,       axis=2  )  )  )  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Any   =   np.random.randn(3							,       4  )
								self.assertTrue(np.allclose(expand_dims(__lowerCAmelCase							,       axis=1  )							,       np.expand_dims(__lowerCAmelCase							,       axis=1  )  )  )







				@require_torch
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[Any]   =   np.random.randn(3							,       4  )
								__magic_name__		:Any   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(expand_dims(__lowerCAmelCase							,       axis=1  )							,       expand_dims(__lowerCAmelCase							,       axis=1  ).numpy()  )  )







				@require_tf
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Union[str, Any]   =   np.random.randn(3							,       4  )
								__magic_name__		:Union[str, Any]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(expand_dims(__lowerCAmelCase							,       axis=1  )							,       expand_dims(__lowerCAmelCase							,       axis=1  ).numpy()  )  )







				@require_flax
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[str]   =   np.random.randn(3							,       4  )
								__magic_name__		:Tuple   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(expand_dims(__lowerCAmelCase							,       axis=1  )							,       np.asarray(expand_dims(__lowerCAmelCase							,       axis=1  )  )  )  )






