




'''simple docstring'''

from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


a       :						str								=		logging.get_logger(__name__)

a       :						int								=		{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

a       :						Optional[Any]								=		{
    '''vocab_file''': {
        '''allegro/herbert-base-cased''': '''https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json'''
    },
    '''merges_file''': {
        '''allegro/herbert-base-cased''': '''https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt'''
    },
}

a       :						Union[str, Any]								=		{'''allegro/herbert-base-cased''': 514}
a       :						str								=		{}



class 		SCREAMING_SNAKE_CASE__  (			_UpperCamelCase	):
			__SCREAMING_SNAKE_CASE    =			VOCAB_FILES_NAMES
			__SCREAMING_SNAKE_CASE    =			PRETRAINED_VOCAB_FILES_MAP
			__SCREAMING_SNAKE_CASE    =			PRETRAINED_INIT_CONFIGURATION
			__SCREAMING_SNAKE_CASE    =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__SCREAMING_SNAKE_CASE    =			HerbertTokenizer





			def __init__(      self						:    Optional[Any]      ,		a_						:    str=None      ,		a_						:    Dict=None      ,		a_						:    Any=None      ,		a_						:    Any="<s>"      ,		a_						:    List[str]="<unk>"      ,		a_						:    int="<pad>"      ,		a_						:    Dict="<mask>"      ,		a_						:    Optional[int]="</s>"      ,		**a_						:    Optional[int]      ,		):



						"""simple docstring"""




						super().__init__(
						    a_      ,		a_      ,		tokenizer_file=a_      ,		cls_token=a_      ,		unk_token=a_      ,		pad_token=a_      ,		mask_token=a_      ,		sep_token=a_      ,		**a_      ,		)





			def 							A (      self						:    Optional[Any]      ,		a_						:    List[int]      ,		a_						:    Optional[List[int]] = None							):



						"""simple docstring"""




						__snake_case			=		[self.cls_token_id]
						__snake_case			=		[self.sep_token_id]
						if token_ids_a is None:
									return cls + token_ids_a + sep

						return cls + token_ids_a + sep + token_ids_a + sep





			def 							A (      self						:    Tuple      ,		a_						:    List[int]      ,		a_						:    Optional[List[int]] = None      ,		a_						:    bool = False							):



						"""simple docstring"""




						if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=a_      ,		token_ids_a=a_      ,		already_has_special_tokens=a_							)

						if token_ids_a is None:
									return [1] + ([0] * len(a_							)) + [1]
						return [1] + ([0] * len(a_							)) + [1] + ([0] * len(a_							)) + [1]





			def 							A (      self						:    Optional[int]      ,		a_						:    List[int]      ,		a_						:    Optional[List[int]] = None							):



						"""simple docstring"""




						__snake_case			=		[self.sep_token_id]
						__snake_case			=		[self.cls_token_id]

						if token_ids_a is None:
									return len(cls + token_ids_a + sep							) * [0]
						return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]






			def 							A (      self						:    Union[str, Any]      ,		a_						:    str      ,		a_						:    Optional[str] = None							):



						"""simple docstring"""




						__snake_case			=		self._tokenizer.model.save(a_      ,		name=a_							)
						return tuple(a_							)







"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


A_    							=					{
    """configuration_nllb_moe""": [
        """NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """NllbMoeConfig""",
    ]
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A_    							=					[
						    """NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """NllbMoeForConditionalGeneration""",
						    """NllbMoeModel""",
						    """NllbMoePreTrainedModel""",
						    """NllbMoeTop2Router""",
						    """NllbMoeSparseMLP""",
						]


if TYPE_CHECKING:
						from .configuration_nllb_moe import (
						    NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP,
						    NllbMoeConfig,
						)

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_nllb_moe import (
												    NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST,
												    NllbMoeForConditionalGeneration,
												    NllbMoeModel,
												    NllbMoePreTrainedModel,
												    NllbMoeSparseMLP,
												    NllbMoeTopaRouter,
												)


else:
						import sys

						A_    							=					_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



