
'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowercase					:							Union[str, Any]   =      {
    "configuration_distilbert": [
        "DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "DistilBertConfig",
        "DistilBertOnnxConfig",
    ],
    "tokenization_distilbert": ["DistilBertTokenizer"],
}

try:
					if not is_tokenizers_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowercase					:							Any   =      ["DistilBertTokenizerFast"]

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowercase					:							List[str]   =      [
					    "DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "DistilBertForMaskedLM",
					    "DistilBertForMultipleChoice",
					    "DistilBertForQuestionAnswering",
					    "DistilBertForSequenceClassification",
					    "DistilBertForTokenClassification",
					    "DistilBertModel",
					    "DistilBertPreTrainedModel",
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowercase					:							Optional[Any]   =      [
					    "TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "TFDistilBertForMaskedLM",
					    "TFDistilBertForMultipleChoice",
					    "TFDistilBertForQuestionAnswering",
					    "TFDistilBertForSequenceClassification",
					    "TFDistilBertForTokenClassification",
					    "TFDistilBertMainLayer",
					    "TFDistilBertModel",
					    "TFDistilBertPreTrainedModel",
					]

try:
					if not is_flax_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowercase					:							Dict   =      [
					    "FlaxDistilBertForMaskedLM",
					    "FlaxDistilBertForMultipleChoice",
					    "FlaxDistilBertForQuestionAnswering",
					    "FlaxDistilBertForSequenceClassification",
					    "FlaxDistilBertForTokenClassification",
					    "FlaxDistilBertModel",
					    "FlaxDistilBertPreTrainedModel",
					]


if TYPE_CHECKING:
					from .configuration_distilbert import (
					    DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
					    DistilBertConfig,
					    DistilBertOnnxConfig,
					)
					from .tokenization_distilbert import DistilBertTokenizer

					try:
										if not is_tokenizers_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_distilbert_fast import DistilBertTokenizerFast

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_distilbert import (
										    DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
										    DistilBertForMaskedLM,
										    DistilBertForMultipleChoice,
										    DistilBertForQuestionAnswering,
										    DistilBertForSequenceClassification,
										    DistilBertForTokenClassification,
										    DistilBertModel,
										    DistilBertPreTrainedModel,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_distilbert import (
										    TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFDistilBertForMaskedLM,
										    TFDistilBertForMultipleChoice,
										    TFDistilBertForQuestionAnswering,
										    TFDistilBertForSequenceClassification,
										    TFDistilBertForTokenClassification,
										    TFDistilBertMainLayer,
										    TFDistilBertModel,
										    TFDistilBertPreTrainedModel,
										)

					try:
										if not is_flax_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_flax_distilbert import (
										    FlaxDistilBertForMaskedLM,
										    FlaxDistilBertForMultipleChoice,
										    FlaxDistilBertForQuestionAnswering,
										    FlaxDistilBertForSequenceClassification,
										    FlaxDistilBertForTokenClassification,
										    FlaxDistilBertModel,
										    FlaxDistilBertPreTrainedModel,
										)

else:
					import sys

					lowercase					:							Tuple   =      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



'''simple docstring'''







from typing import List, Optional

from tokenizers import ByteLevelBPETokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_blenderbot_small import BlenderbotSmallTokenizer


lowercase					:							Optional[Any]   =      logging.get_logger(__name__)

lowercase					:							str   =      {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
    "tokenizer_config_file": "tokenizer_config.json",
}

lowercase					:							Optional[Any]   =      {
    "vocab_file": {
        "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json"
    },
    "merges_file": {
        "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt"
    },
    "tokenizer_config_file": {
        "facebook/blenderbot_small-90M": (
            "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json"
        )
    },
}

lowercase					:							List[Any]   =      {
    "facebook/blenderbot_small-90M": 512,
}


class 		__UpperCAmelCase   (      _lowerCamelCase		):
							__lowercase   	= VOCAB_FILES_NAMES
							__lowercase   	= PRETRAINED_VOCAB_FILES_MAP
							__lowercase   	= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__lowercase   	= BlenderbotSmallTokenizer


							def __init__( self					,     lowerCAmelCase_=None					,     lowerCAmelCase_=None					,     lowerCAmelCase_="<|endoftext|>"					,     lowerCAmelCase_="<|endoftext|>"					,     lowerCAmelCase_="<|endoftext|>"					,     lowerCAmelCase_=False					,     lowerCAmelCase_=True					,     **lowerCAmelCase_					,     ):





								"""simple docstring"""






								super().__init__(
								    ByteLevelBPETokenizer(
								        vocab=lowerCAmelCase_					,     merges=lowerCAmelCase_					,     add_prefix_space=lowerCAmelCase_					,     trim_offsets=lowerCAmelCase_					,     )					,     bos_token=lowerCAmelCase_					,     eos_token=lowerCAmelCase_					,     unk_token=lowerCAmelCase_					,     **lowerCAmelCase_					,     )
								_snake_case     		=	add_prefix_space


							def 				lowerCamelCase       ( self					,     lowerCAmelCase_					,     lowerCAmelCase_=None						):





								"""simple docstring"""






								_snake_case     		=	[self.bos_token_id] + token_ids_a + [self.eos_token_id]
								if token_ids_a is None:
									return output

								return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




							def 				lowerCamelCase       ( self					,     lowerCAmelCase_					,     lowerCAmelCase_ = None						):





								"""simple docstring"""






								_snake_case     		=	[self.sep_token_id]
								_snake_case     		=	[self.cls_token_id]

								if token_ids_a is None:
									return len(cls + token_ids_a + sep						) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]

