
'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_squeezebert import SqueezeBertTokenizer


lowercase					:							int   =      logging.get_logger(__name__)

lowercase					:							Optional[Any]   =      {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowercase					:							Union[str, Any]   =      {
    "vocab_file": {
        "squeezebert/squeezebert-uncased": (
            "https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/vocab.txt"
        ),
        "squeezebert/squeezebert-mnli": "https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/vocab.txt",
        "squeezebert/squeezebert-mnli-headless": (
            "https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "squeezebert/squeezebert-uncased": (
            "https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/tokenizer.json"
        ),
        "squeezebert/squeezebert-mnli": (
            "https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/tokenizer.json"
        ),
        "squeezebert/squeezebert-mnli-headless": (
            "https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/tokenizer.json"
        ),
    },
}

lowercase					:							Optional[Any]   =      {
    "squeezebert/squeezebert-uncased": 512,
    "squeezebert/squeezebert-mnli": 512,
    "squeezebert/squeezebert-mnli-headless": 512,
}


lowercase					:							Dict   =      {
    "squeezebert/squeezebert-uncased": {"do_lower_case": True},
    "squeezebert/squeezebert-mnli": {"do_lower_case": True},
    "squeezebert/squeezebert-mnli-headless": {"do_lower_case": True},
}


class 		__UpperCAmelCase   (      _lowerCamelCase		):
							__lowercase   	= VOCAB_FILES_NAMES
							__lowercase   	= PRETRAINED_VOCAB_FILES_MAP
							__lowercase   	= PRETRAINED_INIT_CONFIGURATION
							__lowercase   	= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__lowercase   	= SqueezeBertTokenizer


							def __init__( self					,     lowerCAmelCase_=None					,     lowerCAmelCase_=None					,     lowerCAmelCase_=True					,     lowerCAmelCase_="[UNK]"					,     lowerCAmelCase_="[SEP]"					,     lowerCAmelCase_="[PAD]"					,     lowerCAmelCase_="[CLS]"					,     lowerCAmelCase_="[MASK]"					,     lowerCAmelCase_=True					,     lowerCAmelCase_=None					,     **lowerCAmelCase_					,     ):





								"""simple docstring"""






								super().__init__(
								    lowerCAmelCase_					,     tokenizer_file=lowerCAmelCase_					,     do_lower_case=lowerCAmelCase_					,     unk_token=lowerCAmelCase_					,     sep_token=lowerCAmelCase_					,     pad_token=lowerCAmelCase_					,     cls_token=lowerCAmelCase_					,     mask_token=lowerCAmelCase_					,     tokenize_chinese_chars=lowerCAmelCase_					,     strip_accents=lowerCAmelCase_					,     **lowerCAmelCase_					,     )

								_snake_case     		=	json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
								if (
								    normalizer_state.get('lowercase'					,     lowerCAmelCase_						) != do_lower_case
								    or normalizer_state.get('strip_accents'					,     lowerCAmelCase_						) != strip_accents
								    or normalizer_state.get('handle_chinese_chars'					,     lowerCAmelCase_						) != tokenize_chinese_chars
								):
									_snake_case     		=	getattr(lowerCAmelCase_					,     normalizer_state.pop('type'						)						)
									_snake_case     		=	do_lower_case
									_snake_case     		=	strip_accents
									_snake_case     		=	tokenize_chinese_chars
									_snake_case     		=	normalizer_class(**lowerCAmelCase_						)

								_snake_case     		=	do_lower_case


							def 				lowerCamelCase       ( self					,     lowerCAmelCase_					,     lowerCAmelCase_=None						):





								"""simple docstring"""






								_snake_case     		=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

								return output


							def 				lowerCamelCase       ( self					,     lowerCAmelCase_					,     lowerCAmelCase_ = None						):





								"""simple docstring"""






								_snake_case     		=	[self.sep_token_id]
								_snake_case     		=	[self.cls_token_id]
								if token_ids_a is None:
									return len(cls + token_ids_a + sep						) * [0]
								return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]




							def 				lowerCamelCase       ( self					,     lowerCAmelCase_					,     lowerCAmelCase_ = None						):





								"""simple docstring"""






								_snake_case     		=	self._tokenizer.model.save(lowerCAmelCase_					,     name=lowerCAmelCase_						)
								return tuple(lowerCAmelCase_						)



'''simple docstring'''







from typing import List, Optional

from tokenizers import ByteLevelBPETokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_blenderbot_small import BlenderbotSmallTokenizer


lowercase					:							Optional[Any]   =      logging.get_logger(__name__)

lowercase					:							str   =      {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
    "tokenizer_config_file": "tokenizer_config.json",
}

lowercase					:							Optional[Any]   =      {
    "vocab_file": {
        "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json"
    },
    "merges_file": {
        "facebook/blenderbot_small-90M": "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt"
    },
    "tokenizer_config_file": {
        "facebook/blenderbot_small-90M": (
            "https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json"
        )
    },
}

lowercase					:							List[Any]   =      {
    "facebook/blenderbot_small-90M": 512,
}


class 		__UpperCAmelCase   (      _lowerCamelCase		):
							__lowercase   	= VOCAB_FILES_NAMES
							__lowercase   	= PRETRAINED_VOCAB_FILES_MAP
							__lowercase   	= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__lowercase   	= BlenderbotSmallTokenizer


							def __init__( self					,     lowerCAmelCase_=None					,     lowerCAmelCase_=None					,     lowerCAmelCase_="<|endoftext|>"					,     lowerCAmelCase_="<|endoftext|>"					,     lowerCAmelCase_="<|endoftext|>"					,     lowerCAmelCase_=False					,     lowerCAmelCase_=True					,     **lowerCAmelCase_					,     ):





								"""simple docstring"""






								super().__init__(
								    ByteLevelBPETokenizer(
								        vocab=lowerCAmelCase_					,     merges=lowerCAmelCase_					,     add_prefix_space=lowerCAmelCase_					,     trim_offsets=lowerCAmelCase_					,     )					,     bos_token=lowerCAmelCase_					,     eos_token=lowerCAmelCase_					,     unk_token=lowerCAmelCase_					,     **lowerCAmelCase_					,     )
								_snake_case     		=	add_prefix_space


							def 				lowerCamelCase       ( self					,     lowerCAmelCase_					,     lowerCAmelCase_=None						):





								"""simple docstring"""






								_snake_case     		=	[self.bos_token_id] + token_ids_a + [self.eos_token_id]
								if token_ids_a is None:
									return output

								return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




							def 				lowerCamelCase       ( self					,     lowerCAmelCase_					,     lowerCAmelCase_ = None						):





								"""simple docstring"""






								_snake_case     		=	[self.sep_token_id]
								_snake_case     		=	[self.cls_token_id]

								if token_ids_a is None:
									return len(cls + token_ids_a + sep						) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]

