






'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_squeezebert import SqueezeBertTokenizer


UpperCamelCase     :   List[str]	      =		logging.get_logger(__name__)

UpperCamelCase     :   Dict	      =		{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCamelCase     :   int	      =		{
    'vocab_file': {
        'squeezebert/squeezebert-uncased': (
            'https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/vocab.txt'
        ),
        'squeezebert/squeezebert-mnli': 'https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/vocab.txt',
        'squeezebert/squeezebert-mnli-headless': (
            'https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'squeezebert/squeezebert-uncased': (
            'https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/tokenizer.json'
        ),
        'squeezebert/squeezebert-mnli': (
            'https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/tokenizer.json'
        ),
        'squeezebert/squeezebert-mnli-headless': (
            'https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/tokenizer.json'
        ),
    },
}

UpperCamelCase     :   Tuple	      =		{
    'squeezebert/squeezebert-uncased': 5_12,
    'squeezebert/squeezebert-mnli': 5_12,
    'squeezebert/squeezebert-mnli-headless': 5_12,
}


UpperCamelCase     :   Dict	      =		{
    'squeezebert/squeezebert-uncased': {'do_lower_case': True},
    'squeezebert/squeezebert-mnli': {'do_lower_case': True},
    'squeezebert/squeezebert-mnli-headless': {'do_lower_case': True},
}




class 		UpperCamelCase__     (a      ):




		'''simple docstring'''





		_UpperCamelCase						=       VOCAB_FILES_NAMES
		_UpperCamelCase						=       PRETRAINED_VOCAB_FILES_MAP
		_UpperCamelCase						=       PRETRAINED_INIT_CONFIGURATION
		_UpperCamelCase						=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_UpperCamelCase						=       SqueezeBertTokenizer






		def __init__(   self ,_lowerCAmelCase=None ,_lowerCAmelCase=None ,_lowerCAmelCase=True ,_lowerCAmelCase="[UNK]" ,_lowerCAmelCase="[SEP]" ,_lowerCAmelCase="[PAD]" ,_lowerCAmelCase="[CLS]" ,_lowerCAmelCase="[MASK]" ,_lowerCAmelCase=True ,_lowerCAmelCase=None ,**_lowerCAmelCase ,):
								super().__init__(
								    _lowerCAmelCase ,tokenizer_file=_lowerCAmelCase ,do_lower_case=_lowerCAmelCase ,unk_token=_lowerCAmelCase ,sep_token=_lowerCAmelCase ,pad_token=_lowerCAmelCase ,cls_token=_lowerCAmelCase ,mask_token=_lowerCAmelCase ,tokenize_chinese_chars=_lowerCAmelCase ,strip_accents=_lowerCAmelCase ,**_lowerCAmelCase ,)

								lowerCamelCase__								=	json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
								if (
								    normalizer_state.get("""lowercase""" ,_lowerCAmelCase							) != do_lower_case
								    or normalizer_state.get("""strip_accents""" ,_lowerCAmelCase							) != strip_accents
								    or normalizer_state.get("""handle_chinese_chars""" ,_lowerCAmelCase							) != tokenize_chinese_chars
								):
														lowerCamelCase__								=	getattr(_lowerCAmelCase ,normalizer_state.pop("""type"""							)							)
														lowerCamelCase__								=	do_lower_case
														lowerCamelCase__								=	strip_accents
														lowerCamelCase__								=	tokenize_chinese_chars
														lowerCamelCase__								=	normalizer_class(**_lowerCAmelCase							)

								lowerCamelCase__								=	do_lower_case






		def       UpperCamelCase_       (   self ,_lowerCAmelCase ,_lowerCAmelCase=None							):
								lowerCamelCase__								=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
														output += token_ids_a + [self.sep_token_id]

								return output






		def       UpperCamelCase_       (   self ,_lowerCAmelCase ,_lowerCAmelCase = None							):
								lowerCamelCase__								=	[self.sep_token_id]
								lowerCamelCase__								=	[self.cls_token_id]
								if token_ids_a is None:
														return len(cls + token_ids_a + sep							) * [0]
								return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]






		def       UpperCamelCase_       (   self ,_lowerCAmelCase ,_lowerCAmelCase = None							):
								lowerCamelCase__								=	self._tokenizer.model.save(_lowerCAmelCase ,name=_lowerCAmelCase							)
								return tuple(_lowerCAmelCase							)








"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


__UpperCamelCase     :						Dict						       =  {'''processing_layoutxlm''': ['''LayoutXLMProcessor''']}

try:
       if not is_sentencepiece_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __UpperCamelCase     :						int						       =  ['''LayoutXLMTokenizer''']

try:
       if not is_tokenizers_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __UpperCamelCase     :						Any						       =  ['''LayoutXLMTokenizerFast''']

if TYPE_CHECKING:
       from .processing_layoutxlm import LayoutXLMProcessor

       try:
              if not is_sentencepiece_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_layoutxlm import LayoutXLMTokenizer

       try:
              if not is_tokenizers_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_layoutxlm_fast import LayoutXLMTokenizerFast

else:
       import sys

       __UpperCamelCase     :						Dict						       =  _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





