





"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


_lowerCamelCase       =	logging.get_logger(__name__)

_lowerCamelCase       =	{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_lowerCamelCase       =	{
    '''vocab_file''': {
        '''microsoft/layoutlm-base-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt'''
        ),
        '''microsoft/layoutlm-large-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''microsoft/layoutlm-base-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json'''
        ),
        '''microsoft/layoutlm-large-uncased''': (
            '''https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

_lowerCamelCase       =	{
    '''microsoft/layoutlm-base-uncased''': 5_12,
    '''microsoft/layoutlm-large-uncased''': 5_12,
}

_lowerCamelCase       =	{
    '''microsoft/layoutlm-base-uncased''': {'''do_lower_case''': True},
    '''microsoft/layoutlm-large-uncased''': {'''do_lower_case''': True},
}





class 			snake_case		(   __UpperCAmelCase   ):
      lowerCamelCase__						= VOCAB_FILES_NAMES
      lowerCamelCase__						= PRETRAINED_VOCAB_FILES_MAP
      lowerCamelCase__						= PRETRAINED_INIT_CONFIGURATION
      lowerCamelCase__						= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCamelCase__						= LayoutLMTokenizer

      def __init__(			self   :List[Any]						,							_lowerCamelCase   :Optional[Any]=None						,							_lowerCamelCase   :Tuple=None						,							_lowerCamelCase   :Optional[int]=True						,							_lowerCamelCase   :Tuple="[UNK]"						,							_lowerCamelCase   :Optional[int]="[SEP]"						,							_lowerCamelCase   :List[Any]="[PAD]"						,							_lowerCamelCase   :List[str]="[CLS]"						,							_lowerCamelCase   :List[str]="[MASK]"						,							_lowerCamelCase   :str=True						,							_lowerCamelCase   :Dict=None						,							**_lowerCamelCase   :List[str]						,							):
       super().__init__(
           _lowerCamelCase						,							tokenizer_file=_lowerCamelCase						,							do_lower_case=_lowerCamelCase						,							unk_token=_lowerCamelCase						,							sep_token=_lowerCamelCase						,							pad_token=_lowerCamelCase						,							cls_token=_lowerCamelCase						,							mask_token=_lowerCamelCase						,							tokenize_chinese_chars=_lowerCamelCase						,							strip_accents=_lowerCamelCase						,							**_lowerCamelCase						,							)

       __SCREAMING_SNAKE_CASE :			List[Any]												=       json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
       if (
           normalizer_state.get('''lowercase'''						,							_lowerCamelCase						) != do_lower_case
           or normalizer_state.get('''strip_accents'''						,							_lowerCamelCase						) != strip_accents
           or normalizer_state.get('''handle_chinese_chars'''						,							_lowerCamelCase						) != tokenize_chinese_chars
       ):
        __SCREAMING_SNAKE_CASE :			Tuple												=       getattr(_lowerCamelCase						,							normalizer_state.pop('''type'''						)						)
        __SCREAMING_SNAKE_CASE :			List[str]												=       do_lower_case
        __SCREAMING_SNAKE_CASE :			Optional[Any]												=       strip_accents
        __SCREAMING_SNAKE_CASE :			Any												=       tokenize_chinese_chars
        __SCREAMING_SNAKE_CASE :			str												=       normalizer_class(**_lowerCamelCase						)

       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       do_lower_case

      def 						SCREAMING_SNAKE_CASE_			(			self   :str						,							_lowerCamelCase   :Union[str, Any]						,							_lowerCamelCase   :Optional[Any]=None						):
       __SCREAMING_SNAKE_CASE :			Tuple												=       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

       return output

      def 						SCREAMING_SNAKE_CASE_			(			self   :Any						,							_lowerCamelCase   :List[int]						,							_lowerCamelCase   :Optional[List[int]] = None						):
       __SCREAMING_SNAKE_CASE :			int												=       [self.sep_token_id]
       __SCREAMING_SNAKE_CASE :			int												=       [self.cls_token_id]
       if token_ids_a is None:
        return len(cls + token_ids_a + sep						) * [0]
       return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]



      def 						SCREAMING_SNAKE_CASE_			(			self   :Union[str, Any]						,							_lowerCamelCase   :str						,							_lowerCamelCase   :Optional[str] = None						):
       __SCREAMING_SNAKE_CASE :			List[str]												=       self._tokenizer.model.save(_lowerCamelCase						,							name=_lowerCamelCase						)
       return tuple(_lowerCamelCase						)








"""simple docstring"""





from ctypes import c_float, sizeof
from enum import Enum
from typing import TYPE_CHECKING, Optional, Union


if TYPE_CHECKING:
    from .. import AutoFeatureExtractor, AutoProcessor, AutoTokenizer  # tests_ignore





class 			snake_case		(   __UpperCAmelCase   ):
      lowerCamelCase__						= c_float

      @property
      def 						SCREAMING_SNAKE_CASE_			(			self   :Optional[Any]						):
       return sizeof(self.value						)







def 		lowerCAmelCase_	(  lowercase_   :							int ,      lowercase_   :							int ,      lowercase_   :							int = 0		):

 '''simple docstring'''






 if dimension <= 0:
  __SCREAMING_SNAKE_CASE :			str												=       fixed_dimension

 dimension -= num_token_to_add
 return dimension




def 		lowerCAmelCase_	(  lowercase_   :							int ,      lowercase_   :							ParameterFormat		):

 '''simple docstring'''






 return num_parameters * dtype.size






def 		lowerCAmelCase_	(  lowercase_   :							str		):

 '''simple docstring'''






 from .. import AutoFeatureExtractor, AutoProcessor, AutoTokenizer  # tests_ignore

 try:
  return AutoProcessor.from_pretrained(lowercase_		)
 except (ValueError, OSError, KeyError):
  __SCREAMING_SNAKE_CASE							,      __SCREAMING_SNAKE_CASE :			int												=       None, None
  try:
   __SCREAMING_SNAKE_CASE :			Optional[Any]												=       AutoTokenizer.from_pretrained(lowercase_		)
  except (OSError, KeyError):
   pass
  try:
   __SCREAMING_SNAKE_CASE :			Optional[Any]												=       AutoFeatureExtractor.from_pretrained(lowercase_		)
  except (OSError, KeyError):
   pass

  if tokenizer is not None and feature_extractor is not None:
   raise ValueError(
       F'''Couldn\'t auto-detect preprocessor for {model_name}. Found both a tokenizer and a feature extractor.'''		)
  elif tokenizer is None and feature_extractor is None:
   return None
  elif tokenizer is not None:
   return tokenizer
  else:
   return feature_extractor

