
"""simple docstring"""






import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType





class 				a__					(						A__       ):
     UpperCAmelCase__         =      ['''image_processor''', '''tokenizer''']
     UpperCAmelCase__         =      '''LayoutLMv2ImageProcessor'''
     UpperCAmelCase__         =      ('''LayoutXLMTokenizer''', '''LayoutXLMTokenizerFast''')



     def __init__(							self :Tuple      ,  _lowerCamelCase :Optional[Any]=None      ,  _lowerCamelCase :Tuple=None      ,  **_lowerCamelCase :Optional[Any]   ):







      '''simple docstring'''



      if "feature_extractor" in kwargs:
       warnings.warn(
           'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
           ' instead.'      ,  _lowerCamelCase      ,  )
       UpperCamelCase_     :					Optional[int]   	=kwargs.pop('feature_extractor'   )

      UpperCamelCase_     :					List[Any]   	=image_processor if image_processor is not None else feature_extractor
      if image_processor is None:
       raise ValueError('You need to specify an `image_processor`.'   )
      if tokenizer is None:
       raise ValueError('You need to specify a `tokenizer`.'   )

      super().__init__(_lowerCamelCase      ,  _lowerCamelCase   )



     def __call__(							self :str      ,  _lowerCamelCase :Optional[int]      ,  _lowerCamelCase :Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None      ,  _lowerCamelCase :Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None      ,  _lowerCamelCase :Union[List[List[int]], List[List[List[int]]]] = None      ,  _lowerCamelCase :Optional[Union[List[int], List[List[int]]]] = None      ,  _lowerCamelCase :bool = True      ,  _lowerCamelCase :Union[bool, str, PaddingStrategy] = False      ,  _lowerCamelCase :Union[bool, str, TruncationStrategy] = None      ,  _lowerCamelCase :Optional[int] = None      ,  _lowerCamelCase :int = 0      ,  _lowerCamelCase :Optional[int] = None      ,  _lowerCamelCase :Optional[bool] = None      ,  _lowerCamelCase :Optional[bool] = None      ,  _lowerCamelCase :bool = False      ,  _lowerCamelCase :bool = False      ,  _lowerCamelCase :bool = False      ,  _lowerCamelCase :bool = False      ,  _lowerCamelCase :bool = True      ,  _lowerCamelCase :Optional[Union[str, TensorType]] = None      ,  **_lowerCamelCase :Optional[int]      ,  ):







      '''simple docstring'''



      if self.image_processor.apply_ocr and (boxes is not None):
       raise ValueError(
           'You cannot provide bounding boxes '
           'if you initialized the image processor with apply_ocr set to True.'   )

      if self.image_processor.apply_ocr and (word_labels is not None):
       raise ValueError(
           'You cannot provide word labels if you initialized the image processor with apply_ocr set to True.'   )

      if return_overflowing_tokens is True and return_offsets_mapping is False:
       raise ValueError('You cannot return overflowing tokens without returning the offsets mapping.'   )

      # first, apply the image processor
      UpperCamelCase_     :					Optional[int]   	=self.image_processor(images=_lowerCamelCase      ,  return_tensors=_lowerCamelCase   )

      # second, apply the tokenizer
      if text is not None and self.image_processor.apply_ocr and text_pair is None:
       if isinstance(_lowerCamelCase      ,  _lowerCamelCase   ):
        UpperCamelCase_     :					int   	=[text]  # add batch dimension (as the image processor always adds a batch dimension)
       UpperCamelCase_     :					Tuple   	=features['words']

      UpperCamelCase_     :					Union[str, Any]   	=self.tokenizer(
          text=text if text is not None else features['words']      ,  text_pair=text_pair if text_pair is not None else None      ,  boxes=boxes if boxes is not None else features['boxes']      ,  word_labels=_lowerCamelCase      ,  add_special_tokens=_lowerCamelCase      ,  padding=_lowerCamelCase      ,  truncation=_lowerCamelCase      ,  max_length=_lowerCamelCase      ,  stride=_lowerCamelCase      ,  pad_to_multiple_of=_lowerCamelCase      ,  return_token_type_ids=_lowerCamelCase      ,  return_attention_mask=_lowerCamelCase      ,  return_overflowing_tokens=_lowerCamelCase      ,  return_special_tokens_mask=_lowerCamelCase      ,  return_offsets_mapping=_lowerCamelCase      ,  return_length=_lowerCamelCase      ,  verbose=_lowerCamelCase      ,  return_tensors=_lowerCamelCase      ,  **_lowerCamelCase      ,  )

      # add pixel values
      UpperCamelCase_     :					Tuple   	=features.pop('pixel_values'   )
      if return_overflowing_tokens is True:
       UpperCamelCase_     :					int   	=self.get_overflowing_images(_lowerCamelCase      ,  encoded_inputs['overflow_to_sample_mapping']   )
      UpperCamelCase_     :					Tuple   	=images

      return encoded_inputs



     def  lowerCamelCase_						(							self :int      ,  _lowerCamelCase :List[Any]      ,  _lowerCamelCase :List[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					Dict   	=[]
      for sample_idx in overflow_to_sample_mapping:
       images_with_overflow.append(images[sample_idx]   )

      if len(_lowerCamelCase   ) != len(_lowerCamelCase   ):
       raise ValueError(
           'Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got'
           f''' {len(_lowerCamelCase   )} and {len(_lowerCamelCase   )}'''   )

      return images_with_overflow



     def  lowerCamelCase_						(							self :Tuple      ,  *_lowerCamelCase :Any      ,  **_lowerCamelCase :str   ):







      '''simple docstring'''



      return self.tokenizer.batch_decode(*_lowerCamelCase      ,  **_lowerCamelCase   )



     def  lowerCamelCase_						(							self :Dict      ,  *_lowerCamelCase :Union[str, Any]      ,  **_lowerCamelCase :Any   ):







      '''simple docstring'''



      return self.tokenizer.decode(*_lowerCamelCase      ,  **_lowerCamelCase   )



     @property
     def  lowerCamelCase_						(							self :Any   ):







      '''simple docstring'''



      return ["input_ids", "bbox", "attention_mask", "image"]



     @property
     def  lowerCamelCase_						(							self :Union[str, Any]   ):







      '''simple docstring'''



      warnings.warn(
          '`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'      ,  _lowerCamelCase      ,  )
      return self.image_processor_class




     @property
     def  lowerCamelCase_						(							self :Union[str, Any]   ):







      '''simple docstring'''



      warnings.warn(
          '`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'      ,  _lowerCamelCase      ,  )
      return self.image_processor


"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


__SCREAMING_SNAKE_CASE							=     logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							=     {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__SCREAMING_SNAKE_CASE							=     {
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

__SCREAMING_SNAKE_CASE							=     {
    'junnyu/roformer_chinese_small': 1_536,
    'junnyu/roformer_chinese_base': 1_536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


__SCREAMING_SNAKE_CASE							=     {
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}





class 				a__					(						A__       ):
     UpperCAmelCase__         =      VOCAB_FILES_NAMES
     UpperCAmelCase__         =      PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__         =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__         =      PRETRAINED_INIT_CONFIGURATION
     UpperCAmelCase__         =      RoFormerTokenizer



     def __init__(							self :Tuple      ,  _lowerCamelCase :Dict=None      ,  _lowerCamelCase :Dict=None      ,  _lowerCamelCase :List[Any]=True      ,  _lowerCamelCase :Dict="[UNK]"      ,  _lowerCamelCase :List[str]="[SEP]"      ,  _lowerCamelCase :str="[PAD]"      ,  _lowerCamelCase :Optional[Any]="[CLS]"      ,  _lowerCamelCase :Optional[int]="[MASK]"      ,  _lowerCamelCase :str=True      ,  _lowerCamelCase :Tuple=None      ,  **_lowerCamelCase :Any      ,  ):







      '''simple docstring'''



      super().__init__(
          _lowerCamelCase      ,  tokenizer_file=_lowerCamelCase      ,  do_lower_case=_lowerCamelCase      ,  unk_token=_lowerCamelCase      ,  sep_token=_lowerCamelCase      ,  pad_token=_lowerCamelCase      ,  cls_token=_lowerCamelCase      ,  mask_token=_lowerCamelCase      ,  tokenize_chinese_chars=_lowerCamelCase      ,  strip_accents=_lowerCamelCase      ,  **_lowerCamelCase      ,  )

      UpperCamelCase_     :					Any   	=json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
      if (
          pre_tok_state.get('lowercase'      ,  _lowerCamelCase   ) != do_lower_case
          or pre_tok_state.get('strip_accents'      ,  _lowerCamelCase   ) != strip_accents
      ):
       UpperCamelCase_     :					Optional[Any]   	=getattr(_lowerCamelCase      ,  pre_tok_state.pop('type'   )   )
       UpperCamelCase_     :					Tuple   	=do_lower_case
       UpperCamelCase_     :					Union[str, Any]   	=strip_accents
       UpperCamelCase_     :					Union[str, Any]   	=pre_tok_class(**_lowerCamelCase   )

      UpperCamelCase_     :					List[str]   	=do_lower_case



     def __getstate__(							self :Any   ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	=self.__dict__.copy()
      UpperCamelCase_     :					Union[str, Any]   	=BertPreTokenizer()
      return state



     def __setstate__(							self :str      ,  _lowerCamelCase :Optional[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=d
      UpperCamelCase_     :					Optional[int]   	=self.__dict__['_tokenizer'].get_vocab()
      UpperCamelCase_     :					Any   	=PreTokenizer.custom(JiebaPreTokenizer(_lowerCamelCase   )   )



     def  lowerCamelCase_						(							self :Dict      ,  _lowerCamelCase :Optional[Any]      ,  _lowerCamelCase :Any=None   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

      return output



     def  lowerCamelCase_						(							self :Union[str, Any]      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[Any]   	=[self.sep_token_id]
      UpperCamelCase_     :					List[Any]   	=[self.cls_token_id]
      if token_ids_a is None:
       return len(cls + token_ids_a + sep   ) * [0]
      return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]



     def  lowerCamelCase_						(							self :Optional[int]      ,  _lowerCamelCase :str      ,  _lowerCamelCase :Optional[str] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[Any]   	=self._tokenizer.model.save(_lowerCamelCase      ,  name=_lowerCamelCase   )
      return tuple(_lowerCamelCase   )




     def  lowerCamelCase_						(							self :str      ,  _lowerCamelCase :List[str]      ,  _lowerCamelCase :List[Any]=None      ,  _lowerCamelCase :Optional[Any]=None      ,  _lowerCamelCase :int=False      ,  **_lowerCamelCase :Optional[int]      ,  ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	=BertPreTokenizer()
      return super().save_pretrained(_lowerCamelCase      ,  _lowerCamelCase      ,  _lowerCamelCase      ,  _lowerCamelCase      ,  **_lowerCamelCase   )
