





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


SCREAMING_SNAKE_CASE__							:      List[str]         =    logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__							:      List[str]         =    {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE__							:      Union[str, Any]         =    {
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

SCREAMING_SNAKE_CASE__							:      Optional[int]         =    {
    'junnyu/roformer_chinese_small': 1_5_3_6,
    'junnyu/roformer_chinese_base': 1_5_3_6,
    'junnyu/roformer_chinese_char_small': 5_1_2,
    'junnyu/roformer_chinese_char_base': 5_1_2,
    'junnyu/roformer_small_discriminator': 1_2_8,
    'junnyu/roformer_small_generator': 1_2_8,
}


SCREAMING_SNAKE_CASE__							:      str         =    {
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}







class 	__lowerCAmelCase      (       _UpperCamelCase	):
   _UpperCamelCase			:    Any  		=					VOCAB_FILES_NAMES
   _UpperCamelCase			:    Optional[int]  		=					PRETRAINED_VOCAB_FILES_MAP
   _UpperCamelCase			:    List[Any]  		=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _UpperCamelCase			:    Tuple  		=					PRETRAINED_INIT_CONFIGURATION
   _UpperCamelCase			:    Tuple  		=					RoFormerTokenizer





   def __init__(			self       ,      snake_case=None       ,      snake_case=None       ,      snake_case=True       ,      snake_case="[UNK]"       ,      snake_case="[SEP]"       ,      snake_case="[PAD]"       ,      snake_case="[CLS]"       ,      snake_case="[MASK]"       ,      snake_case=True       ,      snake_case=None       ,      **snake_case       ,      )					->	List[str]:
      """simple docstring"""





      super().__init__(
          _A       ,      tokenizer_file=_A       ,      do_lower_case=_A       ,      unk_token=_A       ,      sep_token=_A       ,      pad_token=_A       ,      cls_token=_A       ,      mask_token=_A       ,      tokenize_chinese_chars=_A       ,      strip_accents=_A       ,      **_A       ,      )

      a__						:    str   		=	json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
      if (
          pre_tok_state.get("lowercase"       ,      _A   ) != do_lower_case
          or pre_tok_state.get("strip_accents"       ,      _A   ) != strip_accents
      ):
         a__						:    Dict   		=	getattr(_A       ,      pre_tok_state.pop("type"   )   )
         a__						:    Union[str, Any]   		=	do_lower_case
         a__						:    Optional[Any]   		=	strip_accents
         a__						:    int   		=	pre_tok_class(**_A   )

      a__						:    List[str]   		=	do_lower_case





   def __getstate__(			self   )					->	Dict:
      """simple docstring"""





      a__						:    List[str]   		=	self.__dict__.copy()
      a__						:    List[str]   		=	BertPreTokenizer()
      return state





   def __setstate__(			self       ,      snake_case   )					->	List[str]:
      """simple docstring"""





      a__						:    Any   		=	d
      a__						:    List[str]   		=	self.__dict__['''_tokenizer'''].get_vocab()
      a__						:    Dict   		=	PreTokenizer.custom(JiebaPreTokenizer(_A   )   )





   def 						_snake_case			(			self       ,      snake_case       ,      snake_case=None   )					->	List[str]:
      """simple docstring"""





      a__						:    Union[str, Any]   		=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

      return output





   def 						_snake_case			(			self       ,      snake_case       ,      snake_case = None   )					->	List[int]:
      """simple docstring"""





      a__						:    Any   		=	[self.sep_token_id]
      a__						:    Dict   		=	[self.cls_token_id]
      if token_ids_a is None:
         return len(cls + token_ids_a + sep   ) * [0]
      return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]





   def 						_snake_case			(			self       ,      snake_case       ,      snake_case = None   )					->	Tuple[str]:
      """simple docstring"""





      a__						:    Optional[int]   		=	self._tokenizer.model.save(_A       ,      name=_A   )
      return tuple(_A   )





   def 						_snake_case			(			self       ,      snake_case       ,      snake_case=None       ,      snake_case=None       ,      snake_case=False       ,      **snake_case       ,      )					->	str:
      """simple docstring"""





      a__						:    Optional[Any]   		=	BertPreTokenizer()
      return super().save_pretrained(_A       ,      _A       ,      _A       ,      _A       ,      **_A   )









'''simple docstring'''





import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType


class  snake_case__      (					UpperCamelCase):
					a_        =					["image_processor", "tokenizer"]
					a_        =					"LayoutLMv2ImageProcessor"
					a_        =					("LayoutXLMTokenizer", "LayoutXLMTokenizerFast")




					def __init__(  self  :		Optional[int]				, _A  :		str=None				, _A  :		Optional[Any]=None				, **_A  :		Any      )    ->							Tuple:
								if "feature_extractor" in kwargs:
											warnings.warn(
											    '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
											    ''' instead.'''				, _A				, )
											UpperCAmelCase_      :       int          =       kwargs.pop('''feature_extractor'''      )

								UpperCAmelCase_      :       str          =       image_processor if image_processor is not None else feature_extractor
								if image_processor is None:
											raise ValueError('''You need to specify an `image_processor`.'''      )
								if tokenizer is None:
											raise ValueError('''You need to specify a `tokenizer`.'''      )

								super().__init__(_A				, _A      )




					def __call__(  self  :		str				, _A  :		Optional[int]				, _A  :		Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None				, _A  :		Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None				, _A  :		Union[List[List[int]], List[List[List[int]]]] = None				, _A  :		Optional[Union[List[int], List[List[int]]]] = None				, _A  :		bool = True				, _A  :		Union[bool, str, PaddingStrategy] = False				, _A  :		Union[bool, str, TruncationStrategy] = None				, _A  :		Optional[int] = None				, _A  :		int = 0				, _A  :		Optional[int] = None				, _A  :		Optional[bool] = None				, _A  :		Optional[bool] = None				, _A  :		bool = False				, _A  :		bool = False				, _A  :		bool = False				, _A  :		bool = False				, _A  :		bool = True				, _A  :		Optional[Union[str, TensorType]] = None				, **_A  :		Dict				, )    ->							BatchEncoding:
								# verify input
								if self.image_processor.apply_ocr and (boxes is not None):
											raise ValueError(
											    '''You cannot provide bounding boxes '''
											    '''if you initialized the image processor with apply_ocr set to True.'''      )

								if self.image_processor.apply_ocr and (word_labels is not None):
											raise ValueError(
											    '''You cannot provide word labels if you initialized the image processor with apply_ocr set to True.'''      )

								if return_overflowing_tokens is True and return_offsets_mapping is False:
											raise ValueError('''You cannot return overflowing tokens without returning the offsets mapping.'''      )

								# first, apply the image processor
								UpperCAmelCase_      :       int          =       self.image_processor(images=_A				, return_tensors=_A      )

								# second, apply the tokenizer
								if text is not None and self.image_processor.apply_ocr and text_pair is None:
											if isinstance(_A				, _A      ):
														UpperCAmelCase_      :       int          =       [text]  # add batch dimension (as the image processor always adds a batch dimension)
											UpperCAmelCase_      :       int          =       features['''words''']

								UpperCAmelCase_      :       str          =       self.tokenizer(
								    text=text if text is not None else features['''words''']				, text_pair=text_pair if text_pair is not None else None				, boxes=boxes if boxes is not None else features['''boxes''']				, word_labels=_A				, add_special_tokens=_A				, padding=_A				, truncation=_A				, max_length=_A				, stride=_A				, pad_to_multiple_of=_A				, return_token_type_ids=_A				, return_attention_mask=_A				, return_overflowing_tokens=_A				, return_special_tokens_mask=_A				, return_offsets_mapping=_A				, return_length=_A				, verbose=_A				, return_tensors=_A				, **_A				, )

								# add pixel values
								UpperCAmelCase_      :       int          =       features.pop('''pixel_values'''      )
								if return_overflowing_tokens is True:
											UpperCAmelCase_      :       List[Any]          =       self.get_overflowing_images(_A				, encoded_inputs['''overflow_to_sample_mapping''']      )
								UpperCAmelCase_      :       Optional[int]          =       images

								return encoded_inputs




					def 		A      (  self  :		Union[str, Any]				, _A  :		int				, _A  :		Tuple      )    ->							Dict:
								# in case there's an overflow, ensure each `input_ids` sample is mapped to its corresponding image
								UpperCAmelCase_      :       Tuple          =       []
								for sample_idx in overflow_to_sample_mapping:
											images_with_overflow.append(images[sample_idx]      )

								if len(_A      ) != len(_A      ):
											raise ValueError(
											    '''Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got'''
											    F" {len(_A      )} and {len(_A      )}"      )

								return images_with_overflow




					def 		A      (  self  :		Optional[Any]				, *_A  :		Union[str, Any]				, **_A  :		Union[str, Any]      )    ->							Tuple:
								return self.tokenizer.batch_decode(*_A				, **_A      )




					def 		A      (  self  :		Any				, *_A  :		Optional[Any]				, **_A  :		Tuple      )    ->							Tuple:
								return self.tokenizer.decode(*_A				, **_A      )




					@property
					def 		A      (  self  :		Union[str, Any]      )    ->							List[Any]:
								return ["input_ids", "bbox", "attention_mask", "image"]




					@property
					def 		A      (  self  :		Tuple      )    ->							Optional[int]:
								warnings.warn(
								    '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''				, _A				, )
								return self.image_processor_class





					@property
					def 		A      (  self  :		Tuple      )    ->							str:
								warnings.warn(
								    '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''				, _A				, )
								return self.image_processor


