
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


lowerCamelCase__     :		Tuple        = logging.get_logger(__name__)

lowerCamelCase__     :		Optional[int]        = {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCamelCase__     :		List[Any]        = {
    """vocab_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-german-cased""": """https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt""",
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-german-cased""": (
            """https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCamelCase__     :		Dict        = {
    """distilbert-base-uncased""": 5_1_2,
    """distilbert-base-uncased-distilled-squad""": 5_1_2,
    """distilbert-base-cased""": 5_1_2,
    """distilbert-base-cased-distilled-squad""": 5_1_2,
    """distilbert-base-german-cased""": 5_1_2,
    """distilbert-base-multilingual-cased""": 5_1_2,
}


lowerCamelCase__     :		List[Any]        = {
    """distilbert-base-uncased""": {"""do_lower_case""": True},
    """distilbert-base-uncased-distilled-squad""": {"""do_lower_case""": True},
    """distilbert-base-cased""": {"""do_lower_case""": False},
    """distilbert-base-cased-distilled-squad""": {"""do_lower_case""": False},
    """distilbert-base-german-cased""": {"""do_lower_case""": False},
    """distilbert-base-multilingual-cased""": {"""do_lower_case""": False},
}



class    _snake_case (						UpperCAmelCase_				):
 __lowerCAmelCase	:  Any	   =      VOCAB_FILES_NAMES
 __lowerCAmelCase	:  Optional[Any]	   =      PRETRAINED_VOCAB_FILES_MAP
 __lowerCAmelCase	:  str	   =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowerCAmelCase	:  Optional[Any]	   =      PRETRAINED_INIT_CONFIGURATION
 __lowerCAmelCase	:  int	   =      ['input_ids', 'attention_mask']
 __lowerCAmelCase	:  Optional[Any]	   =      DistilBertTokenizer


 def __init__(       self       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=True       ,	SCREAMING_SNAKE_CASE_="[UNK]"       ,	SCREAMING_SNAKE_CASE_="[SEP]"       ,	SCREAMING_SNAKE_CASE_="[PAD]"       ,	SCREAMING_SNAKE_CASE_="[CLS]"       ,	SCREAMING_SNAKE_CASE_="[MASK]"       ,	SCREAMING_SNAKE_CASE_=True       ,	SCREAMING_SNAKE_CASE_=None       ,	**SCREAMING_SNAKE_CASE_       ,	):


   '''simple docstring'''




   super().__init__(
       SCREAMING_SNAKE_CASE_       ,	tokenizer_file=SCREAMING_SNAKE_CASE_       ,	do_lower_case=SCREAMING_SNAKE_CASE_       ,	unk_token=SCREAMING_SNAKE_CASE_       ,	sep_token=SCREAMING_SNAKE_CASE_       ,	pad_token=SCREAMING_SNAKE_CASE_       ,	cls_token=SCREAMING_SNAKE_CASE_       ,	mask_token=SCREAMING_SNAKE_CASE_       ,	tokenize_chinese_chars=SCREAMING_SNAKE_CASE_       ,	strip_accents=SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_       ,	)

   lowercase__	: Optional[int]     		=   json.loads(self.backend_tokenizer.normalizer.__getstate__())
   if (
       normalizer_state.get("""lowercase"""       ,	SCREAMING_SNAKE_CASE_) != do_lower_case
       or normalizer_state.get("""strip_accents"""       ,	SCREAMING_SNAKE_CASE_) != strip_accents
       or normalizer_state.get("""handle_chinese_chars"""       ,	SCREAMING_SNAKE_CASE_) != tokenize_chinese_chars
   ):
     lowercase__	: Tuple     		=   getattr(SCREAMING_SNAKE_CASE_       ,	normalizer_state.pop("""type"""))
     lowercase__	: int     		=   do_lower_case
     lowercase__	: str     		=   strip_accents
     lowercase__	: str     		=   tokenize_chinese_chars
     lowercase__	: int     		=   normalizer_class(**SCREAMING_SNAKE_CASE_)

   lowercase__	: Optional[Any]     		=   do_lower_case


 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_=None):


   '''simple docstring'''




   lowercase__	: Tuple     		=   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

   if token_ids_a:
     output += token_ids_a + [self.sep_token_id]

   return output


 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_ = None):


   '''simple docstring'''




   lowercase__	: int     		=   [self.sep_token_id]
   lowercase__	: int     		=   [self.cls_token_id]
   if token_ids_a is None:
     return len(cls + token_ids_a + sep) * [0]
   return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_ = None):


   '''simple docstring'''




   lowercase__	: Dict     		=   self._tokenizer.model.save(SCREAMING_SNAKE_CASE_       ,	name=SCREAMING_SNAKE_CASE_)
   return tuple(SCREAMING_SNAKE_CASE_)








from typing import Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature
from ...image_transforms import get_image_size, pad, rescale, to_channel_dimension_format
from ...image_utils import ChannelDimension, ImageInput, make_list_of_images, to_numpy_array, valid_images
from ...utils import TensorType, logging


lowerCamelCase__     :		Union[str, Any]        = logging.get_logger(__name__)



class    _snake_case (						UpperCAmelCase_				):
 __lowerCAmelCase	:  Any	   =      ['pixel_values']


 def __init__(       self       ,	SCREAMING_SNAKE_CASE_ = True       ,	SCREAMING_SNAKE_CASE_ = 1 / 2_55       ,	SCREAMING_SNAKE_CASE_ = True       ,	SCREAMING_SNAKE_CASE_ = 8       ,	**SCREAMING_SNAKE_CASE_       ,	):


   '''simple docstring'''




   super().__init__(**SCREAMING_SNAKE_CASE_)

   lowercase__	: List[str]     		=   do_rescale
   lowercase__	: List[Any]     		=   rescale_factor
   lowercase__	: Tuple     		=   do_pad
   lowercase__	: Optional[Any]     		=   pad_size


 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_ = None       ,	**SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   return rescale(SCREAMING_SNAKE_CASE_       ,	scale=SCREAMING_SNAKE_CASE_       ,	data_format=SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)


 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_ = None):


   '''simple docstring'''




   lowercase__		,     lowercase__	: Optional[int]     		=   get_image_size(SCREAMING_SNAKE_CASE_)
   lowercase__	: Optional[Any]     		=   (old_height // size + 1) * size - old_height
   lowercase__	: str     		=   (old_width // size + 1) * size - old_width

   return pad(SCREAMING_SNAKE_CASE_       ,	((0, pad_height), (0, pad_width))       ,	mode="""symmetric"""       ,	data_format=SCREAMING_SNAKE_CASE_)



 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_ = None       ,	SCREAMING_SNAKE_CASE_ = None       ,	SCREAMING_SNAKE_CASE_ = None       ,	SCREAMING_SNAKE_CASE_ = None       ,	SCREAMING_SNAKE_CASE_ = None       ,	SCREAMING_SNAKE_CASE_ = ChannelDimension.FIRST       ,	**SCREAMING_SNAKE_CASE_       ,	):


   '''simple docstring'''




   lowercase__	: Union[str, Any]     		=   do_rescale if do_rescale is not None else self.do_rescale
   lowercase__	: int     		=   rescale_factor if rescale_factor is not None else self.rescale_factor
   lowercase__	: Union[str, Any]     		=   do_pad if do_pad is not None else self.do_pad
   lowercase__	: Optional[Any]     		=   pad_size if pad_size is not None else self.pad_size

   lowercase__	: str     		=   make_list_of_images(SCREAMING_SNAKE_CASE_)

   if not valid_images(SCREAMING_SNAKE_CASE_):
     raise ValueError(
         """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
         """torch.Tensor, tf.Tensor or jax.ndarray.""")

   if do_rescale and rescale_factor is None:
     raise ValueError("""Rescale factor must be specified if do_rescale is True.""")

   # All transformations expect numpy arrays.
   lowercase__	: List[Any]     		=   [to_numpy_array(SCREAMING_SNAKE_CASE_) for image in images]

   if do_rescale:
     lowercase__	: str     		=   [self.rescale(image=SCREAMING_SNAKE_CASE_       ,	scale=SCREAMING_SNAKE_CASE_) for image in images]

   if do_pad:
     lowercase__	: List[str]     		=   [self.pad(SCREAMING_SNAKE_CASE_       ,	size=SCREAMING_SNAKE_CASE_) for image in images]

   lowercase__	: Optional[Any]     		=   [to_channel_dimension_format(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_) for image in images]

   lowercase__	: Dict     		=   {"""pixel_values""": images}
   return BatchFeature(data=SCREAMING_SNAKE_CASE_       ,	tensor_type=SCREAMING_SNAKE_CASE_)






