





'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__magic_name__   :						Any    =			logging.get_logger(__name__)

__magic_name__   :						Optional[Any]    =			{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

__magic_name__   :						List[str]    =			{
    """vocab_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-german-cased""": """https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt""",
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-german-cased""": (
            """https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"""
        ),
    },
}

__magic_name__   :						Union[str, Any]    =			{
    """distilbert-base-uncased""": 512,
    """distilbert-base-uncased-distilled-squad""": 512,
    """distilbert-base-cased""": 512,
    """distilbert-base-cased-distilled-squad""": 512,
    """distilbert-base-german-cased""": 512,
    """distilbert-base-multilingual-cased""": 512,
}


__magic_name__   :						Tuple    =			{
    """distilbert-base-uncased""": {"""do_lower_case""": True},
    """distilbert-base-uncased-distilled-squad""": {"""do_lower_case""": True},
    """distilbert-base-cased""": {"""do_lower_case""": False},
    """distilbert-base-cased-distilled-squad""": {"""do_lower_case""": False},
    """distilbert-base-german-cased""": {"""do_lower_case""": False},
    """distilbert-base-multilingual-cased""": {"""do_lower_case""": False},
}


class       __SCREAMING_SNAKE_CASE	(						_UpperCAmelCase			):



     '''simple docstring'''






     UpperCAmelCase__       :						Tuple      =   VOCAB_FILES_NAMES
     UpperCAmelCase__       :						List[Any]      =   PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__       :						int      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__       :						Optional[int]      =   PRETRAINED_INIT_CONFIGURATION
     UpperCAmelCase__       :						Tuple      =   ['''input_ids''', '''attention_mask''']
     UpperCAmelCase__       :						Dict      =   DistilBertTokenizer

     def __init__(					self						,   lowerCamelCase=None						,   lowerCamelCase=None						,   lowerCamelCase=True						,   lowerCamelCase="[UNK]"						,   lowerCamelCase="[SEP]"						,   lowerCamelCase="[PAD]"						,   lowerCamelCase="[CLS]"						,   lowerCamelCase="[MASK]"						,   lowerCamelCase=True						,   lowerCamelCase=None						,   **lowerCamelCase						,   ):
       super().__init__(
           lowercase__						,   tokenizer_file=lowercase__						,   do_lower_case=lowercase__						,   unk_token=lowercase__						,   sep_token=lowercase__						,   pad_token=lowercase__						,   cls_token=lowercase__						,   mask_token=lowercase__						,   tokenize_chinese_chars=lowercase__						,   strip_accents=lowercase__						,   **lowercase__						,   )

       _snake_case     		=				json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
       if (
           normalizer_state.get("lowercase"						,   lowercase__	) != do_lower_case
           or normalizer_state.get("strip_accents"						,   lowercase__	) != strip_accents
           or normalizer_state.get("handle_chinese_chars"						,   lowercase__	) != tokenize_chinese_chars
       ):
         _snake_case     		=				getattr(lowercase__						,   normalizer_state.pop("type"	)	)
         _snake_case     		=				do_lower_case
         _snake_case     		=				strip_accents
         _snake_case     		=				tokenize_chinese_chars
         _snake_case     		=				normalizer_class(**lowercase__	)

       _snake_case     		=				do_lower_case

     def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase=None	):
       _snake_case     		=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

       return output

     def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase = None	):
       _snake_case     		=				[self.sep_token_id]
       _snake_case     		=				[self.cls_token_id]
       if token_ids_a is None:
         return len(cls + token_ids_a + sep	) * [0]
       return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]






     def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase = None	):
       _snake_case     		=				self._tokenizer.model.save(lowercase__						,   name=lowercase__	)
       return tuple(lowercase__	)









'''simple docstring'''



from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    convert_to_rgb,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


__magic_name__   :						Tuple    =			logging.get_logger(__name__)


if is_vision_available():
 import PIL


class       __SCREAMING_SNAKE_CASE	(						__UpperCamelCase			):



     '''simple docstring'''






     UpperCAmelCase__       :						str      =   ['''pixel_values''']

     def __init__(					self						,   lowerCamelCase = True						,   lowerCamelCase = None						,   lowerCamelCase = PILImageResampling.BICUBIC						,   lowerCamelCase = True						,   lowerCamelCase = None						,   lowerCamelCase = True						,   lowerCamelCase = 1 / 255						,   lowerCamelCase = True						,   lowerCamelCase = None						,   lowerCamelCase = None						,   lowerCamelCase = True						,   **lowerCamelCase						,   ):
       super().__init__(**lowerCamelCase	)
       _snake_case     		=				size if size is not None else {"shortest_edge": 224}
       _snake_case     		=				get_size_dict(lowerCamelCase						,   default_to_square=lowerCamelCase	)
       _snake_case     		=				crop_size if crop_size is not None else {"height": 224, "width": 224}
       _snake_case     		=				get_size_dict(lowerCamelCase						,   default_to_square=lowerCamelCase						,   param_name="crop_size"	)

       _snake_case     		=				do_resize
       _snake_case     		=				size
       _snake_case     		=				resample
       _snake_case     		=				do_center_crop
       _snake_case     		=				crop_size
       _snake_case     		=				do_rescale
       _snake_case     		=				rescale_factor
       _snake_case     		=				do_normalize
       _snake_case     		=				image_mean if image_mean is not None else OPENAI_CLIP_MEAN
       _snake_case     		=				image_std if image_std is not None else OPENAI_CLIP_STD
       _snake_case     		=				do_convert_rgb

     def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase = PILImageResampling.BICUBIC						,   lowerCamelCase = None						,   **lowerCamelCase						,   ):
       _snake_case     		=				get_size_dict(lowerCamelCase						,   default_to_square=lowerCamelCase	)
       if "shortest_edge" not in size:
         raise ValueError(F'''The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}'''	)
       _snake_case     		=				get_resize_output_image_size(lowerCamelCase						,   size=size["shortest_edge"]						,   default_to_square=lowerCamelCase	)
       return resize(lowerCamelCase						,   size=lowerCamelCase						,   resample=lowerCamelCase						,   data_format=lowerCamelCase						,   **lowerCamelCase	)

     def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase = None						,   **lowerCamelCase						,   ):
       _snake_case     		=				get_size_dict(lowerCamelCase	)
       if "height" not in size or "width" not in size:
         raise ValueError(F'''The `size` parameter must contain the keys (height, width). Got {size.keys()}'''	)
       return center_crop(lowerCamelCase						,   size=(size["height"], size["width"])						,   data_format=lowerCamelCase						,   **lowerCamelCase	)

     def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase = None						,   **lowerCamelCase						,   ):
       return rescale(lowerCamelCase						,   scale=lowerCamelCase						,   data_format=lowerCamelCase						,   **lowerCamelCase	)

     def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase						,   lowerCamelCase = None						,   **lowerCamelCase						,   ):
       return normalize(lowerCamelCase						,   mean=lowerCamelCase						,   std=lowerCamelCase						,   data_format=lowerCamelCase						,   **lowerCamelCase	)






     def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase = None						,   lowerCamelCase = None						,   lowerCamelCase = None						,   lowerCamelCase = None						,   lowerCamelCase = None						,   lowerCamelCase = None						,   lowerCamelCase = None						,   lowerCamelCase = None						,   lowerCamelCase = None						,   lowerCamelCase = None						,   lowerCamelCase = None						,   lowerCamelCase = None						,   lowerCamelCase = ChannelDimension.FIRST						,   **lowerCamelCase						,   ):
       _snake_case     		=				do_resize if do_resize is not None else self.do_resize
       _snake_case     		=				size if size is not None else self.size
       _snake_case     		=				get_size_dict(lowerCamelCase						,   param_name="size"						,   default_to_square=lowerCamelCase	)
       _snake_case     		=				resample if resample is not None else self.resample
       _snake_case     		=				do_center_crop if do_center_crop is not None else self.do_center_crop
       _snake_case     		=				crop_size if crop_size is not None else self.crop_size
       _snake_case     		=				get_size_dict(lowerCamelCase						,   param_name="crop_size"						,   default_to_square=lowerCamelCase	)
       _snake_case     		=				do_rescale if do_rescale is not None else self.do_rescale
       _snake_case     		=				rescale_factor if rescale_factor is not None else self.rescale_factor
       _snake_case     		=				do_normalize if do_normalize is not None else self.do_normalize
       _snake_case     		=				image_mean if image_mean is not None else self.image_mean
       _snake_case     		=				image_std if image_std is not None else self.image_std
       _snake_case     		=				do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

       _snake_case     		=				make_list_of_images(lowerCamelCase	)

       if not valid_images(lowerCamelCase	):
         raise ValueError(
             "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
             "torch.Tensor, tf.Tensor or jax.ndarray."	)

       if do_resize and size is None:
         raise ValueError("Size must be specified if do_resize is True."	)

       if do_center_crop and crop_size is None:
         raise ValueError("Crop size must be specified if do_center_crop is True."	)

       if do_rescale and rescale_factor is None:
         raise ValueError("Rescale factor must be specified if do_rescale is True."	)

       if do_normalize and (image_mean is None or image_std is None):
         raise ValueError("Image mean and std must be specified if do_normalize is True."	)

       # PIL RGBA images are converted to RGB
       if do_convert_rgb:
         _snake_case     		=				[convert_to_rgb(lowerCamelCase	) for image in images]

       # All transformations expect numpy arrays.
       _snake_case     		=				[to_numpy_array(lowerCamelCase	) for image in images]

       if do_resize:
         _snake_case     		=				[self.resize(image=lowerCamelCase						,   size=lowerCamelCase						,   resample=lowerCamelCase	) for image in images]

       if do_center_crop:
         _snake_case     		=				[self.center_crop(image=lowerCamelCase						,   size=lowerCamelCase	) for image in images]

       if do_rescale:
         _snake_case     		=				[self.rescale(image=lowerCamelCase						,   scale=lowerCamelCase	) for image in images]

       if do_normalize:
         _snake_case     		=				[self.normalize(image=lowerCamelCase						,   mean=lowerCamelCase						,   std=lowerCamelCase	) for image in images]

       _snake_case     		=				[to_channel_dimension_format(lowerCamelCase						,   lowerCamelCase	) for image in images]

       _snake_case     		=				{"pixel_values": images}
       return BatchFeature(data=lowerCamelCase						,   tensor_type=lowerCamelCase	)


