




'''simple docstring'''



from typing import Dict, Iterable, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


UpperCamelCase_    	=				logging.get_logger(__name__)






class 							_a				(			SCREAMING_SNAKE_CASE ):





       '''simple docstring'''






       A						:				List[str]        =						['''pixel_values''']







       def __init__(      self,							A = True,							A = None,							A = PILImageResampling.BICUBIC,							A = True,							A = None,							A = True,							A = 1 / 255,							A = True,							A = IMAGENET_DEFAULT_MEAN,							A = IMAGENET_DEFAULT_STD,							**A,							):







             '''simple docstring'''







             super().__init__(**A  )
             SCREAMING_SNAKE_CASE   :      Any    					=						size if size is not None else {'shortest_edge': 224}
             SCREAMING_SNAKE_CASE   :      List[str]    					=						get_size_dict(A,							default_to_square=A  )
             SCREAMING_SNAKE_CASE   :      Tuple    					=						crop_size if crop_size is not None else {'height': 224, 'width': 224}
             SCREAMING_SNAKE_CASE   :      List[str]    					=						get_size_dict(A,							param_name='crop_size'  )

             SCREAMING_SNAKE_CASE   :      Any    					=						do_resize
             SCREAMING_SNAKE_CASE   :      List[str]    					=						size
             SCREAMING_SNAKE_CASE   :      Any    					=						resample
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						do_center_crop
             SCREAMING_SNAKE_CASE   :      Any    					=						crop_size
             SCREAMING_SNAKE_CASE   :      Tuple    					=						do_rescale
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						rescale_factor
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						do_normalize
             SCREAMING_SNAKE_CASE   :      int    					=						image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
             SCREAMING_SNAKE_CASE   :      List[str]    					=						image_std if image_std is not None else IMAGENET_DEFAULT_STD







       def        UpperCamelCase_							(      self,							A,							A,							A = PILImageResampling.BICUBIC,							A = None,							**A,							):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      int    					=						get_size_dict(A,							default_to_square=A  )
             # size_dict is a dict with either keys "height" and "width" or "shortest_edge"
             if "shortest_edge" in size:
                   SCREAMING_SNAKE_CASE   :      str    					=						int((256 / 224) * size['shortest_edge']  )
                   SCREAMING_SNAKE_CASE   :      str    					=						get_resize_output_image_size(A,							size=A,							default_to_square=A  )
                   SCREAMING_SNAKE_CASE   :      Dict    					=						{'height': output_size[0], 'width': output_size[1]}
             if "height" not in size_dict or "width" not in size_dict:
                   raise ValueError(
                       F"Size dict must have keys 'height' and 'width' or 'shortest_edge'. Got {size_dict.keys()}"  )
             return resize(
                 A,							size=(size_dict['height'], size_dict['width']),							resample=A,							data_format=A,							**A  )







       def        UpperCamelCase_							(      self,							A,							A,							A = None,							**A,							):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						get_size_dict(A  )
             if "height" not in size or "width" not in size:
                   raise ValueError(F"Size dict must have keys 'height' and 'width'. Got {size.keys()}"  )
             return center_crop(A,							size=(size['height'], size['width']),							data_format=A,							**A  )







       def        UpperCamelCase_							(      self,							A,							A,							A = None,							**A,							):







             '''simple docstring'''







             return rescale(A,							scale=A,							data_format=A,							**A  )







       def        UpperCamelCase_							(      self,							A,							A,							A,							A = None,							**A,							):







             '''simple docstring'''







             return normalize(A,							mean=A,							std=A,							data_format=A,							**A  )







       def        UpperCamelCase_							(      self,							A,							A = None,							A = None,							A = None,							A = None,							A = None,							A = None,							A = None,							A = None,							A = None,							A = None,							A = None,							A = ChannelDimension.FIRST,							**A,							):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						do_resize if do_resize is not None else self.do_resize
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						resample if resample is not None else self.resample
             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						do_center_crop if do_center_crop is not None else self.do_center_crop
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						do_rescale if do_rescale is not None else self.do_rescale
             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						rescale_factor if rescale_factor is not None else self.rescale_factor
             SCREAMING_SNAKE_CASE   :      List[str]    					=						do_normalize if do_normalize is not None else self.do_normalize
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						image_mean if image_mean is not None else self.image_mean
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						image_std if image_std is not None else self.image_std

             SCREAMING_SNAKE_CASE   :      Dict    					=						size if size is not None else self.size
             SCREAMING_SNAKE_CASE   :      List[str]    					=						get_size_dict(A,							default_to_square=A  )
             SCREAMING_SNAKE_CASE   :      Tuple    					=						crop_size if crop_size is not None else self.crop_size
             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						get_size_dict(A,							param_name='crop_size'  )

             SCREAMING_SNAKE_CASE   :      int    					=						make_list_of_images(A  )

             if not valid_images(A  ):
                   raise ValueError(
                       'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
                       'torch.Tensor, tf.Tensor or jax.ndarray.'  )

             if do_resize and size is None:
                   raise ValueError('Size must be specified if do_resize is True.'  )

             if do_center_crop and crop_size is None:
                   raise ValueError('Crop size must be specified if do_center_crop is True.'  )

             if do_rescale and rescale_factor is None:
                   raise ValueError('Rescale factor must be specified if do_rescale is True.'  )

             if do_normalize and (image_mean is None or image_std is None):
                   raise ValueError('Image mean and std must be specified if do_normalize is True.'  )

             # All transformations expect numpy arrays.
             SCREAMING_SNAKE_CASE   :      Tuple    					=						[to_numpy_array(A  ) for image in images]

             if do_resize:
                   SCREAMING_SNAKE_CASE   :      List[str]    					=						[self.resize(A,							A,							A  ) for image in images]

             if do_center_crop:
                   SCREAMING_SNAKE_CASE   :      int    					=						[self.center_crop(A,							A  ) for image in images]

             if do_rescale:
                   SCREAMING_SNAKE_CASE   :      int    					=						[self.rescale(A,							A  ) for image in images]

             if do_normalize:
                   SCREAMING_SNAKE_CASE   :      List[str]    					=						[self.normalize(A,							A,							A  ) for image in images]

             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						[to_channel_dimension_format(A,							A  ) for image in images]

             SCREAMING_SNAKE_CASE   :      Tuple    					=						{'pixel_values': images}
             return BatchFeature(data=A,							tensor_type=A  )










'''simple docstring'''



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_xlnet import XLNetTokenizer
else:
      SCREAMING_SNAKE_CASE							=						None


SCREAMING_SNAKE_CASE							=						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE							=						{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE							=						{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json',
    },
}

SCREAMING_SNAKE_CASE							=						{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

SCREAMING_SNAKE_CASE							=						'▁'

# Segments (not really needed)
SCREAMING_SNAKE_CASE							=						0
SCREAMING_SNAKE_CASE							=						1
SCREAMING_SNAKE_CASE							=						2
SCREAMING_SNAKE_CASE							=						3
SCREAMING_SNAKE_CASE							=						4
class UpperCAmelCase_       (     __A					):




      """simple docstring"""




      UpperCamelCase_             =      VOCAB_FILES_NAMES
      UpperCamelCase_             =      PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase_             =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase_             =      '''left'''
      UpperCamelCase_             =      XLNetTokenizer






      def __init__(			self      :		int  ,   UpperCAmelCase      :		Dict=None  ,   UpperCAmelCase      :		str=None  ,   UpperCAmelCase      :		str=False  ,   UpperCAmelCase      :		List[Any]=True  ,   UpperCAmelCase      :		Optional[Any]=False  ,   UpperCAmelCase      :		int="<s>"  ,   UpperCAmelCase      :		Optional[int]="</s>"  ,   UpperCAmelCase      :		str="<unk>"  ,   UpperCAmelCase      :		Optional[Any]="<sep>"  ,   UpperCAmelCase      :		Optional[int]="<pad>"  ,   UpperCAmelCase      :		Optional[Any]="<cls>"  ,   UpperCAmelCase      :		Dict="<mask>"  ,   UpperCAmelCase      :		int=["<eop>", "<eod>"]  ,   **UpperCAmelCase      :		List[Any]  ,   )	->	List[str]:


       '''simple docstring'''

       lowercase       : Dict						=AddedToken(UpperCAmelCase  ,   lstrip=UpperCAmelCase  ,   rstrip=UpperCAmelCase		) if isinstance(UpperCAmelCase  ,   UpperCAmelCase		) else mask_token

       super().__init__(
           vocab_file=UpperCAmelCase  ,   tokenizer_file=UpperCAmelCase  ,   do_lower_case=UpperCAmelCase  ,   remove_space=UpperCAmelCase  ,   keep_accents=UpperCAmelCase  ,   bos_token=UpperCAmelCase  ,   eos_token=UpperCAmelCase  ,   unk_token=UpperCAmelCase  ,   sep_token=UpperCAmelCase  ,   pad_token=UpperCAmelCase  ,   cls_token=UpperCAmelCase  ,   mask_token=UpperCAmelCase  ,   additional_special_tokens=UpperCAmelCase  ,   **UpperCAmelCase  ,   )

       lowercase       : Tuple						=3
       lowercase       : Union[str, Any]						=do_lower_case
       lowercase       : Any						=remove_space
       lowercase       : int						=keep_accents
       lowercase       : int						=vocab_file
       lowercase       : Union[str, Any]						=False if not self.vocab_file else True






      def 					A__   (			self      :		Any  ,   UpperCAmelCase      :		List[int]  ,   UpperCAmelCase      :		Optional[List[int]] = None		)	->	List[int]:


       '''simple docstring'''

       lowercase       : Union[str, Any]						=[self.sep_token_id]
       lowercase       : Optional[Any]						=[self.cls_token_id]
       if token_ids_a is None:
        return token_ids_a + sep + cls
       return token_ids_a + sep + token_ids_a + sep + cls






      def 					A__   (			self      :		str  ,   UpperCAmelCase      :		List[int]  ,   UpperCAmelCase      :		Optional[List[int]] = None		)	->	List[int]:


       '''simple docstring'''

       lowercase       : Optional[int]						=[self.sep_token_id]
       lowercase       : Union[str, Any]						=[2]

       if token_ids_a is None:
        return len(token_ids_a + sep		) * [0] + cls_segment_id
       return len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1] + cls_segment_id






      def 					A__   (			self      :		str  ,   UpperCAmelCase      :		str  ,   UpperCAmelCase      :		Optional[str] = None		)	->	Tuple[str]:


       '''simple docstring'''

       if not self.can_save_slow_tokenizer:
        raise ValueError(
            '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
            '''tokenizer.'''		)

       if not os.path.isdir(UpperCAmelCase		):
        logger.error(f'Vocabulary path ({save_directory}) should be a directory'		)
        return
       lowercase       : Dict						=os.path.join(
           UpperCAmelCase  ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

       if os.path.abspath(self.vocab_file		) != os.path.abspath(UpperCAmelCase		):
        copyfile(self.vocab_file  ,   UpperCAmelCase		)

       return (out_vocab_file,)



