
'''simple docstring'''


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_fnet import FNetTokenizer
else:
   snake_case_ :  List[Any]     =					None

snake_case_ :  Optional[int]     =					logging.get_logger(__name__)
snake_case_ :  List[str]     =					{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

snake_case_ :  List[Any]     =					{
    'vocab_file': {
        'google/fnet-base': 'https://huggingface.co/google/fnet-base/resolve/main/spiece.model',
        'google/fnet-large': 'https://huggingface.co/google/fnet-large/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'google/fnet-base': 'https://huggingface.co/google/fnet-base/resolve/main/tokenizer.json',
        'google/fnet-large': 'https://huggingface.co/google/fnet-large/resolve/main/tokenizer.json',
    },
}

snake_case_ :  Dict     =					{
    'google/fnet-base': 512,
    'google/fnet-large': 512,
}

snake_case_ :  Any     =					'▁'


class 						lowercase__	(  snake_case_   ):






   '''simple docstring'''







   _snake_case		   =							VOCAB_FILES_NAMES
   _snake_case		   =							PRETRAINED_VOCAB_FILES_MAP
   _snake_case		   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _snake_case		   =							['''input_ids''', '''token_type_ids''']
   _snake_case		   =							FNetTokenizer







   def __init__(			self ,   lowerCamelCase__=None ,   lowerCamelCase__=None ,   lowerCamelCase__=False ,   lowerCamelCase__=True ,   lowerCamelCase__=True ,   lowerCamelCase__="<unk>" ,   lowerCamelCase__="[SEP]" ,   lowerCamelCase__="<pad>" ,   lowerCamelCase__="[CLS]" ,   lowerCamelCase__="[MASK]" ,   **lowerCamelCase__ ,   ):

      '''simple docstring'''







      UpperCamelCase       =  (
          AddedToken(lowerCamelCase__ ,   lstrip=lowerCamelCase__ ,   rstrip=lowerCamelCase__ ,   normalized=lowerCamelCase__   )
          if isinstance(lowerCamelCase__ ,   lowerCamelCase__   )
          else mask_token
      )

      super().__init__(
          lowerCamelCase__ ,   tokenizer_file=lowerCamelCase__ ,   do_lower_case=lowerCamelCase__ ,   remove_space=lowerCamelCase__ ,   keep_accents=lowerCamelCase__ ,   unk_token=lowerCamelCase__ ,   sep_token=lowerCamelCase__ ,   pad_token=lowerCamelCase__ ,   cls_token=lowerCamelCase__ ,   mask_token=lowerCamelCase__ ,   **lowerCamelCase__ ,   )

      UpperCamelCase       =  do_lower_case
      UpperCamelCase       =  remove_space
      UpperCamelCase       =  keep_accents
      UpperCamelCase       =  vocab_file
      UpperCamelCase       =  False if not self.vocab_file else True







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      UpperCamelCase       =  [self.sep_token_id]
      UpperCamelCase       =  [self.cls_token_id]
      if token_ids_a is None:
         return cls + token_ids_a + sep
      return cls + token_ids_a + sep + token_ids_a + sep







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      UpperCamelCase       =  [self.sep_token_id]
      UpperCamelCase       =  [self.cls_token_id]

      if token_ids_a is None:
         return len(cls + token_ids_a + sep   ) * [0]
      return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      if not os.path.isdir(lowerCamelCase__   ):
         logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
         return
      UpperCamelCase       =  os.path.join(
          lowerCamelCase__ ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )

      if os.path.abspath(self.vocab_file   ) != os.path.abspath(lowerCamelCase__   ):
         copyfile(self.vocab_file ,   lowerCamelCase__   )

      return (out_vocab_file,)







'''simple docstring'''


import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
   import torch

if is_vision_available():
   from PIL import Image

   from transformers import LevitImageProcessor


class 						lowercase__	(  unittest.TestCase   ):






   '''simple docstring'''







   def __init__(			self ,   lowerCamelCase__ ,   lowerCamelCase__=7 ,   lowerCamelCase__=3 ,   lowerCamelCase__=1_8 ,   lowerCamelCase__=3_0 ,   lowerCamelCase__=4_0_0 ,   lowerCamelCase__=True ,   lowerCamelCase__=None ,   lowerCamelCase__=True ,   lowerCamelCase__=None ,   lowerCamelCase__=True ,   lowerCamelCase__=[0.5, 0.5, 0.5] ,   lowerCamelCase__=[0.5, 0.5, 0.5] ,   ):

      '''simple docstring'''







      UpperCamelCase       =  size if size is not None else {'''shortest_edge''': 1_8}
      UpperCamelCase       =  crop_size if crop_size is not None else {'''height''': 1_8, '''width''': 1_8}
      UpperCamelCase       =  parent
      UpperCamelCase       =  batch_size
      UpperCamelCase       =  num_channels
      UpperCamelCase       =  image_size
      UpperCamelCase       =  min_resolution
      UpperCamelCase       =  max_resolution
      UpperCamelCase       =  do_resize
      UpperCamelCase       =  size
      UpperCamelCase       =  do_center_crop
      UpperCamelCase       =  crop_size
      UpperCamelCase       =  do_normalize
      UpperCamelCase       =  image_mean
      UpperCamelCase       =  image_std







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      return {
          "image_mean": self.image_mean,
          "image_std": self.image_std,
          "do_normalize": self.do_normalize,
          "do_resize": self.do_resize,
          "do_center_crop": self.do_center_crop,
          "size": self.size,
          "crop_size": self.crop_size,
      }




@require_torch
@require_vision
class 						lowercase__	(  snake_case_,					unittest.TestCase   ):






   '''simple docstring'''







   _snake_case		   =							LevitImageProcessor if is_vision_available() else None







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  LevitImageProcessingTester(self   )







   @property
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      return self.image_processor_tester.prepare_image_processor_dict()







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.image_processing_class(**self.image_processor_dict   )
      self.assertTrue(hasattr(lowerCamelCase__ ,   '''image_mean'''   )   )
      self.assertTrue(hasattr(lowerCamelCase__ ,   '''image_std'''   )   )
      self.assertTrue(hasattr(lowerCamelCase__ ,   '''do_normalize'''   )   )
      self.assertTrue(hasattr(lowerCamelCase__ ,   '''do_resize'''   )   )
      self.assertTrue(hasattr(lowerCamelCase__ ,   '''do_center_crop'''   )   )
      self.assertTrue(hasattr(lowerCamelCase__ ,   '''size'''   )   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.image_processing_class.from_dict(self.image_processor_dict   )
      self.assertEqual(image_processor.size ,   {'''shortest_edge''': 1_8}   )
      self.assertEqual(image_processor.crop_size ,   {'''height''': 1_8, '''width''': 1_8}   )

      UpperCamelCase       =  self.image_processing_class.from_dict(self.image_processor_dict ,   size=4_2 ,   crop_size=8_4   )
      self.assertEqual(image_processor.size ,   {'''shortest_edge''': 4_2}   )
      self.assertEqual(image_processor.crop_size ,   {'''height''': 8_4, '''width''': 8_4}   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      pass







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.image_processing_class(**self.image_processor_dict   )
      # create random PIL images
      UpperCamelCase       =  prepare_image_inputs(self.image_processor_tester ,   equal_resolution=lowerCamelCase__   )
      for image in image_inputs:
         self.assertIsInstance(lowerCamelCase__ ,   Image.Image   )

      # Test not batched input
      UpperCamelCase       =  image_processing(image_inputs[0] ,   return_tensors='''pt'''   ).pixel_values
      self.assertEqual(
          encoded_images.shape ,   (
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          ) ,   )

      # Test batched
      UpperCamelCase       =  image_processing(lowerCamelCase__ ,   return_tensors='''pt'''   ).pixel_values
      self.assertEqual(
          encoded_images.shape ,   (
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          ) ,   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.image_processing_class(**self.image_processor_dict   )
      # create random numpy tensors
      UpperCamelCase       =  prepare_image_inputs(self.image_processor_tester ,   equal_resolution=lowerCamelCase__ ,   numpify=lowerCamelCase__   )
      for image in image_inputs:
         self.assertIsInstance(lowerCamelCase__ ,   np.ndarray   )

      # Test not batched input
      UpperCamelCase       =  image_processing(image_inputs[0] ,   return_tensors='''pt'''   ).pixel_values
      self.assertEqual(
          encoded_images.shape ,   (
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          ) ,   )

      # Test batched
      UpperCamelCase       =  image_processing(lowerCamelCase__ ,   return_tensors='''pt'''   ).pixel_values
      self.assertEqual(
          encoded_images.shape ,   (
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          ) ,   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.image_processing_class(**self.image_processor_dict   )
      # create random PyTorch tensors
      UpperCamelCase       =  prepare_image_inputs(self.image_processor_tester ,   equal_resolution=lowerCamelCase__ ,   torchify=lowerCamelCase__   )
      for image in image_inputs:
         self.assertIsInstance(lowerCamelCase__ ,   torch.Tensor   )

      # Test not batched input
      UpperCamelCase       =  image_processing(image_inputs[0] ,   return_tensors='''pt'''   ).pixel_values
      self.assertEqual(
          encoded_images.shape ,   (
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          ) ,   )

      # Test batched
      UpperCamelCase       =  image_processing(lowerCamelCase__ ,   return_tensors='''pt'''   ).pixel_values
      self.assertEqual(
          encoded_images.shape ,   (
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          ) ,   )





