

import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__A      		=     get_tests_dir('''fixtures/test_sentencepiece.model''')



@require_sentencepiece
@require_tokenizers
class   lowercase_   (  __lowercase       ,							unittest.TestCase					):
      UpperCamelCase_      :		Optional[int]										=    XLNetTokenizer
      UpperCamelCase_      :		Dict										=    XLNetTokenizerFast
      UpperCamelCase_      :		Optional[Any]										=    True
      UpperCamelCase_      :		List[str]										=    True



      def 						UpperCamelCase_		(      self				:    Union[str, Any]		)   ->					str:
        super().setUp()

        # We have a SentencePiece fixture for testing
        _snake_case						=							XLNetTokenizer(A__      , keep_accents=A__		)
        tokenizer.sanitize_special_tokens()
        tokenizer.save_pretrained(self.tmpdirname		)



      def 						UpperCamelCase_		(      self				:    str		)   ->					Optional[Any]:
        _snake_case						=							'''<s>'''
        _snake_case						=							1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(A__		)      , A__		)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(A__		)      , A__		)



      def 						UpperCamelCase_		(      self				:    Dict		)   ->					Any:
        _snake_case						=							list(self.get_tokenizer().get_vocab().keys()		)

        self.assertEqual(vocab_keys[0]      , '''<unk>'''		)
        self.assertEqual(vocab_keys[1]      , '''<s>'''		)
        self.assertEqual(vocab_keys[-1]      , '''<eod>'''		)
        self.assertEqual(len(A__		)      , 1006		)



      def 						UpperCamelCase_		(      self				:    Dict		)   ->					str:
        self.assertEqual(self.get_tokenizer().vocab_size      , 1000		)



      def 						UpperCamelCase_		(      self				:    Optional[Any]		)   ->					Tuple:
        _snake_case						=							XLNetTokenizer(A__      , keep_accents=A__		)

        _snake_case						=							tokenizer.tokenize('''This is a test'''		)
        self.assertListEqual(A__      , ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']		)

        self.assertListEqual(tokenizer.convert_tokens_to_ids(A__		)      , [285, 46, 10, 170, 382]		)

        _snake_case						=							tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
        self.assertListEqual(
            A__      , [
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''9''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''s''',
                '''é''',
                '''.''',
            ]      , )
        _snake_case						=							tokenizer.convert_tokens_to_ids(A__		)
        self.assertListEqual(A__      , [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]		)

        _snake_case						=							tokenizer.convert_ids_to_tokens(A__		)
        self.assertListEqual(
            A__      , [
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''<unk>''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''s''',
                '''<unk>''',
                '''.''',
            ]      , )



      def 						UpperCamelCase_		(      self				:    int		)   ->					Union[str, Any]:
        _snake_case						=							XLNetTokenizer(A__      , do_lower_case=A__		)
        _snake_case						=							tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
        self.assertListEqual(
            A__      , [
                SPIECE_UNDERLINE + '''''',
                '''i''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''9''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''se''',
                '''.''',
            ]      , )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)      , ['''▁he''', '''ll''', '''o''']		)



      def 						UpperCamelCase_		(      self				:    List[Any]		)   ->					List[str]:
        _snake_case						=							XLNetTokenizer(A__      , do_lower_case=A__		)
        _snake_case						=							tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
        self.assertListEqual(
            A__      , [
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''9''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''se''',
                '''.''',
            ]      , )



      @slow
      def 						UpperCamelCase_		(      self				:    str		)   ->					Optional[Any]:
        _snake_case						=							XLNetTokenizer.from_pretrained('''xlnet-base-cased'''		)

        _snake_case						=							tokenizer.encode('''sequence builders'''      , add_special_tokens=A__		)
        _snake_case						=							tokenizer.encode('''multi-sequence build'''      , add_special_tokens=A__		)

        _snake_case						=							tokenizer.build_inputs_with_special_tokens(A__		)
        _snake_case						=							tokenizer.build_inputs_with_special_tokens(A__      , A__		)

        assert encoded_sentence == text + [4, 3]
        assert encoded_pair == text + [4] + text_a + [4, 3]







      @slow
      def 						UpperCamelCase_		(      self				:    int		)   ->					List[str]:
        # fmt: off
        _snake_case						=							{'''input_ids''': [[17, 21442, 270, 17, 10, 14645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 22018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 14431, 13, 5500, 11, 1176, 580, 13, 16819, 4797, 23, 17, 10, 17135, 658, 19, 457, 7932, 13, 184, 19, 3154, 17135, 6468, 19, 1404, 12269, 19, 4229, 5356, 16264, 46, 19, 17, 20545, 10395, 9, 9, 9, 11, 28, 6421, 9531, 20729, 17, 10, 353, 17022, 11, 21, 6421, 9531, 16949, 17, 10, 11509, 753, 11, 33, 95, 2421, 7385, 956, 14431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 24738, 19, 13203, 658, 218, 787, 21, 430, 18482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22178, 27, 1064, 22, 956, 13, 11101, 1429, 5854, 24313, 18953, 40, 422, 24366, 68, 1758, 37, 10483, 14257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 13894, 3380, 23, 95, 18, 17634, 2288, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=A__      , model_name='''xlnet-base-cased'''      , revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''      , )




import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_video_inputs


if is_torch_available():
 import torch

if is_vision_available():
 from PIL import Image

 from transformers import VivitImageProcessor



class   lowercase_   (  unittest.TestCase					):



      def __init__(      self				:    Union[str, Any]      , A__				:    int      , A__				:    List[str]=7      , A__				:    Tuple=3      , A__				:    List[str]=10      , A__				:    Optional[int]=18      , A__				:    int=30      , A__				:    Tuple=400      , A__				:    Dict=True      , A__				:    str=None      , A__				:    str=True      , A__				:    List[str]=[0.5, 0.5, 0.5]      , A__				:    int=[0.5, 0.5, 0.5]      , A__				:    List[Any]=None      , )   ->					int:
        _snake_case						=							size if size is not None else {'''shortest_edge''': 18}
        _snake_case						=							crop_size if crop_size is not None else {'''height''': 18, '''width''': 18}

        _snake_case						=							parent
        _snake_case						=							batch_size
        _snake_case						=							num_channels
        _snake_case						=							num_frames
        _snake_case						=							image_size
        _snake_case						=							min_resolution
        _snake_case						=							max_resolution
        _snake_case						=							do_resize
        _snake_case						=							size
        _snake_case						=							do_normalize
        _snake_case						=							image_mean
        _snake_case						=							image_std
        _snake_case						=							crop_size







      def 						UpperCamelCase_		(      self				:    List[str]		)   ->					Union[str, Any]:
        return {
            "image_mean": self.image_mean,
            "image_std": self.image_std,
            "do_normalize": self.do_normalize,
            "do_resize": self.do_resize,
            "size": self.size,
            "crop_size": self.crop_size,
        }



@require_torch
@require_vision
class   lowercase_   (  __lowercase       ,							unittest.TestCase					):
      UpperCamelCase_      :		Tuple										=    VivitImageProcessor if is_vision_available() else None



      def 						UpperCamelCase_		(      self				:    Union[str, Any]		)   ->					List[str]:
        _snake_case						=							VivitImageProcessingTester(self		)



      @property
      def 						UpperCamelCase_		(      self				:    Union[str, Any]		)   ->					Optional[Any]:
        return self.image_processor_tester.prepare_image_processor_dict()



      def 						UpperCamelCase_		(      self				:    Optional[int]		)   ->					Optional[Any]:
        _snake_case						=							self.image_processing_class(**self.image_processor_dict		)
        self.assertTrue(hasattr(A__      , '''image_mean'''		)		)
        self.assertTrue(hasattr(A__      , '''image_std'''		)		)
        self.assertTrue(hasattr(A__      , '''do_normalize'''		)		)
        self.assertTrue(hasattr(A__      , '''do_resize'''		)		)
        self.assertTrue(hasattr(A__      , '''do_center_crop'''		)		)
        self.assertTrue(hasattr(A__      , '''size'''		)		)



      def 						UpperCamelCase_		(      self				:    int		)   ->					List[Any]:
        _snake_case						=							self.image_processing_class.from_dict(self.image_processor_dict		)
        self.assertEqual(image_processor.size      , {'''shortest_edge''': 18}		)
        self.assertEqual(image_processor.crop_size      , {'''height''': 18, '''width''': 18}		)

        _snake_case						=							self.image_processing_class.from_dict(self.image_processor_dict      , size=42      , crop_size=84		)
        self.assertEqual(image_processor.size      , {'''shortest_edge''': 42}		)
        self.assertEqual(image_processor.crop_size      , {'''height''': 84, '''width''': 84}		)



      def 						UpperCamelCase_		(      self				:    List[Any]		)   ->					Optional[Any]:
        # Initialize image_processing
        _snake_case						=							self.image_processing_class(**self.image_processor_dict		)
        # create random PIL videos
        _snake_case						=							prepare_video_inputs(self.image_processor_tester      , equal_resolution=A__		)
        for video in video_inputs:
          self.assertIsInstance(A__      , A__		)
          self.assertIsInstance(video[0]      , Image.Image		)

        # Test not batched input
        _snake_case						=							image_processing(video_inputs[0]      , return_tensors='''pt'''		).pixel_values
        self.assertEqual(
            encoded_videos.shape      , (
                1,
                self.image_processor_tester.num_frames,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size['''height'''],
                self.image_processor_tester.crop_size['''width'''],
            )      , )

        # Test batched
        _snake_case						=							image_processing(A__      , return_tensors='''pt'''		).pixel_values
        self.assertEqual(
            encoded_videos.shape      , (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_frames,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size['''height'''],
                self.image_processor_tester.crop_size['''width'''],
            )      , )



      def 						UpperCamelCase_		(      self				:    Any		)   ->					List[str]:
        # Initialize image_processing
        _snake_case						=							self.image_processing_class(**self.image_processor_dict		)
        # create random numpy tensors
        _snake_case						=							prepare_video_inputs(self.image_processor_tester      , equal_resolution=A__      , numpify=A__		)
        for video in video_inputs:
          self.assertIsInstance(A__      , A__		)
          self.assertIsInstance(video[0]      , np.ndarray		)

        # Test not batched input
        _snake_case						=							image_processing(video_inputs[0]      , return_tensors='''pt'''		).pixel_values
        self.assertEqual(
            encoded_videos.shape      , (
                1,
                self.image_processor_tester.num_frames,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size['''height'''],
                self.image_processor_tester.crop_size['''width'''],
            )      , )

        # Test batched
        _snake_case						=							image_processing(A__      , return_tensors='''pt'''		).pixel_values
        self.assertEqual(
            encoded_videos.shape      , (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_frames,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size['''height'''],
                self.image_processor_tester.crop_size['''width'''],
            )      , )







      def 						UpperCamelCase_		(      self				:    Optional[Any]		)   ->					int:
        # Initialize image_processing
        _snake_case						=							self.image_processing_class(**self.image_processor_dict		)
        # create random PyTorch tensors
        _snake_case						=							prepare_video_inputs(self.image_processor_tester      , equal_resolution=A__      , torchify=A__		)
        for video in video_inputs:
          self.assertIsInstance(A__      , A__		)
          self.assertIsInstance(video[0]      , torch.Tensor		)

        # Test not batched input
        _snake_case						=							image_processing(video_inputs[0]      , return_tensors='''pt'''		).pixel_values
        self.assertEqual(
            encoded_videos.shape      , (
                1,
                self.image_processor_tester.num_frames,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size['''height'''],
                self.image_processor_tester.crop_size['''width'''],
            )      , )

        # Test batched
        _snake_case						=							image_processing(A__      , return_tensors='''pt'''		).pixel_values
        self.assertEqual(
            encoded_videos.shape      , (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_frames,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size['''height'''],
                self.image_processor_tester.crop_size['''width'''],
            )      , )

