


"""simple docstring"""



import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class     __magic_name__						(	__UpperCAmelCase	,  unittest.TestCase   ):
       __A	:					Optional[int]					     =      GPTNeoXJapaneseTokenizer
       __A	:					int					     =      False
       __A	:					Any					     =      {"do_clean_text": False, "add_prefix_space": False}






       def        __snake_case  (      self     :			Union[str, Any] ):

              '''simple docstring'''






              super().setUp()

              lowercase						:Optional[int]											= [
                  'こん',
                  'こんに',
                  'にちは',
                  'ばんは',
                  '世界,㔺界',
                  '、',
                  '。',
                  '<BR>',
                  '<SP>',
                  '<TAB>',
                  '<URL>',
                  '<EMAIL>',
                  '<TEL>',
                  '<DATE>',
                  '<PRICE>',
                  '<BLOCK>',
                  '<KIGOU>',
                  '<U2000U2BFF>',
                  '<|emoji1|>',
                  '<unk>',
                  '<|startoftext|>',
                  '<|endoftext|>',
              ]
              lowercase						:Any											= {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
              lowercase						:Any											= {'unk_token': '<unk>'}

              lowercase						:str											= os.path.join(self.tmpdirname     ,   VOCAB_FILES_NAMES['''vocab_file'''] )
              lowercase						:Dict											= os.path.join(self.tmpdirname     ,   VOCAB_FILES_NAMES['''emoji_file'''] )
              with open(self.vocab_file     ,   '''w'''     ,   encoding='''utf-8''' ) as vocab_writer:
                     vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )
              with open(self.emoji_file     ,   '''w''' ) as emoji_writer:
                     emoji_writer.write(json.dumps(__lowercase ) )






       def        __snake_case  (      self     :			Optional[int]     ,   **snake_case__     :			Union[str, Any] ):

              '''simple docstring'''






              kwargs.update(self.special_tokens_map )
              return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname     ,   **__lowercase )






       def        __snake_case  (      self     :			Tuple     ,   snake_case__     :			List[str] ):

              '''simple docstring'''






              lowercase						:Optional[int]											= 'こんにちは、世界。 \nこんばんは、㔺界。😀'
              lowercase						:Union[str, Any]											= 'こんにちは、世界。 \nこんばんは、世界。😀'
              return input_text, output_text






       def        __snake_case  (      self     :			Optional[int]     ,   snake_case__     :			List[str] ):

              '''simple docstring'''






              lowercase						:Union[str, Any]											= self.get_input_output_texts(__lowercase )
              lowercase						:Optional[Any]											= tokenizer.encode(__lowercase     ,   add_special_tokens=__lowercase )
              lowercase						:Union[str, Any]											= tokenizer.decode(__lowercase     ,   clean_up_tokenization_spaces=__lowercase )
              return text, ids






       def        __snake_case  (      self     :			Any ):

              '''simple docstring'''






              pass  # TODO add if relevant






       def        __snake_case  (      self     :			Tuple ):

              '''simple docstring'''






              pass  # TODO add if relevant






       def        __snake_case  (      self     :			List[Any] ):

              '''simple docstring'''






              pass  # TODO add if relevant






       def        __snake_case  (      self     :			str ):

              '''simple docstring'''






              lowercase						:Union[str, Any]											= self.get_tokenizer()

              # Testing tokenization
              lowercase						:Optional[Any]											= 'こんにちは、世界。　こんばんは、㔺界。'
              lowercase						:str											= ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
              lowercase						:Tuple											= tokenizer.tokenize(__lowercase )
              self.assertListEqual(__lowercase     ,   __lowercase )

              # Testing conversion to ids without special tokens
              lowercase						:Tuple											= [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
              lowercase						:List[Any]											= tokenizer.convert_tokens_to_ids(__lowercase )
              self.assertListEqual(__lowercase     ,   __lowercase )

              # Testing conversion to ids with special tokens
              lowercase						:List[str]											= tokens + [tokenizer.unk_token]
              lowercase						:List[Any]											= [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
              lowercase						:int											= tokenizer.convert_tokens_to_ids(__lowercase )
              self.assertListEqual(__lowercase     ,   __lowercase )






       @slow
       def        __snake_case  (      self     :			Union[str, Any] ):

              '''simple docstring'''






              lowercase						:Tuple											= self.tokenizer_class.from_pretrained('''abeja/gpt-neox-japanese-2.7b''' )

              lowercase						:List[Any]											= tokenizer.encode('''ありがとう。'''     ,   add_special_tokens=__lowercase )
              lowercase						:Tuple											= tokenizer.encode('''どういたしまして。'''     ,   add_special_tokens=__lowercase )

              lowercase						:List[Any]											= tokenizer.build_inputs_with_special_tokens(__lowercase )
              lowercase						:Union[str, Any]											= tokenizer.build_inputs_with_special_tokens(__lowercase     ,   __lowercase )

              assert encoded_sentence == ids_a
              assert encoded_pair == ids_a + ids_a






       def        __snake_case  (      self     :			Dict ):

              '''simple docstring'''






              pass







       def        __snake_case  (      self     :			Any ):

              '''simple docstring'''






              pass







import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
							import torch

if is_vision_available():
							from PIL import Image

							from transformers import TvltImageProcessor






def 					snake_case					(							A						,   A=10						,   A=10						,   A=False						,   A=False			)							->							str:




	'''simple docstring'''


	lowerCamelCase_							:		Any							   =							[]
	for i in range(image_processor_tester.num_frames			):
		video.append(np.random.randint(255						,   size=(image_processor_tester.num_channels, width, height)						,   dtype=np.uinta			)			)

	if not numpify and not torchify:
		# PIL expects the channel dimension as last dimension
		lowerCamelCase_							:		Any							   =							[Image.fromarray(np.moveaxis(A						,   0						,   -1			)			) for frame in video]

	if torchify:
		lowerCamelCase_							:		List[str]							   =							[torch.from_numpy(A			) for frame in video]

	return video






def 					snake_case					(							A						,   A=False						,   A=False						,   A=False			)							->							Dict:




	'''simple docstring'''


	assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

	lowerCamelCase_							:		int							   =							[]
	for i in range(image_processor_tester.batch_size			):
		if equal_resolution:
			lowerCamelCase_							:		Union[str, Any]							   =							image_processor_tester.max_resolution
		else:
			lowerCamelCase_   ,			lowerCamelCase_							:		Optional[int]							   =							np.random.choice(
			    np.arange(image_processor_tester.min_resolution						,   image_processor_tester.max_resolution			)						,   2			)
			lowerCamelCase_							:		str							   =							prepare_video(
			    image_processor_tester=A						,   width=A						,   height=A						,   numpify=A						,   torchify=A						,   )
		video_inputs.append(A			)

	return video_inputs


class 				__SCREAMING_SNAKE_CASE							(		unittest.TestCase  ):

					def __init__(				self ,       __lowercase ,       __lowercase=7 ,       __lowercase=3 ,       __lowercase=4 ,       __lowercase=18 ,       __lowercase=30 ,       __lowercase=400 ,       __lowercase=True ,       __lowercase=None ,       __lowercase=True ,       __lowercase=[0.5, 0.5, 0.5] ,       __lowercase=[0.5, 0.5, 0.5] ,       __lowercase=True ,       __lowercase=None ,       ):







						'''simple docstring'''




						lowerCamelCase_							:		Optional[int]							   =							size if size is not None else {'shortest_edge': 18}
						lowerCamelCase_							:		Union[str, Any]							   =							crop_size if crop_size is not None else {'height': 18, 'width': 18}

						lowerCamelCase_							:		List[Any]							   =							parent
						lowerCamelCase_							:		Optional[Any]							   =							batch_size
						lowerCamelCase_							:		Tuple							   =							num_channels
						lowerCamelCase_							:		Dict							   =							num_frames
						lowerCamelCase_							:		Dict							   =							image_size
						lowerCamelCase_							:		Optional[Any]							   =							min_resolution
						lowerCamelCase_							:		List[Any]							   =							max_resolution
						lowerCamelCase_							:		Dict							   =							do_resize
						lowerCamelCase_							:		List[Any]							   =							size
						lowerCamelCase_							:		Dict							   =							do_normalize
						lowerCamelCase_							:		Optional[Any]							   =							image_mean
						lowerCamelCase_							:		List[str]							   =							image_std
						lowerCamelCase_							:		Tuple							   =							do_center_crop
						lowerCamelCase_							:		List[Any]							   =							crop_size





					def 	lowercase_							(				self					):







						'''simple docstring'''




						return {
						    "image_mean": self.image_mean,
						    "image_std": self.image_std,
						    "do_normalize": self.do_normalize,
						    "do_resize": self.do_resize,
						    "size": self.size,
						    "do_center_crop": self.do_center_crop,
						    "crop_size": self.crop_size,
						}


@require_torch
@require_vision
class 				__SCREAMING_SNAKE_CASE							(		lowerCamelCase   ,     unittest.TestCase  ):
					_lowerCamelCase		  = TvltImageProcessor if is_vision_available() else None

					def 	lowercase_							(				self					):







						'''simple docstring'''




						lowerCamelCase_							:		Optional[Any]							   =							TvltImageProcessorTester(self					)

					@property
					def 	lowercase_							(				self					):







						'''simple docstring'''




						return self.image_processor_tester.prepare_image_processor_dict()

					def 	lowercase_							(				self					):







						'''simple docstring'''




						lowerCamelCase_							:		int							   =							self.image_processing_class(**self.image_processor_dict					)
						self.assertTrue(hasattr(__lowercase ,       'image_mean'					)					)
						self.assertTrue(hasattr(__lowercase ,       'image_std'					)					)
						self.assertTrue(hasattr(__lowercase ,       'do_normalize'					)					)
						self.assertTrue(hasattr(__lowercase ,       'do_resize'					)					)
						self.assertTrue(hasattr(__lowercase ,       'do_center_crop'					)					)
						self.assertTrue(hasattr(__lowercase ,       'size'					)					)

					def 	lowercase_							(				self					):







						'''simple docstring'''




						lowerCamelCase_							:		Optional[Any]							   =							self.image_processing_class(**self.image_processor_dict					)
						# create random PIL videos
						lowerCamelCase_							:		Dict							   =							prepare_video_inputs(self.image_processor_tester ,       equal_resolution=__lowercase					)
						for video in video_inputs:
							self.assertIsInstance(__lowercase ,       __lowercase					)
							self.assertIsInstance(video[0] ,       Image.Image					)

						# Test not batched input
						lowerCamelCase_							:		Any							   =							image_processor(video_inputs[0] ,       return_tensors='pt'					).pixel_values
						self.assertEqual(
						    encoded_videos.shape ,       (
						        1,
						        self.image_processor_tester.num_frames,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size['height'],
						        self.image_processor_tester.crop_size['width'],
						    ) ,       )

						# Test batched
						lowerCamelCase_							:		List[Any]							   =							image_processor(__lowercase ,       return_tensors='pt'					).pixel_values
						self.assertEqual(
						    encoded_videos.shape ,       (
						        self.image_processor_tester.batch_size,
						        self.image_processor_tester.num_frames,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size['height'],
						        self.image_processor_tester.crop_size['width'],
						    ) ,       )

					def 	lowercase_							(				self					):







						'''simple docstring'''




						lowerCamelCase_							:		Dict							   =							self.image_processing_class(**self.image_processor_dict					)
						# create random numpy tensors
						lowerCamelCase_							:		Union[str, Any]							   =							prepare_video_inputs(self.image_processor_tester ,       equal_resolution=__lowercase ,       numpify=__lowercase					)
						for video in video_inputs:
							self.assertIsInstance(__lowercase ,       __lowercase					)
							self.assertIsInstance(video[0] ,       np.ndarray					)

						# Test not batched input
						lowerCamelCase_							:		Union[str, Any]							   =							image_processor(video_inputs[0] ,       return_tensors='pt'					).pixel_values
						self.assertEqual(
						    encoded_videos.shape ,       (
						        1,
						        self.image_processor_tester.num_frames,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size['height'],
						        self.image_processor_tester.crop_size['width'],
						    ) ,       )

						# Test batched
						lowerCamelCase_							:		Tuple							   =							image_processor(__lowercase ,       return_tensors='pt'					).pixel_values
						self.assertEqual(
						    encoded_videos.shape ,       (
						        self.image_processor_tester.batch_size,
						        self.image_processor_tester.num_frames,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size['height'],
						        self.image_processor_tester.crop_size['width'],
						    ) ,       )





					def 	lowercase_							(				self					):







						'''simple docstring'''




						lowerCamelCase_							:		str							   =							self.image_processing_class(**self.image_processor_dict					)
						# create random PyTorch tensors
						lowerCamelCase_							:		Union[str, Any]							   =							prepare_video_inputs(self.image_processor_tester ,       equal_resolution=__lowercase ,       torchify=__lowercase					)
						for video in video_inputs:
							self.assertIsInstance(__lowercase ,       __lowercase					)
							self.assertIsInstance(video[0] ,       torch.Tensor					)

						# Test not batched input
						lowerCamelCase_							:		Optional[Any]							   =							image_processor(video_inputs[0] ,       return_tensors='pt'					).pixel_values
						self.assertEqual(
						    encoded_videos.shape ,       (
						        1,
						        self.image_processor_tester.num_frames,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size['height'],
						        self.image_processor_tester.crop_size['width'],
						    ) ,       )

						# Test batched
						lowerCamelCase_							:		Tuple							   =							image_processor(__lowercase ,       return_tensors='pt'					).pixel_values
						self.assertEqual(
						    encoded_videos.shape ,       (
						        self.image_processor_tester.batch_size,
						        self.image_processor_tester.num_frames,
						        self.image_processor_tester.num_channels,
						        self.image_processor_tester.crop_size['height'],
						        self.image_processor_tester.crop_size['width'],
						    ) ,       )
