




"""simple docstring"""

import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class     lowercase_	(					__lowerCAmelCase      ,     unittest.TestCase							):



 '''simple docstring'''
 UpperCAmelCase   :					List[str]				  =     CLIPTokenizer
 UpperCAmelCase   :					Union[str, Any]				  =     CLIPTokenizerFast
 UpperCAmelCase   :					List[str]				  =     True
 UpperCAmelCase   :					Dict				  =     {}
 UpperCAmelCase   :					Union[str, Any]				  =     False

 def       lowerCAmelCase_							(			self	:	Dict		):
       super().setUp()

       # fmt: off
       _A        =       ['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
       # fmt: on
       _A        =       dict(zip(_UpperCAmelCase   ,							range(len(_UpperCAmelCase		)		)		)		)
       _A        =       ['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
       _A        =       {'unk_token': '<unk>'}

       _A        =       os.path.join(self.tmpdirname   ,							VOCAB_FILES_NAMES['vocab_file']		)
       _A        =       os.path.join(self.tmpdirname   ,							VOCAB_FILES_NAMES['merges_file']		)
       with open(self.vocab_file   ,							'w'   ,							encoding='utf-8'		) as fp:
             fp.write(json.dumps(_UpperCAmelCase		) + '\n'		)
       with open(self.merges_file   ,							'w'   ,							encoding='utf-8'		) as fp:
             fp.write('\n'.join(_UpperCAmelCase		)		)

 def       lowerCAmelCase_							(			self	:	Dict   ,							**_UpperCAmelCase	:	Union[str, Any]		):
       kwargs.update(self.special_tokens_map		)
       return CLIPTokenizer.from_pretrained(self.tmpdirname   ,							**_UpperCAmelCase		)

 def       lowerCAmelCase_							(			self	:	Optional[Any]   ,							**_UpperCAmelCase	:	List[str]		):
       kwargs.update(self.special_tokens_map		)
       return CLIPTokenizerFast.from_pretrained(self.tmpdirname   ,							**_UpperCAmelCase		)

 def       lowerCAmelCase_							(			self	:	Tuple   ,							_UpperCAmelCase	:	List[str]		):
       _A        =       'lower newer'
       _A        =       'lower newer'
       return input_text, output_text

 def       lowerCAmelCase_							(			self	:	List[str]		):
       _A        =       CLIPTokenizer(self.vocab_file   ,							self.merges_file   ,							**self.special_tokens_map		)
       _A        =       'lower newer'
       _A        =       ['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
       _A        =       tokenizer.tokenize(_UpperCAmelCase		)
       self.assertListEqual(_UpperCAmelCase   ,							_UpperCAmelCase		)

       _A        =       tokens + [tokenizer.unk_token]
       _A        =       [10, 2, 16, 9, 3, 2, 16, 20]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase		)   ,							_UpperCAmelCase		)

 @require_ftfy
 def       lowerCAmelCase_							(			self	:	List[Any]		):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
                   _A        =       self.tokenizer_class.from_pretrained(_UpperCAmelCase   ,							**_UpperCAmelCase		)
                   _A        =       self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase   ,							**_UpperCAmelCase		)

                   _A        =       'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
                   _A        =       tokenizer_s.tokenize(_UpperCAmelCase		)
                   _A        =       tokenizer_r.tokenize(_UpperCAmelCase		)

                   self.assertListEqual(_UpperCAmelCase   ,							_UpperCAmelCase		)

                   # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                   # with Tilde) encoded in 2 different ways
                   _A        =       'xa\u0303y' + ' ' + 'x\xe3y'
                   _A        =       tokenizer_s.tokenize(_UpperCAmelCase		)
                   _A        =       tokenizer_r.tokenize(_UpperCAmelCase		)

                   self.assertListEqual(_UpperCAmelCase   ,							_UpperCAmelCase		)

                   # Test that the tokenization is identical on unicode of space type
                   _A        =       [
                       '\u0009',  # (horizontal tab, '\t')
                       '\u000B',  # (vertical tab)
                       '\u000C',  # (form feed)
                       '\u0020',  # (space, ' ')
                       '\u200E',  # (left-to-right mark):w
                       '\u200F',  # (right-to-left mark)
                   ]
                   for unicode_seq in spaces_unicodes:
                         _A        =       tokenizer_s.tokenize(_UpperCAmelCase		)
                         _A        =       tokenizer_r.tokenize(_UpperCAmelCase		)

                         self.assertListEqual(_UpperCAmelCase   ,							_UpperCAmelCase		)

                   # Test that the tokenization is identical on unicode of line break type
                   _A        =       [
                       '\u000A',  # (line feed, '\n')
                       '\r\n',  # (carriage return and line feed, '\r\n')
                       '\u000D',  # (carriage return, '\r')
                       '\r',  # (carriage return, '\r')
                       '\u000D',  # (carriage return, '\r')
                       '\u2028',  # (line separator)
                       '\u2029',  # (paragraph separator)
                       # "\u0085", # (next line)
                   ]

                   # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                   # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                   # space (and thus into an empty list).

                   for unicode_seq in line_break_unicodes:
                         _A        =       tokenizer_s.tokenize(_UpperCAmelCase		)
                         _A        =       tokenizer_r.tokenize(_UpperCAmelCase		)

                         self.assertListEqual(_UpperCAmelCase   ,							_UpperCAmelCase		)

 def       lowerCAmelCase_							(			self	:	str		):
       # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
                   _A        =       'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
                   _A        =       F'''{text_of_1_token} {text_of_1_token}'''

                   _A        =       self.rust_tokenizer_class.from_pretrained(
                       _UpperCAmelCase   ,							use_fast=_UpperCAmelCase   ,							)
                   _A        =       tokenizer_r(_UpperCAmelCase   ,							return_offsets_mapping=_UpperCAmelCase   ,							add_special_tokens=_UpperCAmelCase		)
                   self.assertEqual(encoding.offset_mapping[0]   ,							(0, len(_UpperCAmelCase		))		)
                   self.assertEqual(
                       encoding.offset_mapping[1]   ,							(len(_UpperCAmelCase		) + 1, len(_UpperCAmelCase		) + 1 + len(_UpperCAmelCase		))   ,							)

                   _A        =       F''' {text}'''

                   _A        =       self.rust_tokenizer_class.from_pretrained(
                       _UpperCAmelCase   ,							use_fast=_UpperCAmelCase   ,							)
                   _A        =       tokenizer_r(_UpperCAmelCase   ,							return_offsets_mapping=_UpperCAmelCase   ,							add_special_tokens=_UpperCAmelCase		)
                   self.assertEqual(encoding.offset_mapping[0]   ,							(1, 1 + len(_UpperCAmelCase		))		)
                   self.assertEqual(
                       encoding.offset_mapping[1]   ,							(1 + len(_UpperCAmelCase		) + 1, 1 + len(_UpperCAmelCase		) + 1 + len(_UpperCAmelCase		))   ,							)

 def       lowerCAmelCase_							(			self	:	str		):
       # Test related to the breaking change introduced in transformers v4.17.0
       # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
       with self.assertRaises(_UpperCAmelCase		) as context:
             self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'		)

       self.assertTrue(
           context.exception.args[0].startswith(
               'The `backend_tokenizer` provided does not match the expected format.'		)		)

 @require_ftfy
 def       lowerCAmelCase_							(			self	:	str		):
       super().test_tokenization_python_rust_equals()



 def       lowerCAmelCase_							(			self	:	Tuple		):
       # CLIP always lower cases letters
       pass







"""simple docstring"""

import inspect
import os
import unittest
from pathlib import Path

import torch

import accelerate
from accelerate.test_utils import execute_subprocess_async
from accelerate.test_utils.testing import run_command





class     lowercase_	(					unittest.TestCase							):



 '''simple docstring'''
 UpperCAmelCase   :					str				  =     inspect.getfile(accelerate.test_utils							)
 UpperCAmelCase   :					Optional[Any]				  =     os.path.sep.join(mod_file.split(os.path.sep							)[:-1] + ['''scripts''', '''test_cli.py''']							)

 UpperCAmelCase   :					List[Any]				  =     ['''accelerate''', '''launch''']
 UpperCAmelCase   :					Dict				  =     Path.home() / '''.cache/huggingface/accelerate'''
 UpperCAmelCase   :					Union[str, Any]				  =     '''default_config.yaml'''
 UpperCAmelCase   :					Union[str, Any]				  =     config_folder / config_file
 UpperCAmelCase   :					Union[str, Any]				  =     config_folder / '''_default_config.yaml'''

 UpperCAmelCase   :					List[Any]				  =     Path('''tests/test_configs'''							)

 @classmethod
 def       lowerCAmelCase_							(			cls	:	List[Any]		):
       if cls.config_path.is_file():
             cls.config_path.rename(cls.changed_path		)

 @classmethod
 def       lowerCAmelCase_							(			cls	:	Tuple		):
       if cls.changed_path.is_file():
             cls.changed_path.rename(cls.config_path		)

 def       lowerCAmelCase_							(			self	:	List[Any]		):
       _A        =       self.base_cmd
       if torch.cuda.is_available() and (torch.cuda.device_count() > 1):
             cmd += ["--multi_gpu"]
       execute_subprocess_async(cmd + [self.test_file_path]   ,							env=os.environ.copy()		)

 def       lowerCAmelCase_							(			self	:	Optional[int]		):
       for config in sorted(self.test_config_path.glob('**/*.yaml'		)		):
             with self.subTest(config_file=_UpperCAmelCase		):
                   execute_subprocess_async(
                       self.base_cmd + ['--config_file', str(_UpperCAmelCase		), self.test_file_path]   ,							env=os.environ.copy()		)



 def       lowerCAmelCase_							(			self	:	Any		):
       execute_subprocess_async(['accelerate', 'test']   ,							env=os.environ.copy()		)





class     lowercase_	(					unittest.TestCase							):



 '''simple docstring'''
 UpperCAmelCase   :					Dict				  =     '''test-tpu'''
 UpperCAmelCase   :					Optional[int]				  =     '''us-central1-a'''
 UpperCAmelCase   :					List[str]				  =     '''ls'''
 UpperCAmelCase   :					str				  =     ['''accelerate''', '''tpu-config''']
 UpperCAmelCase   :					Optional[Any]				  =     '''cd /usr/share'''
 UpperCAmelCase   :					Optional[Any]				  =     '''tests/test_samples/test_command_file.sh'''
 UpperCAmelCase   :					str				  =     '''Running gcloud compute tpus tpu-vm ssh'''

 def       lowerCAmelCase_							(			self	:	Any		):
       _A        =       run_command(
           self.cmd
           + ['--command', self.command, '--tpu_zone', self.tpu_zone, '--tpu_name', self.tpu_name, '--debug']   ,							return_stdout=_UpperCAmelCase   ,							)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all'''   ,							_UpperCAmelCase   ,							)

 def       lowerCAmelCase_							(			self	:	Dict		):
       _A        =       run_command(
           self.cmd
           + [
               '--config_file',
               'tests/test_configs/0_12_0.yaml',
               '--command',
               self.command,
               '--tpu_zone',
               self.tpu_zone,
               '--tpu_name',
               self.tpu_name,
               '--debug',
           ]   ,							return_stdout=_UpperCAmelCase   ,							)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all'''   ,							_UpperCAmelCase   ,							)

 def       lowerCAmelCase_							(			self	:	Optional[int]		):
       _A        =       run_command(
           self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--debug']   ,							return_stdout=_UpperCAmelCase		)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all'''   ,							_UpperCAmelCase   ,							)

 def       lowerCAmelCase_							(			self	:	str		):
       _A        =       run_command(
           self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--command', self.command, '--debug']   ,							return_stdout=_UpperCAmelCase   ,							)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all'''   ,							_UpperCAmelCase   ,							)

 def       lowerCAmelCase_							(			self	:	List[str]		):
       _A        =       run_command(
           self.cmd
           + [
               '--config_file',
               'tests/test_configs/latest.yaml',
               '--command',
               self.command,
               '--command',
               'echo "Hello World"',
               '--debug',
           ]   ,							return_stdout=_UpperCAmelCase   ,							)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls; echo "Hello World" --worker all'''   ,							_UpperCAmelCase   ,							)

 def       lowerCAmelCase_							(			self	:	str		):
       _A        =       run_command(
           self.cmd
           + ['--config_file', 'tests/test_configs/latest.yaml', '--command_file', self.command_file, '--debug']   ,							return_stdout=_UpperCAmelCase   ,							)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all'''   ,							_UpperCAmelCase   ,							)

 def       lowerCAmelCase_							(			self	:	List[Any]		):
       _A        =       run_command(
           self.cmd
           + [
               '--config_file',
               'tests/test_configs/0_12_0.yaml',
               '--command_file',
               self.command_file,
               '--tpu_zone',
               self.tpu_zone,
               '--tpu_name',
               self.tpu_name,
               '--debug',
           ]   ,							return_stdout=_UpperCAmelCase   ,							)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo "hello world"; echo "this is a second command" --worker all'''   ,							_UpperCAmelCase   ,							)

 def       lowerCAmelCase_							(			self	:	int		):
       _A        =       run_command(
           self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--install_accelerate', '--debug']   ,							return_stdout=_UpperCAmelCase   ,							)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate -U; echo "hello world"; echo "this is a second command" --worker all'''   ,							_UpperCAmelCase   ,							)



 def       lowerCAmelCase_							(			self	:	Optional[int]		):
       _A        =       run_command(
           self.cmd
           + [
               '--config_file',
               'tests/test_configs/latest.yaml',
               '--install_accelerate',
               '--accelerate_version',
               '12.0.0',
               '--debug',
           ]   ,							return_stdout=_UpperCAmelCase   ,							)
       self.assertIn(
           F'''{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate==12.0.0; echo "hello world"; echo "this is a second command" --worker all'''   ,							_UpperCAmelCase   ,							)

