
import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class   lowerCamelCase	(						A_							,				unittest.TestCase						):
     UpperCAmelCase__       :							List[Any]     =	CLIPTokenizer
     UpperCAmelCase__       :							int     =	CLIPTokenizerFast
     UpperCAmelCase__       :							int     =	True
     UpperCAmelCase__       :							int     =	{}
     UpperCAmelCase__       :							Any     =	False





     def 	UpperCAmelCase(self   :					str     )		->		int:
         super().setUp()

         # fmt: off
         snake_case				  =      ["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
         # fmt: on
         snake_case				  =      dict(zip(_A       ,					range(len(_A     )     )     )     )
         snake_case				  =      ["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
         snake_case				  =      {"unk_token": "<unk>"}

         snake_case				  =      os.path.join(self.tmpdirname       ,					VOCAB_FILES_NAMES["vocab_file"]     )
         snake_case				  =      os.path.join(self.tmpdirname       ,					VOCAB_FILES_NAMES["merges_file"]     )
         with open(self.vocab_file       ,					"w"       ,					encoding="utf-8"     ) as fp:
             fp.write(json.dumps(_A     ) + "\n"     )
         with open(self.merges_file       ,					"w"       ,					encoding="utf-8"     ) as fp:
             fp.write("\n".join(_A     )     )





     def 	UpperCAmelCase(self   :					List[Any]       ,					**_A   :					Union[str, Any]     )		->		List[Any]:
         kwargs.update(self.special_tokens_map     )
         return CLIPTokenizer.from_pretrained(self.tmpdirname       ,					**_A     )





     def 	UpperCAmelCase(self   :					Any       ,					**_A   :					Union[str, Any]     )		->		Tuple:
         kwargs.update(self.special_tokens_map     )
         return CLIPTokenizerFast.from_pretrained(self.tmpdirname       ,					**_A     )





     def 	UpperCAmelCase(self   :					str       ,					_A   :					Union[str, Any]     )		->		List[Any]:
         snake_case				  =      "lower newer"
         snake_case				  =      "lower newer"
         return input_text, output_text





     def 	UpperCAmelCase(self   :					Tuple     )		->		Optional[Any]:
         snake_case				  =      CLIPTokenizer(self.vocab_file       ,					self.merges_file       ,					**self.special_tokens_map     )
         snake_case				  =      "lower newer"
         snake_case				  =      ["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
         snake_case				  =      tokenizer.tokenize(_A     )
         self.assertListEqual(_A       ,					_A     )

         snake_case				  =      tokens + [tokenizer.unk_token]
         snake_case				  =      [1_0, 2, 1_6, 9, 3, 2, 1_6, 2_0]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(_A     )       ,					_A     )





     @require_ftfy
     def 	UpperCAmelCase(self   :					Optional[Any]     )		->		List[Any]:
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
                 snake_case				  =      self.tokenizer_class.from_pretrained(_A       ,					**_A     )
                 snake_case				  =      self.rust_tokenizer_class.from_pretrained(_A       ,					**_A     )

                 snake_case				  =      "A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
                 snake_case				  =      tokenizer_s.tokenize(_A     )
                 snake_case				  =      tokenizer_r.tokenize(_A     )

                 self.assertListEqual(_A       ,					_A     )

                 # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                 # with Tilde) encoded in 2 different ways
                 snake_case				  =      "xa\u0303y" + " " + "x\xe3y"
                 snake_case				  =      tokenizer_s.tokenize(_A     )
                 snake_case				  =      tokenizer_r.tokenize(_A     )

                 self.assertListEqual(_A       ,					_A     )

                 # Test that the tokenization is identical on unicode of space type
                 snake_case				  =      [
                     "\u0009",  # (horizontal tab, '\t')
                     "\u000B",  # (vertical tab)
                     "\u000C",  # (form feed)
                     "\u0020",  # (space, ' ')
                     "\u200E",  # (left-to-right mark):w
                     "\u200F",  # (right-to-left mark)
                 ]
                 for unicode_seq in spaces_unicodes:
                     snake_case				  =      tokenizer_s.tokenize(_A     )
                     snake_case				  =      tokenizer_r.tokenize(_A     )

                     self.assertListEqual(_A       ,					_A     )

                 # Test that the tokenization is identical on unicode of line break type
                 snake_case				  =      [
                     "\u000A",  # (line feed, '\n')
                     "\r\n",  # (carriage return and line feed, '\r\n')
                     "\u000D",  # (carriage return, '\r')
                     "\r",  # (carriage return, '\r')
                     "\u000D",  # (carriage return, '\r')
                     "\u2028",  # (line separator)
                     "\u2029",  # (paragraph separator)
                     # "\u0085", # (next line)
                 ]

                 # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                 # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                 # space (and thus into an empty list).

                 for unicode_seq in line_break_unicodes:
                     snake_case				  =      tokenizer_s.tokenize(_A     )
                     snake_case				  =      tokenizer_r.tokenize(_A     )

                     self.assertListEqual(_A       ,					_A     )





     def 	UpperCAmelCase(self   :					Dict     )		->		Dict:
         # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
                 snake_case				  =      "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
                 snake_case				  =      f'{text_of_1_token} {text_of_1_token}'

                 snake_case				  =      self.rust_tokenizer_class.from_pretrained(
                     _A       ,					use_fast=_A       ,					)
                 snake_case				  =      tokenizer_r(_A       ,					return_offsets_mapping=_A       ,					add_special_tokens=_A     )
                 self.assertEqual(encoding.offset_mapping[0]       ,					(0, len(_A     ))     )
                 self.assertEqual(
                     encoding.offset_mapping[1]       ,					(len(_A     ) + 1, len(_A     ) + 1 + len(_A     ))       ,					)

                 snake_case				  =      f' {text}'

                 snake_case				  =      self.rust_tokenizer_class.from_pretrained(
                     _A       ,					use_fast=_A       ,					)
                 snake_case				  =      tokenizer_r(_A       ,					return_offsets_mapping=_A       ,					add_special_tokens=_A     )
                 self.assertEqual(encoding.offset_mapping[0]       ,					(1, 1 + len(_A     ))     )
                 self.assertEqual(
                     encoding.offset_mapping[1]       ,					(1 + len(_A     ) + 1, 1 + len(_A     ) + 1 + len(_A     ))       ,					)





     def 	UpperCAmelCase(self   :					Optional[int]     )		->		List[str]:
         # Test related to the breaking change introduced in transformers v4.17.0
         # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
         with self.assertRaises(_A     ) as context:
             self.rust_tokenizer_class.from_pretrained("robot-test/old-clip-tokenizer"     )

         self.assertTrue(
             context.exception.args[0].startswith(
                 "The `backend_tokenizer` provided does not match the expected format."     )     )





     @require_ftfy
     def 	UpperCAmelCase(self   :					List[str]     )		->		str:
         super().test_tokenization_python_rust_equals()





     def 	UpperCAmelCase(self   :					Any     )		->		List[Any]:
         # CLIP always lower cases letters
         pass



import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment



def 		lowercase_    (     A__				,	A__=()				,	A__=None				,	A__="no"				,	A__="29500"	)							->							Optional[int]:
    """simple docstring"""

    snake_case				  =      False
    snake_case				  =      False
    if any(key.startswith("KAGGLE"	) for key in os.environ.keys()	):
        snake_case				  =      True
    elif "IPython" in sys.modules:
        snake_case				  =      "google.colab" in str(sys.modules["IPython"].get_ipython()	)

    try:
        snake_case				  =      PrecisionType(mixed_precision.lower()	)
    except ValueError:
        raise ValueError(
            F'Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}.'	)

    if (in_colab or in_kaggle) and (os.environ.get("TPU_NAME"				,	A__	) is not None):
        # TPU launch
        import torch_xla.distributed.xla_multiprocessing as xmp

        if len(AcceleratorState._shared_state	) > 0:
            raise ValueError(
                "To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside "
                "your training function. Restart your notebook and make sure no cells initializes an "
                "`Accelerator`."	)
        if num_processes is None:
            snake_case				  =      8

        snake_case				  =      PrepareForLaunch(A__				,	distributed_type="TPU"	)
        print(F'Launching a training on {num_processes} TPU cores.'	)
        xmp.spawn(A__				,	args=A__				,	nprocs=A__				,	start_method="fork"	)
    elif in_colab:
        # No need for a distributed launch otherwise as it's either CPU or one GPU.
        if torch.cuda.is_available():
            print("Launching training on one GPU."	)
        else:
            print("Launching training on one CPU."	)
        function(*A__	)
    else:
        if num_processes is None:
            raise ValueError(
                "You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call."	)

        if num_processes > 1:
            # Multi-GPU launch
            from torch.multiprocessing import start_processes
            from torch.multiprocessing.spawn import ProcessRaisedException

            if len(AcceleratorState._shared_state	) > 0:
                raise ValueError(
                    "To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized "
                    "inside your training function. Restart your notebook and make sure no cells initializes an "
                    "`Accelerator`."	)

            if torch.cuda.is_initialized():
                raise ValueError(
                    "To launch a multi-GPU training from your notebook, you need to avoid running any instruction "
                    "using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA "
                    "function."	)

            # torch.distributed will expect a few environment variable to be here. We set the ones common to each
            # process here (the other ones will be set be the launcher).
            with patch_environment(
                world_size=A__				,	master_addr="127.0.01"				,	master_port=A__				,	mixed_precision=A__	):
                snake_case				  =      PrepareForLaunch(A__				,	distributed_type="MULTI_GPU"	)
                print(F'Launching training on {num_processes} GPUs.'	)
                try:
                    start_processes(A__				,	args=A__				,	nprocs=A__				,	start_method="fork"	)
                except ProcessRaisedException as e:
                    if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
                        raise RuntimeError(
                            "CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. "
                            "This likely stems from an outside import causing issues once the `notebook_launcher()` is called. "
                            "Please review your imports and test them when running the `notebook_launcher()` to identify "
                            "which one is problematic."	) from e

        else:
            # No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
            if is_mps_available():
                snake_case				  =      "1"
                print("Launching training on MPS."	)
            elif torch.cuda.is_available():
                print("Launching training on one GPU."	)
            else:
                print("Launching training on CPU."	)
            function(*A__	)





def 		lowercase_    (     A__				,	A__=()				,	A__=2	)							->							Dict:
    """simple docstring"""

    from torch.multiprocessing import start_processes

    with tempfile.NamedTemporaryFile() as tmp_file:
        # torch.distributed will expect a few environment variable to be here. We set the ones common to each
        # process here (the other ones will be set be the launcher).
        with patch_environment(
            world_size=A__				,	master_addr="127.0.01"				,	master_port="29500"				,	accelerate_mixed_precision="no"				,	accelerate_debug_rdv_file=tmp_file.name				,	accelerate_use_cpu="yes"				,	):
            snake_case				  =      PrepareForLaunch(A__				,	debug=A__	)
            start_processes(A__				,	args=A__				,	nprocs=A__				,	start_method="fork"	)

