






import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 				UpperCAmelCase_  (		__lowerCamelCase							,      unittest.TestCase		):
     __lowerCamelCase      				=  CLIPTokenizer
     __lowerCamelCase      				=  CLIPTokenizerFast
     __lowerCamelCase      				=  True
     __lowerCamelCase      				=  {}
     __lowerCamelCase      				=  False




     def   __UpperCAmelCase  (				self		):
          super().setUp()

          # fmt: off
          UpperCAmelCase__      :  Dict				      =			["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
          # fmt: on
          UpperCAmelCase__      :  Union[str, Any]				      =			dict(zip(_lowerCAmelCase	,						range(len(_lowerCAmelCase		)		)		)		)
          UpperCAmelCase__      :  Optional[int]				      =			["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>"""]
          UpperCAmelCase__      :  Any				      =			{"""unk_token""": """<unk>"""}

          UpperCAmelCase__      :  int				      =			os.path.join(self.tmpdirname	,						VOCAB_FILES_NAMES["""vocab_file"""]		)
          UpperCAmelCase__      :  str				      =			os.path.join(self.tmpdirname	,						VOCAB_FILES_NAMES["""merges_file"""]		)
          with open(self.vocab_file	,						"""w"""	,						encoding="""utf-8"""		) as fp:
               fp.write(json.dumps(_lowerCAmelCase		) + """\n"""		)
          with open(self.merges_file	,						"""w"""	,						encoding="""utf-8"""		) as fp:
               fp.write("""\n""".join(_lowerCAmelCase		)		)




     def   __UpperCAmelCase  (				self	,						**_lowerCAmelCase		):
          kwargs.update(self.special_tokens_map		)
          return CLIPTokenizer.from_pretrained(self.tmpdirname	,						**_lowerCAmelCase		)




     def   __UpperCAmelCase  (				self	,						**_lowerCAmelCase		):
          kwargs.update(self.special_tokens_map		)
          return CLIPTokenizerFast.from_pretrained(self.tmpdirname	,						**_lowerCAmelCase		)




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase		):
          UpperCAmelCase__      :  Union[str, Any]				      =			"""lower newer"""
          UpperCAmelCase__      :  str				      =			"""lower newer"""
          return input_text, output_text




     def   __UpperCAmelCase  (				self		):
          UpperCAmelCase__      :  Optional[int]				      =			CLIPTokenizer(self.vocab_file	,						self.merges_file	,						**self.special_tokens_map		)
          UpperCAmelCase__      :  int				      =			"""lower newer"""
          UpperCAmelCase__      :  List[str]				      =			["""lo""", """w""", """er</w>""", """n""", """e""", """w""", """er</w>"""]
          UpperCAmelCase__      :  str				      =			tokenizer.tokenize(_lowerCAmelCase		)
          self.assertListEqual(_lowerCAmelCase	,						_lowerCAmelCase		)

          UpperCAmelCase__      :  List[str]				      =			tokens + [tokenizer.unk_token]
          UpperCAmelCase__      :  Dict				      =			[10, 2, 16, 9, 3, 2, 16, 20]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase		)	,						_lowerCAmelCase		)




     @require_ftfy
     def   __UpperCAmelCase  (				self		):
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                    UpperCAmelCase__      :  Tuple				      =			self.tokenizer_class.from_pretrained(_lowerCAmelCase	,						**_lowerCAmelCase		)
                    UpperCAmelCase__      :  List[Any]				      =			self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase	,						**_lowerCAmelCase		)

                    UpperCAmelCase__      :  Union[str, Any]				      =			"""A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."""
                    UpperCAmelCase__      :  Dict				      =			tokenizer_s.tokenize(_lowerCAmelCase		)
                    UpperCAmelCase__      :  List[str]				      =			tokenizer_r.tokenize(_lowerCAmelCase		)

                    self.assertListEqual(_lowerCAmelCase	,						_lowerCAmelCase		)

                    # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                    # with Tilde) encoded in 2 different ways
                    UpperCAmelCase__      :  Optional[int]				      =			"""xa\u0303y""" + """ """ + """x\xe3y"""
                    UpperCAmelCase__      :  int				      =			tokenizer_s.tokenize(_lowerCAmelCase		)
                    UpperCAmelCase__      :  int				      =			tokenizer_r.tokenize(_lowerCAmelCase		)

                    self.assertListEqual(_lowerCAmelCase	,						_lowerCAmelCase		)

                    # Test that the tokenization is identical on unicode of space type
                    UpperCAmelCase__      :  Tuple				      =			[
                        """\u0009""",  # (horizontal tab, '\t')
                        """\u000B""",  # (vertical tab)
                        """\u000C""",  # (form feed)
                        """\u0020""",  # (space, ' ')
                        """\u200E""",  # (left-to-right mark):w
                        """\u200F""",  # (right-to-left mark)
                    ]
                    for unicode_seq in spaces_unicodes:
                         UpperCAmelCase__      :  Optional[Any]				      =			tokenizer_s.tokenize(_lowerCAmelCase		)
                         UpperCAmelCase__      :  List[Any]				      =			tokenizer_r.tokenize(_lowerCAmelCase		)

                         self.assertListEqual(_lowerCAmelCase	,						_lowerCAmelCase		)

                    # Test that the tokenization is identical on unicode of line break type
                    UpperCAmelCase__      :  List[Any]				      =			[
                        """\u000A""",  # (line feed, '\n')
                        """\r\n""",  # (carriage return and line feed, '\r\n')
                        """\u000D""",  # (carriage return, '\r')
                        """\r""",  # (carriage return, '\r')
                        """\u000D""",  # (carriage return, '\r')
                        """\u2028""",  # (line separator)
                        """\u2029""",  # (paragraph separator)
                        # "\u0085", # (next line)
                    ]

                    # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                    # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                    # space (and thus into an empty list).

                    for unicode_seq in line_break_unicodes:
                         UpperCAmelCase__      :  Optional[int]				      =			tokenizer_s.tokenize(_lowerCAmelCase		)
                         UpperCAmelCase__      :  int				      =			tokenizer_r.tokenize(_lowerCAmelCase		)

                         self.assertListEqual(_lowerCAmelCase	,						_lowerCAmelCase		)




     def   __UpperCAmelCase  (				self		):
          # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                    UpperCAmelCase__      :  Union[str, Any]				      =			"""hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
                    UpperCAmelCase__      :  Dict				      =			f"{text_of_1_token} {text_of_1_token}"

                    UpperCAmelCase__      :  Dict				      =			self.rust_tokenizer_class.from_pretrained(
                        _lowerCAmelCase	,						use_fast=_lowerCAmelCase	,						)
                    UpperCAmelCase__      :  Optional[int]				      =			tokenizer_r(_lowerCAmelCase	,						return_offsets_mapping=_lowerCAmelCase	,						add_special_tokens=_lowerCAmelCase		)
                    self.assertEqual(encoding.offset_mapping[0]	,						(0, len(_lowerCAmelCase		))		)
                    self.assertEqual(
                        encoding.offset_mapping[1]	,						(len(_lowerCAmelCase		) + 1, len(_lowerCAmelCase		) + 1 + len(_lowerCAmelCase		))	,						)

                    UpperCAmelCase__      :  int				      =			f" {text}"

                    UpperCAmelCase__      :  Optional[int]				      =			self.rust_tokenizer_class.from_pretrained(
                        _lowerCAmelCase	,						use_fast=_lowerCAmelCase	,						)
                    UpperCAmelCase__      :  Optional[Any]				      =			tokenizer_r(_lowerCAmelCase	,						return_offsets_mapping=_lowerCAmelCase	,						add_special_tokens=_lowerCAmelCase		)
                    self.assertEqual(encoding.offset_mapping[0]	,						(1, 1 + len(_lowerCAmelCase		))		)
                    self.assertEqual(
                        encoding.offset_mapping[1]	,						(1 + len(_lowerCAmelCase		) + 1, 1 + len(_lowerCAmelCase		) + 1 + len(_lowerCAmelCase		))	,						)




     def   __UpperCAmelCase  (				self		):
          # Test related to the breaking change introduced in transformers v4.17.0
          # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
          with self.assertRaises(_lowerCAmelCase		) as context:
               self.rust_tokenizer_class.from_pretrained("""robot-test/old-clip-tokenizer"""		)

          self.assertTrue(
              context.exception.args[0].startswith(
                  """The `backend_tokenizer` provided does not match the expected format."""		)		)




     @require_ftfy
     def   __UpperCAmelCase  (				self		):
          super().test_tokenization_python_rust_equals()




     def   __UpperCAmelCase  (				self		):
          # CLIP always lower cases letters
          pass














"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_lowerCAmelCase       =				{
    "configuration_timesformer": ["TIMESFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP", "TimesformerConfig"],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							_lowerCAmelCase       =				[
							    "TIMESFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
							    "TimesformerModel",
							    "TimesformerForVideoClassification",
							    "TimesformerPreTrainedModel",
							]

if TYPE_CHECKING:
							from .configuration_timesformer import TIMESFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, TimesformerConfig

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_timesformer import (
														    TIMESFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TimesformerForVideoClassification,
														    TimesformerModel,
														    TimesformerPreTrainedModel,
														)

else:
							import sys

							_lowerCAmelCase       =				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

