





import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class      lowerCAmelCase							(      __UpperCamelCase,						unittest.TestCase    ):
 UpperCAmelCase__									=	CLIPTokenizer
 UpperCAmelCase__									=	CLIPTokenizerFast
 UpperCAmelCase__									=	True
 UpperCAmelCase__									=	{}
 UpperCAmelCase__									=	False
 def   A_			(   self						:   Union[str, Any]			)    ->			List[Any]:
     super().setUp()

     # fmt: off
     lowerCamelCase__					:   str      				=	['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
     # fmt: on
     lowerCamelCase__					:   Union[str, Any]      				=	dict(zip(UpperCAmelCase    ,  range(len(UpperCAmelCase			)			)			)			)
     lowerCamelCase__					:   str      				=	['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
     lowerCamelCase__					:   int      				=	{'unk_token': '<unk>'}

     lowerCamelCase__					:   int      				=	os.path.join(self.tmpdirname    ,  VOCAB_FILES_NAMES['vocab_file']			)
     lowerCamelCase__					:   List[Any]      				=	os.path.join(self.tmpdirname    ,  VOCAB_FILES_NAMES['merges_file']			)
     with open(self.vocab_file    ,  'w'    ,  encoding='utf-8'			) as fp:
         fp.write(json.dumps(UpperCAmelCase			) + '\n'			)
     with open(self.merges_file    ,  'w'    ,  encoding='utf-8'			) as fp:
         fp.write('\n'.join(UpperCAmelCase			)			)
 def   A_			(   self						:   Union[str, Any]    ,  **UpperCAmelCase						:   Union[str, Any]			)    ->			Dict:
     kwargs.update(self.special_tokens_map			)
     return CLIPTokenizer.from_pretrained(self.tmpdirname    ,  **UpperCAmelCase			)
 def   A_			(   self						:   str    ,  **UpperCAmelCase						:   List[str]			)    ->			Optional[Any]:
     kwargs.update(self.special_tokens_map			)
     return CLIPTokenizerFast.from_pretrained(self.tmpdirname    ,  **UpperCAmelCase			)
 def   A_			(   self						:   Optional[int]    ,  UpperCAmelCase						:   Union[str, Any]			)    ->			List[str]:
     lowerCamelCase__					:   Union[str, Any]      				=	'lower newer'
     lowerCamelCase__					:   str      				=	'lower newer'
     return input_text, output_text
 def   A_			(   self						:   int			)    ->			Dict:
     lowerCamelCase__					:   str      				=	CLIPTokenizer(self.vocab_file    ,  self.merges_file    ,  **self.special_tokens_map			)
     lowerCamelCase__					:   Optional[int]      				=	'lower newer'
     lowerCamelCase__					:   List[Any]      				=	['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
     lowerCamelCase__					:   Tuple      				=	tokenizer.tokenize(UpperCAmelCase			)
     self.assertListEqual(UpperCAmelCase    ,  UpperCAmelCase			)

     lowerCamelCase__					:   List[Any]      				=	tokens + [tokenizer.unk_token]
     lowerCamelCase__					:   List[str]      				=	[10, 2, 16, 9, 3, 2, 16, 20]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase			)    ,  UpperCAmelCase			)
 @require_ftfy
 def   A_			(   self						:   str			)    ->			Optional[Any]:
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
             lowerCamelCase__					:   List[Any]      				=	self.tokenizer_class.from_pretrained(UpperCAmelCase    ,  **UpperCAmelCase			)
             lowerCamelCase__					:   Tuple      				=	self.rust_tokenizer_class.from_pretrained(UpperCAmelCase    ,  **UpperCAmelCase			)

             lowerCamelCase__					:   Optional[Any]      				=	'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
             lowerCamelCase__					:   str      				=	tokenizer_s.tokenize(UpperCAmelCase			)
             lowerCamelCase__					:   Any      				=	tokenizer_r.tokenize(UpperCAmelCase			)

             self.assertListEqual(UpperCAmelCase    ,  UpperCAmelCase			)

             # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
             # with Tilde) encoded in 2 different ways
             lowerCamelCase__					:   Tuple      				=	'xa\u0303y' + ' ' + 'x\xe3y'
             lowerCamelCase__					:   int      				=	tokenizer_s.tokenize(UpperCAmelCase			)
             lowerCamelCase__					:   int      				=	tokenizer_r.tokenize(UpperCAmelCase			)

             self.assertListEqual(UpperCAmelCase    ,  UpperCAmelCase			)

             # Test that the tokenization is identical on unicode of space type
             lowerCamelCase__					:   Any      				=	[
                 '\u0009',  # (horizontal tab, '\t')
                 '\u000B',  # (vertical tab)
                 '\u000C',  # (form feed)
                 '\u0020',  # (space, ' ')
                 '\u200E',  # (left-to-right mark):w
                 '\u200F',  # (right-to-left mark)
             ]
             for unicode_seq in spaces_unicodes:
                 lowerCamelCase__					:   str      				=	tokenizer_s.tokenize(UpperCAmelCase			)
                 lowerCamelCase__					:   Optional[Any]      				=	tokenizer_r.tokenize(UpperCAmelCase			)

                 self.assertListEqual(UpperCAmelCase    ,  UpperCAmelCase			)

             # Test that the tokenization is identical on unicode of line break type
             lowerCamelCase__					:   int      				=	[
                 '\u000A',  # (line feed, '\n')
                 '\r\n',  # (carriage return and line feed, '\r\n')
                 '\u000D',  # (carriage return, '\r')
                 '\r',  # (carriage return, '\r')
                 '\u000D',  # (carriage return, '\r')
                 '\u2028',  # (line separator)
                 '\u2029',  # (paragraph separator)
                 # "\u0085", # (next line)
             ]

             # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
             # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
             # space (and thus into an empty list).

             for unicode_seq in line_break_unicodes:
                 lowerCamelCase__					:   Optional[Any]      				=	tokenizer_s.tokenize(UpperCAmelCase			)
                 lowerCamelCase__					:   int      				=	tokenizer_r.tokenize(UpperCAmelCase			)

                 self.assertListEqual(UpperCAmelCase    ,  UpperCAmelCase			)
 def   A_			(   self						:   Optional[int]			)    ->			int:
     # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
             lowerCamelCase__					:   Tuple      				=	'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
             lowerCamelCase__					:   str      				=	F"""{text_of_1_token} {text_of_1_token}"""

             lowerCamelCase__					:   Union[str, Any]      				=	self.rust_tokenizer_class.from_pretrained(
                 UpperCAmelCase    ,  use_fast=UpperCAmelCase    ,  )
             lowerCamelCase__					:   int      				=	tokenizer_r(UpperCAmelCase    ,  return_offsets_mapping=UpperCAmelCase    ,  add_special_tokens=UpperCAmelCase			)
             self.assertEqual(encoding.offset_mapping[0]    ,  (0, len(UpperCAmelCase			))			)
             self.assertEqual(
                 encoding.offset_mapping[1]    ,  (len(UpperCAmelCase			) + 1, len(UpperCAmelCase			) + 1 + len(UpperCAmelCase			))    ,  )

             lowerCamelCase__					:   Optional[Any]      				=	F""" {text}"""

             lowerCamelCase__					:   List[str]      				=	self.rust_tokenizer_class.from_pretrained(
                 UpperCAmelCase    ,  use_fast=UpperCAmelCase    ,  )
             lowerCamelCase__					:   Optional[Any]      				=	tokenizer_r(UpperCAmelCase    ,  return_offsets_mapping=UpperCAmelCase    ,  add_special_tokens=UpperCAmelCase			)
             self.assertEqual(encoding.offset_mapping[0]    ,  (1, 1 + len(UpperCAmelCase			))			)
             self.assertEqual(
                 encoding.offset_mapping[1]    ,  (1 + len(UpperCAmelCase			) + 1, 1 + len(UpperCAmelCase			) + 1 + len(UpperCAmelCase			))    ,  )
 def   A_			(   self						:   Any			)    ->			Union[str, Any]:
     # Test related to the breaking change introduced in transformers v4.17.0
     # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
     with self.assertRaises(UpperCAmelCase			) as context:
         self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'			)

     self.assertTrue(
         context.exception.args[0].startswith(
             'The `backend_tokenizer` provided does not match the expected format.'			)			)
 @require_ftfy
 def   A_			(   self						:   Any			)    ->			Tuple:
     super().test_tokenization_python_rust_equals()







 def   A_			(   self						:   Tuple			)    ->			str:
     # CLIP always lower cases letters
     pass












class      lowerCAmelCase							:
 def __init__(   self						:   Union[str, Any]    ,  UpperCAmelCase						:   list			)    ->			None:
     lowerCamelCase__					:   int      				=	set_counts
     lowerCamelCase__					:   List[str]      				=	max(UpperCAmelCase			)
     lowerCamelCase__					:   Dict      				=	len(UpperCAmelCase			)
     lowerCamelCase__					:   List[str]      				=	[1] * num_sets
     lowerCamelCase__					:   List[str]      				=	list(range(UpperCAmelCase			)			)
 def   A_			(   self						:   Dict    ,  UpperCAmelCase						:   int    ,  UpperCAmelCase						:   int			)    ->			bool:
     lowerCamelCase__					:   List[str]      				=	self.get_parent(UpperCAmelCase			)
     lowerCamelCase__					:   List[str]      				=	self.get_parent(UpperCAmelCase			)

     if src_parent == dst_parent:
         return False

     if self.ranks[dst_parent] >= self.ranks[src_parent]:
         self.set_counts[dst_parent] += self.set_counts[src_parent]
         lowerCamelCase__					:   Union[str, Any]      				=	0
         lowerCamelCase__					:   str      				=	dst_parent
         if self.ranks[dst_parent] == self.ranks[src_parent]:
             self.ranks[dst_parent] += 1
         lowerCamelCase__					:   int      				=	self.set_counts[dst_parent]
     else:
         self.set_counts[src_parent] += self.set_counts[dst_parent]
         lowerCamelCase__					:   List[str]      				=	0
         lowerCamelCase__					:   Tuple      				=	src_parent
         lowerCamelCase__					:   List[str]      				=	self.set_counts[src_parent]

     lowerCamelCase__					:   Optional[int]      				=	max(self.max_set    ,  UpperCAmelCase			)
     return True







 def   A_			(   self						:   str    ,  UpperCAmelCase						:   int			)    ->			int:
     if self.parents[disj_set] == disj_set:
         return disj_set
     lowerCamelCase__					:   Dict      				=	self.get_parent(self.parents[disj_set]			)
     return self.parents[disj_set]





