





'''simple docstring'''







import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class        _UpperCAmelCase				(		_lowerCAmelCase      ,   unittest.TestCase  ):
		a  :  Union[str, Any]      =CLIPTokenizer
		a  :  Any      =CLIPTokenizerFast
		a  :  Any      =True
		a  :  Optional[Any]      ={}
		a  :  Any      =False
		def     lowerCamelCase__     (    self     ):



					'''simple docstring'''

					super().setUp()

					# fmt: off
					__lowerCAmelCase   		=							["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
					# fmt: on
					__lowerCAmelCase   		=							dict(zip(SCREAMING_SNAKE_CASE_,range(len(SCREAMING_SNAKE_CASE_     )     )     )     )
					__lowerCAmelCase   		=							["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>"""]
					__lowerCAmelCase   		=							{"""unk_token""": """<unk>"""}

					__lowerCAmelCase   		=							os.path.join(self.tmpdirname,VOCAB_FILES_NAMES["""vocab_file"""]     )
					__lowerCAmelCase   		=							os.path.join(self.tmpdirname,VOCAB_FILES_NAMES["""merges_file"""]     )
					with open(self.vocab_file,"""w""",encoding="""utf-8"""     ) as fp:
								fp.write(json.dumps(SCREAMING_SNAKE_CASE_     ) + """\n"""     )
					with open(self.merges_file,"""w""",encoding="""utf-8"""     ) as fp:
								fp.write("""\n""".join(SCREAMING_SNAKE_CASE_     )     )
		def     lowerCamelCase__     (    self,**__SCREAMING_SNAKE_CASE     ):



					'''simple docstring'''

					kwargs.update(self.special_tokens_map     )
					return CLIPTokenizer.from_pretrained(self.tmpdirname,**SCREAMING_SNAKE_CASE_     )
		def     lowerCamelCase__     (    self,**__SCREAMING_SNAKE_CASE     ):



					'''simple docstring'''

					kwargs.update(self.special_tokens_map     )
					return CLIPTokenizerFast.from_pretrained(self.tmpdirname,**SCREAMING_SNAKE_CASE_     )
		def     lowerCamelCase__     (    self,__SCREAMING_SNAKE_CASE     ):



					'''simple docstring'''

					__lowerCAmelCase   		=							"""lower newer"""
					__lowerCAmelCase   		=							"""lower newer"""
					return input_text, output_text
		def     lowerCamelCase__     (    self     ):



					'''simple docstring'''

					__lowerCAmelCase   		=							CLIPTokenizer(self.vocab_file,self.merges_file,**self.special_tokens_map     )
					__lowerCAmelCase   		=							"""lower newer"""
					__lowerCAmelCase   		=							["""lo""", """w""", """er</w>""", """n""", """e""", """w""", """er</w>"""]
					__lowerCAmelCase   		=							tokenizer.tokenize(SCREAMING_SNAKE_CASE_     )
					self.assertListEqual(SCREAMING_SNAKE_CASE_,SCREAMING_SNAKE_CASE_     )

					__lowerCAmelCase   		=							tokens + [tokenizer.unk_token]
					__lowerCAmelCase   		=							[10, 2, 16, 9, 3, 2, 16, 20]
					self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_     ),SCREAMING_SNAKE_CASE_     )
		@require_ftfy
		def     lowerCamelCase__     (    self     ):



					'''simple docstring'''

					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
											__lowerCAmelCase   		=							self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_,**SCREAMING_SNAKE_CASE_     )
											__lowerCAmelCase   		=							self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_,**SCREAMING_SNAKE_CASE_     )

											__lowerCAmelCase   		=							"""A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."""
											__lowerCAmelCase   		=							tokenizer_s.tokenize(SCREAMING_SNAKE_CASE_     )
											__lowerCAmelCase   		=							tokenizer_r.tokenize(SCREAMING_SNAKE_CASE_     )

											self.assertListEqual(SCREAMING_SNAKE_CASE_,SCREAMING_SNAKE_CASE_     )

											# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
											# with Tilde) encoded in 2 different ways
											__lowerCAmelCase   		=							"""xa\u0303y""" + """ """ + """x\xe3y"""
											__lowerCAmelCase   		=							tokenizer_s.tokenize(SCREAMING_SNAKE_CASE_     )
											__lowerCAmelCase   		=							tokenizer_r.tokenize(SCREAMING_SNAKE_CASE_     )

											self.assertListEqual(SCREAMING_SNAKE_CASE_,SCREAMING_SNAKE_CASE_     )

											# Test that the tokenization is identical on unicode of space type
											__lowerCAmelCase   		=							[
											    """\u0009""",  # (horizontal tab, '\t')
											    """\u000B""",  # (vertical tab)
											    """\u000C""",  # (form feed)
											    """\u0020""",  # (space, ' ')
											    """\u200E""",  # (left-to-right mark):w
											    """\u200F""",  # (right-to-left mark)
											]
											for unicode_seq in spaces_unicodes:
														__lowerCAmelCase   		=							tokenizer_s.tokenize(SCREAMING_SNAKE_CASE_     )
														__lowerCAmelCase   		=							tokenizer_r.tokenize(SCREAMING_SNAKE_CASE_     )

														self.assertListEqual(SCREAMING_SNAKE_CASE_,SCREAMING_SNAKE_CASE_     )

											# Test that the tokenization is identical on unicode of line break type
											__lowerCAmelCase   		=							[
											    """\u000A""",  # (line feed, '\n')
											    """\r\n""",  # (carriage return and line feed, '\r\n')
											    """\u000D""",  # (carriage return, '\r')
											    """\r""",  # (carriage return, '\r')
											    """\u000D""",  # (carriage return, '\r')
											    """\u2028""",  # (line separator)
											    """\u2029""",  # (paragraph separator)
											    # "\u0085", # (next line)
											]

											# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
											# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
											# space (and thus into an empty list).

											for unicode_seq in line_break_unicodes:
														__lowerCAmelCase   		=							tokenizer_s.tokenize(SCREAMING_SNAKE_CASE_     )
														__lowerCAmelCase   		=							tokenizer_r.tokenize(SCREAMING_SNAKE_CASE_     )

														self.assertListEqual(SCREAMING_SNAKE_CASE_,SCREAMING_SNAKE_CASE_     )
		def     lowerCamelCase__     (    self     ):



					'''simple docstring'''

					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
											__lowerCAmelCase   		=							"""hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
											__lowerCAmelCase   		=							f'{text_of_1_token} {text_of_1_token}'

											__lowerCAmelCase   		=							self.rust_tokenizer_class.from_pretrained(
											    SCREAMING_SNAKE_CASE_,use_fast=SCREAMING_SNAKE_CASE_,)
											__lowerCAmelCase   		=							tokenizer_r(SCREAMING_SNAKE_CASE_,return_offsets_mapping=SCREAMING_SNAKE_CASE_,add_special_tokens=SCREAMING_SNAKE_CASE_     )
											self.assertEqual(encoding.offset_mapping[0],(0, len(SCREAMING_SNAKE_CASE_     ))     )
											self.assertEqual(
											    encoding.offset_mapping[1],(len(SCREAMING_SNAKE_CASE_     ) + 1, len(SCREAMING_SNAKE_CASE_     ) + 1 + len(SCREAMING_SNAKE_CASE_     )),)

											__lowerCAmelCase   		=							f' {text}'

											__lowerCAmelCase   		=							self.rust_tokenizer_class.from_pretrained(
											    SCREAMING_SNAKE_CASE_,use_fast=SCREAMING_SNAKE_CASE_,)
											__lowerCAmelCase   		=							tokenizer_r(SCREAMING_SNAKE_CASE_,return_offsets_mapping=SCREAMING_SNAKE_CASE_,add_special_tokens=SCREAMING_SNAKE_CASE_     )
											self.assertEqual(encoding.offset_mapping[0],(1, 1 + len(SCREAMING_SNAKE_CASE_     ))     )
											self.assertEqual(
											    encoding.offset_mapping[1],(1 + len(SCREAMING_SNAKE_CASE_     ) + 1, 1 + len(SCREAMING_SNAKE_CASE_     ) + 1 + len(SCREAMING_SNAKE_CASE_     )),)
		def     lowerCamelCase__     (    self     ):



					'''simple docstring'''

					with self.assertRaises(SCREAMING_SNAKE_CASE_     ) as context:
								self.rust_tokenizer_class.from_pretrained("""robot-test/old-clip-tokenizer"""     )

					self.assertTrue(
					    context.exception.args[0].startswith(
					        """The `backend_tokenizer` provided does not match the expected format."""     )     )
		@require_ftfy
		def     lowerCamelCase__     (    self     ):



					'''simple docstring'''

					super().test_tokenization_python_rust_equals()







		def     lowerCamelCase__     (    self     ):



					'''simple docstring'''

					pass






from ...configuration_utils import PretrainedConfig
from ...utils import logging


a      :  List[Any]			     =			logging.get_logger(__name__)







class 				_a				(   _lowerCAmelCase					):
			A      		=	'''timm_backbone'''

			def __init__(self,   SCREAMING_SNAKE_CASE_=None,   SCREAMING_SNAKE_CASE_=3,   SCREAMING_SNAKE_CASE_=True,   SCREAMING_SNAKE_CASE_=True,   SCREAMING_SNAKE_CASE_=None,   **SCREAMING_SNAKE_CASE_,   )       ->   Tuple:
						super().__init__(**SCREAMING_SNAKE_CASE_					)
						UpperCAmelCase_:		Dict							      =     backbone
						UpperCAmelCase_:		Optional[Any]							      =     num_channels
						UpperCAmelCase_:		Optional[Any]							      =     features_only
						UpperCAmelCase_:		Any							      =     use_pretrained_backbone
						UpperCAmelCase_:		List[str]							      =     True
						UpperCAmelCase_:		Union[str, Any]							      =     out_indices if out_indices is not None else (-1,)






