


import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 		__magic_name__						(snake_case_   ,unittest.TestCase				):





     '''simple docstring'''



     __lowercase				:							Union[str, Any]	   =       CLIPTokenizer
     __lowercase				:							str	   =       CLIPTokenizerFast
     __lowercase				:							Dict	   =       True
     __lowercase				:							Any	   =       {}
     __lowercase				:							Tuple	   =       False

     def 	SCREAMING_SNAKE_CASE__						(					self:str   ):
       super().setUp()

       # fmt: off
       snake_case__						= ['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
       # fmt: on
       snake_case__						= dict(zip(_a     ,		range(len(_a   )   )   )   )
       snake_case__						= ['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
       snake_case__						= {'''unk_token''': '''<unk>'''}

       snake_case__						= os.path.join(self.tmpdirname     ,		VOCAB_FILES_NAMES['''vocab_file''']   )
       snake_case__						= os.path.join(self.tmpdirname     ,		VOCAB_FILES_NAMES['''merges_file''']   )
       with open(self.vocab_file     ,		'''w'''     ,		encoding='''utf-8'''   ) as fp:
         fp.write(json.dumps(_a   ) + '''\n'''   )
       with open(self.merges_file     ,		'''w'''     ,		encoding='''utf-8'''   ) as fp:
         fp.write('''\n'''.join(_a   )   )

     def 	SCREAMING_SNAKE_CASE__						(					self:List[Any]     ,		**_a:Optional[int]   ):
       kwargs.update(self.special_tokens_map   )
       return CLIPTokenizer.from_pretrained(self.tmpdirname     ,		**_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:List[Any]     ,		**_a:Union[str, Any]   ):
       kwargs.update(self.special_tokens_map   )
       return CLIPTokenizerFast.from_pretrained(self.tmpdirname     ,		**_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Tuple     ,		_a:List[str]   ):
       snake_case__						= '''lower newer'''
       snake_case__						= '''lower newer'''
       return input_text, output_text

     def 	SCREAMING_SNAKE_CASE__						(					self:Optional[Any]   ):
       snake_case__						= CLIPTokenizer(self.vocab_file     ,		self.merges_file     ,		**self.special_tokens_map   )
       snake_case__						= '''lower newer'''
       snake_case__						= ['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
       snake_case__						= tokenizer.tokenize(_a   )
       self.assertListEqual(_a     ,		_a   )

       snake_case__						= tokens + [tokenizer.unk_token]
       snake_case__						= [10, 2, 16, 9, 3, 2, 16, 20]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_a   )     ,		_a   )

     @require_ftfy
     def 	SCREAMING_SNAKE_CASE__						(					self:Optional[int]   ):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
           snake_case__						= self.tokenizer_class.from_pretrained(_a     ,		**_a   )
           snake_case__						= self.rust_tokenizer_class.from_pretrained(_a     ,		**_a   )

           snake_case__						= '''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
           snake_case__						= tokenizer_s.tokenize(_a   )
           snake_case__						= tokenizer_r.tokenize(_a   )

           self.assertListEqual(_a     ,		_a   )

           # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
           # with Tilde) encoded in 2 different ways
           snake_case__						= '''xa\u0303y''' + ''' ''' + '''x\xe3y'''
           snake_case__						= tokenizer_s.tokenize(_a   )
           snake_case__						= tokenizer_r.tokenize(_a   )

           self.assertListEqual(_a     ,		_a   )

           # Test that the tokenization is identical on unicode of space type
           snake_case__						= [
               '''\u0009''',  # (horizontal tab, '\t')
               '''\u000B''',  # (vertical tab)
               '''\u000C''',  # (form feed)
               '''\u0020''',  # (space, ' ')
               '''\u200E''',  # (left-to-right mark):w
               '''\u200F''',  # (right-to-left mark)
           ]
           for unicode_seq in spaces_unicodes:
             snake_case__						= tokenizer_s.tokenize(_a   )
             snake_case__						= tokenizer_r.tokenize(_a   )

             self.assertListEqual(_a     ,		_a   )

           # Test that the tokenization is identical on unicode of line break type
           snake_case__						= [
               '''\u000A''',  # (line feed, '\n')
               '''\r\n''',  # (carriage return and line feed, '\r\n')
               '''\u000D''',  # (carriage return, '\r')
               '''\r''',  # (carriage return, '\r')
               '''\u000D''',  # (carriage return, '\r')
               '''\u2028''',  # (line separator)
               '''\u2029''',  # (paragraph separator)
               # "\u0085", # (next line)
           ]

           # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
           # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
           # space (and thus into an empty list).

           for unicode_seq in line_break_unicodes:
             snake_case__						= tokenizer_s.tokenize(_a   )
             snake_case__						= tokenizer_r.tokenize(_a   )

             self.assertListEqual(_a     ,		_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Any   ):
       # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
           snake_case__						= '''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
           snake_case__						= F"""{text_of_1_token} {text_of_1_token}"""

           snake_case__						= self.rust_tokenizer_class.from_pretrained(
               _a     ,		use_fast=_a     ,		)
           snake_case__						= tokenizer_r(_a     ,		return_offsets_mapping=_a     ,		add_special_tokens=_a   )
           self.assertEqual(encoding.offset_mapping[0]     ,		(0, len(_a   ))   )
           self.assertEqual(
               encoding.offset_mapping[1]     ,		(len(_a   ) + 1, len(_a   ) + 1 + len(_a   ))     ,		)

           snake_case__						= F""" {text}"""

           snake_case__						= self.rust_tokenizer_class.from_pretrained(
               _a     ,		use_fast=_a     ,		)
           snake_case__						= tokenizer_r(_a     ,		return_offsets_mapping=_a     ,		add_special_tokens=_a   )
           self.assertEqual(encoding.offset_mapping[0]     ,		(1, 1 + len(_a   ))   )
           self.assertEqual(
               encoding.offset_mapping[1]     ,		(1 + len(_a   ) + 1, 1 + len(_a   ) + 1 + len(_a   ))     ,		)

     def 	SCREAMING_SNAKE_CASE__						(					self:str   ):
       # Test related to the breaking change introduced in transformers v4.17.0
       # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
       with self.assertRaises(_a   ) as context:
         self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer'''   )

       self.assertTrue(
           context.exception.args[0].startswith(
               '''The `backend_tokenizer` provided does not match the expected format.'''   )   )

     @require_ftfy
     def 	SCREAMING_SNAKE_CASE__						(					self:int   ):
       super().test_tokenization_python_rust_equals()







     def 	SCREAMING_SNAKE_CASE__						(					self:Optional[Any]   ):
       # CLIP always lower cases letters
       pass





from copy import deepcopy
class 		__magic_name__						:





     '''simple docstring'''



     def __init__(					self:int     ,		_a:list[int] | None = None     ,		_a:int | None = None   ):

       if arr is None and size is not None:
         snake_case__						= size
         snake_case__						= [0] * size
       elif arr is not None:
         self.init(_a   )
       else:
         raise ValueError('''Either arr or size must be specified'''   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Any     ,		_a:list[int]   ):
       snake_case__						= len(_a   )
       snake_case__						= deepcopy(_a   )
       for i in range(1     ,		self.size   ):
         snake_case__						= self.next_(_a   )
         if j < self.size:
           self.tree[j] += self.tree[i]

     def 	SCREAMING_SNAKE_CASE__						(					self:Any   ):
       snake_case__						= self.tree[:]
       for i in range(self.size - 1     ,		0     ,		-1   ):
         snake_case__						= self.next_(_a   )
         if j < self.size:
           arr[j] -= arr[i]
       return arr

     @staticmethod
     def 	SCREAMING_SNAKE_CASE__						(					_a:int   ):
       return index + (index & (-index))

     @staticmethod
     def 	SCREAMING_SNAKE_CASE__						(					_a:int   ):
       return index - (index & (-index))

     def 	SCREAMING_SNAKE_CASE__						(					self:List[Any]     ,		_a:int     ,		_a:int   ):
       if index == 0:
         self.tree[0] += value
         return
       while index < self.size:
         self.tree[index] += value
         snake_case__						= self.next_(_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Dict     ,		_a:int     ,		_a:int   ):
       self.add(_a     ,		value - self.get(_a   )   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Optional[Any]     ,		_a:int   ):
       if right == 0:
         return 0
       snake_case__						= self.tree[0]
       right -= 1  # make right inclusive
       while right > 0:
         result += self.tree[right]
         snake_case__						= self.prev(_a   )
       return result

     def 	SCREAMING_SNAKE_CASE__						(					self:Dict     ,		_a:int     ,		_a:int   ):
       return self.prefix(_a   ) - self.prefix(_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:str     ,		_a:int   ):
       return self.query(_a     ,		index + 1   )







     def 	SCREAMING_SNAKE_CASE__						(					self:str     ,		_a:int   ):
       value -= self.tree[0]
       if value < 0:
         return -1

       snake_case__						= 1  # Largest power of 2 <= size
       while j * 2 < self.size:
         j *= 2

       snake_case__						= 0

       while j > 0:
         if i + j < self.size and self.tree[i + j] <= value:
           value -= self.tree[i + j]
           i += j
         j //= 2
       return i


if __name__ == "__main__":
   import doctest

   doctest.testmod()

