


'''simple docstring'''

import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class     __lowerCAmelCase		(lowercase_    ,	unittest.TestCase	):





      '''simple docstring'''






      lowerCAmelCase__   :	Any       =				CLIPTokenizer
      lowerCAmelCase__   :	List[str]       =				CLIPTokenizerFast
      lowerCAmelCase__   :	Optional[Any]       =				True
      lowerCAmelCase__   :	Optional[Any]       =				{}
      lowerCAmelCase__   :	Union[str, Any]       =				False



      def  UpperCamelCase__						(self       :       Tuple					):

             '''simple docstring'''







             super().setUp()

             # fmt: off
             lowercase__      =					['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
             # fmt: on
             lowercase__      =					dict(zip(UpperCamelCase  ,		range(len(UpperCamelCase					)					)					)					)
             lowercase__      =					['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
             lowercase__      =					{'''unk_token''': '''<unk>'''}

             lowercase__      =					os.path.join(self.tmpdirname  ,		VOCAB_FILES_NAMES['''vocab_file''']					)
             lowercase__      =					os.path.join(self.tmpdirname  ,		VOCAB_FILES_NAMES['''merges_file''']					)
             with open(self.vocab_file  ,		'''w'''  ,		encoding='''utf-8'''					) as fp:
                    fp.write(json.dumps(UpperCamelCase					) + '''\n'''					)
             with open(self.merges_file  ,		'''w'''  ,		encoding='''utf-8'''					) as fp:
                    fp.write('''\n'''.join(UpperCamelCase					)					)



      def  UpperCamelCase__						(self       :       List[Any]  ,		**UpperCamelCase       :       Optional[int]					):

             '''simple docstring'''







             kwargs.update(self.special_tokens_map					)
             return CLIPTokenizer.from_pretrained(self.tmpdirname  ,		**UpperCamelCase					)



      def  UpperCamelCase__						(self       :       Any  ,		**UpperCamelCase       :       Tuple					):

             '''simple docstring'''







             kwargs.update(self.special_tokens_map					)
             return CLIPTokenizerFast.from_pretrained(self.tmpdirname  ,		**UpperCamelCase					)



      def  UpperCamelCase__						(self       :       Dict  ,		UpperCamelCase       :       Optional[Any]					):

             '''simple docstring'''







             lowercase__      =					'''lower newer'''
             lowercase__      =					'''lower newer'''
             return input_text, output_text



      def  UpperCamelCase__						(self       :       Dict					):

             '''simple docstring'''







             lowercase__      =					CLIPTokenizer(self.vocab_file  ,		self.merges_file  ,		**self.special_tokens_map					)
             lowercase__      =					'''lower newer'''
             lowercase__      =					['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
             lowercase__      =					tokenizer.tokenize(UpperCamelCase					)
             self.assertListEqual(UpperCamelCase  ,		UpperCamelCase					)

             lowercase__      =					tokens + [tokenizer.unk_token]
             lowercase__      =					[10, 2, 16, 9, 3, 2, 16, 20]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase					)  ,		UpperCamelCase					)



      @require_ftfy
      def  UpperCamelCase__						(self       :       int					):

             '''simple docstring'''







             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"					):
                           lowercase__      =					self.tokenizer_class.from_pretrained(UpperCamelCase  ,		**UpperCamelCase					)
                           lowercase__      =					self.rust_tokenizer_class.from_pretrained(UpperCamelCase  ,		**UpperCamelCase					)

                           lowercase__      =					'''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
                           lowercase__      =					tokenizer_s.tokenize(UpperCamelCase					)
                           lowercase__      =					tokenizer_r.tokenize(UpperCamelCase					)

                           self.assertListEqual(UpperCamelCase  ,		UpperCamelCase					)

                           # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                           # with Tilde) encoded in 2 different ways
                           lowercase__      =					'''xa\u0303y''' + ''' ''' + '''x\xe3y'''
                           lowercase__      =					tokenizer_s.tokenize(UpperCamelCase					)
                           lowercase__      =					tokenizer_r.tokenize(UpperCamelCase					)

                           self.assertListEqual(UpperCamelCase  ,		UpperCamelCase					)

                           # Test that the tokenization is identical on unicode of space type
                           lowercase__      =					[
                               '''\u0009''',  # (horizontal tab, '\t')
                               '''\u000B''',  # (vertical tab)
                               '''\u000C''',  # (form feed)
                               '''\u0020''',  # (space, ' ')
                               '''\u200E''',  # (left-to-right mark):w
                               '''\u200F''',  # (right-to-left mark)
                           ]
                           for unicode_seq in spaces_unicodes:
                                  lowercase__      =					tokenizer_s.tokenize(UpperCamelCase					)
                                  lowercase__      =					tokenizer_r.tokenize(UpperCamelCase					)

                                  self.assertListEqual(UpperCamelCase  ,		UpperCamelCase					)

                           # Test that the tokenization is identical on unicode of line break type
                           lowercase__      =					[
                               '''\u000A''',  # (line feed, '\n')
                               '''\r\n''',  # (carriage return and line feed, '\r\n')
                               '''\u000D''',  # (carriage return, '\r')
                               '''\r''',  # (carriage return, '\r')
                               '''\u000D''',  # (carriage return, '\r')
                               '''\u2028''',  # (line separator)
                               '''\u2029''',  # (paragraph separator)
                               # "\u0085", # (next line)
                           ]

                           # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                           # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                           # space (and thus into an empty list).

                           for unicode_seq in line_break_unicodes:
                                  lowercase__      =					tokenizer_s.tokenize(UpperCamelCase					)
                                  lowercase__      =					tokenizer_r.tokenize(UpperCamelCase					)

                                  self.assertListEqual(UpperCamelCase  ,		UpperCamelCase					)



      def  UpperCamelCase__						(self       :       List[str]					):

             '''simple docstring'''







             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"					):
                           lowercase__      =					'''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
                           lowercase__      =					f"{text_of_1_token} {text_of_1_token}"

                           lowercase__      =					self.rust_tokenizer_class.from_pretrained(
                               UpperCamelCase  ,		use_fast=UpperCamelCase  ,		)
                           lowercase__      =					tokenizer_r(UpperCamelCase  ,		return_offsets_mapping=UpperCamelCase  ,		add_special_tokens=UpperCamelCase					)
                           self.assertEqual(encoding.offset_mapping[0]  ,		(0, len(UpperCamelCase					))					)
                           self.assertEqual(
                               encoding.offset_mapping[1]  ,		(len(UpperCamelCase					) + 1, len(UpperCamelCase					) + 1 + len(UpperCamelCase					))  ,		)

                           lowercase__      =					f" {text}"

                           lowercase__      =					self.rust_tokenizer_class.from_pretrained(
                               UpperCamelCase  ,		use_fast=UpperCamelCase  ,		)
                           lowercase__      =					tokenizer_r(UpperCamelCase  ,		return_offsets_mapping=UpperCamelCase  ,		add_special_tokens=UpperCamelCase					)
                           self.assertEqual(encoding.offset_mapping[0]  ,		(1, 1 + len(UpperCamelCase					))					)
                           self.assertEqual(
                               encoding.offset_mapping[1]  ,		(1 + len(UpperCamelCase					) + 1, 1 + len(UpperCamelCase					) + 1 + len(UpperCamelCase					))  ,		)



      def  UpperCamelCase__						(self       :       str					):

             '''simple docstring'''







             with self.assertRaises(UpperCamelCase					) as context:
                    self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer'''					)

             self.assertTrue(
                 context.exception.args[0].startswith(
                     '''The `backend_tokenizer` provided does not match the expected format.'''					)					)



      @require_ftfy
      def  UpperCamelCase__						(self       :       Any					):

             '''simple docstring'''







             super().test_tokenization_python_rust_equals()






      def  UpperCamelCase__						(self       :       str					):

             '''simple docstring'''







             pass








'''simple docstring'''

from __future__ import annotations

from typing import Any
class     __lowerCAmelCase		:





      '''simple docstring'''






      def __init__(self       :       Optional[Any]  ,		UpperCamelCase       :       int = 6					):

             '''simple docstring'''







             lowercase__      =					None
             lowercase__      =					None
             self.create_linked_list(UpperCamelCase					)



      def  UpperCamelCase__						(self       :       List[str]  ,		UpperCamelCase       :       int					):

             '''simple docstring'''







             lowercase__      =					Node()
             lowercase__      =					current_node
             lowercase__      =					current_node
             lowercase__      =					current_node
             for _ in range(1  ,		UpperCamelCase					):
                    lowercase__      =					Node()
                    lowercase__      =					current_node
                    lowercase__      =					previous_node
                    lowercase__      =					current_node
             lowercase__      =					self.front
             lowercase__      =					previous_node



      def  UpperCamelCase__						(self       :       Union[str, Any]					):

             '''simple docstring'''







             return (
                 self.front == self.rear
                 and self.front is not None
                 and self.front.data is None
             )



      def  UpperCamelCase__						(self       :       Dict					):

             '''simple docstring'''







             self.check_can_perform_operation()
             return self.front.data if self.front else None



      def  UpperCamelCase__						(self       :       Any  ,		UpperCamelCase       :       Any					):

             '''simple docstring'''







             if self.rear is None:
                    return

             self.check_is_full()
             if not self.is_empty():
                    lowercase__      =					self.rear.next
             if self.rear:
                    lowercase__      =					data



      def  UpperCamelCase__						(self       :       List[Any]					):

             '''simple docstring'''







             self.check_can_perform_operation()
             if self.rear is None or self.front is None:
                    return None
             if self.front == self.rear:
                    lowercase__      =					self.front.data
                    lowercase__      =					None
                    return data

             lowercase__      =					self.front
             lowercase__      =					old_front.next
             lowercase__      =					old_front.data
             lowercase__      =					None
             return data



      def  UpperCamelCase__						(self       :       Union[str, Any]					):

             '''simple docstring'''







             if self.is_empty():
                    raise Exception('''Empty Queue'''					)






      def  UpperCamelCase__						(self       :       int					):

             '''simple docstring'''







             if self.rear and self.rear.next == self.front:
                    raise Exception('''Full Queue'''					)






class     __lowerCAmelCase		:





      '''simple docstring'''






      def __init__(self       :       Any					):

             '''simple docstring'''







             lowercase__      =					None
             lowercase__      =					None
             lowercase__      =					None


if __name__ == "__main__":
 import doctest

 doctest.testmod()




