






import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
   from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


A_		:		List[str]		     =			get_tests_dir('fixtures/test_sentencepiece.model')

A_		:		str		     =			{'target_lang': 'fi', 'source_lang': 'en'}
A_		:		List[Any]		     =			'>>zh<<'
A_		:		Optional[int]		     =			'Helsinki-NLP/'

if is_torch_available():
   A_		:		Any		     =			'pt'
elif is_tf_available():
   A_		:		Tuple		     =			'tf'
else:
   A_		:		List[str]		     =			'jax'




@require_sentencepiece
class     _lowerCAmelCase(	UpperCAmelCase_							,     unittest.TestCase ):





       """simple docstring"""







       a    :		Optional[int]									=MarianTokenizer
       a    :		Union[str, Any]									=False
       a    :		Optional[int]									=True





       def       _a   ( self							):
         super().setUp()
         UpperCamelCase_: Dict   				=    ['</s>', '<unk>', '▁This', '▁is', '▁a', '▁t', 'est', '\u0120', '<pad>']
         UpperCamelCase_: int   				=    dict(zip(_lowerCamelCase  ,      range(len(_lowerCamelCase							)							)							)							)
         UpperCamelCase_: Dict   				=    Path(self.tmpdirname							)
         save_json(_lowerCamelCase  ,      save_dir / VOCAB_FILES_NAMES['vocab']							)
         save_json(_lowerCamelCase  ,      save_dir / VOCAB_FILES_NAMES['tokenizer_config_file']							)
         if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
           copyfile(_lowerCamelCase  ,      save_dir / VOCAB_FILES_NAMES['source_spm']							)
           copyfile(_lowerCamelCase  ,      save_dir / VOCAB_FILES_NAMES['target_spm']							)

         UpperCamelCase_: Any   				=    MarianTokenizer.from_pretrained(self.tmpdirname							)
         tokenizer.save_pretrained(self.tmpdirname							)





       def       _a   ( self  ,      **_lowerCamelCase							):
         return MarianTokenizer.from_pretrained(self.tmpdirname  ,      **_lowerCamelCase							)





       def       _a   ( self  ,      _lowerCamelCase							):
         return (
             "This is a test",
             "This is a test",
         )





       def       _a   ( self							):
         UpperCamelCase_: str   				=    '</s>'
         UpperCamelCase_: Optional[Any]   				=    0

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCamelCase							)  ,      _lowerCamelCase							)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCamelCase							)  ,      _lowerCamelCase							)





       def       _a   ( self							):
         UpperCamelCase_: List[str]   				=    list(self.get_tokenizer().get_vocab().keys()							)

         self.assertEqual(vocab_keys[0]  ,      '</s>'							)
         self.assertEqual(vocab_keys[1]  ,      '<unk>'							)
         self.assertEqual(vocab_keys[-1]  ,      '<pad>'							)
         self.assertEqual(len(_lowerCamelCase							)  ,      9							)





       def       _a   ( self							):
         self.assertEqual(self.get_tokenizer().vocab_size  ,      9							)





       def       _a   ( self							):
         UpperCamelCase_: Optional[Any]   				=    MarianTokenizer.from_pretrained(f'''{ORG_NAME}opus-mt-en-de'''							)
         UpperCamelCase_: List[str]   				=    en_de_tokenizer(['I am a small frog']  ,      return_tensors=_lowerCamelCase							)
         self.assertIsInstance(_lowerCamelCase  ,      _lowerCamelCase							)
         UpperCamelCase_: List[str]   				=    [3_8, 1_2_1, 1_4, 6_9_7, 3_8_8_4_8, 0]
         self.assertListEqual(_lowerCamelCase  ,      batch.input_ids[0]							)

         UpperCamelCase_: Union[str, Any]   				=    tempfile.mkdtemp()
         en_de_tokenizer.save_pretrained(_lowerCamelCase							)
         UpperCamelCase_: List[Any]   				=    [x.name for x in Path(_lowerCamelCase							).glob('*'							)]
         self.assertIn('source.spm'  ,      _lowerCamelCase							)
         MarianTokenizer.from_pretrained(_lowerCamelCase							)





       def       _a   ( self							):
         UpperCamelCase_: Optional[int]   				=    self.get_tokenizer()

         UpperCamelCase_: Any   				=    tok(
             ['I am a small frog' * 1_0_0_0, 'I am a small frog']  ,      padding=_lowerCamelCase  ,      truncation=_lowerCamelCase  ,      return_tensors=_lowerCamelCase							)
         self.assertIsInstance(_lowerCamelCase  ,      _lowerCamelCase							)
         self.assertEqual(batch.input_ids.shape  ,      (2, 5_1_2)							)





       def       _a   ( self							):
         UpperCamelCase_: Union[str, Any]   				=    self.get_tokenizer()
         UpperCamelCase_: Any   				=    tok(['I am a tiny frog', 'I am a small frog']  ,      padding=_lowerCamelCase  ,      return_tensors=_lowerCamelCase							)
         self.assertIsInstance(_lowerCamelCase  ,      _lowerCamelCase							)
         self.assertEqual(batch_smaller.input_ids.shape  ,      (2, 1_0)							)





       @slow
       def       _a   ( self							):
         # fmt: off
         UpperCamelCase_: Tuple   				=    {'input_ids': [[4_3_4_9_5, 4_6_2, 2_0, 4_2_1_6_4, 1_3_6_9, 5_2, 4_6_4, 1_3_2, 1_7_0_3, 4_9_2, 1_3, 7_4_9_1, 3_8_9_9_9, 6, 8, 4_6_4, 1_3_2, 1_7_0_3, 4_9_2, 1_3, 4_6_6_9, 3_7_8_6_7, 1_3, 7_5_2_5, 2_7, 1_5_9_3, 9_8_8, 1_3, 3_3_9_7_2, 7_0_2_9, 6, 2_0, 8_2_5_1, 3_8_3, 2, 2_7_0, 5_8_6_6, 3_7_8_8, 2, 2_3_5_3, 8_2_5_1, 1_2_3_3_8, 2, 1_3_9_5_8, 3_8_7, 2, 3_6_2_9, 6_9_5_3, 1_8_8, 2_9_0_0, 2, 1_3_9_5_8, 8_0_1_1, 1_1_5_0_1, 2_3, 8_4_6_0, 4_0_7_3, 3_4_0_0_9, 2_0, 4_3_5, 1_1_4_3_9, 2_7, 8, 8_4_6_0, 4_0_7_3, 6_0_0_4, 2_0, 9_9_8_8, 3_7_5, 2_7, 3_3, 2_6_6, 1_9_4_5, 1_0_7_6, 1_3_5_0, 3_7_8_6_7, 3_2_8_8, 5, 5_7_7, 1_0_7_6, 4_3_7_4, 8, 5_0_8_2, 5, 2_6_4_5_3, 2_5_7, 5_5_6, 4_0_3, 2, 2_4_2, 1_3_2, 3_8_3, 3_1_6, 4_9_2, 8, 1_0_7_6_7, 6, 3_1_6, 3_0_4, 4_2_3_9, 3, 0], [1_4_8, 1_5_7_2_2, 1_9, 1_8_3_9, 1_2, 1_3_5_0, 1_3, 2_2_3_2_7, 5_0_8_2, 5_4_1_8, 4_7_5_6_7, 3_5_9_3_8, 5_9, 3_1_8, 1_9_5_5_2, 1_0_8, 2_1_8_3, 5_4, 1_4_9_7_6, 4_8_3_5, 3_2, 5_4_7, 1_1_1_4, 8, 3_1_5, 2_4_1_7, 5, 9_2, 1_9_0_8_8, 3, 0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0], [3_6, 6_3_9_5, 1_2_5_7_0, 3_9_1_4_7, 1_1_5_9_7, 6, 2_6_6, 4, 4_5_4_0_5, 7_2_9_6, 3, 0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=_lowerCamelCase  ,      model_name='Helsinki-NLP/opus-mt-en-de'  ,      revision='1a8c2263da11e68e50938f97e10cd57820bd504c'  ,      decode_kwargs={'use_source_tokenizer': True}  ,      )





       def       _a   ( self							):
         UpperCamelCase_: Union[str, Any]   				=    MarianTokenizer.from_pretrained('hf-internal-testing/test-marian-two-vocabs'							)

         UpperCamelCase_: List[Any]   				=    'Tämä on testi'
         UpperCamelCase_: Dict   				=    'This is a test'

         UpperCamelCase_: Dict   				=    [7_6, 7, 2_0_4_7, 2]
         UpperCamelCase_: List[str]   				=    [6_9, 1_2, 1_1, 9_4_0, 2]

         UpperCamelCase_: Tuple   				=    tokenizer(_lowerCamelCase							).input_ids
         self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         UpperCamelCase_: List[str]   				=    tokenizer(text_target=_lowerCamelCase							).input_ids
         self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         UpperCamelCase_: str   				=    tokenizer.decode(_lowerCamelCase  ,      skip_special_tokens=_lowerCamelCase							)
         self.assertEqual(_lowerCamelCase  ,      _lowerCamelCase							)







import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class     _lowerCAmelCase(	UpperCAmelCase_							,     unittest.TestCase ):





       """simple docstring"""







       a    :		Optional[Any]									=CLIPTokenizer
       a    :		List[Any]									=CLIPTokenizerFast
       a    :		str									=True
       a    :		List[Any]									={}
       a    :		str									=False





       def       _a   ( self							):
         super().setUp()

         # fmt: off
         UpperCamelCase_: str   				=    ['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
         # fmt: on
         UpperCamelCase_: Any   				=    dict(zip(_lowerCamelCase  ,      range(len(_lowerCamelCase							)							)							)							)
         UpperCamelCase_: Optional[int]   				=    ['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
         UpperCamelCase_: Any   				=    {'unk_token': '<unk>'}

         UpperCamelCase_: List[Any]   				=    os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES['vocab_file']							)
         UpperCamelCase_: Optional[int]   				=    os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES['merges_file']							)
         with open(self.vocab_file  ,      'w'  ,      encoding='utf-8'							) as fp:
           fp.write(json.dumps(_lowerCamelCase							) + '\n'							)
         with open(self.merges_file  ,      'w'  ,      encoding='utf-8'							) as fp:
           fp.write('\n'.join(_lowerCamelCase							)							)





       def       _a   ( self  ,      **_lowerCamelCase							):
         kwargs.update(self.special_tokens_map							)
         return CLIPTokenizer.from_pretrained(self.tmpdirname  ,      **_lowerCamelCase							)





       def       _a   ( self  ,      **_lowerCamelCase							):
         kwargs.update(self.special_tokens_map							)
         return CLIPTokenizerFast.from_pretrained(self.tmpdirname  ,      **_lowerCamelCase							)





       def       _a   ( self  ,      _lowerCamelCase							):
         UpperCamelCase_: Union[str, Any]   				=    'lower newer'
         UpperCamelCase_: Tuple   				=    'lower newer'
         return input_text, output_text





       def       _a   ( self							):
         UpperCamelCase_: Dict   				=    CLIPTokenizer(self.vocab_file  ,      self.merges_file  ,      **self.special_tokens_map							)
         UpperCamelCase_: Optional[int]   				=    'lower newer'
         UpperCamelCase_: Any   				=    ['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
         UpperCamelCase_: Union[str, Any]   				=    tokenizer.tokenize(_lowerCamelCase							)
         self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)

         UpperCamelCase_: str   				=    tokens + [tokenizer.unk_token]
         UpperCamelCase_: Optional[Any]   				=    [1_0, 2, 1_6, 9, 3, 2, 1_6, 2_0]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase							)  ,      _lowerCamelCase							)





       @require_ftfy
       def       _a   ( self							):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
             UpperCamelCase_: Optional[int]   				=    self.tokenizer_class.from_pretrained(_lowerCamelCase  ,      **_lowerCamelCase							)
             UpperCamelCase_: List[str]   				=    self.rust_tokenizer_class.from_pretrained(_lowerCamelCase  ,      **_lowerCamelCase							)

             UpperCamelCase_: Optional[int]   				=    'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
             UpperCamelCase_: Tuple   				=    tokenizer_s.tokenize(_lowerCamelCase							)
             UpperCamelCase_: Any   				=    tokenizer_r.tokenize(_lowerCamelCase							)

             self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)

             # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
             # with Tilde) encoded in 2 different ways
             UpperCamelCase_: List[str]   				=    'xa\u0303y' + ' ' + 'x\xe3y'
             UpperCamelCase_: List[Any]   				=    tokenizer_s.tokenize(_lowerCamelCase							)
             UpperCamelCase_: int   				=    tokenizer_r.tokenize(_lowerCamelCase							)

             self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)

             # Test that the tokenization is identical on unicode of space type
             UpperCamelCase_: Dict   				=    [
                 '\u0009',  # (horizontal tab, '\t')
                 '\u000B',  # (vertical tab)
                 '\u000C',  # (form feed)
                 '\u0020',  # (space, ' ')
                 '\u200E',  # (left-to-right mark):w
                 '\u200F',  # (right-to-left mark)
             ]
             for unicode_seq in spaces_unicodes:
               UpperCamelCase_: int   				=    tokenizer_s.tokenize(_lowerCamelCase							)
               UpperCamelCase_: Dict   				=    tokenizer_r.tokenize(_lowerCamelCase							)

               self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)

             # Test that the tokenization is identical on unicode of line break type
             UpperCamelCase_: List[str]   				=    [
                 '\u000A',  # (line feed, '\n')
                 '\r\n',  # (carriage return and line feed, '\r\n')
                 '\u000D',  # (carriage return, '\r')
                 '\r',  # (carriage return, '\r')
                 '\u000D',  # (carriage return, '\r')
                 '\u2028',  # (line separator)
                 '\u2029',  # (paragraph separator)
                 # "\u0085", # (next line)
             ]

             # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
             # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
             # space (and thus into an empty list).

             for unicode_seq in line_break_unicodes:
               UpperCamelCase_: Optional[Any]   				=    tokenizer_s.tokenize(_lowerCamelCase							)
               UpperCamelCase_: str   				=    tokenizer_r.tokenize(_lowerCamelCase							)

               self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)





       def       _a   ( self							):
         # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
             UpperCamelCase_: str   				=    'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
             UpperCamelCase_: str   				=    f'''{text_of_1_token} {text_of_1_token}'''

             UpperCamelCase_: Optional[Any]   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      )
             UpperCamelCase_: int   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (0, len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (len(_lowerCamelCase							) + 1, len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )

             UpperCamelCase_: List[str]   				=    f''' {text}'''

             UpperCamelCase_: str   				=    self.rust_tokenizer_class.from_pretrained(
                 _lowerCamelCase  ,      use_fast=_lowerCamelCase  ,      )
             UpperCamelCase_: Dict   				=    tokenizer_r(_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             self.assertEqual(encoding.offset_mapping[0]  ,      (1, 1 + len(_lowerCamelCase							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]  ,      (1 + len(_lowerCamelCase							) + 1, 1 + len(_lowerCamelCase							) + 1 + len(_lowerCamelCase							))  ,      )





       def       _a   ( self							):
         # Test related to the breaking change introduced in transformers v4.17.0
         # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
         with self.assertRaises(_lowerCamelCase							) as context:
           self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'							)

         self.assertTrue(
             context.exception.args[0].startswith(
                 'The `backend_tokenizer` provided does not match the expected format.'							)							)





       @require_ftfy
       def       _a   ( self							):
         super().test_tokenization_python_rust_equals()





       def       _a   ( self							):
         # CLIP always lower cases letters
         pass