'''simple docstring'''






import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class lowercase_       (		_UpperCamelCase    ,    unittest.TestCase			):







   """simple docstring"""

   __lowerCAmelCase         =   CLIPTokenizer
   __lowerCAmelCase         =   CLIPTokenizerFast
   __lowerCAmelCase         =   True
   __lowerCAmelCase         =   {}
   __lowerCAmelCase         =   False
   def 	__UpperCAmelCase   (      self      :			Dict     )  ->			Tuple:
        super().setUp()

        # fmt: off
        _A    		=  ['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
        # fmt: on
        _A    		=  dict(zip(UpperCamelCase__, range(len(UpperCamelCase__     )     )     )     )
        _A    		=  ['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>']
        _A    		=  {'unk_token': '<unk>'}

        _A    		=  os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['vocab_file']     )
        _A    		=  os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['merges_file']     )
        with open(self.vocab_file, 'w', encoding='utf-8'     ) as fp:
             fp.write(json.dumps(UpperCamelCase__     ) + '\n'     )
        with open(self.merges_file, 'w', encoding='utf-8'     ) as fp:
             fp.write('\n'.join(UpperCamelCase__     )     )
   def 	__UpperCAmelCase   (      self      :			List[Any], **UpperCamelCase__      :			Optional[Any]     )  ->			int:
        kwargs.update(self.special_tokens_map     )
        return CLIPTokenizer.from_pretrained(self.tmpdirname, **UpperCamelCase__     )
   def 	__UpperCAmelCase   (      self      :			str, **UpperCamelCase__      :			List[str]     )  ->			Tuple:
        kwargs.update(self.special_tokens_map     )
        return CLIPTokenizerFast.from_pretrained(self.tmpdirname, **UpperCamelCase__     )
   def 	__UpperCAmelCase   (      self      :			Dict, UpperCamelCase__      :			str     )  ->			Tuple:
        _A    		=  'lower newer'
        _A    		=  'lower newer'
        return input_text, output_text
   def 	__UpperCAmelCase   (      self      :			List[Any]     )  ->			List[Any]:
        _A    		=  CLIPTokenizer(self.vocab_file, self.merges_file, **self.special_tokens_map     )
        _A    		=  'lower newer'
        _A    		=  ['lo', 'w', 'er</w>', 'n', 'e', 'w', 'er</w>']
        _A    		=  tokenizer.tokenize(UpperCamelCase__     )
        self.assertListEqual(UpperCamelCase__, UpperCamelCase__     )

        _A    		=  tokens + [tokenizer.unk_token]
        _A    		=  [10, 2, 16, 9, 3, 2, 16, 20]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__     ), UpperCamelCase__     )
   @require_ftfy
   def 	__UpperCAmelCase   (      self      :			int     )  ->			Any:
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
                  _A    		=  self.tokenizer_class.from_pretrained(UpperCamelCase__, **UpperCamelCase__     )
                  _A    		=  self.rust_tokenizer_class.from_pretrained(UpperCamelCase__, **UpperCamelCase__     )

                  _A    		=  'A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'
                  _A    		=  tokenizer_s.tokenize(UpperCamelCase__     )
                  _A    		=  tokenizer_r.tokenize(UpperCamelCase__     )

                  self.assertListEqual(UpperCamelCase__, UpperCamelCase__     )

                  # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                  # with Tilde) encoded in 2 different ways
                  _A    		=  'xa\u0303y' + ' ' + 'x\xe3y'
                  _A    		=  tokenizer_s.tokenize(UpperCamelCase__     )
                  _A    		=  tokenizer_r.tokenize(UpperCamelCase__     )

                  self.assertListEqual(UpperCamelCase__, UpperCamelCase__     )

                  # Test that the tokenization is identical on unicode of space type
                  _A    		=  [
                      '\u0009',  # (horizontal tab, '\t')
                      '\u000B',  # (vertical tab)
                      '\u000C',  # (form feed)
                      '\u0020',  # (space, ' ')
                      '\u200E',  # (left-to-right mark):w
                      '\u200F',  # (right-to-left mark)
                  ]
                  for unicode_seq in spaces_unicodes:
                       _A    		=  tokenizer_s.tokenize(UpperCamelCase__     )
                       _A    		=  tokenizer_r.tokenize(UpperCamelCase__     )

                       self.assertListEqual(UpperCamelCase__, UpperCamelCase__     )

                  # Test that the tokenization is identical on unicode of line break type
                  _A    		=  [
                      '\u000A',  # (line feed, '\n')
                      '\r\n',  # (carriage return and line feed, '\r\n')
                      '\u000D',  # (carriage return, '\r')
                      '\r',  # (carriage return, '\r')
                      '\u000D',  # (carriage return, '\r')
                      '\u2028',  # (line separator)
                      '\u2029',  # (paragraph separator)
                      # "\u0085", # (next line)
                  ]

                  # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                  # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                  # space (and thus into an empty list).

                  for unicode_seq in line_break_unicodes:
                       _A    		=  tokenizer_s.tokenize(UpperCamelCase__     )
                       _A    		=  tokenizer_r.tokenize(UpperCamelCase__     )

                       self.assertListEqual(UpperCamelCase__, UpperCamelCase__     )
   def 	__UpperCAmelCase   (      self      :			Union[str, Any]     )  ->			str:
        # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
                  _A    		=  'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
                  _A    		=  f'{text_of_1_token} {text_of_1_token}'

                  _A    		=  self.rust_tokenizer_class.from_pretrained(
                      UpperCamelCase__, use_fast=UpperCamelCase__, )
                  _A    		=  tokenizer_r(UpperCamelCase__, return_offsets_mapping=UpperCamelCase__, add_special_tokens=UpperCamelCase__     )
                  self.assertEqual(encoding.offset_mapping[0], (0, len(UpperCamelCase__     ))     )
                  self.assertEqual(
                      encoding.offset_mapping[1], (len(UpperCamelCase__     ) + 1, len(UpperCamelCase__     ) + 1 + len(UpperCamelCase__     )), )

                  _A    		=  f' {text}'

                  _A    		=  self.rust_tokenizer_class.from_pretrained(
                      UpperCamelCase__, use_fast=UpperCamelCase__, )
                  _A    		=  tokenizer_r(UpperCamelCase__, return_offsets_mapping=UpperCamelCase__, add_special_tokens=UpperCamelCase__     )
                  self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(UpperCamelCase__     ))     )
                  self.assertEqual(
                      encoding.offset_mapping[1], (1 + len(UpperCamelCase__     ) + 1, 1 + len(UpperCamelCase__     ) + 1 + len(UpperCamelCase__     )), )
   def 	__UpperCAmelCase   (      self      :			Optional[Any]     )  ->			Dict:
        # Test related to the breaking change introduced in transformers v4.17.0
        # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
        with self.assertRaises(UpperCamelCase__     ) as context:
             self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'     )

        self.assertTrue(
            context.exception.args[0].startswith(
                'The `backend_tokenizer` provided does not match the expected format.'     )     )
   @require_ftfy
   def 	__UpperCAmelCase   (      self      :			Any     )  ->			List[Any]:
        super().test_tokenization_python_rust_equals()

   def 	__UpperCAmelCase   (      self      :			str     )  ->			str:
        # CLIP always lower cases letters
        pass




'''simple docstring'''






import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


_UpperCAmelCase    :     Optional[int]         =       '''\
    Text data.
    Second line of data.'''

_UpperCAmelCase    :     Optional[Any]         =       '''file'''





@pytest.fixture(scope='session'				)
def  _SCREAMING_SNAKE_CASE		(      __snake_case							:			Any				):
     _A    		=  tmp_path_factory.mktemp('data'				) / (FILE_PATH + '.zstd')
     _A    		=  bytes(__snake_case   ,		'utf-8'				)
     with zstd.open(__snake_case   ,		'wb'				) as f:
          f.write(__snake_case				)
     return path





@pytest.fixture
def  _SCREAMING_SNAKE_CASE		(      __snake_case							:			Union[str, Any]				):
     with open(os.path.join(tmpfs.local_root_dir   ,		__snake_case				)   ,		'w'				) as f:
          f.write(__snake_case				)
     return FILE_PATH





@pytest.mark.parametrize('compression_format'   ,		['gzip', 'xz', 'zstd']				)
def  _SCREAMING_SNAKE_CASE		(      __snake_case							:			Dict   ,		__snake_case							:			str   ,		__snake_case							:			Any   ,		__snake_case							:			Any   ,		__snake_case							:			List[str]   ,		__snake_case							:			List[str]				):
     _A    		=  {'gzip': gz_file, 'xz': xz_file, 'zstd': zstd_path}
     _A    		=  input_paths[compression_format]
     _A    		=  tmp_path / 'cache'
     _A    		=  DownloadConfig(cache_dir=__snake_case   ,		extract_compressed_file=__snake_case				)
     _A    		=  cached_path(__snake_case   ,		download_config=__snake_case				)
     with open(__snake_case				) as f:
          _A    		=  f.read()
     with open(__snake_case				) as f:
          _A    		=  f.read()
     assert extracted_file_content == expected_file_content





@pytest.mark.parametrize('default_extracted'   ,		[True, False]				)
@pytest.mark.parametrize('default_cache_dir'   ,		[True, False]				)
def  _SCREAMING_SNAKE_CASE		(      __snake_case							:			Union[str, Any]   ,		__snake_case							:			List[str]   ,		__snake_case							:			Union[str, Any]   ,		__snake_case							:			int   ,		__snake_case							:			List[str]				):
     _A    		=  'custom_cache'
     _A    		=  'custom_extracted_dir'
     _A    		=  tmp_path / 'custom_extracted_path'
     if default_extracted:
          _A    		=  ('downloads' if default_cache_dir else custom_cache_dir, 'extracted')
     else:
          monkeypatch.setattr('datasets.config.EXTRACTED_DATASETS_DIR'   ,		__snake_case				)
          monkeypatch.setattr('datasets.config.EXTRACTED_DATASETS_PATH'   ,		str(__snake_case				)				)
          _A    		=  custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

     _A    		=  xz_file
     _A    		=  (
         DownloadConfig(extract_compressed_file=__snake_case				)
         if default_cache_dir
         else DownloadConfig(cache_dir=tmp_path / custom_cache_dir   ,		extract_compressed_file=__snake_case				)
     )
     _A    		=  cached_path(__snake_case   ,		download_config=__snake_case				)
     assert Path(__snake_case				).parent.parts[-2:] == expected





def  _SCREAMING_SNAKE_CASE		(      __snake_case							:			str				):
     # absolute path
     _A    		=  str(Path(__snake_case				).resolve()				)
     assert cached_path(__snake_case				) == text_file
     # relative path
     _A    		=  str(Path(__snake_case				).resolve().relative_to(Path(os.getcwd()				)				)				)
     assert cached_path(__snake_case				) == text_file





def  _SCREAMING_SNAKE_CASE		(      __snake_case							:			List[str]				):
     # absolute path
     _A    		=  str(tmp_path.resolve() / '__missing_file__.txt'				)
     with pytest.raises(__snake_case				):
          cached_path(__snake_case				)
     # relative path
     _A    		=  './__missing_file__.txt'
     with pytest.raises(__snake_case				):
          cached_path(__snake_case				)





def  _SCREAMING_SNAKE_CASE		(      __snake_case							:			int				):
     _A    		=  get_from_cache(F'tmp://{tmpfs_file}'				)
     with open(__snake_case				) as f:
          _A    		=  f.read()
     assert output_file_content == FILE_CONTENT





@patch('datasets.config.HF_DATASETS_OFFLINE'   ,		__snake_case				)
def  _SCREAMING_SNAKE_CASE		(      ):
     with pytest.raises(__snake_case				):
          cached_path('https://huggingface.co'				)





@patch('datasets.config.HF_DATASETS_OFFLINE'   ,		__snake_case				)
def  _SCREAMING_SNAKE_CASE		(      __snake_case							:			Tuple				):
     _A    		=  tmp_path_factory.mktemp('data'				) / 'file.html'
     with pytest.raises(__snake_case				):
          http_get('https://huggingface.co'   ,		temp_file=__snake_case				)
     with pytest.raises(__snake_case				):
          http_head('https://huggingface.co'				)





@patch('datasets.config.HF_DATASETS_OFFLINE'   ,		__snake_case				)
def  _SCREAMING_SNAKE_CASE		(      __snake_case							:			Optional[int]				):
     _A    		=  tmp_path_factory.mktemp('data'				) / 'file.html'
     with pytest.raises(__snake_case				):
          ftp_get('ftp://huggingface.co'   ,		temp_file=__snake_case				)
     with pytest.raises(__snake_case				):
          ftp_head('ftp://huggingface.co'				)





@patch('datasets.config.HF_DATASETS_OFFLINE'   ,		__snake_case				)
def  _SCREAMING_SNAKE_CASE		(      __snake_case							:			List[str]				):
     _A    		=  tmp_path_factory.mktemp('data'				) / 'file.html'
     with pytest.raises(__snake_case				):
          fsspec_get('s3://huggingface.co'   ,		temp_file=__snake_case				)
     with pytest.raises(__snake_case				):
          fsspec_head('s3://huggingface.co'				)



