

import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class  _A		(  a__   ,     unittest.TestCase):
  SCREAMING_SNAKE_CASE    :      str       =  CLIPTokenizer
  SCREAMING_SNAKE_CASE    :      Tuple       =  CLIPTokenizerFast
  SCREAMING_SNAKE_CASE    :      List[str]       =  True
  SCREAMING_SNAKE_CASE    :      int       =  {}
  SCREAMING_SNAKE_CASE    :      int       =  False

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        super().setUp()

        # fmt: off
        SCREAMING_SNAKE_CASE_				:  Dict           =     ["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
        # fmt: on
        SCREAMING_SNAKE_CASE_				:  Any           =     dict(zip(lowerCAmelCase__ ,					range(len(lowerCAmelCase__					)					)					)					)
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     ["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
        SCREAMING_SNAKE_CASE_				:  List[str]           =     {"unk_token": "<unk>"}

        SCREAMING_SNAKE_CASE_				:  Dict           =     os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES['vocab_file']					)
        SCREAMING_SNAKE_CASE_				:  List[str]           =     os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES['merges_file']					)
        with open(self.vocab_file ,					'w' ,					encoding='utf-8'					) as fp:
              fp.write(json.dumps(lowerCAmelCase__					) + '\n'					)
        with open(self.merges_file ,					'w' ,					encoding='utf-8'					) as fp:
              fp.write('\n'.join(lowerCAmelCase__					)					)

  def  UpperCAmelCase    (				self ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        kwargs.update(self.special_tokens_map					)
        return CLIPTokenizer.from_pretrained(self.tmpdirname ,					**lowerCAmelCase__					)

  def  UpperCAmelCase    (				self ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        kwargs.update(self.special_tokens_map					)
        return CLIPTokenizerFast.from_pretrained(self.tmpdirname ,					**lowerCAmelCase__					)

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  List[Any]           =     "lower newer"
        SCREAMING_SNAKE_CASE_				:  int           =     "lower newer"
        return input_text, output_text

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     CLIPTokenizer(self.vocab_file ,					self.merges_file ,					**self.special_tokens_map					)
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     "lower newer"
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     ["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
        SCREAMING_SNAKE_CASE_				:  List[str]           =     tokenizer.tokenize(lowerCAmelCase__					)
        self.assertListEqual(lowerCAmelCase__ ,					lowerCAmelCase__					)

        SCREAMING_SNAKE_CASE_				:  List[Any]           =     tokens + [tokenizer.unk_token]
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     [10, 2, 16, 9, 3, 2, 16, 20]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__					) ,					lowerCAmelCase__					)

  @require_ftfy
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"					):
                    SCREAMING_SNAKE_CASE_				:  List[str]           =     self.tokenizer_class.from_pretrained(lowerCAmelCase__ ,					**lowerCAmelCase__					)
                    SCREAMING_SNAKE_CASE_				:  List[Any]           =     self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__ ,					**lowerCAmelCase__					)

                    SCREAMING_SNAKE_CASE_				:  Any           =     "A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
                    SCREAMING_SNAKE_CASE_				:  str           =     tokenizer_s.tokenize(lowerCAmelCase__					)
                    SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     tokenizer_r.tokenize(lowerCAmelCase__					)

                    self.assertListEqual(lowerCAmelCase__ ,					lowerCAmelCase__					)

                    # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                    # with Tilde) encoded in 2 different ways
                    SCREAMING_SNAKE_CASE_				:  List[Any]           =     "xa\u0303y" + " " + "x\xe3y"
                    SCREAMING_SNAKE_CASE_				:  str           =     tokenizer_s.tokenize(lowerCAmelCase__					)
                    SCREAMING_SNAKE_CASE_				:  Optional[int]           =     tokenizer_r.tokenize(lowerCAmelCase__					)

                    self.assertListEqual(lowerCAmelCase__ ,					lowerCAmelCase__					)

                    # Test that the tokenization is identical on unicode of space type
                    SCREAMING_SNAKE_CASE_				:  Any           =     [
                        "\u0009",  # (horizontal tab, '\t')
                        "\u000B",  # (vertical tab)
                        "\u000C",  # (form feed)
                        "\u0020",  # (space, ' ')
                        "\u200E",  # (left-to-right mark):w
                        "\u200F",  # (right-to-left mark)
                    ]
                    for unicode_seq in spaces_unicodes:
                          SCREAMING_SNAKE_CASE_				:  Any           =     tokenizer_s.tokenize(lowerCAmelCase__					)
                          SCREAMING_SNAKE_CASE_				:  Dict           =     tokenizer_r.tokenize(lowerCAmelCase__					)

                          self.assertListEqual(lowerCAmelCase__ ,					lowerCAmelCase__					)

                    # Test that the tokenization is identical on unicode of line break type
                    SCREAMING_SNAKE_CASE_				:  Any           =     [
                        "\u000A",  # (line feed, '\n')
                        "\r\n",  # (carriage return and line feed, '\r\n')
                        "\u000D",  # (carriage return, '\r')
                        "\r",  # (carriage return, '\r')
                        "\u000D",  # (carriage return, '\r')
                        "\u2028",  # (line separator)
                        "\u2029",  # (paragraph separator)
                        # "\u0085", # (next line)
                    ]

                    # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                    # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                    # space (and thus into an empty list).

                    for unicode_seq in line_break_unicodes:
                          SCREAMING_SNAKE_CASE_				:  List[str]           =     tokenizer_s.tokenize(lowerCAmelCase__					)
                          SCREAMING_SNAKE_CASE_				:  Tuple           =     tokenizer_r.tokenize(lowerCAmelCase__					)

                          self.assertListEqual(lowerCAmelCase__ ,					lowerCAmelCase__					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"					):
                    SCREAMING_SNAKE_CASE_				:  Tuple           =     "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
                    SCREAMING_SNAKE_CASE_				:  Tuple           =     f"{text_of_1_token} {text_of_1_token}"

                    SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     self.rust_tokenizer_class.from_pretrained(
                        lowerCAmelCase__ ,					use_fast=lowerCAmelCase__ ,					)
                    SCREAMING_SNAKE_CASE_				:  str           =     tokenizer_r(lowerCAmelCase__ ,					return_offsets_mapping=lowerCAmelCase__ ,					add_special_tokens=lowerCAmelCase__					)
                    self.assertEqual(encoding.offset_mapping[0] ,					(0, len(lowerCAmelCase__					))					)
                    self.assertEqual(
                        encoding.offset_mapping[1] ,					(len(lowerCAmelCase__					) + 1, len(lowerCAmelCase__					) + 1 + len(lowerCAmelCase__					)) ,					)

                    SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     f" {text}"

                    SCREAMING_SNAKE_CASE_				:  Optional[int]           =     self.rust_tokenizer_class.from_pretrained(
                        lowerCAmelCase__ ,					use_fast=lowerCAmelCase__ ,					)
                    SCREAMING_SNAKE_CASE_				:  Any           =     tokenizer_r(lowerCAmelCase__ ,					return_offsets_mapping=lowerCAmelCase__ ,					add_special_tokens=lowerCAmelCase__					)
                    self.assertEqual(encoding.offset_mapping[0] ,					(1, 1 + len(lowerCAmelCase__					))					)
                    self.assertEqual(
                        encoding.offset_mapping[1] ,					(1 + len(lowerCAmelCase__					) + 1, 1 + len(lowerCAmelCase__					) + 1 + len(lowerCAmelCase__					)) ,					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        with self.assertRaises(lowerCAmelCase__					) as context:
              self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'					)

        self.assertTrue(
            context.exception.args[0].startswith(
                'The `backend_tokenizer` provided does not match the expected format.'					)					)

  @require_ftfy
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        super().test_tokenization_python_rust_equals()



  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        pass






# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import importlib.metadata
import json
import os
from dataclasses import dataclass
from typing import Any, Dict, Union

from packaging import version

from ..utils import is_torch_available, logging


if is_torch_available():
    import torch


lowerCAmelCase     :   Optional[int]						     =  logging.get_logger(__name__)
@dataclass
class  _A		:

  def __init__(				self ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE=6.0 ,					_SCREAMING_SNAKE_CASE=None ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE=None ,					_SCREAMING_SNAKE_CASE="fp4" ,					_SCREAMING_SNAKE_CASE=False ,					**_SCREAMING_SNAKE_CASE ,					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  str           =     load_in_abit
        SCREAMING_SNAKE_CASE_				:  Tuple           =     load_in_abit
        SCREAMING_SNAKE_CASE_				:  Dict           =     llm_inta_threshold
        SCREAMING_SNAKE_CASE_				:  Tuple           =     llm_inta_skip_modules
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     llm_inta_enable_fpaa_cpu_offload
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     llm_inta_has_fpaa_weight
        SCREAMING_SNAKE_CASE_				:  List[str]           =     bnb_abit_quant_type
        SCREAMING_SNAKE_CASE_				:  int           =     bnb_abit_use_double_quant

        if bnb_abit_compute_dtype is None:
              SCREAMING_SNAKE_CASE_				:  List[Any]           =     torch.floataa
        elif isinstance(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					):
              SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     getattr(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)
        elif isinstance(_SCREAMING_SNAKE_CASE ,					torch.dtype					):
              SCREAMING_SNAKE_CASE_				:  Optional[int]           =     bnb_abit_compute_dtype
        else:
              raise ValueError('bnb_4bit_compute_dtype must be a string or a torch.dtype'					)

        self.post_init()

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        if not isinstance(self.llm_inta_threshold ,					_SCREAMING_SNAKE_CASE					):
              raise ValueError('llm_int8_threshold must be a float'					)

        if self.llm_inta_skip_modules is not None and not isinstance(self.llm_inta_skip_modules ,					_SCREAMING_SNAKE_CASE					):
              raise ValueError('llm_int8_skip_modules must be a list of strings'					)
        if not isinstance(self.llm_inta_enable_fpaa_cpu_offload ,					_SCREAMING_SNAKE_CASE					):
              raise ValueError('llm_int8_enable_fp32_cpu_offload must be a boolean'					)

        if not isinstance(self.llm_inta_has_fpaa_weight ,					_SCREAMING_SNAKE_CASE					):
              raise ValueError('llm_int8_has_fp16_weight must be a boolean'					)

        if self.bnb_abit_compute_dtype is not None and not isinstance(self.bnb_abit_compute_dtype ,					torch.dtype					):
              raise ValueError('bnb_4bit_compute_dtype must be torch.dtype'					)

        if not isinstance(self.bnb_abit_quant_type ,					_SCREAMING_SNAKE_CASE					):
              raise ValueError('bnb_4bit_quant_type must be a string'					)

        if not isinstance(self.bnb_abit_use_double_quant ,					_SCREAMING_SNAKE_CASE					):
              raise ValueError('bnb_4bit_use_double_quant must be a boolean'					)

        if self.load_in_abit and not version.parse(importlib.metadata.version('bitsandbytes'					)					) >= version.parse(
            '0.39.0'					):
              raise ValueError(
                  '4 bit quantization requires bitsandbytes>=0.39.0 - please upgrade your bitsandbytes version'					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        return self.load_in_abit or self.load_in_abit

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        if self.load_in_abit:
              return "llm_int8"
        elif self.load_in_abit and self.bnb_abit_quant_type == "fp4":
              return "fp4"
        elif self.load_in_abit and self.bnb_abit_quant_type == "nf4":
              return "nf4"
        else:
              return None

  @classmethod
  def  UpperCAmelCase    (				cls ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     cls(**_SCREAMING_SNAKE_CASE					)

        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     []
        for key, value in kwargs.items():
              if hasattr(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					):
                    setattr(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)
                    to_remove.append(_SCREAMING_SNAKE_CASE					)
        for key in to_remove:
              kwargs.pop(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)

        if return_unused_kwargs:
              return config, kwargs
        else:
              return config

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        with open(_SCREAMING_SNAKE_CASE ,					'w' ,					encoding='utf-8'					) as writer:
              SCREAMING_SNAKE_CASE_				:  Optional[int]           =     self.to_dict()
              SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     json.dumps(_SCREAMING_SNAKE_CASE ,					indent=2 ,					sort_keys=_SCREAMING_SNAKE_CASE					) + '\n'

              writer.write(_SCREAMING_SNAKE_CASE					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  str           =     copy.deepcopy(self.__dict__					)
        SCREAMING_SNAKE_CASE_				:  str           =     str(output['bnb_4bit_compute_dtype']					).split('.'					)[1]

        return output

  def __repr__(				self					):





        """simple docstring"""







        return f"{self.__class__.__name__} {self.to_json_string()}"

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE = True					):





        """simple docstring"""







        if use_diff is True:
              SCREAMING_SNAKE_CASE_				:  int           =     self.to_diff_dict()
        else:
              SCREAMING_SNAKE_CASE_				:  List[str]           =     self.to_dict()
        return json.dumps(_SCREAMING_SNAKE_CASE ,					indent=2 ,					sort_keys=_SCREAMING_SNAKE_CASE					) + "\n"



  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  int           =     self.to_dict()

        # get the default config dict
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     BitsAndBytesConfig().to_dict()

        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     {}

        # only serialize values that differ from the default config
        for key, value in config_dict.items():
              if value != default_config_dict[key]:
                    SCREAMING_SNAKE_CASE_				:  int           =     value

        return serializable_config_dict






