

import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin

class     __snake_case    (				snake_case__			,	unittest.TestCase  ):
       """simple docstring"""







       __SCREAMING_SNAKE_CASE				    =							CanineTokenizer
       __SCREAMING_SNAKE_CASE				    =							False


       def 			a		(		self   )      ->							Union[str, Any]:






              """simple docstring"""


              super().setUp()
              __snake_case       	=							CanineTokenizer()
              tokenizer.save_pretrained(self.tmpdirname   )


       @cached_property
       def 			a		(		self   )      ->							Any:






              """simple docstring"""


              return CanineTokenizer.from_pretrained("""google/canine-s"""   )


       def 			a		(		self      , **_UpperCamelCase   )      ->							CanineTokenizer:






              """simple docstring"""


              __snake_case       	=							self.tokenizer_class.from_pretrained(self.tmpdirname      , **_UpperCamelCase   )
              __snake_case       	=							10_24
              return tokenizer


       @require_torch
       def 			a		(		self   )      ->							Dict:






              """simple docstring"""


              __snake_case       	=							self.canine_tokenizer
              __snake_case       	=							["""Life is like a box of chocolates.""", """You never know what you're gonna get."""]
              # fmt: off
              __snake_case       	=							[5_73_44, 76, 1_05, 1_02, 1_01, 32, 1_05, 1_15, 32, 1_08, 1_05, 1_07, 1_01, 32, 97, 32, 98, 1_11, 1_20, 32, 1_11, 1_02, 32, 99, 1_04, 1_11, 99, 1_11, 1_08, 97, 1_16, 1_01, 1_15, 46, 5_73_45, 0, 0, 0, 0]
              # fmt: on
              __snake_case       	=							tokenizer(_UpperCamelCase      , padding=_UpperCamelCase      , return_tensors="""pt"""   )
              self.assertIsInstance(_UpperCamelCase      , _UpperCamelCase   )

              __snake_case       	=							list(batch.input_ids.numpy()[0]   )

              self.assertListEqual(_UpperCamelCase      , _UpperCamelCase   )

              self.assertEqual((2, 39)      , batch.input_ids.shape   )
              self.assertEqual((2, 39)      , batch.attention_mask.shape   )


       @require_torch
       def 			a		(		self   )      ->							Tuple:






              """simple docstring"""


              __snake_case       	=							self.canine_tokenizer
              __snake_case       	=							["""Once there was a man.""", """He wrote a test in HuggingFace Tranformers."""]
              __snake_case       	=							tokenizer(_UpperCamelCase      , padding=_UpperCamelCase      , return_tensors="""pt"""   )
              # check if input_ids, attention_mask and token_type_ids are returned
              self.assertIn("""input_ids"""      , _UpperCamelCase   )
              self.assertIn("""attention_mask"""      , _UpperCamelCase   )
              self.assertIn("""token_type_ids"""      , _UpperCamelCase   )


       @require_torch
       def 			a		(		self   )      ->							Optional[int]:






              """simple docstring"""


              __snake_case       	=							self.canine_tokenizer
              __snake_case       	=							[
                  """What's the weater?""",
                  """It's about 25 degrees.""",
              ]
              __snake_case       	=							tokenizer(
                  text_target=_UpperCamelCase      , max_length=32      , padding="""max_length"""      , truncation=_UpperCamelCase      , return_tensors="""pt"""   )
              self.assertEqual(32      , targets["""input_ids"""].shape[1]   )


       def 			a		(		self   )      ->							Optional[Any]:






              """simple docstring"""


              __snake_case       	=							self.get_tokenizers()
              for tokenizer in tokenizers:
                     with self.subTest(F'{tokenizer.__class__.__name__}'   ):
                            self.assertNotEqual(tokenizer.model_max_length      , 42   )

        # Now let's start the test
              __snake_case       	=							self.get_tokenizers()
              for tokenizer in tokenizers:
                     with self.subTest(F'{tokenizer.__class__.__name__}'   ):
                            # Isolate this from the other tests because we save additional tokens/etc
                            __snake_case       	=							tempfile.mkdtemp()

                            __snake_case       	=							""" He is very happy, UNwant\u00E9d,running"""
                            __snake_case       	=							tokenizer.encode(_UpperCamelCase      , add_special_tokens=_UpperCamelCase   )
                            tokenizer.save_pretrained(_UpperCamelCase   )

                            __snake_case       	=							tokenizer.__class__.from_pretrained(_UpperCamelCase   )
                            __snake_case       	=							after_tokenizer.encode(_UpperCamelCase      , add_special_tokens=_UpperCamelCase   )
                            self.assertListEqual(_UpperCamelCase      , _UpperCamelCase   )

                            shutil.rmtree(_UpperCamelCase   )

              __snake_case       	=							self.get_tokenizers(model_max_length=42   )
              for tokenizer in tokenizers:
                     with self.subTest(F'{tokenizer.__class__.__name__}'   ):
                            # Isolate this from the other tests because we save additional tokens/etc
                            __snake_case       	=							tempfile.mkdtemp()

                            __snake_case       	=							""" He is very happy, UNwant\u00E9d,running"""

                            __snake_case       	=							tokenizer.additional_special_tokens

                            # We can add a new special token for Canine as follows:
                            __snake_case       	=							chr(0xE0_07   )
                            additional_special_tokens.append(_UpperCamelCase   )
                            tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}   )
                            __snake_case       	=							tokenizer.encode(_UpperCamelCase      , add_special_tokens=_UpperCamelCase   )
                            tokenizer.save_pretrained(_UpperCamelCase   )

                            __snake_case       	=							tokenizer.__class__.from_pretrained(_UpperCamelCase   )
                            __snake_case       	=							after_tokenizer.encode(_UpperCamelCase      , add_special_tokens=_UpperCamelCase   )
                            self.assertListEqual(_UpperCamelCase      , _UpperCamelCase   )
                            self.assertIn(_UpperCamelCase      , after_tokenizer.additional_special_tokens   )
                            self.assertEqual(after_tokenizer.model_max_length      , 42   )

                            __snake_case       	=							tokenizer.__class__.from_pretrained(_UpperCamelCase      , model_max_length=43   )
                            self.assertEqual(tokenizer.model_max_length      , 43   )

                            shutil.rmtree(_UpperCamelCase   )


       def 			a		(		self   )      ->							str:






              """simple docstring"""


              __snake_case       	=							self.get_tokenizers(do_lower_case=_UpperCamelCase   )
              for tokenizer in tokenizers:
                     with self.subTest(F'{tokenizer.__class__.__name__}'   ):
                            __snake_case	,    __snake_case       	=							self.get_clean_sequence(_UpperCamelCase   )

                            # a special token for Canine can be defined as follows:
                            __snake_case       	=							0xE0_05
                            __snake_case       	=							chr(_UpperCamelCase   )

                            tokenizer.add_special_tokens({"""cls_token""": special_token}   )
                            __snake_case       	=							tokenizer.encode(_UpperCamelCase      , add_special_tokens=_UpperCamelCase   )
                            self.assertEqual(len(_UpperCamelCase   )      , 1   )

                            __snake_case       	=							tokenizer.decode(ids + encoded_special_token      , clean_up_tokenization_spaces=_UpperCamelCase   )
                            __snake_case       	=							tokenizer.encode(_UpperCamelCase      , add_special_tokens=_UpperCamelCase   )

                            __snake_case       	=							tokenizer.encode(_UpperCamelCase      , add_special_tokens=_UpperCamelCase   )
                            __snake_case       	=							tokenizer.encode(_UpperCamelCase      , add_special_tokens=_UpperCamelCase   )
                            self.assertEqual(_UpperCamelCase      , input_encoded + special_token_id   )

                            __snake_case       	=							tokenizer.decode(_UpperCamelCase      , skip_special_tokens=_UpperCamelCase   )
                            self.assertTrue(special_token not in decoded   )


       def 			a		(		self   )      ->							Tuple:






              """simple docstring"""


              __snake_case       	=							self.get_tokenizers(do_lower_case=_UpperCamelCase   )
              for tokenizer in tokenizers:
                     with self.subTest(F'{tokenizer.__class__.__name__}'   ):
                            __snake_case       	=							chr(0xE0_05   )
                            __snake_case       	=							chr(0xE0_06   )

                            # `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
                            tokenizer.add_tokens([SPECIAL_TOKEN_1]      , special_tokens=_UpperCamelCase   )
                            # `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
                            # which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
                            tokenizer.add_special_tokens({"""additional_special_tokens""": [SPECIAL_TOKEN_2]}   )

                            __snake_case       	=							tokenizer.tokenize(_UpperCamelCase   )
                            __snake_case       	=							tokenizer.tokenize(_UpperCamelCase   )

                            self.assertEqual(len(_UpperCamelCase   )      , 1   )
                            self.assertEqual(len(_UpperCamelCase   )      , 1   )
                            self.assertEqual(token_a[0]      , _UpperCamelCase   )
                            self.assertEqual(token_a[0]      , _UpperCamelCase   )


       @require_tokenizers
       def 			a		(		self   )      ->							Tuple:






              """simple docstring"""


              __snake_case       	=							self.get_tokenizers(do_lower_case=_UpperCamelCase   )
              for tokenizer in tokenizers:
                     with self.subTest(F'{tokenizer.__class__.__name__}'   ):
                            # a special token for Canine can be defined as follows:
                            __snake_case       	=							0xE0_06
                            __snake_case       	=							chr(_UpperCamelCase   )

                            __snake_case       	=							AddedToken(_UpperCamelCase      , lstrip=_UpperCamelCase   )
                            tokenizer.add_special_tokens({"""additional_special_tokens""": [new_token]}   )

                            with tempfile.TemporaryDirectory() as tmp_dir_name:
                                   tokenizer.save_pretrained(_UpperCamelCase   )
                                   tokenizer.from_pretrained(_UpperCamelCase   )


       def 			a		(		self   )      ->							Optional[int]:






              """simple docstring"""


              __snake_case       	=							[]
              if self.test_slow_tokenizer:
                     tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())   )

              if self.test_rust_tokenizer:
                     tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())   )

              for tokenizer_class, tokenizer_utils in tokenizer_list:
                     with tempfile.TemporaryDirectory() as tmp_dir:
                            tokenizer_utils.save_pretrained(_UpperCamelCase   )

                            with open(os.path.join(_UpperCamelCase      , """special_tokens_map.json"""   )      , encoding="""utf-8"""   ) as json_file:
                                   __snake_case       	=							json.load(_UpperCamelCase   )

                            with open(os.path.join(_UpperCamelCase      , """tokenizer_config.json"""   )      , encoding="""utf-8"""   ) as json_file:
                                   __snake_case       	=							json.load(_UpperCamelCase   )

                            # a special token for Canine can be defined as follows:
                            __snake_case       	=							0xE0_06
                            __snake_case       	=							chr(_UpperCamelCase   )

                            __snake_case       	=							[new_token_a]
                            __snake_case       	=							[new_token_a]

                            with open(os.path.join(_UpperCamelCase      , """special_tokens_map.json"""   )      , """w"""      , encoding="""utf-8"""   ) as outfile:
                                   json.dump(_UpperCamelCase      , _UpperCamelCase   )
                            with open(os.path.join(_UpperCamelCase      , """tokenizer_config.json"""   )      , """w"""      , encoding="""utf-8"""   ) as outfile:
                                   json.dump(_UpperCamelCase      , _UpperCamelCase   )

                            # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                            # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                            # "special_tokens_map.json" files
                            __snake_case       	=							tokenizer_class.from_pretrained(_UpperCamelCase      , extra_ids=0   )
                            self.assertIn(_UpperCamelCase      , tokenizer_without_change_in_init.additional_special_tokens   )
                            # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                            self.assertEqual(
                                [new_token_a]      , tokenizer_without_change_in_init.convert_ids_to_tokens(
                                    tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]   )   )      , )

                            __snake_case       	=							0xE0_07
                            __snake_case       	=							chr(_UpperCamelCase   )
                            # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                            __snake_case       	=							[AddedToken(_UpperCamelCase      , lstrip=_UpperCamelCase   )]
                            __snake_case       	=							tokenizer_class.from_pretrained(
                                _UpperCamelCase      , additional_special_tokens=_UpperCamelCase      , extra_ids=0   )

                            self.assertIn(_UpperCamelCase      , tokenizer.additional_special_tokens   )
                            # self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
                            self.assertEqual(
                                [new_token_a]      , tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]   )   )   )


       @require_tokenizers
       def 			a		(		self   )      ->							int:






              """simple docstring"""


              __snake_case       	=							self.get_tokenizers(do_lower_case=_UpperCamelCase   )
              for tokenizer in tokenizers:
                     with self.subTest(F'{tokenizer.__class__.__name__}'   ):
                            __snake_case       	=							"""hello world"""
                            if self.space_between_special_tokens:
                                   __snake_case       	=							"""[CLS] hello world [SEP]"""
                            else:
                                   __snake_case       	=							input
                            __snake_case       	=							tokenizer.encode(_UpperCamelCase      , add_special_tokens=_UpperCamelCase   )
                            __snake_case       	=							tokenizer.decode(_UpperCamelCase      , spaces_between_special_tokens=self.space_between_special_tokens   )
                            self.assertIn(_UpperCamelCase      , [output, output.lower()]   )


       def 			a		(		self   )      ->							Dict:






              """simple docstring"""


              __snake_case       	=							self.get_tokenizers()
              for tokenizer in tokenizers:
                     with self.subTest(F'{tokenizer.__class__.__name__}'   ):
                            __snake_case       	=							[
                                """bos_token""",
                                """eos_token""",
                                """unk_token""",
                                """sep_token""",
                                """pad_token""",
                                """cls_token""",
                                """mask_token""",
                            ]

              __snake_case       	=							"""a"""
              __snake_case       	=							ord(_UpperCamelCase   )

              for attr in attributes_list:
                     setattr(_UpperCamelCase      , attr + """_id"""      , _UpperCamelCase   )
                     self.assertEqual(getattr(_UpperCamelCase      , _UpperCamelCase   )      , _UpperCamelCase   )
                     self.assertEqual(getattr(_UpperCamelCase      , attr + """_id"""   )      , _UpperCamelCase   )

                     setattr(_UpperCamelCase      , attr + """_id"""      , _UpperCamelCase   )
                     self.assertEqual(getattr(_UpperCamelCase      , _UpperCamelCase   )      , _UpperCamelCase   )
                     self.assertEqual(getattr(_UpperCamelCase      , attr + """_id"""   )      , _UpperCamelCase   )

              setattr(_UpperCamelCase      , """additional_special_tokens_ids"""      , []   )
              self.assertListEqual(getattr(_UpperCamelCase      , """additional_special_tokens"""   )      , []   )
              self.assertListEqual(getattr(_UpperCamelCase      , """additional_special_tokens_ids"""   )      , []   )

              __snake_case       	=							0xE0_06
              __snake_case       	=							chr(_UpperCamelCase   )
              setattr(_UpperCamelCase      , """additional_special_tokens_ids"""      , [additional_special_token_id]   )
              self.assertListEqual(getattr(_UpperCamelCase      , """additional_special_tokens"""   )      , [additional_special_token]   )
              self.assertListEqual(getattr(_UpperCamelCase      , """additional_special_tokens_ids"""   )      , [additional_special_token_id]   )


       def 			a		(		self   )      ->							Optional[int]:






              """simple docstring"""


              pass


       def 			a		(		self   )      ->							List[str]:






              """simple docstring"""


              pass


       def 			a		(		self   )      ->							str:






              """simple docstring"""


              pass


       def 			a		(		self   )      ->							str:






              """simple docstring"""


              pass


       def 			a		(		self   )      ->							Tuple:






              """simple docstring"""


              pass


       def 			a		(		self   )      ->							Any:






              """simple docstring"""


              pass


       def 			a		(		self   )      ->							Dict:






              """simple docstring"""


              pass






       def 			a		(		self   )      ->							List[Any]:






              """simple docstring"""


              pass




# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import importlib.metadata
import json
import os
from dataclasses import dataclass
from typing import Any, Dict, Union

from packaging import version

from ..utils import is_torch_available, logging


if is_torch_available():
    import torch


UpperCamelCase__		 =     logging.get_logger(__name__)

@dataclass
class     __snake_case    :
       """simple docstring"""







       def __init__(		self      , _UpperCamelCase=False      , _UpperCamelCase=False      , _UpperCamelCase=6.0      , _UpperCamelCase=None      , _UpperCamelCase=False      , _UpperCamelCase=False      , _UpperCamelCase=None      , _UpperCamelCase="fp4"      , _UpperCamelCase=False      , **_UpperCamelCase      , )      ->							int:






              """simple docstring"""


              __snake_case       	=							load_in_abit
              __snake_case       	=							load_in_abit
              __snake_case       	=							llm_inta_threshold
              __snake_case       	=							llm_inta_skip_modules
              __snake_case       	=							llm_inta_enable_fpaa_cpu_offload
              __snake_case       	=							llm_inta_has_fpaa_weight
              __snake_case       	=							bnb_abit_quant_type
              __snake_case       	=							bnb_abit_use_double_quant

              if bnb_abit_compute_dtype is None:
                     __snake_case       	=							torch.floataa
              elif isinstance(_UpperCamelCase      , _UpperCamelCase   ):
                     __snake_case       	=							getattr(_UpperCamelCase      , _UpperCamelCase   )
              elif isinstance(_UpperCamelCase      , torch.dtype   ):
                     __snake_case       	=							bnb_abit_compute_dtype
              else:
                     raise ValueError("""bnb_4bit_compute_dtype must be a string or a torch.dtype"""   )

              self.post_init()


       def 			a		(		self   )      ->							Union[str, Any]:






              """simple docstring"""


              if not isinstance(self.llm_inta_threshold      , _UpperCamelCase   ):
                     raise ValueError("""llm_int8_threshold must be a float"""   )

              if self.llm_inta_skip_modules is not None and not isinstance(self.llm_inta_skip_modules      , _UpperCamelCase   ):
                     raise ValueError("""llm_int8_skip_modules must be a list of strings"""   )
              if not isinstance(self.llm_inta_enable_fpaa_cpu_offload      , _UpperCamelCase   ):
                     raise ValueError("""llm_int8_enable_fp32_cpu_offload must be a boolean"""   )

              if not isinstance(self.llm_inta_has_fpaa_weight      , _UpperCamelCase   ):
                     raise ValueError("""llm_int8_has_fp16_weight must be a boolean"""   )

              if self.bnb_abit_compute_dtype is not None and not isinstance(self.bnb_abit_compute_dtype      , torch.dtype   ):
                     raise ValueError("""bnb_4bit_compute_dtype must be torch.dtype"""   )

              if not isinstance(self.bnb_abit_quant_type      , _UpperCamelCase   ):
                     raise ValueError("""bnb_4bit_quant_type must be a string"""   )

              if not isinstance(self.bnb_abit_use_double_quant      , _UpperCamelCase   ):
                     raise ValueError("""bnb_4bit_use_double_quant must be a boolean"""   )

              if self.load_in_abit and not version.parse(importlib.metadata.version("""bitsandbytes"""   )   ) >= version.parse(
                  """0.39.0"""   ):
                     raise ValueError(
                         """4 bit quantization requires bitsandbytes>=0.39.0 - please upgrade your bitsandbytes version"""   )


       def 			a		(		self   )      ->							int:






              """simple docstring"""


              return self.load_in_abit or self.load_in_abit


       def 			a		(		self   )      ->							int:






              """simple docstring"""


              if self.load_in_abit:
                     return "llm_int8"
              elif self.load_in_abit and self.bnb_abit_quant_type == "fp4":
                     return "fp4"
              elif self.load_in_abit and self.bnb_abit_quant_type == "nf4":
                     return "nf4"
              else:
                     return None


       @classmethod
       def 			a		(		cls      , _UpperCamelCase      , _UpperCamelCase      , **_UpperCamelCase   )      ->							Union[str, Any]:






              """simple docstring"""


              __snake_case       	=							cls(**_UpperCamelCase   )

              __snake_case       	=							[]
              for key, value in kwargs.items():
                     if hasattr(_UpperCamelCase      , _UpperCamelCase   ):
                            setattr(_UpperCamelCase      , _UpperCamelCase      , _UpperCamelCase   )
                            to_remove.append(_UpperCamelCase   )
              for key in to_remove:
                     kwargs.pop(_UpperCamelCase      , _UpperCamelCase   )

              if return_unused_kwargs:
                     return config, kwargs
              else:
                     return config


       def 			a		(		self      , _UpperCamelCase   )      ->							int:






              """simple docstring"""


              with open(_UpperCamelCase      , """w"""      , encoding="""utf-8"""   ) as writer:
                     __snake_case       	=							self.to_dict()
                     __snake_case       	=							json.dumps(_UpperCamelCase      , indent=2      , sort_keys=_UpperCamelCase   ) + """\n"""

                     writer.write(_UpperCamelCase   )


       def 			a		(		self   )      ->							Dict[str, Any]:






              """simple docstring"""


              __snake_case       	=							copy.deepcopy(self.__dict__   )
              __snake_case       	=							str(output["""bnb_4bit_compute_dtype"""]   ).split("""."""   )[1]

              return output


       def __repr__(		self   )      ->							Dict:






              """simple docstring"""


              return F'{self.__class__.__name__} {self.to_json_string()}'


       def 			a		(		self      , _UpperCamelCase = True   )      ->							str:






              """simple docstring"""


              if use_diff is True:
                     __snake_case       	=							self.to_diff_dict()
              else:
                     __snake_case       	=							self.to_dict()
              return json.dumps(_UpperCamelCase      , indent=2      , sort_keys=_UpperCamelCase   ) + "\n"






       def 			a		(		self   )      ->							Dict[str, Any]:






              """simple docstring"""


              __snake_case       	=							self.to_dict()

              # get the default config dict
              __snake_case       	=							BitsAndBytesConfig().to_dict()

              __snake_case       	=							{}

              # only serialize values that differ from the default config
              for key, value in config_dict.items():
                     if value != default_config_dict[key]:
                            __snake_case       	=							value

              return serializable_config_dict

