






'''simple docstring'''



import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE							=						get_tests_dir('fixtures/test_sentencepiece.model')
@require_sentencepiece
@require_tokenizers
class UpperCAmelCase_       (     __A					,							unittest.TestCase					):




      """simple docstring"""




      UpperCamelCase_             =      ReformerTokenizer
      UpperCamelCase_             =      ReformerTokenizerFast
      UpperCamelCase_             =      True
      UpperCamelCase_             =      False
      UpperCamelCase_             =      True






      def 					A__   (			self      :		Dict		)	->	Tuple:


       '''simple docstring'''

       super().setUp()

       lowercase       : Optional[Any]						=ReformerTokenizer(UpperCAmelCase  ,   keep_accents=UpperCAmelCase		)
       tokenizer.save_pretrained(self.tmpdirname		)






      def 					A__   (			self      :		Optional[int]		)	->	Optional[Any]:


       '''simple docstring'''

       lowercase       : List[Any]						='''<s>'''
       lowercase       : List[Any]						=1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase		)  ,   UpperCAmelCase		)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase		)  ,   UpperCAmelCase		)






      def 					A__   (			self      :		Optional[Any]		)	->	Any:


       '''simple docstring'''

       lowercase       : Any						=list(self.get_tokenizer().get_vocab().keys()		)

       self.assertEqual(vocab_keys[0]  ,   '''<unk>'''		)
       self.assertEqual(vocab_keys[1]  ,   '''<s>'''		)
       self.assertEqual(vocab_keys[-1]  ,   '''j'''		)
       self.assertEqual(len(UpperCAmelCase		)  ,   1000		)






      def 					A__   (			self      :		Tuple		)	->	str:


       '''simple docstring'''

       self.assertEqual(self.get_tokenizer().vocab_size  ,   1000		)






      def 					A__   (			self      :		List[str]		)	->	Tuple:


       '''simple docstring'''

       if not self.test_rust_tokenizer:
        return

       lowercase       : List[str]						=self.get_tokenizer()
       lowercase       : int						=self.get_rust_tokenizer()

       lowercase       : Any						='''I was born in 92000, and this is falsé.'''

       lowercase       : Optional[Any]						=tokenizer.tokenize(UpperCAmelCase		)
       lowercase       : List[Any]						=rust_tokenizer.tokenize(UpperCAmelCase		)
       self.assertListEqual(UpperCAmelCase  ,   UpperCAmelCase		)

       lowercase       : List[Any]						=tokenizer.encode(UpperCAmelCase  ,   add_special_tokens=UpperCAmelCase		)
       lowercase       : Optional[int]						=rust_tokenizer.encode(UpperCAmelCase  ,   add_special_tokens=UpperCAmelCase		)
       self.assertListEqual(UpperCAmelCase  ,   UpperCAmelCase		)

       lowercase       : Optional[Any]						=self.get_rust_tokenizer()
       lowercase       : str						=tokenizer.encode(UpperCAmelCase		)
       lowercase       : Any						=rust_tokenizer.encode(UpperCAmelCase		)
       self.assertListEqual(UpperCAmelCase  ,   UpperCAmelCase		)






      def 					A__   (			self      :		Tuple  ,   UpperCAmelCase      :		int=15		)	->	Optional[int]:


       '''simple docstring'''

       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'		):
         lowercase       : Tuple						=self.rust_tokenizer_class.from_pretrained(UpperCAmelCase  ,   **UpperCAmelCase		)

         # Simple input
         lowercase       : List[str]						='''This is a simple input'''
         lowercase       : List[Any]						=['''This is a simple input 1''', '''This is a simple input 2''']
         lowercase       : Optional[int]						=('''This is a simple input''', '''This is a pair''')
         lowercase       : List[Any]						=[
             ('''This is a simple input 1''', '''This is a simple input 2'''),
             ('''This is a simple pair 1''', '''This is a simple pair 2'''),
         ]

         # Simple input tests
         self.assertRaises(UpperCAmelCase  ,   tokenizer_r.encode  ,   UpperCAmelCase  ,   max_length=UpperCAmelCase  ,   padding='''max_length'''		)

         # Simple input
         self.assertRaises(UpperCAmelCase  ,   tokenizer_r.encode_plus  ,   UpperCAmelCase  ,   max_length=UpperCAmelCase  ,   padding='''max_length'''		)

         # Simple input
         self.assertRaises(
             UpperCAmelCase  ,   tokenizer_r.batch_encode_plus  ,   UpperCAmelCase  ,   max_length=UpperCAmelCase  ,   padding='''max_length'''  ,   )

         # Pair input
         self.assertRaises(UpperCAmelCase  ,   tokenizer_r.encode  ,   UpperCAmelCase  ,   max_length=UpperCAmelCase  ,   padding='''max_length'''		)

         # Pair input
         self.assertRaises(UpperCAmelCase  ,   tokenizer_r.encode_plus  ,   UpperCAmelCase  ,   max_length=UpperCAmelCase  ,   padding='''max_length'''		)

         # Pair input
         self.assertRaises(
             UpperCAmelCase  ,   tokenizer_r.batch_encode_plus  ,   UpperCAmelCase  ,   max_length=UpperCAmelCase  ,   padding='''max_length'''  ,   )






      def 					A__   (			self      :		str		)	->	int:


       '''simple docstring'''

       pass






      def 					A__   (			self      :		List[Any]		)	->	Optional[Any]:


       '''simple docstring'''

       lowercase       : Any						=ReformerTokenizer(UpperCAmelCase  ,   keep_accents=UpperCAmelCase		)

       lowercase       : str						=tokenizer.tokenize('''This is a test'''		)
       self.assertListEqual(UpperCAmelCase  ,   ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']		)

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(UpperCAmelCase		)  ,   [285, 46, 10, 170, 382]  ,   )

       lowercase       : Any						=tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
       self.assertListEqual(
           UpperCAmelCase  ,   [
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''9''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''é''',
               '''.''',
           ]  ,   )
       lowercase       : Dict						=tokenizer.convert_tokens_to_ids(UpperCAmelCase		)
       self.assertListEqual(
           UpperCAmelCase  ,   [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]  ,   )

       lowercase       : str						=tokenizer.convert_ids_to_tokens(UpperCAmelCase		)
       self.assertListEqual(
           UpperCAmelCase  ,   [
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''<unk>''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''<unk>''',
               '''.''',
           ]  ,   )






      @cached_property
      def 					A__   (			self      :		str		)	->	Optional[int]:


       '''simple docstring'''

       return ReformerTokenizer.from_pretrained('''google/reformer-crime-and-punishment'''		)






      @slow
      def 					A__   (			self      :		str		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : Tuple						='''Hello World!'''
       lowercase       : Any						=[126, 32, 262, 152, 38, 72, 287]

       self.assertListEqual(UpperCAmelCase  ,   self.big_tokenizer.encode(UpperCAmelCase		)		)






      @slow
      def 					A__   (			self      :		Optional[Any]		)	->	Tuple:


       '''simple docstring'''

       lowercase       : str						=(
           '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
           ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
       )
       lowercase       : Union[str, Any]						=[
           108,
           265,
           24,
           111,
           4,
           258,
           156,
           35,
           28,
           275,
           3,
           259,
           297,
           260,
           84,
           4,
           35,
           110,
           44,
           8,
           259,
           91,
           268,
           21,
           11,
           209,
           274,
           109,
           266,
           277,
           117,
           86,
           93,
           315,
           258,
           278,
           258,
           277,
           258,
           0,
           258,
           288,
           258,
           319,
           258,
           0,
           258,
           0,
           258,
           0,
           258,
           0,
           258,
           287,
           258,
           315,
           258,
           289,
           258,
           278,
           99,
           269,
           266,
           262,
           8,
           259,
           241,
           4,
           217,
           230,
           268,
           266,
           55,
           168,
           106,
           75,
           193,
           266,
           223,
           27,
           49,
           26,
           282,
           25,
           264,
           299,
           19,
           26,
           0,
           258,
           277,
           117,
           86,
           93,
           176,
           183,
           270,
           11,
           262,
           42,
           61,
           265,
       ]

       self.assertListEqual(UpperCAmelCase  ,   self.big_tokenizer.encode(UpperCAmelCase		)		)






      @require_torch
      @slow
      def 					A__   (			self      :		Optional[Any]		)	->	List[Any]:


       '''simple docstring'''

       import torch

       from transformers import ReformerConfig, ReformerModel

       # Build sequence
       lowercase       : List[Any]						=list(self.big_tokenizer.get_vocab().keys()		)[:10]
       lowercase       : Union[str, Any]						=''' '''.join(UpperCAmelCase		)
       lowercase       : int						=self.big_tokenizer.encode_plus(UpperCAmelCase  ,   return_tensors='''pt'''		)
       lowercase       : List[str]						=self.big_tokenizer.batch_encode_plus([sequence, sequence]  ,   return_tensors='''pt'''		)

       lowercase       : Optional[Any]						=ReformerConfig()
       # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
       lowercase       : str						=encoded_sequence['''input_ids'''].shape
       lowercase       : Any						=ReformerModel(UpperCAmelCase		)

       # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
       assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

       with torch.no_grad():
        model(**UpperCAmelCase		)
        model(**UpperCAmelCase		)






      @slow
      def 					A__   (			self      :		int		)	->	Dict:


       '''simple docstring'''

       lowercase       : Optional[Any]						={'''input_ids''': [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
       # fmt: on

       # This tokenizer does not know some characters like ")".
       # That is the reason why we use very simple texts here.
       # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
       lowercase       : Optional[Any]						=[
           '''This is a very simple sentence.''',
           '''The quick brown fox jumps over the lazy dog.''',
       ]

       self.tokenizer_integration_test_util(
           expected_encoding=UpperCAmelCase  ,   model_name='''google/reformer-crime-and-punishment'''  ,   revision='''0e6c3decb8211d49bf881013425dc8b0448b3f5a'''  ,   padding=UpperCAmelCase  ,   sequences=UpperCAmelCase  ,   )











'''simple docstring'''



# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import importlib.metadata
import json
import os
from dataclasses import dataclass
from typing import Any, Dict, Union

from packaging import version

from ..utils import is_torch_available, logging


if is_torch_available():
      import torch


SCREAMING_SNAKE_CASE							=						logging.get_logger(__name__)
@dataclass
class UpperCAmelCase_       :




      """simple docstring"""




      def __init__(			self      :		List[Any]  ,   UpperCAmelCase      :		List[Any]=False  ,   UpperCAmelCase      :		Union[str, Any]=False  ,   UpperCAmelCase      :		Union[str, Any]=6.0  ,   UpperCAmelCase      :		Any=None  ,   UpperCAmelCase      :		Optional[Any]=False  ,   UpperCAmelCase      :		List[Any]=False  ,   UpperCAmelCase      :		Dict=None  ,   UpperCAmelCase      :		Tuple="fp4"  ,   UpperCAmelCase      :		str=False  ,   **UpperCAmelCase      :		Union[str, Any]  ,   )	->	str:


       '''simple docstring'''

       lowercase       : Optional[int]						=load_in_abit
       lowercase       : Union[str, Any]						=load_in_abit
       lowercase       : Tuple						=llm_inta_threshold
       lowercase       : Optional[Any]						=llm_inta_skip_modules
       lowercase       : int						=llm_inta_enable_fpaa_cpu_offload
       lowercase       : Dict						=llm_inta_has_fpaa_weight
       lowercase       : str						=bnb_abit_quant_type
       lowercase       : int						=bnb_abit_use_double_quant

       if bnb_abit_compute_dtype is None:
        lowercase       : str						=torch.floataa
       elif isinstance(UpperCAmelCase  ,   UpperCAmelCase		):
        lowercase       : Tuple						=getattr(UpperCAmelCase  ,   UpperCAmelCase		)
       elif isinstance(UpperCAmelCase  ,   torch.dtype		):
        lowercase       : Optional[int]						=bnb_abit_compute_dtype
       else:
        raise ValueError('''bnb_4bit_compute_dtype must be a string or a torch.dtype'''		)

       self.post_init()






      def 					A__   (			self      :		List[Any]		)	->	Union[str, Any]:


       '''simple docstring'''

       if not isinstance(self.llm_inta_threshold  ,   UpperCAmelCase		):
        raise ValueError('''llm_int8_threshold must be a float'''		)

       if self.llm_inta_skip_modules is not None and not isinstance(self.llm_inta_skip_modules  ,   UpperCAmelCase		):
        raise ValueError('''llm_int8_skip_modules must be a list of strings'''		)
       if not isinstance(self.llm_inta_enable_fpaa_cpu_offload  ,   UpperCAmelCase		):
        raise ValueError('''llm_int8_enable_fp32_cpu_offload must be a boolean'''		)

       if not isinstance(self.llm_inta_has_fpaa_weight  ,   UpperCAmelCase		):
        raise ValueError('''llm_int8_has_fp16_weight must be a boolean'''		)

       if self.bnb_abit_compute_dtype is not None and not isinstance(self.bnb_abit_compute_dtype  ,   torch.dtype		):
        raise ValueError('''bnb_4bit_compute_dtype must be torch.dtype'''		)

       if not isinstance(self.bnb_abit_quant_type  ,   UpperCAmelCase		):
        raise ValueError('''bnb_4bit_quant_type must be a string'''		)

       if not isinstance(self.bnb_abit_use_double_quant  ,   UpperCAmelCase		):
        raise ValueError('''bnb_4bit_use_double_quant must be a boolean'''		)

       if self.load_in_abit and not version.parse(importlib.metadata.version('''bitsandbytes'''		)		) >= version.parse(
           '''0.39.0'''		):
        raise ValueError(
            '''4 bit quantization requires bitsandbytes>=0.39.0 - please upgrade your bitsandbytes version'''		)






      def 					A__   (			self      :		Any		)	->	Tuple:


       '''simple docstring'''

       return self.load_in_abit or self.load_in_abit






      def 					A__   (			self      :		Tuple		)	->	Any:


       '''simple docstring'''

       if self.load_in_abit:
        return "llm_int8"
       elif self.load_in_abit and self.bnb_abit_quant_type == "fp4":
        return "fp4"
       elif self.load_in_abit and self.bnb_abit_quant_type == "nf4":
        return "nf4"
       else:
        return None






      @classmethod
      def 					A__   (			cls      :		Union[str, Any]  ,   UpperCAmelCase      :		Optional[Any]  ,   UpperCAmelCase      :		Any  ,   **UpperCAmelCase      :		List[str]		)	->	Tuple:


       '''simple docstring'''

       lowercase       : Dict						=cls(**UpperCAmelCase		)

       lowercase       : Dict						=[]
       for key, value in kwargs.items():
        if hasattr(UpperCAmelCase  ,   UpperCAmelCase		):
         setattr(UpperCAmelCase  ,   UpperCAmelCase  ,   UpperCAmelCase		)
         to_remove.append(UpperCAmelCase		)
       for key in to_remove:
        kwargs.pop(UpperCAmelCase  ,   UpperCAmelCase		)

       if return_unused_kwargs:
        return config, kwargs
       else:
        return config






      def 					A__   (			self      :		List[Any]  ,   UpperCAmelCase      :		Union[str, os.PathLike]		)	->	Dict:


       '''simple docstring'''

       with open(UpperCAmelCase  ,   '''w'''  ,   encoding='''utf-8'''		) as writer:
        lowercase       : Any						=self.to_dict()
        lowercase       : List[Any]						=json.dumps(UpperCAmelCase  ,   indent=2  ,   sort_keys=UpperCAmelCase		) + '''\n'''

        writer.write(UpperCAmelCase		)






      def 					A__   (			self      :		Optional[Any]		)	->	Dict[str, Any]:


       '''simple docstring'''

       lowercase       : Tuple						=copy.deepcopy(self.__dict__		)
       lowercase       : Optional[Any]						=str(output['''bnb_4bit_compute_dtype''']		).split('''.'''		)[1]

       return output






      def __repr__(			self      :		Optional[Any]		)	->	Union[str, Any]:


       '''simple docstring'''

       return f'{self.__class__.__name__} {self.to_json_string()}'






      def 					A__   (			self      :		List[Any]  ,   UpperCAmelCase      :		bool = True		)	->	str:


       '''simple docstring'''

       if use_diff is True:
        lowercase       : int						=self.to_diff_dict()
       else:
        lowercase       : List[str]						=self.to_dict()
       return json.dumps(UpperCAmelCase  ,   indent=2  ,   sort_keys=UpperCAmelCase		) + "\n"






      def 					A__   (			self      :		Any		)	->	Dict[str, Any]:


       '''simple docstring'''

       lowercase       : Any						=self.to_dict()

       # get the default config dict
       lowercase       : Union[str, Any]						=BitsAndBytesConfig().to_dict()

       lowercase       : int						={}

       # only serialize values that differ from the default config
       for key, value in config_dict.items():
        if value != default_config_dict[key]:
         lowercase       : Dict						=value

       return serializable_config_dict



