


# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import importlib.metadata
import json
import os
from dataclasses import dataclass
from typing import Any, Dict, Union

from packaging import version

from ..utils import is_torch_available, logging


if is_torch_available():
 import torch


SCREAMING_SNAKE_CASE			: Optional[int]				   =					logging.get_logger(__name__)






@dataclass
class _lowerCamelCase:
       def __init__(   self, lowerCamelCase=False, lowerCamelCase=False, lowerCamelCase=6.0, lowerCamelCase=None, lowerCamelCase=False, lowerCamelCase=False, lowerCamelCase=None, lowerCamelCase="fp4", lowerCamelCase=False, **lowerCamelCase, )			->  List[str]:




           """simple docstring"""

           _lowercase			:				Optional[Any] 						=						load_in_abit
           _lowercase			:				List[Any] 						=						load_in_abit
           _lowercase			:				Union[str, Any] 						=						llm_inta_threshold
           _lowercase			:				List[Any] 						=						llm_inta_skip_modules
           _lowercase			:				Any 						=						llm_inta_enable_fpaa_cpu_offload
           _lowercase			:				Tuple 						=						llm_inta_has_fpaa_weight
           _lowercase			:				Dict 						=						bnb_abit_quant_type
           _lowercase			:				List[str] 						=						bnb_abit_use_double_quant

           if bnb_abit_compute_dtype is None:
               _lowercase			:				Union[str, Any] 						=						torch.floataa
           elif isinstance(lowerCamelCase, lowerCamelCase):
               _lowercase			:				Tuple 						=						getattr(lowerCamelCase, lowerCamelCase)
           elif isinstance(lowerCamelCase, torch.dtype):
               _lowercase			:				int 						=						bnb_abit_compute_dtype
           else:
               raise ValueError('bnb_4bit_compute_dtype must be a string or a torch.dtype')

           self.post_init()
       def      UpperCamelCase	(   self)			->  List[Any]:




           """simple docstring"""

           if not isinstance(self.llm_inta_threshold, lowerCamelCase):
               raise ValueError('llm_int8_threshold must be a float')

           if self.llm_inta_skip_modules is not None and not isinstance(self.llm_inta_skip_modules, lowerCamelCase):
               raise ValueError('llm_int8_skip_modules must be a list of strings')
           if not isinstance(self.llm_inta_enable_fpaa_cpu_offload, lowerCamelCase):
               raise ValueError('llm_int8_enable_fp32_cpu_offload must be a boolean')

           if not isinstance(self.llm_inta_has_fpaa_weight, lowerCamelCase):
               raise ValueError('llm_int8_has_fp16_weight must be a boolean')

           if self.bnb_abit_compute_dtype is not None and not isinstance(self.bnb_abit_compute_dtype, torch.dtype):
               raise ValueError('bnb_4bit_compute_dtype must be torch.dtype')

           if not isinstance(self.bnb_abit_quant_type, lowerCamelCase):
               raise ValueError('bnb_4bit_quant_type must be a string')

           if not isinstance(self.bnb_abit_use_double_quant, lowerCamelCase):
               raise ValueError('bnb_4bit_use_double_quant must be a boolean')

           if self.load_in_abit and not version.parse(importlib.metadata.version('bitsandbytes')) >= version.parse(
               '0.39.0'):
               raise ValueError(
                   '4 bit quantization requires bitsandbytes>=0.39.0 - please upgrade your bitsandbytes version')
       def      UpperCamelCase	(   self)			->  int:




           """simple docstring"""

           return self.load_in_abit or self.load_in_abit
       def      UpperCamelCase	(   self)			->  Optional[int]:




           """simple docstring"""

           if self.load_in_abit:
               return "llm_int8"
           elif self.load_in_abit and self.bnb_abit_quant_type == "fp4":
               return "fp4"
           elif self.load_in_abit and self.bnb_abit_quant_type == "nf4":
               return "nf4"
           else:
               return None
       @classmethod
       def      UpperCamelCase	(   cls, lowerCamelCase, lowerCamelCase, **lowerCamelCase)			->  List[str]:




           """simple docstring"""

           _lowercase			:				Optional[int] 						=						cls(**lowerCamelCase)

           _lowercase			:				int 						=						[]
           for key, value in kwargs.items():
               if hasattr(lowerCamelCase, lowerCamelCase):
                   setattr(lowerCamelCase, lowerCamelCase, lowerCamelCase)
                   to_remove.append(lowerCamelCase)
           for key in to_remove:
               kwargs.pop(lowerCamelCase, lowerCamelCase)

           if return_unused_kwargs:
               return config, kwargs
           else:
               return config
       def      UpperCamelCase	(   self, lowerCamelCase)			->  Optional[int]:




           """simple docstring"""

           with open(lowerCamelCase, 'w', encoding='utf-8') as writer:
               _lowercase			:				Union[str, Any] 						=						self.to_dict()
               _lowercase			:				int 						=						json.dumps(lowerCamelCase, indent=2, sort_keys=lowerCamelCase) + '\n'

               writer.write(lowerCamelCase)
       def      UpperCamelCase	(   self)			->  Dict[str, Any]:




           """simple docstring"""

           _lowercase			:				Dict 						=						copy.deepcopy(self.__dict__)
           _lowercase			:				str 						=						str(output['bnb_4bit_compute_dtype']).split('.')[1]

           return output
       def __repr__(   self)			->  Any:




           """simple docstring"""

           return F'''{self.__class__.__name__} {self.to_json_string()}'''
       def      UpperCamelCase	(   self, lowerCamelCase = True)			->  str:




           """simple docstring"""

           if use_diff is True:
               _lowercase			:				int 						=						self.to_diff_dict()
           else:
               _lowercase			:				List[str] 						=						self.to_dict()
           return json.dumps(lowerCamelCase, indent=2, sort_keys=lowerCamelCase) + "\n"




       def      UpperCamelCase	(   self)			->  Dict[str, Any]:




           """simple docstring"""

           _lowercase			:				Optional[Any] 						=						self.to_dict()

           # get the default config dict
           _lowercase			:				Optional[Any] 						=						BitsAndBytesConfig().to_dict()

           _lowercase			:				str 						=						{}

           # only serialize values that differ from the default config
           for key, value in config_dict.items():
               if value != default_config_dict[key]:
                   _lowercase			:				Dict 						=						value

           return serializable_config_dict










'''simple docstring'''







import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_torch, require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_torch_available, is_vision_available


if is_torch_available():
		import torch


if is_vision_available():
		from PIL import Image

		from transformers import MgpstrProcessor, ViTImageProcessor





@require_torch
@require_vision
class 						SCREAMING_SNAKE_CASE       (       unittest.TestCase							):
			snake_case__	=					ViTImageProcessor if is_vision_available() else None






			@property
			def       SCREAMING_SNAKE_CASE   (			self   :      List[str]       )      ->      List[str]:
							return self.image_processor_tester.prepare_image_processor_dict()






			def       SCREAMING_SNAKE_CASE   (			self   :      Tuple       )      ->      Optional[int]:
							a_		:	Optional[Any]									=   (3, 32, 128)
							a_		:	Optional[Any]									=   tempfile.mkdtemp()

							# fmt: off
							a_		:	Union[str, Any]									=   ['''[GO]''', '''[s]''', '''0''', '''1''', '''2''', '''3''', '''4''', '''5''', '''6''', '''7''', '''8''', '''9''', '''a''', '''b''', '''c''', '''d''', '''e''', '''f''', '''g''', '''h''', '''i''', '''j''', '''k''', '''l''', '''m''', '''n''', '''o''', '''p''', '''q''', '''r''', '''s''', '''t''', '''u''', '''v''', '''w''', '''x''', '''y''', '''z''']
							# fmt: on
							a_		:	List[Any]									=   dict(zip(__SCREAMING_SNAKE_CASE	,    range(len(__SCREAMING_SNAKE_CASE       )       )       )       )

							a_		:	Union[str, Any]									=   os.path.join(self.tmpdirname	,    VOCAB_FILES_NAMES['''vocab_file''']       )
							with open(self.vocab_file	,    '''w'''	,    encoding='''utf-8'''       ) as fp:
											fp.write(json.dumps(__SCREAMING_SNAKE_CASE       ) + '''\n'''       )

							a_		:	Union[str, Any]									=   {
							    '''do_normalize''': False,
							    '''do_resize''': True,
							    '''image_processor_type''': '''ViTImageProcessor''',
							    '''resample''': 3,
							    '''size''': {'''height''': 32, '''width''': 128},
							}
							a_		:	Any									=   os.path.join(self.tmpdirname	,    __SCREAMING_SNAKE_CASE       )
							with open(self.image_processor_file	,    '''w'''	,    encoding='''utf-8'''       ) as fp:
											json.dump(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )






			def       SCREAMING_SNAKE_CASE   (			self   :      List[Any]	,    **__SCREAMING_SNAKE_CASE   :      Union[str, Any]       )      ->      Dict:
							return MgpstrTokenizer.from_pretrained(self.tmpdirname	,    **__SCREAMING_SNAKE_CASE       )






			def       SCREAMING_SNAKE_CASE   (			self   :      List[str]	,    **__SCREAMING_SNAKE_CASE   :      Union[str, Any]       )      ->      List[Any]:
							return ViTImageProcessor.from_pretrained(self.tmpdirname	,    **__SCREAMING_SNAKE_CASE       )






			def       SCREAMING_SNAKE_CASE   (			self   :      Optional[Any]       )      ->      Union[str, Any]:
							shutil.rmtree(self.tmpdirname       )






			def       SCREAMING_SNAKE_CASE   (			self   :      str       )      ->      str:
							a_		:	Optional[Any]									=   np.random.randint(255	,    size=(3, 30, 400)	,    dtype=np.uinta       )

							a_		:	Optional[int]									=   Image.fromarray(np.moveaxis(__SCREAMING_SNAKE_CASE	,    0	,    -1       )       )

							return image_input






			def       SCREAMING_SNAKE_CASE   (			self   :      int       )      ->      int:
							a_		:	str									=   self.get_tokenizer()
							a_		:	List[Any]									=   self.get_image_processor()

							a_		:	Union[str, Any]									=   MgpstrProcessor(tokenizer=__SCREAMING_SNAKE_CASE	,    image_processor=__SCREAMING_SNAKE_CASE       )
							processor.save_pretrained(self.tmpdirname       )
							a_		:	Tuple									=   MgpstrProcessor.from_pretrained(self.tmpdirname	,    use_fast=__SCREAMING_SNAKE_CASE       )

							self.assertEqual(processor.char_tokenizer.get_vocab()	,    tokenizer.get_vocab()       )
							self.assertIsInstance(processor.char_tokenizer	,    __SCREAMING_SNAKE_CASE       )

							self.assertEqual(processor.image_processor.to_json_string()	,    image_processor.to_json_string()       )
							self.assertIsInstance(processor.image_processor	,    __SCREAMING_SNAKE_CASE       )






			def       SCREAMING_SNAKE_CASE   (			self   :      Optional[int]       )      ->      int:
							a_		:	Any									=   self.get_tokenizer()
							a_		:	Dict									=   self.get_image_processor()

							a_		:	Optional[Any]									=   MgpstrProcessor(tokenizer=__SCREAMING_SNAKE_CASE	,    image_processor=__SCREAMING_SNAKE_CASE       )
							processor.save_pretrained(self.tmpdirname       )

							a_		:	str									=   self.get_tokenizer(bos_token='''(BOS)'''	,    eos_token='''(EOS)'''       )
							a_		:	str									=   self.get_image_processor(do_normalize=__SCREAMING_SNAKE_CASE	,    padding_value=1.0       )

							a_		:	Optional[int]									=   MgpstrProcessor.from_pretrained(
							    self.tmpdirname	,    bos_token='''(BOS)'''	,    eos_token='''(EOS)'''	,    do_normalize=__SCREAMING_SNAKE_CASE	,    padding_value=1.0       )

							self.assertEqual(processor.char_tokenizer.get_vocab()	,    tokenizer_add_kwargs.get_vocab()       )
							self.assertIsInstance(processor.char_tokenizer	,    __SCREAMING_SNAKE_CASE       )

							self.assertEqual(processor.image_processor.to_json_string()	,    image_processor_add_kwargs.to_json_string()       )
							self.assertIsInstance(processor.image_processor	,    __SCREAMING_SNAKE_CASE       )






			def       SCREAMING_SNAKE_CASE   (			self   :      str       )      ->      Optional[Any]:
							a_		:	Dict									=   self.get_image_processor()
							a_		:	Tuple									=   self.get_tokenizer()

							a_		:	Any									=   MgpstrProcessor(tokenizer=__SCREAMING_SNAKE_CASE	,    image_processor=__SCREAMING_SNAKE_CASE       )

							a_		:	Optional[Any]									=   self.prepare_image_inputs()

							a_		:	List[Any]									=   image_processor(__SCREAMING_SNAKE_CASE	,    return_tensors='''np'''       )
							a_		:	int									=   processor(images=__SCREAMING_SNAKE_CASE	,    return_tensors='''np'''       )

							for key in input_image_proc.keys():
											self.assertAlmostEqual(input_image_proc[key].sum()	,    input_processor[key].sum()	,    delta=1e-2       )






			def       SCREAMING_SNAKE_CASE   (			self   :      str       )      ->      Optional[Any]:
							a_		:	Dict									=   self.get_image_processor()
							a_		:	List[Any]									=   self.get_tokenizer()

							a_		:	int									=   MgpstrProcessor(tokenizer=__SCREAMING_SNAKE_CASE	,    image_processor=__SCREAMING_SNAKE_CASE       )

							a_		:	Tuple									=   '''test'''

							a_		:	Any									=   processor(text=__SCREAMING_SNAKE_CASE       )

							a_		:	Optional[int]									=   tokenizer(__SCREAMING_SNAKE_CASE       )
							for key in encoded_tok.keys():
											self.assertListEqual(encoded_tok[key]	,    encoded_processor[key]       )






			def       SCREAMING_SNAKE_CASE   (			self   :      int       )      ->      int:
							a_		:	Union[str, Any]									=   self.get_image_processor()
							a_		:	Tuple									=   self.get_tokenizer()

							a_		:	Optional[Any]									=   MgpstrProcessor(tokenizer=__SCREAMING_SNAKE_CASE	,    image_processor=__SCREAMING_SNAKE_CASE       )

							a_		:	Union[str, Any]									=   '''test'''
							a_		:	str									=   self.prepare_image_inputs()

							a_		:	Tuple									=   processor(text=__SCREAMING_SNAKE_CASE	,    images=__SCREAMING_SNAKE_CASE       )

							self.assertListEqual(list(inputs.keys()       )	,    ['''pixel_values''', '''labels''']       )

							# test if it raises when no input is passed
							with pytest.raises(__SCREAMING_SNAKE_CASE       ):
											processor()






			def       SCREAMING_SNAKE_CASE   (			self   :      List[str]       )      ->      int:
							a_		:	int									=   self.get_image_processor()
							a_		:	List[Any]									=   self.get_tokenizer()

							a_		:	Dict									=   MgpstrProcessor(tokenizer=__SCREAMING_SNAKE_CASE	,    image_processor=__SCREAMING_SNAKE_CASE       )

							a_		:	Tuple									=   [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9], [3, 4, 3, 1, 1, 8, 9]]

							a_		:	List[Any]									=   processor.char_decode(__SCREAMING_SNAKE_CASE       )
							a_		:	Optional[int]									=   tokenizer.batch_decode(__SCREAMING_SNAKE_CASE       )
							a_		:	Union[str, Any]									=   [seq.replace(''' '''	,    ''''''       ) for seq in decoded_tok]

							self.assertListEqual(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )






			def       SCREAMING_SNAKE_CASE   (			self   :      Any       )      ->      List[Any]:
							a_		:	int									=   self.get_image_processor()
							a_		:	Tuple									=   self.get_tokenizer()

							a_		:	Any									=   MgpstrProcessor(tokenizer=__SCREAMING_SNAKE_CASE	,    image_processor=__SCREAMING_SNAKE_CASE       )

							a_		:	Dict									=   None
							a_		:	Tuple									=   self.prepare_image_inputs()

							a_		:	Optional[int]									=   processor(text=__SCREAMING_SNAKE_CASE	,    images=__SCREAMING_SNAKE_CASE       )

							self.assertListEqual(list(inputs.keys()       )	,    processor.model_input_names       )







			def       SCREAMING_SNAKE_CASE   (			self   :      Tuple       )      ->      Union[str, Any]:
							a_		:	List[str]									=   self.get_image_processor()
							a_		:	str									=   self.get_tokenizer()

							a_		:	Any									=   MgpstrProcessor(tokenizer=__SCREAMING_SNAKE_CASE	,    image_processor=__SCREAMING_SNAKE_CASE       )

							a_		:	str									=   torch.randn(1	,    27	,    38       )
							a_		:	int									=   torch.randn(1	,    27	,    5_0257       )
							a_		:	Union[str, Any]									=   torch.randn(1	,    27	,    3_0522       )

							a_		:	Tuple									=   processor.batch_decode([char_input, bpe_input, wp_input]       )

							self.assertListEqual(list(results.keys()       )	,    ['''generated_text''', '''scores''', '''char_preds''', '''bpe_preds''', '''wp_preds''']       )




