
import os
import unittest
from tempfile import TemporaryDirectory

import torch
import torch.nn as nn

from accelerate.utils import (
    OffloadedWeightsLoader,
    extract_submodules_state_dict,
    load_offloaded_weight,
    offload_state_dict,
    offload_weight,
)




class 			UpperCAmelCase_     (					nn.Module	):


  '''simple docstring'''







  def __init__(					self     :					List[str]       )     ->      List[str]:







    """simple docstring"""






    super().__init__()
    __magic_name__								=			nn.Linear(3     ,				4       )
    __magic_name__								=			nn.BatchNormad(4       )
    __magic_name__								=			nn.Linear(4     ,				5       )





  def 				_lowercase				(					self     :					Dict     ,				UpperCamelCase__     :					Any       )     ->      Any:







    """simple docstring"""






    return self.lineara(self.batchnorm(self.lineara(lowerCamelCase__       )       )       )







class 			UpperCAmelCase_     (					unittest.TestCase	):


  '''simple docstring'''







  def 				_lowercase				(					self     :					Any       )     ->      Tuple:







    """simple docstring"""






    __magic_name__								=			ModelForTest()
    with TemporaryDirectory() as tmp_dir:
      offload_state_dict(lowerCamelCase__     ,				model.state_dict()       )
      __magic_name__								=			os.path.join(lowerCamelCase__     ,				"""index.json"""       )
      self.assertTrue(os.path.isfile(lowerCamelCase__       )       )
      # TODO: add tests on what is inside the index

      for key in ["linear1.weight", "linear1.bias", "linear2.weight", "linear2.bias"]:
        __magic_name__								=			os.path.join(lowerCamelCase__     ,				F'''{key}.dat'''       )
        self.assertTrue(os.path.isfile(lowerCamelCase__       )       )
        # TODO: add tests on the fact weights are properly loaded
  def 				_lowercase				(					self     :					List[str]       )     ->      Dict:







    """simple docstring"""






    __magic_name__								=			[torch.floataa, torch.floataa, torch.bfloataa]

    for dtype in dtypes:
      __magic_name__								=			torch.randn(2     ,				3     ,				dtype=lowerCamelCase__       )
      with TemporaryDirectory() as tmp_dir:
        __magic_name__								=			offload_weight(lowerCamelCase__     ,				"""weight"""     ,				lowerCamelCase__     ,				{}       )
        __magic_name__								=			os.path.join(lowerCamelCase__     ,				"""weight.dat"""       )
        self.assertTrue(os.path.isfile(lowerCamelCase__       )       )
        self.assertDictEqual(lowerCamelCase__     ,				{"""weight""": {"""shape""": [2, 3], """dtype""": str(lowerCamelCase__       ).split("""."""       )[1]}}       )

        __magic_name__								=			load_offloaded_weight(lowerCamelCase__     ,				index["""weight"""]       )
        self.assertTrue(torch.equal(lowerCamelCase__     ,				lowerCamelCase__       )       )
  def 				_lowercase				(					self     :					Union[str, Any]       )     ->      Dict:







    """simple docstring"""






    __magic_name__								=			ModelForTest()
    __magic_name__								=			model.state_dict()
    __magic_name__								=			{k: v for k, v in state_dict.items() if "linear2" not in k}
    __magic_name__								=			{k: v for k, v in state_dict.items() if "linear2" in k}

    with TemporaryDirectory() as tmp_dir:
      offload_state_dict(lowerCamelCase__     ,				lowerCamelCase__       )
      __magic_name__								=			OffloadedWeightsLoader(state_dict=lowerCamelCase__     ,				save_folder=lowerCamelCase__       )

      # Every key is there with the right value
      self.assertEqual(sorted(lowerCamelCase__       )     ,				sorted(state_dict.keys()       )       )
      for key, param in state_dict.items():
        self.assertTrue(torch.allclose(lowerCamelCase__     ,				weight_map[key]       )       )

    __magic_name__								=			{k: v for k, v in state_dict.items() if "weight" in k}
    __magic_name__								=			{k: v for k, v in state_dict.items() if "weight" not in k}

    with TemporaryDirectory() as tmp_dir:
      offload_state_dict(lowerCamelCase__     ,				lowerCamelCase__       )
      __magic_name__								=			OffloadedWeightsLoader(state_dict=lowerCamelCase__     ,				save_folder=lowerCamelCase__       )

      # Every key is there with the right value
      self.assertEqual(sorted(lowerCamelCase__       )     ,				sorted(state_dict.keys()       )       )
      for key, param in state_dict.items():
        self.assertTrue(torch.allclose(lowerCamelCase__     ,				weight_map[key]       )       )

    with TemporaryDirectory() as tmp_dir:
      offload_state_dict(lowerCamelCase__     ,				lowerCamelCase__       )
      # Duplicates are removed
      __magic_name__								=			OffloadedWeightsLoader(state_dict=lowerCamelCase__     ,				save_folder=lowerCamelCase__       )

      # Every key is there with the right value
      self.assertEqual(sorted(lowerCamelCase__       )     ,				sorted(state_dict.keys()       )       )
      for key, param in state_dict.items():
        self.assertTrue(torch.allclose(lowerCamelCase__     ,				weight_map[key]       )       )





  def 				_lowercase				(					self     :					str       )     ->      Optional[Any]:







    """simple docstring"""






    __magic_name__								=			{"a.1": 0, "a.10": 1, "a.2": 2}
    __magic_name__								=			extract_submodules_state_dict(lowerCamelCase__     ,				["""a.1""", """a.2"""]       )
    self.assertDictEqual(lowerCamelCase__     ,				{"""a.1""": 0, """a.2""": 2}       )

    __magic_name__								=			{"a.1.a": 0, "a.10.a": 1, "a.2.a": 2}
    __magic_name__								=			extract_submodules_state_dict(lowerCamelCase__     ,				["""a.1""", """a.2"""]       )
    self.assertDictEqual(lowerCamelCase__     ,				{"""a.1.a""": 0, """a.2.a""": 2}       )




import unittest

from transformers import DonutProcessor


_lowerCAmelCase  :      str			 ="""naver-clova-ix/donut-base"""

class __UpperCamelCase     (				unittest.TestCase		):






					'''simple docstring'''



					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Any									= DonutProcessor.from_pretrained(lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Any									= {
						    "name": "John Doe",
						    "age": "99",
						    "city": "Atlanta",
						    "state": "GA",
						    "zip": "30301",
						    "phone": "123-4567",
						    "nicknames": [{"nickname": "Johnny"}, {"nickname": "JD"}],
						}

						UpperCAmelCase__:	Optional[Any]									= (
						    "<s_name>John Doe</s_name><s_age>99</s_age><s_city>Atlanta</s_city>"
						    "<s_state>GA</s_state><s_zip>30301</s_zip><s_phone>123-4567</s_phone>"
						    "<s_nicknames><s_nickname>Johnny</s_nickname>"
						    "<sep/><s_nickname>JD</s_nickname></s_nicknames>"
						)
						UpperCAmelCase__:	str									= self.processor.tokenajson(lowerCamelCase__						)

						self.assertDictEqual(lowerCamelCase__				,      lowerCamelCase__						)