




"""simple docstring"""





import os
import unittest
from tempfile import TemporaryDirectory

import torch
import torch.nn as nn

from accelerate.utils import (
    OffloadedWeightsLoader,
    extract_submodules_state_dict,
    load_offloaded_weight,
    offload_state_dict,
    offload_weight,
)



class   a__					(						nn.Module ):


			def __init__(   self   :     Union[str, Any]):


						"""simple docstring"""




						super().__init__()
						__UpperCAmelCase						:			Optional[int]												=       nn.Linear(3   ,	4)
						__UpperCAmelCase						:			str												=       nn.BatchNormad(4)
						__UpperCAmelCase						:			int												=       nn.Linear(4   ,	5)


			def    a_  (   self   :     str   ,	UpperCamelCase_   :     List[str]):


						"""simple docstring"""




						return self.lineara(self.batchnorm(self.lineara(UpperCamelCase_)))






class   a__					(						unittest.TestCase ):


			def    a_  (   self   :     Dict):


						"""simple docstring"""




						__UpperCAmelCase						:			Optional[Any]												=       ModelForTest()
						with TemporaryDirectory() as tmp_dir:
									offload_state_dict(UpperCamelCase_   ,	model.state_dict())
									__UpperCAmelCase						:			Union[str, Any]												=       os.path.join(UpperCamelCase_   ,	"index.json")
									self.assertTrue(os.path.isfile(UpperCamelCase_))
									# TODO: add tests on what is inside the index

									for key in ["linear1.weight", "linear1.bias", "linear2.weight", "linear2.bias"]:
												__UpperCAmelCase						:			Optional[int]												=       os.path.join(UpperCamelCase_   ,	F"{key}.dat")
												self.assertTrue(os.path.isfile(UpperCamelCase_))
												# TODO: add tests on the fact weights are properly loaded


			def    a_  (   self   :     Any):


						"""simple docstring"""




						__UpperCAmelCase						:			int												=       [torch.floataa, torch.floataa, torch.bfloataa]

						for dtype in dtypes:
									__UpperCAmelCase						:			List[Any]												=       torch.randn(2   ,	3   ,	dtype=UpperCamelCase_)
									with TemporaryDirectory() as tmp_dir:
												__UpperCAmelCase						:			Tuple												=       offload_weight(UpperCamelCase_   ,	"weight"   ,	UpperCamelCase_   ,	{})
												__UpperCAmelCase						:			Dict												=       os.path.join(UpperCamelCase_   ,	"weight.dat")
												self.assertTrue(os.path.isfile(UpperCamelCase_))
												self.assertDictEqual(UpperCamelCase_   ,	{"weight": {"shape": [2, 3], "dtype": str(UpperCamelCase_).split(".")[1]}})

												__UpperCAmelCase						:			Optional[Any]												=       load_offloaded_weight(UpperCamelCase_   ,	index["weight"])
												self.assertTrue(torch.equal(UpperCamelCase_   ,	UpperCamelCase_))


			def    a_  (   self   :     List[str]):


						"""simple docstring"""




						__UpperCAmelCase						:			List[Any]												=       ModelForTest()
						__UpperCAmelCase						:			Optional[int]												=       model.state_dict()
						__UpperCAmelCase						:			List[str]												=       {k: v for k, v in state_dict.items() if "linear2" not in k}
						__UpperCAmelCase						:			Optional[int]												=       {k: v for k, v in state_dict.items() if "linear2" in k}

						with TemporaryDirectory() as tmp_dir:
									offload_state_dict(UpperCamelCase_   ,	UpperCamelCase_)
									__UpperCAmelCase						:			List[str]												=       OffloadedWeightsLoader(state_dict=UpperCamelCase_   ,	save_folder=UpperCamelCase_)

									# Every key is there with the right value
									self.assertEqual(sorted(UpperCamelCase_)   ,	sorted(state_dict.keys()))
									for key, param in state_dict.items():
												self.assertTrue(torch.allclose(UpperCamelCase_   ,	weight_map[key]))

						__UpperCAmelCase						:			Optional[int]												=       {k: v for k, v in state_dict.items() if "weight" in k}
						__UpperCAmelCase						:			Optional[Any]												=       {k: v for k, v in state_dict.items() if "weight" not in k}

						with TemporaryDirectory() as tmp_dir:
									offload_state_dict(UpperCamelCase_   ,	UpperCamelCase_)
									__UpperCAmelCase						:			Optional[Any]												=       OffloadedWeightsLoader(state_dict=UpperCamelCase_   ,	save_folder=UpperCamelCase_)

									# Every key is there with the right value
									self.assertEqual(sorted(UpperCamelCase_)   ,	sorted(state_dict.keys()))
									for key, param in state_dict.items():
												self.assertTrue(torch.allclose(UpperCamelCase_   ,	weight_map[key]))

						with TemporaryDirectory() as tmp_dir:
									offload_state_dict(UpperCamelCase_   ,	UpperCamelCase_)
									# Duplicates are removed
									__UpperCAmelCase						:			str												=       OffloadedWeightsLoader(state_dict=UpperCamelCase_   ,	save_folder=UpperCamelCase_)

									# Every key is there with the right value
									self.assertEqual(sorted(UpperCamelCase_)   ,	sorted(state_dict.keys()))
									for key, param in state_dict.items():
												self.assertTrue(torch.allclose(UpperCamelCase_   ,	weight_map[key]))


			def    a_  (   self   :     Union[str, Any]):


						"""simple docstring"""




						__UpperCAmelCase						:			Any												=       {"a.1": 0, "a.10": 1, "a.2": 2}
						__UpperCAmelCase						:			Union[str, Any]												=       extract_submodules_state_dict(UpperCamelCase_   ,	["a.1", "a.2"])
						self.assertDictEqual(UpperCamelCase_   ,	{"a.1": 0, "a.2": 2})

						__UpperCAmelCase						:			int												=       {"a.1.a": 0, "a.10.a": 1, "a.2.a": 2}
						__UpperCAmelCase						:			int												=       extract_submodules_state_dict(UpperCamelCase_   ,	["a.1", "a.2"])
						self.assertDictEqual(UpperCamelCase_   ,	{"a.1.a": 0, "a.2.a": 2})




"""simple docstring"""



import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType





class 				a			(					lowercase      ):
      UpperCamelCase							:   Tuple           =     ["""image_processor""", """tokenizer"""]
      UpperCamelCase							:   List[str]           =     """LayoutLMv2ImageProcessor"""
      UpperCamelCase							:   Optional[int]           =     ("""LayoutXLMTokenizer""", """LayoutXLMTokenizerFast""")



      def __init__(   self     ,	UpperCamelCase_=None     ,	UpperCamelCase_=None     ,	**UpperCamelCase_      ):
          if "feature_extractor" in kwargs:
              warnings.warn(
                  'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
                  ' instead.'     ,	UpperCamelCase_     ,	)
              UpperCAmelCase__	:      Tuple     							=					kwargs.pop('feature_extractor'      )

          UpperCAmelCase__	:      Optional[Any]     							=					image_processor if image_processor is not None else feature_extractor
          if image_processor is None:
              raise ValueError('You need to specify an `image_processor`.'      )
          if tokenizer is None:
              raise ValueError('You need to specify a `tokenizer`.'      )

          super().__init__(UpperCamelCase_     ,	UpperCamelCase_      )



      def __call__(   self     ,	UpperCamelCase_     ,	UpperCamelCase_ = None     ,	UpperCamelCase_ = None     ,	UpperCamelCase_ = None     ,	UpperCamelCase_ = None     ,	UpperCamelCase_ = True     ,	UpperCamelCase_ = False     ,	UpperCamelCase_ = None     ,	UpperCamelCase_ = None     ,	UpperCamelCase_ = 0     ,	UpperCamelCase_ = None     ,	UpperCamelCase_ = None     ,	UpperCamelCase_ = None     ,	UpperCamelCase_ = False     ,	UpperCamelCase_ = False     ,	UpperCamelCase_ = False     ,	UpperCamelCase_ = False     ,	UpperCamelCase_ = True     ,	UpperCamelCase_ = None     ,	**UpperCamelCase_     ,	):
          # verify input
          if self.image_processor.apply_ocr and (boxes is not None):
              raise ValueError(
                  'You cannot provide bounding boxes '
                  'if you initialized the image processor with apply_ocr set to True.'      )

          if self.image_processor.apply_ocr and (word_labels is not None):
              raise ValueError(
                  'You cannot provide word labels if you initialized the image processor with apply_ocr set to True.'      )

          if return_overflowing_tokens is True and return_offsets_mapping is False:
              raise ValueError('You cannot return overflowing tokens without returning the offsets mapping.'      )

          # first, apply the image processor
          UpperCAmelCase__	:      List[Any]     							=					self.image_processor(images=UpperCamelCase_     ,	return_tensors=UpperCamelCase_      )

          # second, apply the tokenizer
          if text is not None and self.image_processor.apply_ocr and text_pair is None:
              if isinstance(UpperCamelCase_     ,	UpperCamelCase_      ):
                  UpperCAmelCase__	:      Any     							=					[text]  # add batch dimension (as the image processor always adds a batch dimension)
              UpperCAmelCase__	:      Union[str, Any]     							=					features['words']

          UpperCAmelCase__	:      List[Any]     							=					self.tokenizer(
              text=text if text is not None else features['words']     ,	text_pair=text_pair if text_pair is not None else None     ,	boxes=boxes if boxes is not None else features['boxes']     ,	word_labels=UpperCamelCase_     ,	add_special_tokens=UpperCamelCase_     ,	padding=UpperCamelCase_     ,	truncation=UpperCamelCase_     ,	max_length=UpperCamelCase_     ,	stride=UpperCamelCase_     ,	pad_to_multiple_of=UpperCamelCase_     ,	return_token_type_ids=UpperCamelCase_     ,	return_attention_mask=UpperCamelCase_     ,	return_overflowing_tokens=UpperCamelCase_     ,	return_special_tokens_mask=UpperCamelCase_     ,	return_offsets_mapping=UpperCamelCase_     ,	return_length=UpperCamelCase_     ,	verbose=UpperCamelCase_     ,	return_tensors=UpperCamelCase_     ,	**UpperCamelCase_     ,	)

          # add pixel values
          UpperCAmelCase__	:      List[Any]     							=					features.pop('pixel_values'      )
          if return_overflowing_tokens is True:
              UpperCAmelCase__	:      int     							=					self.get_overflowing_images(UpperCamelCase_     ,	encoded_inputs['overflow_to_sample_mapping']      )
          UpperCAmelCase__	:      Dict     							=					images

          return encoded_inputs



      def  __snake_case							(   self     ,	UpperCamelCase_     ,	UpperCamelCase_      ):
          # in case there's an overflow, ensure each `input_ids` sample is mapped to its corresponding image
          UpperCAmelCase__	:      Dict     							=					[]
          for sample_idx in overflow_to_sample_mapping:
              images_with_overflow.append(images[sample_idx]      )

          if len(UpperCamelCase_      ) != len(UpperCamelCase_      ):
              raise ValueError(
                  'Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got'
                  F''' {len(UpperCamelCase_      )} and {len(UpperCamelCase_      )}'''      )

          return images_with_overflow



      def  __snake_case							(   self     ,	*UpperCamelCase_     ,	**UpperCamelCase_      ):
          return self.tokenizer.batch_decode(*UpperCamelCase_     ,	**UpperCamelCase_      )



      def  __snake_case							(   self     ,	*UpperCamelCase_     ,	**UpperCamelCase_      ):
          return self.tokenizer.decode(*UpperCamelCase_     ,	**UpperCamelCase_      )



      @property
      def  __snake_case							(   self      ):
          return ["input_ids", "bbox", "attention_mask", "image"]



      @property
      def  __snake_case							(   self      ):
          warnings.warn(
              '`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'     ,	UpperCamelCase_     ,	)
          return self.image_processor_class



      @property
      def  __snake_case							(   self      ):
          warnings.warn(
              '`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'     ,	UpperCamelCase_     ,	)
          return self.image_processor

