



'''simple docstring'''




import logging
import math
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union

import torch

from .tensor_utils import tensor_tree_map, tree_map



def       a     (					lowerCamelCase__       ):
			'''simple docstring'''





			A_      :		Tuple      =	[]
			if isinstance(_UpperCAmelCase		, _UpperCAmelCase       ):
						for v in tree.values():
									shapes.extend(_fetch_dims(_UpperCAmelCase       )       )
			elif isinstance(_UpperCAmelCase		, (list, tuple)       ):
						for t in tree:
									shapes.extend(_fetch_dims(_UpperCAmelCase       )       )
			elif isinstance(_UpperCAmelCase		, torch.Tensor       ):
						shapes.append(tree.shape       )
			else:
						raise ValueError("""Not supported"""       )

			return shapes



@torch.jit.ignore
def       a     (					lowerCamelCase__		, lowerCamelCase__       ):
			'''simple docstring'''





			A_      :		Optional[Any]      =	[]
			for d in reversed(_UpperCAmelCase       ):
						idx.append(flat_idx % d       )
						A_      :		Any      =	flat_idx // d

			return tuple(reversed(_UpperCAmelCase       )       )



@torch.jit.ignore
def       a     (					lowerCamelCase__		, lowerCamelCase__		, lowerCamelCase__		, lowerCamelCase__ = None		, lowerCamelCase__ = None		, ):
			'''simple docstring'''





			def reduce_edge_list(lowerCamelCase__       ) -> None:
						A_      :		Optional[int]      =	True
						for i in range(len(_UpperCAmelCase       )       ):
									A_      :		int      =	-1 * (i + 1)
									l[reversed_idx] &= tally
									A_      :		Optional[int]      =	l[reversed_idx]

			if start_edges is None:
						A_      :		Union[str, Any]      =	[s == 0 for s in start]
						reduce_edge_list(_UpperCAmelCase       )
			if end_edges is None:
						A_      :		Union[str, Any]      =	[e == (d - 1) for e, d in zip(_UpperCAmelCase		, _UpperCAmelCase       )]
						reduce_edge_list(_UpperCAmelCase       )

			# Base cases. Either start/end are empty and we're done, or the final,
			# one-dimensional tensor can be simply sliced
			if len(_UpperCAmelCase       ) == 0:
						return [()]
			elif len(_UpperCAmelCase       ) == 1:
						return [(slice(start[0]		, end[0] + 1       ),)]

			A_      :		List[Tuple[slice, ...]]      =	[]
			A_      :		List[slice]      =	[]

			# Dimensions common to start and end can be selected directly
			for s, e in zip(_UpperCAmelCase		, _UpperCAmelCase       ):
						if s == e:
									path_list.append(slice(_UpperCAmelCase		, s + 1       )       )
						else:
									break

			A_      :		Tuple[slice, ...]      =	tuple(_UpperCAmelCase       )
			A_      :		List[Any]      =	len(_UpperCAmelCase       )

			# start == end, and we're done
			if divergence_idx == len(_UpperCAmelCase       ):
						return [path]

			def upper() -> Tuple[Tuple[slice, ...], ...]:
						assert start_edges is not None
						assert end_edges is not None

						A_      :		List[Any]      =	start[divergence_idx]
						return tuple(
						    path + (slice(_UpperCAmelCase		, sdi + 1       ),) + s
						    for s in _get_minimal_slice_set(
						        start[divergence_idx + 1 :]		, [d - 1 for d in dims[divergence_idx + 1 :]]		, dims[divergence_idx + 1 :]		, start_edges=start_edges[divergence_idx + 1 :]		, end_edges=[True for _ in end_edges[divergence_idx + 1 :]]		, )       )

			def lower() -> Tuple[Tuple[slice, ...], ...]:
						assert start_edges is not None
						assert end_edges is not None

						A_      :		str      =	end[divergence_idx]
						return tuple(
						    path + (slice(_UpperCAmelCase		, edi + 1       ),) + s
						    for s in _get_minimal_slice_set(
						        [0 for _ in start[divergence_idx + 1 :]]		, end[divergence_idx + 1 :]		, dims[divergence_idx + 1 :]		, start_edges=[True for _ in start_edges[divergence_idx + 1 :]]		, end_edges=end_edges[divergence_idx + 1 :]		, )       )

			# If both start and end are at the edges of the subtree rooted at
			# divergence_idx, we can just select the whole subtree at once
			if start_edges[divergence_idx] and end_edges[divergence_idx]:
						slices.append(path + (slice(start[divergence_idx]		, end[divergence_idx] + 1       ),)       )
			# If just start is at the edge, we can grab almost all of the subtree,
			# treating only the ragged bottom edge as an edge case
			elif start_edges[divergence_idx]:
						slices.append(path + (slice(start[divergence_idx]		, end[divergence_idx]       ),)       )
						slices.extend(lower()       )
			# Analogous to the previous case, but the top is ragged this time
			elif end_edges[divergence_idx]:
						slices.extend(upper()       )
						slices.append(path + (slice(start[divergence_idx] + 1		, end[divergence_idx] + 1       ),)       )
			# If both sides of the range are ragged, we need to handle both sides
			# separately. If there's contiguous meat in between them, we can index it
			# in one big chunk
			else:
						slices.extend(upper()       )
						A_      :		Dict      =	end[divergence_idx] - start[divergence_idx]
						if middle_ground > 1:
									slices.append(path + (slice(start[divergence_idx] + 1		, end[divergence_idx]       ),)       )
						slices.extend(lower()       )

			return slices



@torch.jit.ignore
def       a     (					lowerCamelCase__		, lowerCamelCase__		, lowerCamelCase__		, lowerCamelCase__       ):
			'''simple docstring'''





			A_      :		Optional[int]      =	t.shape[:no_batch_dims]
			A_      :		Any      =	list(_flat_idx_to_idx(_UpperCAmelCase		, _UpperCAmelCase       )       )
			# _get_minimal_slice_set is inclusive
			A_      :		Optional[int]      =	list(_flat_idx_to_idx(flat_end - 1		, _UpperCAmelCase       )       )

			# Get an ordered list of slices to perform
			A_      :		Union[str, Any]      =	_get_minimal_slice_set(
			    _UpperCAmelCase		, _UpperCAmelCase		, _UpperCAmelCase		, )

			A_      :		Any      =	[t[s] for s in slices]

			return torch.cat([s.view((-1,) + t.shape[no_batch_dims:]       ) for s in sliced_tensors]       )



def       a     (					lowerCamelCase__		, lowerCamelCase__		, lowerCamelCase__		, lowerCamelCase__		, lowerCamelCase__ = False		, lowerCamelCase__ = None		, lowerCamelCase__ = False		, ):
			'''simple docstring'''





			if not (len(_UpperCAmelCase       ) > 0):
						raise ValueError("""Must provide at least one input"""       )

			A_      :		int      =	[shape[:no_batch_dims] for shape in _fetch_dims(_UpperCAmelCase       )]
			A_      :		Optional[int]      =	tuple([max(_UpperCAmelCase       ) for s in zip(*_UpperCAmelCase       )]       )

			def _prep_inputs(lowerCamelCase__       ) -> torch.Tensor:
						if not low_mem:
									if not sum(t.shape[:no_batch_dims]       ) == no_batch_dims:
												A_      :		Optional[Any]      =	t.expand(orig_batch_dims + t.shape[no_batch_dims:]       )
									A_      :		Optional[Any]      =	t.reshape(-1		, *t.shape[no_batch_dims:]       )
						else:
									A_      :		List[str]      =	t.expand(orig_batch_dims + t.shape[no_batch_dims:]       )
						return t

			A_      :		Dict[str, Any]      =	tensor_tree_map(_prep_inputs		, _UpperCAmelCase       )
			A_      :		Any      =	None
			if _out is not None:
						A_      :		Any      =	tensor_tree_map(lambda lowerCamelCase__       : t.view([-1] + list(t.shape[no_batch_dims:]       )       )		, _out       )

			A_      :		Union[str, Any]      =	1
			for d in orig_batch_dims:
						flat_batch_dim *= d

			A_      :		Any      =	flat_batch_dim // chunk_size + (flat_batch_dim % chunk_size != 0)

			def _select_chunk(lowerCamelCase__       ) -> torch.Tensor:
						return t[i : i + chunk_size] if t.shape[0] != 1 else t

			A_      :		int      =	0
			A_      :		List[str]      =	prepped_outputs
			for _ in range(_UpperCAmelCase       ):
						# Chunk the input
						if not low_mem:
									A_      :		str      =	_select_chunk
						else:
									A_      :		Tuple      =	partial(
									    _chunk_slice		, flat_start=_UpperCAmelCase		, flat_end=min(_UpperCAmelCase		, i + chunk_size       )		, no_batch_dims=len(_UpperCAmelCase       )		, )

						A_      :		Dict[str, Any]      =	tensor_tree_map(_UpperCAmelCase		, _UpperCAmelCase       )

						# Run the layer on the chunk
						A_      :		Union[str, Any]      =	layer(**_UpperCAmelCase       )

						# Allocate space for the output
						if out is None:
									A_      :		Union[str, Any]      =	tensor_tree_map(lambda lowerCamelCase__       : t.new_zeros((flat_batch_dim,) + t.shape[1:]       )		, _UpperCAmelCase       )

						# Put the chunk in its pre-allocated space
						if isinstance(_UpperCAmelCase		, _UpperCAmelCase       ):

									def assign(lowerCamelCase__		, lowerCamelCase__       ) -> None:
												for k, v in da.items():
															if isinstance(_UpperCAmelCase		, _UpperCAmelCase       ):
																		assign(_UpperCAmelCase		, da[k]       )
															else:
																		if _add_into_out:
																					v[i : i + chunk_size] += da[k]
																		else:
																					A_      :		List[Any]      =	da[k]

									assign(_UpperCAmelCase		, _UpperCAmelCase       )
						elif isinstance(_UpperCAmelCase		, _UpperCAmelCase       ):
									for xa, xa in zip(_UpperCAmelCase		, _UpperCAmelCase       ):
												if _add_into_out:
															xa[i : i + chunk_size] += xa
												else:
															A_      :		List[str]      =	xa
						elif isinstance(_UpperCAmelCase		, torch.Tensor       ):
									if _add_into_out:
												out[i : i + chunk_size] += output_chunk
									else:
												A_      :		Dict      =	output_chunk
						else:
									raise ValueError("""Not supported"""       )

						i += chunk_size

			A_      :		List[str]      =	tensor_tree_map(lambda lowerCamelCase__       : t.view(orig_batch_dims + t.shape[1:]       )		, _UpperCAmelCase       )

			return out







class       _lowerCAmelCase				:


	def __init__(self			, lowercase = 512			, ):
				A_      :		List[Any]      =	max_chunk_size
				A_      :		Optional[int]      =	None
				A_      :		Optional[tuple]      =	None


	def        _a      (self			, lowercase			, lowercase			, lowercase				):
				logging.info("""Tuning chunk size..."""				)

				if min_chunk_size >= self.max_chunk_size:
							return min_chunk_size

				A_      :		List[int]      =	[2**l for l in range(int(math.log(self.max_chunk_size			, 2				)				) + 1				)]
				A_      :		List[str]      =	[c for c in candidates if c > min_chunk_size]
				A_      :		List[str]      =	[min_chunk_size] + candidates
				candidates[-1] += 4

				def test_chunk_size(lowercase				) -> bool:
							try:
										with torch.no_grad():
													fn(*__a			, chunk_size=__a				)
										return True
							except RuntimeError:
										return False

				A_      :		Optional[int]      =	0
				A_      :		str      =	len(__a				) - 1
				while i > min_viable_chunk_size_index:
							A_      :		str      =	test_chunk_size(candidates[i]				)
							if not viable:
										A_      :		List[str]      =	(min_viable_chunk_size_index + i) // 2
							else:
										A_      :		str      =	i
										A_      :		Optional[int]      =	(i + len(__a				) - 1) // 2

				return candidates[min_viable_chunk_size_index]


	def        _a      (self			, lowercase			, lowercase				):
				A_      :		int      =	True
				for aa, aa in zip(__a			, __a				):
							assert type(__a				) == type(__a				)
							if isinstance(__a			, (list, tuple)				):
										consistent &= self._compare_arg_caches(__a			, __a				)
							elif isinstance(__a			, __a				):
										A_      :		str      =	[v for _, v in sorted(aa.items()			, key=lambda lowercase				: x[0]				)]
										A_      :		str      =	[v for _, v in sorted(aa.items()			, key=lambda lowercase				: x[0]				)]
										consistent &= self._compare_arg_caches(__a			, __a				)
							else:
										consistent &= aa == aa

				return consistent


	def        _a      (self			, lowercase			, lowercase			, lowercase			, ):
				A_      :		Dict      =	True
				A_      :		tuple      =	tree_map(lambda lowercase				: a.shape if isinstance(__a			, torch.Tensor				) else a			, __a			, __a				)
				if self.cached_arg_data is not None:
							# If args have changed shape/value, we need to re-tune
							assert len(self.cached_arg_data				) == len(__a				)
							A_      :		str      =	self._compare_arg_caches(self.cached_arg_data			, __a				)
				else:
							# Otherwise, we can reuse the precomputed value
							A_      :		List[str]      =	False

				if not consistent:
							A_      :		str      =	self._determine_favorable_chunk_size(
							    __a			, __a			, __a			, )
							A_      :		List[Any]      =	arg_data

				assert self.cached_chunk_size is not None

				return self.cached_chunk_size




'''simple docstring'''





import json
import logging
import os
import sys
from time import time
from unittest.mock import patch

from transformers.testing_utils import TestCasePlus, require_torch_tpu


logging.basicConfig(level=logging.DEBUG)

A__    :   List[str]           =					logging.getLogger()





def   a_   (		_UpperCAmelCase    :      int				)    ->  Optional[int]:
      __snake_case :   List[str]													=    {}
      __snake_case :   List[str]													=    os.path.join(_UpperCAmelCase ,'all_results.json'				)
      if os.path.exists(_UpperCAmelCase				):
            with open(_UpperCAmelCase ,'r'				) as f:
                  __snake_case :   Dict													=    json.load(_UpperCAmelCase				)
      else:
            raise ValueError(f'''can\'t find {path}'''				)
      return results


A__    :   Dict           =					logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)





@require_torch_tpu
class  snake_case__						(    SCREAMING_SNAKE_CASE_				):
 def A_   (   self						:							Optional[Any]  )      ->   List[str]:







       '''simple docstring'''
       import xla_spawn

       __snake_case :   int													=    self.get_auto_remove_tmp_dir()
       __snake_case :   Tuple													=    f'''
            ./examples/pytorch/text-classification/run_glue.py
            --num_cores=8
            ./examples/pytorch/text-classification/run_glue.py
            --model_name_or_path distilbert-base-uncased
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv
            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv
            --do_train
            --do_eval
            --debug tpu_metrics_debug
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --learning_rate=1e-4
            --max_steps=10
            --warmup_steps=2
            --seed=42
            --max_seq_length=128
            '''.split()

       with patch.object(__a      ,     'argv'      ,     __a  ):
             __snake_case :   Tuple													=    time()
             xla_spawn.main()
             __snake_case :   Optional[int]													=    time()

             __snake_case :   List[Any]													=    get_results(__a  )
             self.assertGreaterEqual(result['eval_accuracy']      ,     0.7_5  )

             # Assert that the script takes less than 500 seconds to make sure it doesn't hang.
             self.assertLess(end - start      ,     500  )





 def A_   (   self						:							List[str]  )      ->   Optional[int]:







       '''simple docstring'''
       import xla_spawn

       __snake_case :   Optional[Any]													=    '\n            ./tests/test_trainer_tpu.py\n            --num_cores=8\n            ./tests/test_trainer_tpu.py\n            '.split()
       with patch.object(__a      ,     'argv'      ,     __a  ):
             xla_spawn.main()



