
'''simple docstring'''
import logging
import math
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union

import torch

from .tensor_utils import tensor_tree_map, tree_map






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE				):
							_snake_case       =  []
							if isinstance(_a							, _a				):
														for v in tree.values():
																					shapes.extend(_fetch_dims(_a				)				)
							elif isinstance(_a							, (list, tuple)				):
														for t in tree:
																					shapes.extend(_fetch_dims(_a				)				)
							elif isinstance(_a							, torch.Tensor				):
														shapes.append(tree.shape				)
							else:
														raise ValueError("""Not supported"""				)

							return shapes






@torch.jit.ignore
def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):
							_snake_case       =  []
							for d in reversed(_a				):
														idx.append(flat_idx % d				)
														_snake_case       =  flat_idx // d

							return tuple(reversed(_a				)				)






@torch.jit.ignore
def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE = None							, _SCREAMING_SNAKE_CASE = None							, ):

							# start_edges and end_edges both indicate whether, starting from any given
							# dimension, the start/end index is at the top/bottom edge of the
							# corresponding tensor, modeled as a tree
							def reduce_edge_list(_SCREAMING_SNAKE_CASE				) -> None:
														_snake_case       =  True
														for i in range(len(_a				)				):
																					_snake_case       =  -1 * (i + 1)
																					l[reversed_idx] &= tally
																					_snake_case       =  l[reversed_idx]

							if start_edges is None:
														_snake_case       =  [s == 0 for s in start]
														reduce_edge_list(_a				)
							if end_edges is None:
														_snake_case       =  [e == (d - 1) for e, d in zip(_a							, _a				)]
														reduce_edge_list(_a				)

							# Base cases. Either start/end are empty and we're done, or the final,
							# one-dimensional tensor can be simply sliced
							if len(_a				) == 0:
														return [()]
							elif len(_a				) == 1:
														return [(slice(start[0]							, end[0] + 1				),)]

							_snake_case       =  []
							_snake_case       =  []

							# Dimensions common to start and end can be selected directly
							for s, e in zip(_a							, _a				):
														if s == e:
																					path_list.append(slice(_a							, s + 1				)				)
														else:
																					break

							_snake_case       =  tuple(_a				)
							_snake_case       =  len(_a				)

							# start == end, and we're done
							if divergence_idx == len(_a				):
														return [path]

							def upper() -> Tuple[Tuple[slice, ...], ...]:
														assert start_edges is not None
														assert end_edges is not None

														_snake_case       =  start[divergence_idx]
														return tuple(
														    path + (slice(_a							, sdi + 1				),) + s
														    for s in _get_minimal_slice_set(
														        start[divergence_idx + 1 :]							, [d - 1 for d in dims[divergence_idx + 1 :]]							, dims[divergence_idx + 1 :]							, start_edges=start_edges[divergence_idx + 1 :]							, end_edges=[True for _ in end_edges[divergence_idx + 1 :]]							, )				)

							def lower() -> Tuple[Tuple[slice, ...], ...]:
														assert start_edges is not None
														assert end_edges is not None

														_snake_case       =  end[divergence_idx]
														return tuple(
														    path + (slice(_a							, edi + 1				),) + s
														    for s in _get_minimal_slice_set(
														        [0 for _ in start[divergence_idx + 1 :]]							, end[divergence_idx + 1 :]							, dims[divergence_idx + 1 :]							, start_edges=[True for _ in start_edges[divergence_idx + 1 :]]							, end_edges=end_edges[divergence_idx + 1 :]							, )				)

							# If both start and end are at the edges of the subtree rooted at
							# divergence_idx, we can just select the whole subtree at once
							if start_edges[divergence_idx] and end_edges[divergence_idx]:
														slices.append(path + (slice(start[divergence_idx]							, end[divergence_idx] + 1				),)				)
							# If just start is at the edge, we can grab almost all of the subtree,
							# treating only the ragged bottom edge as an edge case
							elif start_edges[divergence_idx]:
														slices.append(path + (slice(start[divergence_idx]							, end[divergence_idx]				),)				)
														slices.extend(lower()				)
							# Analogous to the previous case, but the top is ragged this time
							elif end_edges[divergence_idx]:
														slices.extend(upper()				)
														slices.append(path + (slice(start[divergence_idx] + 1							, end[divergence_idx] + 1				),)				)
							# If both sides of the range are ragged, we need to handle both sides
							# separately. If there's contiguous meat in between them, we can index it
							# in one big chunk
							else:
														slices.extend(upper()				)
														_snake_case       =  end[divergence_idx] - start[divergence_idx]
														if middle_ground > 1:
																					slices.append(path + (slice(start[divergence_idx] + 1							, end[divergence_idx]				),)				)
														slices.extend(lower()				)

							return slices






@torch.jit.ignore
def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):
							_snake_case       =  t.shape[:no_batch_dims]
							_snake_case       =  list(_flat_idx_to_idx(_a							, _a				)				)
							# _get_minimal_slice_set is inclusive
							_snake_case       =  list(_flat_idx_to_idx(flat_end - 1							, _a				)				)

							# Get an ordered list of slices to perform
							_snake_case       =  _get_minimal_slice_set(
							    _a							, _a							, _a							, )

							_snake_case       =  [t[s] for s in slices]

							return torch.cat([s.view((-1,) + t.shape[no_batch_dims:]				) for s in sliced_tensors]				)






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE = False							, _SCREAMING_SNAKE_CASE = None							, _SCREAMING_SNAKE_CASE = False							, ):
							if not (len(_a				) > 0):
														raise ValueError("""Must provide at least one input"""				)

							_snake_case       =  [shape[:no_batch_dims] for shape in _fetch_dims(_a				)]
							_snake_case       =  tuple([max(_a				) for s in zip(*_a				)]				)

							def _prep_inputs(_SCREAMING_SNAKE_CASE				) -> torch.Tensor:
														if not low_mem:
																					if not sum(t.shape[:no_batch_dims]				) == no_batch_dims:
																												_snake_case       =  t.expand(orig_batch_dims + t.shape[no_batch_dims:]				)
																					_snake_case       =  t.reshape(-1							, *t.shape[no_batch_dims:]				)
														else:
																					_snake_case       =  t.expand(orig_batch_dims + t.shape[no_batch_dims:]				)
														return t

							_snake_case       =  tensor_tree_map(_prep_inputs							, _a				)
							_snake_case       =  None
							if _out is not None:
														_snake_case       =  tensor_tree_map(lambda _SCREAMING_SNAKE_CASE				: t.view([-1] + list(t.shape[no_batch_dims:]				)				)							, _out				)

							_snake_case       =  1
							for d in orig_batch_dims:
														flat_batch_dim *= d

							_snake_case       =  flat_batch_dim // chunk_size + (flat_batch_dim % chunk_size != 0)

							def _select_chunk(_SCREAMING_SNAKE_CASE				) -> torch.Tensor:
														return t[i : i + chunk_size] if t.shape[0] != 1 else t

							_snake_case       =  0
							_snake_case       =  prepped_outputs
							for _ in range(_a				):
														# Chunk the input
														if not low_mem:
																					_snake_case       =  _select_chunk
														else:
																					_snake_case       =  partial(
																					    _chunk_slice							, flat_start=_a							, flat_end=min(_a							, i + chunk_size				)							, no_batch_dims=len(_a				)							, )

														_snake_case       =  tensor_tree_map(_a							, _a				)

														# Run the layer on the chunk
														_snake_case       =  layer(**_a				)

														# Allocate space for the output
														if out is None:
																					_snake_case       =  tensor_tree_map(lambda _SCREAMING_SNAKE_CASE				: t.new_zeros((flat_batch_dim,) + t.shape[1:]				)							, _a				)

														# Put the chunk in its pre-allocated space
														if isinstance(_a							, _a				):

																					def assign(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				) -> None:
																												for k, v in da.items():
																																			if isinstance(_a							, _a				):
																																										assign(_a							, da[k]				)
																																			else:
																																										if _add_into_out:
																																																	v[i : i + chunk_size] += da[k]
																																										else:
																																																	_snake_case       =  da[k]

																					assign(_a							, _a				)
														elif isinstance(_a							, _a				):
																					for xa, xa in zip(_a							, _a				):
																												if _add_into_out:
																																			xa[i : i + chunk_size] += xa
																												else:
																																			_snake_case       =  xa
														elif isinstance(_a							, torch.Tensor				):
																					if _add_into_out:
																												out[i : i + chunk_size] += output_chunk
																					else:
																												_snake_case       =  output_chunk
														else:
																					raise ValueError("""Not supported"""				)

														i += chunk_size

							_snake_case       =  tensor_tree_map(lambda _SCREAMING_SNAKE_CASE				: t.view(orig_batch_dims + t.shape[1:]				)							, _a				)

							return out


class       _lowerCAmelCase   :







							'''simple docstring'''

							def __init__(self  ,  UpperCAmelCase = 512  ,  )					->       List[Any]:
														_snake_case       =  max_chunk_size
														_snake_case       =  None
														_snake_case       =  None

							def 	lowercase							(self  ,  UpperCAmelCase  ,  UpperCAmelCase  ,  UpperCAmelCase  )					->       int:
														logging.info("""Tuning chunk size..."""  )

														if min_chunk_size >= self.max_chunk_size:
																					return min_chunk_size

														_snake_case       =  [2**l for l in range(int(math.log(self.max_chunk_size  ,  2  )  ) + 1  )]
														_snake_case       =  [c for c in candidates if c > min_chunk_size]
														_snake_case       =  [min_chunk_size] + candidates
														candidates[-1] += 4

														def test_chunk_size(UpperCAmelCase  ) -> bool:
																					try:
																												with torch.no_grad():
																																			fn(*_SCREAMING_SNAKE_CASE  ,  chunk_size=_SCREAMING_SNAKE_CASE  )
																												return True
																					except RuntimeError:
																												return False

														_snake_case       =  0
														_snake_case       =  len(_SCREAMING_SNAKE_CASE  ) - 1
														while i > min_viable_chunk_size_index:
																					_snake_case       =  test_chunk_size(candidates[i]  )
																					if not viable:
																												_snake_case       =  (min_viable_chunk_size_index + i) // 2
																					else:
																												_snake_case       =  i
																												_snake_case       =  (i + len(_SCREAMING_SNAKE_CASE  ) - 1) // 2

														return candidates[min_viable_chunk_size_index]

							def 	lowercase							(self  ,  UpperCAmelCase  ,  UpperCAmelCase  )					->       bool:
														_snake_case       =  True
														for aa, aa in zip(_SCREAMING_SNAKE_CASE  ,  _SCREAMING_SNAKE_CASE  ):
																					assert type(_SCREAMING_SNAKE_CASE  ) == type(_SCREAMING_SNAKE_CASE  )
																					if isinstance(_SCREAMING_SNAKE_CASE  ,  (list, tuple)  ):
																												consistent &= self._compare_arg_caches(_SCREAMING_SNAKE_CASE  ,  _SCREAMING_SNAKE_CASE  )
																					elif isinstance(_SCREAMING_SNAKE_CASE  ,  _SCREAMING_SNAKE_CASE  ):
																												_snake_case       =  [v for _, v in sorted(aa.items()  ,  key=lambda UpperCAmelCase  : x[0]  )]
																												_snake_case       =  [v for _, v in sorted(aa.items()  ,  key=lambda UpperCAmelCase  : x[0]  )]
																												consistent &= self._compare_arg_caches(_SCREAMING_SNAKE_CASE  ,  _SCREAMING_SNAKE_CASE  )
																					else:
																												consistent &= aa == aa

														return consistent





							def 	lowercase							(self  ,  UpperCAmelCase  ,  UpperCAmelCase  ,  UpperCAmelCase  ,  )					->       int:
														_snake_case       =  True
														_snake_case       =  tree_map(lambda UpperCAmelCase  : a.shape if isinstance(_SCREAMING_SNAKE_CASE  ,  torch.Tensor  ) else a  ,  _SCREAMING_SNAKE_CASE  ,  _SCREAMING_SNAKE_CASE  )
														if self.cached_arg_data is not None:
																					# If args have changed shape/value, we need to re-tune
																					assert len(self.cached_arg_data  ) == len(_SCREAMING_SNAKE_CASE  )
																					_snake_case       =  self._compare_arg_caches(self.cached_arg_data  ,  _SCREAMING_SNAKE_CASE  )
														else:
																					# Otherwise, we can reuse the precomputed value
																					_snake_case       =  False

														if not consistent:
																					_snake_case       =  self._determine_favorable_chunk_size(
																					    _SCREAMING_SNAKE_CASE  ,  _SCREAMING_SNAKE_CASE  ,  _SCREAMING_SNAKE_CASE  ,  )
																					_snake_case       =  arg_data

														assert self.cached_chunk_size is not None

														return self.cached_chunk_size

'''simple docstring'''
import logging

from transformers.configuration_utils import PretrainedConfig


__lowerCAmelCase												=	logging.getLogger(__name__)


class       _lowerCAmelCase   (   __snake_case				):







							'''simple docstring'''

							lowerCAmelCase_ 						=						"masked_bert"

							def __init__(self  ,  UpperCAmelCase=30522  ,  UpperCAmelCase=768  ,  UpperCAmelCase=12  ,  UpperCAmelCase=12  ,  UpperCAmelCase=3072  ,  UpperCAmelCase="gelu"  ,  UpperCAmelCase=0.1  ,  UpperCAmelCase=0.1  ,  UpperCAmelCase=512  ,  UpperCAmelCase=2  ,  UpperCAmelCase=0.02  ,  UpperCAmelCase=1e-1_2  ,  UpperCAmelCase=0  ,  UpperCAmelCase="topK"  ,  UpperCAmelCase="constant"  ,  UpperCAmelCase=0.0  ,  **UpperCAmelCase  ,  )					->       int:
														super().__init__(pad_token_id=UpperCAmelCase  ,  **UpperCAmelCase  )

														_snake_case       =  vocab_size
														_snake_case       =  hidden_size
														_snake_case       =  num_hidden_layers
														_snake_case       =  num_attention_heads
														_snake_case       =  hidden_act
														_snake_case       =  intermediate_size
														_snake_case       =  hidden_dropout_prob
														_snake_case       =  attention_probs_dropout_prob
														_snake_case       =  max_position_embeddings
														_snake_case       =  type_vocab_size
														_snake_case       =  initializer_range
														_snake_case       =  layer_norm_eps
														_snake_case       =  pruning_method
														_snake_case       =  mask_init
														_snake_case       =  mask_scale