






import json
import os
from dataclasses import dataclass
from functools import partial
from typing import Callable

import flax.linen as nn
import jax
import jax.numpy as jnp
import joblib
import optax
import wandb
from flax import jax_utils, struct, traverse_util
from flax.serialization import from_bytes, to_bytes
from flax.training import train_state
from flax.training.common_utils import shard
from tqdm.auto import tqdm

from transformers import BigBirdConfig, FlaxBigBirdForQuestionAnswering
from transformers.models.big_bird.modeling_flax_big_bird import FlaxBigBirdForQuestionAnsweringModule


class 							__UpperCamelCase     (  _lowerCAmelCase ):
	__snake_case						:BigBirdConfig
	__snake_case						:jnp.dtype =							jnp.floataa
	__snake_case						:bool =							True







	def 			_a (	self   :				Optional[Any]      )		->			Optional[Any]:
		"""simple docstring"""







		super().setup()
		__lowercase   							=	nn.Dense(5							,					dtype=self.dtype      )







	def __call__(	self   :				Optional[int]							,					*_lowerCAmelCase   :				List[str]							,					**_lowerCAmelCase   :				Optional[Any]      )		->			Dict:
		"""simple docstring"""







		__lowercase   							=	super().__call__(*_lowerCAmelCase							,					**_lowerCAmelCase      )
		__lowercase   							=	self.cls(outputs[2]      )
		return outputs[:2] + (cls_out,)


class 							__UpperCamelCase     (  _lowerCAmelCase ):
	__snake_case						:Tuple =							FlaxBigBirdForNaturalQuestionsModule





def 					snake_case	(  lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase       ):




	'''simple docstring'''







	def cross_entropy(lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase=None       ):
		__lowercase   							=	logits.shape[-1]
		__lowercase   							=	(labels[..., None] == jnp.arange(lowerCamelCase       )[None]).astype("""f4"""       )
		__lowercase   							=	jax.nn.log_softmax(lowerCamelCase					,	axis=-1       )
		__lowercase   							=	-jnp.sum(labels * logits					,	axis=-1       )
		if reduction is not None:
			__lowercase   							=	reduction(lowerCamelCase       )
		return loss

	__lowercase   							=	partial(lowerCamelCase					,	reduction=jnp.mean       )
	__lowercase   							=	cross_entropy(lowerCamelCase					,	lowerCamelCase       )
	__lowercase   							=	cross_entropy(lowerCamelCase					,	lowerCamelCase       )
	__lowercase   							=	cross_entropy(lowerCamelCase					,	lowerCamelCase       )
	return (start_loss + end_loss + pooled_loss) / 3


@dataclass
class 							__UpperCamelCase     :
	__snake_case						:str =							"google/bigbird-roberta-base"
	__snake_case						:int =							3_0_0_0
	__snake_case						:int =							1_0_5_0_0

	__snake_case						:int =							1_2_8
	__snake_case						:int =							3

	__snake_case						:int =							1
	__snake_case						:int =							5

	# tx_args
	__snake_case						:float =							3e-5
	__snake_case						:float =							0.0
	__snake_case						:int =							2_0_0_0_0
	__snake_case						:float =							0.00_95

	__snake_case						:str =							"bigbird-roberta-natural-questions"
	__snake_case						:str =							"training-expt"
	__snake_case						:str =							"data/nq-training.jsonl"
	__snake_case						:str =							"data/nq-validation.jsonl"







	def 			_a (	self   :				Dict      )		->			List[str]:
		"""simple docstring"""







		os.makedirs(self.base_dir							,					exist_ok=_lowerCAmelCase      )
		__lowercase   							=	os.path.join(self.base_dir							,					self.save_dir      )
		__lowercase   							=	self.batch_size_per_device * jax.device_count()


@dataclass
class 							__UpperCamelCase     :
	__snake_case						:int
	__snake_case						:int =							4_0_9_6  # no dynamic padding on TPUs







	def __call__(	self   :				Any							,					_lowerCAmelCase   :				str      )		->			Tuple:
		"""simple docstring"""







		__lowercase   							=	self.collate_fn(_lowerCAmelCase      )
		__lowercase   							=	jax.tree_util.tree_map(_lowerCAmelCase							,					_lowerCAmelCase      )
		return batch







	def 			_a (	self   :				Dict							,					_lowerCAmelCase   :				Tuple      )		->			Optional[int]:
		"""simple docstring"""







		__lowercase							,    __lowercase   							=	self.fetch_inputs(features["""input_ids"""]      )
		__lowercase   							=	{
		    """input_ids""": jnp.array(_lowerCAmelCase							,					dtype=jnp.intaa      ),
		    """attention_mask""": jnp.array(_lowerCAmelCase							,					dtype=jnp.intaa      ),
		    """start_labels""": jnp.array(features["""start_token"""]							,					dtype=jnp.intaa      ),
		    """end_labels""": jnp.array(features["""end_token"""]							,					dtype=jnp.intaa      ),
		    """pooled_labels""": jnp.array(features["""category"""]							,					dtype=jnp.intaa      ),
		}
		return batch







	def 			_a (	self   :				Optional[Any]							,					_lowerCAmelCase   :				list      )		->			Optional[Any]:
		"""simple docstring"""







		__lowercase   							=	[self._fetch_inputs(_lowerCAmelCase      ) for ids in input_ids]
		return zip(*_lowerCAmelCase      )







	def 			_a (	self   :				int							,					_lowerCAmelCase   :				list      )		->			Tuple:
		"""simple docstring"""







		__lowercase   							=	[1 for _ in range(len(_lowerCAmelCase      )      )]
		while len(_lowerCAmelCase      ) < self.max_length:
			input_ids.append(self.pad_id      )
			attention_mask.append(0      )
		return input_ids, attention_mask





def 					snake_case	(  lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase=None       ):




	'''simple docstring'''







	if seed is not None:
		__lowercase   							=	dataset.shuffle(seed=lowerCamelCase       )
	for i in range(len(lowerCamelCase       ) // batch_size       ):
		__lowercase   							=	dataset[i * batch_size : (i + 1) * batch_size]
		yield dict(lowerCamelCase       )





@partial(jax.pmap					,	axis_name="""batch"""       )
def 					snake_case	(  lowerCamelCase					,	lowerCamelCase					,	**lowerCamelCase       ):




	'''simple docstring'''







	def loss_fn(lowerCamelCase       ):
		__lowercase   							=	model_inputs.pop("""start_labels"""       )
		__lowercase   							=	model_inputs.pop("""end_labels"""       )
		__lowercase   							=	model_inputs.pop("""pooled_labels"""       )

		__lowercase   							=	state.apply_fn(**lowerCamelCase					,	params=lowerCamelCase					,	dropout_rng=lowerCamelCase					,	train=lowerCamelCase       )
		__lowercase							,    __lowercase							,    __lowercase   							=	outputs

		return state.loss_fn(
		    lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	)

	__lowercase							,    __lowercase   							=	jax.random.split(lowerCamelCase       )
	__lowercase   							=	jax.value_and_grad(lowerCamelCase       )
	__lowercase							,    __lowercase   							=	grad_fn(state.params       )
	__lowercase   							=	jax.lax.pmean({"""loss""": loss}					,	axis_name="""batch"""       )
	__lowercase   							=	jax.lax.pmean(lowerCamelCase					,	"""batch"""       )

	__lowercase   							=	state.apply_gradients(grads=lowerCamelCase       )
	return state, metrics, new_drp_rng





@partial(jax.pmap					,	axis_name="""batch"""       )
def 					snake_case	(  lowerCamelCase					,	**lowerCamelCase       ):




	'''simple docstring'''







	__lowercase   							=	model_inputs.pop("""start_labels"""       )
	__lowercase   							=	model_inputs.pop("""end_labels"""       )
	__lowercase   							=	model_inputs.pop("""pooled_labels"""       )

	__lowercase   							=	state.apply_fn(**lowerCamelCase					,	params=state.params					,	train=lowerCamelCase       )
	__lowercase							,    __lowercase							,    __lowercase   							=	outputs

	__lowercase   							=	state.loss_fn(lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase       )
	__lowercase   							=	jax.lax.pmean({"""loss""": loss}					,	axis_name="""batch"""       )
	return metrics


class 							__UpperCamelCase     (  train_state.TrainState ):
	__snake_case						:Callable =							struct.field(pytree_node=_lowerCAmelCase )


@dataclass
class 							__UpperCamelCase     :
	__snake_case						:Args
	__snake_case						:Callable
	__snake_case						:Callable
	__snake_case						:Callable
	__snake_case						:Callable
	__snake_case						:wandb
	__snake_case						:Callable =							None







	def 			_a (	self   :				Optional[Any]							,					_lowerCAmelCase   :				List[str]							,					_lowerCAmelCase   :				Optional[int]							,					_lowerCAmelCase   :				Tuple							,					_lowerCAmelCase   :				Tuple=None      )		->			int:
		"""simple docstring"""







		__lowercase   							=	model.params
		__lowercase   							=	TrainState.create(
		    apply_fn=model.__call__							,					params=_lowerCAmelCase							,					tx=_lowerCAmelCase							,					loss_fn=_lowerCAmelCase							,					)
		if ckpt_dir is not None:
			__lowercase							,    __lowercase							,    __lowercase							,    __lowercase							,    __lowercase   							=	restore_checkpoint(_lowerCAmelCase							,					_lowerCAmelCase      )
			__lowercase   							=	{
			    """lr""": args.lr,
			    """init_lr""": args.init_lr,
			    """warmup_steps""": args.warmup_steps,
			    """num_train_steps""": num_train_steps,
			    """weight_decay""": args.weight_decay,
			}
			__lowercase							,    __lowercase   							=	build_tx(**_lowerCAmelCase      )
			__lowercase   							=	train_state.TrainState(
			    step=_lowerCAmelCase							,					apply_fn=model.__call__							,					params=_lowerCAmelCase							,					tx=_lowerCAmelCase							,					opt_state=_lowerCAmelCase							,					)
			__lowercase   							=	args
			__lowercase   							=	data_collator
			__lowercase   							=	lr
			__lowercase   							=	params
		__lowercase   							=	jax_utils.replicate(_lowerCAmelCase      )
		return state







	def 			_a (	self   :				str							,					_lowerCAmelCase   :				Any							,					_lowerCAmelCase   :				str							,					_lowerCAmelCase   :				List[str]      )		->			Tuple:
		"""simple docstring"""







		__lowercase   							=	self.args
		__lowercase   							=	len(_lowerCAmelCase      ) // args.batch_size

		__lowercase   							=	jax.random.PRNGKey(0      )
		__lowercase   							=	jax.random.split(_lowerCAmelCase							,					jax.device_count()      )
		for epoch in range(args.max_epochs      ):
			__lowercase   							=	jnp.array(0							,					dtype=jnp.floataa      )
			__lowercase   							=	get_batched_dataset(_lowerCAmelCase							,					args.batch_size							,					seed=_lowerCAmelCase      )
			__lowercase   							=	0
			for batch in tqdm(_lowerCAmelCase							,					total=_lowerCAmelCase							,					desc=F'Running EPOCH-{epoch}'      ):
				__lowercase   							=	self.data_collator(_lowerCAmelCase      )
				__lowercase							,    __lowercase							,    __lowercase   							=	self.train_step_fn(_lowerCAmelCase							,					_lowerCAmelCase							,					**_lowerCAmelCase      )
				running_loss += jax_utils.unreplicate(metrics["""loss"""]      )
				i += 1
				if i % args.logging_steps == 0:
					__lowercase   							=	jax_utils.unreplicate(state.step      )
					__lowercase   							=	running_loss.item() / i
					__lowercase   							=	self.scheduler_fn(state_step - 1      )

					__lowercase   							=	self.evaluate(_lowerCAmelCase							,					_lowerCAmelCase      )
					__lowercase   							=	{
					    """step""": state_step.item(),
					    """eval_loss""": eval_loss.item(),
					    """tr_loss""": tr_loss,
					    """lr""": lr.item(),
					}
					tqdm.write(str(_lowerCAmelCase      )      )
					self.logger.log(_lowerCAmelCase							,					commit=_lowerCAmelCase      )

				if i % args.save_steps == 0:
					self.save_checkpoint(args.save_dir + F'-e{epoch}-s{i}'							,					state=_lowerCAmelCase      )







	def 			_a (	self   :				int							,					_lowerCAmelCase   :				Union[str, Any]							,					_lowerCAmelCase   :				Dict      )		->			List[Any]:
		"""simple docstring"""







		__lowercase   							=	get_batched_dataset(_lowerCAmelCase							,					self.args.batch_size      )
		__lowercase   							=	len(_lowerCAmelCase      ) // self.args.batch_size
		__lowercase   							=	jnp.array(0							,					dtype=jnp.floataa      )
		__lowercase   							=	0
		for batch in tqdm(_lowerCAmelCase							,					total=_lowerCAmelCase							,					desc="""Evaluating ... """      ):
			__lowercase   							=	self.data_collator(_lowerCAmelCase      )
			__lowercase   							=	self.val_step_fn(_lowerCAmelCase							,					**_lowerCAmelCase      )
			running_loss += jax_utils.unreplicate(metrics["""loss"""]      )
			i += 1
		return running_loss / i







	def 			_a (	self   :				Union[str, Any]							,					_lowerCAmelCase   :				Union[str, Any]							,					_lowerCAmelCase   :				List[str]      )		->			Union[str, Any]:
		"""simple docstring"""







		__lowercase   							=	jax_utils.unreplicate(_lowerCAmelCase      )
		print(F'SAVING CHECKPOINT IN {save_dir}'							,					end=""" ... """      )
		self.model_save_fn(_lowerCAmelCase							,					params=state.params      )
		with open(os.path.join(_lowerCAmelCase							,					"""opt_state.msgpack"""      )							,					"""wb"""      ) as f:
			f.write(to_bytes(state.opt_state      )      )
		joblib.dump(self.args							,					os.path.join(_lowerCAmelCase							,					"""args.joblib"""      )      )
		joblib.dump(self.data_collator							,					os.path.join(_lowerCAmelCase							,					"""data_collator.joblib"""      )      )
		with open(os.path.join(_lowerCAmelCase							,					"""training_state.json"""      )							,					"""w"""      ) as f:
			json.dump({"""step""": state.step.item()}							,					_lowerCAmelCase      )
		print("""DONE"""      )





def 					snake_case	(  lowerCamelCase					,	lowerCamelCase       ):




	'''simple docstring'''







	print(F'RESTORING CHECKPOINT FROM {save_dir}'					,	end=""" ... """       )
	with open(os.path.join(lowerCamelCase					,	"""flax_model.msgpack"""       )					,	"""rb"""       ) as f:
		__lowercase   							=	from_bytes(state.params					,	f.read()       )

	with open(os.path.join(lowerCamelCase					,	"""opt_state.msgpack"""       )					,	"""rb"""       ) as f:
		__lowercase   							=	from_bytes(state.opt_state					,	f.read()       )

	__lowercase   							=	joblib.load(os.path.join(lowerCamelCase					,	"""args.joblib"""       )       )
	__lowercase   							=	joblib.load(os.path.join(lowerCamelCase					,	"""data_collator.joblib"""       )       )

	with open(os.path.join(lowerCamelCase					,	"""training_state.json"""       )					,	"""r"""       ) as f:
		__lowercase   							=	json.load(lowerCamelCase       )
	__lowercase   							=	training_state["""step"""]

	print("""DONE"""       )
	return params, opt_state, step, args, data_collator





def 					snake_case	(  lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase       ):




	'''simple docstring'''







	__lowercase   							=	num_train_steps - warmup_steps
	__lowercase   							=	optax.linear_schedule(init_value=lowerCamelCase					,	end_value=lowerCamelCase					,	transition_steps=lowerCamelCase       )
	__lowercase   							=	optax.linear_schedule(init_value=lowerCamelCase					,	end_value=1e-7					,	transition_steps=lowerCamelCase       )
	__lowercase   							=	optax.join_schedules(schedules=[warmup_fn, decay_fn]					,	boundaries=[warmup_steps]       )
	return lr





def 					snake_case	(  lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase       ):




	'''simple docstring'''







	def weight_decay_mask(lowerCamelCase       ):
		__lowercase   							=	traverse_util.flatten_dict(lowerCamelCase       )
		__lowercase   							=	{k: (v[-1] != """bias""" and v[-2:] != ("""LayerNorm""", """scale""")) for k, v in params.items()}
		return traverse_util.unflatten_dict(lowerCamelCase       )

	__lowercase   							=	scheduler_fn(lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase					,	lowerCamelCase       )

	__lowercase   							=	optax.adamw(learning_rate=lowerCamelCase					,	weight_decay=lowerCamelCase					,	mask=lowerCamelCase       )
	return tx, lr







'''simple docstring'''


from __future__ import annotations

from collections import deque
from collections.abc import Iterator
from dataclasses import dataclass

@dataclass
class      A       :
							lowercase_							     =							42
							lowercase_							     =							42



class      A       :







							def __init__(    self			:    Optional[Any]				,     lowerCAmelCase_			:    int   )     ->				str:


														"""simple docstring"""


														_a      	=							[[] for _ in range(lowerCAmelCase_   )]
														_a      	=							size







							def __getitem__(    self			:    Any				,     lowerCAmelCase_			:    int   )     ->				Iterator[Edge]:


														"""simple docstring"""


														return iter(self._graph[vertex]   )







							@property
							def   __lowerCAmelCase     (    self			:    str   )     ->				Tuple:


														"""simple docstring"""


														return self._size







							def   __lowerCAmelCase     (    self			:    Union[str, Any]				,     lowerCAmelCase_			:    int				,     lowerCAmelCase_			:    int				,     lowerCAmelCase_			:    int   )     ->				Dict:


														"""simple docstring"""


														if weight not in (0, 1):
																					raise ValueError('''Edge weight must be either 0 or 1.'''   )

														if to_vertex < 0 or to_vertex >= self.size:
																					raise ValueError('''Vertex indexes must be in [0; size).'''   )

														self._graph[from_vertex].append(Edge(lowerCAmelCase_				,     lowerCAmelCase_   )   )







							def   __lowerCAmelCase     (    self			:    Tuple				,     lowerCAmelCase_			:    int				,     lowerCAmelCase_			:    int   )     ->				int | None:


														"""simple docstring"""


														_a      	=							deque([start_vertex]   )
														_a      	=							[None] * self.size
														_a      	=							0

														while queue:
																					_a      	=							queue.popleft()
																					_a      	=							distances[current_vertex]
																					if current_distance is None:
																												continue

																					for edge in self[current_vertex]:
																												_a      	=							current_distance + edge.weight
																												_a      	=							distances[edge.destination_vertex]
																												if (
																												    isinstance(lowerCAmelCase_				,     lowerCAmelCase_   )
																												    and new_distance >= dest_vertex_distance
																												):
																																			continue
																												_a      	=							new_distance
																												if edge.weight == 0:
																																			queue.appendleft(edge.destination_vertex   )
																												else:
																																			queue.append(edge.destination_vertex   )

														if distances[finish_vertex] is None:
																					raise ValueError('''No path from start_vertex to finish_vertex.'''   )

														return distances[finish_vertex]


if __name__ == "__main__":
				import doctest

				doctest.testmod()

