





import json
import os
from dataclasses import dataclass
from functools import partial
from typing import Callable

import flax.linen as nn
import jax
import jax.numpy as jnp
import joblib
import optax
import wandb
from flax import jax_utils, struct, traverse_util
from flax.serialization import from_bytes, to_bytes
from flax.training import train_state
from flax.training.common_utils import shard
from tqdm.auto import tqdm

from transformers import BigBirdConfig, FlaxBigBirdForQuestionAnswering
from transformers.models.big_bird.modeling_flax_big_bird import FlaxBigBirdForQuestionAnsweringModule

class 		snake_case_   (							UpperCAmelCase_						):







	'''simple docstring'''
	__UpperCamelCase   =		42
	__UpperCamelCase   =		jnp.floataa
	__UpperCamelCase   =		True




	def    UpperCAmelCase			(						self							:     str   )       -> Any:

			'''simple docstring'''




			super().setup()
			__lowercase											=     nn.Dense(5					,      dtype=self.dtype   )






	def __call__(						self							:     List[Any]					,      *__lowerCamelCase							:     Optional[int]					,      **__lowerCamelCase							:     int   )       -> List[str]:

			'''simple docstring'''




			__lowercase											=     super().__call__(*__lowerCamelCase					,      **__lowerCamelCase   )
			__lowercase											=     self.cls(outputs[2]   )
			return outputs[:2] + (cls_out,)





class 		snake_case_   (							UpperCAmelCase_						):







	'''simple docstring'''
	__UpperCamelCase   =		FlaxBigBirdForNaturalQuestionsModule





def  SCREAMING_SNAKE_CASE				(      snake_case						,   snake_case						,   snake_case						,   snake_case						,   snake_case						,   snake_case							)		->    List[str]:
		def cross_entropy(snake_case						,   snake_case						,   snake_case=None							):
				__lowercase											=     logits.shape[-1]
				__lowercase											=     (labels[..., None] == jnp.arange(snake_case							)[None]).astype('f4'							)
				__lowercase											=     jax.nn.log_softmax(snake_case						,   axis=-1							)
				__lowercase											=     -jnp.sum(labels * logits						,   axis=-1							)
				if reduction is not None:
						__lowercase											=     reduction(snake_case							)
				return loss

		__lowercase											=     partial(snake_case						,   reduction=jnp.mean							)
		__lowercase											=     cross_entropy(snake_case						,   snake_case							)
		__lowercase											=     cross_entropy(snake_case						,   snake_case							)
		__lowercase											=     cross_entropy(snake_case						,   snake_case							)
		return (start_loss + end_loss + pooled_loss) / 3

@dataclass
class 		snake_case_   :







	'''simple docstring'''
	__UpperCamelCase   =		"google/bigbird-roberta-base"
	__UpperCamelCase   =		30_00
	__UpperCamelCase   =		1_05_00

	__UpperCamelCase   =		1_28
	__UpperCamelCase   =		3

	__UpperCamelCase   =		1
	__UpperCamelCase   =		5

	# tx_args
	__UpperCamelCase   =		3E-5
	__UpperCamelCase   =		0.0
	__UpperCamelCase   =		2_00_00
	__UpperCamelCase   =		0.0_0_9_5

	__UpperCamelCase   =		"bigbird-roberta-natural-questions"
	__UpperCamelCase   =		"training-expt"
	__UpperCamelCase   =		"data/nq-training.jsonl"
	__UpperCamelCase   =		"data/nq-validation.jsonl"




	def    UpperCAmelCase			(						self							:     Tuple   )       -> Union[str, Any]:

			'''simple docstring'''




			os.makedirs(self.base_dir					,      exist_ok=__lowerCamelCase   )
			__lowercase											=     os.path.join(self.base_dir					,      self.save_dir   )
			__lowercase											=     self.batch_size_per_device * jax.device_count()





@dataclass
class 		snake_case_   :







	'''simple docstring'''
	__UpperCamelCase   =		42
	__UpperCamelCase   =		40_96  # no dynamic padding on TPUs




	def __call__(						self							:     Any					,      __lowerCamelCase							:     Tuple   )       -> int:

			'''simple docstring'''




			__lowercase											=     self.collate_fn(__lowerCamelCase   )
			__lowercase											=     jax.tree_util.tree_map(__lowerCamelCase					,      __lowerCamelCase   )
			return batch




	def    UpperCAmelCase			(						self							:     Dict					,      __lowerCamelCase							:     Union[str, Any]   )       -> Dict:

			'''simple docstring'''




			__lowercase						,      __lowercase											=     self.fetch_inputs(features['input_ids']   )
			__lowercase											=     {
			    'input_ids': jnp.array(__lowerCamelCase					,      dtype=jnp.intaa   ),
			    'attention_mask': jnp.array(__lowerCamelCase					,      dtype=jnp.intaa   ),
			    'start_labels': jnp.array(features['start_token']					,      dtype=jnp.intaa   ),
			    'end_labels': jnp.array(features['end_token']					,      dtype=jnp.intaa   ),
			    'pooled_labels': jnp.array(features['category']					,      dtype=jnp.intaa   ),
			}
			return batch




	def    UpperCAmelCase			(						self							:     Any					,      __lowerCamelCase							:     list   )       -> Union[str, Any]:

			'''simple docstring'''




			__lowercase											=     [self._fetch_inputs(__lowerCamelCase   ) for ids in input_ids]
			return zip(*__lowerCamelCase   )






	def    UpperCAmelCase			(						self							:     str					,      __lowerCamelCase							:     list   )       -> Union[str, Any]:

			'''simple docstring'''




			__lowercase											=     [1 for _ in range(len(__lowerCamelCase   )   )]
			while len(__lowerCamelCase   ) < self.max_length:
					input_ids.append(self.pad_id   )
					attention_mask.append(0   )
			return input_ids, attention_mask





def  SCREAMING_SNAKE_CASE				(      snake_case						,   snake_case						,   snake_case=None							)		->    Union[str, Any]:
		if seed is not None:
				__lowercase											=     dataset.shuffle(seed=snake_case							)
		for i in range(len(snake_case							) // batch_size							):
				__lowercase											=     dataset[i * batch_size : (i + 1) * batch_size]
				yield dict(snake_case							)



@partial(jax.pmap						,   axis_name='batch'							)
def  SCREAMING_SNAKE_CASE				(      snake_case						,   snake_case						,   **snake_case							)		->    Dict:
		def loss_fn(snake_case							):
				__lowercase											=     model_inputs.pop('start_labels'							)
				__lowercase											=     model_inputs.pop('end_labels'							)
				__lowercase											=     model_inputs.pop('pooled_labels'							)

				__lowercase											=     state.apply_fn(**snake_case						,   params=snake_case						,   dropout_rng=snake_case						,   train=snake_case							)
				__lowercase						,      __lowercase						,      __lowercase											=     outputs

				return state.loss_fn(
				    snake_case						,   snake_case						,   snake_case						,   snake_case						,   snake_case						,   snake_case						,   )

		__lowercase						,      __lowercase											=     jax.random.split(snake_case							)
		__lowercase											=     jax.value_and_grad(snake_case							)
		__lowercase						,      __lowercase											=     grad_fn(state.params							)
		__lowercase											=     jax.lax.pmean({'loss': loss}						,   axis_name='batch'							)
		__lowercase											=     jax.lax.pmean(snake_case						,   'batch'							)

		__lowercase											=     state.apply_gradients(grads=snake_case							)
		return state, metrics, new_drp_rng



@partial(jax.pmap						,   axis_name='batch'							)
def  SCREAMING_SNAKE_CASE				(      snake_case						,   **snake_case							)		->    Optional[int]:
		__lowercase											=     model_inputs.pop('start_labels'							)
		__lowercase											=     model_inputs.pop('end_labels'							)
		__lowercase											=     model_inputs.pop('pooled_labels'							)

		__lowercase											=     state.apply_fn(**snake_case						,   params=state.params						,   train=snake_case							)
		__lowercase						,      __lowercase						,      __lowercase											=     outputs

		__lowercase											=     state.loss_fn(snake_case						,   snake_case						,   snake_case						,   snake_case						,   snake_case						,   snake_case							)
		__lowercase											=     jax.lax.pmean({'loss': loss}						,   axis_name='batch'							)
		return metrics

class 		snake_case_   (							train_state.TrainState						):







	'''simple docstring'''
	__UpperCamelCase   =		struct.field(pytree_node=UpperCAmelCase_						)





@dataclass
class 		snake_case_   :







	'''simple docstring'''
	__UpperCamelCase   =		42
	__UpperCamelCase   =		42
	__UpperCamelCase   =		42
	__UpperCamelCase   =		42
	__UpperCamelCase   =		42
	__UpperCamelCase   =		42
	__UpperCamelCase   =		None




	def    UpperCAmelCase			(						self							:     Optional[Any]					,      __lowerCamelCase							:     Tuple					,      __lowerCamelCase							:     List[str]					,      __lowerCamelCase							:     List[str]					,      __lowerCamelCase							:     Optional[int]=None   )       -> Tuple:

			'''simple docstring'''




			__lowercase											=     model.params
			__lowercase											=     TrainState.create(
			    apply_fn=model.__call__					,      params=__lowerCamelCase					,      tx=__lowerCamelCase					,      loss_fn=__lowerCamelCase					,      )
			if ckpt_dir is not None:
					__lowercase						,      __lowercase						,      __lowercase						,      __lowercase						,      __lowercase											=     restore_checkpoint(__lowerCamelCase					,      __lowerCamelCase   )
					__lowercase											=     {
					    'lr': args.lr,
					    'init_lr': args.init_lr,
					    'warmup_steps': args.warmup_steps,
					    'num_train_steps': num_train_steps,
					    'weight_decay': args.weight_decay,
					}
					__lowercase						,      __lowercase											=     build_tx(**__lowerCamelCase   )
					__lowercase											=     train_state.TrainState(
					    step=__lowerCamelCase					,      apply_fn=model.__call__					,      params=__lowerCamelCase					,      tx=__lowerCamelCase					,      opt_state=__lowerCamelCase					,      )
					__lowercase											=     args
					__lowercase											=     data_collator
					__lowercase											=     lr
					__lowercase											=     params
			__lowercase											=     jax_utils.replicate(__lowerCamelCase   )
			return state




	def    UpperCAmelCase			(						self							:     Dict					,      __lowerCamelCase							:     Any					,      __lowerCamelCase							:     Optional[int]					,      __lowerCamelCase							:     Dict   )       -> Tuple:

			'''simple docstring'''




			__lowercase											=     self.args
			__lowercase											=     len(__lowerCamelCase   ) // args.batch_size

			__lowercase											=     jax.random.PRNGKey(0   )
			__lowercase											=     jax.random.split(__lowerCamelCase					,      jax.device_count()   )
			for epoch in range(args.max_epochs   ):
					__lowercase											=     jnp.array(0					,      dtype=jnp.floataa   )
					__lowercase											=     get_batched_dataset(__lowerCamelCase					,      args.batch_size					,      seed=__lowerCamelCase   )
					__lowercase											=     0
					for batch in tqdm(__lowerCamelCase					,      total=__lowerCamelCase					,      desc=F"Running EPOCH-{epoch}"   ):
							__lowercase											=     self.data_collator(__lowerCamelCase   )
							__lowercase						,      __lowercase						,      __lowercase											=     self.train_step_fn(__lowerCamelCase					,      __lowerCamelCase					,      **__lowerCamelCase   )
							running_loss += jax_utils.unreplicate(metrics['loss']   )
							i += 1
							if i % args.logging_steps == 0:
									__lowercase											=     jax_utils.unreplicate(state.step   )
									__lowercase											=     running_loss.item() / i
									__lowercase											=     self.scheduler_fn(state_step - 1   )

									__lowercase											=     self.evaluate(__lowerCamelCase					,      __lowerCamelCase   )
									__lowercase											=     {
									    'step': state_step.item(),
									    'eval_loss': eval_loss.item(),
									    'tr_loss': tr_loss,
									    'lr': lr.item(),
									}
									tqdm.write(str(__lowerCamelCase   )   )
									self.logger.log(__lowerCamelCase					,      commit=__lowerCamelCase   )

							if i % args.save_steps == 0:
									self.save_checkpoint(args.save_dir + F"-e{epoch}-s{i}"					,      state=__lowerCamelCase   )




	def    UpperCAmelCase			(						self							:     Union[str, Any]					,      __lowerCamelCase							:     int					,      __lowerCamelCase							:     Optional[int]   )       -> List[Any]:

			'''simple docstring'''




			__lowercase											=     get_batched_dataset(__lowerCamelCase					,      self.args.batch_size   )
			__lowercase											=     len(__lowerCamelCase   ) // self.args.batch_size
			__lowercase											=     jnp.array(0					,      dtype=jnp.floataa   )
			__lowercase											=     0
			for batch in tqdm(__lowerCamelCase					,      total=__lowerCamelCase					,      desc='Evaluating ... '   ):
					__lowercase											=     self.data_collator(__lowerCamelCase   )
					__lowercase											=     self.val_step_fn(__lowerCamelCase					,      **__lowerCamelCase   )
					running_loss += jax_utils.unreplicate(metrics['loss']   )
					i += 1
			return running_loss / i






	def    UpperCAmelCase			(						self							:     List[str]					,      __lowerCamelCase							:     Optional[int]					,      __lowerCamelCase							:     Optional[int]   )       -> Optional[Any]:

			'''simple docstring'''




			__lowercase											=     jax_utils.unreplicate(__lowerCamelCase   )
			print(F"SAVING CHECKPOINT IN {save_dir}"					,      end=' ... '   )
			self.model_save_fn(__lowerCamelCase					,      params=state.params   )
			with open(os.path.join(__lowerCamelCase					,      'opt_state.msgpack'   )					,      'wb'   ) as f:
					f.write(to_bytes(state.opt_state   )   )
			joblib.dump(self.args					,      os.path.join(__lowerCamelCase					,      'args.joblib'   )   )
			joblib.dump(self.data_collator					,      os.path.join(__lowerCamelCase					,      'data_collator.joblib'   )   )
			with open(os.path.join(__lowerCamelCase					,      'training_state.json'   )					,      'w'   ) as f:
					json.dump({'step': state.step.item()}					,      __lowerCamelCase   )
			print('DONE'   )





def  SCREAMING_SNAKE_CASE				(      snake_case						,   snake_case							)		->    Any:
		print(F"RESTORING CHECKPOINT FROM {save_dir}"						,   end=' ... '							)
		with open(os.path.join(snake_case						,   'flax_model.msgpack'							)						,   'rb'							) as f:
				__lowercase											=     from_bytes(state.params						,   f.read()							)

		with open(os.path.join(snake_case						,   'opt_state.msgpack'							)						,   'rb'							) as f:
				__lowercase											=     from_bytes(state.opt_state						,   f.read()							)

		__lowercase											=     joblib.load(os.path.join(snake_case						,   'args.joblib'							)							)
		__lowercase											=     joblib.load(os.path.join(snake_case						,   'data_collator.joblib'							)							)

		with open(os.path.join(snake_case						,   'training_state.json'							)						,   'r'							) as f:
				__lowercase											=     json.load(snake_case							)
		__lowercase											=     training_state['step']

		print('DONE'							)
		return params, opt_state, step, args, data_collator



def  SCREAMING_SNAKE_CASE				(      snake_case						,   snake_case						,   snake_case						,   snake_case							)		->    int:
		__lowercase											=     num_train_steps - warmup_steps
		__lowercase											=     optax.linear_schedule(init_value=snake_case						,   end_value=snake_case						,   transition_steps=snake_case							)
		__lowercase											=     optax.linear_schedule(init_value=snake_case						,   end_value=1E-7						,   transition_steps=snake_case							)
		__lowercase											=     optax.join_schedules(schedules=[warmup_fn, decay_fn]						,   boundaries=[warmup_steps]							)
		return lr



def  SCREAMING_SNAKE_CASE				(      snake_case						,   snake_case						,   snake_case						,   snake_case						,   snake_case							)		->    str:
		def weight_decay_mask(snake_case							):
				__lowercase											=     traverse_util.flatten_dict(snake_case							)
				__lowercase											=     {k: (v[-1] != 'bias' and v[-2:] != ('LayerNorm', 'scale')) for k, v in params.items()}
				return traverse_util.unflatten_dict(snake_case							)

		__lowercase											=     scheduler_fn(snake_case						,   snake_case						,   snake_case						,   snake_case							)

		__lowercase											=     optax.adamw(learning_rate=snake_case						,   weight_decay=snake_case						,   mask=snake_case							)
		return tx, lr











class 		snake_case_   :







	'''simple docstring'''




	def __init__(						self							:     Dict					,      __lowerCamelCase							:     int					,      __lowerCamelCase							:     Optional[Any]=None					,      __lowerCamelCase							:     Optional[Any]=None   )       -> Dict:

			'''simple docstring'''




			__lowercase											=     data
			__lowercase											=     previous
			__lowercase											=     next_node




	def __str__(						self							:     int   )       -> str:

			'''simple docstring'''




			return F"{self.data}"




	def    UpperCAmelCase			(						self							:     int   )       -> int:

			'''simple docstring'''




			return self.data




	def    UpperCAmelCase			(						self							:     Tuple   )       -> Optional[int]:

			'''simple docstring'''




			return self.next






	def    UpperCAmelCase			(						self							:     Optional[int]   )       -> Any:

			'''simple docstring'''




			return self.previous





class 		snake_case_   :







	'''simple docstring'''




	def __init__(						self							:     Tuple					,      __lowerCamelCase							:     Optional[Any]   )       -> str:

			'''simple docstring'''




			__lowercase											=     head




	def __iter__(						self							:     Union[str, Any]   )       -> List[Any]:

			'''simple docstring'''




			return self






	def    UpperCAmelCase			(						self							:     str   )       -> str:

			'''simple docstring'''




			if not self.current:
					raise StopIteration
			else:
					__lowercase											=     self.current.get_data()
					__lowercase											=     self.current.get_next()
					return value





class 		snake_case_   :







	'''simple docstring'''




	def __init__(						self							:     List[str]   )       -> Any:

			'''simple docstring'''




			__lowercase											=     None  # First node in list
			__lowercase											=     None  # Last node in list




	def __str__(						self							:     int   )       -> Any:

			'''simple docstring'''




			__lowercase											=     self.head
			__lowercase											=     []
			while current is not None:
					nodes.append(current.get_data()   )
					__lowercase											=     current.get_next()
			return " ".join(str(__lowerCamelCase   ) for node in nodes   )




	def __contains__(						self							:     Tuple					,      __lowerCamelCase							:     int   )       -> List[Any]:

			'''simple docstring'''




			__lowercase											=     self.head
			while current:
					if current.get_data() == value:
							return True
					__lowercase											=     current.get_next()
			return False




	def __iter__(						self							:     Optional[int]   )       -> List[str]:

			'''simple docstring'''




			return LinkedListIterator(self.head   )




	def    UpperCAmelCase			(						self							:     Any   )       -> str:

			'''simple docstring'''




			if self.head:
					return self.head.get_data()
			return None




	def    UpperCAmelCase			(						self							:     str   )       -> Union[str, Any]:

			'''simple docstring'''




			if self.tail:
					return self.tail.get_data()
			return None




	def    UpperCAmelCase			(						self							:     str					,      __lowerCamelCase							:     Node   )       -> None:

			'''simple docstring'''




			if self.head is None:
					__lowercase											=     node
					__lowercase											=     node
			else:
					self.insert_before_node(self.head					,      __lowerCamelCase   )




	def    UpperCAmelCase			(						self							:     Dict					,      __lowerCamelCase							:     Node   )       -> None:

			'''simple docstring'''




			if self.head is None:
					self.set_head(__lowerCamelCase   )
			else:
					self.insert_after_node(self.tail					,      __lowerCamelCase   )




	def    UpperCAmelCase			(						self							:     Optional[Any]					,      __lowerCamelCase							:     int   )       -> None:

			'''simple docstring'''




			__lowercase											=     Node(__lowerCamelCase   )
			if self.head is None:
					self.set_head(__lowerCamelCase   )
			else:
					self.set_tail(__lowerCamelCase   )




	def    UpperCAmelCase			(						self							:     int					,      __lowerCamelCase							:     Node					,      __lowerCamelCase							:     Node   )       -> None:

			'''simple docstring'''




			__lowercase											=     node
			__lowercase											=     node.previous

			if node.get_previous() is None:
					__lowercase											=     node_to_insert
			else:
					__lowercase											=     node_to_insert

			__lowercase											=     node_to_insert




	def    UpperCAmelCase			(						self							:     Optional[Any]					,      __lowerCamelCase							:     Node					,      __lowerCamelCase							:     Node   )       -> None:

			'''simple docstring'''




			__lowercase											=     node
			__lowercase											=     node.next

			if node.get_next() is None:
					__lowercase											=     node_to_insert
			else:
					__lowercase											=     node_to_insert

			__lowercase											=     node_to_insert




	def    UpperCAmelCase			(						self							:     List[str]					,      __lowerCamelCase							:     int					,      __lowerCamelCase							:     int   )       -> None:

			'''simple docstring'''




			__lowercase											=     1
			__lowercase											=     Node(__lowerCamelCase   )
			__lowercase											=     self.head
			while node:
					if current_position == position:
							self.insert_before_node(__lowerCamelCase					,      __lowerCamelCase   )
							return
					current_position += 1
					__lowercase											=     node.next
			self.insert_after_node(self.tail					,      __lowerCamelCase   )




	def    UpperCAmelCase			(						self							:     Optional[Any]					,      __lowerCamelCase							:     int   )       -> Node:

			'''simple docstring'''




			__lowercase											=     self.head
			while node:
					if node.get_data() == item:
							return node
					__lowercase											=     node.get_next()
			raise Exception('Node not found'   )




	def    UpperCAmelCase			(						self							:     Optional[Any]					,      __lowerCamelCase							:     List[str]   )       -> Dict:

			'''simple docstring'''




			if (node := self.get_node(__lowerCamelCase   )) is not None:
					if node == self.head:
							__lowercase											=     self.head.get_next()

					if node == self.tail:
							__lowercase											=     self.tail.get_previous()

					self.remove_node_pointers(__lowerCamelCase   )




	@staticmethod
	def    UpperCAmelCase			(						__lowerCamelCase							:     Node   )       -> None:

			'''simple docstring'''




			if node.get_next():
					__lowercase											=     node.previous

			if node.get_previous():
					__lowercase											=     node.next

			__lowercase											=     None
			__lowercase											=     None






	def    UpperCAmelCase			(						self							:     str   )       -> List[str]:

			'''simple docstring'''




			return self.head is None





def  SCREAMING_SNAKE_CASE				(      )		->    None:
		pass


if __name__ == "__main__":
		import doctest

		doctest.testmod()




