
'''simple docstring'''







import json
import os
from dataclasses import dataclass
from functools import partial
from typing import Callable

import flax.linen as nn
import jax
import jax.numpy as jnp
import joblib
import optax
import wandb
from flax import jax_utils, struct, traverse_util
from flax.serialization import from_bytes, to_bytes
from flax.training import train_state
from flax.training.common_utils import shard
from tqdm.auto import tqdm

from transformers import BigBirdConfig, FlaxBigBirdForQuestionAnswering
from transformers.models.big_bird.modeling_flax_big_bird import FlaxBigBirdForQuestionAnsweringModule







class UpperCAmelCase__      (					UpperCamelCase__      ):





  """simple docstring"""

  __UpperCAmelCase							:							str    					=   42
  __UpperCAmelCase							:							Union[str, Any]    					=   jnp.floataa
  __UpperCAmelCase							:							List[Any]    					=   True


  def     __lowercase						(						self :	Tuple			):


     '''simple docstring'''






     super().setup()
     _a							:      List[str]     =  nn.Dense(5				,dtype=self.dtype			)


  def __call__(						self :	List[Any]				,*_a :	int				,**_a :	Optional[Any]			):


     '''simple docstring'''






     _a							:      Optional[Any]     =  super().__call__(*__A				,**__A			)
     _a							:      Tuple     =  self.cls(outputs[2]			)
     return outputs[:2] + (cls_out,)







class UpperCAmelCase__      (					UpperCamelCase__      ):





  """simple docstring"""

  __UpperCAmelCase							:							List[Any]    					=   FlaxBigBirdForNaturalQuestionsModule





def    UpperCAmelCase_						(__a :       List[Any]					,      __a :       List[Any]					,      __a :       Optional[int]					,      __a :       Tuple					,      __a :       Union[str, Any]					,      __a :       Tuple					):






   """simple docstring"""

   def cross_entropy(__a :       List[str]					,      __a :       List[str]					,      __a :       List[str]=None					):
      _a							:      List[Any]     =  logits.shape[-1]
      _a							:      Dict     =  (labels[..., None] == jnp.arange(UpperCamelCase__					)[None]).astype('f4'					)
      _a							:      str     =  jax.nn.log_softmax(UpperCamelCase__					,      axis=-1					)
      _a							:      Optional[Any]     =  -jnp.sum(labels * logits					,      axis=-1					)
      if reduction is not None:
         _a							:      List[Any]     =  reduction(UpperCamelCase__					)
      return loss

   _a							:      Optional[Any]     =  partial(UpperCamelCase__					,      reduction=jnp.mean					)
   _a							:      Optional[Any]     =  cross_entropy(UpperCamelCase__					,      UpperCamelCase__					)
   _a							:      str     =  cross_entropy(UpperCamelCase__					,      UpperCamelCase__					)
   _a							:      List[str]     =  cross_entropy(UpperCamelCase__					,      UpperCamelCase__					)
   return (start_loss + end_loss + pooled_loss) / 3







@dataclass
class UpperCAmelCase__      :





  """simple docstring"""

  __UpperCAmelCase							:							str    					=   '''google/bigbird-roberta-base'''
  __UpperCAmelCase							:							Optional[Any]    					=   3000
  __UpperCAmelCase							:							List[Any]    					=   1_0500

  __UpperCAmelCase							:							Optional[int]    					=   128
  __UpperCAmelCase							:							int    					=   3

  __UpperCAmelCase							:							Any    					=   1
  __UpperCAmelCase							:							List[Any]    					=   5

  # tx_args
  __UpperCAmelCase							:							Optional[int]    					=   3e-5
  __UpperCAmelCase							:							List[Any]    					=   0.0
  __UpperCAmelCase							:							Tuple    					=   2_0000
  __UpperCAmelCase							:							Any    					=   0.0_0_9_5

  __UpperCAmelCase							:							int    					=   '''bigbird-roberta-natural-questions'''
  __UpperCAmelCase							:							Any    					=   '''training-expt'''
  __UpperCAmelCase							:							int    					=   '''data/nq-training.jsonl'''
  __UpperCAmelCase							:							int    					=   '''data/nq-validation.jsonl'''


  def     __lowercase						(						self :	str			):


     '''simple docstring'''






     os.makedirs(self.base_dir				,exist_ok=__A			)
     _a							:      int     =  os.path.join(self.base_dir				,self.save_dir			)
     _a							:      str     =  self.batch_size_per_device * jax.device_count()







@dataclass
class UpperCAmelCase__      :





  """simple docstring"""

  __UpperCAmelCase							:							Optional[int]    					=   42
  __UpperCAmelCase							:							int    					=   4096  # no dynamic padding on TPUs


  def __call__(						self :	Optional[Any]				,_a :	Optional[int]			):


     '''simple docstring'''






     _a							:      str     =  self.collate_fn(__A			)
     _a							:      List[str]     =  jax.tree_util.tree_map(__A				,__A			)
     return batch


  def     __lowercase						(						self :	List[Any]				,_a :	Union[str, Any]			):


     '''simple docstring'''






     _a,    _a							:      Optional[int]     =  self.fetch_inputs(features['input_ids']			)
     _a							:      List[str]     =  {
         'input_ids': jnp.array(__A				,dtype=jnp.intaa			),
         'attention_mask': jnp.array(__A				,dtype=jnp.intaa			),
         'start_labels': jnp.array(features['start_token']				,dtype=jnp.intaa			),
         'end_labels': jnp.array(features['end_token']				,dtype=jnp.intaa			),
         'pooled_labels': jnp.array(features['category']				,dtype=jnp.intaa			),
     }
     return batch


  def     __lowercase						(						self :	Tuple				,_a :	list			):


     '''simple docstring'''






     _a							:      Dict     =  [self._fetch_inputs(__A			) for ids in input_ids]
     return zip(*__A			)


  def     __lowercase						(						self :	List[str]				,_a :	list			):


     '''simple docstring'''






     _a							:      str     =  [1 for _ in range(len(__A			)			)]
     while len(__A			) < self.max_length:
        input_ids.append(self.pad_id			)
        attention_mask.append(0			)
     return input_ids, attention_mask





def    UpperCAmelCase_						(__a :       str					,      __a :       List[str]					,      __a :       Optional[Any]=None					):






   """simple docstring"""

   if seed is not None:
      _a							:      Union[str, Any]     =  dataset.shuffle(seed=UpperCamelCase__					)
   for i in range(len(UpperCamelCase__					) // batch_size					):
      _a							:      Dict     =  dataset[i * batch_size : (i + 1) * batch_size]
      yield dict(UpperCamelCase__					)





@partial(jax.pmap					,      axis_name='batch'					)
def    UpperCAmelCase_						(__a :       Dict					,      __a :       Optional[int]					,      **__a :       Optional[int]					):






   """simple docstring"""

   def loss_fn(__a :       List[Any]					):
      _a							:      Optional[int]     =  model_inputs.pop('start_labels'					)
      _a							:      int     =  model_inputs.pop('end_labels'					)
      _a							:      Optional[int]     =  model_inputs.pop('pooled_labels'					)

      _a							:      int     =  state.apply_fn(**UpperCamelCase__					,      params=UpperCamelCase__					,      dropout_rng=UpperCamelCase__					,      train=UpperCamelCase__					)
      _a,    _a,    _a							:      str     =  outputs

      return state.loss_fn(
          UpperCamelCase__					,      UpperCamelCase__					,      UpperCamelCase__					,      UpperCamelCase__					,      UpperCamelCase__					,      UpperCamelCase__					,      )

   _a,    _a							:      Dict     =  jax.random.split(UpperCamelCase__					)
   _a							:      int     =  jax.value_and_grad(UpperCamelCase__					)
   _a,    _a							:      str     =  grad_fn(state.params					)
   _a							:      Any     =  jax.lax.pmean({'loss': loss}					,      axis_name='batch'					)
   _a							:      int     =  jax.lax.pmean(UpperCamelCase__					,      'batch'					)

   _a							:      Optional[int]     =  state.apply_gradients(grads=UpperCamelCase__					)
   return state, metrics, new_drp_rng





@partial(jax.pmap					,      axis_name='batch'					)
def    UpperCAmelCase_						(__a :       Optional[Any]					,      **__a :       Dict					):






   """simple docstring"""

   _a							:      Dict     =  model_inputs.pop('start_labels'					)
   _a							:      Dict     =  model_inputs.pop('end_labels'					)
   _a							:      Union[str, Any]     =  model_inputs.pop('pooled_labels'					)

   _a							:      int     =  state.apply_fn(**UpperCamelCase__					,      params=state.params					,      train=UpperCamelCase__					)
   _a,    _a,    _a							:      str     =  outputs

   _a							:      str     =  state.loss_fn(UpperCamelCase__					,      UpperCamelCase__					,      UpperCamelCase__					,      UpperCamelCase__					,      UpperCamelCase__					,      UpperCamelCase__					)
   _a							:      Optional[int]     =  jax.lax.pmean({'loss': loss}					,      axis_name='batch'					)
   return metrics







class UpperCAmelCase__      (					train_state.TrainState      ):





  """simple docstring"""

  __UpperCAmelCase							:							Union[str, Any]    					=   struct.field(pytree_node=UpperCamelCase__      )







@dataclass
class UpperCAmelCase__      :





  """simple docstring"""

  __UpperCAmelCase							:							Dict    					=   42
  __UpperCAmelCase							:							Optional[int]    					=   42
  __UpperCAmelCase							:							List[str]    					=   42
  __UpperCAmelCase							:							Tuple    					=   42
  __UpperCAmelCase							:							Any    					=   42
  __UpperCAmelCase							:							Tuple    					=   42
  __UpperCAmelCase							:							Any    					=   None


  def     __lowercase						(						self :	List[Any]				,_a :	str				,_a :	str				,_a :	str				,_a :	Tuple=None			):


     '''simple docstring'''






     _a							:      Tuple     =  model.params
     _a							:      Union[str, Any]     =  TrainState.create(
         apply_fn=model.__call__				,params=__A				,tx=__A				,loss_fn=__A				,)
     if ckpt_dir is not None:
        _a,    _a,    _a,    _a,    _a							:      Tuple     =  restore_checkpoint(__A				,__A			)
        _a							:      List[Any]     =  {
            'lr': args.lr,
            'init_lr': args.init_lr,
            'warmup_steps': args.warmup_steps,
            'num_train_steps': num_train_steps,
            'weight_decay': args.weight_decay,
        }
        _a,    _a							:      str     =  build_tx(**__A			)
        _a							:      Union[str, Any]     =  train_state.TrainState(
            step=__A				,apply_fn=model.__call__				,params=__A				,tx=__A				,opt_state=__A				,)
        _a							:      Tuple     =  args
        _a							:      Optional[int]     =  data_collator
        _a							:      Optional[Any]     =  lr
        _a							:      Union[str, Any]     =  params
     _a							:      Any     =  jax_utils.replicate(__A			)
     return state


  def     __lowercase						(						self :	Optional[Any]				,_a :	Optional[int]				,_a :	int				,_a :	int			):


     '''simple docstring'''






     _a							:      Union[str, Any]     =  self.args
     _a							:      str     =  len(__A			) // args.batch_size

     _a							:      Optional[Any]     =  jax.random.PRNGKey(0			)
     _a							:      str     =  jax.random.split(__A				,jax.device_count()			)
     for epoch in range(args.max_epochs			):
        _a							:      List[Any]     =  jnp.array(0				,dtype=jnp.floataa			)
        _a							:      int     =  get_batched_dataset(__A				,args.batch_size				,seed=__A			)
        _a							:      Optional[int]     =  0
        for batch in tqdm(__A				,total=__A				,desc=F"""Running EPOCH-{epoch}"""			):
           _a							:      List[str]     =  self.data_collator(__A			)
           _a,    _a,    _a							:      List[str]     =  self.train_step_fn(__A				,__A				,**__A			)
           running_loss += jax_utils.unreplicate(metrics['loss']			)
           i += 1
           if i % args.logging_steps == 0:
              _a							:      Optional[int]     =  jax_utils.unreplicate(state.step			)
              _a							:      Tuple     =  running_loss.item() / i
              _a							:      int     =  self.scheduler_fn(state_step - 1			)

              _a							:      Dict     =  self.evaluate(__A				,__A			)
              _a							:      Dict     =  {
                  'step': state_step.item(),
                  'eval_loss': eval_loss.item(),
                  'tr_loss': tr_loss,
                  'lr': lr.item(),
              }
              tqdm.write(str(__A			)			)
              self.logger.log(__A				,commit=__A			)

           if i % args.save_steps == 0:
              self.save_checkpoint(args.save_dir + F"""-e{epoch}-s{i}"""				,state=__A			)


  def     __lowercase						(						self :	List[str]				,_a :	Dict				,_a :	str			):


     '''simple docstring'''






     _a							:      str     =  get_batched_dataset(__A				,self.args.batch_size			)
     _a							:      int     =  len(__A			) // self.args.batch_size
     _a							:      Optional[int]     =  jnp.array(0				,dtype=jnp.floataa			)
     _a							:      str     =  0
     for batch in tqdm(__A				,total=__A				,desc='Evaluating ... '			):
        _a							:      Dict     =  self.data_collator(__A			)
        _a							:      int     =  self.val_step_fn(__A				,**__A			)
        running_loss += jax_utils.unreplicate(metrics['loss']			)
        i += 1
     return running_loss / i


  def     __lowercase						(						self :	List[Any]				,_a :	Any				,_a :	Dict			):


     '''simple docstring'''






     _a							:      int     =  jax_utils.unreplicate(__A			)
     print(F"""SAVING CHECKPOINT IN {save_dir}"""				,end=' ... '			)
     self.model_save_fn(__A				,params=state.params			)
     with open(os.path.join(__A				,'opt_state.msgpack'			)				,'wb'			) as f:
        f.write(to_bytes(state.opt_state			)			)
     joblib.dump(self.args				,os.path.join(__A				,'args.joblib'			)			)
     joblib.dump(self.data_collator				,os.path.join(__A				,'data_collator.joblib'			)			)
     with open(os.path.join(__A				,'training_state.json'			)				,'w'			) as f:
        json.dump({'step': state.step.item()}				,__A			)
     print('DONE'			)





def    UpperCAmelCase_						(__a :       int					,      __a :       Optional[Any]					):






   """simple docstring"""

   print(f"""RESTORING CHECKPOINT FROM {save_dir}"""					,      end=' ... '					)
   with open(os.path.join(UpperCamelCase__					,      'flax_model.msgpack'					)					,      'rb'					) as f:
      _a							:      int     =  from_bytes(state.params					,      f.read()					)

   with open(os.path.join(UpperCamelCase__					,      'opt_state.msgpack'					)					,      'rb'					) as f:
      _a							:      int     =  from_bytes(state.opt_state					,      f.read()					)

   _a							:      List[str]     =  joblib.load(os.path.join(UpperCamelCase__					,      'args.joblib'					)					)
   _a							:      Union[str, Any]     =  joblib.load(os.path.join(UpperCamelCase__					,      'data_collator.joblib'					)					)

   with open(os.path.join(UpperCamelCase__					,      'training_state.json'					)					,      'r'					) as f:
      _a							:      Dict     =  json.load(UpperCamelCase__					)
   _a							:      Dict     =  training_state['step']

   print('DONE'					)
   return params, opt_state, step, args, data_collator





def    UpperCAmelCase_						(__a :       int					,      __a :       Union[str, Any]					,      __a :       List[Any]					,      __a :       Dict					):






   """simple docstring"""

   _a							:      List[Any]     =  num_train_steps - warmup_steps
   _a							:      List[str]     =  optax.linear_schedule(init_value=UpperCamelCase__					,      end_value=UpperCamelCase__					,      transition_steps=UpperCamelCase__					)
   _a							:      List[Any]     =  optax.linear_schedule(init_value=UpperCamelCase__					,      end_value=1e-7					,      transition_steps=UpperCamelCase__					)
   _a							:      Any     =  optax.join_schedules(schedules=[warmup_fn, decay_fn]					,      boundaries=[warmup_steps]					)
   return lr





def    UpperCAmelCase_						(__a :       List[str]					,      __a :       Union[str, Any]					,      __a :       Tuple					,      __a :       Tuple					,      __a :       Tuple					):






   """simple docstring"""

   def weight_decay_mask(__a :       Any					):
      _a							:      Tuple     =  traverse_util.flatten_dict(UpperCamelCase__					)
      _a							:      Tuple     =  {k: (v[-1] != 'bias' and v[-2:] != ('LayerNorm', 'scale')) for k, v in params.items()}
      return traverse_util.unflatten_dict(UpperCamelCase__					)

   _a							:      Any     =  scheduler_fn(UpperCamelCase__					,      UpperCamelCase__					,      UpperCamelCase__					,      UpperCamelCase__					)

   _a							:      Union[str, Any]     =  optax.adamw(learning_rate=UpperCamelCase__					,      weight_decay=UpperCamelCase__					,      mask=UpperCamelCase__					)
   return tx, lr










from torch import nn






def 					SCREAMING_SNAKE_CASE__				(							UpperCamelCase__:				str     ):
		if act_fn in ["swish", "silu"]:
				return nn.SiLU()
		elif act_fn == "mish":
				return nn.Mish()
		elif act_fn == "gelu":
				return nn.GELU()
		else:
				raise ValueError(f'''Unsupported activation function: {act_fn}'''     )