





'''simple docstring'''






import json
import os
from dataclasses import dataclass
from functools import partial
from typing import Callable

import flax.linen as nn
import jax
import jax.numpy as jnp
import joblib
import optax
import wandb
from flax import jax_utils, struct, traverse_util
from flax.serialization import from_bytes, to_bytes
from flax.training import train_state
from flax.training.common_utils import shard
from tqdm.auto import tqdm

from transformers import BigBirdConfig, FlaxBigBirdForQuestionAnswering
from transformers.models.big_bird.modeling_flax_big_bird import FlaxBigBirdForQuestionAnsweringModule


class   _lowercase						(							a				):
   _UpperCamelCase       		=   42
   _UpperCamelCase       		=   jnp.floataa
   _UpperCamelCase       		=   True


   def 					snake_case				( self					):
       super().setup()
       A       :       int							=						nn.Dense(5    ,   dtype=self.dtype					)


   def __call__( self    ,   *_UpperCAmelCase    ,   **_UpperCAmelCase					):
       A       :       str							=						super().__call__(*_UpperCAmelCase    ,   **_UpperCAmelCase					)
       A       :       int							=						self.cls(outputs[2]					)
       return outputs[:2] + (cls_out,)







class   _lowercase						(							a				):
   _UpperCamelCase       		=   FlaxBigBirdForNaturalQuestionsModule







def 							_lowerCamelCase(  UpperCamelCase__    : Optional[int]    ,       UpperCamelCase__    : Optional[int]    ,       UpperCamelCase__    : List[Any]    ,       UpperCamelCase__    : Any    ,       UpperCamelCase__    : List[Any]    ,       UpperCamelCase__    : List[Any]		)		->  Optional[int]:
    def cross_entropy(UpperCamelCase__    : List[str]    ,       UpperCamelCase__    : Tuple    ,       UpperCamelCase__    : Optional[int]=None		):
        A       :       List[Any]							=						logits.shape[-1]
        A       :       List[str]							=						(labels[..., None] == jnp.arange(UpperCamelCase__		)[None]).astype('''f4'''		)
        A       :       Tuple							=						jax.nn.log_softmax(UpperCamelCase__    ,       axis=-1		)
        A       :       Any							=						-jnp.sum(labels * logits    ,       axis=-1		)
        if reduction is not None:
            A       :       int							=						reduction(UpperCamelCase__		)
        return loss

    A       :       Any							=						partial(UpperCamelCase__    ,       reduction=jnp.mean		)
    A       :       Optional[int]							=						cross_entropy(UpperCamelCase__    ,       UpperCamelCase__		)
    A       :       Optional[Any]							=						cross_entropy(UpperCamelCase__    ,       UpperCamelCase__		)
    A       :       Optional[Any]							=						cross_entropy(UpperCamelCase__    ,       UpperCamelCase__		)
    return (start_loss + end_loss + pooled_loss) / 3


@dataclass
class   _lowercase						:
   _UpperCamelCase       		=   "google/bigbird-roberta-base"
   _UpperCamelCase       		=   30_00
   _UpperCamelCase       		=   1_05_00

   _UpperCamelCase       		=   1_28
   _UpperCamelCase       		=   3

   _UpperCamelCase       		=   1
   _UpperCamelCase       		=   5

   # tx_args
   _UpperCamelCase       		=   3e-5
   _UpperCamelCase       		=   0.0
   _UpperCamelCase       		=   2_00_00
   _UpperCamelCase       		=   0.0095

   _UpperCamelCase       		=   "bigbird-roberta-natural-questions"
   _UpperCamelCase       		=   "training-expt"
   _UpperCamelCase       		=   "data/nq-training.jsonl"
   _UpperCamelCase       		=   "data/nq-validation.jsonl"


   def 					snake_case				( self					):
       os.makedirs(self.base_dir    ,   exist_ok=_UpperCAmelCase					)
       A       :       Optional[Any]							=						os.path.join(self.base_dir    ,   self.save_dir					)
       A       :       Dict							=						self.batch_size_per_device * jax.device_count()







@dataclass
class   _lowercase						:
   _UpperCamelCase       		=   42
   _UpperCamelCase       		=   40_96  # no dynamic padding on TPUs


   def __call__( self    ,   _UpperCAmelCase					):
       A       :       str							=						self.collate_fn(_UpperCAmelCase					)
       A       :       Dict							=						jax.tree_util.tree_map(_UpperCAmelCase    ,   _UpperCAmelCase					)
       return batch


   def 					snake_case				( self    ,   _UpperCAmelCase					):
       A,	A       :       Tuple							=						self.fetch_inputs(features['''input_ids''']					)
       A       :       Optional[Any]							=						{
           '''input_ids''': jnp.array(_UpperCAmelCase    ,   dtype=jnp.intaa					),
           '''attention_mask''': jnp.array(_UpperCAmelCase    ,   dtype=jnp.intaa					),
           '''start_labels''': jnp.array(features['''start_token''']    ,   dtype=jnp.intaa					),
           '''end_labels''': jnp.array(features['''end_token''']    ,   dtype=jnp.intaa					),
           '''pooled_labels''': jnp.array(features['''category''']    ,   dtype=jnp.intaa					),
       }
       return batch


   def 					snake_case				( self    ,   _UpperCAmelCase					):
       A       :       str							=						[self._fetch_inputs(_UpperCAmelCase					) for ids in input_ids]
       return zip(*_UpperCAmelCase					)


   def 					snake_case				( self    ,   _UpperCAmelCase					):
       A       :       Dict							=						[1 for _ in range(len(_UpperCAmelCase					)					)]
       while len(_UpperCAmelCase					) < self.max_length:
           input_ids.append(self.pad_id					)
           attention_mask.append(0					)
       return input_ids, attention_mask







def 							_lowerCamelCase(  UpperCamelCase__    : Dict    ,       UpperCamelCase__    : List[Any]    ,       UpperCamelCase__    : Any=None		)		->  int:
    if seed is not None:
        A       :       int							=						dataset.shuffle(seed=UpperCamelCase__		)
    for i in range(len(UpperCamelCase__		) // batch_size		):
        A       :       List[Any]							=						dataset[i * batch_size : (i + 1) * batch_size]
        yield dict(UpperCamelCase__		)
@partial(jax.pmap    ,       axis_name='''batch'''		)
def 							_lowerCamelCase(  UpperCamelCase__    : Dict    ,       UpperCamelCase__    : str    ,       **UpperCamelCase__    : List[str]		)		->  Optional[Any]:
    def loss_fn(UpperCamelCase__    : Optional[Any]		):
        A       :       Union[str, Any]							=						model_inputs.pop('''start_labels'''		)
        A       :       Optional[int]							=						model_inputs.pop('''end_labels'''		)
        A       :       List[str]							=						model_inputs.pop('''pooled_labels'''		)

        A       :       Any							=						state.apply_fn(**UpperCamelCase__    ,       params=UpperCamelCase__    ,       dropout_rng=UpperCamelCase__    ,       train=UpperCamelCase__		)
        A,	A,	A       :       Optional[int]							=						outputs

        return state.loss_fn(
            UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__    ,       )

    A,	A       :       Tuple							=						jax.random.split(UpperCamelCase__		)
    A       :       Optional[Any]							=						jax.value_and_grad(UpperCamelCase__		)
    A,	A       :       Any							=						grad_fn(state.params		)
    A       :       Tuple							=						jax.lax.pmean({'''loss''': loss}    ,       axis_name='''batch'''		)
    A       :       Any							=						jax.lax.pmean(UpperCamelCase__    ,       '''batch'''		)

    A       :       Any							=						state.apply_gradients(grads=UpperCamelCase__		)
    return state, metrics, new_drp_rng
@partial(jax.pmap    ,       axis_name='''batch'''		)
def 							_lowerCamelCase(  UpperCamelCase__    : Union[str, Any]    ,       **UpperCamelCase__    : Dict		)		->  Any:
    A       :       Tuple							=						model_inputs.pop('''start_labels'''		)
    A       :       List[Any]							=						model_inputs.pop('''end_labels'''		)
    A       :       Optional[Any]							=						model_inputs.pop('''pooled_labels'''		)

    A       :       Optional[Any]							=						state.apply_fn(**UpperCamelCase__    ,       params=state.params    ,       train=UpperCamelCase__		)
    A,	A,	A       :       int							=						outputs

    A       :       List[Any]							=						state.loss_fn(UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__		)
    A       :       Any							=						jax.lax.pmean({'''loss''': loss}    ,       axis_name='''batch'''		)
    return metrics


class   _lowercase						(							train_state.TrainState				):
   _UpperCamelCase       		=   struct.field(pytree_node=a				)







@dataclass
class   _lowercase						:
   _UpperCamelCase       		=   42
   _UpperCamelCase       		=   42
   _UpperCamelCase       		=   42
   _UpperCamelCase       		=   42
   _UpperCamelCase       		=   42
   _UpperCamelCase       		=   42
   _UpperCamelCase       		=   None


   def 					snake_case				( self    ,   _UpperCAmelCase    ,   _UpperCAmelCase    ,   _UpperCAmelCase    ,   _UpperCAmelCase=None					):
       A       :       str							=						model.params
       A       :       List[str]							=						TrainState.create(
           apply_fn=model.__call__    ,   params=_UpperCAmelCase    ,   tx=_UpperCAmelCase    ,   loss_fn=_UpperCAmelCase    ,   )
       if ckpt_dir is not None:
           A,	A,	A,	A,	A       :       Union[str, Any]							=						restore_checkpoint(_UpperCAmelCase    ,   _UpperCAmelCase					)
           A       :       str							=						{
               '''lr''': args.lr,
               '''init_lr''': args.init_lr,
               '''warmup_steps''': args.warmup_steps,
               '''num_train_steps''': num_train_steps,
               '''weight_decay''': args.weight_decay,
           }
           A,	A       :       Tuple							=						build_tx(**_UpperCAmelCase					)
           A       :       Optional[int]							=						train_state.TrainState(
               step=_UpperCAmelCase    ,   apply_fn=model.__call__    ,   params=_UpperCAmelCase    ,   tx=_UpperCAmelCase    ,   opt_state=_UpperCAmelCase    ,   )
           A       :       Optional[int]							=						args
           A       :       Dict							=						data_collator
           A       :       Optional[int]							=						lr
           A       :       Optional[int]							=						params
       A       :       int							=						jax_utils.replicate(_UpperCAmelCase					)
       return state


   def 					snake_case				( self    ,   _UpperCAmelCase    ,   _UpperCAmelCase    ,   _UpperCAmelCase					):
       A       :       Any							=						self.args
       A       :       Tuple							=						len(_UpperCAmelCase					) // args.batch_size

       A       :       str							=						jax.random.PRNGKey(0					)
       A       :       List[Any]							=						jax.random.split(_UpperCAmelCase    ,   jax.device_count()					)
       for epoch in range(args.max_epochs					):
           A       :       List[Any]							=						jnp.array(0    ,   dtype=jnp.floataa					)
           A       :       Tuple							=						get_batched_dataset(_UpperCAmelCase    ,   args.batch_size    ,   seed=_UpperCAmelCase					)
           A       :       List[str]							=						0
           for batch in tqdm(_UpperCAmelCase    ,   total=_UpperCAmelCase    ,   desc=f'''Running EPOCH-{epoch}'''					):
               A       :       List[str]							=						self.data_collator(_UpperCAmelCase					)
               A,	A,	A       :       Optional[Any]							=						self.train_step_fn(_UpperCAmelCase    ,   _UpperCAmelCase    ,   **_UpperCAmelCase					)
               running_loss += jax_utils.unreplicate(metrics['''loss''']					)
               i += 1
               if i % args.logging_steps == 0:
                   A       :       Dict							=						jax_utils.unreplicate(state.step					)
                   A       :       int							=						running_loss.item() / i
                   A       :       Dict							=						self.scheduler_fn(state_step - 1					)

                   A       :       Optional[Any]							=						self.evaluate(_UpperCAmelCase    ,   _UpperCAmelCase					)
                   A       :       Optional[int]							=						{
                       '''step''': state_step.item(),
                       '''eval_loss''': eval_loss.item(),
                       '''tr_loss''': tr_loss,
                       '''lr''': lr.item(),
                   }
                   tqdm.write(str(_UpperCAmelCase					)					)
                   self.logger.log(_UpperCAmelCase    ,   commit=_UpperCAmelCase					)

               if i % args.save_steps == 0:
                   self.save_checkpoint(args.save_dir + f'''-e{epoch}-s{i}'''    ,   state=_UpperCAmelCase					)


   def 					snake_case				( self    ,   _UpperCAmelCase    ,   _UpperCAmelCase					):
       A       :       int							=						get_batched_dataset(_UpperCAmelCase    ,   self.args.batch_size					)
       A       :       str							=						len(_UpperCAmelCase					) // self.args.batch_size
       A       :       str							=						jnp.array(0    ,   dtype=jnp.floataa					)
       A       :       Tuple							=						0
       for batch in tqdm(_UpperCAmelCase    ,   total=_UpperCAmelCase    ,   desc='''Evaluating ... '''					):
           A       :       Optional[int]							=						self.data_collator(_UpperCAmelCase					)
           A       :       Union[str, Any]							=						self.val_step_fn(_UpperCAmelCase    ,   **_UpperCAmelCase					)
           running_loss += jax_utils.unreplicate(metrics['''loss''']					)
           i += 1
       return running_loss / i


   def 					snake_case				( self    ,   _UpperCAmelCase    ,   _UpperCAmelCase					):
       A       :       List[Any]							=						jax_utils.unreplicate(_UpperCAmelCase					)
       print(f'''SAVING CHECKPOINT IN {save_dir}'''    ,   end=''' ... '''					)
       self.model_save_fn(_UpperCAmelCase    ,   params=state.params					)
       with open(os.path.join(_UpperCAmelCase    ,   '''opt_state.msgpack'''					)    ,   '''wb'''					) as f:
           f.write(to_bytes(state.opt_state					)					)
       joblib.dump(self.args    ,   os.path.join(_UpperCAmelCase    ,   '''args.joblib'''					)					)
       joblib.dump(self.data_collator    ,   os.path.join(_UpperCAmelCase    ,   '''data_collator.joblib'''					)					)
       with open(os.path.join(_UpperCAmelCase    ,   '''training_state.json'''					)    ,   '''w'''					) as f:
           json.dump({'''step''': state.step.item()}    ,   _UpperCAmelCase					)
       print('''DONE'''					)







def 							_lowerCamelCase(  UpperCamelCase__    : Optional[Any]    ,       UpperCamelCase__    : Any		)		->  Union[str, Any]:
    print(F'''RESTORING CHECKPOINT FROM {save_dir}'''    ,       end=''' ... '''		)
    with open(os.path.join(UpperCamelCase__    ,       '''flax_model.msgpack'''		)    ,       '''rb'''		) as f:
        A       :       str							=						from_bytes(state.params    ,       f.read()		)

    with open(os.path.join(UpperCamelCase__    ,       '''opt_state.msgpack'''		)    ,       '''rb'''		) as f:
        A       :       Tuple							=						from_bytes(state.opt_state    ,       f.read()		)

    A       :       Tuple							=						joblib.load(os.path.join(UpperCamelCase__    ,       '''args.joblib'''		)		)
    A       :       int							=						joblib.load(os.path.join(UpperCamelCase__    ,       '''data_collator.joblib'''		)		)

    with open(os.path.join(UpperCamelCase__    ,       '''training_state.json'''		)    ,       '''r'''		) as f:
        A       :       List[str]							=						json.load(UpperCamelCase__		)
    A       :       Any							=						training_state['''step''']

    print('''DONE'''		)
    return params, opt_state, step, args, data_collator
def 							_lowerCamelCase(  UpperCamelCase__    : Tuple    ,       UpperCamelCase__    : Tuple    ,       UpperCamelCase__    : List[str]    ,       UpperCamelCase__    : str		)		->  Tuple:
    A       :       List[Any]							=						num_train_steps - warmup_steps
    A       :       int							=						optax.linear_schedule(init_value=UpperCamelCase__    ,       end_value=UpperCamelCase__    ,       transition_steps=UpperCamelCase__		)
    A       :       Any							=						optax.linear_schedule(init_value=UpperCamelCase__    ,       end_value=1e-7    ,       transition_steps=UpperCamelCase__		)
    A       :       Union[str, Any]							=						optax.join_schedules(schedules=[warmup_fn, decay_fn]    ,       boundaries=[warmup_steps]		)
    return lr





def 							_lowerCamelCase(  UpperCamelCase__    : Optional[int]    ,       UpperCamelCase__    : Any    ,       UpperCamelCase__    : str    ,       UpperCamelCase__    : str    ,       UpperCamelCase__    : List[str]		)		->  Tuple:
    def weight_decay_mask(UpperCamelCase__    : Optional[int]		):
        A       :       str							=						traverse_util.flatten_dict(UpperCamelCase__		)
        A       :       List[Any]							=						{k: (v[-1] != '''bias''' and v[-2:] != ('''LayerNorm''', '''scale''')) for k, v in params.items()}
        return traverse_util.unflatten_dict(UpperCamelCase__		)

    A       :       List[str]							=						scheduler_fn(UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__		)

    A       :       Dict							=						optax.adamw(learning_rate=UpperCamelCase__    ,       weight_decay=UpperCamelCase__    ,       mask=UpperCamelCase__		)
    return tx, lr







'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


snake_case_     =	{
    """configuration_biogpt""": ["""BIOGPT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """BioGptConfig"""],
    """tokenization_biogpt""": ["""BioGptTokenizer"""],
}

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     snake_case_     =	[
         """BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST""",
         """BioGptForCausalLM""",
         """BioGptForTokenClassification""",
         """BioGptForSequenceClassification""",
         """BioGptModel""",
         """BioGptPreTrainedModel""",
     ]


if TYPE_CHECKING:
     from .configuration_biogpt import BIOGPT_PRETRAINED_CONFIG_ARCHIVE_MAP, BioGptConfig
     from .tokenization_biogpt import BioGptTokenizer

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_biogpt import (
              BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST,
              BioGptForCausalLM,
              BioGptForSequenceClassification,
              BioGptForTokenClassification,
              BioGptModel,
              BioGptPreTrainedModel,
          )


else:
     import sys

     snake_case_     =	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
