import json
import os
from dataclasses import dataclass
from functools import partial
from typing import Callable

import flax.linen as nn
import jax
import jax.numpy as jnp
import joblib
import optax
import wandb
from flax import jax_utils, struct, traverse_util
from flax.serialization import from_bytes, to_bytes
from flax.training import train_state
from flax.training.common_utils import shard
from tqdm.auto import tqdm

from transformers import BigBirdConfig, FlaxBigBirdForQuestionAnswering
from transformers.models.big_bird.modeling_flax_big_bird import FlaxBigBirdForQuestionAnsweringModule





class 	__lowercase    (			A       ):




				'''simple docstring'''


				_A      :		BigBirdConfig
				_A      :		jnp.dtype   						=			jnp.floataa
				_A      :		bool   						=			True





				def      A_							(		self    :		List[Any]				):
								super().setup()
								UpperCamelCase__					=				nn.Dense(5      ,							dtype=self.dtype				)





				def __call__(		self    :		Dict      ,							*_a    :		List[str]      ,							**_a    :		Optional[int]				):
								UpperCamelCase__					=				super().__call__(*A__      ,							**A__				)
								UpperCamelCase__					=				self.cls(outputs[2]				)
								return outputs[:2] + (cls_out,)






class 	__lowercase    (			A       ):




				'''simple docstring'''


				_A      :		Tuple   						=			FlaxBigBirdForNaturalQuestionsModule






def        lowerCamelCase_					(     UpperCamelCase__       :						Any,		UpperCamelCase__       :						Optional[int],		UpperCamelCase__       :						Any,		UpperCamelCase__       :						List[str],		UpperCamelCase__       :						List[Any],		UpperCamelCase__       :						Dict			):







				'''simple docstring'''







				def cross_entropy(UpperCamelCase__       :						str,		UpperCamelCase__       :						Optional[int],		UpperCamelCase__       :						Any=None			):
								UpperCamelCase__					=				logits.shape[-1]
								UpperCamelCase__					=				(labels[..., None] == jnp.arange(lowercase_			)[None]).astype('''f4'''			)
								UpperCamelCase__					=				jax.nn.log_softmax(lowercase_,		axis=-1			)
								UpperCamelCase__					=				-jnp.sum(labels * logits,		axis=-1			)
								if reduction is not None:
												UpperCamelCase__					=				reduction(lowercase_			)
								return loss

				UpperCamelCase__					=				partial(lowercase_,		reduction=jnp.mean			)
				UpperCamelCase__					=				cross_entropy(lowercase_,		lowercase_			)
				UpperCamelCase__					=				cross_entropy(lowercase_,		lowercase_			)
				UpperCamelCase__					=				cross_entropy(lowercase_,		lowercase_			)
				return (start_loss + end_loss + pooled_loss) / 3





@dataclass
class 	__lowercase    :




				'''simple docstring'''


				_A      :		str   						=			"google/bigbird-roberta-base"
				_A      :		int   						=			3000
				_A      :		int   						=			1_0500

				_A      :		int   						=			128
				_A      :		int   						=			3

				_A      :		int   						=			1
				_A      :		int   						=			5

				# tx_args
				_A      :		float   						=			3e-5
				_A      :		float   						=			0.0
				_A      :		int   						=			2_0000
				_A      :		float   						=			0.00_95

				_A      :		str   						=			"bigbird-roberta-natural-questions"
				_A      :		str   						=			"training-expt"
				_A      :		str   						=			"data/nq-training.jsonl"
				_A      :		str   						=			"data/nq-validation.jsonl"





				def      A_							(		self    :		int				):
								os.makedirs(self.base_dir      ,							exist_ok=A__				)
								UpperCamelCase__					=				os.path.join(self.base_dir      ,							self.save_dir				)
								UpperCamelCase__					=				self.batch_size_per_device * jax.device_count()






@dataclass
class 	__lowercase    :




				'''simple docstring'''


				_A      :		int
				_A      :		int   						=			4096  # no dynamic padding on TPUs





				def __call__(		self    :		List[Any]      ,							_a    :		Tuple				):
								UpperCamelCase__					=				self.collate_fn(A__				)
								UpperCamelCase__					=				jax.tree_util.tree_map(A__      ,							A__				)
								return batch





				def      A_							(		self    :		Any      ,							_a    :		str				):
								UpperCamelCase__					=				self.fetch_inputs(features['''input_ids''']				)
								UpperCamelCase__					=				{
								    """input_ids""": jnp.array(A__      ,							dtype=jnp.intaa				),
								    """attention_mask""": jnp.array(A__      ,							dtype=jnp.intaa				),
								    """start_labels""": jnp.array(features['''start_token''']      ,							dtype=jnp.intaa				),
								    """end_labels""": jnp.array(features['''end_token''']      ,							dtype=jnp.intaa				),
								    """pooled_labels""": jnp.array(features['''category''']      ,							dtype=jnp.intaa				),
								}
								return batch





				def      A_							(		self    :		Dict      ,							_a    :		Tuple				):
								UpperCamelCase__					=				[self._fetch_inputs(A__				) for ids in input_ids]
								return zip(*A__				)





				def      A_							(		self    :		Optional[int]      ,							_a    :		Dict				):
								UpperCamelCase__					=				[1 for _ in range(len(A__				)				)]
								while len(A__				) < self.max_length:
												input_ids.append(self.pad_id				)
												attention_mask.append(0				)
								return input_ids, attention_mask






def        lowerCamelCase_					(     UpperCamelCase__       :						Optional[int],		UpperCamelCase__       :						List[str],		UpperCamelCase__       :						Optional[Any]=None			):







				'''simple docstring'''







				if seed is not None:
								UpperCamelCase__					=				dataset.shuffle(seed=lowercase_			)
				for i in range(len(lowercase_			) // batch_size			):
								UpperCamelCase__					=				dataset[i * batch_size : (i + 1) * batch_size]
								yield dict(lowercase_			)






@partial(jax.pmap,		axis_name='''batch'''			)
def        lowerCamelCase_					(     UpperCamelCase__       :						int,		UpperCamelCase__       :						Union[str, Any],		**UpperCamelCase__       :						Union[str, Any]			):







				'''simple docstring'''







				def loss_fn(UpperCamelCase__       :						Optional[int]			):
								UpperCamelCase__					=				model_inputs.pop('''start_labels'''			)
								UpperCamelCase__					=				model_inputs.pop('''end_labels'''			)
								UpperCamelCase__					=				model_inputs.pop('''pooled_labels'''			)

								UpperCamelCase__					=				state.apply_fn(**lowercase_,		params=lowercase_,		dropout_rng=lowercase_,		train=lowercase_			)
								UpperCamelCase__					=				outputs

								return state.loss_fn(
								    lowercase_,		lowercase_,		lowercase_,		lowercase_,		lowercase_,		lowercase_,		)

				UpperCamelCase__					=				jax.random.split(lowercase_			)
				UpperCamelCase__					=				jax.value_and_grad(lowercase_			)
				UpperCamelCase__					=				grad_fn(state.params			)
				UpperCamelCase__					=				jax.lax.pmean({'''loss''': loss},		axis_name='''batch'''			)
				UpperCamelCase__					=				jax.lax.pmean(lowercase_,		'''batch'''			)

				UpperCamelCase__					=				state.apply_gradients(grads=lowercase_			)
				return state, metrics, new_drp_rng






@partial(jax.pmap,		axis_name='''batch'''			)
def        lowerCamelCase_					(     UpperCamelCase__       :						str,		**UpperCamelCase__       :						Optional[Any]			):







				'''simple docstring'''







				UpperCamelCase__					=				model_inputs.pop('''start_labels'''			)
				UpperCamelCase__					=				model_inputs.pop('''end_labels'''			)
				UpperCamelCase__					=				model_inputs.pop('''pooled_labels'''			)

				UpperCamelCase__					=				state.apply_fn(**lowercase_,		params=state.params,		train=lowercase_			)
				UpperCamelCase__					=				outputs

				UpperCamelCase__					=				state.loss_fn(lowercase_,		lowercase_,		lowercase_,		lowercase_,		lowercase_,		lowercase_			)
				UpperCamelCase__					=				jax.lax.pmean({'''loss''': loss},		axis_name='''batch'''			)
				return metrics





class 	__lowercase    (			train_state.TrainState       ):




				'''simple docstring'''


				_A      :		Callable   						=			struct.field(pytree_node=A       )






@dataclass
class 	__lowercase    :




				'''simple docstring'''


				_A      :		Args
				_A      :		Callable
				_A      :		Callable
				_A      :		Callable
				_A      :		Callable
				_A      :		wandb
				_A      :		Callable   						=			None





				def      A_							(		self    :		Any      ,							_a    :		Any      ,							_a    :		Optional[int]      ,							_a    :		int      ,							_a    :		str=None				):
								UpperCamelCase__					=				model.params
								UpperCamelCase__					=				TrainState.create(
								    apply_fn=model.__call__      ,							params=A__      ,							tx=A__      ,							loss_fn=A__      ,							)
								if ckpt_dir is not None:
												UpperCamelCase__					=				restore_checkpoint(A__      ,							A__				)
												UpperCamelCase__					=				{
												    """lr""": args.lr,
												    """init_lr""": args.init_lr,
												    """warmup_steps""": args.warmup_steps,
												    """num_train_steps""": num_train_steps,
												    """weight_decay""": args.weight_decay,
												}
												UpperCamelCase__					=				build_tx(**A__				)
												UpperCamelCase__					=				train_state.TrainState(
												    step=A__      ,							apply_fn=model.__call__      ,							params=A__      ,							tx=A__      ,							opt_state=A__      ,							)
												UpperCamelCase__					=				args
												UpperCamelCase__					=				data_collator
												UpperCamelCase__					=				lr
												UpperCamelCase__					=				params
								UpperCamelCase__					=				jax_utils.replicate(A__				)
								return state





				def      A_							(		self    :		Optional[int]      ,							_a    :		Dict      ,							_a    :		Union[str, Any]      ,							_a    :		Optional[int]				):
								UpperCamelCase__					=				self.args
								UpperCamelCase__					=				len(A__				) // args.batch_size

								UpperCamelCase__					=				jax.random.PRNGKey(0				)
								UpperCamelCase__					=				jax.random.split(A__      ,							jax.device_count()				)
								for epoch in range(args.max_epochs				):
												UpperCamelCase__					=				jnp.array(0      ,							dtype=jnp.floataa				)
												UpperCamelCase__					=				get_batched_dataset(A__      ,							args.batch_size      ,							seed=A__				)
												UpperCamelCase__					=				0
												for batch in tqdm(A__      ,							total=A__      ,							desc=F"""Running EPOCH-{epoch}"""				):
																UpperCamelCase__					=				self.data_collator(A__				)
																UpperCamelCase__					=				self.train_step_fn(A__      ,							A__      ,							**A__				)
																running_loss += jax_utils.unreplicate(metrics['''loss''']				)
																i += 1
																if i % args.logging_steps == 0:
																				UpperCamelCase__					=				jax_utils.unreplicate(state.step				)
																				UpperCamelCase__					=				running_loss.item() / i
																				UpperCamelCase__					=				self.scheduler_fn(state_step - 1				)

																				UpperCamelCase__					=				self.evaluate(A__      ,							A__				)
																				UpperCamelCase__					=				{
																				    """step""": state_step.item(),
																				    """eval_loss""": eval_loss.item(),
																				    """tr_loss""": tr_loss,
																				    """lr""": lr.item(),
																				}
																				tqdm.write(str(A__				)				)
																				self.logger.log(A__      ,							commit=A__				)

																if i % args.save_steps == 0:
																				self.save_checkpoint(args.save_dir + F"""-e{epoch}-s{i}"""      ,							state=A__				)





				def      A_							(		self    :		Any      ,							_a    :		Any      ,							_a    :		Any				):
								UpperCamelCase__					=				get_batched_dataset(A__      ,							self.args.batch_size				)
								UpperCamelCase__					=				len(A__				) // self.args.batch_size
								UpperCamelCase__					=				jnp.array(0      ,							dtype=jnp.floataa				)
								UpperCamelCase__					=				0
								for batch in tqdm(A__      ,							total=A__      ,							desc='''Evaluating ... '''				):
												UpperCamelCase__					=				self.data_collator(A__				)
												UpperCamelCase__					=				self.val_step_fn(A__      ,							**A__				)
												running_loss += jax_utils.unreplicate(metrics['''loss''']				)
												i += 1
								return running_loss / i





				def      A_							(		self    :		Any      ,							_a    :		List[Any]      ,							_a    :		Union[str, Any]				):
								UpperCamelCase__					=				jax_utils.unreplicate(A__				)
								print(F"""SAVING CHECKPOINT IN {save_dir}"""      ,							end=''' ... '''				)
								self.model_save_fn(A__      ,							params=state.params				)
								with open(os.path.join(A__      ,							'''opt_state.msgpack'''				)      ,							'''wb'''				) as f:
												f.write(to_bytes(state.opt_state				)				)
								joblib.dump(self.args      ,							os.path.join(A__      ,							'''args.joblib'''				)				)
								joblib.dump(self.data_collator      ,							os.path.join(A__      ,							'''data_collator.joblib'''				)				)
								with open(os.path.join(A__      ,							'''training_state.json'''				)      ,							'''w'''				) as f:
												json.dump({'''step''': state.step.item()}      ,							A__				)
								print('''DONE'''				)






def        lowerCamelCase_					(     UpperCamelCase__       :						List[str],		UpperCamelCase__       :						str			):







				'''simple docstring'''







				print(F"""RESTORING CHECKPOINT FROM {save_dir}""",		end=''' ... '''			)
				with open(os.path.join(lowercase_,		'''flax_model.msgpack'''			),		'''rb'''			) as f:
								UpperCamelCase__					=				from_bytes(state.params,		f.read()			)

				with open(os.path.join(lowercase_,		'''opt_state.msgpack'''			),		'''rb'''			) as f:
								UpperCamelCase__					=				from_bytes(state.opt_state,		f.read()			)

				UpperCamelCase__					=				joblib.load(os.path.join(lowercase_,		'''args.joblib'''			)			)
				UpperCamelCase__					=				joblib.load(os.path.join(lowercase_,		'''data_collator.joblib'''			)			)

				with open(os.path.join(lowercase_,		'''training_state.json'''			),		'''r'''			) as f:
								UpperCamelCase__					=				json.load(lowercase_			)
				UpperCamelCase__					=				training_state["""step"""]

				print('''DONE'''			)
				return params, opt_state, step, args, data_collator






def        lowerCamelCase_					(     UpperCamelCase__       :						str,		UpperCamelCase__       :						Any,		UpperCamelCase__       :						Tuple,		UpperCamelCase__       :						Any			):







				'''simple docstring'''







				UpperCamelCase__					=				num_train_steps - warmup_steps
				UpperCamelCase__					=				optax.linear_schedule(init_value=lowercase_,		end_value=lowercase_,		transition_steps=lowercase_			)
				UpperCamelCase__					=				optax.linear_schedule(init_value=lowercase_,		end_value=1e-7,		transition_steps=lowercase_			)
				UpperCamelCase__					=				optax.join_schedules(schedules=[warmup_fn, decay_fn],		boundaries=[warmup_steps]			)
				return lr







def        lowerCamelCase_					(     UpperCamelCase__       :						Any,		UpperCamelCase__       :						List[str],		UpperCamelCase__       :						int,		UpperCamelCase__       :						int,		UpperCamelCase__       :						int			):







				'''simple docstring'''







				def weight_decay_mask(UpperCamelCase__       :						List[Any]			):
								UpperCamelCase__					=				traverse_util.flatten_dict(lowercase_			)
								UpperCamelCase__					=				{k: (v[-1] != """bias""" and v[-2:] != ("""LayerNorm""", """scale""")) for k, v in params.items()}
								return traverse_util.unflatten_dict(lowercase_			)

				UpperCamelCase__					=				scheduler_fn(lowercase_,		lowercase_,		lowercase_,		lowercase_			)

				UpperCamelCase__					=				optax.adamw(learning_rate=lowercase_,		weight_decay=lowercase_,		mask=lowercase_			)
				return tx, lr






def 		UpperCamelCase   (lowercase_:					list   )	-> list:
	A__				:       Union[str, Any]	  =						False
	while is_sorted is False:  # Until all the indices are traversed keep looping
		A__				:       Dict	  =						True
		for i in range(0       , len(lowercase_   ) - 1       , 2   ):  # iterating over all even indices
			if input_list[i] > input_list[i + 1]:
				A__  ,  A__				:       Dict	  =						input_list[i + 1], input_list[i]
				# swapping if elements not in order
				A__				:       Optional[Any]	  =						False

		for i in range(1       , len(lowercase_   ) - 1       , 2   ):  # iterating over all odd indices
			if input_list[i] > input_list[i + 1]:
				A__  ,  A__				:       Dict	  =						input_list[i + 1], input_list[i]
				# swapping if elements not in order
				A__				:       Optional[Any]	  =						False
	return input_list


if __name__ == "__main__":
	print('Enter list to be sorted')
	A_			:					Optional[int]    			=     [int(x) for x in input().split()]
	# inputing elements of the list in one line
	A_			:					Optional[int]    			=     odd_even_sort(input_list)
	print('The sorted list is')
	print(sorted_list)



