
"""simple docstring"""
from typing import Optional, Tuple, Union

import flax
import flax.linen as nn
import jax
import jax.numpy as jnp
from flax.core.frozen_dict import FrozenDict

from ..configuration_utils import ConfigMixin, flax_register_to_config
from ..utils import BaseOutput
from .embeddings_flax import FlaxTimestepEmbedding, FlaxTimesteps
from .modeling_flax_utils import FlaxModelMixin
from .unet_ad_blocks_flax import (
    FlaxCrossAttnDownBlockaD,
    FlaxCrossAttnUpBlockaD,
    FlaxDownBlockaD,
    FlaxUNetMidBlockaDCrossAttn,
    FlaxUpBlockaD,
)
@flax.struct.dataclass
class 			UpperCamelCase      (							__SCREAMING_SNAKE_CASE  ):
     A__      =			42
@flax_register_to_config
class 			UpperCamelCase      (							nn.Module      ,       __SCREAMING_SNAKE_CASE      ,       __SCREAMING_SNAKE_CASE  ):
     A__      =			32
     A__      =			4
     A__      =			4
     A__      =			(
         """CrossAttnDownBlock2D""",
         """CrossAttnDownBlock2D""",
         """CrossAttnDownBlock2D""",
         """DownBlock2D""",
     )
     A__      =			("""UpBlock2D""", """CrossAttnUpBlock2D""", """CrossAttnUpBlock2D""", """CrossAttnUpBlock2D""")
     A__      =			False
     A__      =			(320, 640, 1280, 1280)
     A__      =			2
     A__      =			8
     A__      =			None
     A__      =			1280
     A__      =			0.0
     A__      =			False
     A__      =			jnp.floataa
     A__      =			True
     A__      =			0
     A__      =			False
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = (1, self.in_channels, self.sample_size, self.sample_size)
       _SCREAMING_SNAKE_CASE   :				int							    = jnp.zeros(snake_case__  ,     dtype=jnp.floataa				)
       _SCREAMING_SNAKE_CASE   :				str							    = jnp.ones((1,)  ,     dtype=jnp.intaa				)
       _SCREAMING_SNAKE_CASE   :				Any							    = jnp.zeros((1, 1, self.cross_attention_dim)  ,     dtype=jnp.floataa				)

       _SCREAMING_SNAKE_CASE   :				Tuple							    = jax.random.split(snake_case__				)
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = {"params": params_rng, "dropout": dropout_rng}

       return self.init(snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__				)["params"]
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				List[str]							    = self.block_out_channels
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = block_out_channels[0] * 4

       if self.num_attention_heads is not None:
         raise ValueError(
             "At the moment it is not possible to define the number of attention heads via `num_attention_heads` because of a naming issue as described in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. Passing `num_attention_heads` will only be supported in diffusers v0.19."				)

       # If `num_attention_heads` is not defined (which is the case for most models)
       # it will default to `attention_head_dim`. This looks weird upon first reading it and it is.
       # The reason for this behavior is to correct for incorrectly named variables that were introduced
       # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131
       # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking
       # which is why we correct for the naming here.
       _SCREAMING_SNAKE_CASE   :				List[str]							    = self.num_attention_heads or self.attention_head_dim

       # input
       _SCREAMING_SNAKE_CASE   :				int							    = nn.Conv(
           block_out_channels[0]  ,     kernel_size=(3, 3)  ,     strides=(1, 1)  ,     padding=((1, 1), (1, 1))  ,     dtype=self.dtype  ,     )

       # time
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = FlaxTimesteps(
           block_out_channels[0]  ,     flip_sin_to_cos=self.flip_sin_to_cos  ,     freq_shift=self.config.freq_shift				)
       _SCREAMING_SNAKE_CASE   :				Dict							    = FlaxTimestepEmbedding(snake_case__  ,     dtype=self.dtype				)

       _SCREAMING_SNAKE_CASE   :				str							    = self.only_cross_attention
       if isinstance(snake_case__  ,     snake_case__				):
         _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = (only_cross_attention,) * len(self.down_block_types				)

       if isinstance(snake_case__  ,     snake_case__				):
         _SCREAMING_SNAKE_CASE   :				int							    = (num_attention_heads,) * len(self.down_block_types				)

       # down
       _SCREAMING_SNAKE_CASE   :				Tuple							    = []
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = block_out_channels[0]
       for i, down_block_type in enumerate(self.down_block_types				):
         _SCREAMING_SNAKE_CASE   :				str							    = output_channel
         _SCREAMING_SNAKE_CASE   :				List[str]							    = block_out_channels[i]
         _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = i == len(snake_case__				) - 1

         if down_block_type == "CrossAttnDownBlock2D":
           _SCREAMING_SNAKE_CASE   :				Optional[int]							    = FlaxCrossAttnDownBlockaD(
               in_channels=snake_case__  ,     out_channels=snake_case__  ,     dropout=self.dropout  ,     num_layers=self.layers_per_block  ,     num_attention_heads=num_attention_heads[i]  ,     add_downsample=not is_final_block  ,     use_linear_projection=self.use_linear_projection  ,     only_cross_attention=only_cross_attention[i]  ,     use_memory_efficient_attention=self.use_memory_efficient_attention  ,     dtype=self.dtype  ,     )
         else:
           _SCREAMING_SNAKE_CASE   :				str							    = FlaxDownBlockaD(
               in_channels=snake_case__  ,     out_channels=snake_case__  ,     dropout=self.dropout  ,     num_layers=self.layers_per_block  ,     add_downsample=not is_final_block  ,     dtype=self.dtype  ,     )

         down_blocks.append(snake_case__				)
       _SCREAMING_SNAKE_CASE   :				Tuple							    = down_blocks

       # mid
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = FlaxUNetMidBlockaDCrossAttn(
           in_channels=block_out_channels[-1]  ,     dropout=self.dropout  ,     num_attention_heads=num_attention_heads[-1]  ,     use_linear_projection=self.use_linear_projection  ,     use_memory_efficient_attention=self.use_memory_efficient_attention  ,     dtype=self.dtype  ,     )

       # up
       _SCREAMING_SNAKE_CASE   :				int							    = []
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = list(reversed(snake_case__				)				)
       _SCREAMING_SNAKE_CASE   :				Any							    = list(reversed(snake_case__				)				)
       _SCREAMING_SNAKE_CASE   :				Any							    = list(reversed(snake_case__				)				)
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = reversed_block_out_channels[0]
       for i, up_block_type in enumerate(self.up_block_types				):
         _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = output_channel
         _SCREAMING_SNAKE_CASE   :				List[str]							    = reversed_block_out_channels[i]
         _SCREAMING_SNAKE_CASE   :				List[Any]							    = reversed_block_out_channels[min(i + 1  ,     len(snake_case__				) - 1				)]

         _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = i == len(snake_case__				) - 1

         if up_block_type == "CrossAttnUpBlock2D":
           _SCREAMING_SNAKE_CASE   :				int							    = FlaxCrossAttnUpBlockaD(
               in_channels=snake_case__  ,     out_channels=snake_case__  ,     prev_output_channel=snake_case__  ,     num_layers=self.layers_per_block + 1  ,     num_attention_heads=reversed_num_attention_heads[i]  ,     add_upsample=not is_final_block  ,     dropout=self.dropout  ,     use_linear_projection=self.use_linear_projection  ,     only_cross_attention=only_cross_attention[i]  ,     use_memory_efficient_attention=self.use_memory_efficient_attention  ,     dtype=self.dtype  ,     )
         else:
           _SCREAMING_SNAKE_CASE   :				str							    = FlaxUpBlockaD(
               in_channels=snake_case__  ,     out_channels=snake_case__  ,     prev_output_channel=snake_case__  ,     num_layers=self.layers_per_block + 1  ,     add_upsample=not is_final_block  ,     dropout=self.dropout  ,     dtype=self.dtype  ,     )

         up_blocks.append(snake_case__				)
         _SCREAMING_SNAKE_CASE   :				Any							    = output_channel
       _SCREAMING_SNAKE_CASE   :				int							    = up_blocks

       # out
       _SCREAMING_SNAKE_CASE   :				str							    = nn.GroupNorm(num_groups=32  ,     epsilon=1E-5				)
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = nn.Conv(
           self.out_channels  ,     kernel_size=(3, 3)  ,     strides=(1, 1)  ,     padding=((1, 1), (1, 1))  ,     dtype=self.dtype  ,     )




     def __call__(     self  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__=None  ,     snake_case__=None  ,     snake_case__ = True  ,     snake_case__ = False  ,     ):




       """simple docstring"""


       if not isinstance(snake_case__  ,     jnp.ndarray				):
         _SCREAMING_SNAKE_CASE   :				Any							    = jnp.array([timesteps]  ,     dtype=jnp.intaa				)
       elif isinstance(snake_case__  ,     jnp.ndarray				) and len(timesteps.shape				) == 0:
         _SCREAMING_SNAKE_CASE   :				Tuple							    = timesteps.astype(dtype=jnp.floataa				)
         _SCREAMING_SNAKE_CASE   :				Tuple							    = jnp.expand_dims(snake_case__  ,     0				)

       _SCREAMING_SNAKE_CASE   :				List[str]							    = self.time_proj(snake_case__				)
       _SCREAMING_SNAKE_CASE   :				Dict							    = self.time_embedding(snake_case__				)

       # 2. pre-process
       _SCREAMING_SNAKE_CASE   :				Dict							    = jnp.transpose(snake_case__  ,     (0, 2, 3, 1)				)
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = self.conv_in(snake_case__				)

       # 3. down
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = (sample,)
       for down_block in self.down_blocks:
         if isinstance(snake_case__  ,     snake_case__				):
           _SCREAMING_SNAKE_CASE   :				str							    = down_block(snake_case__  ,     snake_case__  ,     snake_case__  ,     deterministic=not train				)
         else:
           _SCREAMING_SNAKE_CASE   :				int							    = down_block(snake_case__  ,     snake_case__  ,     deterministic=not train				)
         down_block_res_samples += res_samples

       if down_block_additional_residuals is not None:
         _SCREAMING_SNAKE_CASE   :				str							    = ()

         for down_block_res_sample, down_block_additional_residual in zip(
             snake_case__  ,     snake_case__				):
           down_block_res_sample += down_block_additional_residual
           new_down_block_res_samples += (down_block_res_sample,)

         _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = new_down_block_res_samples

       # 4. mid
       _SCREAMING_SNAKE_CASE   :				Dict							    = self.mid_block(snake_case__  ,     snake_case__  ,     snake_case__  ,     deterministic=not train				)

       if mid_block_additional_residual is not None:
         sample += mid_block_additional_residual

       # 5. up
       for up_block in self.up_blocks:
         _SCREAMING_SNAKE_CASE   :				List[str]							    = down_block_res_samples[-(self.layers_per_block + 1) :]
         _SCREAMING_SNAKE_CASE   :				int							    = down_block_res_samples[: -(self.layers_per_block + 1)]
         if isinstance(snake_case__  ,     snake_case__				):
           _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = up_block(
               snake_case__  ,     temb=snake_case__  ,     encoder_hidden_states=snake_case__  ,     res_hidden_states_tuple=snake_case__  ,     deterministic=not train  ,     )
         else:
           _SCREAMING_SNAKE_CASE   :				List[str]							    = up_block(snake_case__  ,     temb=snake_case__  ,     res_hidden_states_tuple=snake_case__  ,     deterministic=not train				)

        # 6. post-process
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = self.conv_norm_out(snake_case__				)
       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = nn.silu(snake_case__				)
       _SCREAMING_SNAKE_CASE   :				List[str]							    = self.conv_out(snake_case__				)
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = jnp.transpose(snake_case__  ,     (0, 3, 1, 2)				)

       if not return_dict:
         return (sample,)

       return FlaxUNetaDConditionOutput(sample=snake_case__				)





"""simple docstring"""
from dataclasses import dataclass, field
from typing import Optional
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Model name or path of model to be trained."""}  )
     A__      =			field(
         default="""./"""      ,       metadata={"""help""": """Save dir where model repo is cloned and models updates are saved to."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot-clean-train"""      ,       metadata={"""help""": """Name or path of training dataset."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot-clean-valid"""      ,       metadata={"""help""": """Name or path of validation dataset."""}  )
     A__      =			field(default=2      ,       metadata={"""help""": """Batch size for training."""}  )
     A__      =			field(default=2      ,       metadata={"""help""": """Batch size for evaluation."""}  )
     A__      =			field(default=0.1      ,       metadata={"""help""": """Value of weight decay."""}  )
     A__      =			field(
         default=10000      ,       metadata={"""help""": """Size of buffer used to shuffle streaming dataset."""}  )
     A__      =			field(default=2e-4      ,       metadata={"""help""": """Learning rate fo training."""}  )
     A__      =			field(default="""cosine"""      ,       metadata={"""help""": """Learning rate."""}  )
     A__      =			field(
         default=750      ,       metadata={"""help""": """Number of warmup steps in the learning rate schedule."""}  )
     A__      =			field(
         default=16      ,       metadata={"""help""": """Number of gradient accumulation steps."""}  )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Use gradient checkpointing to reduce memory footprint."""}  )
     A__      =			field(default=50000      ,       metadata={"""help""": """Maximum number of training steps."""}  )
     A__      =			field(
         default=-1      ,       metadata={"""help""": """Maximum number of evaluation steps. If -1 the full dataset is evaluated."""}  )
     A__      =			field(default=1024      ,       metadata={"""help""": """Sequence lengths used for training."""}  )
     A__      =			field(default=1      ,       metadata={"""help""": """Training seed."""}  )
     A__      =			field(
         default=1024      ,       metadata={"""help""": """Interval to save checkpoints. Measured as number of forward passes not training steps."""}      ,       )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """States path if the training should continue from a checkpoint folder."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """If True the data is pretokenized."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Model name or path of model to be evaluated."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot-clean-valid"""      ,       metadata={"""help""": """Name or path of validation dataset."""}  )
     A__      =			field(default=2      ,       metadata={"""help""": """Batch size used for evaluation."""}  )
     A__      =			field(
         default=-1      ,       metadata={"""help""": """Maximum number of evaluation steps. If -1 the full dataset is evaluated."""}  )
     A__      =			field(default=1024      ,       metadata={"""help""": """Length of sequences to be evaluated."""}  )
     A__      =			field(default=1      ,       metadata={"""help""": """Random seed used for evaluation."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Model name or path of model to be evaluated."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Number of workers used for code evaluation."""}  )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """The number of human-eval tasks to run. If not included all tasks are evaluated."""}      ,       )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Sample from the language model's output distribution."""}  )
     A__      =			field(default=0.2      ,       metadata={"""help""": """Sampling temperature used for generation."""}  )
     A__      =			field(default=256      ,       metadata={"""help""": """Maximum number of newly generated tokens."""}  )
     A__      =			field(default=0      ,       metadata={"""help""": """Top-k parameter used for generation."""}  )
     A__      =			field(default=0.95      ,       metadata={"""help""": """Top-p parameter used for nucleus sampling."""}  )
     A__      =			field(default=10      ,       metadata={"""help""": """Number of generations to run in parallel."""}  )
     A__      =			field(
         default=200      ,       metadata={"""help""": """Number of completions to generate for each sample."""}  )
     A__      =			field(default=1      ,       metadata={"""help""": """Random seed used for evaluation."""}  )
     A__      =			field(
         default="""eval_results.json"""      ,       metadata={"""help""": """Random seed used for evaluation."""}  )
     A__      =			field(
         default="""0"""      ,       metadata={"""help""": """Allow `code_eval` to execute Python code on machine"""}  )
     A__      =			field(
         default=-1      ,       metadata={
             """help""": (
                 """Determine which device to run the `text-generation` Pipeline on. -1 is CPU and any zero or positive"""
                 """ number corresponds to which GPU device id to run on."""
             )
         }      ,       )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={
             """help""": """The number of CPU cores to use for parallel preprocessing. Default uses the maximum available."""
         }      ,       )
     A__      =			field(
         default="""transformersbook/codeparrot"""      ,       metadata={"""help""": """Folder or name of dataset to process."""}  )
     A__      =			field(
         default="""codeparrot-clean"""      ,       metadata={"""help""": """Folder to save processed processed dataset."""}  )
     A__      =			field(
         default=100000      ,       metadata={"""help""": """Number of files to save per JSON output file."""}  )
     A__      =			field(default="""content"""      ,       metadata={"""help""": """Column containing text data to process."""}  )
     A__      =			field(
         default=1000      ,       metadata={"""help""": """Maximum line length in file, otherwise file is filtered."""}  )
     A__      =			field(
         default=100      ,       metadata={"""help""": """Maximum mean line length in file, otherwise file is filtered."""}  )
     A__      =			field(
         default=0.25      ,       metadata={"""help""": """Maximum fraction of non-alphanumeric characters, otherwise file is filtered."""}  )
     A__      =			field(
         default=1.5      ,       metadata={"""help""": """Minimum character token ratio for the file, otherwise file is filtered."""}  )
     A__      =			field(
         default=0.7      ,       metadata={"""help""": """Probability for filtering config, test and uncommon files."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Name or path to the tokenizer."""}      ,       )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """If True, near-duplicate samples are removed."""}  )
     A__      =			field(
         default=0.85      ,       metadata={"""help""": """Jaccard threshold for near-duplicate samples."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""gpt2"""      ,       metadata={"""help""": """Base tokenizer to build new tokenizer from."""}  )
     A__      =			field(
         default="""transformersbook/codeparrot-train"""      ,       metadata={"""help""": """Dataset to train tokenizer on."""}  )
     A__      =			field(default="""content"""      ,       metadata={"""help""": """Column containing text data to process."""}  )
     A__      =			field(default=200000      ,       metadata={"""help""": """Number of examples to train tokenizer on."""}  )
     A__      =			field(
         default=32768      ,       metadata={"""help""": """Number of examples to train the tokenizer on."""}  )
     A__      =			field(default="""codeparrot"""      ,       metadata={"""help""": """Name of new tokenizer."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Push saved tokenizer to the hub."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Name or path to the tokenizer."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot-clean-train"""      ,       metadata={"""help""": """Name or path to the dataset to pretokenize."""}  )
     A__      =			field(
         default="""tokenized-codeparrot-train"""      ,       metadata={"""help""": """Repo name of the pretokenized data."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Number of workers used for code evaluation."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""gpt2-large"""      ,       metadata={"""help""": """Configuration to use for model initialization."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Tokenizer attached to model."""}  )
     A__      =			field(default="""codeparrot"""      ,       metadata={"""help""": """Name of the created model."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Push saved tokenizer to the hub."""}  )



