
"""simple docstring"""



from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin





class 				a			(					lowercase						,		lowercase						,		lowercase      ):
      UpperCamelCase							:   str           =     [r"""h\.\d+\.attn\.bias""", r"""h\.\d+\.attn\.masked_bias"""]



      @register_to_config
      def __init__(   self     ,	UpperCamelCase_     ,	UpperCamelCase_     ,	UpperCamelCase_ = None     ,	UpperCamelCase_ = 50_257     ,	UpperCamelCase_ = 1_024     ,	UpperCamelCase_ = 768     ,	UpperCamelCase_ = 12     ,	UpperCamelCase_ = 12     ,	UpperCamelCase_ = None     ,	UpperCamelCase_ = "gelu_new"     ,	UpperCamelCase_ = 0.1     ,	UpperCamelCase_ = 0.1     ,	UpperCamelCase_ = 0.1     ,	UpperCamelCase_ = 1E-5     ,	UpperCamelCase_ = 0.02     ,	UpperCamelCase_ = True     ,	UpperCamelCase_ = True     ,	UpperCamelCase_ = False     ,	UpperCamelCase_ = False     ,	):
          super().__init__()

          UpperCAmelCase__	:      str     							=					prefix_length

          if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
              raise ValueError(
                  F'''`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and'''
                  F''' `n_embd`: {n_embd} are not equal.'''      )

          UpperCAmelCase__	:      str     							=					prefix_inner_dim
          UpperCAmelCase__	:      int     							=					prefix_hidden_dim

          UpperCAmelCase__	:      Tuple     							=					(
              nn.Linear(self.prefix_inner_dim     ,	self.prefix_hidden_dim      )
              if self.prefix_hidden_dim is not None
              else nn.Identity()
          )
          UpperCAmelCase__	:      Dict     							=					(
              nn.Linear(self.prefix_hidden_dim     ,	UpperCamelCase_      ) if self.prefix_hidden_dim is not None else nn.Identity()
          )

          UpperCAmelCase__	:      List[Any]     							=					GPTaConfig(
              vocab_size=UpperCamelCase_     ,	n_positions=UpperCamelCase_     ,	n_embd=UpperCamelCase_     ,	n_layer=UpperCamelCase_     ,	n_head=UpperCamelCase_     ,	n_inner=UpperCamelCase_     ,	activation_function=UpperCamelCase_     ,	resid_pdrop=UpperCamelCase_     ,	embd_pdrop=UpperCamelCase_     ,	attn_pdrop=UpperCamelCase_     ,	layer_norm_epsilon=UpperCamelCase_     ,	initializer_range=UpperCamelCase_     ,	scale_attn_weights=UpperCamelCase_     ,	use_cache=UpperCamelCase_     ,	scale_attn_by_inverse_layer_idx=UpperCamelCase_     ,	reorder_and_upcast_attn=UpperCamelCase_     ,	)
          UpperCAmelCase__	:      Optional[Any]     							=					GPTaLMHeadModel(UpperCamelCase_      )



      def  __snake_case							(   self     ,	UpperCamelCase_     ,	UpperCamelCase_     ,	UpperCamelCase_ = None     ,	UpperCamelCase_ = None     ,	):
          UpperCAmelCase__	:      List[Any]     							=					self.transformer.transformer.wte(UpperCamelCase_      )
          UpperCAmelCase__	:      List[Any]     							=					self.encode_prefix(UpperCamelCase_      )
          UpperCAmelCase__	:      Dict     							=					self.decode_prefix(UpperCamelCase_      )
          UpperCAmelCase__	:      Any     							=					torch.cat((prefix_embeds, embedding_text)     ,	dim=1      )

          if labels is not None:
              UpperCAmelCase__	:      Any     							=					self.get_dummy_token(input_ids.shape[0]     ,	input_ids.device      )
              UpperCAmelCase__	:      Any     							=					torch.cat((dummy_token, input_ids)     ,	dim=1      )
          UpperCAmelCase__	:      Union[str, Any]     							=					self.transformer(inputs_embeds=UpperCamelCase_     ,	labels=UpperCamelCase_     ,	attention_mask=UpperCamelCase_      )
          if self.prefix_hidden_dim is not None:
              return out, hidden
          else:
              return out



      def  __snake_case							(   self     ,	UpperCamelCase_     ,	UpperCamelCase_      ):
          return torch.zeros(UpperCamelCase_     ,	self.prefix_length     ,	dtype=torch.intaa     ,	device=UpperCamelCase_      )



      def  __snake_case							(   self     ,	UpperCamelCase_      ):
          return self.encode_prefix(UpperCamelCase_      )



      @torch.no_grad()
      def  __snake_case							(   self     ,	UpperCamelCase_     ,	UpperCamelCase_     ,	UpperCamelCase_      ):
          UpperCAmelCase__	:      int     							=					torch.split(UpperCamelCase_     ,	1     ,	dim=0      )
          UpperCAmelCase__	:      Optional[Any]     							=					[]
          UpperCAmelCase__	:      Dict     							=					[]
          for feature in features:
              UpperCAmelCase__	:      str     							=					self.decode_prefix(feature.to(UpperCamelCase_      )      )  # back to the clip feature
              # Only support beam search for now
              UpperCAmelCase__							,							UpperCAmelCase__	:      str     							=					self.generate_beam(
                  input_embeds=UpperCamelCase_     ,	device=UpperCamelCase_     ,	eos_token_id=UpperCamelCase_      )
              generated_tokens.append(output_tokens[0]      )
              generated_seq_lengths.append(seq_lengths[0]      )
          UpperCAmelCase__	:      List[str]     							=					torch.stack(UpperCamelCase_      )
          UpperCAmelCase__	:      Optional[Any]     							=					torch.stack(UpperCamelCase_      )
          return generated_tokens, generated_seq_lengths



      @torch.no_grad()
      def  __snake_case							(   self     ,	UpperCamelCase_=None     ,	UpperCamelCase_=None     ,	UpperCamelCase_=None     ,	UpperCamelCase_ = 5     ,	UpperCamelCase_ = 67     ,	UpperCamelCase_ = 1.0     ,	UpperCamelCase_ = None     ,	):
          UpperCAmelCase__	:      Tuple     							=					eos_token_id
          UpperCAmelCase__	:      Any     							=					None
          UpperCAmelCase__	:      int     							=					None
          UpperCAmelCase__	:      Tuple     							=					torch.ones(UpperCamelCase_     ,	device=UpperCamelCase_     ,	dtype=torch.int      )
          UpperCAmelCase__	:      str     							=					torch.zeros(UpperCamelCase_     ,	device=UpperCamelCase_     ,	dtype=torch.bool      )

          if input_embeds is not None:
              UpperCAmelCase__	:      Union[str, Any]     							=					input_embeds
          else:
              UpperCAmelCase__	:      Optional[int]     							=					self.transformer.transformer.wte(UpperCamelCase_      )

          for i in range(UpperCamelCase_      ):
              UpperCAmelCase__	:      Dict     							=					self.transformer(inputs_embeds=UpperCamelCase_      )
              UpperCAmelCase__	:      Optional[Any]     							=					outputs.logits
              UpperCAmelCase__	:      Tuple     							=					logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
              UpperCAmelCase__	:      Dict     							=					logits.softmax(-1      ).log()

              if scores is None:
                  UpperCAmelCase__							,							UpperCAmelCase__	:      Optional[Any]     							=					logits.topk(UpperCamelCase_     ,	-1      )
                  UpperCAmelCase__	:      Union[str, Any]     							=					generated.expand(UpperCamelCase_     ,	*generated.shape[1:]      )
                  UpperCAmelCase__							,							UpperCAmelCase__	:      Any     							=					next_tokens.permute(1     ,	0      ), scores.squeeze(0      )
                  if tokens is None:
                      UpperCAmelCase__	:      List[Any]     							=					next_tokens
                  else:
                      UpperCAmelCase__	:      Union[str, Any]     							=					tokens.expand(UpperCamelCase_     ,	*tokens.shape[1:]      )
                      UpperCAmelCase__	:      int     							=					torch.cat((tokens, next_tokens)     ,	dim=1      )
              else:
                  UpperCAmelCase__	:      str     							=					-float(np.inf      )
                  UpperCAmelCase__	:      Union[str, Any]     							=					0
                  UpperCAmelCase__	:      List[Any]     							=					scores[:, None] + logits
                  seq_lengths[~is_stopped] += 1
                  UpperCAmelCase__	:      int     							=					scores_sum / seq_lengths[:, None]
                  UpperCAmelCase__							,							UpperCAmelCase__	:      Union[str, Any]     							=					scores_sum_average.view(-1      ).topk(UpperCamelCase_     ,	-1      )
                  UpperCAmelCase__	:      List[str]     							=					next_tokens // scores_sum.shape[1]
                  UpperCAmelCase__	:      Dict     							=					seq_lengths[next_tokens_source]
                  UpperCAmelCase__	:      Any     							=					next_tokens % scores_sum.shape[1]
                  UpperCAmelCase__	:      Optional[int]     							=					next_tokens.unsqueeze(1      )
                  UpperCAmelCase__	:      Tuple     							=					tokens[next_tokens_source]
                  UpperCAmelCase__	:      Any     							=					torch.cat((tokens, next_tokens)     ,	dim=1      )
                  UpperCAmelCase__	:      Union[str, Any]     							=					generated[next_tokens_source]
                  UpperCAmelCase__	:      Optional[Any]     							=					scores_sum_average * seq_lengths
                  UpperCAmelCase__	:      Any     							=					is_stopped[next_tokens_source]

              UpperCAmelCase__	:      List[str]     							=					self.transformer.transformer.wte(next_tokens.squeeze()      ).view(generated.shape[0]     ,	1     ,	-1      )
              UpperCAmelCase__	:      Tuple     							=					torch.cat((generated, next_token_embed)     ,	dim=1      )
              UpperCAmelCase__	:      List[Any]     							=					is_stopped + next_tokens.eq(UpperCamelCase_      ).squeeze()
              if is_stopped.all():
                  break

          UpperCAmelCase__	:      Dict     							=					scores / seq_lengths
          UpperCAmelCase__	:      List[str]     							=					scores.argsort(descending=UpperCamelCase_      )
          # tokens tensors are already padded to max_seq_length
          UpperCAmelCase__	:      Dict     							=					[tokens[i] for i in order]
          UpperCAmelCase__	:      int     							=					torch.stack(UpperCamelCase_     ,	dim=0      )
          UpperCAmelCase__	:      List[Any]     							=					torch.tensor([seq_lengths[i] for i in order]     ,	dtype=seq_lengths.dtype      )
          return output_texts, seq_lengths



"""simple docstring"""



from itertools import permutations
def 					lowerCamelCase   (     _snake_case ):
    if num[3] % 2 != 0:
        return False

    if (num[2] + num[3] + num[4]) % 3 != 0:
        return False

    if num[5] % 5 != 0:
        return False

    UpperCAmelCase__	:      List[str]     							=					[7, 11, 13, 17]
    for i, test in enumerate(_snake_case ):
        if (num[i + 4] * 100 + num[i + 5] * 10 + num[i + 6]) % test != 0:
            return False
    return True
def 					lowerCamelCase   (     _snake_case = 10 ):
    return sum(
        int(''.join(map(_snake_case    ,_snake_case ) ) )
        for num in permutations(range(_snake_case ) )
        if is_substring_divisible(_snake_case ) )


if __name__ == "__main__":
    print(f'{solution() = }')

