
'''simple docstring'''





import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
      if is_torch_available():
            import torch
      if is_tf_available():
            import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


SCREAMING_SNAKE_CASE     =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE     =       {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE     =       {
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """Salesforce/codegen-350M-mono""": (
            """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"""
        ),
    },
}

SCREAMING_SNAKE_CASE     =       {
    """Salesforce/codegen-350M-mono""": 2_0_4_8,
}






class 							UpperCAmelCase_							(    A				):

 '''simple docstring'''
 lowercase_					:    Tuple				=							VOCAB_FILES_NAMES
 lowercase_					:    Union[str, Any]				=							PRETRAINED_VOCAB_FILES_MAP
 lowercase_					:    Dict				=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowercase_					:    int				=							["input_ids", "attention_mask"]
 lowercase_					:    int				=							CodeGenTokenizer







 def __init__(					self		:  str			,				snake_case__		:  Optional[int]=None			,				snake_case__		:  Optional[Any]=None			,				snake_case__		:  Dict=None			,				snake_case__		:  Tuple="<|endoftext|>"			,				snake_case__		:  Union[str, Any]="<|endoftext|>"			,				snake_case__		:  Dict="<|endoftext|>"			,				snake_case__		:  Union[str, Any]=False			,				**snake_case__		:  Any			,				):



   '''simple docstring'''




   super().__init__(
       snake_case__			,				snake_case__			,				tokenizer_file=snake_case__			,				unk_token=snake_case__			,				bos_token=snake_case__			,				eos_token=snake_case__			,				add_prefix_space=snake_case__			,				**snake_case__			,				)

   if kwargs.pop("add_bos_token"			,				snake_case__ ):
     UpperCAmelCase__      :					Dict				       =							kwargs.pop("name_or_path"			,				"" )
     raise ValueError(
         "Currenty GPT2's fast tokenizer does NOT support adding a BOS token."
         "Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"
         F"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
         F"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
         "This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."
         " so that the fast tokenizer works correctly." )

   UpperCAmelCase__      :					Tuple				       =							json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__() )
   if pre_tok_state.get("add_prefix_space"			,				snake_case__ ) != add_prefix_space:
     UpperCAmelCase__      :					Optional[Any]				       =							getattr(snake_case__			,				pre_tok_state.pop("type" ) )
     UpperCAmelCase__      :					Union[str, Any]				       =							add_prefix_space
     UpperCAmelCase__      :					Dict				       =							pre_tok_class(**snake_case__ )

   UpperCAmelCase__      :					List[str]				       =							add_prefix_space







 def 							UpperCamelCase  (					self		:  Tuple			,				*snake_case__		:  List[str]			,				**snake_case__		:  Union[str, Any] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Union[str, Any]				       =							kwargs.get("is_split_into_words"			,				snake_case__ )
   assert self.add_prefix_space or not is_split_into_words, (
       F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
       "to use it with pretokenized inputs."
   )

   return super()._batch_encode_plus(*snake_case__			,				**snake_case__ )







 def 							UpperCamelCase  (					self		:  Dict			,				*snake_case__		:  Tuple			,				**snake_case__		:  List[str] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Tuple				       =							kwargs.get("is_split_into_words"			,				snake_case__ )

   assert self.add_prefix_space or not is_split_into_words, (
       F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
       "to use it with pretokenized inputs."
   )

   return super()._encode_plus(*snake_case__			,				**snake_case__ )







 def 							UpperCamelCase  (					self		:  Tuple			,				snake_case__		:  str			,				snake_case__		:  Optional[str] = None ):



   '''simple docstring'''




   UpperCAmelCase__      :					Dict				       =							self._tokenizer.model.save(snake_case__			,				name=snake_case__ )
   return tuple(snake_case__ )







 def 							UpperCamelCase  (					self		:  Tuple			,				snake_case__		:  Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]			,				snake_case__		:  bool = False			,				snake_case__		:  bool = None			,				snake_case__		:  Optional[List[str]] = None			,				**snake_case__		:  Optional[Any]			,				):



   '''simple docstring'''




   UpperCAmelCase__      :					List[Any]				       =							super().decode(
       token_ids=snake_case__			,				skip_special_tokens=snake_case__			,				clean_up_tokenization_spaces=snake_case__			,				**snake_case__			,				)

   if truncate_before_pattern is not None and len(snake_case__ ) > 0:
     UpperCAmelCase__      :					int				       =							self.truncate(snake_case__			,				snake_case__ )

   return decoded_text







 def 							UpperCamelCase  (					self		:  Optional[int]			,				snake_case__		:  str			,				snake_case__		:  Dict ):



   '''simple docstring'''




   def find_re(snake_case__		:  List[Any]			,				snake_case__		:  Dict			,				snake_case__		:  Dict ):
     UpperCAmelCase__      :					Optional[Any]				       =							pattern.search(snake_case__			,				snake_case__ )
     return m.start() if m else -1

   UpperCAmelCase__      :					List[str]				       =							[re.compile(snake_case__			,				re.MULTILINE ) for pattern in truncate_before_pattern]

   UpperCAmelCase__      :					Union[str, Any]				       =							list(re.finditer("^print"			,				snake_case__			,				re.MULTILINE ) )

   if len(snake_case__ ) > 1:
     UpperCAmelCase__      :					str				       =							completion[: prints[1].start()]

   UpperCAmelCase__      :					Optional[Any]				       =							list(re.finditer("^def"			,				snake_case__			,				re.MULTILINE ) )

   if len(snake_case__ ) > 1:
     UpperCAmelCase__      :					int				       =							completion[: defs[1].start()]

   UpperCAmelCase__      :					int				       =							0

   UpperCAmelCase__      :					List[Any]				       =							[
       pos for pos in [find_re(snake_case__			,				snake_case__			,				snake_case__ ) for terminal in terminals] if pos != -1
   ]

   if len(snake_case__ ) > 0:
     return completion[: min(snake_case__ )]
   else:
     return completion





'''simple docstring'''





from dataclasses import dataclass
from typing import Optional, Tuple, Union

import torch
import torch.nn as nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput, apply_forward_hook
from .modeling_utils import ModelMixin
from .vae import Decoder, DecoderOutput, Encoder, VectorQuantizer






@dataclass
class 							UpperCAmelCase_							(    A				):

 '''simple docstring'''
 lowercase_					:    torch.FloatTensor






class 							UpperCAmelCase_							(    A					,				A				):

 '''simple docstring'''







 @register_to_config
 def __init__(					self		:  List[str]			,				snake_case__		:  int = 3			,				snake_case__		:  int = 3			,				snake_case__		:  Tuple[str] = ("DownEncoderBlock2D",)			,				snake_case__		:  Tuple[str] = ("UpDecoderBlock2D",)			,				snake_case__		:  Tuple[int] = (64,)			,				snake_case__		:  int = 1			,				snake_case__		:  str = "silu"			,				snake_case__		:  int = 3			,				snake_case__		:  int = 32			,				snake_case__		:  int = 2_56			,				snake_case__		:  int = 32			,				snake_case__		:  Optional[int] = None			,				snake_case__		:  float = 0.18215			,				snake_case__		:  str = "group"			,				):



   '''simple docstring'''




   super().__init__()

   # pass init params to Encoder
   UpperCAmelCase__      :					Tuple				       =							Encoder(
       in_channels=snake_case__			,				out_channels=snake_case__			,				down_block_types=snake_case__			,				block_out_channels=snake_case__			,				layers_per_block=snake_case__			,				act_fn=snake_case__			,				norm_num_groups=snake_case__			,				double_z=snake_case__			,				)

   UpperCAmelCase__      :					str				       =							vq_embed_dim if vq_embed_dim is not None else latent_channels

   UpperCAmelCase__      :					Tuple				       =							nn.Convad(snake_case__			,				snake_case__			,				1 )
   UpperCAmelCase__      :					int				       =							VectorQuantizer(snake_case__			,				snake_case__			,				beta=0.25			,				remap=snake_case__			,				sane_index_shape=snake_case__ )
   UpperCAmelCase__      :					Tuple				       =							nn.Convad(snake_case__			,				snake_case__			,				1 )

   # pass init params to Decoder
   UpperCAmelCase__      :					Union[str, Any]				       =							Decoder(
       in_channels=snake_case__			,				out_channels=snake_case__			,				up_block_types=snake_case__			,				block_out_channels=snake_case__			,				layers_per_block=snake_case__			,				act_fn=snake_case__			,				norm_num_groups=snake_case__			,				norm_type=snake_case__			,				)







 @apply_forward_hook
 def 							UpperCamelCase  (					self		:  Union[str, Any]			,				snake_case__		:  torch.FloatTensor			,				snake_case__		:  bool = True ):



   '''simple docstring'''




   UpperCAmelCase__      :					List[Any]				       =							self.encoder(snake_case__ )
   UpperCAmelCase__      :					List[str]				       =							self.quant_conv(snake_case__ )

   if not return_dict:
     return (h,)

   return VQEncoderOutput(latents=snake_case__ )







 @apply_forward_hook
 def 							UpperCamelCase  (					self		:  List[str]			,				snake_case__		:  torch.FloatTensor			,				snake_case__		:  bool = False			,				snake_case__		:  bool = True ):



   '''simple docstring'''




   if not force_not_quantize:
     UpperCAmelCase__       ,			UpperCAmelCase__       ,			UpperCAmelCase__      :					Any				       =							self.quantize(snake_case__ )
   else:
     UpperCAmelCase__      :					Tuple				       =							h
   UpperCAmelCase__      :					int				       =							self.post_quant_conv(snake_case__ )
   UpperCAmelCase__      :					Optional[int]				       =							self.decoder(snake_case__			,				quant if self.config.norm_type == "spatial" else None )

   if not return_dict:
     return (dec,)

   return DecoderOutput(sample=snake_case__ )







 def 							UpperCamelCase  (					self		:  Any			,				snake_case__		:  torch.FloatTensor			,				snake_case__		:  bool = True ):



   '''simple docstring'''




   UpperCAmelCase__      :					Dict				       =							sample
   UpperCAmelCase__      :					int				       =							self.encode(snake_case__ ).latents
   UpperCAmelCase__      :					Any				       =							self.decode(snake_case__ ).sample

   if not return_dict:
     return (dec,)

   return DecoderOutput(sample=snake_case__ )



