

"""simple docstring"""




import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
     if is_torch_available():
          import torch
     if is_tf_available():
          import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


SCREAMING_SNAKE_CASE_     :			List[str]	  =			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_     :			Any	  =			{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

SCREAMING_SNAKE_CASE_     :			Tuple	  =			{
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''Salesforce/codegen-350M-mono''': (
            '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'''
        ),
    },
}

SCREAMING_SNAKE_CASE_     :			List[str]	  =			{
    '''Salesforce/codegen-350M-mono''': 2048,
}


class 		_A       (			__a					):
      __a								=	VOCAB_FILES_NAMES
      __a								=	PRETRAINED_VOCAB_FILES_MAP
      __a								=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __a								=	['input_ids', 'attention_mask']
      __a								=	CodeGenTokenizer





      def __init__(  self    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__="<|endoftext|>"    ,						SCREAMING_SNAKE_CASE__="<|endoftext|>"    ,						SCREAMING_SNAKE_CASE__="<|endoftext|>"    ,						SCREAMING_SNAKE_CASE__=False    ,						**SCREAMING_SNAKE_CASE__    ,						)   ->					Tuple:
         super().__init__(
             SCREAMING_SNAKE_CASE__    ,						SCREAMING_SNAKE_CASE__    ,						tokenizer_file=SCREAMING_SNAKE_CASE__    ,						unk_token=SCREAMING_SNAKE_CASE__    ,						bos_token=SCREAMING_SNAKE_CASE__    ,						eos_token=SCREAMING_SNAKE_CASE__    ,						add_prefix_space=SCREAMING_SNAKE_CASE__    ,						**SCREAMING_SNAKE_CASE__    ,						)

         if kwargs.pop("add_bos_token"    ,						SCREAMING_SNAKE_CASE__		):
            lowerCamelCase__						=			kwargs.pop("name_or_path"    ,						""		)
            raise ValueError(
                "Currenty GPT2's fast tokenizer does NOT support adding a BOS token."
                "Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"
                f'`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'
                f'`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'
                "This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."
                " so that the fast tokenizer works correctly."		)

         lowerCamelCase__						=			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
         if pre_tok_state.get("add_prefix_space"    ,						SCREAMING_SNAKE_CASE__		) != add_prefix_space:
            lowerCamelCase__						=			getattr(SCREAMING_SNAKE_CASE__    ,						pre_tok_state.pop("type"		)		)
            lowerCamelCase__						=			add_prefix_space
            lowerCamelCase__						=			pre_tok_class(**SCREAMING_SNAKE_CASE__		)

         lowerCamelCase__						=			add_prefix_space





      def 					_lowerCamelCase  (  self    ,						*SCREAMING_SNAKE_CASE__    ,						**SCREAMING_SNAKE_CASE__		)   ->					BatchEncoding:
         lowerCamelCase__						=			kwargs.get("is_split_into_words"    ,						SCREAMING_SNAKE_CASE__		)
         assert self.add_prefix_space or not is_split_into_words, (
             f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
             "to use it with pretokenized inputs."
         )

         return super()._batch_encode_plus(*SCREAMING_SNAKE_CASE__    ,						**SCREAMING_SNAKE_CASE__		)





      def 					_lowerCamelCase  (  self    ,						*SCREAMING_SNAKE_CASE__    ,						**SCREAMING_SNAKE_CASE__		)   ->					BatchEncoding:
         lowerCamelCase__						=			kwargs.get("is_split_into_words"    ,						SCREAMING_SNAKE_CASE__		)

         assert self.add_prefix_space or not is_split_into_words, (
             f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
             "to use it with pretokenized inputs."
         )

         return super()._encode_plus(*SCREAMING_SNAKE_CASE__    ,						**SCREAMING_SNAKE_CASE__		)





      def 					_lowerCamelCase  (  self    ,						SCREAMING_SNAKE_CASE__    ,						SCREAMING_SNAKE_CASE__ = None		)   ->					Tuple[str]:
         lowerCamelCase__						=			self._tokenizer.model.save(SCREAMING_SNAKE_CASE__    ,						name=SCREAMING_SNAKE_CASE__		)
         return tuple(SCREAMING_SNAKE_CASE__		)





      def 					_lowerCamelCase  (  self    ,						SCREAMING_SNAKE_CASE__    ,						SCREAMING_SNAKE_CASE__ = False    ,						SCREAMING_SNAKE_CASE__ = None    ,						SCREAMING_SNAKE_CASE__ = None    ,						**SCREAMING_SNAKE_CASE__    ,						)   ->					str:
         lowerCamelCase__						=			super().decode(
             token_ids=SCREAMING_SNAKE_CASE__    ,						skip_special_tokens=SCREAMING_SNAKE_CASE__    ,						clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE__    ,						**SCREAMING_SNAKE_CASE__    ,						)

         if truncate_before_pattern is not None and len(SCREAMING_SNAKE_CASE__		) > 0:
            lowerCamelCase__						=			self.truncate(SCREAMING_SNAKE_CASE__    ,						SCREAMING_SNAKE_CASE__		)

         return decoded_text





      def 					_lowerCamelCase  (  self    ,						SCREAMING_SNAKE_CASE__    ,						SCREAMING_SNAKE_CASE__		)   ->					Optional[Any]:
         def find_re(SCREAMING_SNAKE_CASE__    ,						SCREAMING_SNAKE_CASE__    ,						SCREAMING_SNAKE_CASE__		):
            lowerCamelCase__						=			pattern.search(SCREAMING_SNAKE_CASE__    ,						SCREAMING_SNAKE_CASE__		)
            return m.start() if m else -1

         lowerCamelCase__						=			[re.compile(SCREAMING_SNAKE_CASE__    ,						re.MULTILINE		) for pattern in truncate_before_pattern]

         lowerCamelCase__						=			list(re.finditer("^print"    ,						SCREAMING_SNAKE_CASE__    ,						re.MULTILINE		)		)

         if len(SCREAMING_SNAKE_CASE__		) > 1:
            lowerCamelCase__						=			completion[: prints[1].start()]

         lowerCamelCase__						=			list(re.finditer("^def"    ,						SCREAMING_SNAKE_CASE__    ,						re.MULTILINE		)		)

         if len(SCREAMING_SNAKE_CASE__		) > 1:
            lowerCamelCase__						=			completion[: defs[1].start()]

         lowerCamelCase__						=			0

         lowerCamelCase__						=			[
             pos for pos in [find_re(SCREAMING_SNAKE_CASE__    ,						SCREAMING_SNAKE_CASE__    ,						SCREAMING_SNAKE_CASE__		) for terminal in terminals] if pos != -1
         ]

         if len(SCREAMING_SNAKE_CASE__		) > 0:
            return completion[: min(SCREAMING_SNAKE_CASE__		)]
         else:
            return completion







"""simple docstring"""




import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_     :			Dict	  =			logging.get_logger(__name__)


class 		_A       (			__a					):
      __a								=	'encoder-decoder'
      __a								=	True





      def __init__(  self    ,						**SCREAMING_SNAKE_CASE__		)   ->					Optional[int]:
         super().__init__(**SCREAMING_SNAKE_CASE__		)
         assert (
             "encoder" in kwargs and "decoder" in kwargs
         ), "Config has to be initialized with encoder and decoder config"
         lowerCamelCase__						=			kwargs.pop("encoder"		)
         lowerCamelCase__						=			encoder_config.pop("model_type"		)
         lowerCamelCase__						=			kwargs.pop("decoder"		)
         lowerCamelCase__						=			decoder_config.pop("model_type"		)

         from ..auto.configuration_auto import AutoConfig

         lowerCamelCase__						=			AutoConfig.for_model(SCREAMING_SNAKE_CASE__    ,						**SCREAMING_SNAKE_CASE__		)
         lowerCamelCase__						=			AutoConfig.for_model(SCREAMING_SNAKE_CASE__    ,						**SCREAMING_SNAKE_CASE__		)
         lowerCamelCase__						=			True





      @classmethod
      def 					_lowerCamelCase  (  cls    ,						SCREAMING_SNAKE_CASE__    ,						SCREAMING_SNAKE_CASE__    ,						**SCREAMING_SNAKE_CASE__		)   ->					PretrainedConfig:
         logger.info("Set `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config"		)
         lowerCamelCase__						=			True
         lowerCamelCase__						=			True

         return cls(encoder=encoder_config.to_dict()    ,						decoder=decoder_config.to_dict()    ,						**SCREAMING_SNAKE_CASE__		)





      def 					_lowerCamelCase  (  self		)   ->					int:
         lowerCamelCase__						=			copy.deepcopy(self.__dict__		)
         lowerCamelCase__						=			self.encoder.to_dict()
         lowerCamelCase__						=			self.decoder.to_dict()
         lowerCamelCase__						=			self.__class__.model_type
         return output




