


'''simple docstring'''

import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
   if is_torch_available():
      import torch
   if is_tf_available():
      import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


lowercase__				     =		logging.get_logger(__name__)

lowercase__				     =		{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

lowercase__				     =		{
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "Salesforce/codegen-350M-mono": (
            "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"
        ),
    },
}

lowercase__				     =		{
    "Salesforce/codegen-350M-mono": 2_048,
}





class 						__SCREAMING_SNAKE_CASE					(      lowerCamelCase__ ):
     UpperCAmelCase 					=				VOCAB_FILES_NAMES
     UpperCAmelCase 					=				PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase 					=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase 					=				['''input_ids''', '''attention_mask''']
     UpperCAmelCase 					=				CodeGenTokenizer






     def __init__(			self     ,							__UpperCamelCase=None     ,							__UpperCamelCase=None     ,							__UpperCamelCase=None     ,							__UpperCamelCase="<|endoftext|>"     ,							__UpperCamelCase="<|endoftext|>"     ,							__UpperCamelCase="<|endoftext|>"     ,							__UpperCamelCase=False     ,							**__UpperCamelCase     ,							)       ->    Any:
          super().__init__(
              __UpperCamelCase     ,							__UpperCamelCase     ,							tokenizer_file=__UpperCamelCase     ,							unk_token=__UpperCamelCase     ,							bos_token=__UpperCamelCase     ,							eos_token=__UpperCamelCase     ,							add_prefix_space=__UpperCamelCase     ,							**__UpperCamelCase     ,							)

          if kwargs.pop("add_bos_token"     ,							__UpperCamelCase		):
               _a						=		kwargs.pop("name_or_path"     ,							""		)
               raise ValueError(
                   "Currenty GPT2's fast tokenizer does NOT support adding a BOS token."
                   "Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"
                   f"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
                   f"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
                   "This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."
                   " so that the fast tokenizer works correctly."		)

          _a						=		json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
          if pre_tok_state.get("add_prefix_space"     ,							__UpperCamelCase		) != add_prefix_space:
               _a						=		getattr(__UpperCamelCase     ,							pre_tok_state.pop("type"		)		)
               _a						=		add_prefix_space
               _a						=		pre_tok_class(**__UpperCamelCase		)

          _a						=		add_prefix_space






     def  a_					(			self     ,							*__UpperCamelCase     ,							**__UpperCamelCase		)       ->    BatchEncoding:
          _a						=		kwargs.get("is_split_into_words"     ,							__UpperCamelCase		)
          assert self.add_prefix_space or not is_split_into_words, (
              f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
              "to use it with pretokenized inputs."
          )

          return super()._batch_encode_plus(*__UpperCamelCase     ,							**__UpperCamelCase		)






     def  a_					(			self     ,							*__UpperCamelCase     ,							**__UpperCamelCase		)       ->    BatchEncoding:
          _a						=		kwargs.get("is_split_into_words"     ,							__UpperCamelCase		)

          assert self.add_prefix_space or not is_split_into_words, (
              f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
              "to use it with pretokenized inputs."
          )

          return super()._encode_plus(*__UpperCamelCase     ,							**__UpperCamelCase		)






     def  a_					(			self     ,							__UpperCamelCase     ,							__UpperCamelCase = None		)       ->    Tuple[str]:
          _a						=		self._tokenizer.model.save(__UpperCamelCase     ,							name=__UpperCamelCase		)
          return tuple(__UpperCamelCase		)






     def  a_					(			self     ,							__UpperCamelCase     ,							__UpperCamelCase = False     ,							__UpperCamelCase = None     ,							__UpperCamelCase = None     ,							**__UpperCamelCase     ,							)       ->    str:
          _a						=		super().decode(
              token_ids=__UpperCamelCase     ,							skip_special_tokens=__UpperCamelCase     ,							clean_up_tokenization_spaces=__UpperCamelCase     ,							**__UpperCamelCase     ,							)

          if truncate_before_pattern is not None and len(__UpperCamelCase		) > 0:
               _a						=		self.truncate(__UpperCamelCase     ,							__UpperCamelCase		)

          return decoded_text






     def  a_					(			self     ,							__UpperCamelCase     ,							__UpperCamelCase		)       ->    List[Any]:
          def find_re(__UpperCamelCase     ,							__UpperCamelCase     ,							__UpperCamelCase		):
               _a						=		pattern.search(__UpperCamelCase     ,							__UpperCamelCase		)
               return m.start() if m else -1

          _a						=		[re.compile(__UpperCamelCase     ,							re.MULTILINE		) for pattern in truncate_before_pattern]

          _a						=		list(re.finditer("^print"     ,							__UpperCamelCase     ,							re.MULTILINE		)		)

          if len(__UpperCamelCase		) > 1:
               _a						=		completion[: prints[1].start()]

          _a						=		list(re.finditer("^def"     ,							__UpperCamelCase     ,							re.MULTILINE		)		)

          if len(__UpperCamelCase		) > 1:
               _a						=		completion[: defs[1].start()]

          _a						=		0

          _a						=		[
              pos for pos in [find_re(__UpperCamelCase     ,							__UpperCamelCase     ,							__UpperCamelCase		) for terminal in terminals] if pos != -1
          ]

          if len(__UpperCamelCase		) > 0:
               return completion[: min(__UpperCamelCase		)]
          else:
               return completion





'''simple docstring'''

from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowercase__				     =		logging.get_logger(__name__)

lowercase__				     =		{
    "YituTech/conv-bert-base": "https://huggingface.co/YituTech/conv-bert-base/resolve/main/config.json",
    "YituTech/conv-bert-medium-small": (
        "https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/config.json"
    ),
    "YituTech/conv-bert-small": "https://huggingface.co/YituTech/conv-bert-small/resolve/main/config.json",
    # See all ConvBERT models at https://huggingface.co/models?filter=convbert
}





class 						__SCREAMING_SNAKE_CASE					(      lowerCamelCase__ ):
     UpperCAmelCase 					=				'''convbert'''






     def __init__(			self     ,							__UpperCamelCase=30_522     ,							__UpperCamelCase=768     ,							__UpperCamelCase=12     ,							__UpperCamelCase=12     ,							__UpperCamelCase=3_072     ,							__UpperCamelCase="gelu"     ,							__UpperCamelCase=0.1     ,							__UpperCamelCase=0.1     ,							__UpperCamelCase=512     ,							__UpperCamelCase=2     ,							__UpperCamelCase=0.02     ,							__UpperCamelCase=1e-12     ,							__UpperCamelCase=1     ,							__UpperCamelCase=0     ,							__UpperCamelCase=2     ,							__UpperCamelCase=768     ,							__UpperCamelCase=2     ,							__UpperCamelCase=9     ,							__UpperCamelCase=1     ,							__UpperCamelCase=None     ,							**__UpperCamelCase     ,							)       ->    Optional[int]:
          super().__init__(
              pad_token_id=__UpperCamelCase     ,							bos_token_id=__UpperCamelCase     ,							eos_token_id=__UpperCamelCase     ,							**__UpperCamelCase     ,							)

          _a						=		vocab_size
          _a						=		hidden_size
          _a						=		num_hidden_layers
          _a						=		num_attention_heads
          _a						=		intermediate_size
          _a						=		hidden_act
          _a						=		hidden_dropout_prob
          _a						=		attention_probs_dropout_prob
          _a						=		max_position_embeddings
          _a						=		type_vocab_size
          _a						=		initializer_range
          _a						=		layer_norm_eps
          _a						=		embedding_size
          _a						=		head_ratio
          _a						=		conv_kernel_size
          _a						=		num_groups
          _a						=		classifier_dropout







class 						__SCREAMING_SNAKE_CASE					(      lowerCamelCase__ ):






     @property
     def  a_					(			self		)       ->    Mapping[str, Mapping[int, str]]:
          if self.task == "multiple-choice":
               _a						=		{0: "batch", 1: "choice", 2: "sequence"}
          else:
               _a						=		{0: "batch", 1: "sequence"}
          return OrderedDict(
              [
                  ("input_ids", dynamic_axis),
                  ("attention_mask", dynamic_axis),
                  ("token_type_ids", dynamic_axis),
              ]		)

