



'''simple docstring'''






import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
						if is_torch_available():
												import torch
						if is_tf_available():
												import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


UpperCamelCase__:     List[Any]										=    logging.get_logger(__name__)

UpperCamelCase__:     str										=    {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

UpperCamelCase__:     Any										=    {
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "Salesforce/codegen-350M-mono": (
            "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"
        ),
    },
}

UpperCamelCase__:     int										=    {
    "Salesforce/codegen-350M-mono": 2048,
}

class        SCREAMING_SNAKE_CASE(							A__    ):







	"""simple docstring"""






	lowerCamelCase__    				= VOCAB_FILES_NAMES
	lowerCamelCase__    				= PRETRAINED_VOCAB_FILES_MAP
	lowerCamelCase__    				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCamelCase__    				= ["""input_ids""", """attention_mask"""]
	lowerCamelCase__    				= CodeGenTokenizer







	def __init__(		self    :  Union[str, Any]	,    __snake_case    :  Any=None	,    __snake_case    :  int=None	,    __snake_case    :  List[str]=None	,    __snake_case    :  Optional[int]="<|endoftext|>"	,    __snake_case    :  Optional[int]="<|endoftext|>"	,    __snake_case    :  Optional[int]="<|endoftext|>"	,    __snake_case    :  List[Any]=False	,    **__snake_case    :  Optional[int]	,    )    ->	List[str]:
								super().__init__(
								    __snake_case	,    __snake_case	,    tokenizer_file=__snake_case	,    unk_token=__snake_case	,    bos_token=__snake_case	,    eos_token=__snake_case	,    add_prefix_space=__snake_case	,    **__snake_case	,    )

								if kwargs.pop('''add_bos_token'''	,    __snake_case	):
															UpperCAmelCase				: Optional[int]											=	kwargs.pop('''name_or_path'''	,    ''''''	)
															raise ValueError(
															    '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
															    '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
															    F"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
															    F"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
															    '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
															    ''' so that the fast tokenizer works correctly.'''	)

								UpperCAmelCase				: Any											=	json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
								if pre_tok_state.get('''add_prefix_space'''	,    __snake_case	) != add_prefix_space:
															UpperCAmelCase				: str											=	getattr(__snake_case	,    pre_tok_state.pop('''type'''	)	)
															UpperCAmelCase				: List[str]											=	add_prefix_space
															UpperCAmelCase				: List[Any]											=	pre_tok_class(**__snake_case	)

								UpperCAmelCase				: Any											=	add_prefix_space







	def   A			(		self    :  str	,    *__snake_case    :  Dict	,    **__snake_case    :  List[Any]	)    ->	BatchEncoding:
								UpperCAmelCase				: int											=	kwargs.get('''is_split_into_words'''	,    __snake_case	)
								assert self.add_prefix_space or not is_split_into_words, (
								    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
								    "to use it with pretokenized inputs."
								)

								return super()._batch_encode_plus(*__snake_case	,    **__snake_case	)







	def   A			(		self    :  Dict	,    *__snake_case    :  str	,    **__snake_case    :  Dict	)    ->	BatchEncoding:
								UpperCAmelCase				: Any											=	kwargs.get('''is_split_into_words'''	,    __snake_case	)

								assert self.add_prefix_space or not is_split_into_words, (
								    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
								    "to use it with pretokenized inputs."
								)

								return super()._encode_plus(*__snake_case	,    **__snake_case	)







	def   A			(		self    :  Optional[Any]	,    __snake_case    :  str	,    __snake_case    :  Optional[str] = None	)    ->	Tuple[str]:
								UpperCAmelCase				: List[Any]											=	self._tokenizer.model.save(__snake_case	,    name=__snake_case	)
								return tuple(__snake_case	)







	def   A			(		self    :  int	,    __snake_case    :  Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]	,    __snake_case    :  bool = False	,    __snake_case    :  bool = None	,    __snake_case    :  Optional[List[str]] = None	,    **__snake_case    :  Optional[Any]	,    )    ->	str:
								UpperCAmelCase				: List[str]											=	super().decode(
								    token_ids=__snake_case	,    skip_special_tokens=__snake_case	,    clean_up_tokenization_spaces=__snake_case	,    **__snake_case	,    )

								if truncate_before_pattern is not None and len(__snake_case	) > 0:
															UpperCAmelCase				: List[Any]											=	self.truncate(__snake_case	,    __snake_case	)

								return decoded_text







	def   A			(		self    :  int	,    __snake_case    :  int	,    __snake_case    :  Tuple	)    ->	List[str]:
								def find_re(__snake_case    :  List[str]	,    __snake_case    :  Optional[Any]	,    __snake_case    :  Tuple	):
															UpperCAmelCase				: str											=	pattern.search(__snake_case	,    __snake_case	)
															return m.start() if m else -1

								UpperCAmelCase				: List[str]											=	[re.compile(__snake_case	,    re.MULTILINE	) for pattern in truncate_before_pattern]

								UpperCAmelCase				: List[Any]											=	list(re.finditer('''^print'''	,    __snake_case	,    re.MULTILINE	)	)

								if len(__snake_case	) > 1:
															UpperCAmelCase				: Union[str, Any]											=	completion[: prints[1].start()]

								UpperCAmelCase				: List[Any]											=	list(re.finditer('''^def'''	,    __snake_case	,    re.MULTILINE	)	)

								if len(__snake_case	) > 1:
															UpperCAmelCase				: Optional[Any]											=	completion[: defs[1].start()]

								UpperCAmelCase				: Tuple											=	0

								UpperCAmelCase				: int											=	[
								    pos for pos in [find_re(__snake_case	,    __snake_case	,    __snake_case	) for terminal in terminals] if pos != -1
								]

								if len(__snake_case	) > 0:
															return completion[: min(__snake_case	)]
								else:
															return completion










'''simple docstring'''






import math

import tensorflow as tf
from packaging import version


def 				snake_case_  (      _lowerCAmelCase						:		List[str]						)   ->     Dict:
							UpperCAmelCase				: Tuple											=	tf.convert_to_tensor(_lowerCAmelCase						)
							UpperCAmelCase				: Dict											=	0.5 * (1.0 + tf.math.erf(x / tf.cast(tf.sqrt(2.0						)     ,   x.dtype						)						))

							return x * cdf


def 				snake_case_  (      _lowerCAmelCase						:		int						)   ->     List[Any]:
							UpperCAmelCase				: str											=	tf.convert_to_tensor(_lowerCAmelCase						)
							UpperCAmelCase				: Union[str, Any]											=	tf.cast(math.pi     ,   x.dtype						)
							UpperCAmelCase				: Tuple											=	tf.cast(0.0_4_4_7_1_5     ,   x.dtype						)
							UpperCAmelCase				: List[Any]											=	0.5 * (1.0 + tf.tanh(tf.sqrt(2.0 / pi						) * (x + coeff * tf.pow(_lowerCAmelCase     ,   3						))						))

							return x * cdf


def 				snake_case_  (      _lowerCAmelCase						:		Optional[Any]						)   ->     Any:
							UpperCAmelCase				: List[Any]											=	tf.convert_to_tensor(_lowerCAmelCase						)

							return x * tf.tanh(tf.math.softplus(_lowerCAmelCase						)						)


def 				snake_case_  (      _lowerCAmelCase						:		str						)   ->     Tuple:
							UpperCAmelCase				: Union[str, Any]											=	tf.convert_to_tensor(_lowerCAmelCase						)
							UpperCAmelCase				: Union[str, Any]											=	tf.cast(0.0_4_4_7_1_5     ,   x.dtype						)
							UpperCAmelCase				: Tuple											=	tf.cast(0.7_9_7_8_8_4_5_6_0_8     ,   x.dtype						)

							return 0.5 * x * (1.0 + tf.tanh(x * coeffa * (1.0 + coeffa * x * x)						))


def 				snake_case_  (      _lowerCAmelCase						:		Optional[Any]						)   ->     Optional[int]:
							UpperCAmelCase				: str											=	tf.convert_to_tensor(_lowerCAmelCase						)
							UpperCAmelCase				: Optional[int]											=	tf.cast(1.7_0_2     ,   x.dtype						)
							return x * tf.math.sigmoid(coeff * x						)


def 				snake_case_  (      _lowerCAmelCase						:		Any						)   ->     Any:
							return tf.clip_by_value(_gelu(_lowerCAmelCase						)     ,   -10     ,   10						)


def 				snake_case_  (      _lowerCAmelCase						:		Dict     ,   _lowerCAmelCase						:		Optional[Any]=-1						)   ->     Union[str, Any]:
							UpperCAmelCase							,	UpperCAmelCase				: Optional[Any]											=	tf.split(_lowerCAmelCase     ,   2     ,   axis=_lowerCAmelCase						)
							return a * tf.math.sigmoid(_lowerCAmelCase						)


if version.parse(tf.version.VERSION) >= version.parse("2.4"):


						def 				snake_case_  (      _lowerCAmelCase						:		Tuple						)   ->     Any:
													return tf.keras.activations.gelu(_lowerCAmelCase     ,   approximate=_lowerCAmelCase						)

						UpperCamelCase__:     List[Any]										=    tf.keras.activations.gelu




						UpperCamelCase__:     List[Any]										=    approximate_gelu_wrap
else:
						UpperCamelCase__:     List[str]										=    _gelu
						UpperCamelCase__:     Tuple										=    _gelu_new


UpperCamelCase__:     List[Any]										=    {
    "gelu": gelu,
    "gelu_10": gelu_aa,
    "gelu_fast": gelu_fast,
    "gelu_new": gelu_new,
    "glu": glu,
    "mish": mish,
    "quick_gelu": quick_gelu,
    "relu": tf.keras.activations.relu,
    "sigmoid": tf.keras.activations.sigmoid,
    "silu": tf.keras.activations.swish,
    "swish": tf.keras.activations.swish,
    "tanh": tf.keras.activations.tanh,
}




def 				snake_case_  (      _lowerCAmelCase						:		Dict						)   ->     List[Any]:
							if activation_string in ACTaFN:
														return ACTaFN[activation_string]
							else:
														raise KeyError(f"""function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()						)}"""						)





