



import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
		if is_torch_available():
				import torch
		if is_tf_available():
				import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


snake_case    :						Union[str, Any]  =						logging.get_logger(__name__)

snake_case    :						Dict  =						{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

snake_case    :						Any  =						{
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'Salesforce/codegen-350M-mono': (
            'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'
        ),
    },
}

snake_case    :						str  =						{
    'Salesforce/codegen-350M-mono': 2_0_4_8,
}





class 							lowerCAmelCase__  (	__A						):
					__A     :    Optional[int]           =						VOCAB_FILES_NAMES
					__A     :    List[Any]           =						PRETRAINED_VOCAB_FILES_MAP
					__A     :    Optional[Any]           =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__A     :    Any           =						['input_ids', 'attention_mask']
					__A     :    str           =						CodeGenTokenizer







					def __init__(       self      :     Any			,		_A      :     Union[str, Any]=None			,		_A      :     Any=None			,		_A      :     Union[str, Any]=None			,		_A      :     str="<|endoftext|>"			,		_A      :     Dict="<|endoftext|>"			,		_A      :     Any="<|endoftext|>"			,		_A      :     Optional[int]=False			,		**_A      :     Union[str, Any]			,		):
											super().__init__(
											    _A			,		_A			,		tokenizer_file=_A			,		unk_token=_A			,		bos_token=_A			,		eos_token=_A			,		add_prefix_space=_A			,		**_A			,		)

											if kwargs.pop("add_bos_token"			,		_A):
																	A__	:							List[str]						    = kwargs.pop("name_or_path"			,		"")
																	raise ValueError(
																	    "Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token."
																	    "Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n"
																	    F'`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'
																	    F'`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'
																	    "This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."
																	    " so that the fast tokenizer works correctly.")

											A__	:							List[str]						    = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
											if pre_tok_state.get("add_prefix_space"			,		_A) != add_prefix_space:
																	A__	:							List[Any]						    = getattr(_A			,		pre_tok_state.pop("type"))
																	A__	:							Tuple						    = add_prefix_space
																	A__	:							Union[str, Any]						    = pre_tok_class(**_A)

											A__	:							str						    = add_prefix_space







					def  _lowercase			(       self      :     Tuple			,		*_A      :     Optional[int]			,		**_A      :     str):
											A__	:							List[str]						    = kwargs.get("is_split_into_words"			,		_A)
											assert self.add_prefix_space or not is_split_into_words, (
											    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
											    "to use it with pretokenized inputs."
											)

											return super()._batch_encode_plus(*_A			,		**_A)







					def  _lowercase			(       self      :     Optional[int]			,		*_A      :     Any			,		**_A      :     str):
											A__	:							str						    = kwargs.get("is_split_into_words"			,		_A)

											assert self.add_prefix_space or not is_split_into_words, (
											    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
											    "to use it with pretokenized inputs."
											)

											return super()._encode_plus(*_A			,		**_A)







					def  _lowercase			(       self      :     int			,		_A      :     str			,		_A      :     Optional[str] = None):
											A__	:							Union[str, Any]						    = self._tokenizer.model.save(_A			,		name=_A)
											return tuple(_A)







					def  _lowercase			(       self      :     Optional[int]			,		_A      :     Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]			,		_A      :     bool = False			,		_A      :     bool = None			,		_A      :     Optional[List[str]] = None			,		**_A      :     Dict			,		):
											A__	:							str						    = super().decode(
											    token_ids=_A			,		skip_special_tokens=_A			,		clean_up_tokenization_spaces=_A			,		**_A			,		)

											if truncate_before_pattern is not None and len(_A) > 0:
																	A__	:							List[str]						    = self.truncate(_A			,		_A)

											return decoded_text







					def  _lowercase			(       self      :     int			,		_A      :     Tuple			,		_A      :     List[Any]):

											def find_re(_A      :     Dict			,		_A      :     Optional[int]			,		_A      :     int):
																	A__	:							Union[str, Any]						    = pattern.search(_A			,		_A)
																	return m.start() if m else -1

											A__	:							str						    = [re.compile(_A			,		re.MULTILINE) for pattern in truncate_before_pattern]

											A__	:							Any						    = list(re.finditer("^print"			,		_A			,		re.MULTILINE))

											if len(_A) > 1:
																	A__	:							List[str]						    = completion[: prints[1].start()]

											A__	:							Optional[int]						    = list(re.finditer("^def"			,		_A			,		re.MULTILINE))

											if len(_A) > 1:
																	A__	:							Optional[Any]						    = completion[: defs[1].start()]

											A__	:							Any						    = 0

											A__	:							int						    = [
											    pos for pos in [find_re(_A			,		_A			,		_A) for terminal in terminals] if pos != -1
											]

											if len(_A) > 0:
																	return completion[: min(_A)]
											else:
																	return completion




# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING, Dict, Optional

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.logging import get_logger
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
		import jax
		import jaxlib

snake_case    :						Optional[int]  =						get_logger()

snake_case    :						Optional[dict]  =						None




class 							lowerCAmelCase__  (	TensorFormatter[Mapping, 'jax.Array', Mapping]						):







					def __init__(       self      :     Optional[Any]			,		_A      :     Optional[int]=None			,		_A      :     str=None			,		**_A      :     Tuple):
											super().__init__(features=_A)
											import jax
											from jaxlib.xla_client import Device

											if isinstance(_A			,		_A):
																	raise ValueError(
																	    F'Expected {device} to be a `str` not {type(_A)}, as `jaxlib.xla_extension.Device` '
																	    "is not serializable neither with `pickle` nor with `dill`. Instead you can surround "
																	    "the device with `str()` to get its string identifier that will be internally mapped "
																	    "to the actual `jaxlib.xla_extension.Device`.")
											A__	:							Union[str, Any]						    = device if isinstance(_A			,		_A) else str(jax.devices()[0])
											# using global variable since `jaxlib.xla_extension.Device` is not serializable neither
											# with `pickle` nor with `dill`, so we need to use a global variable instead
											global DEVICE_MAPPING
											if DEVICE_MAPPING is None:
																	A__	:							Any						    = self._map_devices_to_str()
											if self.device not in list(DEVICE_MAPPING.keys()):
																	logger.warning(
																	    F'Device with string identifier {self.device} not listed among the available '
																	    F'devices: {list(DEVICE_MAPPING.keys())}, so falling back to the default '
																	    F'device: {str(jax.devices()[0])}.')
																	A__	:							str						    = str(jax.devices()[0])
											A__	:							List[Any]						    = jnp_array_kwargs







					@staticmethod
					def  _lowercase			(       ):
											import jax

											return {str(_A): device for device in jax.devices()}







					def  _lowercase			(       self      :     Tuple			,		_A      :     int):
											import jax
											import jax.numpy as jnp

											if isinstance(_A			,		_A) and column:
																	if all(
																	    isinstance(_A			,		jax.Array) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column):
																							return jnp.stack(_A			,		axis=0)
											return column







					def  _lowercase			(       self      :     List[str]			,		_A      :     Union[str, Any]):
											import jax
											import jax.numpy as jnp

											if isinstance(_A			,		(str, bytes, type(_A))):
																	return value
											elif isinstance(_A			,		(np.character, np.ndarray)) and np.issubdtype(value.dtype			,		np.character):
																	return value.tolist()

											A__	:							Dict						    = {}

											if isinstance(_A			,		(np.number, np.ndarray)) and np.issubdtype(value.dtype			,		np.integer):
																	# the default int precision depends on the jax config
																	# see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
																	if jax.config.jax_enable_xaa:
																							A__	:							Union[str, Any]						    = {"dtype": jnp.intaa}
																	else:
																							A__	:							int						    = {"dtype": jnp.intaa}
											elif isinstance(_A			,		(np.number, np.ndarray)) and np.issubdtype(value.dtype			,		np.floating):
																	A__	:							str						    = {"dtype": jnp.floataa}
											elif config.PIL_AVAILABLE and "PIL" in sys.modules:
																	import PIL.Image

																	if isinstance(_A			,		PIL.Image.Image):
																							A__	:							Any						    = np.asarray(_A)

        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
        # with `pickle` nor with `dill`, so we need to use a global variable instead
											global DEVICE_MAPPING
											if DEVICE_MAPPING is None:
																	A__	:							Optional[Any]						    = self._map_devices_to_str()

											with jax.default_device(DEVICE_MAPPING[self.device]):
																	# calling jnp.array on a np.ndarray does copy the data
																	# see https://github.com/google/jax/issues/4486
																	return jnp.array(_A			,		**{**default_dtype, **self.jnp_array_kwargs})







					def  _lowercase			(       self      :     Dict			,		_A      :     Any):
											import jax

											# support for torch, tf, jax etc.
											if config.TORCH_AVAILABLE and "torch" in sys.modules:
																	import torch

																	if isinstance(_A			,		torch.Tensor):
																							return self._tensorize(data_struct.detach().cpu().numpy()[()])
											if hasattr(_A			,		"__array__") and not isinstance(_A			,		jax.Array):
																	A__	:							Any						    = data_struct.__array__()
											# support for nested types like struct of list of struct
											if isinstance(_A			,		np.ndarray):
																	if data_struct.dtype == object:  # jax arrays cannot be instantied from an array of objects
																							return self._consolidate([self.recursive_tensorize(_A) for substruct in data_struct])
											elif isinstance(_A			,		(list, tuple)):
																	return self._consolidate([self.recursive_tensorize(_A) for substruct in data_struct])
											return self._tensorize(_A)







					def  _lowercase			(       self      :     Optional[Any]			,		_A      :     dict):
											return map_nested(self._recursive_tensorize			,		_A			,		map_list=_A)







					def  _lowercase			(       self      :     List[str]			,		_A      :     pa.Table):
											A__	:							Dict						    = self.numpy_arrow_extractor().extract_row(_A)
											A__	:							int						    = self.python_features_decoder.decode_row(_A)
											return self.recursive_tensorize(_A)







					def  _lowercase			(       self      :     int			,		_A      :     pa.Table):
											A__	:							int						    = self.numpy_arrow_extractor().extract_column(_A)
											A__	:							Optional[int]						    = self.python_features_decoder.decode_column(_A			,		pa_table.column_names[0])
											A__	:							Dict						    = self.recursive_tensorize(_A)
											A__	:							List[Any]						    = self._consolidate(_A)
											return column







					def  _lowercase			(       self      :     Tuple			,		_A      :     pa.Table):
											A__	:							Any						    = self.numpy_arrow_extractor().extract_batch(_A)
											A__	:							str						    = self.python_features_decoder.decode_batch(_A)
											A__	:							Optional[Any]						    = self.recursive_tensorize(_A)
											for column_name in batch:
																	A__	:							Optional[int]						    = self._consolidate(batch[column_name])
											return batch