
"""simple docstring"""


import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
						if is_torch_available():
												import torch
						if is_tf_available():
												import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


A_    							=					logging.get_logger(__name__)

A_    							=					{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

A_    							=					{
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """Salesforce/codegen-350M-mono""": (
            """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"""
        ),
    },
}

A_    							=					{
    """Salesforce/codegen-350M-mono""": 2048,
}




class  __lowerCamelCase  ( lowerCAmelCase			):
				a__:      List[str]          =    VOCAB_FILES_NAMES
				a__:      Any          =    PRETRAINED_VOCAB_FILES_MAP
				a__:      int          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				a__:      Union[str, Any]          =    ['input_ids', 'attention_mask']
				a__:      List[str]          =    CodeGenTokenizer




				def __init__(				self							,						UpperCAmelCase=None							,						UpperCAmelCase=None							,						UpperCAmelCase=None							,						UpperCAmelCase="<|endoftext|>"							,						UpperCAmelCase="<|endoftext|>"							,						UpperCAmelCase="<|endoftext|>"							,						UpperCAmelCase=False							,						**UpperCAmelCase							,						):
							super().__init__(
							    UpperCAmelCase							,						UpperCAmelCase							,						tokenizer_file=UpperCAmelCase							,						unk_token=UpperCAmelCase							,						bos_token=UpperCAmelCase							,						eos_token=UpperCAmelCase							,						add_prefix_space=UpperCAmelCase							,						**UpperCAmelCase							,						)

							if kwargs.pop('''add_bos_token'''							,						UpperCAmelCase   ):
										lowerCamelCase_					=  kwargs.pop('''name_or_path'''							,						''''''   )
										raise ValueError(
										    '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
										    '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
										    f"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
										    f"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
										    '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
										    ''' so that the fast tokenizer works correctly.'''   )

							lowerCamelCase_					=  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
							if pre_tok_state.get('''add_prefix_space'''							,						UpperCAmelCase   ) != add_prefix_space:
										lowerCamelCase_					=  getattr(UpperCAmelCase							,						pre_tok_state.pop('''type'''   )   )
										lowerCamelCase_					=  add_prefix_space
										lowerCamelCase_					=  pre_tok_class(**UpperCAmelCase   )

							lowerCamelCase_					=  add_prefix_space




				def        UpperCAmelCase__						(				self							,						*UpperCAmelCase							,						**UpperCAmelCase   ):
							lowerCamelCase_					=  kwargs.get('''is_split_into_words'''							,						UpperCAmelCase   )
							assert self.add_prefix_space or not is_split_into_words, (
							    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
							    "to use it with pretokenized inputs."
							)

							return super()._batch_encode_plus(*UpperCAmelCase							,						**UpperCAmelCase   )




				def        UpperCAmelCase__						(				self							,						*UpperCAmelCase							,						**UpperCAmelCase   ):
							lowerCamelCase_					=  kwargs.get('''is_split_into_words'''							,						UpperCAmelCase   )

							assert self.add_prefix_space or not is_split_into_words, (
							    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
							    "to use it with pretokenized inputs."
							)

							return super()._encode_plus(*UpperCAmelCase							,						**UpperCAmelCase   )




				def        UpperCAmelCase__						(				self							,						UpperCAmelCase							,						UpperCAmelCase = None   ):
							lowerCamelCase_					=  self._tokenizer.model.save(UpperCAmelCase							,						name=UpperCAmelCase   )
							return tuple(UpperCAmelCase   )




				def        UpperCAmelCase__						(				self							,						UpperCAmelCase							,						UpperCAmelCase = False							,						UpperCAmelCase = None							,						UpperCAmelCase = None							,						**UpperCAmelCase							,						):
							lowerCamelCase_					=  super().decode(
							    token_ids=UpperCAmelCase							,						skip_special_tokens=UpperCAmelCase							,						clean_up_tokenization_spaces=UpperCAmelCase							,						**UpperCAmelCase							,						)

							if truncate_before_pattern is not None and len(UpperCAmelCase   ) > 0:
										lowerCamelCase_					=  self.truncate(UpperCAmelCase							,						UpperCAmelCase   )

							return decoded_text





				def        UpperCAmelCase__						(				self							,						UpperCAmelCase							,						UpperCAmelCase   ):
							def find_re(UpperCAmelCase							,						UpperCAmelCase							,						UpperCAmelCase   ):
										lowerCamelCase_					=  pattern.search(UpperCAmelCase							,						UpperCAmelCase   )
										return m.start() if m else -1

							lowerCamelCase_					=  [re.compile(UpperCAmelCase							,						re.MULTILINE   ) for pattern in truncate_before_pattern]

							lowerCamelCase_					=  list(re.finditer('''^print'''							,						UpperCAmelCase							,						re.MULTILINE   )   )

							if len(UpperCAmelCase   ) > 1:
										lowerCamelCase_					=  completion[: prints[1].start()]

							lowerCamelCase_					=  list(re.finditer('''^def'''							,						UpperCAmelCase							,						re.MULTILINE   )   )

							if len(UpperCAmelCase   ) > 1:
										lowerCamelCase_					=  completion[: defs[1].start()]

							lowerCamelCase_					=  0

							lowerCamelCase_					=  [
							    pos for pos in [find_re(UpperCAmelCase							,						UpperCAmelCase							,						UpperCAmelCase   ) for terminal in terminals] if pos != -1
							]

							if len(UpperCAmelCase   ) > 0:
										return completion[: min(UpperCAmelCase   )]
							else:
										return completion





"""simple docstring"""





# Imports
import numpy as np






class        _UpperCAmelCase:



		def __init__(   self							,						__a=None							,						__a=None							,						__a=None							,						__a=None							,						__a=None)			->			Dict:






							'''simple docstring'''
							self.set_matricies(red=__a							,						green=__a							,						blue=__a							,						red_edge=__a							,						nir=__a)



		def 		UpperCAmelCase				(   self							,						__a=None							,						__a=None							,						__a=None							,						__a=None							,						__a=None)			->			Dict:






							'''simple docstring'''
							if red is not None:
												_UpperCamelCase									=    red
							if green is not None:
												_UpperCamelCase									=    green
							if blue is not None:
												_UpperCamelCase									=    blue
							if red_edge is not None:
												_UpperCamelCase									=    red_edge
							if nir is not None:
												_UpperCamelCase									=    nir
							return True



		def 		UpperCAmelCase				(   self							,						__a=""							,						__a=None							,						__a=None							,						__a=None							,						__a=None							,						__a=None)			->			List[str]:






							'''simple docstring'''
							self.set_matricies(red=__a							,						green=__a							,						blue=__a							,						red_edge=__a							,						nir=__a)
							_UpperCamelCase									=    {
							    '''ARVI2''': self.arvaa,
							    '''CCCI''': self.ccci,
							    '''CVI''': self.cvi,
							    '''GLI''': self.gli,
							    '''NDVI''': self.ndvi,
							    '''BNDVI''': self.bndvi,
							    '''redEdgeNDVI''': self.red_edge_ndvi,
							    '''GNDVI''': self.gndvi,
							    '''GBNDVI''': self.gbndvi,
							    '''GRNDVI''': self.grndvi,
							    '''RBNDVI''': self.rbndvi,
							    '''PNDVI''': self.pndvi,
							    '''ATSAVI''': self.atsavi,
							    '''BWDRVI''': self.bwdrvi,
							    '''CIgreen''': self.ci_green,
							    '''CIrededge''': self.ci_rededge,
							    '''CI''': self.ci,
							    '''CTVI''': self.ctvi,
							    '''GDVI''': self.gdvi,
							    '''EVI''': self.evi,
							    '''GEMI''': self.gemi,
							    '''GOSAVI''': self.gosavi,
							    '''GSAVI''': self.gsavi,
							    '''Hue''': self.hue,
							    '''IVI''': self.ivi,
							    '''IPVI''': self.ipvi,
							    '''I''': self.i,
							    '''RVI''': self.rvi,
							    '''MRVI''': self.mrvi,
							    '''MSAVI''': self.m_savi,
							    '''NormG''': self.norm_g,
							    '''NormNIR''': self.norm_nir,
							    '''NormR''': self.norm_r,
							    '''NGRDI''': self.ngrdi,
							    '''RI''': self.ri,
							    '''S''': self.s,
							    '''IF''': self._if,
							    '''DVI''': self.dvi,
							    '''TVI''': self.tvi,
							    '''NDRE''': self.ndre,
							}

							try:
												return funcs[index]()
							except KeyError:
												print('''Index not in the list!''')
												return False



		def 		UpperCAmelCase				(   self)			->			List[Any]:






							'''simple docstring'''
							return -0.18 + (1.17 * ((self.nir - self.red) / (self.nir + self.red)))



		def 		UpperCAmelCase				(   self)			->			Any:






							'''simple docstring'''
							return ((self.nir - self.redEdge) / (self.nir + self.redEdge)) / (
							    (self.nir - self.red) / (self.nir + self.red)
							)



		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							return self.nir * (self.red / (self.green**2))



		def 		UpperCAmelCase				(   self)			->			str:






							'''simple docstring'''
							return (2 * self.green - self.red - self.blue) / (
							    2 * self.green + self.red + self.blue
							)



		def 		UpperCAmelCase				(   self)			->			List[str]:






							'''simple docstring'''
							return (self.nir - self.red) / (self.nir + self.red)



		def 		UpperCAmelCase				(   self)			->			str:






							'''simple docstring'''
							return (self.nir - self.blue) / (self.nir + self.blue)



		def 		UpperCAmelCase				(   self)			->			List[Any]:






							'''simple docstring'''
							return (self.redEdge - self.red) / (self.redEdge + self.red)



		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							return (self.nir - self.green) / (self.nir + self.green)



		def 		UpperCAmelCase				(   self)			->			Optional[Any]:






							'''simple docstring'''
							return (self.nir - (self.green + self.blue)) / (
							    self.nir + (self.green + self.blue)
							)



		def 		UpperCAmelCase				(   self)			->			Tuple:






							'''simple docstring'''
							return (self.nir - (self.green + self.red)) / (
							    self.nir + (self.green + self.red)
							)



		def 		UpperCAmelCase				(   self)			->			List[Any]:






							'''simple docstring'''
							return (self.nir - (self.blue + self.red)) / (self.nir + (self.blue + self.red))



		def 		UpperCAmelCase				(   self)			->			List[str]:






							'''simple docstring'''
							return (self.nir - (self.green + self.red + self.blue)) / (
							    self.nir + (self.green + self.red + self.blue)
							)



		def 		UpperCAmelCase				(   self							,						__a=0.08							,						__a=1.22							,						__a=0.03)			->			Optional[Any]:






							'''simple docstring'''
							return a * (
							    (self.nir - a * self.red - b)
							    / (a * self.nir + self.red - a * b + x * (1 + a**2))
							)



		def 		UpperCAmelCase				(   self)			->			Dict:






							'''simple docstring'''
							return (0.1 * self.nir - self.blue) / (0.1 * self.nir + self.blue)



		def 		UpperCAmelCase				(   self)			->			List[str]:






							'''simple docstring'''
							return (self.nir / self.green) - 1



		def 		UpperCAmelCase				(   self)			->			List[Any]:






							'''simple docstring'''
							return (self.nir / self.redEdge) - 1



		def 		UpperCAmelCase				(   self)			->			Union[str, Any]:






							'''simple docstring'''
							return (self.red - self.blue) / self.red



		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							_UpperCamelCase									=    self.ndvi()
							return ((ndvi + 0.5) / (abs(ndvi + 0.5))) * (abs(ndvi + 0.5) ** (1 / 2))



		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							return self.nir - self.green



		def 		UpperCAmelCase				(   self)			->			List[str]:






							'''simple docstring'''
							return 2.5 * (
							    (self.nir - self.red) / (self.nir + 6 * self.red - 7.5 * self.blue + 1)
							)



		def 		UpperCAmelCase				(   self)			->			Tuple:






							'''simple docstring'''
							_UpperCamelCase									=    (2 * (self.nir**2 - self.red**2) + 1.5 * self.nir + 0.5 * self.red) / (
							    self.nir + self.red + 0.5
							)
							return n * (1 - 0.25 * n) - (self.red - 0.125) / (1 - self.red)



		def 		UpperCAmelCase				(   self							,						__a=0.16)			->			Optional[Any]:






							'''simple docstring'''
							return (self.nir - self.green) / (self.nir + self.green + y)



		def 		UpperCAmelCase				(   self							,						__a=0.5)			->			Dict:






							'''simple docstring'''
							return ((self.nir - self.green) / (self.nir + self.green + n)) * (1 + n)



		def 		UpperCAmelCase				(   self)			->			Dict:






							'''simple docstring'''
							return np.arctan(
							    ((2 * self.red - self.green - self.blue) / 30.5) * (self.green - self.blue))



		def 		UpperCAmelCase				(   self							,						__a=None							,						__a=None)			->			Any:






							'''simple docstring'''
							return (self.nir - b) / (a * self.red)



		def 		UpperCAmelCase				(   self)			->			Optional[Any]:






							'''simple docstring'''
							return (self.nir / ((self.nir + self.red) / 2)) * (self.ndvi() + 1)



		def 		UpperCAmelCase				(   self)			->			Optional[Any]:






							'''simple docstring'''
							return (self.red + self.green + self.blue) / 30.5



		def 		UpperCAmelCase				(   self)			->			Any:






							'''simple docstring'''
							return self.nir / self.red



		def 		UpperCAmelCase				(   self)			->			Tuple:






							'''simple docstring'''
							return (self.rvi() - 1) / (self.rvi() + 1)



		def 		UpperCAmelCase				(   self)			->			List[Any]:






							'''simple docstring'''
							return (
							    (2 * self.nir + 1)
							    - ((2 * self.nir + 1) ** 2 - 8 * (self.nir - self.red)) ** (1 / 2)
							) / 2



		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							return self.green / (self.nir + self.red + self.green)



		def 		UpperCAmelCase				(   self)			->			str:






							'''simple docstring'''
							return self.nir / (self.nir + self.red + self.green)



		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							return self.red / (self.nir + self.red + self.green)



		def 		UpperCAmelCase				(   self)			->			Tuple:






							'''simple docstring'''
							return (self.green - self.red) / (self.green + self.red)



		def 		UpperCAmelCase				(   self)			->			Dict:






							'''simple docstring'''
							return (self.red - self.green) / (self.red + self.green)



		def 		UpperCAmelCase				(   self)			->			Any:






							'''simple docstring'''
							_UpperCamelCase									=    np.max([np.max(self.red), np.max(self.green), np.max(self.blue)])
							_UpperCamelCase									=    np.min([np.min(self.red), np.min(self.green), np.min(self.blue)])
							return (max_value - min_value) / max_value



		def 		UpperCAmelCase				(   self)			->			str:






							'''simple docstring'''
							return (2 * self.red - self.green - self.blue) / (self.green - self.blue)



		def 		UpperCAmelCase				(   self)			->			int:






							'''simple docstring'''
							return self.nir / self.red



		def 		UpperCAmelCase				(   self)			->			Any:






							'''simple docstring'''
							return (self.ndvi() + 0.5) ** (1 / 2)




		def 		UpperCAmelCase				(   self)			->			Union[str, Any]:






							'''simple docstring'''
							return (self.nir - self.redEdge) / (self.nir + self.redEdge)






