"""simple docstring"""


import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
	if is_torch_available():
		import torch
	if is_tf_available():
		import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


a							=logging.get_logger(__name__)

a							={
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

a							={
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

a							={
    'Salesforce/codegen-350M-mono': 2048,
}





@lru_cache()
def 			lowerCamelCase_ (					)							->					Tuple:




						'''simple docstring'''



						lowerCamelCase__					=(
						    list(range(ord("!"				)   ,    ord("~"				) + 1				)				) + list(range(ord("¡"				)   ,    ord("¬"				) + 1				)				) + list(range(ord("®"				)   ,    ord("ÿ"				) + 1				)				)
						)
						lowerCamelCase__					=bs[:]
						lowerCamelCase__					=0
						for b in range(2**8				):
												if b not in bs:
																		bs.append(__lowerCAmelCase				)
																		cs.append(2**8 + n				)
																		n += 1
						lowerCamelCase__					=[chr(__lowerCAmelCase				) for n in cs]
						return dict(zip(__lowerCAmelCase   ,    __lowerCAmelCase				)				)





def 			lowerCamelCase_ (					__lowerCAmelCase				)							->					List[Any]:




						'''simple docstring'''



						lowerCamelCase__					=set()
						lowerCamelCase__					=word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)				)
												lowerCamelCase__					=char
						return pairs






class    __UpperCAmelCase						(	__lowerCAmelCase	):
				A__      :   List[Any]  							=       VOCAB_FILES_NAMES
				A__      :   Any  							=       PRETRAINED_VOCAB_FILES_MAP
				A__      :   Tuple  							=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				A__      :   Optional[Any]  							=       ['''input_ids''', '''attention_mask''']

				def __init__(      self	, _lowerCamelCase	, _lowerCamelCase	, _lowerCamelCase="replace"	, _lowerCamelCase="<|endoftext|>"	, _lowerCamelCase="<|endoftext|>"	, _lowerCamelCase="<|endoftext|>"	, _lowerCamelCase=None	, _lowerCamelCase=False	, _lowerCamelCase=False	, **_lowerCamelCase	, ):
										lowerCamelCase__					=AddedToken(_lowerCamelCase	, lstrip=_lowerCamelCase	, rstrip=_lowerCamelCase      ) if isinstance(_lowerCamelCase	, _lowerCamelCase      ) else bos_token
										lowerCamelCase__					=AddedToken(_lowerCamelCase	, lstrip=_lowerCamelCase	, rstrip=_lowerCamelCase      ) if isinstance(_lowerCamelCase	, _lowerCamelCase      ) else eos_token
										lowerCamelCase__					=AddedToken(_lowerCamelCase	, lstrip=_lowerCamelCase	, rstrip=_lowerCamelCase      ) if isinstance(_lowerCamelCase	, _lowerCamelCase      ) else unk_token
										lowerCamelCase__					=AddedToken(_lowerCamelCase	, lstrip=_lowerCamelCase	, rstrip=_lowerCamelCase      ) if isinstance(_lowerCamelCase	, _lowerCamelCase      ) else pad_token
										super().__init__(
										    errors=_lowerCamelCase	, unk_token=_lowerCamelCase	, bos_token=_lowerCamelCase	, eos_token=_lowerCamelCase	, pad_token=_lowerCamelCase	, add_prefix_space=_lowerCamelCase	, add_bos_token=_lowerCamelCase	, **_lowerCamelCase	, )
										lowerCamelCase__					=add_bos_token

										with open(_lowerCamelCase	, encoding="utf-8"      ) as vocab_handle:
																lowerCamelCase__					=json.load(_lowerCamelCase      )
										lowerCamelCase__					={v: k for k, v in self.encoder.items()}
										lowerCamelCase__					=errors  # how to handle errors in decoding
										lowerCamelCase__					=bytes_to_unicode()
										lowerCamelCase__					={v: k for k, v in self.byte_encoder.items()}
										with open(_lowerCamelCase	, encoding="utf-8"      ) as merges_handle:
																lowerCamelCase__					=merges_handle.read().split("\n"      )[1:-1]
										lowerCamelCase__					=[tuple(merge.split()      ) for merge in bpe_merges]
										lowerCamelCase__					=dict(zip(_lowerCamelCase	, range(len(_lowerCamelCase      )      )      )      )
										lowerCamelCase__					={}
										lowerCamelCase__					=add_prefix_space

										# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
										lowerCamelCase__					=re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"      )

				@property
				def 		_a	(      self      ):
										return len(self.encoder      )

				def 		_a	(      self      ):
										return dict(self.encoder	, **self.added_tokens_encoder      )

				def 		_a	(      self	, _lowerCamelCase      ):
										if token in self.cache:
																return self.cache[token]
										lowerCamelCase__					=tuple(_lowerCamelCase      )
										lowerCamelCase__					=get_pairs(_lowerCamelCase      )

										if not pairs:
																return token

										while True:
																lowerCamelCase__					=min(_lowerCamelCase	, key=lambda _lowerCamelCase      : self.bpe_ranks.get(_lowerCamelCase	, float("inf"      )      )      )
																if bigram not in self.bpe_ranks:
																						break
																lowerCamelCase__					,      lowerCamelCase__					=bigram
																lowerCamelCase__					=[]
																lowerCamelCase__					=0
																while i < len(_lowerCamelCase      ):
																						try:
																												lowerCamelCase__					=word.index(_lowerCamelCase	, _lowerCamelCase      )
																						except ValueError:
																												new_word.extend(word[i:]      )
																												break
																						else:
																												new_word.extend(word[i:j]      )
																												lowerCamelCase__					=j

																						if word[i] == first and i < len(_lowerCamelCase      ) - 1 and word[i + 1] == second:
																												new_word.append(first + second      )
																												i += 2
																						else:
																												new_word.append(word[i]      )
																												i += 1
																lowerCamelCase__					=tuple(_lowerCamelCase      )
																lowerCamelCase__					=new_word
																if len(_lowerCamelCase      ) == 1:
																						break
																else:
																						lowerCamelCase__					=get_pairs(_lowerCamelCase      )
										lowerCamelCase__					=" ".join(_lowerCamelCase      )
										lowerCamelCase__					=word
										return word

				def 		_a	(      self	, _lowerCamelCase	, _lowerCamelCase=None      ):
										if self.add_bos_token:
																lowerCamelCase__					=[self.bos_token_id]
										else:
																lowerCamelCase__					=[]

										lowerCamelCase__					=bos_token_ids + token_ids_a

										if token_ids_a is None:
																return output

										return output + bos_token_ids + token_ids_a

				def 		_a	(      self	, _lowerCamelCase      ):
										lowerCamelCase__					=[]
										for token in re.findall(self.pat	, _lowerCamelCase      ):
																lowerCamelCase__					="".join(
																    self.byte_encoder[b] for b in token.encode("utf-8"      )      )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCamelCase      ).split(" "      )      )
										return bpe_tokens

				def 		_a	(      self	, _lowerCamelCase      ):
										return self.encoder.get(_lowerCamelCase	, self.encoder.get(self.unk_token      )      )

				def 		_a	(      self	, _lowerCamelCase      ):
										return self.decoder.get(_lowerCamelCase      )

				def 		_a	(      self	, _lowerCamelCase      ):
										lowerCamelCase__					="".join(_lowerCamelCase      )
										lowerCamelCase__					=bytearray([self.byte_decoder[c] for c in text]      ).decode("utf-8"	, errors=self.errors      )
										return text

				def 		_a	(      self	, _lowerCamelCase	, _lowerCamelCase = None      ):
										if not os.path.isdir(_lowerCamelCase      ):
																logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''      )
																return
										lowerCamelCase__					=os.path.join(
										    _lowerCamelCase	, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )
										lowerCamelCase__					=os.path.join(
										    _lowerCamelCase	, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]      )

										with open(_lowerCamelCase	, "w"	, encoding="utf-8"      ) as f:
																f.write(json.dumps(self.encoder	, indent=2	, sort_keys=_lowerCamelCase	, ensure_ascii=_lowerCamelCase      ) + "\n"      )

										lowerCamelCase__					=0
										with open(_lowerCamelCase	, "w"	, encoding="utf-8"      ) as writer:
																writer.write("#version: 0.2\n"      )
																for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	, key=lambda _lowerCamelCase      : kv[1]      ):
																						if index != token_index:
																												logger.warning(
																												    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																												    " Please check that the tokenizer is not corrupted!"      )
																												lowerCamelCase__					=token_index
																						writer.write(" ".join(_lowerCamelCase      ) + "\n"      )
																						index += 1

										return vocab_file, merge_file

				def 		_a	(      self	, _lowerCamelCase	, _lowerCamelCase=False	, **_lowerCamelCase      ):
										lowerCamelCase__					=kwargs.pop("add_prefix_space"	, self.add_prefix_space      )
										if is_split_into_words or add_prefix_space:
																lowerCamelCase__					=" " + text
										return (text, kwargs)

				def 		_a	(      self	, _lowerCamelCase	, _lowerCamelCase = False	, _lowerCamelCase = None	, _lowerCamelCase = None	, **_lowerCamelCase	, ):
										lowerCamelCase__					=super()._decode(
										    token_ids=_lowerCamelCase	, skip_special_tokens=_lowerCamelCase	, clean_up_tokenization_spaces=_lowerCamelCase	, **_lowerCamelCase	, )

										if truncate_before_pattern is not None and len(_lowerCamelCase      ) > 0:
																lowerCamelCase__					=self.truncate(_lowerCamelCase	, _lowerCamelCase      )

										return decoded_text






				def 		_a	(      self	, _lowerCamelCase	, _lowerCamelCase      ):
										def find_re(_lowerCamelCase	, _lowerCamelCase	, _lowerCamelCase      ):
																lowerCamelCase__					=pattern.search(_lowerCamelCase	, _lowerCamelCase      )
																return m.start() if m else -1

										lowerCamelCase__					=[re.compile(_lowerCamelCase	, re.MULTILINE      ) for pattern in truncate_before_pattern]

										lowerCamelCase__					=list(re.finditer("^print"	, _lowerCamelCase	, re.MULTILINE      )      )

										if len(_lowerCamelCase      ) > 1:
																lowerCamelCase__					=completion[: prints[1].start()]

										lowerCamelCase__					=list(re.finditer("^def"	, _lowerCamelCase	, re.MULTILINE      )      )

										if len(_lowerCamelCase      ) > 1:
																lowerCamelCase__					=completion[: defs[1].start()]

										lowerCamelCase__					=0

										lowerCamelCase__					=[
										    pos for pos in [find_re(_lowerCamelCase	, _lowerCamelCase	, _lowerCamelCase      ) for terminal in terminals] if pos != -1
										]

										if len(_lowerCamelCase      ) > 0:
																return completion[: min(_lowerCamelCase      )]
										else:
																return completion



"""simple docstring"""


import abc
import copy
import dataclasses
from dataclasses import dataclass
from typing import ClassVar, Dict, Type, TypeVar

from ..features import Features


a							=TypeVar('T', bound='TaskTemplate')






@dataclass(frozen=__lowerCAmelCase	)
class    __UpperCAmelCase						(	abc.ABC	):
				# `task` is not a ClassVar since we want it to be part of the `asdict` output for JSON serialization
				A__      :   str
				A__      :   ClassVar[Features]
				A__      :   ClassVar[Features]

				def 		_a	(      self	, _lowerCamelCase      ):
										return copy.deepcopy(self      )

				@property
				def 		_a	(      self      ):
										return Features(**self.input_schema	, **self.label_schema      )

				@property
				@abc.abstractmethod
				def 		_a	(      self      ):
										raise NotImplementedError






				@classmethod
				def 		_a	(      cls	, _lowerCamelCase      ):
										lowerCamelCase__					={f.name for f in dataclasses.fields(cls      )}
										return cls(**{k: v for k, v in template_dict.items() if k in field_names}      )


