





"""simple docstring"""





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
    if is_torch_available():
        import torch
    if is_tf_available():
        import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


_lowerCamelCase       =	logging.get_logger(__name__)

_lowerCamelCase       =	{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_lowerCamelCase       =	{
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
}

_lowerCamelCase       =	{
    '''Salesforce/codegen-350M-mono''': 20_48,
}




@lru_cache()
def 		lowerCAmelCase_	(  ):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			List[Any]												=       (
     list(range(ord('''!'''		) ,      ord('''~'''		) + 1		)		) + list(range(ord('''¡'''		) ,      ord('''¬'''		) + 1		)		) + list(range(ord('''®'''		) ,      ord('''ÿ'''		) + 1		)		)
 )
 __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       bs[:]
 __SCREAMING_SNAKE_CASE :			Optional[int]												=       0
 for b in range(2**8		):
  if b not in bs:
   bs.append(__UpperCamelCase		)
   cs.append(2**8 + n		)
   n += 1
 __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       [chr(__UpperCamelCase		) for n in cs]
 return dict(zip(__UpperCamelCase ,      __UpperCamelCase		)		)




def 		lowerCAmelCase_	(  lowercase_   :							Any		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			List[str]												=       set()
 __SCREAMING_SNAKE_CASE :			str												=       word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)		)
  __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       char
 return pairs






class 			snake_case		(   __UpperCAmelCase   ):
      lowerCamelCase__						= VOCAB_FILES_NAMES
      lowerCamelCase__						= PRETRAINED_VOCAB_FILES_MAP
      lowerCamelCase__						= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCamelCase__						= ['''input_ids''', '''attention_mask''']

      def __init__(			self   :Tuple						,							_lowerCamelCase   :int						,							_lowerCamelCase   :List[str]						,							_lowerCamelCase   :Any="replace"						,							_lowerCamelCase   :int="<|endoftext|>"						,							_lowerCamelCase   :str="<|endoftext|>"						,							_lowerCamelCase   :Optional[Any]="<|endoftext|>"						,							_lowerCamelCase   :int=None						,							_lowerCamelCase   :Any=False						,							_lowerCamelCase   :Optional[Any]=False						,							**_lowerCamelCase   :List[str]						,							):
       __SCREAMING_SNAKE_CASE :			Any												=       AddedToken(__UpperCAmelCase						,							lstrip=__UpperCAmelCase						,							rstrip=__UpperCAmelCase						) if isinstance(__UpperCAmelCase						,							__UpperCAmelCase						) else bos_token
       __SCREAMING_SNAKE_CASE :			Tuple												=       AddedToken(__UpperCAmelCase						,							lstrip=__UpperCAmelCase						,							rstrip=__UpperCAmelCase						) if isinstance(__UpperCAmelCase						,							__UpperCAmelCase						) else eos_token
       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       AddedToken(__UpperCAmelCase						,							lstrip=__UpperCAmelCase						,							rstrip=__UpperCAmelCase						) if isinstance(__UpperCAmelCase						,							__UpperCAmelCase						) else unk_token
       __SCREAMING_SNAKE_CASE :			List[str]												=       AddedToken(__UpperCAmelCase						,							lstrip=__UpperCAmelCase						,							rstrip=__UpperCAmelCase						) if isinstance(__UpperCAmelCase						,							__UpperCAmelCase						) else pad_token
       super().__init__(
           errors=__UpperCAmelCase						,							unk_token=__UpperCAmelCase						,							bos_token=__UpperCAmelCase						,							eos_token=__UpperCAmelCase						,							pad_token=__UpperCAmelCase						,							add_prefix_space=__UpperCAmelCase						,							add_bos_token=__UpperCAmelCase						,							**__UpperCAmelCase						,							)
       __SCREAMING_SNAKE_CASE :			int												=       add_bos_token

       with open(__UpperCAmelCase						,							encoding='''utf-8'''						) as vocab_handle:
        __SCREAMING_SNAKE_CASE :			List[Any]												=       json.load(__UpperCAmelCase						)
       __SCREAMING_SNAKE_CASE :			Tuple												=       {v: k for k, v in self.encoder.items()}
       __SCREAMING_SNAKE_CASE :			Optional[Any]												=       errors  # how to handle errors in decoding
       __SCREAMING_SNAKE_CASE :			Dict												=       bytes_to_unicode()
       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       {v: k for k, v in self.byte_encoder.items()}
       with open(__UpperCAmelCase						,							encoding='''utf-8'''						) as merges_handle:
        __SCREAMING_SNAKE_CASE :			Dict												=       merges_handle.read().split('''\n'''						)[1:-1]
       __SCREAMING_SNAKE_CASE :			List[str]												=       [tuple(merge.split()						) for merge in bpe_merges]
       __SCREAMING_SNAKE_CASE :			Optional[Any]												=       dict(zip(__UpperCAmelCase						,							range(len(__UpperCAmelCase						)						)						)						)
       __SCREAMING_SNAKE_CASE :			Any												=       {}
       __SCREAMING_SNAKE_CASE :			List[str]												=       add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       __SCREAMING_SNAKE_CASE :			str												=       re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''						)

      @property
      def 						SCREAMING_SNAKE_CASE_			(			self   :int						):
       return len(self.encoder						)

      def 						SCREAMING_SNAKE_CASE_			(			self   :Union[str, Any]						):
       return dict(self.encoder						,							**self.added_tokens_encoder						)

      def 						SCREAMING_SNAKE_CASE_			(			self   :Tuple						,							_lowerCamelCase   :Union[str, Any]						):
       if token in self.cache:
        return self.cache[token]
       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       tuple(__UpperCAmelCase						)
       __SCREAMING_SNAKE_CASE :			str												=       get_pairs(__UpperCAmelCase						)

       if not pairs:
        return token

       while True:
        __SCREAMING_SNAKE_CASE :			Optional[Any]												=       min(__UpperCAmelCase						,							key=lambda _lowerCamelCase						: self.bpe_ranks.get(__UpperCAmelCase						,							float('''inf'''						)						)						)
        if bigram not in self.bpe_ranks:
         break
        __SCREAMING_SNAKE_CASE							,      __SCREAMING_SNAKE_CASE :			Optional[Any]												=       bigram
        __SCREAMING_SNAKE_CASE :			Optional[int]												=       []
        __SCREAMING_SNAKE_CASE :			Dict												=       0
        while i < len(__UpperCAmelCase						):
         try:
          __SCREAMING_SNAKE_CASE :			int												=       word.index(__UpperCAmelCase						,							__UpperCAmelCase						)
         except ValueError:
          new_word.extend(word[i:]						)
          break
         else:
          new_word.extend(word[i:j]						)
          __SCREAMING_SNAKE_CASE :			Optional[int]												=       j

         if word[i] == first and i < len(__UpperCAmelCase						) - 1 and word[i + 1] == second:
          new_word.append(first + second						)
          i += 2
         else:
          new_word.append(word[i]						)
          i += 1
        __SCREAMING_SNAKE_CASE :			List[str]												=       tuple(__UpperCAmelCase						)
        __SCREAMING_SNAKE_CASE :			int												=       new_word
        if len(__UpperCAmelCase						) == 1:
         break
        else:
         __SCREAMING_SNAKE_CASE :			int												=       get_pairs(__UpperCAmelCase						)
       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       ''' '''.join(__UpperCAmelCase						)
       __SCREAMING_SNAKE_CASE :			str												=       word
       return word

      def 						SCREAMING_SNAKE_CASE_			(			self   :str						,							_lowerCamelCase   :List[str]						,							_lowerCamelCase   :Tuple=None						):
       if self.add_bos_token:
        __SCREAMING_SNAKE_CASE :			List[Any]												=       [self.bos_token_id]
       else:
        __SCREAMING_SNAKE_CASE :			int												=       []

       __SCREAMING_SNAKE_CASE :			Tuple												=       bos_token_ids + token_ids_a

       if token_ids_a is None:
        return output

       return output + bos_token_ids + token_ids_a

      def 						SCREAMING_SNAKE_CASE_			(			self   :List[Any]						,							_lowerCamelCase   :List[str]						):
       __SCREAMING_SNAKE_CASE :			Any												=       []
       for token in re.findall(self.pat						,							__UpperCAmelCase						):
        __SCREAMING_SNAKE_CASE :			List[str]												=       ''''''.join(
            self.byte_encoder[b] for b in token.encode('''utf-8'''						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
        bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__UpperCAmelCase						).split(''' '''						)						)
       return bpe_tokens

      def 						SCREAMING_SNAKE_CASE_			(			self   :Tuple						,							_lowerCamelCase   :Any						):
       return self.encoder.get(__UpperCAmelCase						,							self.encoder.get(self.unk_token						)						)

      def 						SCREAMING_SNAKE_CASE_			(			self   :int						,							_lowerCamelCase   :Dict						):
       return self.decoder.get(__UpperCAmelCase						)

      def 						SCREAMING_SNAKE_CASE_			(			self   :Any						,							_lowerCamelCase   :List[Any]						):
       __SCREAMING_SNAKE_CASE :			List[str]												=       ''''''.join(__UpperCAmelCase						)
       __SCREAMING_SNAKE_CASE :			Optional[int]												=       bytearray([self.byte_decoder[c] for c in text]						).decode('''utf-8'''						,							errors=self.errors						)
       return text

      def 						SCREAMING_SNAKE_CASE_			(			self   :str						,							_lowerCamelCase   :List[str]						,							_lowerCamelCase   :List[str] = None						):
       if not os.path.isdir(__UpperCAmelCase						):
        logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
        return
       __SCREAMING_SNAKE_CASE :			List[Any]												=       os.path.join(
           __UpperCAmelCase						,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
       __SCREAMING_SNAKE_CASE :			List[Any]												=       os.path.join(
           __UpperCAmelCase						,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']						)

       with open(__UpperCAmelCase						,							'''w'''						,							encoding='''utf-8'''						) as f:
        f.write(json.dumps(self.encoder						,							indent=2						,							sort_keys=__UpperCAmelCase						,							ensure_ascii=__UpperCAmelCase						) + '''\n'''						)

       __SCREAMING_SNAKE_CASE :			Dict												=       0
       with open(__UpperCAmelCase						,							'''w'''						,							encoding='''utf-8'''						) as writer:
        writer.write('''#version: 0.2\n'''						)
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,							key=lambda _lowerCamelCase						: kv[1]						):
         if index != token_index:
          logger.warning(
              f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
              ''' Please check that the tokenizer is not corrupted!'''						)
          __SCREAMING_SNAKE_CASE :			Dict												=       token_index
         writer.write(''' '''.join(__UpperCAmelCase						) + '''\n'''						)
         index += 1

       return vocab_file, merge_file

      def 						SCREAMING_SNAKE_CASE_			(			self   :List[str]						,							_lowerCamelCase   :Tuple						,							_lowerCamelCase   :Tuple=False						,							**_lowerCamelCase   :List[Any]						):
       __SCREAMING_SNAKE_CASE :			Any												=       kwargs.pop('''add_prefix_space'''						,							self.add_prefix_space						)
       if is_split_into_words or add_prefix_space:
        __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       ''' ''' + text
       return (text, kwargs)

      def 						SCREAMING_SNAKE_CASE_			(			self   :Dict						,							_lowerCamelCase   :Optional[int]						,							_lowerCamelCase   :List[str] = False						,							_lowerCamelCase   :Union[str, Any] = None						,							_lowerCamelCase   :int = None						,							**_lowerCamelCase   :List[str]						,							):
       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       super()._decode(
           token_ids=__UpperCAmelCase						,							skip_special_tokens=__UpperCAmelCase						,							clean_up_tokenization_spaces=__UpperCAmelCase						,							**__UpperCAmelCase						,							)

       if truncate_before_pattern is not None and len(__UpperCAmelCase						) > 0:
        __SCREAMING_SNAKE_CASE :			List[Any]												=       self.truncate(__UpperCAmelCase						,							__UpperCAmelCase						)

       return decoded_text



      def 						SCREAMING_SNAKE_CASE_			(			self   :Union[str, Any]						,							_lowerCamelCase   :List[Any]						,							_lowerCamelCase   :str						):
       def find_re(_lowerCamelCase   :List[Any]						,							_lowerCamelCase   :List[Any]						,							_lowerCamelCase   :Optional[int]						):
        __SCREAMING_SNAKE_CASE :			Optional[Any]												=       pattern.search(__UpperCAmelCase						,							__UpperCAmelCase						)
        return m.start() if m else -1

       __SCREAMING_SNAKE_CASE :			Optional[Any]												=       [re.compile(__UpperCAmelCase						,							re.MULTILINE						) for pattern in truncate_before_pattern]

       __SCREAMING_SNAKE_CASE :			Dict												=       list(re.finditer('''^print'''						,							__UpperCAmelCase						,							re.MULTILINE						)						)

       if len(__UpperCAmelCase						) > 1:
        __SCREAMING_SNAKE_CASE :			Optional[Any]												=       completion[: prints[1].start()]

       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       list(re.finditer('''^def'''						,							__UpperCAmelCase						,							re.MULTILINE						)						)

       if len(__UpperCAmelCase						) > 1:
        __SCREAMING_SNAKE_CASE :			str												=       completion[: defs[1].start()]

       __SCREAMING_SNAKE_CASE :			str												=       0

       __SCREAMING_SNAKE_CASE :			List[str]												=       [
           pos for pos in [find_re(__UpperCAmelCase						,							__UpperCAmelCase						,							__UpperCAmelCase						) for terminal in terminals] if pos != -1
       ]

       if len(__UpperCAmelCase						) > 0:
        return completion[: min(__UpperCAmelCase						)]
       else:
        return completion



"""simple docstring"""


import enum
import inspect
import warnings
from functools import wraps
from typing import Callable, Optional

from .logging import get_logger


__A       						=       set()
__A       						=       get_logger(__name__)



def     _lowerCamelCase(__UpperCamelCase = None       )				-> Optional[Any]:

	def decorator(__UpperCamelCase       ):
		global _emitted_deprecation_warnings

		if inspect.isclass(__UpperCamelCase       ):
			_lowerCAmelCase								=deprecated_class_or_function.__init__
			_lowerCAmelCase								=deprecated_class_or_function.__name__
		else:
			_lowerCAmelCase								=deprecated_class_or_function
			_lowerCAmelCase								=deprecated_function.__name__
			# Support deprecating __init__ class method: class name instead
			_lowerCAmelCase								=name if name != """__init__""" else deprecated_function.__qualname__.split("""."""       )[-2]

		_lowerCAmelCase								=(
		    F'''{name} is deprecated and will be removed in the next major version of datasets.''' + F''' {help_message}'''
		    if help_message
		    else """"""
		)

		@wraps(__UpperCamelCase       )
		def wrapper(*__UpperCamelCase    , **__UpperCamelCase       ):
			_lowerCAmelCase								=hash(__UpperCamelCase       )
			if func_hash not in _emitted_deprecation_warnings:
				warnings.warn(__UpperCamelCase    , category=__UpperCamelCase    , stacklevel=2       )
				_emitted_deprecation_warnings.add(__UpperCamelCase       )
			return deprecated_function(*__UpperCamelCase    , **__UpperCamelCase       )

		_lowerCAmelCase								="""deprecated"""

		if inspect.isclass(__UpperCamelCase       ):
			_lowerCAmelCase								=wrapper
			return deprecated_class_or_function
		else:
			return wrapper

	return decorator







class 		lowerCamelCase__					(					enum.EnumMeta				):





			'''simple docstring'''



			def __getattribute__(							cls     ,     __UpperCAmelCase  ) ->     Optional[int]:
				_lowerCAmelCase								=super().__getattribute__(__UpperCAmelCase  )
				if isinstance(__UpperCAmelCase     ,     enum.Enum  ) and obj._on_access:
					obj._on_access()
				return obj



			def __getitem__(							cls     ,     __UpperCAmelCase  ) ->     List[str]:
				_lowerCAmelCase								=super().__getitem__(__UpperCAmelCase  )
				if member._on_access:
					member._on_access()
				return member



			def __call__(							cls     ,     __UpperCAmelCase     ,     __UpperCAmelCase=None     ,     *, __UpperCAmelCase=None     ,     __UpperCAmelCase=None     ,     __UpperCAmelCase=None     ,     __UpperCAmelCase=1  ) ->     Optional[Any]:
				_lowerCAmelCase								=super().__call__(__UpperCAmelCase     ,     __UpperCAmelCase     ,     module=__UpperCAmelCase     ,     qualname=__UpperCAmelCase     ,     type=__UpperCAmelCase     ,     start=__UpperCAmelCase  )
				if isinstance(__UpperCAmelCase     ,     enum.Enum  ) and obj._on_access:
					obj._on_access()
				return obj







class 		lowerCamelCase__					(					enum.Enum ,	metaclass=__magic_name__				):





			'''simple docstring'''



			def __new__(							cls     ,     __UpperCAmelCase  ) ->     Optional[int]:
				_lowerCAmelCase								=object.__new__(cls  )
				_lowerCAmelCase								=value
				_lowerCAmelCase								=member.deprecate
				return member



			@property
			def 					_lowerCAmelCase		(							self  ) ->     List[Any]:
				return ""



			def 					_lowerCAmelCase		(							self  ) ->     Tuple:
				_lowerCAmelCase								=f''' {self.help_message}''' if self.help_message else """"""
				warnings.warn(
				    f'''\'{self.__objclass__.__name__}\' is deprecated and will be removed in the next major version of datasets.'''
				    + help_message     ,     __UpperCAmelCase     ,     stacklevel=3     ,     )



