





import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
      if is_torch_available():
            import torch
      if is_tf_available():
            import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


_lowerCAmelCase								=  logging.get_logger(__name__)

_lowerCAmelCase								=  {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_lowerCAmelCase								=  {
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''Salesforce/codegen-350M-mono''': (
            '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'''
        ),
    },
}

_lowerCAmelCase								=  {
    '''Salesforce/codegen-350M-mono''': 2_048,
}

class  _UpperCAmelCase				(		_UpperCAmelCase  ):
      a    							=				VOCAB_FILES_NAMES
      a    							=				PRETRAINED_VOCAB_FILES_MAP
      a    							=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      a    							=				['input_ids', 'attention_mask']
      a    							=				CodeGenTokenizer
      def __init__(		self   ,							a__=None   ,							a__=None   ,							a__=None   ,							a__="<|endoftext|>"   ,							a__="<|endoftext|>"   ,							a__="<|endoftext|>"   ,							a__=False   ,							**a__   ,							):
         super().__init__(
             __UpperCamelCase   ,							__UpperCamelCase   ,							tokenizer_file=__UpperCamelCase   ,							unk_token=__UpperCamelCase   ,							bos_token=__UpperCamelCase   ,							eos_token=__UpperCamelCase   ,							add_prefix_space=__UpperCamelCase   ,							**__UpperCamelCase   ,							)

         if kwargs.pop("""add_bos_token"""   ,							__UpperCamelCase							):
            A_  :					Union[str, Any]         =						kwargs.pop("""name_or_path"""   ,							""""""							)
            raise ValueError(
                """Currenty GPT2's fast tokenizer does NOT support adding a BOS token."""
                """Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"""
                F"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
                F"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
                """This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."""
                """ so that the fast tokenizer works correctly."""							)

         A_  :					int         =						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
         if pre_tok_state.get("""add_prefix_space"""   ,							__UpperCamelCase							) != add_prefix_space:
            A_  :					Tuple         =						getattr(__UpperCamelCase   ,							pre_tok_state.pop("""type"""							)							)
            A_  :					str         =						add_prefix_space
            A_  :					List[str]         =						pre_tok_class(**__UpperCamelCase							)

         A_  :					Optional[Any]         =						add_prefix_space
      def 		_lowerCamelCase   (		self   ,							*a__   ,							**a__							):
         A_  :					Optional[int]         =						kwargs.get("""is_split_into_words"""   ,							__UpperCamelCase							)
         assert self.add_prefix_space or not is_split_into_words, (
             F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
             "to use it with pretokenized inputs."
         )

         return super()._batch_encode_plus(*__UpperCamelCase   ,							**__UpperCamelCase							)
      def 		_lowerCamelCase   (		self   ,							*a__   ,							**a__							):
         A_  :					Optional[Any]         =						kwargs.get("""is_split_into_words"""   ,							__UpperCamelCase							)

         assert self.add_prefix_space or not is_split_into_words, (
             F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
             "to use it with pretokenized inputs."
         )

         return super()._encode_plus(*__UpperCamelCase   ,							**__UpperCamelCase							)
      def 		_lowerCamelCase   (		self   ,							a__   ,							a__ = None							):
         A_  :					Tuple         =						self._tokenizer.model.save(__UpperCamelCase   ,							name=__UpperCamelCase							)
         return tuple(__UpperCamelCase							)
      def 		_lowerCamelCase   (		self   ,							a__   ,							a__ = False   ,							a__ = None   ,							a__ = None   ,							**a__   ,							):
         A_  :					Any         =						super().decode(
             token_ids=__UpperCamelCase   ,							skip_special_tokens=__UpperCamelCase   ,							clean_up_tokenization_spaces=__UpperCamelCase   ,							**__UpperCamelCase   ,							)

         if truncate_before_pattern is not None and len(__UpperCamelCase							) > 0:
            A_  :					List[Any]         =						self.truncate(__UpperCamelCase   ,							__UpperCamelCase							)

         return decoded_text






      def 		_lowerCamelCase   (		self   ,							a__   ,							a__							):






         def find_re(a__   ,							a__   ,							a__							):
            A_  :					Optional[Any]         =						pattern.search(__UpperCamelCase   ,							__UpperCamelCase							)
            return m.start() if m else -1

         A_  :					Optional[int]         =						[re.compile(__UpperCamelCase   ,							re.MULTILINE							) for pattern in truncate_before_pattern]

         A_  :					List[Any]         =						list(re.finditer("""^print"""   ,							__UpperCamelCase   ,							re.MULTILINE							)							)

         if len(__UpperCamelCase							) > 1:
            A_  :					str         =						completion[: prints[1].start()]

         A_  :					List[str]         =						list(re.finditer("""^def"""   ,							__UpperCamelCase   ,							re.MULTILINE							)							)

         if len(__UpperCamelCase							) > 1:
            A_  :					int         =						completion[: defs[1].start()]

         A_  :					Optional[int]         =						0

         A_  :					List[Any]         =						[
             pos for pos in [find_re(__UpperCamelCase   ,							__UpperCamelCase   ,							__UpperCamelCase							) for terminal in terminals] if pos != -1
         ]

         if len(__UpperCamelCase							) > 0:
            return completion[: min(__UpperCamelCase							)]
         else:
            return completion








"""simple docstring"""



def 			_UpperCamelCase						(    _A   )     ->	int:





    """simple docstring"""

    if not isinstance(_A       ,   _A   ) or number < 0:
        raise ValueError("""Input must be a non-negative integer"""   )

    _UpperCAmelCase        = 0
    while number:
        # This way we arrive at next set bit (next 1) instead of looping
        # through each bit and checking for 1s hence the
        # loop won't run 32 times it will only run the number of `1` times
        number &= number - 1
        count += 1
    return count


if __name__ == "__main__":
 import doctest

 doctest.testmod()