





import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
       if is_torch_available():
              import torch
       if is_tf_available():
              import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


a__: Tuple							   =						logging.get_logger(__name__)

a__: List[Any]							   =						{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

a__: Optional[Any]							   =						{
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'Salesforce/codegen-350M-mono': (
            'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'
        ),
    },
}

a__: Union[str, Any]							   =						{
    'Salesforce/codegen-350M-mono': 2_048,
}




class        SCREAMING_SNAKE_CASE__					(		UpperCamelCase__ ):
 __SCREAMING_SNAKE_CASE							=      VOCAB_FILES_NAMES
 __SCREAMING_SNAKE_CASE							=      PRETRAINED_VOCAB_FILES_MAP
 __SCREAMING_SNAKE_CASE							=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __SCREAMING_SNAKE_CASE							=      ['''input_ids''', '''attention_mask''']
 __SCREAMING_SNAKE_CASE							=      CodeGenTokenizer



 def __init__(				self,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase="<|endoftext|>",__lowerCamelCase="<|endoftext|>",__lowerCamelCase="<|endoftext|>",__lowerCamelCase=False,**__lowerCamelCase,):
        super().__init__(
            __lowerCamelCase,__lowerCamelCase,tokenizer_file=__lowerCamelCase,unk_token=__lowerCamelCase,bos_token=__lowerCamelCase,eos_token=__lowerCamelCase,add_prefix_space=__lowerCamelCase,**__lowerCamelCase,)

        if kwargs.pop('''add_bos_token''',__lowerCamelCase					):
               A__		   =		kwargs.pop('''name_or_path''',''''''					)
               raise ValueError(
                   '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
                   '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
                   f"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
                   f"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
                   '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
                   ''' so that the fast tokenizer works correctly.'''					)

        A__		   =		json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()					)
        if pre_tok_state.get('''add_prefix_space''',__lowerCamelCase					) != add_prefix_space:
               A__		   =		getattr(__lowerCamelCase,pre_tok_state.pop('''type'''					)					)
               A__		   =		add_prefix_space
               A__		   =		pre_tok_class(**__lowerCamelCase					)

        A__		   =		add_prefix_space



 def 							UpperCamelCase     (				self,*__lowerCamelCase,**__lowerCamelCase					):
        A__		   =		kwargs.get('''is_split_into_words''',__lowerCamelCase					)
        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._batch_encode_plus(*__lowerCamelCase,**__lowerCamelCase					)



 def 							UpperCamelCase     (				self,*__lowerCamelCase,**__lowerCamelCase					):
        A__		   =		kwargs.get('''is_split_into_words''',__lowerCamelCase					)

        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._encode_plus(*__lowerCamelCase,**__lowerCamelCase					)



 def 							UpperCamelCase     (				self,__lowerCamelCase,__lowerCamelCase = None					):
        A__		   =		self._tokenizer.model.save(__lowerCamelCase,name=__lowerCamelCase					)
        return tuple(__lowerCamelCase					)



 def 							UpperCamelCase     (				self,__lowerCamelCase,__lowerCamelCase = False,__lowerCamelCase = None,__lowerCamelCase = None,**__lowerCamelCase,):
        A__		   =		super().decode(
            token_ids=__lowerCamelCase,skip_special_tokens=__lowerCamelCase,clean_up_tokenization_spaces=__lowerCamelCase,**__lowerCamelCase,)

        if truncate_before_pattern is not None and len(__lowerCamelCase					) > 0:
               A__		   =		self.truncate(__lowerCamelCase,__lowerCamelCase					)

        return decoded_text




 def 							UpperCamelCase     (				self,__lowerCamelCase,__lowerCamelCase					):
        def find_re(__lowerCamelCase,__lowerCamelCase,__lowerCamelCase					):
               A__		   =		pattern.search(__lowerCamelCase,__lowerCamelCase					)
               return m.start() if m else -1

        A__		   =		[re.compile(__lowerCamelCase,re.MULTILINE					) for pattern in truncate_before_pattern]

        A__		   =		list(re.finditer('''^print''',__lowerCamelCase,re.MULTILINE					)					)

        if len(__lowerCamelCase					) > 1:
               A__		   =		completion[: prints[1].start()]

        A__		   =		list(re.finditer('''^def''',__lowerCamelCase,re.MULTILINE					)					)

        if len(__lowerCamelCase					) > 1:
               A__		   =		completion[: defs[1].start()]

        A__		   =		0

        A__		   =		[
            pos for pos in [find_re(__lowerCamelCase,__lowerCamelCase,__lowerCamelCase					) for terminal in terminals] if pos != -1
        ]

        if len(__lowerCamelCase					) > 0:
               return completion[: min(__lowerCamelCase					)]
        else:
               return completion












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


a__: List[Any]							   =						{
    'configuration_x_clip': [
        'XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'XCLIPConfig',
        'XCLIPTextConfig',
        'XCLIPVisionConfig',
    ],
    'processing_x_clip': ['XCLIPProcessor'],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a__: Any							   =						[
           'XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST',
           'XCLIPModel',
           'XCLIPPreTrainedModel',
           'XCLIPTextModel',
           'XCLIPVisionModel',
       ]

if TYPE_CHECKING:
       from .configuration_x_clip import (
           XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
           XCLIPConfig,
           XCLIPTextConfig,
           XCLIPVisionConfig,
       )
       from .processing_x_clip import XCLIPProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_x_clip import (
                  XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
                  XCLIPModel,
                  XCLIPPreTrainedModel,
                  XCLIPTextModel,
                  XCLIPVisionModel,
              )

else:
       import sys

       a__: Optional[Any]							   =						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





