
"""simple docstring"""



import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
	if is_torch_available():
		import torch
	if is_tf_available():
		import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


_snake_case        =    logging.get_logger(__name__)

_snake_case        =    {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

_snake_case        =    {
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'Salesforce/codegen-350M-mono': (
            'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'
        ),
    },
}

_snake_case        =    {
    'Salesforce/codegen-350M-mono': 2_0_4_8,
}
class 							_SCREAMING_SNAKE_CASE	(	UpperCamelCase__				):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE_:				Tuple				=       VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE_:				str				=       PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE_:				Union[str, Any]				=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE_:				str				=       ["input_ids", "attention_mask"]
				SCREAMING_SNAKE_CASE_:				Optional[Any]				=       CodeGenTokenizer


				def __init__(       self						:      List[str]					,  UpperCAmelCase_						:      int=None					,  UpperCAmelCase_						:      List[str]=None					,  UpperCAmelCase_						:      int=None					,  UpperCAmelCase_						:      Optional[int]="<|endoftext|>"					,  UpperCAmelCase_						:      Any="<|endoftext|>"					,  UpperCAmelCase_						:      Optional[int]="<|endoftext|>"					,  UpperCAmelCase_						:      Tuple=False					,  **UpperCAmelCase_						:      int					,  )		->	Any:







											"""simple docstring"""




											super().__init__(
											    __lowerCamelCase					,  __lowerCamelCase					,  tokenizer_file=__lowerCamelCase					,  unk_token=__lowerCamelCase					,  bos_token=__lowerCamelCase					,  eos_token=__lowerCamelCase					,  add_prefix_space=__lowerCamelCase					,  **__lowerCamelCase					,  )

											if kwargs.pop('add_bos_token'					,  __lowerCamelCase	):
																		_lowerCAmelCase							=      kwargs.pop('name_or_path'					,  ''	)
																		raise ValueError(
																		    'Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'
																		    'Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'
																		    F"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
																		    F"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
																		    'This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'
																		    ' so that the fast tokenizer works correctly.'	)

											_lowerCAmelCase							=      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
											if pre_tok_state.get('add_prefix_space'					,  __lowerCamelCase	) != add_prefix_space:
																		_lowerCAmelCase							=      getattr(__lowerCamelCase					,  pre_tok_state.pop('type'	)	)
																		_lowerCAmelCase							=      add_prefix_space
																		_lowerCAmelCase							=      pre_tok_class(**__lowerCamelCase	)

											_lowerCAmelCase							=      add_prefix_space


				def 					__lowerCamelCase						(       self						:      List[str]					,  *UpperCAmelCase_						:      Any					,  **UpperCAmelCase_						:      Optional[int]	)		->	int:







											"""simple docstring"""




											_lowerCAmelCase							=      kwargs.get('is_split_into_words'					,  __lowerCamelCase	)
											assert self.add_prefix_space or not is_split_into_words, (
											    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
											    "to use it with pretokenized inputs."
											)

											return super()._batch_encode_plus(*__lowerCamelCase					,  **__lowerCamelCase	)


				def 					__lowerCamelCase						(       self						:      Dict					,  *UpperCAmelCase_						:      Any					,  **UpperCAmelCase_						:      Dict	)		->	List[str]:







											"""simple docstring"""




											_lowerCAmelCase							=      kwargs.get('is_split_into_words'					,  __lowerCamelCase	)

											assert self.add_prefix_space or not is_split_into_words, (
											    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
											    "to use it with pretokenized inputs."
											)

											return super()._encode_plus(*__lowerCamelCase					,  **__lowerCamelCase	)


				def 					__lowerCamelCase						(       self						:      str					,  UpperCAmelCase_						:      int					,  UpperCAmelCase_						:      Dict = None	)		->	Optional[int]:







											"""simple docstring"""




											_lowerCAmelCase							=      self._tokenizer.model.save(__lowerCamelCase					,  name=__lowerCamelCase	)
											return tuple(__lowerCamelCase	)


				def 					__lowerCamelCase						(       self						:      Union[str, Any]					,  UpperCAmelCase_						:      Any					,  UpperCAmelCase_						:      int = False					,  UpperCAmelCase_						:      List[str] = None					,  UpperCAmelCase_						:      Union[str, Any] = None					,  **UpperCAmelCase_						:      int					,  )		->	Optional[int]:







											"""simple docstring"""




											_lowerCAmelCase							=      super().decode(
											    token_ids=__lowerCamelCase					,  skip_special_tokens=__lowerCamelCase					,  clean_up_tokenization_spaces=__lowerCamelCase					,  **__lowerCamelCase					,  )

											if truncate_before_pattern is not None and len(__lowerCamelCase	) > 0:
																		_lowerCAmelCase							=      self.truncate(__lowerCamelCase					,  __lowerCamelCase	)

											return decoded_text







				def 					__lowerCamelCase						(       self						:      Any					,  UpperCAmelCase_						:      Any					,  UpperCAmelCase_						:      Optional[int]	)		->	Union[str, Any]:







											"""simple docstring"""




											def find_re(UpperCAmelCase_						:      List[Any]					,  UpperCAmelCase_						:      int					,  UpperCAmelCase_						:      str	):
																		_lowerCAmelCase							=      pattern.search(__lowerCamelCase					,  __lowerCamelCase	)
																		return m.start() if m else -1

											_lowerCAmelCase							=      [re.compile(__lowerCamelCase					,  re.MULTILINE	) for pattern in truncate_before_pattern]

											_lowerCAmelCase							=      list(re.finditer('^print'					,  __lowerCamelCase					,  re.MULTILINE	)	)

											if len(__lowerCamelCase	) > 1:
																		_lowerCAmelCase							=      completion[: prints[1].start()]

											_lowerCAmelCase							=      list(re.finditer('^def'					,  __lowerCamelCase					,  re.MULTILINE	)	)

											if len(__lowerCamelCase	) > 1:
																		_lowerCAmelCase							=      completion[: defs[1].start()]

											_lowerCAmelCase							=      0

											_lowerCAmelCase							=      [
											    pos for pos in [find_re(__lowerCamelCase					,  __lowerCamelCase					,  __lowerCamelCase	) for terminal in terminals] if pos != -1
											]

											if len(__lowerCamelCase	) > 0:
																		return completion[: min(__lowerCamelCase	)]
											else:
																		return completion













import json
import os
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from requests.exceptions import HTTPError

from transformers.utils import (
    CONFIG_NAME,
    FLAX_WEIGHTS_NAME,
    TF2_WEIGHTS_NAME,
    TRANSFORMERS_CACHE,
    WEIGHTS_NAME,
    cached_file,
    get_file_from_repo,
    has_file,
)


a__: Union[str, Any]							   =						'hf-internal-testing/tiny-random-bert'
a__: Optional[int]							   =						os.path.join(TRANSFORMERS_CACHE, 'models--hf-internal-testing--tiny-random-bert')
a__: List[str]							   =						'9b8c223d42b2188cb49d29af482996f9d0f3e5a6'




class        SCREAMING_SNAKE_CASE__					(		unittest.TestCase ):



 def 							UpperCamelCase     (				self					):
        A__		   =		cached_file(__lowerCamelCase,__lowerCamelCase					)
        # Should have downloaded the file in here
        self.assertTrue(os.path.isdir(__lowerCamelCase					)					)
        # Cache should contain at least those three subfolders:
        for subfolder in ["blobs", "refs", "snapshots"]:
               self.assertTrue(os.path.isdir(os.path.join(__lowerCamelCase,__lowerCamelCase					)					)					)
        with open(os.path.join(__lowerCamelCase,'''refs''','''main'''					)					) as f:
               A__		   =		f.read()
        self.assertEqual(__lowerCamelCase,os.path.join(__lowerCamelCase,'''snapshots''',__lowerCamelCase,__lowerCamelCase					)					)
        self.assertTrue(os.path.isfile(__lowerCamelCase					)					)

        # File is cached at the same place the second time.
        A__		   =		cached_file(__lowerCamelCase,__lowerCamelCase					)
        self.assertEqual(__lowerCamelCase,__lowerCamelCase					)

        # Using a specific revision to test the full commit hash.
        A__		   =		cached_file(__lowerCamelCase,__lowerCamelCase,revision='''9b8c223'''					)
        self.assertEqual(__lowerCamelCase,os.path.join(__lowerCamelCase,'''snapshots''',__lowerCamelCase,__lowerCamelCase					)					)



 def 							UpperCamelCase     (				self					):
        with self.assertRaisesRegex(__lowerCamelCase,'''is not a valid model identifier'''					):
               A__		   =		cached_file('''tiny-random-bert''',__lowerCamelCase					)

        with self.assertRaisesRegex(__lowerCamelCase,'''is not a valid git identifier'''					):
               A__		   =		cached_file(__lowerCamelCase,__lowerCamelCase,revision='''aaaa'''					)

        with self.assertRaisesRegex(__lowerCamelCase,'''does not appear to have a file named'''					):
               A__		   =		cached_file(__lowerCamelCase,'''conf'''					)



 def 							UpperCamelCase     (				self					):
        with self.assertRaisesRegex(__lowerCamelCase,'''does not appear to have a file named'''					):
               A__		   =		cached_file(__lowerCamelCase,'''conf'''					)

        with open(os.path.join(__lowerCamelCase,'''refs''','''main'''					)					) as f:
               A__		   =		f.read()
        self.assertTrue(os.path.isfile(os.path.join(__lowerCamelCase,'''.no_exist''',__lowerCamelCase,'''conf'''					)					)					)

        A__		   =		cached_file(__lowerCamelCase,'''conf''',_raise_exceptions_for_missing_entries=__lowerCamelCase					)
        self.assertIsNone(__lowerCamelCase					)

        A__		   =		cached_file(__lowerCamelCase,'''conf''',local_files_only=__lowerCamelCase,_raise_exceptions_for_missing_entries=__lowerCamelCase					)
        self.assertIsNone(__lowerCamelCase					)

        A__		   =		mock.Mock()
        A__		   =		500
        A__		   =		{}
        A__		   =		HTTPError
        A__		   =		{}

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch('''requests.Session.request''',return_value=__lowerCamelCase					) as mock_head:
               A__		   =		cached_file(__lowerCamelCase,'''conf''',_raise_exceptions_for_connection_errors=__lowerCamelCase					)
               self.assertIsNone(__lowerCamelCase					)
               # This check we did call the fake head request
               mock_head.assert_called()



 def 							UpperCamelCase     (				self					):
        self.assertTrue(has_file('''hf-internal-testing/tiny-bert-pt-only''',__lowerCamelCase					)					)
        self.assertFalse(has_file('''hf-internal-testing/tiny-bert-pt-only''',__lowerCamelCase					)					)
        self.assertFalse(has_file('''hf-internal-testing/tiny-bert-pt-only''',__lowerCamelCase					)					)



 def 							UpperCamelCase     (				self					):
        # `get_file_from_repo` returns None if the file does not exist
        self.assertIsNone(get_file_from_repo('''bert-base-cased''','''ahah.txt'''					)					)

        # The function raises if the repository does not exist.
        with self.assertRaisesRegex(__lowerCamelCase,'''is not a valid model identifier'''					):
               get_file_from_repo('''bert-base-case''',__lowerCamelCase					)

        # The function raises if the revision does not exist.
        with self.assertRaisesRegex(__lowerCamelCase,'''is not a valid git identifier'''					):
               get_file_from_repo('''bert-base-cased''',__lowerCamelCase,revision='''ahaha'''					)

        A__		   =		get_file_from_repo('''bert-base-cased''',__lowerCamelCase					)
        # The name is the cached name which is not very easy to test, so instead we load the content.
        A__		   =		json.loads(open(__lowerCamelCase,'''r'''					).read()					)
        self.assertEqual(config['''hidden_size'''],768					)




 def 							UpperCamelCase     (				self					):
        with tempfile.TemporaryDirectory() as tmp_dir:
               A__		   =		Path(__lowerCamelCase					) / '''a.txt'''
               filename.touch()
               self.assertEqual(get_file_from_repo(__lowerCamelCase,'''a.txt'''					),str(__lowerCamelCase					)					)

               self.assertIsNone(get_file_from_repo(__lowerCamelCase,'''b.txt'''					)					)





