



'''simple docstring'''

import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
	if is_torch_available():
		import torch
	if is_tf_available():
		import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


a							   =				logging.get_logger(__name__)

a							   =				{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

a							   =				{
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'Salesforce/codegen-350M-mono': (
            'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'
        ),
    },
}

a							   =				{
    'Salesforce/codegen-350M-mono': 2_048,
}






class 						a_		(     snake_case    ):
							UpperCAmelCase			:		Union[str, Any]        =				VOCAB_FILES_NAMES
							UpperCAmelCase			:		str        =				PRETRAINED_VOCAB_FILES_MAP
							UpperCAmelCase			:		Any        =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCAmelCase			:		Optional[Any]        =				["""input_ids""", """attention_mask"""]
							UpperCAmelCase			:		Optional[int]        =				CodeGenTokenizer


							def __init__(				self   :						Optional[Any]      ,						a_   :						Optional[int]=None      ,						a_   :						Union[str, Any]=None      ,						a_   :						List[str]=None      ,						a_   :						Any="<|endoftext|>"      ,						a_   :						Union[str, Any]="<|endoftext|>"      ,						a_   :						Any="<|endoftext|>"      ,						a_   :						Union[str, Any]=False      ,						**a_   :						Optional[int]      ,						)			->					List[str]:
										super().__init__(
										    a_      ,						a_      ,						tokenizer_file=a_      ,						unk_token=a_      ,						bos_token=a_      ,						eos_token=a_      ,						add_prefix_space=a_      ,						**a_      ,						)

										if kwargs.pop('add_bos_token'      ,						a_			):
													snake_case:    List[str]      							=kwargs.pop('name_or_path'      ,						''			)
													raise ValueError(
													    'Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'
													    'Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'
													    F'''`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'''
													    F'''`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'''
													    'This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'
													    ' so that the fast tokenizer works correctly.'			)

										snake_case:    int      							=json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()			)
										if pre_tok_state.get('add_prefix_space'      ,						a_			) != add_prefix_space:
													snake_case:    List[str]      							=getattr(a_      ,						pre_tok_state.pop('type'			)			)
													snake_case:    Any      							=add_prefix_space
													snake_case:    Any      							=pre_tok_class(**a_			)

										snake_case:    List[Any]      							=add_prefix_space


							def    UpperCamelCase     (				self   :						Optional[Any]      ,						*a_   :						Optional[Any]      ,						**a_   :						Optional[int]			)			->					BatchEncoding:
										snake_case:    Optional[int]      							=kwargs.get('is_split_into_words'      ,						a_			)
										assert self.add_prefix_space or not is_split_into_words, (
										    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*a_      ,						**a_			)


							def    UpperCamelCase     (				self   :						int      ,						*a_   :						Any      ,						**a_   :						Any			)			->					BatchEncoding:
										snake_case:    List[Any]      							=kwargs.get('is_split_into_words'      ,						a_			)

										assert self.add_prefix_space or not is_split_into_words, (
										    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*a_      ,						**a_			)


							def    UpperCamelCase     (				self   :						List[Any]      ,						a_   :						str      ,						a_   :						Optional[str] = None			)			->					Tuple[str]:
										snake_case:    int      							=self._tokenizer.model.save(a_      ,						name=a_			)
										return tuple(a_			)


							def    UpperCamelCase     (				self   :						Tuple      ,						a_   :						Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]      ,						a_   :						bool = False      ,						a_   :						bool = None      ,						a_   :						Optional[List[str]] = None      ,						**a_   :						List[str]      ,						)			->					str:
										snake_case:    Any      							=super().decode(
										    token_ids=a_      ,						skip_special_tokens=a_      ,						clean_up_tokenization_spaces=a_      ,						**a_      ,						)

										if truncate_before_pattern is not None and len(a_			) > 0:
													snake_case:    int      							=self.truncate(a_      ,						a_			)

										return decoded_text




							def    UpperCamelCase     (				self   :						Dict      ,						a_   :						List[str]      ,						a_   :						str			)			->					Optional[int]:
										def find_re(a_   :						Dict      ,						a_   :						Optional[Any]      ,						a_   :						Tuple			):
													snake_case:    Optional[Any]      							=pattern.search(a_      ,						a_			)
													return m.start() if m else -1

										snake_case:    Union[str, Any]      							=[re.compile(a_      ,						re.MULTILINE			) for pattern in truncate_before_pattern]

										snake_case:    List[Any]      							=list(re.finditer('^print'      ,						a_      ,						re.MULTILINE			)			)

										if len(a_			) > 1:
													snake_case:    List[str]      							=completion[: prints[1].start()]

										snake_case:    Tuple      							=list(re.finditer('^def'      ,						a_      ,						re.MULTILINE			)			)

										if len(a_			) > 1:
													snake_case:    int      							=completion[: defs[1].start()]

										snake_case:    Any      							=0

										snake_case:    List[str]      							=[
										    pos for pos in [find_re(a_      ,						a_      ,						a_			) for terminal in terminals] if pos != -1
										]

										if len(a_			) > 0:
													return completion[: min(a_			)]
										else:
													return completion









'''simple docstring'''

import os


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_doctest_list.py
a							   =				'.'


if __name__ == "__main__":
	a							   =				os.path.join(REPO_PATH, 'utils/documentation_tests.txt')
	a							   =				[]
	a							   =				[]
	with open(doctest_file_path) as fp:
		for line in fp:
			a							   =				line.strip()
			a							   =				os.path.join(REPO_PATH, line)
			if not (os.path.isfile(path) or os.path.isdir(path)):
				non_existent_paths.append(line)
			all_paths.append(path)
	if len(non_existent_paths) > 0:
		a							   =				'\n'.join(non_existent_paths)
		raise ValueError(F"""`utils/documentation_tests.txt` contains non-existent paths:\n{non_existent_paths}""")
	if all_paths != sorted(all_paths):
		raise ValueError('Files in `utils/documentation_tests.txt` are not in alphabetical order.')




