

'''simple docstring'''



import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
		import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__snake_case					:      Optional[int] 		=     logging.get_logger(__name__)

__snake_case					:      Any 		=     {'vocab_file': 'vocab.txt'}

__snake_case					:      Union[str, Any] 		=     {
    'vocab_file': {
        'openbmb/cpm-ant-10b': 'https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt',
    },
}

__snake_case					:      Any 		=     {
    'openbmb/cpm-ant-10b': 1024,
}
def 		__lowerCamelCase			(  __snake_case	:						Optional[int]			)			->				Dict:






						"""simple docstring"""





						A__	:  Optional[int]           =collections.OrderedDict()
						with open(__a,						"""r""",						encoding="""utf-8"""			) as reader:
												A__	:  List[Any]           =reader.readlines()
						for index, token in enumerate(__a			):
												A__	:  Any           =token.rstrip("""\n"""			)
												A__	:  Union[str, Any]           =index
						return vocab



class lowerCamelCase					(    SCREAMING_SNAKE_CASE_		):



			'''simple docstring'''






			def __init__( self			:		int						,     lowerCAmelCase_			:		Tuple						,     lowerCAmelCase_			:		int="<unk>"						,     lowerCAmelCase_			:		Optional[int]=2_00					)     ->						Tuple:





									'''simple docstring'''
									A__	:  int           =vocab
									A__	:  str           =unk_token
									A__	:  Tuple           =max_input_chars_per_word



			def lowercase__		( self			:		Union[str, Any]						,     lowerCAmelCase_			:		List[Any]					)     ->						Union[str, Any]:





									'''simple docstring'''
									A__	:  Optional[int]           =list(lowerCAmelCase_					)
									if len(lowerCAmelCase_					) > self.max_input_chars_per_word:
															return [self.unk_token]

									A__	:  Any           =0
									A__	:  Optional[int]           =[]
									while start < len(lowerCAmelCase_					):
															A__	:  Union[str, Any]           =len(lowerCAmelCase_					)
															A__	:  int           =None
															while start < end:
																					A__	:  Optional[Any]           =''.join(chars[start:end]					)
																					if substr in self.vocab:
																											A__	:  List[str]           =substr
																											break
																					end -= 1
															if cur_substr is None:
																					sub_tokens.append(self.unk_token					)
																					start += 1
															else:
																					sub_tokens.append(lowerCAmelCase_					)
																					A__	:  Union[str, Any]           =end

									return sub_tokens



class lowerCamelCase					(    SCREAMING_SNAKE_CASE_		):



			'''simple docstring'''






			__snake_case             =			VOCAB_FILES_NAMES
			__snake_case             =			PRETRAINED_VOCAB_FILES_MAP
			__snake_case             =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__snake_case             =			["""input_ids""", """attention_mask"""]
			__snake_case             =			False


			def __init__( self			:		Dict						,     lowerCAmelCase_			:		Dict						,     lowerCAmelCase_			:		Tuple="<d>"						,     lowerCAmelCase_			:		Dict="</d>"						,     lowerCAmelCase_			:		Union[str, Any]="<s>"						,     lowerCAmelCase_			:		Optional[int]="</s>"						,     lowerCAmelCase_			:		str="<pad>"						,     lowerCAmelCase_			:		Any="<unk>"						,     lowerCAmelCase_			:		List[Any]="</n>"						,     lowerCAmelCase_			:		Optional[Any]="</_>"						,     lowerCAmelCase_			:		List[Any]="left"						,     **lowerCAmelCase_			:		Any						,     )     ->						Any:





									'''simple docstring'''
									requires_backends(self						,     ["""jieba"""]					)
									super().__init__(
									    bod_token=lowerCAmelCase_						,     eod_token=lowerCAmelCase_						,     bos_token=lowerCAmelCase_						,     eos_token=lowerCAmelCase_						,     pad_token=lowerCAmelCase_						,     unk_token=lowerCAmelCase_						,     line_token=lowerCAmelCase_						,     space_token=lowerCAmelCase_						,     padding_side=lowerCAmelCase_						,     **lowerCAmelCase_						,     )
									A__	:  str           =bod_token
									A__	:  Optional[Any]           =eod_token
									A__	:  Any           =load_vocab(lowerCAmelCase_					)
									A__	:  Union[str, Any]           =self.encoder[space_token]
									A__	:  int           =self.encoder[line_token]

									del self.encoder[space_token]
									del self.encoder[line_token]

									A__	:  Tuple           =collections.OrderedDict(sorted(self.encoder.items()						,     key=lambda lowerCAmelCase_					: x[1]					)					)
									A__	:  Any           ={v: k for k, v in self.encoder.items()}

									A__	:  List[str]           =WordpieceTokenizer(vocab=self.encoder						,     unk_token=self.unk_token					)


			@property
			def lowercase__		( self			:		Optional[int]					)     ->						Tuple:





									'''simple docstring'''
									return self.encoder[self.bod_token]


			@property
			def lowercase__		( self			:		Any					)     ->						Union[str, Any]:





									'''simple docstring'''
									return self.encoder[self.eod_token]


			@property
			def lowercase__		( self			:		Any					)     ->						str:





									'''simple docstring'''
									return self.encoder["\n"]


			@property
			def lowercase__		( self			:		Union[str, Any]					)     ->						int:





									'''simple docstring'''
									return len(self.encoder					)


			def lowercase__		( self			:		Any					)     ->						Any:





									'''simple docstring'''
									return dict(self.encoder						,     **self.added_tokens_encoder					)


			def lowercase__		( self			:		Dict						,     lowerCAmelCase_			:		Tuple					)     ->						Dict:





									'''simple docstring'''
									A__	:  Dict           =[]
									for x in jieba.cut(lowerCAmelCase_						,     cut_all=lowerCAmelCase_					):
															output_tokens.extend(self.wordpiece_tokenizer.tokenize(lowerCAmelCase_					)					)
									return output_tokens


			def lowercase__		( self			:		List[Any]						,     lowerCAmelCase_			:		Any						,     **lowerCAmelCase_			:		Union[str, Any]					)     ->						Optional[Any]:





									'''simple docstring'''
									A__	:  Any           =[i for i in token_ids if i >= 0]
									A__	:  List[Any]           =[
									    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
									]
									return super()._decode(lowerCAmelCase_						,     **lowerCAmelCase_					)


			def lowercase__		( self			:		int						,     lowerCAmelCase_			:		List[str]					)     ->						int:





									'''simple docstring'''
									return token in self.encoder


			def lowercase__		( self			:		Dict						,     lowerCAmelCase_			:		Union[str, Any]					)     ->						int:





									'''simple docstring'''
									return "".join(lowerCAmelCase_					)


			def lowercase__		( self			:		Optional[Any]						,     lowerCAmelCase_			:		int					)     ->						int:





									'''simple docstring'''
									return self.encoder.get(lowerCAmelCase_						,     self.encoder.get(self.unk_token					)					)


			def lowercase__		( self			:		Optional[int]						,     lowerCAmelCase_			:		Union[str, Any]					)     ->						Optional[int]:





									'''simple docstring'''
									return self.decoder.get(lowerCAmelCase_						,     self.unk_token					)


			def lowercase__		( self			:		Any						,     lowerCAmelCase_			:		str						,     lowerCAmelCase_			:		Optional[int] = None					)     ->						List[str]:





									'''simple docstring'''
									if os.path.isdir(lowerCAmelCase_					):
															A__	:  str           =os.path.join(
															    lowerCAmelCase_						,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)
									else:
															A__	:  List[str]           =(filename_prefix + '-' if filename_prefix else '') + save_directory
									A__	:  Tuple           =0
									if " " in self.encoder:
															A__	:  Any           =self.encoder[' ']
															del self.encoder[" "]
									if "\n" in self.encoder:
															A__	:  Optional[Any]           =self.encoder['\n']
															del self.encoder["\n"]
									A__	:  Optional[Any]           =collections.OrderedDict(sorted(self.encoder.items()						,     key=lambda lowerCAmelCase_					: x[1]					)					)
									with open(lowerCAmelCase_						,     """w"""						,     encoding="""utf-8"""					) as writer:
															for token, token_index in self.encoder.items():
																					if index != token_index:
																											logger.warning(
																											    f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
																											    """ Please check that the vocabulary is not corrupted!"""					)
																											A__	:  Optional[int]           =token_index
																					writer.write(token + """\n"""					)
																					index += 1
									return (vocab_file,)


			def lowercase__		( self			:		int						,     lowerCAmelCase_			:		Dict						,     lowerCAmelCase_			:		Dict = None					)     ->						Any:





									'''simple docstring'''
									if token_ids_a is None:
															return [self.bos_token_id] + token_ids_a
									return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a



			def lowercase__		( self			:		Any						,     lowerCAmelCase_			:		Union[str, Any]						,     lowerCAmelCase_			:		str = None						,     lowerCAmelCase_			:		Optional[int] = False					)     ->						List[Any]:





									'''simple docstring'''

									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=lowerCAmelCase_						,     token_ids_a=lowerCAmelCase_						,     already_has_special_tokens=lowerCAmelCase_					)

									if token_ids_a is not None:
															return [1] + ([0] * len(lowerCAmelCase_					)) + [1] + ([0] * len(lowerCAmelCase_					))
									return [1] + ([0] * len(lowerCAmelCase_					))






"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


a_							=    {"""configuration_opt""": ["""OPT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """OPTConfig"""]}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	a_							=    [
	    """OPT_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """OPTForCausalLM""",
	    """OPTModel""",
	    """OPTPreTrainedModel""",
	    """OPTForSequenceClassification""",
	    """OPTForQuestionAnswering""",
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	a_							=    ["""TFOPTForCausalLM""", """TFOPTModel""", """TFOPTPreTrainedModel"""]

try:
	if not is_flax_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	a_							=    [
	    """FlaxOPTForCausalLM""",
	    """FlaxOPTModel""",
	    """FlaxOPTPreTrainedModel""",
	]


if TYPE_CHECKING:
	from .configuration_opt import OPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OPTConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_opt import (
		    OPT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    OPTForCausalLM,
		    OPTForQuestionAnswering,
		    OPTForSequenceClassification,
		    OPTModel,
		    OPTPreTrainedModel,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_opt import TFOPTForCausalLM, TFOPTModel, TFOPTPreTrainedModel

	try:
		if not is_flax_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_flax_opt import FlaxOPTForCausalLM, FlaxOPTModel, FlaxOPTPreTrainedModel

else:
	import sys

	a_							=    _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





