


'''simple docstring'''






import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__A		:		Tuple   =   logging.get_logger(__name__)
__A		:		str   =   {'vocab_file': 'spiece.model'}

__A		:		Any   =   {
    'vocab_file': {
        'google/fnet-base': 'https://huggingface.co/google/fnet-base/resolve/main/spiece.model',
        'google/fnet-large': 'https://huggingface.co/google/fnet-large/resolve/main/spiece.model',
    },
}

__A		:		Tuple   =   {
    'google/fnet-base': 512,
    'google/fnet-large': 512,
}

__A		:		Optional[Any]   =   '▁'
class   __UpperCamelCase		(  a_							):
							lowercase	:						Tuple      						=     VOCAB_FILES_NAMES
							lowercase	:						int      						=     PRETRAINED_VOCAB_FILES_MAP
							lowercase	:						Optional[Any]      						=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase	:						Optional[int]      						=     ['input_ids', 'token_type_ids']





							def __init__(     self						:str     ,_UpperCamelCase						:Optional[int]     ,_UpperCamelCase						:Dict=False     ,_UpperCamelCase						:int=True     ,_UpperCamelCase						:int=True     ,_UpperCamelCase						:Tuple="<unk>"     ,_UpperCamelCase						:List[Any]="[SEP]"     ,_UpperCamelCase						:Tuple="<pad>"     ,_UpperCamelCase						:Optional[Any]="[CLS]"     ,_UpperCamelCase						:Tuple="[MASK]"     ,_UpperCamelCase						:Optional[Dict[str, Any]] = None     ,**_UpperCamelCase						:Optional[Any]     ,):
									# Mask token behave like a normal word, i.e. include the space before it and
									# is included in the raw text, there should be a match in a non-normalized sentence.
									snake_case_						:							Dict					=  (
									    AddedToken(_UpperCamelCase     ,lstrip=_UpperCamelCase     ,rstrip=_UpperCamelCase     ,normalized=_UpperCamelCase	)
									    if isinstance(_UpperCamelCase     ,_UpperCamelCase	)
									    else mask_token
									)

									snake_case_						:							int					=  {} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    do_lower_case=_UpperCamelCase     ,remove_space=_UpperCamelCase     ,keep_accents=_UpperCamelCase     ,unk_token=_UpperCamelCase     ,sep_token=_UpperCamelCase     ,pad_token=_UpperCamelCase     ,cls_token=_UpperCamelCase     ,mask_token=_UpperCamelCase     ,sp_model_kwargs=self.sp_model_kwargs     ,**_UpperCamelCase     ,)

									snake_case_						:							Any					=  do_lower_case
									snake_case_						:							Optional[int]					=  remove_space
									snake_case_						:							Optional[int]					=  keep_accents
									snake_case_						:							Tuple					=  vocab_file

									snake_case_						:							Any					=  spm.SentencePieceProcessor(**self.sp_model_kwargs	)
									self.sp_model.Load(_UpperCamelCase	)





							@property
							def 						a__      (     self						:List[str]	):
									return len(self.sp_model	)





							def 						a__      (     self						:List[str]	):
									snake_case_						:							List[str]					=  {self.convert_ids_to_tokens(_UpperCamelCase	): i for i in range(self.vocab_size	)}
									vocab.update(self.added_tokens_encoder	)
									return vocab





							def __getstate__(     self						:Any	):
									snake_case_						:							Tuple					=  self.__dict__.copy()
									snake_case_						:							List[Any]					=  None
									return state





							def __setstate__(     self						:str     ,_UpperCamelCase						:str	):
									snake_case_						:							int					=  d

									# for backward compatibility
									if not hasattr(self     ,"""sp_model_kwargs"""	):
											snake_case_						:							Any					=  {}

									snake_case_						:							Optional[int]					=  spm.SentencePieceProcessor(**self.sp_model_kwargs	)
									self.sp_model.Load(self.vocab_file	)





							def 						a__      (     self						:str     ,_UpperCamelCase						:Dict	):
									if self.remove_space:
											snake_case_						:							int					=  ' '.join(inputs.strip().split()	)
									else:
											snake_case_						:							Dict					=  inputs
									snake_case_						:							Union[str, Any]					=  outputs.replace("""``"""     ,"""\""""	).replace("""\'\'"""     ,"""\""""	)

									if not self.keep_accents:
											snake_case_						:							str					=  unicodedata.normalize("""NFKD"""     ,_UpperCamelCase	)
											snake_case_						:							List[Any]					=  ''.join([c for c in outputs if not unicodedata.combining(_UpperCamelCase	)]	)
									if self.do_lower_case:
											snake_case_						:							Union[str, Any]					=  outputs.lower()

									return outputs





							def 						a__      (     self						:Union[str, Any]     ,_UpperCamelCase						:str	):
									snake_case_						:							str					=  self.preprocess_text(_UpperCamelCase	)
									snake_case_						:							Tuple					=  self.sp_model.encode(_UpperCamelCase     ,out_type=_UpperCamelCase	)
									snake_case_						:							int					=  []
									for piece in pieces:
											if len(_UpperCamelCase	) > 1 and piece[-1] == str(""","""	) and piece[-2].isdigit():
													snake_case_						:							Any					=  self.sp_model.EncodeAsPieces(piece[:-1].replace(_UpperCamelCase     ,""""""	)	)
													if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
															if len(cur_pieces[0]	) == 1:
																	snake_case_						:							Dict					=  cur_pieces[1:]
															else:
																	snake_case_						:							int					=  cur_pieces[0][1:]
													cur_pieces.append(piece[-1]	)
													new_pieces.extend(_UpperCamelCase	)
											else:
													new_pieces.append(_UpperCamelCase	)

									return new_pieces





							def 						a__      (     self						:str     ,_UpperCamelCase						:List[Any]	):
									return self.sp_model.PieceToId(_UpperCamelCase	)





							def 						a__      (     self						:Tuple     ,_UpperCamelCase						:List[str]	):
									return self.sp_model.IdToPiece(_UpperCamelCase	)





							def 						a__      (     self						:Optional[Any]     ,_UpperCamelCase						:Union[str, Any]	):
									snake_case_						:							Union[str, Any]					=  []
									snake_case_						:							Any					=  ''
									snake_case_						:							Optional[Any]					=  False
									for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
													if not prev_is_special:
															out_string += " "
													out_string += self.sp_model.decode(_UpperCamelCase	) + token
													snake_case_						:							List[Any]					=  True
													snake_case_						:							Any					=  []
											else:
													current_sub_tokens.append(_UpperCamelCase	)
													snake_case_						:							int					=  False
									out_string += self.sp_model.decode(_UpperCamelCase	)
									return out_string.strip()





							def 						a__      (     self						:str     ,_UpperCamelCase						:List[int]     ,_UpperCamelCase						:bool = False     ,_UpperCamelCase						:bool = None     ,_UpperCamelCase						:bool = True     ,**_UpperCamelCase						:str     ,):
									snake_case_						:							Any					=  kwargs.pop("""use_source_tokenizer"""     ,_UpperCamelCase	)

									snake_case_						:							List[str]					=  self.convert_ids_to_tokens(_UpperCamelCase     ,skip_special_tokens=_UpperCamelCase	)

									# To avoid mixing byte-level and unicode for byte-level BPT
									# we need to build string separately for added tokens and byte-level tokens
									# cf. https://github.com/huggingface/transformers/issues/1133
									snake_case_						:							Optional[int]					=  []
									snake_case_						:							Dict					=  []
									for token in filtered_tokens:
											if skip_special_tokens and token in self.all_special_ids:
													continue
											if token in self.added_tokens_encoder:
													if current_sub_text:
															sub_texts.append(self.convert_tokens_to_string(_UpperCamelCase	)	)
															snake_case_						:							Optional[int]					=  []
													sub_texts.append(_UpperCamelCase	)
											else:
													current_sub_text.append(_UpperCamelCase	)
									if current_sub_text:
											sub_texts.append(self.convert_tokens_to_string(_UpperCamelCase	)	)

									# Mimic the behavior of the Rust tokenizer:
									# No space after <unk>
									if spaces_between_special_tokens:
											snake_case_						:							Dict					=  re.sub(R"""(<unk>) """     ,R"""\1"""     ,""" """.join(_UpperCamelCase	)	)
									else:
											snake_case_						:							Any					=  ''.join(_UpperCamelCase	)

									snake_case_						:							Optional[Any]					=  (
									    clean_up_tokenization_spaces
									    if clean_up_tokenization_spaces is not None
									    else self.clean_up_tokenization_spaces
									)
									if clean_up_tokenization_spaces:
											snake_case_						:							str					=  self.clean_up_tokenization(_UpperCamelCase	)
											return clean_text
									else:
											return text





							def 						a__      (     self						:List[Any]     ,_UpperCamelCase						:List[int]     ,_UpperCamelCase						:Optional[List[int]] = None	):
									snake_case_						:							Optional[int]					=  [self.sep_token_id]
									snake_case_						:							Dict					=  [self.cls_token_id]
									if token_ids_a is None:
											return cls + token_ids_a + sep
									return cls + token_ids_a + sep + token_ids_a + sep





							def 						a__      (     self						:Tuple     ,_UpperCamelCase						:List[int]     ,_UpperCamelCase						:Optional[List[int]] = None     ,_UpperCamelCase						:bool = False	):

									if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=_UpperCamelCase     ,token_ids_a=_UpperCamelCase     ,already_has_special_tokens=_UpperCamelCase	)

									if token_ids_a is not None:
											return [1] + ([0] * len(_UpperCamelCase	)) + [1] + ([0] * len(_UpperCamelCase	)) + [1]
									return [1] + ([0] * len(_UpperCamelCase	)) + [1]





							def 						a__      (     self						:Dict     ,_UpperCamelCase						:List[int]     ,_UpperCamelCase						:Optional[List[int]] = None	):
									snake_case_						:							Optional[Any]					=  [self.sep_token_id]
									snake_case_						:							Optional[int]					=  [self.cls_token_id]

									if token_ids_a is None:
											return len(cls + token_ids_a + sep	) * [0]
									return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]





							def 						a__      (     self						:Tuple     ,_UpperCamelCase						:str     ,_UpperCamelCase						:Optional[str] = None	):
									if not os.path.isdir(_UpperCamelCase	):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''	)
											return
									snake_case_						:							Tuple					=  os.path.join(
									    _UpperCamelCase     ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)

									if os.path.abspath(self.vocab_file	) != os.path.abspath(_UpperCamelCase	) and os.path.isfile(self.vocab_file	):
											copyfile(self.vocab_file     ,_UpperCamelCase	)
									elif not os.path.isfile(self.vocab_file	):
											with open(_UpperCamelCase     ,"""wb"""	) as fi:
													snake_case_						:							int					=  self.sp_model.serialized_model_proto()
													fi.write(_UpperCamelCase	)

									return (out_vocab_file,)
import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


UpperCAmelCase							= logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
UpperCAmelCase							= direct_transformers_import(Path(__file__).parent)


UpperCAmelCase							= {
    '''AutoTokenizer''': '''PreTrainedTokenizerBase''',
    '''AutoFeatureExtractor''': '''FeatureExtractionMixin''',
    '''AutoImageProcessor''': '''ImageProcessingMixin''',
}




class       __lowerCAmelCase     (				a_  ):



 '''simple docstring'''
 _lowerCamelCase  :		Optional[int] 			=       ['feature_extractor', 'tokenizer']
 # Names need to be attr_class for attr in attributes
 _lowerCamelCase  :		List[Any] 			=       None
 _lowerCamelCase  :		Optional[Any] 			=       None
 _lowerCamelCase  :		Tuple 			=       None

 def __init__(	self				:						Tuple				,	*lowerCamelCase				:						Optional[Any]				,	**lowerCamelCase				:						List[Any]     )       ->							str:
        # Sanitize args and kwargs
        for key in kwargs:
               if key not in self.attributes:
                      raise TypeError(f"Unexpected keyword argument {key}."     )
        for arg, attribute_name in zip(lowerCamelCase				,	self.attributes     ):
               if attribute_name in kwargs:
                      raise TypeError(f"Got multiple values for argument {attribute_name}."     )
               else:
                      _UpperCamelCase   :				Any       =     arg

        if len(lowerCamelCase     ) != len(self.attributes     ):
               raise ValueError(
                   f"This processor requires {len(self.attributes     )} arguments: {', '.join(self.attributes     )}. Got "
                   f"{len(lowerCamelCase     )} arguments instead."     )

        # Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
        for attribute_name, arg in kwargs.items():
               _UpperCamelCase   :				Union[str, Any]       =     getattr(self				,	f"{attribute_name}_class"     )
               # Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
               _UpperCamelCase   :				str       =     AUTO_TO_BASE_CLASS_MAPPING.get(lowerCamelCase				,	lowerCamelCase     )
               if isinstance(lowerCamelCase				,	lowerCamelCase     ):
                      _UpperCamelCase   :				int       =     tuple(getattr(lowerCamelCase				,	lowerCamelCase     ) for n in class_name if n is not None     )
               else:
                      _UpperCamelCase   :				Optional[int]       =     getattr(lowerCamelCase				,	lowerCamelCase     )

               if not isinstance(lowerCamelCase				,	lowerCamelCase     ):
                      raise ValueError(
                          f"Received a {type(lowerCamelCase     ).__name__} for argument {attribute_name}, but a {class_name} was expected."     )

               setattr(self				,	lowerCamelCase				,	lowerCamelCase     )

 def __repr__(	self				:						Dict     )       ->							str:
        _UpperCamelCase   :				List[str]       =     [f"- {name}: {repr(getattr(self				,	lowerCamelCase     )     )}" for name in self.attributes]
        _UpperCamelCase   :				int       =     '\n'.join(lowerCamelCase     )
        return f"{self.__class__.__name__}:\n{attributes_repr}"

 def     __snake_case    (	self				:						Tuple				,	lowerCamelCase				:						int				,	lowerCamelCase				:						bool = False				,	**lowerCamelCase				:						Optional[Any]     )       ->							Any:
        os.makedirs(lowerCamelCase				,	exist_ok=lowerCamelCase     )

        if push_to_hub:
               _UpperCamelCase   :				Optional[int]       =     kwargs.pop('commit_message'				,	lowerCamelCase     )
               _UpperCamelCase   :				List[Any]       =     kwargs.pop('repo_id'				,	save_directory.split(os.path.sep     )[-1]     )
               _UpperCamelCase   :				Optional[int]       =     self._create_repo(lowerCamelCase				,	**lowerCamelCase     )
               _UpperCamelCase   :				List[str]       =     self._get_files_timestamps(lowerCamelCase     )
        # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
        # loaded from the Hub.
        if self._auto_class is not None:
               _UpperCamelCase   :				Optional[int]       =     [getattr(self				,	lowerCamelCase     ) for attribute_name in self.attributes]
               _UpperCamelCase   :				int       =     [(a.init_kwargs if isinstance(lowerCamelCase				,	lowerCamelCase     ) else a) for a in attrs]
               custom_object_save(self				,	lowerCamelCase				,	config=lowerCamelCase     )

        for attribute_name in self.attributes:
               _UpperCamelCase   :				int       =     getattr(self				,	lowerCamelCase     )
               # Include the processor class in the attribute config so this processor can then be reloaded with the
               # `AutoProcessor` API.
               if hasattr(lowerCamelCase				,	'_set_processor_class'     ):
                      attribute._set_processor_class(self.__class__.__name__     )
               attribute.save_pretrained(lowerCamelCase     )

        if self._auto_class is not None:
               # We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
               for attribute_name in self.attributes:
                      _UpperCamelCase   :				Optional[Any]       =     getattr(self				,	lowerCamelCase     )
                      if isinstance(lowerCamelCase				,	lowerCamelCase     ):
                             del attribute.init_kwargs["auto_map"]

        if push_to_hub:
               self._upload_modified_files(
                   lowerCamelCase				,	lowerCamelCase				,	lowerCamelCase				,	commit_message=lowerCamelCase				,	token=kwargs.get('use_auth_token'     )				,	)

 @classmethod
 def     __snake_case    (	cls				:						Tuple				,	lowerCamelCase				:						Union[str, os.PathLike]				,	lowerCamelCase				:						Optional[Union[str, os.PathLike]] = None				,	lowerCamelCase				:						bool = False				,	lowerCamelCase				:						bool = False				,	lowerCamelCase				:						Optional[Union[str, bool]] = None				,	lowerCamelCase				:						str = "main"				,	**lowerCamelCase				:						Tuple				,	)       ->							Optional[Any]:
        _UpperCamelCase   :				Any       =     cache_dir
        _UpperCamelCase   :				Optional[int]       =     force_download
        _UpperCamelCase   :				int       =     local_files_only
        _UpperCamelCase   :				Dict       =     revision

        _UpperCamelCase   :				Optional[Any]       =     kwargs.pop('use_auth_token'				,	lowerCamelCase     )
        if use_auth_token is not None:
               warnings.warn(
                   'The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.'				,	lowerCamelCase     )
               if token is not None:
                      raise ValueError(
                          '`token` and `use_auth_token` are both specified. Please set only the argument `token`.'     )
               _UpperCamelCase   :				Optional[int]       =     use_auth_token

        if token is not None:
               # change to `token` in a follow-up PR
               _UpperCamelCase   :				List[Any]       =     token

        _UpperCamelCase   :				List[Any]       =     cls._get_arguments_from_pretrained(lowerCamelCase				,	**lowerCamelCase     )
        return cls(*lowerCamelCase     )

 @classmethod
 def     __snake_case    (	cls				:						Dict				,	lowerCamelCase				:						Optional[Any]="AutoProcessor"     )       ->							Optional[int]:
        if not isinstance(lowerCamelCase				,	lowerCamelCase     ):
               _UpperCamelCase   :				Tuple       =     auto_class.__name__

        import transformers.models.auto as auto_module

        if not hasattr(lowerCamelCase				,	lowerCamelCase     ):
               raise ValueError(f"{auto_class} is not a valid auto class."     )

        _UpperCamelCase   :				Union[str, Any]       =     auto_class

 @classmethod
 def     __snake_case    (	cls				:						int				,	lowerCamelCase				:						Any				,	**lowerCamelCase				:						List[Any]     )       ->							Optional[Any]:
        _UpperCamelCase   :				Optional[int]       =     []
        for attribute_name in cls.attributes:
               _UpperCamelCase   :				Dict       =     getattr(cls				,	f"{attribute_name}_class"     )
               if isinstance(lowerCamelCase				,	lowerCamelCase     ):
                      _UpperCamelCase   :				Any       =     tuple(getattr(lowerCamelCase				,	lowerCamelCase     ) if n is not None else None for n in class_name     )
                      _UpperCamelCase   :				List[Any]       =     kwargs.get('use_fast'				,	lowerCamelCase     )
                      if use_fast and classes[1] is not None:
                             _UpperCamelCase   :				Optional[int]       =     classes[1]
                      else:
                             _UpperCamelCase   :				Any       =     classes[0]
               else:
                      _UpperCamelCase   :				Any       =     getattr(lowerCamelCase				,	lowerCamelCase     )

               args.append(attribute_class.from_pretrained(lowerCamelCase				,	**lowerCamelCase     )     )
        return args

 @property
 def     __snake_case    (	self				:						List[Any]     )       ->							Optional[Any]:
        _UpperCamelCase   :				List[str]       =     getattr(self				,	self.attributes[0]     )
        return getattr(lowerCamelCase				,	'model_input_names'				,	lowerCamelCase     )







UpperCAmelCase							= copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
 UpperCAmelCase							= ProcessorMixin.push_to_hub.__doc__.format(
     object='''processor''', object_class='''AutoProcessor''', object_files='''processor files'''
 )



