

'''simple docstring'''

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase					 =	logging.get_logger(__name__)

UpperCAmelCase					 =	'''▁'''

UpperCAmelCase					 =	{'''vocab_file''': '''sentencepiece.bpe.model''', '''monolingual_vocab_file''': '''dict.txt'''}

UpperCAmelCase					 =	{
    '''vocab_file''': {
        '''vinai/bartpho-syllable''': '''https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model''',
    },
    '''monolingual_vocab_file''': {
        '''vinai/bartpho-syllable''': '''https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt''',
    },
}

UpperCAmelCase					 =	{'''vinai/bartpho-syllable''': 1024}


class 	lowerCAmelCase  (   _lowerCAmelCase ):
       lowerCAmelCase_												=			VOCAB_FILES_NAMES
       lowerCAmelCase_												=			PRETRAINED_VOCAB_FILES_MAP
       lowerCAmelCase_												=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowerCAmelCase_												=			["input_ids", "attention_mask"]







       def __init__(				self      :			Optional[Any]    ,   __lowercase      :			List[Any]    ,   __lowercase      :			int    ,   __lowercase      :			Union[str, Any]="<s>"    ,   __lowercase      :			int="</s>"    ,   __lowercase      :			List[str]="</s>"    ,   __lowercase      :			Any="<s>"    ,   __lowercase      :			int="<unk>"    ,   __lowercase      :			Union[str, Any]="<pad>"    ,   __lowercase      :			Dict="<mask>"    ,   __lowercase      :			int = None    ,   **__lowercase      :			Optional[int]    ,   ):







            """simple docstring"""







            __lowercase      		=AddedToken(SCREAMING_SNAKE_CASE__    ,   lstrip=SCREAMING_SNAKE_CASE__    ,   rstrip=SCREAMING_SNAKE_CASE__   ) if isinstance(SCREAMING_SNAKE_CASE__    ,   SCREAMING_SNAKE_CASE__   ) else mask_token

            __lowercase      		={} if sp_model_kwargs is None else sp_model_kwargs

            super().__init__(
                bos_token=SCREAMING_SNAKE_CASE__    ,   eos_token=SCREAMING_SNAKE_CASE__    ,   unk_token=SCREAMING_SNAKE_CASE__    ,   sep_token=SCREAMING_SNAKE_CASE__    ,   cls_token=SCREAMING_SNAKE_CASE__    ,   pad_token=SCREAMING_SNAKE_CASE__    ,   mask_token=SCREAMING_SNAKE_CASE__    ,   sp_model_kwargs=self.sp_model_kwargs    ,   **SCREAMING_SNAKE_CASE__    ,   )

            __lowercase      		=vocab_file
            __lowercase      		=monolingual_vocab_file
            __lowercase      		=spm.SentencePieceProcessor(**self.sp_model_kwargs   )
            self.sp_model.Load(str(SCREAMING_SNAKE_CASE__   )   )

            # Load the reduced vocab

            # Keep order of special tokens for backward compatibility
            __lowercase      		={}
            __lowercase      		=0
            for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
                 if str(SCREAMING_SNAKE_CASE__   ) not in self.fairseq_tokens_to_ids:
                      __lowercase      		=cnt
                      cnt += 1
            with open(SCREAMING_SNAKE_CASE__    ,   'r'    ,   encoding='utf-8'   ) as f:
                 for line in f.readlines():
                      __lowercase      		=line.strip().split()[0]
                      __lowercase      		=len(self.fairseq_tokens_to_ids   )
            if str(SCREAMING_SNAKE_CASE__   ) not in self.fairseq_tokens_to_ids:
                 __lowercase      		=len(self.fairseq_tokens_to_ids   )

            __lowercase      		={v: k for k, v in self.fairseq_tokens_to_ids.items()}







       def __getstate__(				self      :			List[Any]   ):







            """simple docstring"""







            __lowercase      		=self.__dict__.copy()
            __lowercase      		=None
            __lowercase      		=self.sp_model.serialized_model_proto()
            return state







       def __setstate__(				self      :			Tuple    ,   __lowercase      :			Tuple   ):







            """simple docstring"""







            __lowercase      		=d

            # for backward compatibility
            if not hasattr(self    ,   'sp_model_kwargs'   ):
                 __lowercase      		={}

            __lowercase      		=spm.SentencePieceProcessor(**self.sp_model_kwargs   )
            self.sp_model.LoadFromSerializedProto(self.sp_model_proto   )







       def       snake_case (				self      :			List[str]    ,   __lowercase      :			List[str]    ,   __lowercase      :			List[str] = None   ):







            """simple docstring"""







            if token_ids_a is None:
                 return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
            __lowercase      		=[self.cls_token_id]
            __lowercase      		=[self.sep_token_id]
            return cls + token_ids_a + sep + sep + token_ids_a + sep







       def       snake_case (				self      :			Any    ,   __lowercase      :			Tuple    ,   __lowercase      :			str = None    ,   __lowercase      :			List[Any] = False   ):







            """simple docstring"""







            if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=SCREAMING_SNAKE_CASE__    ,   token_ids_a=SCREAMING_SNAKE_CASE__    ,   already_has_special_tokens=SCREAMING_SNAKE_CASE__   )

            if token_ids_a is None:
                 return [1] + ([0] * len(SCREAMING_SNAKE_CASE__   )) + [1]
            return [1] + ([0] * len(SCREAMING_SNAKE_CASE__   )) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE__   )) + [1]







       def       snake_case (				self      :			Tuple    ,   __lowercase      :			Optional[int]    ,   __lowercase      :			str = None   ):







            """simple docstring"""







            __lowercase      		=[self.sep_token_id]
            __lowercase      		=[self.cls_token_id]

            if token_ids_a is None:
                 return len(cls + token_ids_a + sep   ) * [0]
            return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]







       @property
       def       snake_case (				self      :			Dict   ):







            """simple docstring"""







            return len(self.fairseq_ids_to_tokens   )







       def       snake_case (				self      :			int   ):







            """simple docstring"""







            __lowercase      		={self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__   ): i for i in range(self.vocab_size   )}
            vocab.update(self.added_tokens_encoder   )
            return vocab







       def       snake_case (				self      :			str    ,   __lowercase      :			int   ):







            """simple docstring"""







            return self.sp_model.encode(SCREAMING_SNAKE_CASE__    ,   out_type=SCREAMING_SNAKE_CASE__   )







       def       snake_case (				self      :			Any    ,   __lowercase      :			Dict   ):







            """simple docstring"""







            if token in self.fairseq_tokens_to_ids:
                 return self.fairseq_tokens_to_ids[token]
            else:
                 return self.unk_token_id







       def       snake_case (				self      :			Optional[int]    ,   __lowercase      :			List[str]   ):







            """simple docstring"""







            return self.fairseq_ids_to_tokens[index]







       def       snake_case (				self      :			Any    ,   __lowercase      :			Dict   ):







            """simple docstring"""







            __lowercase      		=''.join(SCREAMING_SNAKE_CASE__   ).replace(SCREAMING_SNAKE_CASE__    ,   ' '   ).strip()
            return out_string







       def       snake_case (				self      :			List[Any]    ,   __lowercase      :			Optional[int]    ,   __lowercase      :			Optional[Any] = None   ):







            """simple docstring"""







            if not os.path.isdir(SCREAMING_SNAKE_CASE__   ):
                 logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''   )
                 return
            __lowercase      		=os.path.join(
                SCREAMING_SNAKE_CASE__    ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )
            __lowercase      		=os.path.join(
                SCREAMING_SNAKE_CASE__    ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['monolingual_vocab_file']    ,   )

            if os.path.abspath(self.vocab_file   ) != os.path.abspath(SCREAMING_SNAKE_CASE__   ) and os.path.isfile(self.vocab_file   ):
                 copyfile(self.vocab_file    ,   SCREAMING_SNAKE_CASE__   )
            elif not os.path.isfile(self.vocab_file   ):
                 with open(SCREAMING_SNAKE_CASE__    ,   'wb'   ) as fi:
                      __lowercase      		=self.sp_model.serialized_model_proto()
                      fi.write(SCREAMING_SNAKE_CASE__   )

            if os.path.abspath(self.monolingual_vocab_file   ) != os.path.abspath(
                SCREAMING_SNAKE_CASE__   ) and os.path.isfile(self.monolingual_vocab_file   ):
                 copyfile(self.monolingual_vocab_file    ,   SCREAMING_SNAKE_CASE__   )
            elif not os.path.isfile(self.monolingual_vocab_file   ):
                 with open(SCREAMING_SNAKE_CASE__    ,   'w'    ,   encoding='utf-8'   ) as fp:
                      for token in self.fairseq_tokens_to_ids:
                           if token not in self.all_special_tokens:
                                fp.write(f'''{str(SCREAMING_SNAKE_CASE__   )} \n'''   )

            return out_vocab_file, out_monolingual_vocab_file







"""simple docstring"""







def       _lowerCamelCase				(							UpperCAmelCase_ :       int     )	-> int:






	"""simple docstring"""





	if not isinstance(UpperCAmelCase_,	UpperCAmelCase_     ):
		raise ValueError("Input must be an integer"     )
	if input_num <= 0:
		raise ValueError("Input must be positive"     )
	return sum(
	    divisor for divisor in range(1,	input_num // 2 + 1     ) if input_num % divisor == 0     )


if __name__ == "__main__":
			import doctest

			doctest.testmod()




