





'''simple docstring'''

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowercase__		:     Union[str, Any]			=							logging.get_logger(__name__)

lowercase__		:     List[Any]			=							{'vocab_file': 'spiece.model'}

lowercase__		:     List[Any]			=							{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

lowercase__		:     Optional[Any]			=							{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
lowercase__		:     Tuple			=							0
lowercase__		:     Union[str, Any]			=							1
lowercase__		:     Any			=							2
lowercase__		:     Optional[int]			=							3
lowercase__		:     Optional[Any]			=							4
class 							__lowerCAmelCase	(       __magic_name__					):






       """simple docstring"""
       _snake_case      :	Optional[int]									=       VOCAB_FILES_NAMES
       _snake_case      :	List[str]									=       PRETRAINED_VOCAB_FILES_MAP
       _snake_case      :	Optional[int]									=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       _snake_case      :	Dict									=       'left'





       def __init__(				self  :						Any			,					lowerCAmelCase__  :						int			,					lowerCAmelCase__  :						str=False			,					lowerCAmelCase__  :						Union[str, Any]=True			,					lowerCAmelCase__  :						str=False			,					lowerCAmelCase__  :						int="<s>"			,					lowerCAmelCase__  :						int="</s>"			,					lowerCAmelCase__  :						str="<unk>"			,					lowerCAmelCase__  :						Tuple="<sep>"			,					lowerCAmelCase__  :						Union[str, Any]="<pad>"			,					lowerCAmelCase__  :						str="<cls>"			,					lowerCAmelCase__  :						Any="<mask>"			,					lowerCAmelCase__  :						Any=["<eop>", "<eod>"]			,					lowerCAmelCase__  :						Optional[Dict[str, Any]] = None			,					**lowerCAmelCase__  :						List[Any]			,					)       ->	None:

            '''simple docstring'''




            _UpperCamelCase							       =			AddedToken(lowerCAmelCase__			,					lstrip=lowerCAmelCase__			,					rstrip=lowerCAmelCase__	) if isinstance(lowerCAmelCase__			,					lowerCAmelCase__	) else mask_token

            _UpperCamelCase							       =			{} if sp_model_kwargs is None else sp_model_kwargs

            super().__init__(
                do_lower_case=lowerCAmelCase__			,					remove_space=lowerCAmelCase__			,					keep_accents=lowerCAmelCase__			,					bos_token=lowerCAmelCase__			,					eos_token=lowerCAmelCase__			,					unk_token=lowerCAmelCase__			,					sep_token=lowerCAmelCase__			,					pad_token=lowerCAmelCase__			,					cls_token=lowerCAmelCase__			,					mask_token=lowerCAmelCase__			,					additional_special_tokens=lowerCAmelCase__			,					sp_model_kwargs=self.sp_model_kwargs			,					**lowerCAmelCase__			,					)

            _UpperCamelCase							       =			3

            _UpperCamelCase							       =			do_lower_case
            _UpperCamelCase							       =			remove_space
            _UpperCamelCase							       =			keep_accents
            _UpperCamelCase							       =			vocab_file

            _UpperCamelCase							       =			spm.SentencePieceProcessor(**self.sp_model_kwargs	)
            self.sp_model.Load(lowerCAmelCase__	)





       @property
       def snake_case__		(				self  :						int	)       ->	Union[str, Any]:

            '''simple docstring'''




            return len(self.sp_model	)





       def snake_case__		(				self  :						List[str]	)       ->	Tuple:

            '''simple docstring'''




            _UpperCamelCase							       =			{self.convert_ids_to_tokens(lowerCAmelCase__	): i for i in range(self.vocab_size	)}
            vocab.update(self.added_tokens_encoder	)
            return vocab





       def __getstate__(				self  :						Optional[Any]	)       ->	List[Any]:

            '''simple docstring'''




            _UpperCamelCase							       =			self.__dict__.copy()
            _UpperCamelCase							       =			None
            return state





       def __setstate__(				self  :						Union[str, Any]			,					lowerCAmelCase__  :						str	)       ->	List[str]:

            '''simple docstring'''




            _UpperCamelCase							       =			d

            # for backward compatibility
            if not hasattr(self			,					'''sp_model_kwargs'''	):
                 _UpperCamelCase							       =			{}

            _UpperCamelCase							       =			spm.SentencePieceProcessor(**self.sp_model_kwargs	)
            self.sp_model.Load(self.vocab_file	)





       def snake_case__		(				self  :						int			,					lowerCAmelCase__  :						List[str]	)       ->	Tuple:

            '''simple docstring'''




            if self.remove_space:
                 _UpperCamelCase							       =			''' '''.join(inputs.strip().split()	)
            else:
                 _UpperCamelCase							       =			inputs
            _UpperCamelCase							       =			outputs.replace('''``'''			,					'''"'''	).replace('''\'\''''			,					'''"'''	)

            if not self.keep_accents:
                 _UpperCamelCase							       =			unicodedata.normalize('''NFKD'''			,					lowerCAmelCase__	)
                 _UpperCamelCase							       =			''''''.join([c for c in outputs if not unicodedata.combining(lowerCAmelCase__	)]	)
            if self.do_lower_case:
                 _UpperCamelCase							       =			outputs.lower()

            return outputs





       def snake_case__		(				self  :						Tuple			,					lowerCAmelCase__  :						str	)       ->	List[str]:

            '''simple docstring'''




            _UpperCamelCase							       =			self.preprocess_text(lowerCAmelCase__	)
            _UpperCamelCase							       =			self.sp_model.encode(lowerCAmelCase__			,					out_type=lowerCAmelCase__	)
            _UpperCamelCase							       =			[]
            for piece in pieces:
                 if len(lowerCAmelCase__	) > 1 and piece[-1] == str(''','''	) and piece[-2].isdigit():
                      _UpperCamelCase							       =			self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase__			,					''''''	)	)
                      if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                           if len(cur_pieces[0]	) == 1:
                                _UpperCamelCase							       =			cur_pieces[1:]
                           else:
                                _UpperCamelCase							       =			cur_pieces[0][1:]
                      cur_pieces.append(piece[-1]	)
                      new_pieces.extend(lowerCAmelCase__	)
                 else:
                      new_pieces.append(lowerCAmelCase__	)

            return new_pieces





       def snake_case__		(				self  :						Union[str, Any]			,					lowerCAmelCase__  :						Dict	)       ->	Optional[int]:

            '''simple docstring'''




            return self.sp_model.PieceToId(lowerCAmelCase__	)





       def snake_case__		(				self  :						Union[str, Any]			,					lowerCAmelCase__  :						Any	)       ->	str:

            '''simple docstring'''




            return self.sp_model.IdToPiece(lowerCAmelCase__	)





       def snake_case__		(				self  :						int			,					lowerCAmelCase__  :						Any	)       ->	Tuple:

            '''simple docstring'''




            _UpperCamelCase							       =			''''''.join(lowerCAmelCase__	).replace(lowerCAmelCase__			,					''' '''	).strip()
            return out_string





       def snake_case__		(				self  :						List[Any]			,					lowerCAmelCase__  :						List[int]			,					lowerCAmelCase__  :						bool = False			,					lowerCAmelCase__  :						bool = None			,					lowerCAmelCase__  :						bool = True			,					**lowerCAmelCase__  :						Optional[int]			,					)       ->	str:

            '''simple docstring'''




            _UpperCamelCase							       =			kwargs.pop('''use_source_tokenizer'''			,					lowerCAmelCase__	)

            _UpperCamelCase							       =			self.convert_ids_to_tokens(lowerCAmelCase__			,					skip_special_tokens=lowerCAmelCase__	)

            # To avoid mixing byte-level and unicode for byte-level BPT
            # we need to build string separately for added tokens and byte-level tokens
            # cf. https://github.com/huggingface/transformers/issues/1133
            _UpperCamelCase							       =			[]
            _UpperCamelCase							       =			[]
            for token in filtered_tokens:
                 if skip_special_tokens and token in self.all_special_ids:
                      continue
                 if token in self.added_tokens_encoder:
                      if current_sub_text:
                           sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__	)	)
                           _UpperCamelCase							       =			[]
                      sub_texts.append(lowerCAmelCase__	)
                 else:
                      current_sub_text.append(lowerCAmelCase__	)
            if current_sub_text:
                 sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__	)	)

            # Mimic the behavior of the Rust tokenizer:
            # By default, there are no spaces between special tokens
            _UpperCamelCase							       =			''''''.join(lowerCAmelCase__	)

            _UpperCamelCase							       =			(
                clean_up_tokenization_spaces
                if clean_up_tokenization_spaces is not None
                else self.clean_up_tokenization_spaces
            )
            if clean_up_tokenization_spaces:
                 _UpperCamelCase							       =			self.clean_up_tokenization(lowerCAmelCase__	)
                 return clean_text
            else:
                 return text





       def snake_case__		(				self  :						Union[str, Any]			,					lowerCAmelCase__  :						List[int]			,					lowerCAmelCase__  :						Optional[List[int]] = None	)       ->	List[int]:

            '''simple docstring'''




            _UpperCamelCase							       =			[self.sep_token_id]
            _UpperCamelCase							       =			[self.cls_token_id]
            if token_ids_a is None:
                 return token_ids_a + sep + cls
            return token_ids_a + sep + token_ids_a + sep + cls





       def snake_case__		(				self  :						Optional[Any]			,					lowerCAmelCase__  :						List[int]			,					lowerCAmelCase__  :						Optional[List[int]] = None			,					lowerCAmelCase__  :						bool = False	)       ->	List[int]:

            '''simple docstring'''




            if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=lowerCAmelCase__			,					token_ids_a=lowerCAmelCase__			,					already_has_special_tokens=lowerCAmelCase__	)

            if token_ids_a is not None:
                 return ([0] * len(lowerCAmelCase__	)) + [1] + ([0] * len(lowerCAmelCase__	)) + [1, 1]
            return ([0] * len(lowerCAmelCase__	)) + [1, 1]





       def snake_case__		(				self  :						Union[str, Any]			,					lowerCAmelCase__  :						List[int]			,					lowerCAmelCase__  :						Optional[List[int]] = None	)       ->	List[int]:

            '''simple docstring'''




            _UpperCamelCase							       =			[self.sep_token_id]
            _UpperCamelCase							       =			[2]

            if token_ids_a is None:
                 return len(token_ids_a + sep	) * [0] + cls_segment_id
            return len(token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1] + cls_segment_id







       def snake_case__		(				self  :						List[str]			,					lowerCAmelCase__  :						str			,					lowerCAmelCase__  :						Optional[str] = None	)       ->	Tuple[str]:

            '''simple docstring'''




            if not os.path.isdir(lowerCAmelCase__	):
                 logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""	)
                 return
            _UpperCamelCase							       =			os.path.join(
                lowerCAmelCase__			,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

            if os.path.abspath(self.vocab_file	) != os.path.abspath(lowerCAmelCase__	) and os.path.isfile(self.vocab_file	):
                 copyfile(self.vocab_file			,					lowerCAmelCase__	)
            elif not os.path.isfile(self.vocab_file	):
                 with open(lowerCAmelCase__			,					'''wb'''	) as fi:
                      _UpperCamelCase							       =			self.sp_model.serialized_model_proto()
                      fi.write(lowerCAmelCase__	)

            return (out_vocab_file,)










'''simple docstring'''

import unittest

from accelerate import debug_launcher
from accelerate.test_utils import require_cpu, test_ops, test_script
@require_cpu
class 							__lowerCAmelCase	(       unittest.TestCase					):






       """simple docstring"""





       def snake_case__		(				self  :						str	)       ->	Optional[int]:

            '''simple docstring'''




            debug_launcher(test_script.main	)







       def snake_case__		(				self  :						Any	)       ->	Optional[Any]:

            '''simple docstring'''




            debug_launcher(test_ops.main	)



