



'''simple docstring'''

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


SCREAMING_SNAKE_CASE__     :						List[str]	=  logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__     :						Optional[int]	=  {'''vocab_file''': '''spiece.model'''}

SCREAMING_SNAKE_CASE__     :						List[str]	=  {
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

SCREAMING_SNAKE_CASE__     :						Dict	=  {
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
SCREAMING_SNAKE_CASE__     :						Dict	=  0
SCREAMING_SNAKE_CASE__     :						Tuple	=  1
SCREAMING_SNAKE_CASE__     :						Union[str, Any]	=  2
SCREAMING_SNAKE_CASE__     :						int	=  3
SCREAMING_SNAKE_CASE__     :						Optional[int]	=  4

class 							a__( snake_case__     ):
			a_	: Union[str, Any]									=      VOCAB_FILES_NAMES
			a_	: Union[str, Any]									=      PRETRAINED_VOCAB_FILES_MAP
			a_	: Dict									=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			a_	: Optional[Any]									=      '''left'''






			def __init__(     self					,   _UpperCAmelCase					,   _UpperCAmelCase=False					,   _UpperCAmelCase=True					,   _UpperCAmelCase=False					,   _UpperCAmelCase="<s>"					,   _UpperCAmelCase="</s>"					,   _UpperCAmelCase="<unk>"					,   _UpperCAmelCase="<sep>"					,   _UpperCAmelCase="<pad>"					,   _UpperCAmelCase="<cls>"					,   _UpperCAmelCase="<mask>"					,   _UpperCAmelCase=["<eop>", "<eod>"]					,   _UpperCAmelCase = None					,   **_UpperCAmelCase					,   )							->							None:
					# Mask token behave like a normal word, i.e. include the space before it
					snake_case__	     =AddedToken(_UpperCAmelCase					,   lstrip=_UpperCAmelCase					,   rstrip=_UpperCAmelCase						) if isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) else mask_token

					snake_case__	     ={} if sp_model_kwargs is None else sp_model_kwargs

					super().__init__(
					    do_lower_case=_UpperCAmelCase					,   remove_space=_UpperCAmelCase					,   keep_accents=_UpperCAmelCase					,   bos_token=_UpperCAmelCase					,   eos_token=_UpperCAmelCase					,   unk_token=_UpperCAmelCase					,   sep_token=_UpperCAmelCase					,   pad_token=_UpperCAmelCase					,   cls_token=_UpperCAmelCase					,   mask_token=_UpperCAmelCase					,   additional_special_tokens=_UpperCAmelCase					,   sp_model_kwargs=self.sp_model_kwargs					,   **_UpperCAmelCase					,   )

					snake_case__	     =3

					snake_case__	     =do_lower_case
					snake_case__	     =remove_space
					snake_case__	     =keep_accents
					snake_case__	     =vocab_file

					snake_case__	     =spm.SentencePieceProcessor(**self.sp_model_kwargs						)
					self.sp_model.Load(_UpperCAmelCase						)






			@property
			def 						_lowercase  (     self						)							->							int:
					return len(self.sp_model						)






			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     ={self.convert_ids_to_tokens(_UpperCAmelCase						): i for i in range(self.vocab_size						)}
					vocab.update(self.added_tokens_encoder						)
					return vocab






			def __getstate__(     self						)							->							Union[str, Any]:
					snake_case__	     =self.__dict__.copy()
					snake_case__	     =None
					return state






			def __setstate__(     self					,   _UpperCAmelCase						)							->							Optional[Any]:
					snake_case__	     =d

					# for backward compatibility
					if not hasattr(self					,   'sp_model_kwargs'						):
							snake_case__	     ={}

					snake_case__	     =spm.SentencePieceProcessor(**self.sp_model_kwargs						)
					self.sp_model.Load(self.vocab_file						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Optional[int]:
					if self.remove_space:
							snake_case__	     =' '.join(inputs.strip().split()						)
					else:
							snake_case__	     =inputs
					snake_case__	     =outputs.replace('``'					,   '"'						).replace('\'\''					,   '"'						)

					if not self.keep_accents:
							snake_case__	     =unicodedata.normalize('NFKD'					,   _UpperCAmelCase						)
							snake_case__	     =''.join([c for c in outputs if not unicodedata.combining(_UpperCAmelCase						)]						)
					if self.do_lower_case:
							snake_case__	     =outputs.lower()

					return outputs






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							List[str]:
					snake_case__	     =self.preprocess_text(_UpperCAmelCase						)
					snake_case__	     =self.sp_model.encode(_UpperCAmelCase					,   out_type=_UpperCAmelCase						)
					snake_case__	     =[]
					for piece in pieces:
							if len(_UpperCAmelCase						) > 1 and piece[-1] == str(','						) and piece[-2].isdigit():
									snake_case__	     =self.sp_model.EncodeAsPieces(piece[:-1].replace(_UpperCAmelCase					,   ''						)						)
									if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
											if len(cur_pieces[0]						) == 1:
													snake_case__	     =cur_pieces[1:]
											else:
													snake_case__	     =cur_pieces[0][1:]
									cur_pieces.append(piece[-1]						)
									new_pieces.extend(_UpperCAmelCase						)
							else:
									new_pieces.append(_UpperCAmelCase						)

					return new_pieces






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Dict:
					return self.sp_model.PieceToId(_UpperCAmelCase						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Optional[Any]:
					return self.sp_model.IdToPiece(_UpperCAmelCase						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Dict:
					snake_case__	     =''.join(_UpperCAmelCase						).replace(_UpperCAmelCase					,   ' '						).strip()
					return out_string






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase = False					,   _UpperCAmelCase = None					,   _UpperCAmelCase = True					,   **_UpperCAmelCase					,   )							->							str:
					snake_case__	     =kwargs.pop('use_source_tokenizer'					,   _UpperCAmelCase						)

					snake_case__	     =self.convert_ids_to_tokens(_UpperCAmelCase					,   skip_special_tokens=_UpperCAmelCase						)

					# To avoid mixing byte-level and unicode for byte-level BPT
					# we need to build string separately for added tokens and byte-level tokens
					# cf. https://github.com/huggingface/transformers/issues/1133
					snake_case__	     =[]
					snake_case__	     =[]
					for token in filtered_tokens:
							if skip_special_tokens and token in self.all_special_ids:
									continue
							if token in self.added_tokens_encoder:
									if current_sub_text:
											sub_texts.append(self.convert_tokens_to_string(_UpperCAmelCase						)						)
											snake_case__	     =[]
									sub_texts.append(_UpperCAmelCase						)
							else:
									current_sub_text.append(_UpperCAmelCase						)
					if current_sub_text:
							sub_texts.append(self.convert_tokens_to_string(_UpperCAmelCase						)						)

					# Mimic the behavior of the Rust tokenizer:
					# By default, there are no spaces between special tokens
					snake_case__	     =''.join(_UpperCAmelCase						)

					snake_case__	     =(
					    clean_up_tokenization_spaces
					    if clean_up_tokenization_spaces is not None
					    else self.clean_up_tokenization_spaces
					)
					if clean_up_tokenization_spaces:
							snake_case__	     =self.clean_up_tokenization(_UpperCAmelCase						)
							return clean_text
					else:
							return text






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase = None						)							->							List[int]:
					snake_case__	     =[self.sep_token_id]
					snake_case__	     =[self.cls_token_id]
					if token_ids_a is None:
							return token_ids_a + sep + cls
					return token_ids_a + sep + token_ids_a + sep + cls






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase = None					,   _UpperCAmelCase = False						)							->							List[int]:

					if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=_UpperCAmelCase					,   token_ids_a=_UpperCAmelCase					,   already_has_special_tokens=_UpperCAmelCase						)

					if token_ids_a is not None:
							return ([0] * len(_UpperCAmelCase						)) + [1] + ([0] * len(_UpperCAmelCase						)) + [1, 1]
					return ([0] * len(_UpperCAmelCase						)) + [1, 1]






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase = None						)							->							List[int]:
					snake_case__	     =[self.sep_token_id]
					snake_case__	     =[2]

					if token_ids_a is None:
							return len(token_ids_a + sep						) * [0] + cls_segment_id
					return len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1] + cls_segment_id






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase = None						)							->							Tuple[str]:
					if not os.path.isdir(_UpperCAmelCase						):
							logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
							return
					snake_case__	     =os.path.join(
					    _UpperCAmelCase					,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)

					if os.path.abspath(self.vocab_file						) != os.path.abspath(_UpperCAmelCase						) and os.path.isfile(self.vocab_file						):
							copyfile(self.vocab_file					,   _UpperCAmelCase						)
					elif not os.path.isfile(self.vocab_file						):
							with open(_UpperCAmelCase					,   'wb'						) as fi:
									snake_case__	     =self.sp_model.serialized_model_proto()
									fi.write(_UpperCAmelCase						)

					return (out_vocab_file,)









'''simple docstring'''

import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE__     :						int	=  get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model''')

@require_sentencepiece
@require_tokenizers
class 							a__( snake_case__   ,      unittest.TestCase     ):
			a_	: List[Any]									=      GPTSwaTokenizer
			a_	: int									=      False
			a_	: List[Any]									=      True
			a_	: Any									=      False






			def 						_lowercase  (     self						)							->							str:
					super().setUp()

					# We have a SentencePiece fixture for testing
					snake_case__	     =GPTSwaTokenizer(_UpperCAmelCase					,   eos_token='<unk>'					,   bos_token='<unk>'					,   pad_token='<unk>'						)

					tokenizer.save_pretrained(self.tmpdirname						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Any:
					snake_case__	     ='This is a test'
					snake_case__	     ='This is a test'
					return input_text, output_text






			def 						_lowercase  (     self						)							->							str:
					snake_case__	     ='<s>'
					snake_case__	     =1

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCAmelCase						)					,   _UpperCAmelCase						)
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCAmelCase						)					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =list(self.get_tokenizer().get_vocab().keys()						)

					self.assertEqual(vocab_keys[0]					,   '<unk>'						)
					self.assertEqual(vocab_keys[1]					,   '<s>'						)
					self.assertEqual(vocab_keys[-1]					,   'j'						)
					self.assertEqual(len(_UpperCAmelCase						)					,   2000						)






			def 						_lowercase  (     self						)							->							List[str]:
					self.assertEqual(self.get_tokenizer().vocab_size					,   2000						)






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =GPTSwaTokenizer(_UpperCAmelCase						)

					snake_case__	     =tokenizer.tokenize('This is a test'						)
					self.assertListEqual(_UpperCAmelCase					,   ['▁This', '▁is', '▁a', '▁t', 'est']						)

					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [465, 287, 265, 631, 842]						)

					snake_case__	     =tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
					# fmt: off
					self.assertListEqual(
					    _UpperCAmelCase					,   ['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']					,   )
					# fmt: on

					snake_case__	     =tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)
					self.assertListEqual(
					    _UpperCAmelCase					,   [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]					,   )

					snake_case__	     =tokenizer.convert_ids_to_tokens(_UpperCAmelCase						)
					# fmt: off
					self.assertListEqual(
					    _UpperCAmelCase					,   ['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']						)
					# fmt: on






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =GPTSwaTokenizer(_UpperCAmelCase						)
					snake_case__	     =['This is a test', 'I was born in 92000, and this is falsé.']
					snake_case__	     =[
					    [465, 287, 265, 631, 842],
					    [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
					]

					# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
					for text, expected_ids in zip(_UpperCAmelCase					,   _UpperCAmelCase						):
							self.assertListEqual(tokenizer.encode_fast(_UpperCAmelCase						)					,   _UpperCAmelCase						)

					# Test that decode_fast returns the input text
					for text, token_ids in zip(_UpperCAmelCase					,   _UpperCAmelCase						):
							self.assertEqual(tokenizer.decode_fast(_UpperCAmelCase						)					,   _UpperCAmelCase						)






			@slow
			def 						_lowercase  (     self						)							->							Dict:
					snake_case__	     =[
					    '<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')',
					    'Hey there, how are you doing this fine day?',
					    'This is a text with a trailing spaces followed by a dot     .',
					    'Häj sväjs lillebrör! =)',
					    'Det är inget fel på Mr. Cool',
					]

					# fmt: off
					snake_case__	     ={'input_ids': [[6_3423, 5, 6811, 1_4954, 282, 816, 3821, 6_3466, 6_3425, 6_3462, 18, 6_3978, 678, 301, 1320, 6_3423, 6_3455, 6_3458, 18, 6_3982, 4246, 3940, 1901, 4_7789, 5547, 1_8994], [1_9630, 1100, 6_3446, 1342, 633, 544, 4488, 593, 5102, 2416, 6_3495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 5_8593, 2_2413, 9106, 546, 268, 3_3213, 6_3979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5130, 6_3450, 924, 6_3449, 2249, 4062, 1558, 318, 6_3504, 2_1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 6_3443, 2_6801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
					# fmt: on
					self.tokenizer_integration_test_util(
					    expected_encoding=_UpperCAmelCase					,   model_name='AI-Sweden/gpt-sw3-126m'					,   sequences=_UpperCAmelCase					,   )




