






'''simple docstring'''



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE							=						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE							=						'▁'

SCREAMING_SNAKE_CASE							=						{'vocab_file': 'sentencepiece.bpe.model'}

SCREAMING_SNAKE_CASE							=						{
    'vocab_file': {
        'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model',
    }
}

SCREAMING_SNAKE_CASE							=						{
    'facebook/xglm-564M': 2_048,
}
class UpperCAmelCase_       (     __snake_case					):




      """simple docstring"""




      UpperCamelCase_             =      VOCAB_FILES_NAMES
      UpperCamelCase_             =      PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase_             =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase_             =      ['''input_ids''', '''attention_mask''']






      def __init__(			self      :		Optional[int]  ,   UpperCAmelCase      :		Any  ,   UpperCAmelCase      :		int="<s>"  ,   UpperCAmelCase      :		Optional[Any]="</s>"  ,   UpperCAmelCase      :		List[Any]="</s>"  ,   UpperCAmelCase      :		Optional[Any]="<s>"  ,   UpperCAmelCase      :		List[Any]="<unk>"  ,   UpperCAmelCase      :		Dict="<pad>"  ,   UpperCAmelCase      :		Union[str, Any] = None  ,   **UpperCAmelCase      :		int  ,   )	->	Tuple:


       '''simple docstring'''

       lowercase       : Optional[int]						={} if sp_model_kwargs is None else sp_model_kwargs

       # Compatibility with the original tokenizer
       lowercase       : List[Any]						=7
       lowercase       : Union[str, Any]						=[f'<madeupword{i}>' for i in range(self.num_madeup_words		)]

       lowercase       : int						=kwargs.get('''additional_special_tokens'''  ,   []		)
       kwargs["additional_special_tokens"] += [
           word for word in madeup_words if word not in kwargs["additional_special_tokens"]
       ]

       super().__init__(
           bos_token=__UpperCamelCase  ,   eos_token=__UpperCamelCase  ,   unk_token=__UpperCamelCase  ,   sep_token=__UpperCamelCase  ,   cls_token=__UpperCamelCase  ,   pad_token=__UpperCamelCase  ,   sp_model_kwargs=self.sp_model_kwargs  ,   **__UpperCamelCase  ,   )

       lowercase       : Union[str, Any]						=spm.SentencePieceProcessor(**self.sp_model_kwargs		)
       self.sp_model.Load(str(__UpperCamelCase		)		)
       lowercase       : List[Any]						=vocab_file

       # Original fairseq vocab and spm vocab must be "aligned":
       # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
       # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
       # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
       # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

       # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
       lowercase       : Optional[int]						=1

       # Mimic fairseq token-to-id alignment for the first 4 token
       lowercase       : Optional[Any]						={'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

       lowercase       : Optional[int]						=len(self.sp_model		)
       lowercase       : List[Any]						={f'<madeupword{i}>': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words		)}
       self.fairseq_tokens_to_ids.update(__UpperCamelCase		)

       lowercase       : List[str]						={v: k for k, v in self.fairseq_tokens_to_ids.items()}






      def __getstate__(			self      :		Optional[int]		)	->	int:


       '''simple docstring'''

       lowercase       : Optional[int]						=self.__dict__.copy()
       lowercase       : List[Any]						=None
       lowercase       : Union[str, Any]						=self.sp_model.serialized_model_proto()
       return state






      def __setstate__(			self      :		List[str]  ,   UpperCAmelCase      :		str		)	->	Any:


       '''simple docstring'''

       lowercase       : List[Any]						=d

       # for backward compatibility
       if not hasattr(self  ,   '''sp_model_kwargs'''		):
        lowercase       : Union[str, Any]						={}

       lowercase       : Union[str, Any]						=spm.SentencePieceProcessor(**self.sp_model_kwargs		)
       self.sp_model.LoadFromSerializedProto(self.sp_model_proto		)






      def 					A__   (			self      :		List[Any]  ,   UpperCAmelCase      :		Optional[int]  ,   UpperCAmelCase      :		List[str] = None		)	->	Optional[int]:


       '''simple docstring'''

       if token_ids_a is None:
        return [self.sep_token_id] + token_ids_a
       lowercase       : Any						=[self.sep_token_id]
       return sep + token_ids_a + sep + sep + token_ids_a






      def 					A__   (			self      :		Union[str, Any]  ,   UpperCAmelCase      :		Any  ,   UpperCAmelCase      :		int = None  ,   UpperCAmelCase      :		str = False		)	->	Any:


       '''simple docstring'''

       if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=__UpperCamelCase  ,   token_ids_a=__UpperCamelCase  ,   already_has_special_tokens=__UpperCamelCase		)

       if token_ids_a is None:
        return [1] + ([0] * len(__UpperCamelCase		))
       return [1] + ([0] * len(__UpperCamelCase		)) + [1, 1] + ([0] * len(__UpperCamelCase		))






      def 					A__   (			self      :		List[str]  ,   UpperCAmelCase      :		Union[str, Any]  ,   UpperCAmelCase      :		int = None		)	->	Any:


       '''simple docstring'''

       lowercase       : List[str]						=[self.sep_token_id]

       if token_ids_a is None:
        return len(sep + token_ids_a		) * [0]
       return len(sep + token_ids_a + sep + sep + token_ids_a		) * [0]






      @property
      def 					A__   (			self      :		Optional[Any]		)	->	Optional[Any]:


       '''simple docstring'''

       return len(self.sp_model		) + self.fairseq_offset + self.num_madeup_words






      def 					A__   (			self      :		Tuple		)	->	Optional[int]:


       '''simple docstring'''

       lowercase       : List[Any]						={self.convert_ids_to_tokens(__UpperCamelCase		): i for i in range(self.vocab_size		)}
       vocab.update(self.added_tokens_encoder		)
       return vocab






      def 					A__   (			self      :		int  ,   UpperCAmelCase      :		Optional[int]		)	->	List[Any]:


       '''simple docstring'''

       return self.sp_model.encode(__UpperCamelCase  ,   out_type=__UpperCamelCase		)






      def 					A__   (			self      :		List[Any]  ,   UpperCAmelCase      :		Optional[int]		)	->	List[Any]:


       '''simple docstring'''

       if token in self.fairseq_tokens_to_ids:
        return self.fairseq_tokens_to_ids[token]
       lowercase       : str						=self.sp_model.PieceToId(__UpperCamelCase		)

       # Need to return unknown token if the SP model returned 0
       return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






      def 					A__   (			self      :		str  ,   UpperCAmelCase      :		Any		)	->	Tuple:


       '''simple docstring'''

       if index in self.fairseq_ids_to_tokens:
        return self.fairseq_ids_to_tokens[index]
       return self.sp_model.IdToPiece(index - self.fairseq_offset		)






      def 					A__   (			self      :		Dict  ,   UpperCAmelCase      :		List[str]		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : List[str]						=''''''.join(__UpperCamelCase		).replace(__UpperCamelCase  ,   ''' '''		).strip()
       return out_string






      def 					A__   (			self      :		Optional[int]  ,   UpperCAmelCase      :		Dict  ,   UpperCAmelCase      :		List[Any] = None		)	->	Optional[int]:


       '''simple docstring'''

       if not os.path.isdir(__UpperCamelCase		):
        logger.error(f'Vocabulary path ({save_directory}) should be a directory'		)
        return
       lowercase       : Optional[int]						=os.path.join(
           __UpperCamelCase  ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

       if os.path.abspath(self.vocab_file		) != os.path.abspath(__UpperCamelCase		) and os.path.isfile(self.vocab_file		):
        copyfile(self.vocab_file  ,   __UpperCamelCase		)
       elif not os.path.isfile(self.vocab_file		):
        with open(__UpperCamelCase  ,   '''wb'''		) as fi:
         lowercase       : List[Any]						=self.sp_model.serialized_model_proto()
         fi.write(__UpperCamelCase		)

       return (out_vocab_file,)






import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class SCREAMING_SNAKE_CASE	(		__snake_case				,    unittest.TestCase				):






			"""simple docstring"""




			__A    	=	None
			__A    	=	BloomTokenizerFast
			__A    	=	BloomTokenizerFast
			__A    	=	True
			__A    	=	False
			__A    	=	"""tokenizer_file"""
			__A    	=	{"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				super().setUp()
				snake_case_   		=			BloomTokenizerFast.from_pretrained('bigscience/tokenizer'				)
				tokenizer.save_pretrained(self.tmpdirname				)


			def       __lowerCAmelCase			(     self			,					**__UpperCamelCase				):

				"""simple docstring"""


				kwargs.update(self.special_tokens_map				)
				return BloomTokenizerFast.from_pretrained(self.tmpdirname			,					**__UpperCamelCase				)


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			self.get_rust_tokenizer()

				snake_case_   		=			['The quick brown fox</s>', 'jumps over the lazy dog</s>']
				snake_case_   		=			[[21_75, 2_37_14, 7_31_73, 14_42_52, 2], [77, 13_26_19, 34_78, 3_68, 10_95_86, 3_54_33, 2]]

				snake_case_   		=			tokenizer.batch_encode_plus(__UpperCamelCase				)['input_ids']
				self.assertListEqual(__UpperCamelCase			,					__UpperCamelCase				)

				snake_case_   		=			tokenizer.batch_decode(__UpperCamelCase				)
				self.assertListEqual(__UpperCamelCase			,					__UpperCamelCase				)


			def       __lowerCAmelCase			(     self			,					__UpperCamelCase=6				):

				"""simple docstring"""


				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
						snake_case_   		=			self.rust_tokenizer_class.from_pretrained(__UpperCamelCase			,					**__UpperCamelCase				)
						# tokenizer_r.pad_token = None # Hotfixing padding = None
						# Simple input
						snake_case_   		=			'This is a simple input'
						snake_case_   		=			['This is a simple input 1', 'This is a simple input 2']
						snake_case_   		=			('This is a simple input', 'This is a pair')
						snake_case_   		=			[
						    ('This is a simple input 1', 'This is a simple input 2'),
						    ('This is a simple pair 1', 'This is a simple pair 2'),
						]

						# Simple input tests
						try:
							tokenizer_r.encode(__UpperCamelCase			,					max_length=__UpperCamelCase				)
							tokenizer_r.encode_plus(__UpperCamelCase			,					max_length=__UpperCamelCase				)

							tokenizer_r.batch_encode_plus(__UpperCamelCase			,					max_length=__UpperCamelCase				)
							tokenizer_r.encode(__UpperCamelCase			,					max_length=__UpperCamelCase				)
							tokenizer_r.batch_encode_plus(__UpperCamelCase			,					max_length=__UpperCamelCase				)
						except ValueError:
							self.fail('Bloom Tokenizer should be able to deal with padding'				)

						snake_case_   		=			None  # Hotfixing padding = None
						self.assertRaises(__UpperCamelCase			,					tokenizer_r.encode			,					__UpperCamelCase			,					max_length=__UpperCamelCase			,					padding='max_length'				)

						# Simple input
						self.assertRaises(__UpperCamelCase			,					tokenizer_r.encode_plus			,					__UpperCamelCase			,					max_length=__UpperCamelCase			,					padding='max_length'				)

						# Simple input
						self.assertRaises(
						    __UpperCamelCase			,					tokenizer_r.batch_encode_plus			,					__UpperCamelCase			,					max_length=__UpperCamelCase			,					padding='max_length'			,					)

						# Pair input
						self.assertRaises(__UpperCamelCase			,					tokenizer_r.encode			,					__UpperCamelCase			,					max_length=__UpperCamelCase			,					padding='max_length'				)

						# Pair input
						self.assertRaises(__UpperCamelCase			,					tokenizer_r.encode_plus			,					__UpperCamelCase			,					max_length=__UpperCamelCase			,					padding='max_length'				)

						# Pair input
						self.assertRaises(
						    __UpperCamelCase			,					tokenizer_r.batch_encode_plus			,					__UpperCamelCase			,					max_length=__UpperCamelCase			,					padding='max_length'			,					)


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			self.get_rust_tokenizer()
				snake_case_   		=			load_dataset('xnli'			,					'all_languages'			,					split='test'			,					streaming=__UpperCamelCase				)

				snake_case_   		=			next(iter(__UpperCamelCase				)				)['premise']  # pick up one data
				snake_case_   		=			list(sample_data.values()				)

				snake_case_   		=			list(map(tokenizer.encode			,					__UpperCamelCase				)				)
				snake_case_   		=			[tokenizer.decode(__UpperCamelCase			,					clean_up_tokenization_spaces=__UpperCamelCase				) for x in output_tokens]
				self.assertListEqual(__UpperCamelCase			,					__UpperCamelCase				)







			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map				)			,					1				)
				self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()				)[0]				)			,					1				)
