


import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__       :	Optional[Any]     				=logging.get_logger(__name__)
lowerCAmelCase__       :	Optional[Any]     				={'vocab_file': 'spiece.model'}

lowerCAmelCase__       :	Tuple     				={
    'vocab_file': {
        'google/fnet-base': 'https://huggingface.co/google/fnet-base/resolve/main/spiece.model',
        'google/fnet-large': 'https://huggingface.co/google/fnet-large/resolve/main/spiece.model',
    },
}

lowerCAmelCase__       :	Optional[int]     				={
    'google/fnet-base': 5_12,
    'google/fnet-large': 5_12,
}

lowerCAmelCase__       :	List[Any]     				='▁'







class      __lowercase  (__SCREAMING_SNAKE_CASE   ):





							"""simple docstring"""







							_UpperCAmelCase     =		VOCAB_FILES_NAMES
							_UpperCAmelCase     =		PRETRAINED_VOCAB_FILES_MAP
							_UpperCAmelCase     =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_UpperCAmelCase     =		["""input_ids""", """token_type_ids"""]






							def __init__(	self       ,		lowerCAmelCase__       ,		lowerCAmelCase__=False       ,		lowerCAmelCase__=True       ,		lowerCAmelCase__=True       ,		lowerCAmelCase__="<unk>"       ,		lowerCAmelCase__="[SEP]"       ,		lowerCAmelCase__="<pad>"       ,		lowerCAmelCase__="[CLS]"       ,		lowerCAmelCase__="[MASK]"       ,		lowerCAmelCase__ = None       ,		**lowerCAmelCase__       ,		):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Any       						= (
										    AddedToken(lowerCAmelCase__       ,		lstrip=lowerCAmelCase__       ,		rstrip=lowerCAmelCase__       ,		normalized=lowerCAmelCase__    )
										    if isinstance(lowerCAmelCase__       ,		lowerCAmelCase__    )
										    else mask_token
										)

										SCREAMING_SNAKE_CASE_						:			Tuple       						= {} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    do_lower_case=lowerCAmelCase__       ,		remove_space=lowerCAmelCase__       ,		keep_accents=lowerCAmelCase__       ,		unk_token=lowerCAmelCase__       ,		sep_token=lowerCAmelCase__       ,		pad_token=lowerCAmelCase__       ,		cls_token=lowerCAmelCase__       ,		mask_token=lowerCAmelCase__       ,		sp_model_kwargs=self.sp_model_kwargs       ,		**lowerCAmelCase__       ,		)

										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= do_lower_case
										SCREAMING_SNAKE_CASE_						:			List[str]       						= remove_space
										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= keep_accents
										SCREAMING_SNAKE_CASE_						:			List[str]       						= vocab_file

										SCREAMING_SNAKE_CASE_						:			int       						= spm.SentencePieceProcessor(**self.sp_model_kwargs    )
										self.sp_model.Load(lowerCAmelCase__    )






							@property
							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										return len(self.sp_model    )






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			str       						= {self.convert_ids_to_tokens(lowerCAmelCase__    ): i for i in range(self.vocab_size    )}
										vocab.update(self.added_tokens_encoder    )
										return vocab






							def __getstate__(	self    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= self.__dict__.copy()
										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= None
										return state






							def __setstate__(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= d

										# for backward compatibility
										if not hasattr(self       ,		'sp_model_kwargs'    ):
													SCREAMING_SNAKE_CASE_						:			int       						= {}

										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= spm.SentencePieceProcessor(**self.sp_model_kwargs    )
										self.sp_model.Load(self.vocab_file    )






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										if self.remove_space:
													SCREAMING_SNAKE_CASE_						:			Optional[int]       						= ' '.join(inputs.strip().split()    )
										else:
													SCREAMING_SNAKE_CASE_						:			Optional[int]       						= inputs
										SCREAMING_SNAKE_CASE_						:			List[str]       						= outputs.replace('``'       ,		'"'    ).replace('\'\''       ,		'"'    )

										if not self.keep_accents:
													SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= unicodedata.normalize('NFKD'       ,		lowerCAmelCase__    )
													SCREAMING_SNAKE_CASE_						:			Any       						= ''.join([c for c in outputs if not unicodedata.combining(lowerCAmelCase__    )]    )
										if self.do_lower_case:
													SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= outputs.lower()

										return outputs






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			str       						= self.preprocess_text(lowerCAmelCase__    )
										SCREAMING_SNAKE_CASE_						:			str       						= self.sp_model.encode(lowerCAmelCase__       ,		out_type=lowerCAmelCase__    )
										SCREAMING_SNAKE_CASE_						:			List[Any]       						= []
										for piece in pieces:
													if len(lowerCAmelCase__    ) > 1 and piece[-1] == str(','    ) and piece[-2].isdigit():
																SCREAMING_SNAKE_CASE_						:			Optional[int]       						= self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase__       ,		''    )    )
																if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																			if len(cur_pieces[0]    ) == 1:
																						SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= cur_pieces[1:]
																			else:
																						SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= cur_pieces[0][1:]
																cur_pieces.append(piece[-1]    )
																new_pieces.extend(lowerCAmelCase__    )
													else:
																new_pieces.append(lowerCAmelCase__    )

										return new_pieces






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										return self.sp_model.PieceToId(lowerCAmelCase__    )






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										return self.sp_model.IdToPiece(lowerCAmelCase__    )






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Dict       						= []
										SCREAMING_SNAKE_CASE_						:			List[str]       						= ''
										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= False
										for token in tokens:
													# make sure that special tokens are not decoded using sentencepiece model
													if token in self.all_special_tokens:
																if not prev_is_special:
																			out_string += " "
																out_string += self.sp_model.decode(lowerCAmelCase__    ) + token
																SCREAMING_SNAKE_CASE_						:			List[str]       						= True
																SCREAMING_SNAKE_CASE_						:			List[Any]       						= []
													else:
																current_sub_tokens.append(lowerCAmelCase__    )
																SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= False
										out_string += self.sp_model.decode(lowerCAmelCase__    )
										return out_string.strip()






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__       ,		lowerCAmelCase__ = False       ,		lowerCAmelCase__ = None       ,		lowerCAmelCase__ = True       ,		**lowerCAmelCase__       ,		):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			List[Any]       						= kwargs.pop('use_source_tokenizer'       ,		lowerCAmelCase__    )

										SCREAMING_SNAKE_CASE_						:			List[Any]       						= self.convert_ids_to_tokens(lowerCAmelCase__       ,		skip_special_tokens=lowerCAmelCase__    )

										# To avoid mixing byte-level and unicode for byte-level BPT
										# we need to build string separately for added tokens and byte-level tokens
										# cf. https://github.com/huggingface/transformers/issues/1133
										SCREAMING_SNAKE_CASE_						:			Any       						= []
										SCREAMING_SNAKE_CASE_						:			str       						= []
										for token in filtered_tokens:
													if skip_special_tokens and token in self.all_special_ids:
																continue
													if token in self.added_tokens_encoder:
																if current_sub_text:
																			sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__    )    )
																			SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= []
																sub_texts.append(lowerCAmelCase__    )
													else:
																current_sub_text.append(lowerCAmelCase__    )
										if current_sub_text:
													sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__    )    )

										# Mimic the behavior of the Rust tokenizer:
										# No space after <unk>
										if spaces_between_special_tokens:
													SCREAMING_SNAKE_CASE_						:			List[Any]       						= re.sub(r'(<unk>) '       ,		r'\1'       ,		' '.join(lowerCAmelCase__    )    )
										else:
													SCREAMING_SNAKE_CASE_						:			List[Any]       						= ''.join(lowerCAmelCase__    )

										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= (
										    clean_up_tokenization_spaces
										    if clean_up_tokenization_spaces is not None
										    else self.clean_up_tokenization_spaces
										)
										if clean_up_tokenization_spaces:
													SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= self.clean_up_tokenization(lowerCAmelCase__    )
													return clean_text
										else:
													return text






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__       ,		lowerCAmelCase__ = None    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			str       						= [self.sep_token_id]
										SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= [self.cls_token_id]
										if token_ids_a is None:
													return cls + token_ids_a + sep
										return cls + token_ids_a + sep + token_ids_a + sep






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__       ,		lowerCAmelCase__ = None       ,		lowerCAmelCase__ = False    ):





										"""simple docstring"""




										if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=lowerCAmelCase__       ,		token_ids_a=lowerCAmelCase__       ,		already_has_special_tokens=lowerCAmelCase__    )

										if token_ids_a is not None:
													return [1] + ([0] * len(lowerCAmelCase__    )) + [1] + ([0] * len(lowerCAmelCase__    )) + [1]
										return [1] + ([0] * len(lowerCAmelCase__    )) + [1]






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__       ,		lowerCAmelCase__ = None    ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= [self.sep_token_id]
										SCREAMING_SNAKE_CASE_						:			Any       						= [self.cls_token_id]

										if token_ids_a is None:
													return len(cls + token_ids_a + sep    ) * [0]
										return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__       ,		lowerCAmelCase__ = None    ):





										"""simple docstring"""




										if not os.path.isdir(lowerCAmelCase__    ):
													logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
													return
										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= os.path.join(
										    lowerCAmelCase__       ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )

										if os.path.abspath(self.vocab_file    ) != os.path.abspath(lowerCAmelCase__    ) and os.path.isfile(self.vocab_file    ):
													copyfile(self.vocab_file       ,		lowerCAmelCase__    )
										elif not os.path.isfile(self.vocab_file    ):
													with open(lowerCAmelCase__       ,		'wb'    ) as fi:
																SCREAMING_SNAKE_CASE_						:			Tuple       						= self.sp_model.serialized_model_proto()
																fi.write(lowerCAmelCase__    )

										return (out_vocab_file,)











'''simple docstring'''


import unittest
from pathlib import Path

from transformers import is_vision_available, load_tool
from transformers.testing_utils import get_tests_dir

from .test_tools_common import ToolTesterMixin


if is_vision_available():
				from PIL import Image

class 			A    (		unittest.TestCase     ,_a    ):







							def    __lowerCAmelCase			(  self       :  Optional[int]					)			->							Optional[int]:


														"""simple docstring"""


														_a	     =				load_tool('''image-question-answering'''					)
														self.tool.setup()
														_a	     =				load_tool('''image-question-answering'''							,			remote=lowerCAmelCase_					)







							def    __lowerCAmelCase			(  self       :  Tuple					)			->							Optional[int]:


														"""simple docstring"""


														_a	     =				Image.open(Path(get_tests_dir('''fixtures/tests_samples/COCO'''					)					) / '''000000039769.png'''					)
														_a	     =				self.tool(lowerCAmelCase_							,			'''How many cats are sleeping on the couch?'''					)
														self.assertEqual(lowerCAmelCase_							,			'''2'''					)







							def    __lowerCAmelCase			(  self       :  List[str]					)			->							Union[str, Any]:


														"""simple docstring"""


														_a	     =				Image.open(Path(get_tests_dir('''fixtures/tests_samples/COCO'''					)					) / '''000000039769.png'''					)
														_a	     =				self.remote_tool(lowerCAmelCase_							,			'''How many cats are sleeping on the couch?'''					)
														self.assertEqual(lowerCAmelCase_							,			'''2'''					)







							def    __lowerCAmelCase			(  self       :  Dict					)			->							Tuple:


														"""simple docstring"""


														_a	     =				Image.open(Path(get_tests_dir('''fixtures/tests_samples/COCO'''					)					) / '''000000039769.png'''					)
														_a	     =				self.tool(image=lowerCAmelCase_							,			question='''How many cats are sleeping on the couch?'''					)
														self.assertEqual(lowerCAmelCase_							,			'''2'''					)







							def    __lowerCAmelCase			(  self       :  Tuple					)			->							Dict:


														"""simple docstring"""


														_a	     =				Image.open(Path(get_tests_dir('''fixtures/tests_samples/COCO'''					)					) / '''000000039769.png'''					)
														_a	     =				self.remote_tool(image=lowerCAmelCase_							,			question='''How many cats are sleeping on the couch?'''					)
														self.assertEqual(lowerCAmelCase_							,			'''2'''					)

