

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


a__												=						logging.get_logger(__name__)

a__												=						{'''vocab_file''': '''spiece.model'''}

a__												=						{
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

a__												=						{
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
a__												=						0
a__												=						1
a__												=						2
a__												=						3
a__												=						4

class 							UpperCAmelCase_						(     __lowercase  ):
							"""simple docstring"""







							UpperCAmelCase__			:		Tuple             =       VOCAB_FILES_NAMES
							UpperCAmelCase__			:		Union[str, Any]             =       PRETRAINED_VOCAB_FILES_MAP
							UpperCAmelCase__			:		Optional[int]             =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCAmelCase__			:		List[Any]             =       "left"


							def __init__(     self			,				_a			,				_a=False			,				_a=True			,				_a=False			,				_a="<s>"			,				_a="</s>"			,				_a="<unk>"			,				_a="<sep>"			,				_a="<pad>"			,				_a="<cls>"			,				_a="<mask>"			,				_a=["<eop>", "<eod>"]			,				_a = None			,				**_a			,				)       ->       None:
												# Mask token behave like a normal word, i.e. include the space before it
												_a      :       Optional[int]   =						AddedToken(_a			,				lstrip=_a			,				rstrip=_a			) if isinstance(_a			,				_a			) else mask_token

												_a      :       List[Any]   =						{} if sp_model_kwargs is None else sp_model_kwargs

												super().__init__(
												    do_lower_case=_a			,				remove_space=_a			,				keep_accents=_a			,				bos_token=_a			,				eos_token=_a			,				unk_token=_a			,				sep_token=_a			,				pad_token=_a			,				cls_token=_a			,				mask_token=_a			,				additional_special_tokens=_a			,				sp_model_kwargs=self.sp_model_kwargs			,				**_a			,				)

												_a      :       List[Any]   =						3

												_a      :       List[str]   =						do_lower_case
												_a      :       int   =						remove_space
												_a      :       Optional[Any]   =						keep_accents
												_a      :       List[Any]   =						vocab_file

												_a      :       Union[str, Any]   =						spm.SentencePieceProcessor(**self.sp_model_kwargs			)
												self.sp_model.Load(_a			)


							@property
							def 							__lowercase							(     self			)       ->       int:
												return len(self.sp_model			)


							def 							__lowercase							(     self			)       ->       List[Any]:
												_a      :       str   =						{self.convert_ids_to_tokens(_a			): i for i in range(self.vocab_size			)}
												vocab.update(self.added_tokens_encoder			)
												return vocab


							def __getstate__(     self			)       ->       Any:
												_a      :       Any   =						self.__dict__.copy()
												_a      :       Optional[Any]   =						None
												return state


							def __setstate__(     self			,				_a			)       ->       Optional[Any]:
												_a      :       Tuple   =						d

												# for backward compatibility
												if not hasattr(self			,				'''sp_model_kwargs'''			):
																	_a      :       Any   =						{}

												_a      :       Tuple   =						spm.SentencePieceProcessor(**self.sp_model_kwargs			)
												self.sp_model.Load(self.vocab_file			)


							def 							__lowercase							(     self			,				_a			)       ->       int:
												if self.remove_space:
																	_a      :       Union[str, Any]   =						''' '''.join(inputs.strip().split()			)
												else:
																	_a      :       Tuple   =						inputs
												_a      :       Dict   =						outputs.replace('''``'''			,				'''"'''			).replace('''\'\''''			,				'''"'''			)

												if not self.keep_accents:
																	_a      :       Tuple   =						unicodedata.normalize('''NFKD'''			,				_a			)
																	_a      :       Tuple   =						''''''.join([c for c in outputs if not unicodedata.combining(_a			)]			)
												if self.do_lower_case:
																	_a      :       int   =						outputs.lower()

												return outputs


							def 							__lowercase							(     self			,				_a			)       ->       List[str]:
												_a      :       int   =						self.preprocess_text(_a			)
												_a      :       int   =						self.sp_model.encode(_a			,				out_type=_a			)
												_a      :       List[str]   =						[]
												for piece in pieces:
																	if len(_a			) > 1 and piece[-1] == str(''','''			) and piece[-2].isdigit():
																						_a      :       Optional[int]   =						self.sp_model.EncodeAsPieces(piece[:-1].replace(_a			,				''''''			)			)
																						if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																											if len(cur_pieces[0]			) == 1:
																																_a      :       Tuple   =						cur_pieces[1:]
																											else:
																																_a      :       List[Any]   =						cur_pieces[0][1:]
																						cur_pieces.append(piece[-1]			)
																						new_pieces.extend(_a			)
																	else:
																						new_pieces.append(_a			)

												return new_pieces


							def 							__lowercase							(     self			,				_a			)       ->       Union[str, Any]:
												return self.sp_model.PieceToId(_a			)


							def 							__lowercase							(     self			,				_a			)       ->       Optional[Any]:
												return self.sp_model.IdToPiece(_a			)


							def 							__lowercase							(     self			,				_a			)       ->       Any:
												_a      :       Optional[Any]   =						''''''.join(_a			).replace(_a			,				''' '''			).strip()
												return out_string


							def 							__lowercase							(     self			,				_a			,				_a = False			,				_a = None			,				_a = True			,				**_a			,				)       ->       str:
												_a      :       int   =						kwargs.pop('''use_source_tokenizer'''			,				_a			)

												_a      :       str   =						self.convert_ids_to_tokens(_a			,				skip_special_tokens=_a			)

												# To avoid mixing byte-level and unicode for byte-level BPT
												# we need to build string separately for added tokens and byte-level tokens
												# cf. https://github.com/huggingface/transformers/issues/1133
												_a      :       int   =						[]
												_a      :       str   =						[]
												for token in filtered_tokens:
																	if skip_special_tokens and token in self.all_special_ids:
																						continue
																	if token in self.added_tokens_encoder:
																						if current_sub_text:
																											sub_texts.append(self.convert_tokens_to_string(_a			)			)
																											_a      :       str   =						[]
																						sub_texts.append(_a			)
																	else:
																						current_sub_text.append(_a			)
												if current_sub_text:
																	sub_texts.append(self.convert_tokens_to_string(_a			)			)

												# Mimic the behavior of the Rust tokenizer:
												# By default, there are no spaces between special tokens
												_a      :       Any   =						''''''.join(_a			)

												_a      :       List[str]   =						(
												    clean_up_tokenization_spaces
												    if clean_up_tokenization_spaces is not None
												    else self.clean_up_tokenization_spaces
												)
												if clean_up_tokenization_spaces:
																	_a      :       str   =						self.clean_up_tokenization(_a			)
																	return clean_text
												else:
																	return text


							def 							__lowercase							(     self			,				_a			,				_a = None			)       ->       List[int]:
												_a      :       Optional[Any]   =						[self.sep_token_id]
												_a      :       Union[str, Any]   =						[self.cls_token_id]
												if token_ids_a is None:
																	return token_ids_a + sep + cls
												return token_ids_a + sep + token_ids_a + sep + cls


							def 							__lowercase							(     self			,				_a			,				_a = None			,				_a = False			)       ->       List[int]:

												if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=_a			,				token_ids_a=_a			,				already_has_special_tokens=_a			)

												if token_ids_a is not None:
																	return ([0] * len(_a			)) + [1] + ([0] * len(_a			)) + [1, 1]
												return ([0] * len(_a			)) + [1, 1]


							def 							__lowercase							(     self			,				_a			,				_a = None			)       ->       List[int]:
												_a      :       Optional[int]   =						[self.sep_token_id]
												_a      :       Optional[int]   =						[2]

												if token_ids_a is None:
																	return len(token_ids_a + sep			) * [0] + cls_segment_id
												return len(token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1] + cls_segment_id




							def 							__lowercase							(     self			,				_a			,				_a = None			)       ->       Tuple[str]:
												if not os.path.isdir(_a			):
																	logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""			)
																	return
												_a      :       Optional[int]   =						os.path.join(
												    _a			,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)

												if os.path.abspath(self.vocab_file			) != os.path.abspath(_a			) and os.path.isfile(self.vocab_file			):
																	copyfile(self.vocab_file			,				_a			)
												elif not os.path.isfile(self.vocab_file			):
																	with open(_a			,				'''wb'''			) as fi:
																						_a      :       Any   =						self.sp_model.serialized_model_proto()
																						fi.write(_a			)

												return (out_vocab_file,)





"""simple docstring"""




import unittest

from transformers import is_vision_available
from transformers.pipelines import pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
  from PIL import Image
else:


  class       __magic_name__							:
         @staticmethod
         def 				lowercase_				(  *A_			,					**A_							)  ->						Optional[int]:





           """simple docstring"""




           pass


@is_pipeline_test
@require_vision
class       __magic_name__							(							unittest.TestCase			):
       @require_torch
       def 				lowercase_				(  self							)  ->						int:





         """simple docstring"""




         _lowercase:      Optional[Any]   			=  pipeline(
             model='''hf-internal-testing/tiny-random-clip-zero-shot-image-classification'''			,					)
         _lowercase:      List[Any]   			=  Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''							)
         _lowercase:      str   			=  image_classifier(A_			,					candidate_labels=['''a''', '''b''', '''c''']							)

         # The floating scores are so close, we enter floating error approximation and the order is not guaranteed across
         # python and torch versions.
         self.assertIn(
             nested_simplify(A_							)			,					[
                 [{'''score''': 0.3_33, '''label''': '''a'''}, {'''score''': 0.3_33, '''label''': '''b'''}, {'''score''': 0.3_33, '''label''': '''c'''}],
                 [{'''score''': 0.3_33, '''label''': '''a'''}, {'''score''': 0.3_33, '''label''': '''c'''}, {'''score''': 0.3_33, '''label''': '''b'''}],
             ]			,					)

         _lowercase:      Any   			=  image_classifier([image] * 5			,					candidate_labels=['''A''', '''B''', '''C''']			,					batch_size=2							)
         self.assertEqual(
             nested_simplify(A_							)			,					[
                 [
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                 ],
                 [
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                 ],
                 [
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                 ],
                 [
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                 ],
                 [
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                 ],
             ]			,					)
       @require_tf
       def 				lowercase_				(  self							)  ->						int:





         """simple docstring"""




         _lowercase:      Union[str, Any]   			=  pipeline(
             model='''hf-internal-testing/tiny-random-clip-zero-shot-image-classification'''			,					framework='''tf'''							)
         _lowercase:      List[str]   			=  Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''							)
         _lowercase:      Union[str, Any]   			=  image_classifier(A_			,					candidate_labels=['''a''', '''b''', '''c''']							)

         self.assertEqual(
             nested_simplify(A_							)			,					[{'''score''': 0.3_33, '''label''': '''a'''}, {'''score''': 0.3_33, '''label''': '''b'''}, {'''score''': 0.3_33, '''label''': '''c'''}]			,					)

         _lowercase:      List[str]   			=  image_classifier([image] * 5			,					candidate_labels=['''A''', '''B''', '''C''']			,					batch_size=2							)
         self.assertEqual(
             nested_simplify(A_							)			,					[
                 [
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                 ],
                 [
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                 ],
                 [
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                 ],
                 [
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                 ],
                 [
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                     {'''score''': 0.3_33, '''label''': ANY(A_							)},
                 ],
             ]			,					)
       @slow
       @require_torch
       def 				lowercase_				(  self							)  ->						Any:





         """simple docstring"""




         _lowercase:      Tuple   			=  pipeline(
             task='''zero-shot-image-classification'''			,					model='''openai/clip-vit-base-patch32'''			,					)
         # This is an image of 2 cats with remotes and no planes
         _lowercase:      str   			=  Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''							)
         _lowercase:      Optional[Any]   			=  image_classifier(A_			,					candidate_labels=['''cat''', '''plane''', '''remote''']							)

         self.assertEqual(
             nested_simplify(A_							)			,					[
                 {'''score''': 0.5_11, '''label''': '''remote'''},
                 {'''score''': 0.4_85, '''label''': '''cat'''},
                 {'''score''': 0.0_04, '''label''': '''plane'''},
             ]			,					)

         _lowercase:      Optional[Any]   			=  image_classifier([image] * 5			,					candidate_labels=['''cat''', '''plane''', '''remote''']			,					batch_size=2							)
         self.assertEqual(
             nested_simplify(A_							)			,					[
                 [
                     {'''score''': 0.5_11, '''label''': '''remote'''},
                     {'''score''': 0.4_85, '''label''': '''cat'''},
                     {'''score''': 0.0_04, '''label''': '''plane'''},
                 ],
             ]
             * 5			,					)



       @slow
       @require_tf
       def 				lowercase_				(  self							)  ->						Optional[Any]:





         """simple docstring"""




         _lowercase:      Tuple   			=  pipeline(
             task='''zero-shot-image-classification'''			,					model='''openai/clip-vit-base-patch32'''			,					framework='''tf'''							)
         # This is an image of 2 cats with remotes and no planes
         _lowercase:      str   			=  Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''							)
         _lowercase:      Optional[int]   			=  image_classifier(A_			,					candidate_labels=['''cat''', '''plane''', '''remote''']							)
         self.assertEqual(
             nested_simplify(A_							)			,					[
                 {'''score''': 0.5_11, '''label''': '''remote'''},
                 {'''score''': 0.4_85, '''label''': '''cat'''},
                 {'''score''': 0.0_04, '''label''': '''plane'''},
             ]			,					)

         _lowercase:      Optional[Any]   			=  image_classifier([image] * 5			,					candidate_labels=['''cat''', '''plane''', '''remote''']			,					batch_size=2							)
         self.assertEqual(
             nested_simplify(A_							)			,					[
                 [
                     {'''score''': 0.5_11, '''label''': '''remote'''},
                     {'''score''': 0.4_85, '''label''': '''cat'''},
                     {'''score''': 0.0_04, '''label''': '''plane'''},
                 ],
             ]
             * 5			,					)
