



'''simple docstring'''

import os
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen, xsplitext
from ..table import array_cast
from ..utils.py_utils import no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
	from .features import FeatureType


a   ,      a   ,      a							   =				False, False, False






@dataclass
class 						a_		:
							UpperCAmelCase			:		Optional[int]        =				None
							UpperCAmelCase			:		bool        =				True
							UpperCAmelCase			:		bool        =				True
							UpperCAmelCase			:		Optional[str]        =				None
							# Automatically constructed
							UpperCAmelCase			:		ClassVar[str]        =				"dict"
							UpperCAmelCase			:		ClassVar[Any]        =				pa.struct({"""bytes""": pa.binary(), """path""": pa.string()}    )
							UpperCAmelCase			:		str        =				field(default="""Audio"""    ,   init=snake_case    ,   repr=snake_case    )


							def __call__(				self   :						Optional[Any]			)			->					Tuple:
										return self.pa_type


							def    UpperCamelCase     (				self   :						Union[str, Any]      ,						a_   :						Union[str, bytes, dict]			)			->					dict:
										try:
													import soundfile as sf  # soundfile is a dependency of librosa, needed to decode audio files.
										except ImportError as err:
													raise ImportError('To support encoding audio data, please install \'soundfile\'.'			) from err
										if isinstance(a_      ,						a_			):
													return {"bytes": None, "path": value}
										elif isinstance(a_      ,						a_			):
													return {"bytes": value, "path": None}
										elif "array" in value:
													# convert the audio array to wav bytes
													snake_case:    str      							=BytesIO()
													sf.write(a_      ,						value['array']      ,						value['sampling_rate']      ,						format='wav'			)
													return {"bytes": buffer.getvalue(), "path": None}
										elif value.get('path'			) is not None and os.path.isfile(value['path']			):
													# we set "bytes": None to not duplicate the data if they're already available locally
													if value["path"].endswith('pcm'			):
																# "PCM" only has raw audio bytes
																if value.get('sampling_rate'			) is None:
																			# At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
																			raise KeyError('To use PCM files, please specify a \'sampling_rate\' in Audio object'			)
																if value.get('bytes'			):
																			# If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
																			snake_case:    List[str]      							=np.frombuffer(value['bytes']      ,						dtype=np.intaa			).astype(np.floataa			) / 3_2_7_6_7
																else:
																			snake_case:    List[str]      							=np.memmap(value['path']      ,						dtype='h'      ,						mode='r'			).astype(np.floataa			) / 3_2_7_6_7

																snake_case:    Union[str, Any]      							=BytesIO(bytes()			)
																sf.write(a_      ,						a_      ,						value['sampling_rate']      ,						format='wav'			)
																return {"bytes": buffer.getvalue(), "path": None}
													else:
																return {"bytes": None, "path": value.get('path'			)}
										elif value.get('bytes'			) is not None or value.get('path'			) is not None:
													# store the audio bytes, and path is used to infer the audio format using the file extension
													return {"bytes": value.get('bytes'			), "path": value.get('path'			)}
										else:
													raise ValueError(
													    F'''An audio sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.'''			)


							def    UpperCamelCase     (				self   :						Optional[int]      ,						a_   :						dict      ,						a_   :						Optional[Dict[str, Union[str, bool, None]]] = None			)			->					dict:
										if not self.decode:
													raise RuntimeError('Decoding is disabled for this feature. Please use Audio(decode=True) instead.'			)

										snake_case							,  snake_case:    int      							=(value['path'], BytesIO(value['bytes']			)) if value['bytes'] is not None else (value['path'], None)
										if path is None and file is None:
													raise ValueError(F'''An audio sample should have one of \'path\' or \'bytes\' but both are None in {value}.'''			)

										try:
													import librosa
													import soundfile as sf
										except ImportError as err:
													raise ImportError('To support decoding audio files, please install \'librosa\' and \'soundfile\'.'			) from err

										snake_case:    int      							=xsplitext(a_			)[1][1:].lower() if path is not None else None
										if not config.IS_OPUS_SUPPORTED and audio_format == "opus":
													raise RuntimeError(
													    'Decoding \'opus\' files requires system library \'libsndfile\'>=1.0.31, '
													    'You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. '			)
										elif not config.IS_MP3_SUPPORTED and audio_format == "mp3":
													raise RuntimeError(
													    'Decoding \'mp3\' files requires system library \'libsndfile\'>=1.1.0, '
													    'You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. '			)

										if file is None:
													snake_case:    str      							=token_per_repo_id or {}
													snake_case:    Union[str, Any]      							=path.split('::'			)[-1]
													try:
																snake_case:    Dict      							=string_to_dict(a_      ,						config.HUB_DATASETS_URL			)['repo_id']
																snake_case:    List[str]      							=token_per_repo_id[repo_id]
													except (ValueError, KeyError):
																snake_case:    List[Any]      							=None

													with xopen(a_      ,						'rb'      ,						use_auth_token=a_			) as f:
																snake_case							,  snake_case:    List[str]      							=sf.read(a_			)

										else:
													snake_case							,  snake_case:    int      							=sf.read(a_			)

										snake_case:    str      							=array.T
										if self.mono:
													snake_case:    Dict      							=librosa.to_mono(a_			)
										if self.sampling_rate and self.sampling_rate != sampling_rate:
													snake_case:    int      							=librosa.resample(a_      ,						orig_sr=a_      ,						target_sr=self.sampling_rate			)
													snake_case:    Dict      							=self.sampling_rate

										return {"path": path, "array": array, "sampling_rate": sampling_rate}


							def    UpperCamelCase     (				self   :						List[str]			)			->					Union["FeatureType", Dict[str, "FeatureType"]]:
										from .features import Value

										if self.decode:
													raise ValueError('Cannot flatten a decoded Audio feature.'			)
										return {
										    "bytes": Value('binary'			),
										    "path": Value('string'			),
										}


							def    UpperCamelCase     (				self   :						int      ,						a_   :						Union[pa.StringArray, pa.StructArray]			)			->					pa.StructArray:
										if pa.types.is_string(storage.type			):
													snake_case:    str      							=pa.array([None] * len(a_			)      ,						type=pa.binary()			)
													snake_case:    Optional[int]      							=pa.StructArray.from_arrays([bytes_array, storage]      ,						['bytes', 'path']      ,						mask=storage.is_null()			)
										elif pa.types.is_binary(storage.type			):
													snake_case:    List[str]      							=pa.array([None] * len(a_			)      ,						type=pa.string()			)
													snake_case:    Optional[int]      							=pa.StructArray.from_arrays([storage, path_array]      ,						['bytes', 'path']      ,						mask=storage.is_null()			)
										elif pa.types.is_struct(storage.type			) and storage.type.get_all_field_indices('array'			):
													snake_case:    Tuple      							=pa.array([Audio().encode_example(a_			) if x is not None else None for x in storage.to_pylist()]			)
										elif pa.types.is_struct(storage.type			):
													if storage.type.get_field_index('bytes'			) >= 0:
																snake_case:    Any      							=storage.field('bytes'			)
													else:
																snake_case:    Tuple      							=pa.array([None] * len(a_			)      ,						type=pa.binary()			)
													if storage.type.get_field_index('path'			) >= 0:
																snake_case:    Optional[Any]      							=storage.field('path'			)
													else:
																snake_case:    Optional[int]      							=pa.array([None] * len(a_			)      ,						type=pa.string()			)
													snake_case:    Optional[int]      							=pa.StructArray.from_arrays([bytes_array, path_array]      ,						['bytes', 'path']      ,						mask=storage.is_null()			)
										return array_cast(a_      ,						self.pa_type			)




							def    UpperCamelCase     (				self   :						Optional[int]      ,						a_   :						pa.StructArray			)			->					pa.StructArray:

										@no_op_if_value_is_null
										def path_to_bytes(a_   :						str			):
													with xopen(a_      ,						'rb'			) as f:
																snake_case:    List[str]      							=f.read()
													return bytes_

										snake_case:    str      							=pa.array(
										    [
										        (path_to_bytes(x['path']			) if x['bytes'] is None else x['bytes']) if x is not None else None
										        for x in storage.to_pylist()
										    ]      ,						type=pa.binary()      ,						)
										snake_case:    List[str]      							=pa.array(
										    [os.path.basename(a_			) if path is not None else None for path in storage.field('path'			).to_pylist()]      ,						type=pa.string()      ,						)
										snake_case:    Dict      							=pa.StructArray.from_arrays([bytes_array, path_array]      ,						['bytes', 'path']      ,						mask=bytes_array.is_null()			)
										return array_cast(a_      ,						self.pa_type			)









'''simple docstring'''

import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


a							   =				get_tests_dir('fixtures/test_sentencepiece.model')






@require_sentencepiece
@require_tokenizers
class 						a_		(     snake_case    ,   unittest.TestCase    ):
							UpperCAmelCase			:		Tuple        =				XLNetTokenizer
							UpperCAmelCase			:		str        =				XLNetTokenizerFast
							UpperCAmelCase			:		str        =				True
							UpperCAmelCase			:		str        =				True


							def    UpperCamelCase     (				self   :						Optional[Any]			)			->					int:
										super().setUp()

										# We have a SentencePiece fixture for testing
										snake_case:    Tuple      							=XLNetTokenizer(a_      ,						keep_accents=a_			)
										tokenizer.sanitize_special_tokens()
										tokenizer.save_pretrained(self.tmpdirname			)


							def    UpperCamelCase     (				self   :						str			)			->					str:
										snake_case:    List[str]      							='<s>'
										snake_case:    Tuple      							=1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(a_			)      ,						a_			)
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(a_			)      ,						a_			)


							def    UpperCamelCase     (				self   :						Any			)			->					int:
										snake_case:    str      							=list(self.get_tokenizer().get_vocab().keys()			)

										self.assertEqual(vocab_keys[0]      ,						'<unk>'			)
										self.assertEqual(vocab_keys[1]      ,						'<s>'			)
										self.assertEqual(vocab_keys[-1]      ,						'<eod>'			)
										self.assertEqual(len(a_			)      ,						1_0_0_6			)


							def    UpperCamelCase     (				self   :						Optional[Any]			)			->					List[Any]:
										self.assertEqual(self.get_tokenizer().vocab_size      ,						1_0_0_0			)


							def    UpperCamelCase     (				self   :						Union[str, Any]			)			->					Tuple:
										snake_case:    str      							=XLNetTokenizer(a_      ,						keep_accents=a_			)

										snake_case:    Dict      							=tokenizer.tokenize('This is a test'			)
										self.assertListEqual(a_      ,						['▁This', '▁is', '▁a', '▁t', 'est']			)

										self.assertListEqual(tokenizer.convert_tokens_to_ids(a_			)      ,						[2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]			)

										snake_case:    List[str]      							=tokenizer.tokenize('I was born in 92000, and this is falsé.'			)
										self.assertListEqual(
										    a_      ,						[
										        SPIECE_UNDERLINE + 'I',
										        SPIECE_UNDERLINE + 'was',
										        SPIECE_UNDERLINE + 'b',
										        'or',
										        'n',
										        SPIECE_UNDERLINE + 'in',
										        SPIECE_UNDERLINE + '',
										        '9',
										        '2',
										        '0',
										        '0',
										        '0',
										        ',',
										        SPIECE_UNDERLINE + 'and',
										        SPIECE_UNDERLINE + 'this',
										        SPIECE_UNDERLINE + 'is',
										        SPIECE_UNDERLINE + 'f',
										        'al',
										        's',
										        'é',
										        '.',
										    ]      ,						)
										snake_case:    Optional[int]      							=tokenizer.convert_tokens_to_ids(a_			)
										self.assertListEqual(a_      ,						[8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4]			)

										snake_case:    List[str]      							=tokenizer.convert_ids_to_tokens(a_			)
										self.assertListEqual(
										    a_      ,						[
										        SPIECE_UNDERLINE + 'I',
										        SPIECE_UNDERLINE + 'was',
										        SPIECE_UNDERLINE + 'b',
										        'or',
										        'n',
										        SPIECE_UNDERLINE + 'in',
										        SPIECE_UNDERLINE + '',
										        '<unk>',
										        '2',
										        '0',
										        '0',
										        '0',
										        ',',
										        SPIECE_UNDERLINE + 'and',
										        SPIECE_UNDERLINE + 'this',
										        SPIECE_UNDERLINE + 'is',
										        SPIECE_UNDERLINE + 'f',
										        'al',
										        's',
										        '<unk>',
										        '.',
										    ]      ,						)


							def    UpperCamelCase     (				self   :						Dict			)			->					int:
										snake_case:    Tuple      							=XLNetTokenizer(a_      ,						do_lower_case=a_			)
										snake_case:    Optional[int]      							=tokenizer.tokenize('I was born in 92000, and this is falsé.'			)
										self.assertListEqual(
										    a_      ,						[
										        SPIECE_UNDERLINE + '',
										        'i',
										        SPIECE_UNDERLINE + 'was',
										        SPIECE_UNDERLINE + 'b',
										        'or',
										        'n',
										        SPIECE_UNDERLINE + 'in',
										        SPIECE_UNDERLINE + '',
										        '9',
										        '2',
										        '0',
										        '0',
										        '0',
										        ',',
										        SPIECE_UNDERLINE + 'and',
										        SPIECE_UNDERLINE + 'this',
										        SPIECE_UNDERLINE + 'is',
										        SPIECE_UNDERLINE + 'f',
										        'al',
										        'se',
										        '.',
										    ]      ,						)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'			)      ,						['▁he', 'll', 'o']			)


							def    UpperCamelCase     (				self   :						Tuple			)			->					Dict:
										snake_case:    str      							=XLNetTokenizer(a_      ,						do_lower_case=a_			)
										snake_case:    List[Any]      							=tokenizer.tokenize('I was born in 92000, and this is falsé.'			)
										self.assertListEqual(
										    a_      ,						[
										        SPIECE_UNDERLINE + 'I',
										        SPIECE_UNDERLINE + 'was',
										        SPIECE_UNDERLINE + 'b',
										        'or',
										        'n',
										        SPIECE_UNDERLINE + 'in',
										        SPIECE_UNDERLINE + '',
										        '9',
										        '2',
										        '0',
										        '0',
										        '0',
										        ',',
										        SPIECE_UNDERLINE + 'and',
										        SPIECE_UNDERLINE + 'this',
										        SPIECE_UNDERLINE + 'is',
										        SPIECE_UNDERLINE + 'f',
										        'al',
										        'se',
										        '.',
										    ]      ,						)


							@slow
							def    UpperCamelCase     (				self   :						List[str]			)			->					Optional[Any]:
										snake_case:    Tuple      							=XLNetTokenizer.from_pretrained('xlnet-base-cased'			)

										snake_case:    Optional[Any]      							=tokenizer.encode('sequence builders'      ,						add_special_tokens=a_			)
										snake_case:    List[str]      							=tokenizer.encode('multi-sequence build'      ,						add_special_tokens=a_			)

										snake_case:    Optional[Any]      							=tokenizer.build_inputs_with_special_tokens(a_			)
										snake_case:    Any      							=tokenizer.build_inputs_with_special_tokens(a_      ,						a_			)

										assert encoded_sentence == text + [4, 3]
										assert encoded_pair == text + [4] + text_a + [4, 3]




							@slow
							def    UpperCamelCase     (				self   :						Dict			)			->					Optional[int]:
										# fmt: off
										snake_case:    Dict      							={'input_ids': [[1_7, 2_1_4_4_2, 2_7_0, 1_7, 1_0, 1_4_6_4_5, 3_1_8, 3_4, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 7_7_5_2, 2_2_0_1_8, 2_3, 2_1, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 3_3_5_2, 1_4_4_3_1, 1_3, 5_5_0_0, 1_1, 1_1_7_6, 5_8_0, 1_3, 1_6_8_1_9, 4_7_9_7, 2_3, 1_7, 1_0, 1_7_1_3_5, 6_5_8, 1_9, 4_5_7, 7_9_3_2, 1_3, 1_8_4, 1_9, 3_1_5_4, 1_7_1_3_5, 6_4_6_8, 1_9, 1_4_0_4, 1_2_2_6_9, 1_9, 4_2_2_9, 5_3_5_6, 1_6_2_6_4, 4_6, 1_9, 1_7, 2_0_5_4_5, 1_0_3_9_5, 9, 9, 9, 1_1, 2_8, 6_4_2_1, 9_5_3_1, 2_0_7_2_9, 1_7, 1_0, 3_5_3, 1_7_0_2_2, 1_1, 2_1, 6_4_2_1, 9_5_3_1, 1_6_9_4_9, 1_7, 1_0, 1_1_5_0_9, 7_5_3, 1_1, 3_3, 9_5, 2_4_2_1, 7_3_8_5, 9_5_6, 1_4_4_3_1, 2_6_2_6, 2_5, 8_4_2, 7_3_8_5, 4_8_3_6, 2_1, 1_4_2_9, 2_2_7_2, 9_8_5_5, 3_1_2_0, 1_6_1, 2_4_7_3_8, 1_9, 1_3_2_0_3, 6_5_8, 2_1_8, 7_8_7, 2_1, 4_3_0, 1_8_4_8_2, 8_4_7, 2_6_3_7, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2_2, 2_2_1_7_8, 2_7, 1_0_6_4, 2_2, 9_5_6, 1_3, 1_1_1_0_1, 1_4_2_9, 5_8_5_4, 2_4_3_1_3, 1_8_9_5_3, 4_0, 4_2_2, 2_4_3_6_6, 6_8, 1_7_5_8, 3_7, 1_0_4_8_3, 1_4_2_5_7, 3_1, 2_0_7, 2_6_3, 2_1, 2_0_3, 3_7_7_3, 2_5, 7_1, 9_7_3_5, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2, 2_0_4_9, 3_4_4_2, 1_7, 1_3_8_9_4, 3_3_8_0, 2_3, 9_5, 1_8, 1_7_6_3_4, 2_2_8_8, 9, 4, 3]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=a_      ,						model_name='xlnet-base-cased'      ,						revision='c841166438c31ec7ca9a106dee7bb312b73ae511'      ,						)




