






import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowercase    				=   get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
	from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowercase    				=   256047
lowercase    				=   256145




@require_sentencepiece
@require_tokenizers
class 					UpperCamelCase_ (     snake_case_			,   unittest.TestCase							):



	'''simple docstring'''




	lowerCAmelCase						=					NllbTokenizer
	lowerCAmelCase						=					NllbTokenizerFast
	lowerCAmelCase						=					True
	lowerCAmelCase						=					True
	lowerCAmelCase						=					{}




	def 				_UpperCamelCase     (			self   )			->       Any:
								super().setUp()

								# We have a SentencePiece fixture for testing
								snake_case_           =				NllbTokenizer(a			,     keep_accents=a   )
								tokenizer.save_pretrained(self.tmpdirname   )




	def 				_UpperCamelCase     (			self   )			->       str:
								snake_case_           =				NllbTokenizer(a			,     keep_accents=a   )

								snake_case_           =				tokenizer.tokenize('This is a test'   )
								self.assertListEqual(a			,     ['▁This', '▁is', '▁a', '▁t', 'est']   )

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(a   )			,     [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]			,     )

								snake_case_           =				tokenizer.tokenize('I was born in 92000, and this is falsé.'   )
								self.assertListEqual(
								    a			,     [
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '9',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        's',
								        'é',
								        '.',
								    ]			,     )
								snake_case_           =				tokenizer.convert_tokens_to_ids(a   )
								self.assertListEqual(
								    a			,     [
								        value + tokenizer.fairseq_offset
								        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
								    ]			,     )

								snake_case_           =				tokenizer.convert_ids_to_tokens(a   )
								self.assertListEqual(
								    a			,     [
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '<unk>',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        's',
								        '<unk>',
								        '.',
								    ]			,     )




	def 				_UpperCamelCase     (			self   )			->       str:
								snake_case_           =				(self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
																						snake_case_           =				self.rust_tokenizer_class.from_pretrained(a			,     **a   )
																						snake_case_           =				self.tokenizer_class.from_pretrained(a			,     **a   )

																						snake_case_           =				tempfile.mkdtemp()

																						snake_case_           =				tokenizer_r.save_pretrained(a   )
																						snake_case_           =				tokenizer_p.save_pretrained(a   )

																						# Checks it save with the same files + the tokenizer.json file for the fast one
																						self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files   )   )
																						snake_case_           =				tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f   )
																						self.assertSequenceEqual(a			,     a   )

																						# Checks everything loads correctly in the same way
																						snake_case_           =				tokenizer_r.from_pretrained(a   )
																						snake_case_           =				tokenizer_p.from_pretrained(a   )

																						# Check special tokens are set accordingly on Rust and Python
																						for key in tokenizer_pp.special_tokens_map:
																													self.assertTrue(hasattr(a			,     a   )   )

																						shutil.rmtree(a   )

																						# Save tokenizer rust, legacy_format=True
																						snake_case_           =				tempfile.mkdtemp()

																						snake_case_           =				tokenizer_r.save_pretrained(a			,     legacy_format=a   )
																						snake_case_           =				tokenizer_p.save_pretrained(a   )

																						# Checks it save with the same files
																						self.assertSequenceEqual(a			,     a   )

																						# Checks everything loads correctly in the same way
																						snake_case_           =				tokenizer_r.from_pretrained(a   )
																						snake_case_           =				tokenizer_p.from_pretrained(a   )

																						# Check special tokens are set accordingly on Rust and Python
																						for key in tokenizer_pp.special_tokens_map:
																													self.assertTrue(hasattr(a			,     a   )   )

																						shutil.rmtree(a   )

																						# Save tokenizer rust, legacy_format=False
																						snake_case_           =				tempfile.mkdtemp()

																						snake_case_           =				tokenizer_r.save_pretrained(a			,     legacy_format=a   )
																						snake_case_           =				tokenizer_p.save_pretrained(a   )

																						# Checks it saved the tokenizer.json file
																						self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files   )   )

																						# Checks everything loads correctly in the same way
																						snake_case_           =				tokenizer_r.from_pretrained(a   )
																						snake_case_           =				tokenizer_p.from_pretrained(a   )

																						# Check special tokens are set accordingly on Rust and Python
																						for key in tokenizer_pp.special_tokens_map:
																													self.assertTrue(hasattr(a			,     a   )   )

																						shutil.rmtree(a   )




	@require_torch
	def 				_UpperCamelCase     (			self   )			->       Optional[int]:
								if not self.test_seqaseq:
															return

								snake_case_           =				self.get_tokenizers()
								for tokenizer in tokenizers:
															with self.subTest(F'''{tokenizer.__class__.__name__}'''   ):
																						# Longer text that will definitely require truncation.
																						snake_case_           =				[
																						    ' UN Chief Says There Is No Military Solution in Syria',
																						    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
																						    ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
																						    ' will only worsen the violence and misery for millions of people.',
																						]
																						snake_case_           =				[
																						    'Şeful ONU declară că nu există o soluţie militară în Siria',
																						    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
																						    ' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'
																						    ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
																						]
																						try:
																													snake_case_           =				tokenizer.prepare_seqaseq_batch(
																													    src_texts=a			,     tgt_texts=a			,     max_length=3			,     max_target_length=10			,     return_tensors='pt'			,     src_lang='eng_Latn'			,     tgt_lang='ron_Latn'			,     )
																						except NotImplementedError:
																													return
																						self.assertEqual(batch.input_ids.shape[1]			,     3   )
																						self.assertEqual(batch.labels.shape[1]			,     10   )
																						# max_target_length will default to max_length if not specified
																						snake_case_           =				tokenizer.prepare_seqaseq_batch(
																						    a			,     tgt_texts=a			,     max_length=3			,     return_tensors='pt'   )
																						self.assertEqual(batch.input_ids.shape[1]			,     3   )
																						self.assertEqual(batch.labels.shape[1]			,     3   )

																						snake_case_           =				tokenizer.prepare_seqaseq_batch(
																						    src_texts=a			,     max_length=3			,     max_target_length=10			,     return_tensors='pt'   )
																						self.assertEqual(batch_encoder_only.input_ids.shape[1]			,     3   )
																						self.assertEqual(batch_encoder_only.attention_mask.shape[1]			,     3   )
																						self.assertNotIn('decoder_input_ids'			,     a   )




	@unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'   )
	def 				_UpperCamelCase     (			self   )			->       Union[str, Any]:
								pass






	def 				_UpperCamelCase     (			self   )			->       Optional[Any]:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
																						snake_case_           =				[AddedToken('<special>'			,     lstrip=a   )]

																						snake_case_           =				self.rust_tokenizer_class.from_pretrained(
																						    a			,     additional_special_tokens=a			,     **a   )
																						snake_case_           =				tokenizer_r.encode('Hey this is a <special> token'   )

																						snake_case_           =				tokenizer_r.encode('<special>'			,     add_special_tokens=a   )[0]

																						self.assertTrue(special_token_id in r_output   )

																						if self.test_slow_tokenizer:
																													snake_case_           =				self.rust_tokenizer_class.from_pretrained(
																													    a			,     additional_special_tokens=a			,     **a			,     )
																													snake_case_           =				self.tokenizer_class.from_pretrained(
																													    a			,     additional_special_tokens=a			,     **a   )

																													snake_case_           =				tokenizer_p.encode('Hey this is a <special> token'   )

																													snake_case_           =				tokenizer_cr.encode('Hey this is a <special> token'   )

																													self.assertEqual(a			,     a   )
																													self.assertEqual(a			,     a   )
																													self.assertTrue(special_token_id in p_output   )
																													self.assertTrue(special_token_id in cr_output   )






@require_torch
@require_sentencepiece
@require_tokenizers
class 					UpperCamelCase_ (     unittest.TestCase							):



	'''simple docstring'''




	lowerCAmelCase						=					'''facebook/nllb-200-distilled-600M'''
	lowerCAmelCase						=					[
	    ''' UN Chief Says There Is No Military Solution in Syria''',
	    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
	]
	lowerCAmelCase						=					[
	    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
	    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
	    ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
	    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
	]
	lowerCAmelCase						=					[
	    2_5_6_0_4_7,
	    1_6_2_9_7,
	    1_3_4_4_0_8,
	    8_1_6_5,
	    2_4_8_0_6_6,
	    1_4_7_3_4,
	    9_5_0,
	    1_1_3_5,
	    1_0_5_7_2_1,
	    3_5_7_3,
	    8_3,
	    2_7_3_5_2,
	    1_0_8,
	    4_9_4_8_6,
	    2,
	]




	@classmethod
	def 				_UpperCamelCase     (			cls   )			->       List[Any]:
								snake_case_           =				NllbTokenizer.from_pretrained(
								    cls.checkpoint_name			,     src_lang='eng_Latn'			,     tgt_lang='ron_Latn'   )
								snake_case_           =				1
								return cls




	def 				_UpperCamelCase     (			self   )			->       Optional[int]:
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']			,     25_60_01   )
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']			,     25_60_02   )
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']			,     25_60_57   )




	def 				_UpperCamelCase     (			self   )			->       Union[str, Any]:
								snake_case_           =				self.tokenizer.batch_encode_plus(self.src_text   ).input_ids[0]
								self.assertListEqual(self.expected_src_tokens			,     a   )




	def 				_UpperCamelCase     (			self   )			->       str:
								self.assertIn(a			,     self.tokenizer.all_special_ids   )
								# fmt: off
								snake_case_           =				[RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
								# fmt: on

								snake_case_           =				self.tokenizer.decode(a			,     skip_special_tokens=a   )
								snake_case_           =				self.tokenizer.decode(generated_ids[1:]			,     skip_special_tokens=a   )
								self.assertEqual(a			,     a   )
								self.assertNotIn(self.tokenizer.eos_token			,     a   )




	def 				_UpperCamelCase     (			self   )			->       int:
								snake_case_           =				['this is gunna be a long sentence ' * 20]
								assert isinstance(src_text[0]			,     a   )
								snake_case_           =				10
								snake_case_           =				self.tokenizer(a			,     max_length=a			,     truncation=a   ).input_ids[0]
								self.assertEqual(ids[-1]			,     2   )
								self.assertEqual(ids[0]			,     a   )
								self.assertEqual(len(a   )			,     a   )




	def 				_UpperCamelCase     (			self   )			->       str:
								self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']   )			,     [25_62_03, 3]   )




	def 				_UpperCamelCase     (			self   )			->       str:
								snake_case_           =				tempfile.mkdtemp()
								snake_case_           =				self.tokenizer.fairseq_tokens_to_ids
								self.tokenizer.save_pretrained(a   )
								snake_case_           =				NllbTokenizer.from_pretrained(a   )
								self.assertDictEqual(new_tok.fairseq_tokens_to_ids			,     a   )




	@require_torch
	def 				_UpperCamelCase     (			self   )			->       Tuple:
								snake_case_           =				self.tokenizer(
								    self.src_text			,     text_target=self.tgt_text			,     padding=a			,     truncation=a			,     max_length=len(self.expected_src_tokens   )			,     return_tensors='pt'			,     )
								snake_case_           =				shift_tokens_right(
								    batch['labels']			,     self.tokenizer.pad_token_id			,     self.tokenizer.lang_code_to_id['ron_Latn']   )

								self.assertIsInstance(a			,     a   )

								self.assertEqual((2, 15)			,     batch.input_ids.shape   )
								self.assertEqual((2, 15)			,     batch.attention_mask.shape   )
								snake_case_           =				batch.input_ids.tolist()[0]
								self.assertListEqual(self.expected_src_tokens			,     a   )
								self.assertEqual(a			,     batch.decoder_input_ids[0, 0]   )  # EOS
								# Test that special tokens are reset
								self.assertEqual(self.tokenizer.prefix_tokens			,     [EN_CODE]   )
								self.assertEqual(self.tokenizer.suffix_tokens			,     [self.tokenizer.eos_token_id]   )




	def 				_UpperCamelCase     (			self   )			->       Any:
								snake_case_           =				self.tokenizer(self.src_text			,     padding=a			,     truncation=a			,     max_length=3			,     return_tensors='pt'   )
								snake_case_           =				self.tokenizer(
								    text_target=self.tgt_text			,     padding=a			,     truncation=a			,     max_length=10			,     return_tensors='pt'   )
								snake_case_           =				targets['input_ids']
								snake_case_           =				shift_tokens_right(
								    a			,     self.tokenizer.pad_token_id			,     decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]			,     )

								self.assertEqual(batch.input_ids.shape[1]			,     3   )
								self.assertEqual(batch.decoder_input_ids.shape[1]			,     10   )




	@require_torch
	def 				_UpperCamelCase     (			self   )			->       Union[str, Any]:
								snake_case_           =				self.tokenizer._build_translation_inputs(
								    'A test'			,     return_tensors='pt'			,     src_lang='eng_Latn'			,     tgt_lang='fra_Latn'   )

								self.assertEqual(
								    nested_simplify(a   )			,     {
								        # A, test, EOS, en_XX
								        'input_ids': [[25_60_47, 70, 73_56, 2]],
								        'attention_mask': [[1, 1, 1, 1]],
								        # ar_AR
								        'forced_bos_token_id': 25_60_57,
								    }			,     )






	@require_torch
	def 				_UpperCamelCase     (			self   )			->       Optional[int]:
								snake_case_           =				True
								snake_case_           =				self.tokenizer(
								    'UN Chief says there is no military solution in Syria'			,     src_lang='eng_Latn'			,     tgt_lang='fra_Latn'   )
								self.assertEqual(
								    inputs.input_ids			,     [1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]   )

								snake_case_           =				False
								snake_case_           =				self.tokenizer(
								    'UN Chief says there is no military solution in Syria'			,     src_lang='eng_Latn'			,     tgt_lang='fra_Latn'   )
								self.assertEqual(
								    inputs.input_ids			,     [25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]   )













import itertools
import random
import unittest

import numpy as np

from transformers import ASTFeatureExtractor
from transformers.testing_utils import require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


lowercase    				=   random.Random()

if is_torch_available():
	import torch





def 		__UpperCAmelCase     (	a_				,	a_=1.0				,	a_=None				,	a_=None):
							if rng is None:
														snake_case_           =				global_rng

							snake_case_           =				[]
							for batch_idx in range(shape[0]):
														values.append([])
														for _ in range(shape[1]):
																					values[-1].append(rng.random() * scale)

							return values




class 					UpperCamelCase_ (     unittest.TestCase							):



	'''simple docstring'''




	def __init__(			self			,     a			,     a=7			,     a=4_00			,     a=20_00			,     a=1			,     a=0.0			,     a=1_60_00			,     a=True			,     a=True			,     )			->       Tuple:
								snake_case_           =				parent
								snake_case_           =				batch_size
								snake_case_           =				min_seq_length
								snake_case_           =				max_seq_length
								snake_case_           =				(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
								snake_case_           =				feature_size
								snake_case_           =				padding_value
								snake_case_           =				sampling_rate
								snake_case_           =				return_attention_mask
								snake_case_           =				do_normalize




	def 				_UpperCamelCase     (			self   )			->       Union[str, Any]:
								return {
								    "feature_size": self.feature_size,
								    "padding_value": self.padding_value,
								    "sampling_rate": self.sampling_rate,
								    "return_attention_mask": self.return_attention_mask,
								    "do_normalize": self.do_normalize,
								}






	def 				_UpperCamelCase     (			self			,     a=False			,     a=False   )			->       Optional[int]:
								def _flatten(a   ):
															return list(itertools.chain(*a   )   )

								if equal_length:
															snake_case_           =				floats_list((self.batch_size, self.max_seq_length)   )
								else:
															# make sure that inputs increase in size
															snake_case_           =				[
															    _flatten(floats_list((x, self.feature_size)   )   )
															    for x in range(self.min_seq_length			,     self.max_seq_length			,     self.seq_length_diff   )
															]

								if numpify:
															snake_case_           =				[np.asarray(a   ) for x in speech_inputs]

								return speech_inputs






@require_torch
@require_torchaudio
class 					UpperCamelCase_ (     snake_case_			,   unittest.TestCase							):



	'''simple docstring'''




	lowerCAmelCase						=					ASTFeatureExtractor




	def 				_UpperCamelCase     (			self   )			->       Optional[int]:
								snake_case_           =				ASTFeatureExtractionTester(self   )




	def 				_UpperCamelCase     (			self   )			->       Union[str, Any]:
								# Tests that all call wrap to encode_plus and batch_encode_plus
								snake_case_           =				self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()   )
								# create three inputs of length 800, 1000, and 1200
								snake_case_           =				[floats_list((1, x)   )[0] for x in range(8_00			,     14_00			,     2_00   )]
								snake_case_           =				[np.asarray(a   ) for speech_input in speech_inputs]

								# Test not batched input
								snake_case_           =				feat_extract(speech_inputs[0]			,     return_tensors='np'   ).input_values
								snake_case_           =				feat_extract(np_speech_inputs[0]			,     return_tensors='np'   ).input_values
								self.assertTrue(np.allclose(a			,     a			,     atol=1E-3   )   )

								# Test batched
								snake_case_           =				feat_extract(a			,     padding=a			,     return_tensors='np'   ).input_values
								snake_case_           =				feat_extract(a			,     padding=a			,     return_tensors='np'   ).input_values
								for enc_seq_a, enc_seq_a in zip(a			,     a   ):
															self.assertTrue(np.allclose(a			,     a			,     atol=1E-3   )   )

								# Test 2-D numpy arrays are batched.
								snake_case_           =				[floats_list((1, x)   )[0] for x in (8_00, 8_00, 8_00)]
								snake_case_           =				np.asarray(a   )
								snake_case_           =				feat_extract(a			,     return_tensors='np'   ).input_values
								snake_case_           =				feat_extract(a			,     return_tensors='np'   ).input_values
								for enc_seq_a, enc_seq_a in zip(a			,     a   ):
															self.assertTrue(np.allclose(a			,     a			,     atol=1E-3   )   )




	@require_torch
	def 				_UpperCamelCase     (			self   )			->       List[str]:
								import torch

								snake_case_           =				self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()   )
								snake_case_           =				np.random.rand(1_00   ).astype(np.floataa   )
								snake_case_           =				np_speech_inputs.tolist()

								for inputs in [py_speech_inputs, np_speech_inputs]:
															snake_case_           =				feature_extractor.pad([{'input_values': inputs}]			,     return_tensors='np'   )
															self.assertTrue(np_processed.input_values.dtype == np.floataa   )
															snake_case_           =				feature_extractor.pad([{'input_values': inputs}]			,     return_tensors='pt'   )
															self.assertTrue(pt_processed.input_values.dtype == torch.floataa   )




	def 				_UpperCamelCase     (			self			,     a   )			->       Tuple:
								from datasets import load_dataset

								snake_case_           =				load_dataset('hf-internal-testing/librispeech_asr_dummy'			,     'clean'			,     split='validation'   )
								# automatic decoding with librispeech
								snake_case_           =				ds.sort('id'   ).select(range(a   )   )[:num_samples]['audio']

								return [x["array"] for x in speech_samples]






	@require_torch
	def 				_UpperCamelCase     (			self   )			->       Optional[int]:
								# fmt: off
								snake_case_           =				torch.tensor(
								    [-0.9_894, -1.2_776, -0.9_066, -1.2_776, -0.9_349, -1.2_609, -1.0_386, -1.2_776,
								     -1.1_561, -1.2_776, -1.2_052, -1.2_723, -1.2_190, -1.2_132, -1.2_776, -1.1_133,
								     -1.1_953, -1.1_343, -1.1_584, -1.2_203, -1.1_770, -1.2_474, -1.2_381, -1.1_936,
								     -0.9_270, -0.8_317, -0.8_049, -0.7_706, -0.7_565, -0.7_869]   )
								# fmt: on

								snake_case_           =				self._load_datasamples(1   )
								snake_case_           =				ASTFeatureExtractor()
								snake_case_           =				feature_extractor(a			,     return_tensors='pt'   ).input_values
								self.assertEquals(input_values.shape			,     (1, 10_24, 1_28)   )
								self.assertTrue(torch.allclose(input_values[0, 0, :30]			,     a			,     atol=1E-4   )   )





