


"""simple docstring"""




import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import BertTokenizer, BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
					from PIL import Image

					from transformers import AlignProcessor, EfficientNetImageProcessor




@require_vision
class 			__snake_case   (unittest.TestCase  ):




							def   __a   (     self:    Tuple   ):
										__lowerCamelCase          =      tempfile.mkdtemp()

										__lowerCamelCase          =      [
										    """[UNK]""",
										    """[CLS]""",
										    """[SEP]""",
										    """[PAD]""",
										    """[MASK]""",
										    """want""",
										    """##want""",
										    """##ed""",
										    """wa""",
										    """un""",
										    """runn""",
										    """##ing""",
										    """,""",
										    """low""",
										    """lowest""",
										]
										__lowerCamelCase          =      os.path.join(self.tmpdirname     ,							VOCAB_FILES_NAMES["""vocab_file"""]   )
										with open(self.vocab_file     ,							"""w"""     ,							encoding="""utf-8"""   ) as vocab_writer:
													vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]   )   )

										__lowerCamelCase          =      {
										    """do_resize""": True,
										    """size""": 20,
										    """do_center_crop""": True,
										    """crop_size""": 18,
										    """do_normalize""": True,
										    """image_mean""": [0.48_145_466, 0.4_578_275, 0.40_821_073],
										    """image_std""": [0.26_862_954, 0.26_130_258, 0.27_577_711],
										}
										__lowerCamelCase          =      os.path.join(self.tmpdirname     ,							SCREAMING_SNAKE_CASE_   )
										with open(self.image_processor_file     ,							"""w"""     ,							encoding="""utf-8"""   ) as fp:
													json.dump(SCREAMING_SNAKE_CASE_     ,							SCREAMING_SNAKE_CASE_   )




							def   __a   (     self:    Tuple     ,							**A_:    Tuple   ):
										return BertTokenizer.from_pretrained(self.tmpdirname     ,							**SCREAMING_SNAKE_CASE_   )




							def   __a   (     self:    List[Any]     ,							**A_:    Optional[Any]   ):
										return BertTokenizerFast.from_pretrained(self.tmpdirname     ,							**SCREAMING_SNAKE_CASE_   )




							def   __a   (     self:    str     ,							**A_:    List[Any]   ):
										return EfficientNetImageProcessor.from_pretrained(self.tmpdirname     ,							**SCREAMING_SNAKE_CASE_   )




							def   __a   (     self:    Union[str, Any]   ):
										shutil.rmtree(self.tmpdirname   )




							def   __a   (     self:    int   ):
										__lowerCamelCase          =      [np.random.randint(2_55     ,							size=(3, 30, 4_00)     ,							dtype=np.uinta   )]
										__lowerCamelCase          =      [Image.fromarray(np.moveaxis(SCREAMING_SNAKE_CASE_     ,							0     ,							-1   )   ) for x in image_inputs]
										return image_inputs




							def   __a   (     self:    List[str]   ):
										__lowerCamelCase          =      self.get_tokenizer()
										__lowerCamelCase          =      self.get_rust_tokenizer()
										__lowerCamelCase          =      self.get_image_processor()

										__lowerCamelCase          =      AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE_     ,							image_processor=SCREAMING_SNAKE_CASE_   )
										processor_slow.save_pretrained(self.tmpdirname   )
										__lowerCamelCase          =      AlignProcessor.from_pretrained(self.tmpdirname     ,							use_fast=SCREAMING_SNAKE_CASE_   )

										__lowerCamelCase          =      AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE_     ,							image_processor=SCREAMING_SNAKE_CASE_   )
										processor_fast.save_pretrained(self.tmpdirname   )
										__lowerCamelCase          =      AlignProcessor.from_pretrained(self.tmpdirname   )

										self.assertEqual(processor_slow.tokenizer.get_vocab()     ,							tokenizer_slow.get_vocab()   )
										self.assertEqual(processor_fast.tokenizer.get_vocab()     ,							tokenizer_fast.get_vocab()   )
										self.assertEqual(tokenizer_slow.get_vocab()     ,							tokenizer_fast.get_vocab()   )
										self.assertIsInstance(processor_slow.tokenizer     ,							SCREAMING_SNAKE_CASE_   )
										self.assertIsInstance(processor_fast.tokenizer     ,							SCREAMING_SNAKE_CASE_   )

										self.assertEqual(processor_slow.image_processor.to_json_string()     ,							image_processor.to_json_string()   )
										self.assertEqual(processor_fast.image_processor.to_json_string()     ,							image_processor.to_json_string()   )
										self.assertIsInstance(processor_slow.image_processor     ,							SCREAMING_SNAKE_CASE_   )
										self.assertIsInstance(processor_fast.image_processor     ,							SCREAMING_SNAKE_CASE_   )




							def   __a   (     self:    Tuple   ):
										__lowerCamelCase          =      AlignProcessor(tokenizer=self.get_tokenizer()     ,							image_processor=self.get_image_processor()   )
										processor.save_pretrained(self.tmpdirname   )

										__lowerCamelCase          =      self.get_tokenizer(bos_token="""(BOS)"""     ,							eos_token="""(EOS)"""   )
										__lowerCamelCase          =      self.get_image_processor(do_normalize=SCREAMING_SNAKE_CASE_     ,							padding_value=1.0   )

										__lowerCamelCase          =      AlignProcessor.from_pretrained(
										    self.tmpdirname     ,							bos_token="""(BOS)"""     ,							eos_token="""(EOS)"""     ,							do_normalize=SCREAMING_SNAKE_CASE_     ,							padding_value=1.0   )

										self.assertEqual(processor.tokenizer.get_vocab()     ,							tokenizer_add_kwargs.get_vocab()   )
										self.assertIsInstance(processor.tokenizer     ,							SCREAMING_SNAKE_CASE_   )

										self.assertEqual(processor.image_processor.to_json_string()     ,							image_processor_add_kwargs.to_json_string()   )
										self.assertIsInstance(processor.image_processor     ,							SCREAMING_SNAKE_CASE_   )




							def   __a   (     self:    List[str]   ):
										__lowerCamelCase          =      self.get_image_processor()
										__lowerCamelCase          =      self.get_tokenizer()

										__lowerCamelCase          =      AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE_     ,							image_processor=SCREAMING_SNAKE_CASE_   )

										__lowerCamelCase          =      self.prepare_image_inputs()

										__lowerCamelCase          =      image_processor(SCREAMING_SNAKE_CASE_     ,							return_tensors="""np"""   )
										__lowerCamelCase          =      processor(images=SCREAMING_SNAKE_CASE_     ,							return_tensors="""np"""   )

										for key in input_image_proc.keys():
													self.assertAlmostEqual(input_image_proc[key].sum()     ,							input_processor[key].sum()     ,							delta=1E-2   )




							def   __a   (     self:    Any   ):
										__lowerCamelCase          =      self.get_image_processor()
										__lowerCamelCase          =      self.get_tokenizer()

										__lowerCamelCase          =      AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE_     ,							image_processor=SCREAMING_SNAKE_CASE_   )

										__lowerCamelCase          =      """lower newer"""

										__lowerCamelCase          =      processor(text=SCREAMING_SNAKE_CASE_   )

										__lowerCamelCase          =      tokenizer(SCREAMING_SNAKE_CASE_     ,							padding="""max_length"""     ,							max_length=64   )

										for key in encoded_tok.keys():
													self.assertListEqual(encoded_tok[key]     ,							encoded_processor[key]   )




							def   __a   (     self:    List[Any]   ):
										__lowerCamelCase          =      self.get_image_processor()
										__lowerCamelCase          =      self.get_tokenizer()

										__lowerCamelCase          =      AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE_     ,							image_processor=SCREAMING_SNAKE_CASE_   )

										__lowerCamelCase          =      """lower newer"""
										__lowerCamelCase          =      self.prepare_image_inputs()

										__lowerCamelCase          =      processor(text=SCREAMING_SNAKE_CASE_     ,							images=SCREAMING_SNAKE_CASE_   )

										self.assertListEqual(list(inputs.keys()   )     ,							["""input_ids""", """token_type_ids""", """attention_mask""", """pixel_values"""]   )

										# test if it raises when no input is passed
										with pytest.raises(SCREAMING_SNAKE_CASE_   ):
													processor()




							def   __a   (     self:    Tuple   ):
										__lowerCamelCase          =      self.get_image_processor()
										__lowerCamelCase          =      self.get_tokenizer()

										__lowerCamelCase          =      AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE_     ,							image_processor=SCREAMING_SNAKE_CASE_   )

										__lowerCamelCase          =      [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

										__lowerCamelCase          =      processor.batch_decode(SCREAMING_SNAKE_CASE_   )
										__lowerCamelCase          =      tokenizer.batch_decode(SCREAMING_SNAKE_CASE_   )

										self.assertListEqual(SCREAMING_SNAKE_CASE_     ,							SCREAMING_SNAKE_CASE_   )




							def   __a   (     self:    Any   ):
										__lowerCamelCase          =      self.get_image_processor()
										__lowerCamelCase          =      self.get_tokenizer()

										__lowerCamelCase          =      AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE_     ,							image_processor=SCREAMING_SNAKE_CASE_   )

										__lowerCamelCase          =      """lower newer"""
										__lowerCamelCase          =      self.prepare_image_inputs()

										__lowerCamelCase          =      processor(text=SCREAMING_SNAKE_CASE_     ,							images=SCREAMING_SNAKE_CASE_   )

										self.assertListEqual(list(inputs.keys()   )     ,							processor.model_input_names   )






from typing import List

import datasets
from datasets.tasks import AudioClassification

from ..folder_based_builder import folder_based_builder


lowerCamelCase__     :		Any        = datasets.utils.logging.get_logger(__name__)



class    _snake_case (						folder_based_builder.FolderBasedBuilderConfig				):
 __lowerCAmelCase	:  bool	   =      None
 __lowerCAmelCase	:  bool	   =      None



class    _snake_case (						folder_based_builder.FolderBasedBuilder				):
 __lowerCAmelCase	:  Optional[Any]	   =      datasets.Audio()
 __lowerCAmelCase	:  Union[str, Any]	   =      'audio'
 __lowerCAmelCase	:  str	   =      AudioFolderConfig
 __lowerCAmelCase	:  List[str]  # definition at the bottom of the script
 __lowerCAmelCase	:  Optional[int]	   =      AudioClassification(audio_column='audio'	,    label_column='label'				)

lowerCamelCase__     :		int        = [
    """.aiff""",
    """.au""",
    """.avr""",
    """.caf""",
    """.flac""",
    """.htk""",
    """.svx""",
    """.mat4""",
    """.mat5""",
    """.mpc2k""",
    """.ogg""",
    """.paf""",
    """.pvf""",
    """.raw""",
    """.rf64""",
    """.sd2""",
    """.sds""",
    """.ircam""",
    """.voc""",
    """.w64""",
    """.wav""",
    """.nist""",
    """.wavex""",
    """.wve""",
    """.xi""",
    """.mp3""",
    """.opus""",
]
lowerCamelCase__     :		int        = AUDIO_EXTENSIONS






