
'''simple docstring'''



import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
     from transformers import SpeechTaFeatureExtractor, SpeechTaProcessor

     from .test_feature_extraction_speechta import floats_list


UpperCamelCase__					:						Union[str, Any]											=     get_tests_dir("fixtures/test_sentencepiece_bpe_char.model")

@require_torch
class       _a (unittest.TestCase):





       """simple docstring"""




       def 				UpperCamelCase (							self					)			->	List[str]:
          _SCREAMING_SNAKE_CASE          = tempfile.mkdtemp()

          _SCREAMING_SNAKE_CASE          = SpeechTaTokenizer(A__					)
          tokenizer.save_pretrained(self.tmpdirname					)

          _SCREAMING_SNAKE_CASE          = {
              """feature_size""": 1,
              """padding_value""": 0.0,
              """sampling_rate""": 1_60_00,
              """do_normalize""": False,
              """num_mel_bins""": 80,
              """hop_length""": 16,
              """win_length""": 64,
              """win_function""": """hann_window""",
              """fmin""": 80,
              """fmax""": 76_00,
              """mel_floor""": 1E-10,
              """reduction_factor""": 2,
              """return_attention_mask""": True,
          }

          _SCREAMING_SNAKE_CASE          = os.path.join(self.tmpdirname				,							A__					)
          with open(self.feature_extraction_file				,							"""w"""				,							encoding="""utf-8"""					) as fp:
             fp.write(json.dumps(A__					) + """\n"""					)






       def 				UpperCamelCase (							self				,							**A__					)			->	Optional[int]:
          return SpeechTaTokenizer.from_pretrained(self.tmpdirname				,							**A__					)






       def 				UpperCamelCase (							self				,							**A__					)			->	int:
          return SpeechTaFeatureExtractor.from_pretrained(self.tmpdirname				,							**A__					)






       def 				UpperCamelCase (							self					)			->	Tuple:
          shutil.rmtree(self.tmpdirname					)






       def 				UpperCamelCase (							self					)			->	Union[str, Any]:
          _SCREAMING_SNAKE_CASE          = self.get_tokenizer()
          _SCREAMING_SNAKE_CASE          = self.get_feature_extractor()

          _SCREAMING_SNAKE_CASE          = SpeechTaProcessor(tokenizer=A__				,							feature_extractor=A__					)

          processor.save_pretrained(self.tmpdirname					)
          _SCREAMING_SNAKE_CASE          = SpeechTaProcessor.from_pretrained(self.tmpdirname					)

          self.assertEqual(processor.tokenizer.get_vocab()				,							tokenizer.get_vocab()					)
          self.assertIsInstance(processor.tokenizer				,							A__					)

          self.assertEqual(processor.feature_extractor.to_json_string()				,							feature_extractor.to_json_string()					)
          self.assertIsInstance(processor.feature_extractor				,							A__					)






       def 				UpperCamelCase (							self					)			->	Optional[int]:
          _SCREAMING_SNAKE_CASE          = SpeechTaProcessor(tokenizer=self.get_tokenizer()				,							feature_extractor=self.get_feature_extractor()					)
          processor.save_pretrained(self.tmpdirname					)

          _SCREAMING_SNAKE_CASE          = self.get_tokenizer(bos_token="""(BOS)"""				,							eos_token="""(EOS)"""					)
          _SCREAMING_SNAKE_CASE          = self.get_feature_extractor(do_normalize=A__				,							padding_value=1.0					)

          _SCREAMING_SNAKE_CASE          = SpeechTaProcessor.from_pretrained(
              self.tmpdirname				,							bos_token="""(BOS)"""				,							eos_token="""(EOS)"""				,							do_normalize=A__				,							padding_value=1.0					)

          self.assertEqual(processor.tokenizer.get_vocab()				,							tokenizer_add_kwargs.get_vocab()					)
          self.assertIsInstance(processor.tokenizer				,							A__					)

          self.assertEqual(processor.feature_extractor.to_json_string()				,							feature_extractor_add_kwargs.to_json_string()					)
          self.assertIsInstance(processor.feature_extractor				,							A__					)






       def 				UpperCamelCase (							self					)			->	Dict:
          _SCREAMING_SNAKE_CASE          = self.get_feature_extractor()
          _SCREAMING_SNAKE_CASE          = self.get_tokenizer()

          _SCREAMING_SNAKE_CASE          = SpeechTaProcessor(tokenizer=A__				,							feature_extractor=A__					)

          _SCREAMING_SNAKE_CASE          = floats_list((3, 10_00)					)

          _SCREAMING_SNAKE_CASE          = feature_extractor(audio=A__				,							return_tensors="""np"""					)
          _SCREAMING_SNAKE_CASE          = processor(audio=A__				,							return_tensors="""np"""					)

          for key in input_feat_extract.keys():
             self.assertAlmostEqual(input_feat_extract[key].sum()				,							input_processor[key].sum()				,							delta=1E-2					)






       def 				UpperCamelCase (							self					)			->	str:
          _SCREAMING_SNAKE_CASE          = self.get_feature_extractor()
          _SCREAMING_SNAKE_CASE          = self.get_tokenizer()

          _SCREAMING_SNAKE_CASE          = SpeechTaProcessor(tokenizer=A__				,							feature_extractor=A__					)

          _SCREAMING_SNAKE_CASE          = floats_list((3, 10_00)					)

          _SCREAMING_SNAKE_CASE          = feature_extractor(audio_target=A__				,							return_tensors="""np"""					)
          _SCREAMING_SNAKE_CASE          = processor(audio_target=A__				,							return_tensors="""np"""					)

          for key in input_feat_extract.keys():
             self.assertAlmostEqual(input_feat_extract[key].sum()				,							input_processor[key].sum()				,							delta=1E-2					)






       def 				UpperCamelCase (							self					)			->	Tuple:
          _SCREAMING_SNAKE_CASE          = self.get_feature_extractor()
          _SCREAMING_SNAKE_CASE          = self.get_tokenizer()

          _SCREAMING_SNAKE_CASE          = SpeechTaProcessor(tokenizer=A__				,							feature_extractor=A__					)

          _SCREAMING_SNAKE_CASE          = """This is a test string"""

          _SCREAMING_SNAKE_CASE          = processor(text=A__					)
          _SCREAMING_SNAKE_CASE          = tokenizer(A__					)

          for key in encoded_tok.keys():
             self.assertListEqual(encoded_tok[key]				,							encoded_processor[key]					)






       def 				UpperCamelCase (							self					)			->	str:
          _SCREAMING_SNAKE_CASE          = self.get_feature_extractor()
          _SCREAMING_SNAKE_CASE          = self.get_tokenizer()

          _SCREAMING_SNAKE_CASE          = SpeechTaProcessor(tokenizer=A__				,							feature_extractor=A__					)

          _SCREAMING_SNAKE_CASE          = """This is a test string"""

          _SCREAMING_SNAKE_CASE          = processor(text_target=A__					)
          _SCREAMING_SNAKE_CASE          = tokenizer(A__					)

          for key in encoded_tok.keys():
             self.assertListEqual(encoded_tok[key]				,							encoded_processor[key]					)






       def 				UpperCamelCase (							self					)			->	List[Any]:
          _SCREAMING_SNAKE_CASE          = self.get_feature_extractor()
          _SCREAMING_SNAKE_CASE          = self.get_tokenizer()

          _SCREAMING_SNAKE_CASE          = SpeechTaProcessor(tokenizer=A__				,							feature_extractor=A__					)

          _SCREAMING_SNAKE_CASE          = [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

          _SCREAMING_SNAKE_CASE          = processor.batch_decode(A__					)
          _SCREAMING_SNAKE_CASE          = tokenizer.batch_decode(A__					)

          self.assertListEqual(A__				,							A__					)






       def 				UpperCamelCase (							self					)			->	List[str]:
          _SCREAMING_SNAKE_CASE          = self.get_feature_extractor()
          _SCREAMING_SNAKE_CASE          = self.get_tokenizer()

          _SCREAMING_SNAKE_CASE          = SpeechTaProcessor(tokenizer=A__				,							feature_extractor=A__					)

          self.assertListEqual(
              processor.model_input_names				,							feature_extractor.model_input_names				,							msg="""`processor` and `feature_extractor` model input names do not match"""				,							)








'''simple docstring'''



import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


if TYPE_CHECKING:
     import sqlitea

     import sqlalchemy


UpperCamelCase__					:						Tuple											=     datasets.utils.logging.get_logger(__name__)

@dataclass
class       _a (datasets.BuilderConfig):





       """simple docstring"""




       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     True
       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     None
       SCREAMING_SNAKE_CASE					=     1_00_00
       SCREAMING_SNAKE_CASE					=     None






       def 				UpperCamelCase (							self					)			->	str:
          if self.sql is None:
             raise ValueError("""sql must be specified"""					)
          if self.con is None:
             raise ValueError("""con must be specified"""					)






       def 				UpperCamelCase (							self				,							A__				,							A__ = None				,							)			->	str:
          _SCREAMING_SNAKE_CASE          = config_kwargs.copy()
          # We need to stringify the Selectable object to make its hash deterministic

          # The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
          _SCREAMING_SNAKE_CASE          = config_kwargs["""sql"""]
          if not isinstance(A__				,							A__					):
             if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
                import sqlalchemy

                if isinstance(A__				,							sqlalchemy.sql.Selectable					):
                   _SCREAMING_SNAKE_CASE          = sqlalchemy.create_engine(config_kwargs["""con"""].split("""://"""					)[0] + """://"""					)
                   _SCREAMING_SNAKE_CASE          = str(sql.compile(dialect=engine.dialect					)					)
                   _SCREAMING_SNAKE_CASE          = sql_str
                else:
                   raise TypeError(
                       F"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(A__					)}: {sql}"					)
             else:
                raise TypeError(
                    F"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(A__					)}: {sql}"					)
          _SCREAMING_SNAKE_CASE          = config_kwargs["""con"""]
          if not isinstance(A__				,							A__					):
             _SCREAMING_SNAKE_CASE          = id(A__					)
             logger.info(
                 F"SQL connection 'con' of type {type(A__					)} couldn't be hashed properly. To enable hashing, specify 'con' as URI string instead."					)

          return super().create_config_id(A__				,							custom_features=A__					)






       @property
       def 				UpperCamelCase (							self					)			->	Any:
          _SCREAMING_SNAKE_CASE          = {
              """index_col""": self.index_col,
              """columns""": self.columns,
              """params""": self.params,
              """coerce_float""": self.coerce_float,
              """parse_dates""": self.parse_dates,
          }
          return pd_read_sql_kwargs






class       _a (datasets.ArrowBasedBuilder):





       """simple docstring"""




       SCREAMING_SNAKE_CASE					=     SqlConfig






       def 				UpperCamelCase (							self					)			->	Optional[Any]:
          return datasets.DatasetInfo(features=self.config.features					)






       def 				UpperCamelCase (							self				,							A__					)			->	Dict:
          return [datasets.SplitGenerator(name=datasets.Split.TRAIN				,							gen_kwargs={}					)]






       def 				UpperCamelCase (							self				,							A__					)			->	pa.Table:
          if self.config.features is not None:
             _SCREAMING_SNAKE_CASE          = self.config.features.arrow_schema
             if all(not require_storage_cast(A__					) for feature in self.config.features.values()					):
                # cheaper cast
                _SCREAMING_SNAKE_CASE          = pa.Table.from_arrays([pa_table[field.name] for field in schema]				,							schema=A__					)
             else:
                # more expensive cast; allows str <-> int/float or str to Audio for example
                _SCREAMING_SNAKE_CASE          = table_cast(A__				,							A__					)
          return pa_table






       def 				UpperCamelCase (							self					)			->	Optional[int]:
          _SCREAMING_SNAKE_CASE          = self.config.chunksize
          _SCREAMING_SNAKE_CASE          = pd.read_sql(
              self.config.sql				,							self.config.con				,							chunksize=A__				,							**self.config.pd_read_sql_kwargs					)
          _SCREAMING_SNAKE_CASE          = [sql_reader] if chunksize is None else sql_reader
          for chunk_idx, df in enumerate(A__					):
             _SCREAMING_SNAKE_CASE          = pa.Table.from_pandas(A__					)
             yield chunk_idx, self._cast_table(A__					)






