






'''simple docstring'''

import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to properly calculate the metrics on the
# validation dataset when in a distributed system, and builds off the
# `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


UpperCamelCase							=16
UpperCamelCase							=32



def 							snake_case					(     a_							: Accelerator    ,   a_							: int = 16	)       ->				Tuple:





      """simple docstring"""

      UpperCamelCase_				:   Union[str, Any]					    =							AutoTokenizer.from_pretrained("""bert-base-cased"""	)
      UpperCamelCase_				:   Any					    =							load_dataset("""glue"""    ,   """mrpc"""	)

      def tokenize_function(a_							: Dict	):
            # max_length=None => use the model max length (it's actually the default)
            UpperCamelCase_				:   List[Any]					    =							tokenizer(examples["""sentence1"""]    ,   examples["""sentence2"""]    ,   truncation=a_    ,   max_length=a_	)
            return outputs

      # Apply the method we just defined to all the examples in all the splits of the dataset
      # starting with the main process first:
      with accelerator.main_process_first():
            UpperCamelCase_				:   Any					    =							datasets.map(
                a_    ,   batched=a_    ,   remove_columns=["""idx""", """sentence1""", """sentence2"""]    ,   )

      # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
      # transformers library
      UpperCamelCase_				:   Dict					    =							tokenized_datasets.rename_column("""label"""    ,   """labels"""	)

      def collate_fn(a_							: Dict	):
            # On TPU it's best to pad everything to the same length or training will be very slow.
            UpperCamelCase_				:   List[Any]					    =							128 if accelerator.distributed_type == DistributedType.TPU else None
            # When using mixed precision we want round multiples of 8/16
            if accelerator.mixed_precision == "fp8":
                  UpperCamelCase_				:   Optional[Any]					    =							16
            elif accelerator.mixed_precision != "no":
                  UpperCamelCase_				:   Optional[Any]					    =							8
            else:
                  UpperCamelCase_				:   Union[str, Any]					    =							None

            return tokenizer.pad(
                a_    ,   padding="""longest"""    ,   max_length=a_    ,   pad_to_multiple_of=a_    ,   return_tensors="""pt"""    ,   )

      # Instantiate dataloaders.
      UpperCamelCase_				:   Dict					    =							DataLoader(
          tokenized_datasets["""train"""]    ,   shuffle=a_    ,   collate_fn=a_    ,   batch_size=a_	)
      UpperCamelCase_				:   Optional[Any]					    =							DataLoader(
          tokenized_datasets["""validation"""]    ,   shuffle=a_    ,   collate_fn=a_    ,   batch_size=a_	)

      return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("TESTING_MOCKED_DATALOADERS", None) == "1":
      from accelerate.test_utils.training import mocked_dataloaders

      UpperCamelCase							=mocked_dataloaders  # noqa: F811



def 							snake_case					(     a_							: Dict    ,   a_							: List[str]	)       ->				Any:





      """simple docstring"""

      if os.environ.get("""TESTING_MOCKED_DATALOADERS"""    ,   a_	) == "1":
            UpperCamelCase_				:   List[str]					    =							2
      # Initialize accelerator
      UpperCamelCase_				:   Any					    =							Accelerator(cpu=args.cpu    ,   mixed_precision=args.mixed_precision	)
      # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
      UpperCamelCase_				:   Any					    =							config["""lr"""]
      UpperCamelCase_				:   Optional[Any]					    =							int(config["""num_epochs"""]	)
      UpperCamelCase_				:   Dict					    =							int(config["""seed"""]	)
      UpperCamelCase_				:   List[Any]					    =							int(config["""batch_size"""]	)

      UpperCamelCase_				:   Optional[int]					    =							evaluate.load("""glue"""    ,   """mrpc"""	)

      # If the batch size is too big we use gradient accumulation
      UpperCamelCase_				:   Dict					    =							1
      if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
            UpperCamelCase_				:   List[str]					    =							batch_size // MAX_GPU_BATCH_SIZE
            UpperCamelCase_				:   Optional[int]					    =							MAX_GPU_BATCH_SIZE

      set_seed(a_	)
      UpperCamelCase_				:   Dict					    =							get_dataloaders(a_    ,   a_	)
      # Instantiate the model (we build the model here so that the seed also control new weights initialization)
      UpperCamelCase_				:   Optional[Any]					    =							AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""    ,   return_dict=a_	)

      # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
      # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
      # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
      UpperCamelCase_				:   Tuple					    =							model.to(accelerator.device	)

      # Instantiate optimizer
      UpperCamelCase_				:   Any					    =							AdamW(params=model.parameters()    ,   lr=a_	)

      # Instantiate scheduler
      UpperCamelCase_				:   Optional[Any]					    =							get_linear_schedule_with_warmup(
          optimizer=a_    ,   num_warmup_steps=100    ,   num_training_steps=(len(a_	) * num_epochs) // gradient_accumulation_steps    ,   )

      # Prepare everything
      # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
      # prepare method.
      UpperCamelCase_				:   Optional[int]					    =							accelerator.prepare(
          a_    ,   a_    ,   a_    ,   a_    ,   a_	)

      # Now we train the model
      for epoch in range(a_	):
            model.train()
            for step, batch in enumerate(a_	):
                  # We could avoid this line since we set the accelerator with `device_placement=True`.
                  batch.to(accelerator.device	)
                  UpperCamelCase_				:   str					    =							model(**a_	)
                  UpperCamelCase_				:   str					    =							outputs.loss
                  UpperCamelCase_				:   Union[str, Any]					    =							loss / gradient_accumulation_steps
                  accelerator.backward(a_	)
                  if step % gradient_accumulation_steps == 0:
                        optimizer.step()
                        lr_scheduler.step()
                        optimizer.zero_grad()

            model.eval()
            UpperCamelCase_				:   Optional[Any]					    =							0
            for step, batch in enumerate(a_	):
                  # We could avoid this line since we set the accelerator with `device_placement=True`.
                  batch.to(accelerator.device	)
                  with torch.no_grad():
                        UpperCamelCase_				:   Dict					    =							model(**a_	)
                  UpperCamelCase_				:   Optional[Any]					    =							outputs.logits.argmax(dim=-1	)
                  UpperCamelCase_				:   List[str]					    =							accelerator.gather((predictions, batch["""labels"""])	)
                  # New Code #
                  # First we check if it's a distributed system
                  if accelerator.use_distributed:
                        # Then see if we're on the last batch of our eval dataloader
                        if step == len(a_	) - 1:
                              # Last batch needs to be truncated on distributed systems as it contains additional samples
                              UpperCamelCase_				:   int					    =							predictions[: len(eval_dataloader.dataset	) - samples_seen]
                              UpperCamelCase_				:   str					    =							references[: len(eval_dataloader.dataset	) - samples_seen]
                        else:
                              # Otherwise we add the number of samples seen
                              samples_seen += references.shape[0]
            # All of this can be avoided if you use `Accelerator.gather_for_metrics` instead of `Accelerator.gather`:
            # accelerator.gather_for_metrics((predictions, batch["labels"]))
                  metric.add_batch(
                      predictions=a_    ,   references=a_    ,   )

            UpperCamelCase_				:   Union[str, Any]					    =							metric.compute()
            # Use accelerator.print to print only on the main process.
            accelerator.print(f"epoch {epoch}:"    ,   a_	)



def 							snake_case					(     )       ->				str:





      """simple docstring"""

      UpperCamelCase_				:   Dict					    =							argparse.ArgumentParser(description="""Simple example of training script."""	)
      parser.add_argument(
          """--mixed_precision"""    ,   type=a_    ,   default=a_    ,   choices=["""no""", """fp16""", """bf16""", """fp8"""]    ,   help="""Whether to use mixed precision. Choose"""
          """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
          """and an Nvidia Ampere GPU."""    ,   )
      parser.add_argument("""--cpu"""    ,   action="""store_true"""    ,   help="""If passed, will train on the CPU."""	)
      UpperCamelCase_				:   Tuple					    =							parser.parse_args()
      UpperCamelCase_				:   Any					    =							{"""lr""": 2e-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
      training_function(a_    ,   a_	)


if __name__ == "__main__":
      main()













'''simple docstring'''

import json
import os
import shutil
import tempfile
import unittest
from multiprocessing import get_context
from pathlib import Path

import datasets
import numpy as np
from datasets import load_dataset
from parameterized import parameterized

from transformers import AutoProcessor
from transformers.models.wavaveca import WavaVecaCTCTokenizer, WavaVecaFeatureExtractor
from transformers.models.wavaveca.tokenization_wavaveca import VOCAB_FILES_NAMES
from transformers.testing_utils import require_pyctcdecode, require_torch, require_torchaudio, slow
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_pyctcdecode_available, is_torch_available

from ..wavaveca.test_feature_extraction_wavaveca import floats_list


if is_pyctcdecode_available():
      from huggingface_hub import snapshot_download
      from pyctcdecode import BeamSearchDecoderCTC

      from transformers.models.wavaveca_with_lm import WavaVecaProcessorWithLM
      from transformers.models.wavaveca_with_lm.processing_wavaveca_with_lm import WavaVecaDecoderWithLMOutput

if is_torch_available():
      from transformers import WavaVecaForCTC




@require_pyctcdecode
class     A    (      unittest.TestCase						):






 """simple docstring"""





 def   _UpperCAmelCase       (							self ):
       UpperCamelCase_				:   str					    =							"""| <pad> <unk> <s> </s> a b c d e f g h i j k""".split()
       UpperCamelCase_				:   int					    =							dict(zip(__lowerCAmelCase			,     range(len(__lowerCAmelCase ) ) ) )

       UpperCamelCase_				:   Tuple					    =							{
           """unk_token""": """<unk>""",
           """bos_token""": """<s>""",
           """eos_token""": """</s>""",
       }
       UpperCamelCase_				:   Optional[Any]					    =							{
           """feature_size""": 1,
           """padding_value""": 0.0,
           """sampling_rate""": 1_60_00,
           """return_attention_mask""": False,
           """do_normalize""": True,
       }

       UpperCamelCase_				:   int					    =							tempfile.mkdtemp()
       UpperCamelCase_				:   Tuple					    =							os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES["""vocab_file"""] )
       UpperCamelCase_				:   List[Any]					    =							os.path.join(self.tmpdirname			,     __lowerCAmelCase )
       with open(self.vocab_file			,     """w"""			,     encoding="""utf-8""" ) as fp:
             fp.write(json.dumps(__lowerCAmelCase ) + """\n""" )

       with open(self.feature_extraction_file			,     """w"""			,     encoding="""utf-8""" ) as fp:
             fp.write(json.dumps(__lowerCAmelCase ) + """\n""" )

       # load decoder from hub
       UpperCamelCase_				:   Union[str, Any]					    =							"""hf-internal-testing/ngram-beam-search-decoder"""
 def   _UpperCAmelCase       (							self			,     **__lowerCAmelCase ):
       UpperCamelCase_				:   str					    =							self.add_kwargs_tokens_map.copy()
       kwargs.update(__lowerCAmelCase )
       return WavaVecaCTCTokenizer.from_pretrained(self.tmpdirname			,     **__lowerCAmelCase )
 def   _UpperCAmelCase       (							self			,     **__lowerCAmelCase ):
       return WavaVecaFeatureExtractor.from_pretrained(self.tmpdirname			,     **__lowerCAmelCase )
 def   _UpperCAmelCase       (							self			,     **__lowerCAmelCase ):
       return BeamSearchDecoderCTC.load_from_hf_hub(self.decoder_name			,     **__lowerCAmelCase )
 def   _UpperCAmelCase       (							self ):
       shutil.rmtree(self.tmpdirname )
 def   _UpperCAmelCase       (							self ):
       UpperCamelCase_				:   Union[str, Any]					    =							self.get_tokenizer()
       UpperCamelCase_				:   str					    =							self.get_feature_extractor()
       UpperCamelCase_				:   Tuple					    =							self.get_decoder()

       UpperCamelCase_				:   int					    =							WavaVecaProcessorWithLM(tokenizer=__lowerCAmelCase			,     feature_extractor=__lowerCAmelCase			,     decoder=__lowerCAmelCase )

       processor.save_pretrained(self.tmpdirname )
       UpperCamelCase_				:   Dict					    =							WavaVecaProcessorWithLM.from_pretrained(self.tmpdirname )

       # tokenizer
       self.assertEqual(processor.tokenizer.get_vocab()			,     tokenizer.get_vocab() )
       self.assertIsInstance(processor.tokenizer			,     __lowerCAmelCase )

       # feature extractor
       self.assertEqual(processor.feature_extractor.to_json_string()			,     feature_extractor.to_json_string() )
       self.assertIsInstance(processor.feature_extractor			,     __lowerCAmelCase )

       # decoder
       self.assertEqual(processor.decoder._alphabet.labels			,     decoder._alphabet.labels )
       self.assertEqual(
           processor.decoder.model_container[decoder._model_key]._unigram_set			,     decoder.model_container[decoder._model_key]._unigram_set			,     )
       self.assertIsInstance(processor.decoder			,     __lowerCAmelCase )
 def   _UpperCAmelCase       (							self ):
       UpperCamelCase_				:   Union[str, Any]					    =							WavaVecaProcessorWithLM(
           tokenizer=self.get_tokenizer()			,     feature_extractor=self.get_feature_extractor()			,     decoder=self.get_decoder() )
       processor.save_pretrained(self.tmpdirname )

       # make sure that error is thrown when decoder alphabet doesn't match
       UpperCamelCase_				:   Optional[Any]					    =							WavaVecaProcessorWithLM.from_pretrained(
           self.tmpdirname			,     alpha=5.0			,     beta=3.0			,     score_boundary=-7.0			,     unk_score_offset=3 )

       # decoder
       self.assertEqual(processor.language_model.alpha			,     5.0 )
       self.assertEqual(processor.language_model.beta			,     3.0 )
       self.assertEqual(processor.language_model.score_boundary			,     -7.0 )
       self.assertEqual(processor.language_model.unk_score_offset			,     3 )
 def   _UpperCAmelCase       (							self ):
       UpperCamelCase_				:   int					    =							self.get_tokenizer()
       # add token to trigger raise
       tokenizer.add_tokens(["""xx"""] )
       with self.assertRaisesRegex(__lowerCAmelCase			,     """include""" ):
             WavaVecaProcessorWithLM(
                 tokenizer=__lowerCAmelCase			,     feature_extractor=self.get_feature_extractor()			,     decoder=self.get_decoder() )
 def   _UpperCAmelCase       (							self ):
       UpperCamelCase_				:   Tuple					    =							self.get_feature_extractor()
       UpperCamelCase_				:   Tuple					    =							self.get_tokenizer()
       UpperCamelCase_				:   Any					    =							self.get_decoder()

       UpperCamelCase_				:   List[Any]					    =							WavaVecaProcessorWithLM(tokenizer=__lowerCAmelCase			,     feature_extractor=__lowerCAmelCase			,     decoder=__lowerCAmelCase )

       UpperCamelCase_				:   List[Any]					    =							floats_list((3, 10_00) )

       UpperCamelCase_				:   Tuple					    =							feature_extractor(__lowerCAmelCase			,     return_tensors="""np""" )
       UpperCamelCase_				:   str					    =							processor(__lowerCAmelCase			,     return_tensors="""np""" )

       for key in input_feat_extract.keys():
             self.assertAlmostEqual(input_feat_extract[key].sum()			,     input_processor[key].sum()			,     delta=1E-2 )
 def   _UpperCAmelCase       (							self ):
       UpperCamelCase_				:   Dict					    =							self.get_feature_extractor()
       UpperCamelCase_				:   List[Any]					    =							self.get_tokenizer()
       UpperCamelCase_				:   List[Any]					    =							self.get_decoder()

       UpperCamelCase_				:   Optional[Any]					    =							WavaVecaProcessorWithLM(tokenizer=__lowerCAmelCase			,     feature_extractor=__lowerCAmelCase			,     decoder=__lowerCAmelCase )

       UpperCamelCase_				:   List[str]					    =							"""This is a test string"""

       UpperCamelCase_				:   Optional[Any]					    =							processor(text=__lowerCAmelCase )

       UpperCamelCase_				:   int					    =							tokenizer(__lowerCAmelCase )

       for key in encoded_tok.keys():
             self.assertListEqual(encoded_tok[key]			,     encoded_processor[key] )
 def   _UpperCAmelCase       (							self			,     __lowerCAmelCase=(2, 10, 16)			,     __lowerCAmelCase=77 ):
       np.random.seed(__lowerCAmelCase )
       return np.random.rand(*__lowerCAmelCase )
 def   _UpperCAmelCase       (							self ):
       UpperCamelCase_				:   int					    =							self.get_feature_extractor()
       UpperCamelCase_				:   Tuple					    =							self.get_tokenizer()
       UpperCamelCase_				:   Optional[int]					    =							self.get_decoder()

       UpperCamelCase_				:   int					    =							WavaVecaProcessorWithLM(tokenizer=__lowerCAmelCase			,     feature_extractor=__lowerCAmelCase			,     decoder=__lowerCAmelCase )

       UpperCamelCase_				:   List[str]					    =							self._get_dummy_logits(shape=(10, 16)			,     seed=13 )

       UpperCamelCase_				:   Any					    =							processor.decode(__lowerCAmelCase )

       UpperCamelCase_				:   Any					    =							decoder.decode_beams(__lowerCAmelCase )[0]

       self.assertEqual(decoded_decoder[0]			,     decoded_processor.text )
       self.assertEqual("""</s> <s> </s>"""			,     decoded_processor.text )
       self.assertEqual(decoded_decoder[-2]			,     decoded_processor.logit_score )
       self.assertEqual(decoded_decoder[-1]			,     decoded_processor.lm_score )
 @parameterized.expand([[None], ["""fork"""], ["""spawn"""]] )
 def   _UpperCAmelCase       (							self			,     __lowerCAmelCase ):
       UpperCamelCase_				:   Union[str, Any]					    =							self.get_feature_extractor()
       UpperCamelCase_				:   str					    =							self.get_tokenizer()
       UpperCamelCase_				:   List[Any]					    =							self.get_decoder()

       UpperCamelCase_				:   Optional[Any]					    =							WavaVecaProcessorWithLM(tokenizer=__lowerCAmelCase			,     feature_extractor=__lowerCAmelCase			,     decoder=__lowerCAmelCase )

       UpperCamelCase_				:   Union[str, Any]					    =							self._get_dummy_logits()

       # note: pool should be instantiated *after* Wav2Vec2ProcessorWithLM.
       #       otherwise, the LM won't be available to the pool's sub-processes.
       # manual logic used to allow parameterized test for both pool=None and pool=Pool(...)
       if pool_context is None:
             UpperCamelCase_				:   List[Any]					    =							processor.batch_decode(__lowerCAmelCase )
       else:
             with get_context(__lowerCAmelCase ).Pool() as pool:
                   UpperCamelCase_				:   Any					    =							processor.batch_decode(__lowerCAmelCase			,     __lowerCAmelCase )

       UpperCamelCase_				:   Tuple					    =							list(__lowerCAmelCase )

       with get_context("""fork""" ).Pool() as p:
             UpperCamelCase_				:   Optional[int]					    =							decoder.decode_beams_batch(__lowerCAmelCase			,     __lowerCAmelCase )

       UpperCamelCase_    ,			UpperCamelCase_    ,			UpperCamelCase_				:   Union[str, Any]					    =							[], [], []
       for beams in decoded_beams:
             texts_decoder.append(beams[0][0] )
             logit_scores_decoder.append(beams[0][-2] )
             lm_scores_decoder.append(beams[0][-1] )

       self.assertListEqual(__lowerCAmelCase			,     decoded_processor.text )
       self.assertListEqual(["""<s> <s> </s>""", """<s> <s> <s>"""]			,     decoded_processor.text )
       self.assertListEqual(__lowerCAmelCase			,     decoded_processor.logit_score )
       self.assertListEqual(__lowerCAmelCase			,     decoded_processor.lm_score )
 def   _UpperCAmelCase       (							self ):
       UpperCamelCase_				:   Dict					    =							self.get_feature_extractor()
       UpperCamelCase_				:   Tuple					    =							self.get_tokenizer()
       UpperCamelCase_				:   Tuple					    =							self.get_decoder()

       UpperCamelCase_				:   Optional[Any]					    =							WavaVecaProcessorWithLM(tokenizer=__lowerCAmelCase			,     feature_extractor=__lowerCAmelCase			,     decoder=__lowerCAmelCase )

       UpperCamelCase_				:   List[str]					    =							self._get_dummy_logits()

       UpperCamelCase_				:   Dict					    =							15
       UpperCamelCase_				:   str					    =							-20.0
       UpperCamelCase_				:   Dict					    =							-4.0

       UpperCamelCase_				:   Union[str, Any]					    =							processor.batch_decode(
           __lowerCAmelCase			,     beam_width=__lowerCAmelCase			,     beam_prune_logp=__lowerCAmelCase			,     token_min_logp=__lowerCAmelCase			,     )
       UpperCamelCase_				:   Any					    =							decoded_processor_out.text

       UpperCamelCase_				:   Tuple					    =							list(__lowerCAmelCase )

       with get_context("""fork""" ).Pool() as pool:
             UpperCamelCase_				:   str					    =							decoder.decode_beams_batch(
                 __lowerCAmelCase			,     __lowerCAmelCase			,     beam_width=__lowerCAmelCase			,     beam_prune_logp=__lowerCAmelCase			,     token_min_logp=__lowerCAmelCase			,     )

       UpperCamelCase_				:   str					    =							[d[0][0] for d in decoded_decoder_out]
       UpperCamelCase_				:   List[str]					    =							[d[0][2] for d in decoded_decoder_out]
       UpperCamelCase_				:   Union[str, Any]					    =							[d[0][3] for d in decoded_decoder_out]

       self.assertListEqual(__lowerCAmelCase			,     __lowerCAmelCase )
       self.assertListEqual(["""</s> <s> <s>""", """<s> <s> <s>"""]			,     __lowerCAmelCase )

       self.assertTrue(np.array_equal(__lowerCAmelCase			,     decoded_processor_out.logit_score ) )
       self.assertTrue(np.allclose([-20.0_54, -18.4_47]			,     __lowerCAmelCase			,     atol=1E-3 ) )

       self.assertTrue(np.array_equal(__lowerCAmelCase			,     decoded_processor_out.lm_score ) )
       self.assertTrue(np.allclose([-15.5_54, -13.94_74]			,     __lowerCAmelCase			,     atol=1E-3 ) )
 def   _UpperCAmelCase       (							self ):
       UpperCamelCase_				:   Optional[int]					    =							self.get_feature_extractor()
       UpperCamelCase_				:   Union[str, Any]					    =							self.get_tokenizer()
       UpperCamelCase_				:   int					    =							self.get_decoder()

       UpperCamelCase_				:   Dict					    =							WavaVecaProcessorWithLM(tokenizer=__lowerCAmelCase			,     feature_extractor=__lowerCAmelCase			,     decoder=__lowerCAmelCase )

       UpperCamelCase_				:   str					    =							self._get_dummy_logits()

       UpperCamelCase_				:   Optional[int]					    =							2.0
       UpperCamelCase_				:   List[str]					    =							5.0
       UpperCamelCase_				:   Optional[Any]					    =							-20.0
       UpperCamelCase_				:   Optional[Any]					    =							True

       UpperCamelCase_				:   Union[str, Any]					    =							processor.batch_decode(
           __lowerCAmelCase			,     alpha=__lowerCAmelCase			,     beta=__lowerCAmelCase			,     unk_score_offset=__lowerCAmelCase			,     lm_score_boundary=__lowerCAmelCase			,     )
       UpperCamelCase_				:   List[str]					    =							decoded_processor_out.text

       UpperCamelCase_				:   List[str]					    =							list(__lowerCAmelCase )
       decoder.reset_params(
           alpha=__lowerCAmelCase			,     beta=__lowerCAmelCase			,     unk_score_offset=__lowerCAmelCase			,     lm_score_boundary=__lowerCAmelCase			,     )

       with get_context("""fork""" ).Pool() as pool:
             UpperCamelCase_				:   int					    =							decoder.decode_beams_batch(
                 __lowerCAmelCase			,     __lowerCAmelCase			,     )

       UpperCamelCase_				:   Any					    =							[d[0][0] for d in decoded_decoder_out]

       self.assertListEqual(__lowerCAmelCase			,     __lowerCAmelCase )
       self.assertListEqual(["""<s> </s> <s> </s> </s>""", """</s> </s> <s> </s> </s>"""]			,     __lowerCAmelCase )
       UpperCamelCase_				:   Union[str, Any]					    =							processor.decoder.model_container[processor.decoder._model_key]
       self.assertEqual(lm_model.alpha			,     2.0 )
       self.assertEqual(lm_model.beta			,     5.0 )
       self.assertEqual(lm_model.unk_score_offset			,     -20.0 )
       self.assertEqual(lm_model.score_boundary			,     __lowerCAmelCase )
 def   _UpperCAmelCase       (							self ):
       UpperCamelCase_				:   Dict					    =							WavaVecaProcessorWithLM.from_pretrained("""hf-internal-testing/processor_with_lm""" )

       UpperCamelCase_				:   Optional[int]					    =							processor.decoder.model_container[processor.decoder._model_key]
       UpperCamelCase_				:   int					    =							Path(language_model._kenlm_model.path.decode("""utf-8""" ) ).parent.parent.absolute()

       UpperCamelCase_				:   Any					    =							os.listdir(__lowerCAmelCase )
       UpperCamelCase_				:   Optional[Any]					    =							["""alphabet.json""", """language_model"""]

       downloaded_decoder_files.sort()
       expected_decoder_files.sort()

       # test that only decoder relevant files from
       # https://huggingface.co/hf-internal-testing/processor_with_lm/tree/main
       # are downloaded and none of the rest (e.g. README.md, ...)
       self.assertListEqual(__lowerCAmelCase			,     __lowerCAmelCase )
 def   _UpperCAmelCase       (							self ):
       UpperCamelCase_				:   Union[str, Any]					    =							snapshot_download("""hf-internal-testing/processor_with_lm""" )

       UpperCamelCase_				:   Union[str, Any]					    =							WavaVecaProcessorWithLM.from_pretrained(__lowerCAmelCase )

       UpperCamelCase_				:   Union[str, Any]					    =							processor.decoder.model_container[processor.decoder._model_key]
       UpperCamelCase_				:   Tuple					    =							Path(language_model._kenlm_model.path.decode("""utf-8""" ) ).parent.parent.absolute()

       UpperCamelCase_				:   Union[str, Any]					    =							os.listdir(__lowerCAmelCase )
       UpperCamelCase_				:   str					    =							os.listdir(__lowerCAmelCase )

       local_decoder_files.sort()
       expected_decoder_files.sort()

       # test that both decoder form hub and local files in cache are the same
       self.assertListEqual(__lowerCAmelCase			,     __lowerCAmelCase )
 def   _UpperCAmelCase       (							self ):
       UpperCamelCase_				:   Any					    =							WavaVecaProcessorWithLM.from_pretrained("""hf-internal-testing/processor_with_lm""" )
       UpperCamelCase_				:   Tuple					    =							AutoProcessor.from_pretrained("""hf-internal-testing/processor_with_lm""" )

       UpperCamelCase_				:   Dict					    =							floats_list((3, 10_00) )

       UpperCamelCase_				:   List[Any]					    =							processor_wavaveca(__lowerCAmelCase			,     return_tensors="""np""" )
       UpperCamelCase_				:   Tuple					    =							processor_auto(__lowerCAmelCase			,     return_tensors="""np""" )

       for key in input_wavaveca.keys():
             self.assertAlmostEqual(input_wavaveca[key].sum()			,     input_auto[key].sum()			,     delta=1E-2 )

       UpperCamelCase_				:   Optional[int]					    =							self._get_dummy_logits()

       UpperCamelCase_				:   Dict					    =							processor_wavaveca.batch_decode(__lowerCAmelCase )
       UpperCamelCase_				:   Any					    =							processor_auto.batch_decode(__lowerCAmelCase )

       self.assertListEqual(decoded_wavaveca.text			,     decoded_auto.text )
 def   _UpperCAmelCase       (							self ):
       UpperCamelCase_				:   Tuple					    =							self.get_feature_extractor()
       UpperCamelCase_				:   int					    =							self.get_tokenizer()
       UpperCamelCase_				:   List[Any]					    =							self.get_decoder()

       UpperCamelCase_				:   List[str]					    =							WavaVecaProcessorWithLM(tokenizer=__lowerCAmelCase			,     feature_extractor=__lowerCAmelCase			,     decoder=__lowerCAmelCase )

       self.assertListEqual(
           processor.model_input_names			,     feature_extractor.model_input_names			,     msg="""`processor` and `feature_extractor` model input names do not match"""			,     )
 @staticmethod
 def   _UpperCAmelCase       (							__lowerCAmelCase			,     __lowerCAmelCase ):
       UpperCamelCase_				:   Union[str, Any]					    =							[d[key] for d in offsets]
       return retrieved_list
 def   _UpperCAmelCase       (							self ):
       UpperCamelCase_				:   Dict					    =							WavaVecaProcessorWithLM.from_pretrained("""hf-internal-testing/processor_with_lm""" )
       UpperCamelCase_				:   int					    =							self._get_dummy_logits()[0]

       UpperCamelCase_				:   List[Any]					    =							processor.decode(__lowerCAmelCase			,     output_word_offsets=__lowerCAmelCase )
       # check Wav2Vec2CTCTokenizerOutput keys for word
       self.assertEqual(len(outputs.keys() )			,     4 )
       self.assertTrue("""text""" in outputs )
       self.assertTrue("""word_offsets""" in outputs )
       self.assertTrue(isinstance(__lowerCAmelCase			,     __lowerCAmelCase ) )

       self.assertEqual(""" """.join(self.get_from_offsets(outputs["""word_offsets"""]			,     """word""" ) )			,     outputs.text )
       self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""]			,     """word""" )			,     ["""<s>""", """<s>""", """</s>"""] )
       self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""]			,     """start_offset""" )			,     [0, 2, 4] )
       self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""]			,     """end_offset""" )			,     [1, 3, 5] )
 def   _UpperCAmelCase       (							self ):
       UpperCamelCase_				:   Any					    =							WavaVecaProcessorWithLM.from_pretrained("""hf-internal-testing/processor_with_lm""" )
       UpperCamelCase_				:   Union[str, Any]					    =							self._get_dummy_logits()

       UpperCamelCase_				:   Dict					    =							processor.batch_decode(__lowerCAmelCase			,     output_word_offsets=__lowerCAmelCase )

       # check Wav2Vec2CTCTokenizerOutput keys for word
       self.assertEqual(len(outputs.keys() )			,     4 )
       self.assertTrue("""text""" in outputs )
       self.assertTrue("""word_offsets""" in outputs )
       self.assertTrue(isinstance(__lowerCAmelCase			,     __lowerCAmelCase ) )

       self.assertListEqual(
           [""" """.join(self.get_from_offsets(__lowerCAmelCase			,     """word""" ) ) for o in outputs["""word_offsets"""]]			,     outputs.text )
       self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""][0]			,     """word""" )			,     ["""<s>""", """<s>""", """</s>"""] )
       self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""][0]			,     """start_offset""" )			,     [0, 2, 4] )
       self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""][0]			,     """end_offset""" )			,     [1, 3, 5] )





 @slow
 @require_torch
 @require_torchaudio
 def   _UpperCAmelCase       (							self ):
       import torch

       UpperCamelCase_				:   str					    =							load_dataset("""common_voice"""			,     """en"""			,     split="""train"""			,     streaming=__lowerCAmelCase )
       UpperCamelCase_				:   Union[str, Any]					    =							ds.cast_column("""audio"""			,     datasets.Audio(sampling_rate=1_60_00 ) )
       UpperCamelCase_				:   Union[str, Any]					    =							iter(__lowerCAmelCase )
       UpperCamelCase_				:   int					    =							next(__lowerCAmelCase )

       UpperCamelCase_				:   List[str]					    =							AutoProcessor.from_pretrained("""patrickvonplaten/wav2vec2-base-100h-with-lm""" )
       UpperCamelCase_				:   Tuple					    =							WavaVecaForCTC.from_pretrained("""patrickvonplaten/wav2vec2-base-100h-with-lm""" )

       # compare to filename `common_voice_en_100038.mp3` of dataset viewer on https://huggingface.co/datasets/common_voice/viewer/en/train
       UpperCamelCase_				:   Dict					    =							processor(sample["""audio"""]["""array"""]			,     return_tensors="""pt""" ).input_values

       with torch.no_grad():
             UpperCamelCase_				:   List[Any]					    =							model(__lowerCAmelCase ).logits.cpu().numpy()

       UpperCamelCase_				:   Tuple					    =							processor.decode(logits[0]			,     output_word_offsets=__lowerCAmelCase )

       UpperCamelCase_				:   List[Any]					    =							model.config.inputs_to_logits_ratio / processor.feature_extractor.sampling_rate
       UpperCamelCase_				:   Dict					    =							[
           {
               """start_time""": d["""start_offset"""] * time_offset,
               """end_time""": d["""end_offset"""] * time_offset,
               """word""": d["""word"""],
           }
           for d in output["""word_offsets"""]
       ]

       UpperCamelCase_				:   Optional[Any]					    =							"""WHY DOES MILISANDRA LOOK LIKE SHE WANTS TO CONSUME JOHN SNOW ON THE RIVER AT THE WALL"""

       # output words
       self.assertEqual(""" """.join(self.get_from_offsets(__lowerCAmelCase			,     """word""" ) )			,     __lowerCAmelCase )
       self.assertEqual(""" """.join(self.get_from_offsets(__lowerCAmelCase			,     """word""" ) )			,     output.text )

       # output times
       UpperCamelCase_				:   str					    =							torch.tensor(self.get_from_offsets(__lowerCAmelCase			,     """start_time""" ) )
       UpperCamelCase_				:   Union[str, Any]					    =							torch.tensor(self.get_from_offsets(__lowerCAmelCase			,     """end_time""" ) )

       # fmt: off
       UpperCamelCase_				:   Union[str, Any]					    =							torch.tensor([1.41_99, 1.65_99, 2.25_99, 3.0, 3.24, 3.59_99, 3.79_99, 4.09_99, 4.26, 4.94, 5.28, 5.65_99, 5.78, 5.94, 6.32, 6.53_99, 6.65_99] )
       UpperCamelCase_				:   Union[str, Any]					    =							torch.tensor([1.53_99, 1.89_99, 2.9, 3.16, 3.53_99, 3.72, 4.01_99, 4.17_99, 4.76, 5.15_99, 5.55_99, 5.69_99, 5.86, 6.19_99, 6.38, 6.61_99, 6.94] )
       # fmt: on

       self.assertTrue(torch.allclose(__lowerCAmelCase			,     __lowerCAmelCase			,     atol=0.01 ) )
       self.assertTrue(torch.allclose(__lowerCAmelCase			,     __lowerCAmelCase			,     atol=0.01 ) )






