class_docstr: This lobe enables the integration of HuggingFace pretrained Whisper model.
  
  Source paper whisper:
      https://cdn.openai.com/papers/whisper.pdf
  Transformer from HuggingFace needs to be installed:
  https://huggingface.co/transformers/installation.html
  
  Some part of the code also cis adapted from the official OpenAI repository:
  https://github.com/openai/whisper
  
  The model can be finetuned. It will download automatically the model from
  HuggingFace or use a local path.
  
  Arguments
  ---------
  source : str
      HuggingFace hub name: e.g "openai/whisper-tiny"
  save_path : str
      Path (dir) of the downloaded model.
  sampling_rate : int (default: 16000)
      Sampling rate of the audio signal.
  encoder_only : bool (default: False)
      If True, the forward function outputs the hidden states from the last transformer layer of the encoder.
      If False, one step of the decoder is performed and returned.
  freeze : bool (default: False)
      If True, the model is frozen.
  freeze_encoder : bool (default: False)
      If True, the encoder is frozen.
  output_attentions : bool (default: True)
      If True, the forward function outputs the attention weights.
  output_all_hiddens: bool (default: False)
      If True, the forward function outputs the hidden states from all transformer layers of the encoder.
      For example whisper-base has 6 transformer layers and the output is of shape (7, B, T, C),
      where the output of the CNN output is added to the beginning.
      If False, the forward function outputs the hidden states only from the last transformer layer of the encoder.
  
  Example
  -------
  >>> model_hub = "openai/whisper-tiny"
  >>> save_path = "savedir"
  >>> sampling_rate = 16000
  >>> model = Whisper(model_hub, save_path, sampling_rate)
  >>> tokens = torch.tensor([[1, 1]]) * model.model.config.decoder_start_token_id
  >>> inputs = torch.randn([1, 93680])
  >>> outputs = model(inputs, tokens)
class_docstr_tok: ["This","lobe","enables","the","integration","of","HuggingFace","pretrained","Whisper","model",".","Source","paper","whisper",":","https",":","\/\/cdn.openai.com\/papers\/whisper.pdf","Transformer","from","HuggingFace","needs","to","be","installed",":","https",":","\/\/huggingface.co\/transformers\/installation.html","Some","part","of","the","code","also","cis","adapted","from","the","official","OpenAI","repository",":","https",":","\/\/github.com\/openai\/whisper","The","model","can","be","finetuned",".","It","will","download","automatically","the","model","from","HuggingFace","or","use","a","local","path",".","Arguments","--","--","--","--","-","source",":","str","HuggingFace","hub","name",":","e.g","``","openai\/whisper-tiny","''","save_path",":","str","Path","(","dir",")","of","the","downloaded","model",".","sampling_rate",":","int","(","default",":","16000",")","Sampling","rate","of","the","audio","signal",".","encoder_only",":","bool","(","default",":","False",")","If","True",",","the","forward","function","outputs","the","hidden","states","from","the","last","transformer","layer","of","the","encoder",".","If","False",",","one","step","of","the","decoder","is","performed","and","returned",".","freeze",":","bool","(","default",":","False",")","If","True",",","the","model","is","frozen",".","freeze_encoder",":","bool","(","default",":","False",")","If","True",",","the","encoder","is","frozen",".","output_attentions",":","bool","(","default",":","True",")","If","True",",","the","forward","function","outputs","the","attention","weights",".","output_all_hiddens",":","bool","(","default",":","False",")","If","True",",","the","forward","function","outputs","the","hidden","states","from","all","transformer","layers","of","the","encoder",".","For","example","whisper-base","has","6","transformer","layers","and","the","output","is","of","shape","(","7",",","B",",","T",",","C",")",",","where","the","output","of","the","CNN","output","is","added","to","the","beginning",".","If","False",",","the","forward","function","outputs","the","hidden","states","only","from","the","last","transformer","layer","of","the","encoder",".","Example","--","--","--","-",">",">",">","model_hub","=","``","openai\/whisper-tiny","''",">",">",">","save_path","=","``","savedir","''",">",">",">","sampling_rate","=","16000",">",">",">","model","=","Whisper","(","model_hub",",","save_path",",","sampling_rate",")",">",">",">","tokens","=","torch.tensor","(","[","[","1",",","1","]","]",")","*","model.model.config.decoder_start_token_id",">",">",">","inputs","=","torch.randn","(","[","1",",","93680","]",")",">",">",">","outputs","=","model","(","inputs",",","tokens",")"]
code: def __init__(
      self,
      source,
      save_path,
      sampling_rate=16000,
      encoder_only=False,
      freeze=False,
      freeze_encoder=False,
      output_attentions=True,
      output_all_hiddens=False,
  ):
      super().__init__(
          source=source,
          save_path=save_path,
          freeze=freeze,
          sampling_rate=sampling_rate,
      )
      self.sampling_rate = sampling_rate
      self.encoder_only = encoder_only
      self.freeze_encoder = freeze_encoder
      self.output_attentions = output_attentions
      self.output_all_hiddens = output_all_hiddens
  
      if encoder_only:
          self.tokenizer = None
      else:
          self.load_tokenizer(source)
  
      self.load_feature_extractor(
          source, save_path, sampling_rate=sampling_rate
      )
  
      self._n_fft = self.feature_extractor.n_fft
      self._hop_length = self.feature_extractor.hop_length
      self._n_samples = self.feature_extractor.n_samples
      # The following breaking changes were introduced in transformers>=4.29:
      # 1) mel_filters.shape = (..., feature_extractor.feature_size) instead of (feature_extractor.feature_size, ...)
      # 2) mel_filters.dtype = float64 instead of float32
      # The following code fixes the issue in a backward compatible way
      mel_filters = self.feature_extractor.mel_filters
      if mel_filters.shape[0] != self.feature_extractor.feature_size:
          mel_filters = mel_filters.T
      assert mel_filters.shape[0] == self.feature_extractor.feature_size
      self.register_buffer(
          "_mel_filters",
          torch.as_tensor(mel_filters, dtype=torch.float32),
      )
      #################################################################
  
      if not self.freeze and self.freeze_encoder:
          logger.warning(
              "speechbrain.lobes.models.huggingface_transformers.whisper - whisper encoder is frozen."
          )
          for param in self.model.encoder.parameters():
              param.requires_grad = False
  
file_path: speechbrain/speechbrain/lobes/models/huggingface_transformers/whisper.py
imports: import torch
  import logging
  from torch import nn
  from speechbrain.lobes.models.huggingface_transformers.huggingface import HFTransformersInterface
label_desc: Use image node_id 1 to create a new Whisper object from inherited base classes: HFTransformersInterface with example: obj = Whisper(source, save_path, sampling_rate, encoder_only, freeze, freeze_encoder, output_attentions, output_all_hiddens)