File size: 2,099 Bytes
5fc76ef |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
from typing import Callable
import torch
from loguru import logger
from fish_speech.models.vqgan.modules.firefly import FireflyArchitecture
class VQManager:
def __init__(self):
# Make Pylance happy (attribut/method not defined...)
self.decoder_model: FireflyArchitecture
self.load_audio: Callable
def decode_vq_tokens(self, codes):
feature_lengths = torch.tensor(
[codes.shape[1]], device=self.decoder_model.device
)
logger.info(f"VQ features: {codes.shape}")
if isinstance(self.decoder_model, FireflyArchitecture):
return self.decoder_model.decode(
indices=codes[None],
feature_lengths=feature_lengths,
)[0].squeeze()
raise ValueError(f"Unknown model type: {type(self.decoder_model)}")
def encode_reference(self, reference_audio, enable_reference_audio):
if enable_reference_audio and reference_audio is not None:
# Load audios, and prepare basic info here
reference_audio_content = self.load_audio(
reference_audio, self.decoder_model.spec_transform.sample_rate
)
audios = torch.from_numpy(reference_audio_content).to(
self.decoder_model.device
)[None, None, :]
audio_lengths = torch.tensor(
[audios.shape[2]], device=self.decoder_model.device, dtype=torch.long
)
logger.info(
f"Loaded audio with {audios.shape[2] / self.decoder_model.spec_transform.sample_rate:.2f} seconds"
)
# VQ Encoder
if isinstance(self.decoder_model, FireflyArchitecture):
prompt_tokens = self.decoder_model.encode(audios, audio_lengths)[0][0]
logger.info(f"Encoded prompt: {prompt_tokens.shape}")
else:
raise ValueError(f"Unknown model type: {type(self.decoder_model)}")
else:
prompt_tokens = None
logger.info("No reference audio provided")
return prompt_tokens
|