Spaces:
Running
on
Zero
Running
on
Zero
| from abc import ABC | |
| import torch | |
| import torch.nn.functional as F | |
| from modules.diffusion_transformer import DiT | |
| from modules.commons import sequence_mask | |
| from tqdm import tqdm | |
| class BASECFM(torch.nn.Module, ABC): | |
| def __init__( | |
| self, | |
| args, | |
| ): | |
| super().__init__() | |
| self.sigma_min = 1e-6 | |
| self.estimator = None | |
| self.in_channels = args.DiT.in_channels | |
| self.criterion = torch.nn.MSELoss() if args.reg_loss_type == "l2" else torch.nn.L1Loss() | |
| if hasattr(args.DiT, 'zero_prompt_speech_token'): | |
| self.zero_prompt_speech_token = args.DiT.zero_prompt_speech_token | |
| else: | |
| self.zero_prompt_speech_token = False | |
| def inference(self, mu, x_lens, prompt, style, f0, n_timesteps, temperature=1.0, inference_cfg_rate=0.5): | |
| """Forward diffusion | |
| Args: | |
| mu (torch.Tensor): output of encoder | |
| shape: (batch_size, n_feats, mel_timesteps) | |
| mask (torch.Tensor): output_mask | |
| shape: (batch_size, 1, mel_timesteps) | |
| n_timesteps (int): number of diffusion steps | |
| temperature (float, optional): temperature for scaling noise. Defaults to 1.0. | |
| spks (torch.Tensor, optional): speaker ids. Defaults to None. | |
| shape: (batch_size, spk_emb_dim) | |
| cond: Not used but kept for future purposes | |
| Returns: | |
| sample: generated mel-spectrogram | |
| shape: (batch_size, n_feats, mel_timesteps) | |
| """ | |
| B, T = mu.size(0), mu.size(1) | |
| z = torch.randn([B, self.in_channels, T], device=mu.device) * temperature | |
| t_span = torch.linspace(0, 1, n_timesteps + 1, device=mu.device) | |
| # t_span = t_span + (-1) * (torch.cos(torch.pi / 2 * t_span) - 1 + t_span) | |
| return self.solve_euler(z, x_lens, prompt, mu, style, f0, t_span, inference_cfg_rate) | |
| def solve_euler(self, x, x_lens, prompt, mu, style, f0, t_span, inference_cfg_rate=0.5): | |
| """ | |
| Fixed euler solver for ODEs. | |
| Args: | |
| x (torch.Tensor): random noise | |
| t_span (torch.Tensor): n_timesteps interpolated | |
| shape: (n_timesteps + 1,) | |
| mu (torch.Tensor): output of encoder | |
| shape: (batch_size, n_feats, mel_timesteps) | |
| mask (torch.Tensor): output_mask | |
| shape: (batch_size, 1, mel_timesteps) | |
| spks (torch.Tensor, optional): speaker ids. Defaults to None. | |
| shape: (batch_size, spk_emb_dim) | |
| cond: Not used but kept for future purposes | |
| """ | |
| t, _, _ = t_span[0], t_span[-1], t_span[1] - t_span[0] | |
| # I am storing this because I can later plot it by putting a debugger here and saving it to a file | |
| # Or in future might add like a return_all_steps flag | |
| sol = [] | |
| # apply prompt | |
| prompt_len = prompt.size(-1) | |
| prompt_x = torch.zeros_like(x) | |
| prompt_x[..., :prompt_len] = prompt[..., :prompt_len] | |
| x[..., :prompt_len] = 0 | |
| if self.zero_prompt_speech_token: | |
| mu[..., :prompt_len] = 0 | |
| for step in tqdm(range(1, len(t_span))): | |
| dt = t_span[step] - t_span[step - 1] | |
| if inference_cfg_rate > 0: | |
| # Stack original and CFG (null) inputs for batched processing | |
| stacked_prompt_x = torch.cat([prompt_x, torch.zeros_like(prompt_x)], dim=0) | |
| stacked_style = torch.cat([style, torch.zeros_like(style)], dim=0) | |
| stacked_mu = torch.cat([mu, torch.zeros_like(mu)], dim=0) | |
| stacked_x = torch.cat([x, x], dim=0) | |
| stacked_t = torch.cat([t.unsqueeze(0), t.unsqueeze(0)], dim=0) | |
| # Perform a single forward pass for both original and CFG inputs | |
| stacked_dphi_dt = self.estimator( | |
| stacked_x, stacked_prompt_x, x_lens, stacked_t, stacked_style, stacked_mu, | |
| ) | |
| # Split the output back into the original and CFG components | |
| dphi_dt, cfg_dphi_dt = stacked_dphi_dt.chunk(2, dim=0) | |
| # Apply CFG formula | |
| dphi_dt = (1.0 + inference_cfg_rate) * dphi_dt - inference_cfg_rate * cfg_dphi_dt | |
| else: | |
| dphi_dt = self.estimator(x, prompt_x, x_lens, t.unsqueeze(0), style, mu) | |
| x = x + dt * dphi_dt | |
| t = t + dt | |
| sol.append(x) | |
| if step < len(t_span) - 1: | |
| dt = t_span[step + 1] - t | |
| x[:, :, :prompt_len] = 0 | |
| return sol[-1] | |
| def forward(self, x1, x_lens, prompt_lens, mu, style): | |
| """Computes diffusion loss | |
| Args: | |
| x1 (torch.Tensor): Target | |
| shape: (batch_size, n_feats, mel_timesteps) | |
| mask (torch.Tensor): target mask | |
| shape: (batch_size, 1, mel_timesteps) | |
| mu (torch.Tensor): output of encoder | |
| shape: (batch_size, n_feats, mel_timesteps) | |
| spks (torch.Tensor, optional): speaker embedding. Defaults to None. | |
| shape: (batch_size, spk_emb_dim) | |
| Returns: | |
| loss: conditional flow matching loss | |
| y: conditional flow | |
| shape: (batch_size, n_feats, mel_timesteps) | |
| """ | |
| b, _, t = x1.shape | |
| # random timestep | |
| t = torch.rand([b, 1, 1], device=mu.device, dtype=x1.dtype) | |
| # sample noise p(x_0) | |
| z = torch.randn_like(x1) | |
| y = (1 - (1 - self.sigma_min) * t) * z + t * x1 | |
| u = x1 - (1 - self.sigma_min) * z | |
| prompt = torch.zeros_like(x1) | |
| for bib in range(b): | |
| prompt[bib, :, :prompt_lens[bib]] = x1[bib, :, :prompt_lens[bib]] | |
| # range covered by prompt are set to 0 | |
| y[bib, :, :prompt_lens[bib]] = 0 | |
| if self.zero_prompt_speech_token: | |
| mu[bib, :, :prompt_lens[bib]] = 0 | |
| estimator_out = self.estimator(y, prompt, x_lens, t.squeeze(1).squeeze(1), style, mu, prompt_lens) | |
| loss = 0 | |
| for bib in range(b): | |
| loss += self.criterion(estimator_out[bib, :, prompt_lens[bib]:x_lens[bib]], u[bib, :, prompt_lens[bib]:x_lens[bib]]) | |
| loss /= b | |
| return loss, estimator_out + (1 - self.sigma_min) * z | |
| class CFM(BASECFM): | |
| def __init__(self, args): | |
| super().__init__( | |
| args | |
| ) | |
| if args.dit_type == "DiT": | |
| self.estimator = DiT(args) | |
| else: | |
| raise NotImplementedError(f"Unknown diffusion type {args.dit_type}") | |