File size: 4,467 Bytes
9580089
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
import torch
import torchaudio
import numpy as np
from pathlib import Path
from typing import List, Union
from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan
from speechbrain.pretrained import EncoderClassifier
import tempfile
import os

class VoiceCloneSystem:
    """语音克隆系统:将输入文本转换为目标说话人的语音"""
    
    def __init__(self, device: str = "cpu"):
        """
        初始化语音克隆系统
        
        Args:
            device: 使用的设备,'cpu' 或 'cuda'
        """
        self.device = device
        print("正在加载模型...")
        
        # 加载说话人编码器
        self.speaker_encoder = EncoderClassifier.from_hparams(
            source="speechbrain/spkrec-xvect-voxceleb",
            savedir="tmp/spkrec-xvect-voxceleb",
            run_opts={"device": device}
        )
        
        # 加载文本到语音模型
        self.processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
        self.tts_model = SpeechT5ForTextToSpeech.from_pretrained(
            "microsoft/speecht5_tts"
        ).to(device)
        
        # 加载声码器
        self.vocoder = SpeechT5HifiGan.from_pretrained(
            "microsoft/speecht5_hifigan"
        ).to(device)
        
        print("模型加载完成!")
        
    def extract_speaker_embedding(
        self,
        audio_paths: List[Union[str, Path]]
    ) -> torch.Tensor:
        """
        从参考音频中提取说话人特征
        
        Args:
            audio_paths: 参考音频文件路径列表
            
        Returns:
            说话人特征向量
        """
        embeddings = []
        
        for audio_path in audio_paths:
            # 加载音频
            waveform, sr = torchaudio.load(str(audio_path))
            
            # 重采样到16kHz
            if sr != 16000:
                waveform = torchaudio.functional.resample(waveform, sr, 16000)
            
            # 确保音频是单声道
            if waveform.shape[0] > 1:
                waveform = torch.mean(waveform, dim=0, keepdim=True)
            
            # 提取特征
            with torch.no_grad():
                embedding = self.speaker_encoder.encode_batch(waveform.to(self.device))
                embeddings.append(embedding)
        
        # 计算平均特征
        mean_embedding = torch.mean(torch.stack(embeddings), dim=0)
        return mean_embedding
        
    def generate_speech(
        self,
        text: str,
        speaker_embedding: torch.Tensor
    ) -> torch.Tensor:
        """
        生成语音
        
        Args:
            text: 输入文本
            speaker_embedding: 说话人特征向量
            
        Returns:
            生成的语音波形
        """
        # 处理输入文本
        inputs = self.processor(text=text, return_tensors="pt")
        
        # 生成语音
        speech = self.tts_model.generate_speech(
            inputs["input_ids"].to(self.device),
            speaker_embedding.to(self.device),
            vocoder=self.vocoder
        )
        
        return speech
        
    def clone_voice(
        self,
        text: str,
        reference_audio_paths: List[Union[str, Path]]
    ) -> torch.Tensor:
        """
        主函数:克隆声音
        
        Args:
            text: 要转换的文本
            reference_audio_paths: 参考音频文件路径列表
            
        Returns:
            生成的语音波形
        """
        # 1. 提取说话人特征
        speaker_embedding = self.extract_speaker_embedding(reference_audio_paths)
        
        # 2. 生成语音
        speech = self.generate_speech(text, speaker_embedding)
        
        return speech
        
    def save_audio(
        self,
        waveform: torch.Tensor,
        output_path: Union[str, Path],
        sample_rate: int = 16000
    ):
        """
        保存音频文件
        
        Args:
            waveform: 音频波形
            output_path: 输出文件路径
            sample_rate: 采样率
        """
        # 确保输出目录存在
        output_path = Path(output_path)
        output_path.parent.mkdir(parents=True, exist_ok=True)
        
        # 保存音频
        torchaudio.save(
            str(output_path),
            waveform.unsqueeze(0).cpu(),
            sample_rate
        )