Spaces:
Running
Running
File size: 3,376 Bytes
369ee40 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 |
import os
from random import shuffle
from shutil import copyfile
# file template needed for import script
template = "{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}"
# structure example below
# client_id path sentence up_votes down_votes age gender accent locale segment
structure = template.format("client_id", "path", "sentence", "up_votes",
"down_votes", "age", "gender", "accent", "locale", "segment")
iterator = 1
speaker_iterator = 1
def write_dataset(path, name, data):
"""
Function to write converted data list
"""
global iterator
global speaker_iterator
file_path = os.path.join(path, name)
clip_path = os.path.join(os.path.dirname(path), "wav")
result = open(file_path, mode="w", encoding="utf-8")
result.write(structure)
result.write("\n")
for row in data:
file_name = row[0]
if file_name.endswith(".wav"):
pass
elif file_name.endswith(".mp3"):
pass
elif file_name.find(".") == -1:
file_name += ".wav"
parted_name = file_name.split(".")
new_file_name = f"{iterator}." + parted_name[1]
old_file_path = os.path.join(clip_path, file_name)
new_file_path = os.path.join("clips", new_file_name)
if os.path.exists(old_file_path):
copyfile(old_file_path,
new_file_path)
result.write(template.format(
speaker_iterator, new_file_name, row[1], "", "", "", "", "", "uk", "\n"))
speaker_iterator += 1
iterator += 1
else:
print("File not found", old_file_path)
result.close()
if not os.path.exists("clips"):
os.makedirs("clips") # create folder to contain processed clips
# iterate over all data lists and write converted version near them
for subdir, dirs, files in os.walk(os.path.abspath(os.path.curdir)):
print(subdir)
for file in files:
if file == "txt.final.data":
file_path = os.path.join(subdir, file)
file = open(file_path, mode="r")
data = [row.replace(" \n", "").split(" ", 1)
for row in file.readlines()]
file.close()
shuffle(data)
dataset_size = len(data)
train_point = int(dataset_size*0.8)
dev_point = int(train_point + (dataset_size - train_point) / 2)
# split dataset
write_dataset(subdir, "train.tsv", data[:train_point])
write_dataset(subdir, "dev.tsv", data[train_point:dev_point])
write_dataset(subdir, "test.tsv", data[dev_point:])
# write dataset splits into single files
final_files = {
"train.tsv": open("train.tsv", mode="w", encoding="utf-8"),
"dev.tsv": open("dev.tsv", mode="w", encoding="utf-8"),
"test.tsv": open("test.tsv", mode="w", encoding="utf-8")
}
for file in final_files.values():
file.write(structure)
file.write("\n")
for subdir, dirs, files in os.walk(os.path.curdir):
for file in files:
if file in ["train.tsv", "dev.tsv", "test.tsv"]:
input_file = open(os.path.join(subdir, file))
data = [row for row in input_file.readlines()][1::]
input_file.close()
for row in data:
final_files[file].write(row)
for file in final_files.values():
file.close()
|