|
import os |
|
import sys |
|
import tqdm |
|
|
|
import pandas as pd |
|
import numpy as np |
|
|
|
from PIL import Image |
|
|
|
filepath = os.path.split(os.path.abspath(__file__))[0] |
|
repopath = os.path.split(filepath)[0] |
|
sys.path.append(repopath) |
|
|
|
from utils.eval_functions import * |
|
from utils.misc import * |
|
|
|
BETA = 1.0 |
|
|
|
def evaluate(opt, args): |
|
if os.path.isdir(opt.Eval.result_path) is False: |
|
os.makedirs(opt.Eval.result_path) |
|
|
|
method = os.path.split(opt.Eval.pred_root)[-1] |
|
|
|
if args.verbose is True: |
|
print('#' * 20, 'Start Evaluation', '#' * 20) |
|
datasets = tqdm.tqdm(opt.Eval.datasets, desc='Expr - ' + method, total=len( |
|
opt.Eval.datasets), position=0, bar_format='{desc:<30}{percentage:3.0f}%|{bar:50}{r_bar}') |
|
else: |
|
datasets = opt.Eval.datasets |
|
|
|
results = [] |
|
|
|
for dataset in datasets: |
|
pred_root = os.path.join(opt.Eval.pred_root, dataset) |
|
gt_root = os.path.join(opt.Eval.gt_root, dataset, 'masks') |
|
|
|
preds = os.listdir(pred_root) |
|
gts = os.listdir(gt_root) |
|
|
|
preds = sort(preds) |
|
gts = sort(gts) |
|
|
|
preds = [i for i in preds if i in gts] |
|
gts = [i for i in gts if i in preds] |
|
|
|
FM = Fmeasure() |
|
WFM = WeightedFmeasure() |
|
SM = Smeasure() |
|
EM = Emeasure() |
|
MAE = Mae() |
|
MSE = Mse() |
|
MBA = BoundaryAccuracy() |
|
IOU = IoU() |
|
BIOU = BIoU() |
|
TIOU = TIoU() |
|
|
|
if args.verbose is True: |
|
samples = tqdm.tqdm(enumerate(zip(preds, gts)), desc=dataset + ' - Evaluation', total=len( |
|
preds), position=1, leave=False, bar_format='{desc:<30}{percentage:3.0f}%|{bar:50}{r_bar}') |
|
else: |
|
samples = enumerate(zip(preds, gts)) |
|
|
|
for i, sample in samples: |
|
pred, gt = sample |
|
|
|
pred_mask = np.array(Image.open(os.path.join(pred_root, pred)).convert('L')) |
|
gt_mask = np.array(Image.open(os.path.join(gt_root, gt)).convert('L')) |
|
|
|
if len(pred_mask.shape) != 2: |
|
pred_mask = pred_mask[:, :, 0] |
|
if len(gt_mask.shape) != 2: |
|
gt_mask = gt_mask[:, :, 0] |
|
|
|
assert pred_mask.shape == gt_mask.shape, print(pred, 'does not match the size of', gt) |
|
|
|
|
|
FM.step( pred=pred_mask, gt=gt_mask) |
|
WFM.step(pred=pred_mask, gt=gt_mask) |
|
SM.step( pred=pred_mask, gt=gt_mask) |
|
EM.step( pred=pred_mask, gt=gt_mask) |
|
MAE.step(pred=pred_mask, gt=gt_mask) |
|
MSE.step(pred=pred_mask, gt=gt_mask) |
|
MBA.step(pred=pred_mask, gt=gt_mask) |
|
IOU.step(pred=pred_mask, gt=gt_mask) |
|
BIOU.step(pred=pred_mask, gt=gt_mask) |
|
TIOU.step(pred=pred_mask, gt=gt_mask) |
|
|
|
result = [] |
|
|
|
Sm = SM.get_results()["sm"] |
|
wFm = WFM.get_results()["wfm"] |
|
mae = MAE.get_results()["mae"] |
|
mse = MSE.get_results()["mse"] |
|
mBA = MBA.get_results()["mba"] |
|
|
|
Fm = FM.get_results()["fm"] |
|
Em = EM.get_results()["em"] |
|
Iou = IOU.get_results()["iou"] |
|
BIou = BIOU.get_results()["biou"] |
|
TIou = TIOU.get_results()["tiou"] |
|
|
|
adpEm = Em["adp"] |
|
avgEm = Em["curve"].mean() |
|
maxEm = Em["curve"].max() |
|
adpFm = Fm["adp"] |
|
avgFm = Fm["curve"].mean() |
|
maxFm = Fm["curve"].max() |
|
avgIou = Iou["curve"].mean() |
|
maxIou = Iou["curve"].max() |
|
avgBIou = BIou["curve"].mean() |
|
maxBIou = BIou["curve"].max() |
|
avgTIou = TIou["curve"].mean() |
|
maxTIou = TIou["curve"].max() |
|
|
|
out = dict() |
|
for metric in opt.Eval.metrics: |
|
out[metric] = eval(metric) |
|
|
|
pkl = os.path.join(opt.Eval.result_path, 'result_' + dataset + '.pkl') |
|
if os.path.isfile(pkl) is True: |
|
result = pd.read_pickle(pkl) |
|
result.loc[method] = out |
|
result.to_pickle(pkl) |
|
else: |
|
result = pd.DataFrame(data=out, index=[method]) |
|
result.to_pickle(pkl) |
|
result.to_csv(os.path.join(opt.Eval.result_path, 'result_' + dataset + '.csv')) |
|
results.append(result) |
|
|
|
if args.verbose is True: |
|
for dataset, result in zip(datasets, results): |
|
print('###', dataset, '###', '\n', result.sort_index(), '\n') |
|
|
|
if __name__ == "__main__": |
|
args = parse_args() |
|
opt = load_config(args.config) |
|
evaluate(opt, args) |
|
|
