# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# +
# %load_ext autoreload
# %autoreload 2
# %matplotlib inline

#
# THE                                .-'''-.        .-'''-.                   
#                                   '   _    \     '   _    \                 
#_________   _...._               /   /` '.   \  /   /` '.   \                
#\        |.'      '-.           .   |     \  ' .   |     \  '   _.._         
# \        .'```'.    '. .-,.--. |   '      |  '|   '      |  '.' .._|        
#  \      |       \     \|  .-. |\    \     / / \    \     / / | '            
#   |     |        |    || |  | | `.   ` ..' /   `.   ` ..' /__| |__          
#   |      \      /    . | |  | |    '-...-'`       '-...-'`|__   __|         
#   |     |\`'-.-'   .'  | |  '-                               | |            
#   |     | '-....-'`    | |                                   | |            
#  .'     '.             | |                                   | |            
#'-----------'           |_|           _______                 | |            
#              __.....__               \  ___ `'.   .--.   _...|_|            
#          .-''         '.              ' |--.\  \  |__| .'     '.   .--./)   
#.-,.--.  /     .-''"'-.  `.            | |    \  ' .--..   .-.   . /.''\\    
#|  .-. |/     /________\   \    __     | |     |  '|  ||  '   '  || |  | |   
#| |  | ||                  | .:--.'.   | |     |  ||  ||  |   |  | \`-' /    
#| |  | |\    .-------------'/ |   \ |  | |     ' .'|  ||  |   |  | /("'`     
#| |  '-  \    '-.____...---.`" __ | |  | |___.' /' |  ||  |   |  | \ '---.   
#| |       `.             .'  .'.''| | /_______.'/  |__||  |   |  |  /'""'.\  
#| |         `''-...... -'   / /   | |_\_______|/       |  |   |  | ||     || 
#|_|                         \ \._,\ '/                 |  |   |  | \'. __//  
#                             `--'  `"                  '--'   '--'  `'---'   
#
#
#                                                               BENCHMARK
#

#
# AS REPORTED IN
#
# <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>. 
# Design and Evaluation of Interactive Proofreading Tools for Connectomics.
# IEEE Transactions on Visualization and Computer Graphics 2014;20(12):2466-2475.
#

import _proofreading as p
# -

# # Data

#
# load the first slice of our test data
#
image, gold, segmentation = p.Util.load(0)

p.Util.view(image, gold, segmentation)

#
# load all slices of our test data
#
images, golds, segmentations = p.Util.load_all()

p.Util.view(images, golds, segmentations)

# # Measures

#
# variation of information of our test data
# Meila 2003, Comparing Clusterings by the Variation of Information
#
p.Util.variation_of_information(golds, segmentations)

#
# Adjusted Rand score of our test data
# Rand 1971, Objective Criteria for the Evaluation of Clustering Methods
#
p.Util.rand_index(golds, segmentations)

#
# Edit distance of our test data
# Returns number of 2d splits, number of 3d splits, number of 3d merges
# 
p.Util.edit_distance(golds, segmentations)

#
# Adapted Rand error as defined by http://brainiac2.mit.edu/SNEMI3D/evaluation
#
p.Util.adapted_rand_error(golds, segmentations)

# # User Study

# 
# Loads the participant information of the user study published in:
#
# Haehn 2014, Design and Evaluation of Interactive Proofreading Tools
#   for Connectomics, IEEE Vis
#
# Then, loads the proofreading results for each of the users and calculates the measures
# in comparison to the groundtruth manual segmentation.
#
users = p.Util.load_users()
p.Util.evaluate_users(users, golds)

users

#
# We calculate the VI, RI, ED, Rand Error, and VI Error measures per tool.
#
measures = p.Util.group_users_by_tool(users)

#
# Variation of Information across tools
#
p.Util.plot_metric('Variation of Information', p.Util.variation_of_information(golds, segmentations), measures['vi'])

#
# Rand index across tools
#
p.Util.plot_metric('Rand Index', p.Util.rand_index(golds, segmentations), measures['ri'])

#
# Edit Distance across tools (sum of 2d splits + 3d merges)
#
ed_baseline = p.Util.edit_distance(golds, segmentations)
ed_baseline = ed_baseline[0] + ed_baseline[2] # 2d splits and 3d merges (see Haehn 2014)
p.Util.plot_metric('Edit Distance', ed_baseline, measures['ed'])

#
# Adapted Rand error across tools
#
p.Util.plot_metric('Adapted Rand Error', p.Util.adapted_rand_error(golds, segmentations), measures['rand_error'])



StarcoderdataJupyterScriptsDedupFiltered