code: import vdomr as vd
  import spikeforest as sf
  from cairio import client as ca
  import pandas as pd
  import numpy as np
  from matplotlib import pyplot as plt
  
  
  class AccuracyPlot(vd.components.Pyplot):
      def __init__(self, snrs, accuracies):
          vd.components.Pyplot.__init__(self)
          self._snrs = snrs
          self._accuracies = accuracies
  
      def plot(self):
          plt.scatter(self._snrs, self._accuracies)
  
  
  class StudySorterFigure(vd.Component):
      def __init__(self, sfdata):
          vd.Component.__init__(self)
          self._plot = None
          self._SF_data = sfdata
          self._study = None
          self._sorter = None
  
      def setStudySorter(self, *, study, sorter):
          self._study = study
          self._sorter = sorter
          self._update_plot()
  
      def _update_plot(self):
          SF = self._SF_data
          study = SF.study(self._study)
          b = _get_study_sorting_results(study)
          a = b[self._sorter]
          snrs = a['true_unit_snrs']
          accuracies = a['num_matches'] / \
              (a['num_matches']+a['num_false_positives']+a['num_false_negatives'])
  
          self._plot = AccuracyPlot(snrs, accuracies)
          self.refresh()
  
      def render(self):
          if self._plot is None:
              return vd.div('Nothing')
          return vd.div(
              vd.div('test '+self._study+' '+self._sorter),
              self._plot
          )
  
  
  class SFBrowser(vd.Component):
      def __init__(self, output_id):
          vd.Component.__init__(self)
  
          self._output_id = output_id
  
          a = ca.loadObject(
              key=dict(name='spikeforest_results'),
              subkey=output_id
          )
          if not a:
              print('ERROR: unable to open results: '+output_id)
              return
  
          if ('recordings' not in a) or ('studies' not in a) or ('sorting_results' not in a):
              print('ERROR: problem with output: '+output_id)
              return
  
          studies = a['studies']
          recordings = a['recordings']
          sorting_results = a['sorting_results']
  
          SF = sf.SFData()
          SF.loadStudies(studies)
          SF.loadRecordings2(recordings)
          SF.loadSortingResults(sorting_results)
  
          # sorter_names=[]
          # for SR in sorting_results:
          #     sorter_names.append(SR['sorter']['name'])
          # sorter_names=list(set(sorter_names))
          # sorter_names.sort()
  
          self._SF_data = SF
  
          self._accuracy_threshold_input = vd.components.LineEdit(
              value=0.8, dtype=float, style=dict(width='70px'))
          self._update_button = vd.components.Button(
              onclick=self._on_update, class_='button', label='Update')
          self._study_sorter_fig = StudySorterFigure(SF)
          self._study_sorter_table = vd.div()  # dummy
  
          vd.devel.loadBootstrap()
  
          self._update_accuracy_table()
  
      def _on_update(self):
          self._update_accuracy_table()
  
      def _update_accuracy_table(self):
          accuracy_threshold = self._accuracy_threshold_input.value()
          self._accuracy_table_data, self._sorters = self._get_accuracy_table_data(
              accuracy_threshold=accuracy_threshold)
          self._accuracy_table = self._to_table(
              self._accuracy_table_data, ['study']+self._sorters)
          print(self._accuracy_table_data)
          self.refresh()
  
      def _open_study_sorter_fig(self, *, sorter, study):
          self._study_sorter_fig.setStudySorter(study=study, sorter=sorter)
  
      def _get_accuracy_table_data(self, *, accuracy_threshold):
          SF = self._SF_data
          accuracy_table = []
          sorters = set()
          for sname in SF.studyNames():
              print('STUDY: '+sname)
              study = SF.study(sname)
              b = _get_study_sorting_results(study)
              tmp = dict(
                  study=dict(  # first column
                      text=sname
                  )
              )
              for sorter in b:
                  sorters.add(sorter)
                  a = b[sorter]
                  accuracies = a['num_matches'] / \
                      (a['num_matches']+a['num_false_positives'] +
                       a['num_false_negatives'])
                  tmp[sorter] = dict(
                      text=str(np.count_nonzero(
                          accuracies >= accuracy_threshold)),
                      callback=lambda sorter=sorter, study=sname: self._open_study_sorter_fig(
                          sorter=sorter, study=study)
                  )
              accuracy_table.append(tmp)
  
          sorters = list(sorters)
          sorters.sort()
          return accuracy_table, sorters
  
      def _to_table(self, X, column_names):
          rows = []
          rows.append(vd.tr([vd.th(cname) for cname in column_names]))
          for x in X:
              elmts = []
              for cname in column_names:
                  tmp = x.get(cname)
                  if tmp:
                      if 'callback' in tmp:
                          elmt = vd.a(tmp['text'], onclick=tmp['callback'])
                      else:
                          elmt = vd.span(tmp['text'])
                  else:
                      elmt = vd.span('N/A')
                  elmts.append(elmt)
              rows.append(vd.tr([vd.td(elmt) for elmt in elmts]))
          return vd.table(rows, class_='table')
  
      def render(self):
          return vd.div(
              vd.table(
                  vd.tr(
                      vd.td('Accuracy threshold:'),
                      vd.td(self._accuracy_threshold_input),
                      vd.td(self._update_button)
                  ),
                  class_='table',
                  style={'max-width': '200px'}
              ),
              vd.components.ScrollArea(
                  self._accuracy_table,
                  height=500
              ),
              self._study_sorter_fig,
              style=dict(padding='15px')
          )
  
  
  def _get_study_sorting_results(study):
      results = []
      for rname in study.recordingNames():
          rec = study.recording(rname)
          true_units_info = rec.trueUnitsInfo(format='json')
          true_units_info_by_id = dict()
          for true_unit in true_units_info:
              true_units_info_by_id[true_unit['unit_id']] = true_unit
          for srname in rec.sortingResultNames():
              a = rec.sortingResult(srname)
              res0 = dict(sorter=srname, recording=rname, study=study.name())
              tmp = a.comparisonWithTruth(format='json')
              for i in tmp:
                  tmp[i]['true_unit_info'] = true_units_info_by_id[tmp[i]['unit_id']]
              res0['comparison_with_truth'] = tmp
              results.append(res0)
  
      sorters = list(set([a['sorter'] for a in results]))
      sorters.sort()
  
      units_by_sorter = dict()
      for sorter in sorters:
          units_by_sorter[sorter] = []
  
      for obj in results:
          sorter0 = obj['sorter']
          units = [obj['comparison_with_truth'][i]
                   for i in obj['comparison_with_truth']]
          units_by_sorter[sorter0] = units_by_sorter[sorter0]+units
  
      ret = dict()
      for sorter in sorters:
          units = units_by_sorter[sorter]
          try:
              ret[sorter] = dict(
                  true_unit_ids=[unit['unit_id'] for unit in units],
                  true_unit_snrs=np.array(
                      [unit['true_unit_info']['snr'] for unit in units]),
                  true_unit_firing_rates=np.array(
                      [unit['true_unit_info']['firing_rate'] for unit in units]),
                  num_matches=np.array([unit['num_matches'] for unit in units]),
                  num_false_positives=np.array(
                      [unit['num_false_positives'] for unit in units]),
                  num_false_negatives=np.array(
                      [unit['num_false_negatives'] for unit in units])
              )
          except:
              print('WARNING: Problem loading results for sorter: '+sorter)
              ret[sorter] = dict(
                  true_unit_ids=[],
                  true_unit_snrs=np.array([]),
                  true_unit_firing_rates=np.array([]),
                  num_matches=np.array([]),
                  num_false_positives=np.array([]),
                  num_false_negatives=np.array([])
              )
  
      return ret
   import spikeextractors as si
  #import spikewidgets as sw
  import spiketoolkit as st
  import mlprocessors as mlpr
  import json
  from cairio import client as ca
  import numpy as np
  from copy import deepcopy
  
  def compare_sortings_with_truth(sortings,compute_resource,num_workers=None):
      print('>>>>>> compare sortings with truth')
      container='sha1://3b26155930cc4a4745c67b702ce297c9c968ac94/02-12-2019/mountaintools_basic.simg'
      jobs_gen_table=[]
      for sorting in sortings:
          units_true=sorting.get('units_true',[])
          firings=sorting['firings']
          firings_true=sorting['firings_true']
          units_true=units_true
          job=GenSortingComparisonTable.createJob(
              firings=firings,
              firings_true=firings_true,
              units_true=units_true,
              json_out={'ext':'.json','upload':True},
              html_out={'ext':'.html','upload':True},
              _container=container
          )
          jobs_gen_table.append(job)
      
      all_jobs=jobs_gen_table
      label='Compare sortings with truth'
      mlpr.executeBatch(jobs=all_jobs,label=label,num_workers=num_workers,compute_resource=compute_resource)
      
      sortings_out=[]
      for i,sorting in enumerate(sortings):
          comparison_with_truth=dict()
          comparison_with_truth['json']=jobs_gen_table[i]['result']['outputs']['json_out']
          comparison_with_truth['html']=jobs_gen_table[i]['result']['outputs']['html_out']
          sorting2=deepcopy(sorting)
          sorting2['comparison_with_truth']=comparison_with_truth
          sortings_out.append(sorting2)
  
      return sortings_out
  
  class GenSortingComparisonTable(mlpr.Processor):
      VERSION='0.2.0'
      firings=mlpr.Input('Firings file (sorting)')
      firings_true=mlpr.Input('True firings file')
      units_true=mlpr.IntegerListParameter('List of true units to consider')
      json_out=mlpr.Output('Table as .json file produced from pandas dataframe')
      html_out=mlpr.Output('Table as .html file produced from pandas dataframe')
      
      def run(self):
          sorting=si.MdaSortingExtractor(firings_file=self.firings)
          sorting_true=si.MdaSortingExtractor(firings_file=self.firings_true)
          if (self.units_true is not None) and (len(self.units_true)>0):
              sorting_true=si.SubSortingExtractor(parent_sorting=sorting_true,unit_ids=self.units_true)
          SC=st.comparison.SortingComparison(sorting_true,sorting)
          df=get_comparison_data_frame(comparison=SC)
          #sw.SortingComparisonTable(comparison=SC).getDataframe()
          json=df.transpose().to_dict()
          html=df.to_html(index=False)
          _write_json_file(json,self.json_out)
          _write_json_file(html,self.html_out)
  
  def get_comparison_data_frame(*,comparison):
      import pandas as pd
      SC=comparison
  
      unit_properties=[] #snr, etc? these would need to be properties in the sortings of the comparison
  
      # Compute events counts
      sorting1=SC.getSorting1()
      sorting2=SC.getSorting2()
      unit1_ids = sorting1.getUnitIds()
      unit2_ids = sorting2.getUnitIds()
      N1 = len(unit1_ids)
      N2 = len(unit2_ids)
      event_counts1 = dict()
      for i1, u1 in enumerate(unit1_ids):
          times1 = sorting1.getUnitSpikeTrain(u1)
          event_counts1[u1] = len(times1)
      event_counts2 = dict()
      for i2, u2 in enumerate(unit2_ids):
          times2 = sorting2.getUnitSpikeTrain(u2)
          event_counts2[u2] = len(times2)
  
      rows = []
      for u_1, unit1 in enumerate(unit1_ids):
          unit2 = SC.getBestUnitMatch1(unit1)
          if unit2>=0:
              num_matches=SC.getMatchingEventCount(unit1, unit2)
              num_false_negatives=event_counts1[unit1]-num_matches
              num_false_positives=event_counts2[unit2]-num_matches
          else:
              num_matches=0
              num_false_negatives=event_counts1[unit1]
              num_false_positives=0
          row0 = {
              'unit_id': unit1,
              'accuracy': _safe_frac(num_matches,num_false_positives+num_false_negatives+num_matches),
              'best_unit': unit2,
              'matched_unit': SC.getMappedSorting1().getMappedUnitIds(unit1),
              'num_matches': num_matches,
              'num_false_negatives': num_false_negatives,
              'num_false_positives': num_false_positives,
              'f_n': _safe_frac(num_false_negatives,num_false_negatives+num_matches),
              'f_p': _safe_frac(num_false_positives,num_false_positives+num_matches)
          }
          for prop in unit_properties:
              pname = prop['name']
              row0[pname] = SC.getSorting1().getUnitProperty(unit_id=int(unit1), property_name=pname)
          rows.append(row0)
  
      df = pd.DataFrame(rows)
      fields = ['unit_id']
      fields = fields + ['accuracy', 'best_unit', 'matched_unit', 'num_matches', 'num_false_negatives', 'num_false_positives', 'f_n', 'f_p']
      for prop in unit_properties:
          pname = prop['name']
          fields.append(pname)
      df = df[fields]
      df['accuracy'] = df['accuracy'].map('{:,.4f}'.format)
      # df['Best match'] = df['Accuracy'].map('{:,.2f}'.format)
      df['f_n'] = df['f_n'].map('{:,.4f}'.format)
      df['f_p'] = df['f_p'].map('{:,.4f}'.format)
      return df
  
  def _safe_frac(numer, denom):
      if denom == 0:
          return 0
      return float(numer) / denom
  
  def _write_json_file(obj,path):
    with open(path,'w') as f:
      return json.dump(obj,f)
  
  
apis: numpy.array matplotlib.pyplot.scatter numpy.count_nonzero
apis: pandas.DataFrame