text: #
  # Collective Knowledge: CK-powered Caffe crowdbenchmarking (very early prototyping)
  #
  # See CK LICENSE.txt for licensing details
  # See CK COPYRIGHT.txt for copyright details
  #
  # Developer: Grigori Fursin, Grigori.Fursin@cTuning.org, http://fursin.net
  #
  
  cfg={}  # Will be updated by CK (meta description of this module)
  work={} # Will be updated by CK (temporal data)
  ck=None # Will be updated by CK (initialized CK kernel) 
  
  # Local settings
  
  line='================================================================'
  
  ck_url='http://cknowledge.org/repo/web.php?native_action=show&native_module_uoa=program.optimization&scenario=1eb2f50d4620903e'
  ck_url1='http://cknowledge.org/repo/web.php?wcid=experiment.bench.dnn:'
  
  ffstat='ck-stat-flat-characteristics.json'
  ffmin='ck-stat-flat-min.json'
  
  form_name='wa_web_form'
  onchange='document.'+form_name+'.submit();'
  
  #hextra='<i><center>\n'
  #hextra+='This is an on-going long-term project. Please check our vision [ '
  #hextra+='<a href="http://doi.acm.org/10.1145/2909437.2909449">IWOCL\'16</a>, \n'
  #hextra+='<a href="http://arxiv.org/abs/1506.06256">CPC\'15</a>, \n'
  #hextra+='<a href="https://www.youtube.com/watch?v=Q94yWxXUMP0">YouTube</a>, \n'
  #hextra+='<a href="http://ctuning.org/cm/wiki/index.php?title=CM:data:45741e3fbcf4024b:1db78910464c9d05">wiki</a> ] '
  #hextra+=' and <a href="https://github.com/dividiti/ck-caffe">CK-Caffe GitHub repo</a> for more details!'
  #hextra+='</center></i>\n'
  #hextra+='<br>\n'
  
  hextra='<i><center>\n'
  hextra+=' [ <a href="http://cKnowledge.org/ai">Community-driven AI R&D powered by CK</a> ], '
  hextra+=' [ <a href="https://github.com/ctuning/ck-caffe2">CK-Caffe2</a> / <a href="https://github.com/dividiti/ck-caffe">CK-Caffe</a> ], '
  hextra+=' [ <a href="https://github.com/ctuning/ck-tensorflow">CK-TensorFlow</a> ], '
  hextra+=' [ <a href="https://en.wikipedia.org/wiki/Collective_Knowledge_(software)">Wikipedia</a>, \n'
  hextra+='<a href="https://www.researchgate.net/publication/304010295_Collective_Knowledge_Towards_RD_Sustainability">paper 1</a>, \n'
  hextra+='<a href="https://arxiv.org/abs/1506.06256">Paper 2</a>, \n'
  hextra+='<a href="https://www.youtube.com/watch?v=Q94yWxXUMP0">YouTube CK intro</a> ], \n'
  hextra+=' [ <a href="http://dividiti.blogspot.fr/2017/02/we-received-test-of-time-award-for-our.html">CGO\'17 test of time award for our interdisiplinary R&D</a> ]'
  hextra+='</center></i>\n'
  hextra+='<br>\n'
  
  selector=[{'name':'Type', 'key':'dnn_type'},
            {'name':'DNN engine', 'key':'dnn_engine_name'},
            {'name':'Model', 'key':'nn_type'},
            {'name':'Platform', 'key':'plat_name', 'new_line':'yes'},
            {'name':'CPU', 'key':'cpu_name'},
            {'name':'OS', 'key':'os_name', 'new_line':'yes'},
            {'name':'GPGPU', 'key':'gpgpu_name'}]
  
  replay_clean_vars=['no_compile','host_os','device_id']
  replay_clean_env_vars=['CK_CAFFE_MODEL','CK_CAFFE_MODEL_FILE','CK_ENV_MODEL_CAFFE_WEIGHTS']
    
  ##############################################################################
  # Initialize module
  
  def init(i):
      """
  
      Input:  {}
  
      Output: {
                return       - return code =  0, if successful
                                           >  0, if error
                (error)      - error text if return > 0
              }
  
      """
      return {'return':0}
  
  ##############################################################################
  # crowdsource these experiments
  
  def crowdsource(i):
      """
      Input:  {
                (local)               - if 'yes', local crowd-benchmarking, instead of public
                (user)                - force different user ID/email for demos
  
                (choices)             - force different choices to program pipeline
  
                (repetitions)         - statistical repetitions (default=1), for now statistical analysis is not used (TBD)
              }
  
      Output: {
                return       - return code =  0, if successful
                                           >  0, if error
                (error)      - error text if return > 0
              }
  
      """
  
      import copy
      import os
  
      # Setting output
      o=i.get('out','')
      oo=''
      if o=='con': oo='con'
  
      quiet=i.get('quiet','')
  
      er=i.get('exchange_repo','')
      if er=='': er=ck.cfg['default_exchange_repo_uoa']
      esr=i.get('exchange_subrepo','')
      if esr=='': esr=ck.cfg['default_exchange_subrepo_uoa']
  
      if i.get('local','')=='yes': 
         er='local'
         esr=''
  
      la=i.get('local_autotuning','')
  
      repetitions=i.get('repetitions','')
      if repetitions=='': repetitions=3
      repetitions=int(repetitions)
  
      record='no'
  
      # Check if any input has . and convert to dict
      for k in list(i.keys()):
          if k.find('.')>0:
              v=i[k]
  
              kk='##'+k.replace('.','#')
  
              del(i[k])
  
              r=ck.set_by_flat_key({'dict':i, 'key':kk, 'value':v})
              if r['return']>0: return r
  
      choices=i.get('choices',{})
      env=i.get('env',{})
  
      if 'env' not in choices: choices['env']={}
  
      r=ck.merge_dicts({'dict1':choices['env'], 'dict2':copy.deepcopy(env)})
      env={}
  
      xchoices=copy.deepcopy(choices)
  
      # Get user 
      user=''
  
      mcfg={}
      ii={'action':'load',
          'module_uoa':'module',
          'data_uoa':cfg['module_deps']['program.optimization']}
      r=ck.access(ii)
      if r['return']==0:
         mcfg=r['dict']
  
         dcfg={}
         ii={'action':'load',
             'module_uoa':mcfg['module_deps']['cfg'],
             'data_uoa':mcfg['cfg_uoa']}
         r=ck.access(ii)
         if r['return']>0 and r['return']!=16: return r
         if r['return']!=16:
            dcfg=r['dict']
  
         user=dcfg.get('user_email','')
  
      # Initialize local environment for program optimization ***********************************************************
      pi=i.get('platform_info',{})
      if len(pi)==0:
         ii=copy.deepcopy(i)
         ii['action']='initialize'
         ii['module_uoa']=cfg['module_deps']['program.optimization']
         ii['data_uoa']='caffe'
         ii['exchange_repo']=er
         ii['exchange_subrepo']=esr
         ii['skip_welcome']='yes'
         ii['skip_log_wait']='yes'
         ii['crowdtuning_type']='caffe-crowd-benchmarking'
         r=ck.access(ii)
         if r['return']>0: return r
  
         pi=r['platform_info']
         user=r.get('user','')
  
      hos=pi['host_os_uoa']
      hosd=pi['host_os_dict']
  
      tos=pi['os_uoa']
      tosd=pi['os_dict']
      tbits=tosd.get('bits','')
  
      remote=tosd.get('remote','')
  
      tdid=pi['device_id']
  
      features=pi.get('features',{})
  
      fplat=features.get('platform',{})
      fos=features.get('os',{})
      fcpu=features.get('cpu',{})
      fgpu=features.get('gpu',{})
  
      plat_name=fplat.get('name','')
      plat_uid=features.get('platform_uid','')
      os_name=fos.get('name','')
      os_uid=features.get('os_uid','')
      cpu_name=fcpu.get('name','')
      if cpu_name=='': cpu_name='unknown-'+fcpu.get('cpu_abi','')
      cpu_uid=features.get('cpu_uid','')
      gpu_name=fgpu.get('name','')
      gpgpu_name=''
      sn=fos.get('serial_number','')
  
      # Ask for cmd
      tp=['cpu', 'cuda', 'cuda_fp16', 'opencl']
  
      ck.out(line)
      ck.out('Select Caffe library type:')
      ck.out('')
      r=ck.access({'action':'select_list',
                   'module_uoa':cfg['module_deps']['choice'],
                   'choices':tp})
      if r['return']>0: return r
      xtp=r['choice']
  
      xtp16=''
      if xtp=='cuda_fp16':
         xtp='cuda'
         xtp16='yes'
  
      android=False
      if 'android' in tos: android=True
  
      # Get extra platform features if "cuda" or "opencl"
      if android:
         run_cmd='default'
         prog_uoa='caffe-time'
      else:
         run_cmd='time_cpu'
         prog_uoa='caffe'
  
      gpgpu_uid=''
  
      if xtp=='cuda' or xtp=='opencl':
          r=ck.access({'action':'detect',
                       'module_uoa':cfg['module_deps']['platform.gpgpu'],
                       'host_os':hos,
                       'target_os':tos,
                       'device_id':tdid,
                       'type':xtp,
                       'share':'yes',
                       'exchange_repo':er,
                       'exchange_subrepo':esr})
          if r['return']>0: return r
          gfeat=r.get('features',{})
          gpgpus=gfeat.get('gpgpu',[])
  
          if len(gpgpus)>0:
              gpgpu_name=gpgpus[0].get('gpgpu',{}).get('name','')
              gpgpu_uid=gpgpus[0].get('gpgpu_uoa','')
  
          if android:
             if xtp!='opencl':
                return {'return':1, 'error':'can\'t crowdbenchmark this type of DNN engine on selected target platform'}
  
             run_cmd='default'
             prog_uoa='caffe-time-opencl'
          else:
             run_cmd='time_gpu'
             if xtp16=='yes': run_cmd='time_gpu_fp16'
  
      # Get deps from caffe program
      r=ck.access({'action':'load',
                   'module_uoa':cfg['module_deps']['program'],
                   'data_uoa':prog_uoa})
      if r['return']>0: return r
  
      pp=r['path']
  
  #    lib_dep=r['dict']['run_deps']['lib-caffe']
  
  #    deps={'lib-caffe':lib_dep}
  
      # Check environment for selected type
  #    r=ck.access({'action':'resolve',
  #                 'module_uoa':cfg['module_deps']['env'],
  #                 'deps':deps,
  #                 'host_os':hos,
  #                 'target_os':tos,
  #                 'device_id':tdid,
  #                 'out':o})
  #    if r['return']>0: return r
  #    deps=r['deps']
  
      # Prepare CK pipeline for a given workload
      ii={'action':'pipeline',
  
          'module_uoa':cfg['module_deps']['program'],
          'data_uoa':prog_uoa,
  
          'host_os':hos,
          'target_os':tos,
          'device_id':tdid,
  
          'skip_target':'yes',
  
          'prepare':'yes',
  
          'env':env,
          'choices':choices,
  #        'dependencies':deps,
          'cmd_key':run_cmd,
          'no_state_check':'yes',
          'no_compiler_description':'yes',
          'skip_info_collection':'yes',
          'skip_calibration':'yes',
          'cpu_freq':'max',
          'gpu_freq':'max',
          'env_speed':'yes',
          'energy':'no',
          'skip_print_timers':'yes',
          'generate_rnd_tmp_dir':'no',
  
          'out':oo}
  
      rr=ck.access(ii)
      if rr['return']>0: return rr
  
      fail=rr.get('fail','')
      if fail=='yes':
          return {'return':10, 'error':'pipeline failed ('+rr.get('fail_reason','')+')'}
  
      ready=rr.get('ready','')
      if ready!='yes':
          return {'return':11, 'error':'couldn\'t prepare universal CK program workflow'}
  
      state=rr['state']
      tmp_dir=state.get('tmp_dir','')
      if tmp_dir=='': tmp_dir='tmp' # usually when no_compile
  
      deps=rr['dependencies'] # resolved deps
  
      # Clean pipeline
      if 'ready' in rr: del(rr['ready'])
      if 'fail' in rr: del(rr['fail'])
      if 'return' in rr: del(rr['return'])
  
      # Prepare high-level experiment meta
      meta={'cpu_name':cpu_name,
            'os_name':os_name,
            'plat_name':plat_name,
            'gpu_name':gpu_name,
            'dnn_type':xtp,
            'gpgpu_name':gpgpu_name,
            'cmd_key':run_cmd}
  
      # Process deps
      xdeps={}
      xnn=''
      xblas=''
      for k in deps:
          dp=deps[k]
  
          ptags=dp.get('tags',[])
  
          puoa=dp.get('package_uoa','')
          if puoa=='':
             puoa=dp.get('cus',{}).get('used_package_uoa','')
  
          dname=dp.get('dict',{}).get('data_name','')
  
          if k=='caffemodel':
              xnn=dname
  
              j1=xnn.rfind('(')
              if j1>0:
                  xnn=xnn[j1+1:-1]
  
          xdeps[k]={'name':dp.get('name',''), 'data_name':dname, 'ver':dp.get('ver',''), 'package_uoa':puoa, 'package_tags':ptags}
  
      # versions of engine sub deps
      dvers={}
      mdep=deps['lib-caffe']
      mdeps=mdep.get('dict',{}).get('deps',{})
  
      for k in mdeps:
          dvers[k]=mdeps[k].get('ver','')
  
      meta['xversions']=dvers
      meta['xdeps']=xdeps
      meta['nn_type']=xnn
      meta['choices']=xchoices
  
      mmeta=copy.deepcopy(meta)
  
      # Extra meta which is not used to search similar case ...
      mmeta['platform_uid']=plat_uid
      mmeta['os_uid']=os_uid
      mmeta['cpu_uid']=cpu_uid
      mmeta['gpgpu_uid']=gpgpu_uid
      mmeta['user']=user
  
      # Check if already exists (to aggregate stats)
      aggregated_stats={}
      rduid=''
      found=False
  
      if o=='con':
          ck.out('')
          ck.out('Checking if results already exists in a public repo (to aggregate statistics) ...')
  
      # Find remote entry
      ii={'action':'search',
          'module_uoa':work['self_module_uid'],
          'repo_uoa':er,
          'remote_repo_uoa':esr,
          'search_dict':{'meta':meta}}
      rx=ck.access(ii)
      if rx['return']>0: return rx
  
      lst=rx['lst']
  
      if len(lst)==1:
          rduid=lst[0]['data_uid']
          found=True
  
          if o=='con':
             ck.out('')
             ck.out('Results found. Pre-loading aggregated stats from '+rduid+' ...')
  
          # Load stats
          rx=ck.access({'action':'load',
                        'module_uoa':work['self_module_uid'],
                        'data_uoa':rduid,
                        'repo_uoa':er,
                        'remote_repo_uoa':esr,
                        'load_extra_json_files':[ffstat]})
          if rx['return']==0:
             aggregated_stats=rx.get('extra_json_files',{}).get(ffstat,{})
          else:
             ck.out('')
             ck.out('WARNING: couldn\'t load data ('+rx['error']+')')
      else:
         rx=ck.gen_uid({})
         if rx['return']>0: return rx
         rduid=rx['data_uid']
  
      # Run CK pipeline *****************************************************
      pipeline=copy.deepcopy(rr)
      if len(choices)>0:
          r=ck.merge_dicts({'dict1':pipeline['choices'], 'dict2':xchoices})
          if r['return']>0: return r
  
      ii={'action':'autotune',
          'module_uoa':cfg['module_deps']['pipeline'],
  
          'host_os':hos,
          'target_os':tos,
          'device_id':tdid,
  
          'iterations':1,
          'repetitions':repetitions,
  
          'collect_all':'yes',
          'process_multi_keys':['##characteristics#*'],
  
          'tmp_dir':tmp_dir,
  
          'pipeline':pipeline,
  
          'stat_flat_dict':aggregated_stats,
  
          "features_keys_to_process":["##choices#*"],
  
          "record_params": {
            "search_point_by_features":"yes"
          },
  
          'out':oo}
  
      rrr=ck.access(ii)
      if rrr['return']>0: return rrr
  
      ls=rrr.get('last_iteration_output',{})
      state=ls.get('state',{})
      xchoices=copy.deepcopy(ls.get('choices',{}))
      lsaf=rrr.get('last_stat_analysis',{}).get('dict_flat',{})
  
      real_proto=xchoices.get('env',{}).get('CK_CAFFE_MODEL','') # to push to server
  
      ddd={'meta':mmeta}
  
      ddd['choices']=xchoices
  
      features=ls.get('features',{})
  
      deps=ls.get('dependencies',{})
  
      fail=ls.get('fail','')
      fail_reason=ls.get('fail_reason','')
  
      ch=ls.get('characteristics',{})
  
      # Save pipeline
      ddd['state']={'fail':fail, 'fail_reason':fail_reason}
      ddd['characteristics']=ch
  
      ddd['user']=user
  
      # Add files
      ddd['file_stat']=ffstat
  
      if real_proto!='':
         ddd['file_model_topology']=os.path.basename(real_proto)
  
      if not found:
         if o=='con':
            ck.out('')
            ck.out('Saving results to the remote public repo ('+rduid+') ...')
  
         # Update meta
         rx=ck.access({'action':'add',
                       'module_uoa':work['self_module_uid'],
                       'data_uoa':rduid,
                       'repo_uoa':er,
                       'remote_repo_uoa':esr,
                       'dict':ddd,
                       'sort_keys':'yes'})
         if rx['return']>0: return rx
  
         # Push real proto
         if real_proto!='':
            if o=='con':
               ck.out('')
               ck.out('Pushing prototxt to the remote public repo ...')
  
            rx=ck.access({'action':'push',
                          'module_uoa':work['self_module_uid'],
                          'data_uoa':rduid,
                          'repo_uoa':er,
                          'remote_repo_uoa':esr,
                          'filename':real_proto,
                          'overwrite':'yes'})
            if rx['return']>0: return rx
  
      # Push statistical characteristics
      if o=='con':
         ck.out('')
         ck.out('Pushing file with statistics to server ...')
  
      fstat=os.path.join(pp,tmp_dir,ffstat)
  
      r=ck.save_json_to_file({'json_file':fstat, 'dict':lsaf, 'sort_keys':'yes'})
      if r['return']>0: return r
  
      rx=ck.access({'action':'push',
                    'module_uoa':work['self_module_uid'],
                    'data_uoa':rduid,
                    'repo_uoa':er,
                    'remote_repo_uoa':esr,
                    'filename':fstat,
                    'overwrite':'yes'})
      if rx['return']>0: return rx
  
      os.remove(fstat)
  
      # Info
      if o=='con':
          ck.out('')
          ck.out('Succesfully recorded results in remote repo (Entry UID='+rduid+')')
     
          # Check host URL prefix and default module/action
          url=ck_url+'&highlight_uid='+rduid
          ck.out('')
          ck.out('You can see your results at the following URL:')
          ck.out('')
          ck.out(url)
  
      return {'return':0}
  
  ##############################################################################
  # show results
  
  def show(i):
      """
      Input:  {
                 (crowd_module_uoa) - if rendered from experiment crowdsourcing
                 (crowd_key)        - add extra name to Web keys to avoid overlapping with original crowdsourcing HTML
                 (crowd_on_change)  - reuse onchange doc from original crowdsourcing HTML
              }
  
      Output: {
                return       - return code =  0, if successful
                                           >  0, if error
                (error)      - error text if return > 0
              }
  
      """
  
      import os
  
      st=''
  
      cmuoa=i.get('crowd_module_uoa','')
      ckey=i.get('crowd_key','')
  
      conc=i.get('crowd_on_change','')
      if conc=='':
          conc=onchange
  
      hi_uid=i.get('highlight_uid','')
  
      h=''
  #    h='<hr>\n'
      h+='<center>\n'
      h+='\n\n<script language="JavaScript">function copyToClipboard (text) {window.prompt ("Copy to clipboard: Ctrl+C, Enter", text);}</script>\n\n' 
  
  #    h+='<h2>Aggregated results from Caffe crowd-benchmarking (time, accuracy, energy, cost, ...)</h2>\n'
  
      h+=hextra
  
      # Check host URL prefix and default module/action
      rx=ck.access({'action':'form_url_prefix',
                    'module_uoa':'wfe',
                    'host':i.get('host',''), 
                    'port':i.get('port',''), 
                    'template':i.get('template','')})
      if rx['return']>0: return rx
      url0=rx['url']
      template=rx['template']
  
      url=url0
      action=i.get('action','')
      muoa=i.get('module_uoa','')
  
      st=''
  
      url+='action=index&module_uoa=wfe&native_action='+action+'&'+'native_module_uoa='+muoa
      url1=url
  
      # List entries
      ii={'action':'search',
          'module_uoa':work['self_module_uid'],
          'add_meta':'yes'}
  
      if cmuoa!='':
          ii['module_uoa']=cmuoa
  
      r=ck.access(ii)
      if r['return']>0: return r
  
      lst=r['lst']
  
      # Check unique entries
      choices={}
      wchoices={}
  
      for q in lst:
          d=q['meta']
          meta=d.get('meta',{})
  
          for kk in selector:
              kx=kk['key']
              k=ckey+kx
  
              if k not in choices: 
                  choices[k]=[]
                  wchoices[k]=[{'name':'','value':''}]
  
              kflat=kk.get('flat_key','')
              if kflat=='': kflat='##'+kx
  
              rx=ck.get_by_flat_key({'dict':meta, 'key':kflat})
              if rx['return']>0: return rx
              v=rx['value']
              if v==None: v=''
  
              if v!='':
                  if v not in choices[k]: 
                      choices[k].append(v)
                      wchoices[k].append({'name':v, 'value':v})
  
  
      # Prepare query div ***************************************************************
      if cmuoa=='':
          # Start form + URL (even when viewing entry)
          r=ck.access({'action':'start_form',
                       'module_uoa':cfg['module_deps']['wfe'],
                       'url':url1,
                       'name':form_name})
          if r['return']>0: return r
          h+=r['html']
  
      for kk in selector:
          kx=kk['key']
          k=ckey+kx
          n=kk['name']
  
          nl=kk.get('new_line','')
          if nl=='yes':
              h+='<br>\n<div id="ck_entries_space8"></div>\n'
  
          v=''
          if i.get(k,'')!='':
              v=i[k]
              kk['value']=v
  
          # Show hardware
          ii={'action':'create_selector',
              'module_uoa':cfg['module_deps']['wfe'],
              'data':wchoices.get(k,[]),
              'name':k,
              'onchange':conc, 
              'skip_sort':'no',
              'selected_value':v}
          r=ck.access(ii)
          if r['return']>0: return r
  
          h+='<b>'+n+':</b> '+r['html'].strip()+'\n'
  
      # Check hidden
      if hi_uid!='':
          h+='<input type="hidden" name="highlight_uid" value="'+hi_uid+'">\n'
  
      h+='<br><br>'
  
      # Prune list
      plst=[]
      for q in lst:
          d=q['meta']
          meta=d.get('meta',{})
  
          # Check selector
          skip=False
          for kk in selector:
              k=kk['key']
              n=kk['name']
              v=kk.get('value','')
  
              kflat=kk.get('flat_key','')
              if kflat=='': kflat='##'+k
  
              rx=ck.get_by_flat_key({'dict':meta, 'key':kflat})
              if rx['return']>0: return rx
              vxx=rx['value']
              if vxx==None: vxx=''
  
              if v!='' and vxx!=v:
                  skip=True
  
          if not skip:
              plst.append(q)
  
      # Check if too many
      lplst=len(plst)
      if lplst==0:
          h+='<b>No results found!</b>'
          return {'return':0, 'html':h, 'style':st}
      elif lplst>300:
          h+='<b>Too many entries to show ('+str(lplst)+') - please, prune list further!</b>'
          return {'return':0, 'html':h, 'style':st}
  
      # Prepare table
      h+='<table border="1" cellpadding="7" cellspacing="0">\n'
  
      ha='align="center" valign="top"'
      hb='align="left" valign="top"'
  
      h+='  <tr style="background-color:#dddddd">\n'
      h+='   <td '+ha+'><b>#</b></td>\n'
      h+='   <td '+ha+'><b>Platform</b></td>\n'
      h+='   <td '+ha+'><b>OS</b></td>\n'
      h+='   <td '+ha+'><b>CPU</b></td>\n'
      h+='   <td '+ha+'><b>GPGPU</b></td>\n'
      h+='   <td '+ha+'><b>Type</b></td>\n'
      h+='   <td '+ha+'><b>DNN engine</b></td>\n'
      h+='   <td '+ha+'><b>Model</b></td>\n'
      h+='   <td '+ha+'><b>Choices (env)</b></td>\n'
      h+='   <td '+ha+'><b>FWBW<br>min time</b><br><br>(exp&nbsp;time)<br>stat.&nbsp;repetitions</td>\n'
      h+='   <td '+ha+'><b>FW</b></td>\n'
      h+='   <td '+ha+'><b>BW</b></td>\n'
      h+='   <td '+ha+'><b>Per layer</b></td>\n'
      h+='   <td '+ha+'><b>HW costs</td>\n'
      h+='   <td '+ha+'><b>All usage costs (preparation, training, inference, errors, etc)</td>\n'
      h+='   <td '+ha+'><b>Model size</b></td>\n'
      h+='   <td '+ha+'><b><a href="https://github.com/dividiti/ck-caffe/blob/master/script/explore-accuracy/explore_accuracy.20160808.ipynb">Model accuracy on ImageNet</a></td>\n'
      h+='   <td '+ha+'><b>Model topology and parameters</td>\n'
      h+='   <td '+ha+'><b>Power consumption (W)<br>min&nbsp;/&nbsp;max</td>\n'
      h+='   <td '+ha+'><b>Acoustic noise (dB)<br>min&nbsp;/&nbsp;max</td>\n'
      h+='   <td '+ha+'><b>Memory usage (MB)</td>\n'
      h+='   <td '+ha+'><b>Bug detected?</b></td>\n'
      h+='   <td '+ha+'><b>User</b></td>\n'
      h+='   <td '+ha+'><b>Replay</b></td>\n'
      h+='  <tr>\n'
  
      # Dictionary to hold target meta
      tm={}
  
      ix=0
      bgraph={'0':[]} # Just for graph demo
      if hi_uid!='':
          bgraph['1']=[]
  
      # Load min stat
      for q in plst:
          pmin=os.path.join(q['path'],ffmin)
          dx={'##characteristics#run#time_fwbw_ms#min':1e99}
  
          if os.path.isfile(pmin):
             rx=ck.load_json_file({'json_file':pmin})
             if rx['return']==0:
                dx=rx['dict']
  
                # Fix
                x=dx.get('##characteristics#run#time_fwbw_ms#min','')
                if x==None or x=='' or x>50000: 
                   dx['##characteristics#run#time_fwbw_ms#min']=1e99
                   if q.get('meta',{}).get('state',{}).get('fail_reason','')=='':
                      q['meta']['state']['fail']='yes'
                      q['meta']['state']['fail_reason']='strange timing'
  
          q['min_stat']=dx
  
      # Sort
      splst=sorted(plst, key=lambda x: x.get('min_stat',{}).get('##characteristics#run#time_fwbw_ms#min',0))
  
  #    splst=sorted(plst, key=lambda x: x.get('meta',{}).get('characteristics',{}).get('run',{}).get('time_fwbw_ms',0))
  
      for q in splst:
          ix+=1
  
          duid=q['data_uid']
          path=q['path']
  
          d=q['meta']
  
          # Characteristics
          # Check if has statistics
          dstat={}
          fstat=os.path.join(path,'ck-stat-flat-characteristics.json')
          if os.path.isfile(fstat):
              r=ck.load_json_file({'json_file':fstat, 'dict':dstat})
              if r['return']>0: return r
              dstat=r['dict']
  
          x0=dstat.get("##characteristics#run#time_fwbw_ms#min",None)
  
          meta=d.get('meta',{})
  
          choices=d.get('choices',{})
          env=choices.get('env',{})
          params=choices.get('params',{}).get('params',{})
  
          xdeps=meta.get('xdeps',{})
  
          d_model=xdeps.get('caffemodel',{})
          d_model_name=d_model.get('data_name','')
          d_model_package_uoa=d_model.get('package_uoa','')
          d_model_ver=d_model.get('ver','')
  
          d_engine=xdeps.get('lib-caffe',{})
          d_engine_name=d_engine.get('data_name','')
          d_engine_package_uoa=d_engine.get('package_uoa','')
          d_engine_ver=d_engine.get('ver','')
  
          tp=meta.get('dnn_type','')
          nn=meta.get('nn_type','')
  
          plat_name=meta.get('plat_name','')
          cpu_name=meta.get('cpu_name','')
          os_name=meta.get('os_name','')
          gpgpu_name=meta.get('gpgpu_name','')
  
          plat_uid=meta.get('platform_uid','')
          cpu_uid=meta.get('cpu_uid','')
          os_uid=meta.get('os_uid','')
          gpu_uid=meta.get('gpu_uid','')
          gpgpu_uid=meta.get('gpgpu_uid','')
  
          user=meta.get('user','')
  
          te=d.get('characteristics',{}).get('run',{})
  
  #        bgc='afffaf'
          bgc='dfffdf'
          fail=d.get('state',{}).get('fail','')
          fail_reason=d.get('state',{}).get('fail_reason','')
          if fail=='yes':
              if fail_reason=='': fail_reason='yes'
              bgc='ffafaf'
          elif hi_uid!='' and duid==hi_uid:
              bgc='9fff9f'
  #            bgraph['0'].append([ix,None])
  #            bgraph['1'].append([ix,x0])
  
          bg=' style="background-color:#'+bgc+';"'
  
          h+='  <tr'+bg+'>\n'
  
          # Number
          h+='   <td '+ha+'><a name="'+duid+'">'+str(ix)+'</a></td>\n'
  
          # Platform, etc ...
          x=plat_name
          if plat_uid!='':
              x='<a href="'+url0+'&wcid='+cfg['module_deps']['platform']+':'+plat_uid+'">'+x+'</a>'
          h+='   <td '+ha+'>'+x+'</td>\n'
  
          x=os_name
          if os_uid!='':
              x='<a href="'+url0+'&wcid='+cfg['module_deps']['platform']+':'+os_uid+'">'+x+'</a>'
          h+='   <td '+ha+'>'+x+'</td>\n'
  
          x=cpu_name
          if cpu_uid!='':
              x='<a href="'+url0+'&wcid='+cfg['module_deps']['platform.cpu']+':'+cpu_uid+'">'+x+'</a>'
          h+='   <td '+ha+'>'+x+'</td>\n'
  
          x=gpgpu_name
          if gpgpu_uid!='':
              x='<a href="'+url0+'&wcid='+cfg['module_deps']['platform.gpgpu']+':'+gpgpu_uid+'">'+x+'</a>'
          h+='   <td '+ha+'>'+x+'</td>\n'
  
          # All files
          uu1=work['self_module_uid']
          if cmuoa!='': uu1=cmuoa
          uu2=str(ix)+')&nbsp;<a href="'+url0+'&wcid='+uu1+':'+duid+'">'+duid+'</a>'
          uu3='[&nbsp;<a href="'+url0+'&wcid='+uu1+':'+duid+'">See&nbsp;raw&nbsp;files</a>&nbsp;]<br><br>('+duid+')'
          uu4=uu1+':'+duid
  
          # Type
          h+='   <td '+ha+'>'+tp+'</a></td>\n'
  
          # Engine
          x=d_engine_name
  
          if d_engine_package_uoa!='':
             x='<a href="'+url0+'&wcid=package:'+d_engine_package_uoa+'">'+x+'</a>'
  
          if x!='' and d_engine_ver!='':
             x+='\n<br><br>Version&nbsp;<b>'+d_engine_ver+'</b>'
  
          # Versions
          ver=''
          dver=meta.get('xversions',{})
          for dx in sorted(dver):
              vx=dver[dx]
              if vx!=None and vx!='':
                 ver+=dx+': '+str(dver[dx])+'\n'
  
          ver=ver.replace("\'","'").replace("'","\\'").replace('\"','"').replace('"',"\\'").replace('\n','\\n')
          if ver!='':
              ver='<input type="button" class="ck_small_button" onClick="alert(\''+ver+'\');" value="See versions of all deps">'
  
          h+='   <td '+ha+'>'+x+'<br><br>'+ver+'</td>\n'
  
          # Model
  
          x=nn
  
          msize=''
          mtop=''
          mtop5=''
  
          if d_model_package_uoa!='':
             x='<a href="'+url0+'&wcid=package:'+d_model_package_uoa+'">'+x+'</a>'
  
             # Load features
             rx=ck.access({'action':'load',
                           'module_uoa':'package',
                           'data_uoa':d_model_package_uoa})
             if rx['return']==0: 
                mft=rx['dict'].get('features',{})
  
                msize=str(mft.get('model_size_mb',''))+'&nbsp;MB'
                mtop=str(mft.get('accuracy',''))
                mtop5=str(mft.get('accuracy_top5',''))
  
  #        if x!='' and d_model_ver!='':
  #           x+='\n<br><br>Version&nbsp;<b>'+d_model_ver+'</b>'
  
          h+='   <td '+ha+'>'+x+'</td>\n'
  
          # Choices (for now env)
  #        x='<table border="0" cellpadding="0" cellspacing="2">\n'
          x=''
          for k in sorted(env):
              v=env[k]
              x+=str(k)+'='+str(v)+'\n'
  #            x+='<tr><td>'+str(k)+'=</td><td>'+str(v)+'</td></tr>\n'
  #        x+='</table>\n'
  #        x=x.replace("'","\'").replace('"',"\\'").replace('\n','\\n')
          x=x.replace("\'","'").replace("'","\\'").replace('\"','"').replace('"',"\\'").replace('\n','\\n')
  
          x1=''
          if x!='':
              if env.get('CK_CAFFE_BATCH_SIZE','')!='':
                 x1+='Batch&nbsp;size='+env['CK_CAFFE_BATCH_SIZE']+'<br><br>\n'
              x1+='<input type="button" class="ck_small_button" onClick="alert(\''+x+'\');" value="View all">'
  
          h+='   <td '+ha+'>'+x1+'</td>\n'
  
  
          x=''
  
          # Check if has stats
          x0=dstat.get("##characteristics#run#time_fwbw_ms#min",None)
          x0e=dstat.get("##characteristics#run#time_fwbw_ms#exp",None)
          x1=dstat.get("##characteristics#run#time_fwbw_ms#center",None)
          xr=dstat.get("##characteristics#run#time_fwbw_ms#repeats",None)
          x2=dstat.get("##characteristics#run#time_fwbw_ms#halfrange",None)
          x=''
          if x0!=None:
              x='<b>'+('%.0f'%x0)+'&nbsp;ms.</b>\n'
  #            x+='('+('%.0f'%x1)+'&nbsp;&PlusMinus;&nbsp;'+('%.0f'%x2)+'&nbsp;ms.)'
  
          if x0e!=None and x2!=None:
              x+='<br><br>('+('%.0f'%x0e)+'&nbsp;&PlusMinus;&nbsp;'+('%.0f'%x2)+'&nbsp;ms.)\n'
  
          if xr!=None:
              x+='<br><i>'+str(xr)+' repetitions</i>\n'
  
          h+='   <td '+ha+' style="background-color:#afffaf">'+x+'</td>\n'
  
          if fail=='yes': x0=0
          bgraph['0'].append([ix,x0])
          if fail!='yes' and x0!=None and duid!=hi_uid:
              if hi_uid!='': bgraph['1'].append([ix,None])
  
          x1=dstat.get("##characteristics#run#time_fw_ms#center",None)
          x2=dstat.get("##characteristics#run#time_fw_ms#halfrange",None)
          if x1!=None and x2!=None:
              x=('%.0f'%x1)+'&nbsp;&PlusMinus;&nbsp;'+('%.0f'%x2)+'&nbsp;ms.'
  
          h+='   <td '+ha+'>'+x+'</td>\n'
  
          x1=dstat.get("##characteristics#run#time_bw_ms#center",None)
          x2=dstat.get("##characteristics#run#time_bw_ms#halfrange",None)
          if x1!=None and x2!=None:
              x=('%.0f'%x1)+'&nbsp;&PlusMinus;&nbsp;'+('%.0f'%x2)+'&nbsp;ms.'
  
          h+='   <td '+ha+'>'+x+'</td>\n'
  
          # Check all characteristics
          x=''
          x5=''
          for k in sorted(te):
              v=te[k]
  
              kx="##characteristics#run#"+k
  
              kx1=dstat.get(kx+'#center',None)
              kx2=dstat.get(kx+'#halfrange',None)
  
              x6=''
              if type(v)==int:
                  if kx1!=None and kx2!=None:
                      x6=str(kx1)+' +- '+str(kx2)
                  else:
                      x6=str(v)
              elif type(v)==float:
                  if kx1!=None and kx2!=None:
                      x6=('%.1f'%kx1)+' +- '+('%.1f'%kx2)
                  else:
                      x6=('%.1f'%v)
  
              if x6!='':
                  x5+=str(k)+'='+x6+'\n'
  
          # Also layers
          y5=''
          for j in range(0,1000):
              k1='##characteristics#run#per_layer_info@'+str(j)+'#direction#min'
              k2='##characteristics#run#per_layer_info@'+str(j)+'#label#min'
              k3='##characteristics#run#per_layer_info@'+str(j)+'#time_ms#min'
              k4='##characteristics#run#per_layer_info@'+str(j)+'#time_ms#max'
              k5='##characteristics#run#per_layer_info@'+str(j)+'#time_ms#exp_allx'
  
              v1=dstat.get(k1,'')
              v2=dstat.get(k2,'')
              v3=dstat.get(k3,'')
              v4=dstat.get(k4,'')
              v5=dstat.get(k5,[])
  
              if v1!='' and v2!='' and v3!='' and v4!='':
                 v6=0
                 if len(v5)>0:
                    v6=v5[0]
  
                 xv3=''
                 xv4=''
                 xv5=''
  
                 if v3!='': xv3=('%.1f'%v3)
                 if v4!='': xv4=('%.1f'%v4)
                 if v6!='': xv6=('%.1f'%v6)
  
                 if y5=='': y5='Layers:\nName (direction): min time (ms.) ; expected time (ms.) ; max time (ms.)\n'
  
                 y5+='\n'+v2+' ('+v1+'): '+xv3+';'+xv6+';'+xv4
              else:
                 break
  
          y5=y5.replace("\'","'").replace("'","\\'").replace('\"','"').replace('"',"\\'").replace('\n','\\n')
          if y5!='':
              x+='<a href="'+ck_url1+duid+'">Stats per layer</a><br><br>\n'
              x+='<input type="button" class="ck_small_button" onClick="alert(\''+y5+'\');" value="All layers as pop-up">'
  
  #        x5=x5.replace("'","\'").replace('"',"\\'").replace('\n','\\n')
          x5=x5.replace("\'","'").replace("'","\\'").replace('\"','"').replace('"',"\\'").replace('\n','\\n')
          if x5!='':
              x+='<br><br><input type="button" class="ck_small_button" onClick="alert(\''+x5+'\');" value="CK vars">'
  
          h+='   <td '+ha+'>'+x+'</td>\n'
  
          # Get info about platform
          hd={}
          if plat_uid!='':
             rh=ck.access({'action':'load',
                          'module_uoa':cfg['module_deps']['platform'],
                          'data_uoa':plat_uid})
             if rh['return']==0:
                hd=rh['dict']
  
          # Cost (take from platform meta)
          hc='-'
          if len(hd)>0:
             costs=hd.get('features',{}).get('cost',[])
             hc=''
             for c in costs:
                 if hc!='': hc+='<br>\n'
                 hc+='<b>'+str(c.get('price',''))+' '+c.get('currency','')+ '</b> - '+c.get('desc','')+' ('+c.get('date','')+')'
  
          h+='   <td '+ha+'>'+hc+'</a></td>\n'
  
          # TBD: all other costs
          h+='   <td '+ha+'></a></td>\n'
  
          # Model size
          h+='   <td '+ha+'>'+msize+'</td>\n'
  
          # Accuracy
          x=''
  
          if mtop!='' and mtop5!='':
             x=mtop+'&nbsp;/&nbsp;'+mtop5
  
  #        if nn=='bvlc, alexnet':
  #            x='0.568279&nbsp;/&nbsp;0.799501'
  #        elif nn=='bvlc, googlenet':
  #            x='0.689299&nbsp;/&nbsp;0.891441'
  #        elif nn=='deepscale, squeezenet, 1.1':
  #            x='0.583880&nbsp;/&nbsp;0.810123'
  #        elif nn=='deepscale, squeezenet, 1.0':
  #            x='0.576801&nbsp;/&nbsp;0.803903'
  
          h+='   <td '+ha+'>'+x+'</td>\n'
  
          # Model topology
          x=''
  
          fmt=d.get('file_model_topology','')
          if fmt!='':
             pfmt=os.path.join(path,fmt)
             if os.path.isfile(pfmt):
                 x='<a href="'+url0+'&action=pull&common_action=yes&cid='+work['self_module_uid']+':'+duid+'&filename='+fmt+'">deploy.prototxt</a>\n'
  
          h+='   <td '+ha+'>'+x+'</td>\n'
  
          # Power consumption (TBD - real measurements)
          x='-'
          if len(hd)>0:
             power=hd.get('features',{}).get('power_consumption',{})
             if len(power)>0:
                pmin=power.get('min','')
                pmax=power.get('max','')
  
                x=str(pmin)+' / '+str(pmax)
          h+='   <td '+ha+'>'+x+'</a></td>\n'
  
          # Acoustic noise (TBD - real measurements)
          x='-'
          if len(hd)>0:
             power=hd.get('features',{}).get('acoustic_noise',{})
             if len(power)>0:
                pmin=power.get('min','')
                pmax=power.get('max','')
  
                x=str(pmin)+' / '+str(pmax)
          h+='   <td '+ha+'>'+x+'</a></td>\n'
  
          # Memory usage
          x=''
  
          mem=dstat.get("##characteristics#run#memory_mbytes#max",None)
          if mem!=None:
             x=str(int(mem))+' MB'
  
          h+='   <td '+ha+'>'+x+'</td>\n'
  
  
          # Crowdsourcing bug detection
          x=fail_reason
          if x=='': 
              x=''
          else:
              fail_reason=fail_reason.replace("\'","'").replace("'","\\'").replace('\"','"').replace('"',"\\'").replace('\n','\\n')
              x='Yes <input type="button" class="ck_small_button" onClick="alert(\''+fail_reason+'\');" value="Log">'
  
          h+='   <td '+ha+'>'+x+'</td>\n'
  
  
          h+='   <td '+ha+'><a href="'+url0+'&action=index&module_uoa=wfe&native_action=show&native_module_uoa=experiment.user">'+user+'</a></td>\n'
  
          h+='   <td '+ha+'><input type="button" class="ck_small_button" onClick="copyToClipboard(\'ck replay '+uu4+' '+ck.cfg.get('add_extra_to_replay','')+'\');" value="Replay"><br><br>\n'
          h+='              '+uu3+'</td>\n'
  
          h+='  <tr>\n'
  
      h+='</table>\n'
      h+='</center>\n'
  
      if cmuoa=='':
          h+='</form>\n'
  
      if len(bgraph['0'])>0:
         ii={'action':'plot',
             'module_uoa':cfg['module_deps']['graph'],
  
             "table":bgraph,
  
             "h_lines":[1.0],
  
             "ymin":0,
  
             "ignore_point_if_none":"yes",
  
             "plot_type":"d3_2d_bars",
  
             "display_y_error_bar":"no",
  
             "title":"Powered by Collective Knowledge",
  
             "x_ticks_period":10,
  
             "axis_x_desc":"Experiment",
             "axis_y_desc":"Neural network total time (ms.)",
  
             "plot_grid":"yes",
  
             "d3_div":"ck_interactive",
  
             "image_width":"900",
             "image_height":"400",
  
             "wfe_url":url0}
  
         r=ck.access(ii)
         if r['return']==0:
            x=r.get('html','')
            if x!='':
               st+=r.get('style','')
  
               h+='<br>\n'
               h+='<center>\n'
               h+='<div id="ck_box_with_shadow" style="width:920px;">\n'
               h+=' <div id="ck_interactive" style="text-align:center">\n'
               h+=x+'\n'
               h+=' </div>\n'
               h+='</div>\n'
               h+='</center>\n'
  
      return {'return':0, 'html':h, 'style':st}
  
  ##############################################################################
  # browse public results
  
  def browse(i):
      """
      Input:  {
              }
  
      Output: {
                return       - return code =  0, if successful
                                           >  0, if error
                (error)      - error text if return > 0
              }
  
      """
  
      import webbrowser
  
      ck.out('Opening web page '+ck_url+' ...')
  
      webbrowser.open(ck_url)
  
      return {'return':0}
  
  ##############################################################################
  # show info for all layers
  
  def html_viewer(i):
      """
      Input:  {
                data_uoa - CK entry UOA to view
              }
  
      Output: {
                return       - return code =  0, if successful
                                           >  0, if error
                (error)      - error text if return > 0
              }
  
      """
  
      import os
  
      duoa=i.get('data_uoa','')
  
      # Load entry
      r=ck.access({'action':'load',
                   'module_uoa':work['self_module_uid'],
                   'data_uoa':duoa})
      if r['return']>0: return r
      p=r['path']
      d=r['dict']
  
      dchars=d.get('characteristics',{})
      dchoices=d.get('choices',{})
      dmeta=d.get('meta',{})
  
      # Load stats
      dstat={}
      fstat=os.path.join(p,'ck-stat-flat-characteristics.json')
      if os.path.isfile(fstat):
          r=ck.load_json_file({'json_file':fstat, 'dict':dstat})
          if r['return']>0: return r
          dstat=r['dict']
  
      # Prepare table
      h=''
  #    h+='<hr>\n'
      h+='<br>\n'
      h+='<center>\n'
      h+='<h2>DNN engine and model evaluation statistics per layer (crowd-tuning)</h2><br>\n'
      h+='</center>\n'
  
      xdeps=dmeta.get('xdeps',{})
      lcaffe=xdeps.get('lib-caffe',{})
      lmodel=xdeps.get('caffemodel',{})
  
      # Prepare extra info
      h+='<p>\n'
      h+='<table border="1" cellpadding="8" cellspacing="0">\n'
  
      h+=' <tr>\n'
      h+='  <td><b>DNN engine name:</b></td>\n'
      h+='  <td>'+lcaffe.get('data_name','')+'</td>\n'
      h+=' </tr>\n'
  
      h+=' <tr>\n'
      h+='  <td><b>DNN engine version:</b></td>\n'
      h+='  <td>'+lcaffe.get('ver','')+'</td>\n'
      h+=' </tr>\n'
  
      h+=' <tr>\n'
      h+='  <td><b>DNN engine type:</b></td>\n'
      h+='  <td>'+dmeta.get('dnn_type','')+'</td>\n'
      h+=' </tr>\n'
  
      x=''
  
      dx=dmeta.get('xversions',{})
      for k in sorted(dx):
          v=dx[k]
          if v!='':
             if x!='': x+='<br>\n'
             x+=k+'='+str(v)+'\n'
  
      h+=' <tr>\n'
      h+='  <td><b>DNN engine dependencies:</b></td>\n'
      h+='  <td>'+x+'</td>\n'
      h+=' </tr>\n'
  
      h+=' <tr>\n'
      h+='  <td><b>DNN model name:</b></td>\n'
      h+='  <td>'+lmodel.get('data_name','')+'</td>\n'
      h+=' </tr>\n'
  
      h+=' <tr>\n'
      h+='  <td><b>DNN model version:</b></td>\n'
      h+='  <td>'+lmodel.get('ver','')+'</td>\n'
      h+=' </tr>\n'
  
      h+=' <tr>\n'
      h+='  <td><b>Batch size:</b></td>\n'
      h+='  <td>'+dchars.get('run',{}).get('REAL_ENV_CK_CAFFE_BATCH_SIZE','')+'</td>\n'
      h+=' </tr>\n'
  
  # TBD: Need to show min,exp,max!
  #    h+=' <tr>\n'
  #    h+='  <td><b>FWBW time (ms.):</b></td>\n'
  #    h+='  <td>'+str(dchars.get('run',{}).get('time_bw_ms',''))+'</td>\n'
  #    h+=' </tr>\n'
  
  #    h+=' <tr>\n'
  #    h+='  <td><b>FW time (ms.):</b></td>\n'
  #    h+='  <td>'+str(dchars.get('run',{}).get('time_fw_ms',''))+'</td>\n'
  #    h+=' </tr>\n'
  
  #    h+=' <tr>\n'
  #    h+='  <td><b>BW time (ms.):</b></td>\n'
  #    h+='  <td>'+str(dchars.get('run',{}).get('time_bw_ms',''))+'</td>\n'
  #    h+=' </tr>\n'
  
      h+=' <tr>\n'
      h+='  <td><b>Platform:</b></td>\n'
      h+='  <td>'+dmeta.get('plat_name','')+'</td>\n'
      h+=' </tr>\n'
  
      h+=' <tr>\n'
      h+='  <td><b>OS:</b></td>\n'
      h+='  <td>'+dmeta.get('os_name','')+'</td>\n'
      h+=' </tr>\n'
  
      h+=' <tr>\n'
      h+='  <td><b>CPU:</b></td>\n'
      h+='  <td>'+dmeta.get('cpu_name','')+'</td>\n'
      h+=' </tr>\n'
  
      h+=' <tr>\n'
      h+='  <td><b>GPU:</b></td>\n'
      h+='  <td>'+dmeta.get('gpu_name','')+'</td>\n'
      h+=' </tr>\n'
  
  
      h+=' </tr>\n'
      h+='</table>\n'
  
      h+='<center>\n'
      h+='<p>\n'
      h+='<table border="0" cellpadding="10" cellspacing="0">\n'
  
      h+=' <tr>\n'
      h+='  <td><b>Name</b></td>\n'
      h+='  <td><b>Direction</b></td>\n'
      h+='  <td align="right"><b>Min time (ms.):</b></td>\n'
      h+='  <td align="right"><b>Expected time (ms.):</b></td>\n'
      h+='  <td align="right"><b>Max time (ms.):</b></td>\n'
      h+='  <td align="right"><b>Repetitions:</b></td>\n'
      h+=' </tr>\n'
  
      # Detecting number of layers
      jj={}
  
      for j in range(0,1000):
          k3='##characteristics#run#per_layer_info@'+str(j)+'#time_ms#min'
          v3=dstat.get(k3,'')
  
          if v3=='': break
  
          jj[j]=v3
  
      # Sorting by min time
      if i.get('all_params',{}).get('skip_sort','')!='yes':
         jj=sorted(jj, key=lambda x: jj[x], reverse=True)
  
      # Also layers
      for j in jj:
          k1='##characteristics#run#per_layer_info@'+str(j)+'#direction#min'
          k2='##characteristics#run#per_layer_info@'+str(j)+'#label#min'
          k3='##characteristics#run#per_layer_info@'+str(j)+'#time_ms#min'
          k4='##characteristics#run#per_layer_info@'+str(j)+'#time_ms#max'
          k5='##characteristics#run#per_layer_info@'+str(j)+'#time_ms#exp_allx'
          k7='##characteristics#run#per_layer_info@'+str(j)+'#time_ms#repeats'
  
          v1=dstat.get(k1,'')
          v2=dstat.get(k2,'')
          v3=dstat.get(k3,'')
          v4=dstat.get(k4,'')
          v5=dstat.get(k5,[])
          v7=dstat.get(k7,'')
  
          if v1!='' and v2!='' and v3!='' and v4!='':
             v6=0
             if len(v5)>0:
                v6=v5[0]
  
             xv3=''
             xv4=''
             xv6=''
  
             if v3!='':
                if v3<0.1: xv3='0'
                else: xv3='<b>'+('%.1f'%v3)+'</b>'
  
             if v4!='':
                if v4<0.1: xv4='0'
                else: xv4='<b>'+('%.1f'%v4)+'</b>'
  
             if v6!='':
                if v6<0.1: xv6='0'
                else: xv6='<b>'+('%.1f'%v6)+'</b>'
  
             h+=' <tr>\n'
             h+='  <td>'+v2+'</td>\n'
             h+='  <td>'+v1+'</td>\n'
             h+='  <td align="right">'+xv3+'</td>\n'
             h+='  <td align="right">'+xv6+'</td>\n'
             h+='  <td align="right">'+xv4+'</td>\n'
             h+='  <td align="right">'+str(v7)+'</td>\n'
             h+=' </tr>\n'
  
      h+='</table>\n'
  
      h+='</center>\n'
  
      return {'return':0, 'html':h, 'show_top':'yes'}
  
  ##############################################################################
  # replay experiment (TBD)
  
  def replay(i):
      """
      Input:  {
                (data_uoa)
                (remote)
  
                (host_os)
                (target_os)
                (device_id)
              }
  
      Output: {
                return       - return code =  0, if successful
                                           >  0, if error
                (error)      - error text if return > 0
              }
  
      """
  
      import copy
      import os
  
      # Setting output
      o=i.get('out','')
      oo=''
      if o=='con': oo='con'
  
      duoa=i.get('data_uoa','')
      remote=i.get('remote','')
  
      er=''
      esr=''
  
      if remote=='yes':
         er=i.get('exchange_repo','')
         if er=='': er=ck.cfg['default_exchange_repo_uoa']
         esr=i.get('exchange_subrepo','')
         if esr=='': esr=ck.cfg['default_exchange_subrepo_uoa']
  
      # Try to load info
      if o=='con':
         ck.out('Loading experiment entry ...')
         ck.out('')
  
      r=ck.access({'action':'load',
                   'module_uoa':work['self_module_uid'],
                   'data_uoa':duoa,
                   'repo_uoa':er,
                   'remote_repo_uoa':esr})
      if r['return']>0: return r
  
      d=r['dict']
  
      hos=i.get('host_os','')
      tos=i.get('target_os','')
      tdid=i.get('device_id','')
  
      # Check two main deps (engine and model)
      meta=d.get('meta',{})
      xdeps=meta.get('xdeps',{})
  
  #    TBD: rebuild env by tags!
  #
  #    dnn=xdeps.get('lib-caffe',{})
  #    model=xdeps.get('caffemodel',{})
  #
  #    pdnn=dnn.get('package_uoa','')
  #    pmodel=model.get('package_uoa','')
  #
  #    preset_env={}
  #    penv=[pdnn,pmodel]
  #
  #    for j in range(0, len(penv)):
  #        px=''
  #        py=penv[j]
  #
  #        if py!='':
  #           # Search by package
  #           r=ck.access({'action':'search',
  #                        'module_uoa':cfg['module_deps']['env'],
  #                        'search_dict':{'package_uoa':py}})
  #           if r['return']>0: return r
  #
  #           l=r['lst']
  #
  #        if j==0: preset_env['lib-caffe']=px
  #        elif j==1: preset_env['caffemodel']=px
  
      # Run pipeline
      choices=d.get('choices',{})
      
      # Clean various vars
      for k in replay_clean_vars:
          if k in choices:
             del(choices[k])
  
      if i.get('target_os','')!='' and not i['target_os'].startswith('android'):
         del(i['target_os'])
  
      env=choices.get('env',{})
      for k in replay_clean_env_vars:
          if k in env:
             del(env[k])
      choices['env']=env
  
      if hos!='': choices['host_os']=hos
      if tos!='': choices['target_os']=tos
      if tdid!='': choices['device_id']=tdid
  
      pipeline_data_uoa=choices['module_uoa']
  
      # Prepare pipeline
      ii={'action':'pipeline',
          'module_uoa':cfg['module_deps']['program'],
          'prepare':'yes',
          'choices':choices,
          'out':o}
      rr=ck.access(ii)
      if rr['return']>0: return rr
  
      fail=rr.get('fail','')
      if fail=='yes':
          return {'return':10, 'error':'pipeline failed ('+rr.get('fail_reason','')+')'}
  
      ready=rr.get('ready','')
      if ready!='yes':
          return {'return':11, 'error':'couldn\'t prepare universal CK program workflow'}
  
      # Run pipeline
      ii={'action':'run',
          'module_uoa':cfg['module_deps']['pipeline'],
          'data_uoa':pipeline_data_uoa,
          'pipeline':rr,
          'out':o}
      rr=ck.access(ii)
      if rr['return']>0: return rr
  
      fail=rr.get('fail','')
      if fail=='yes':
          return {'return':10, 'error':'pipeline failed ('+rr.get('fail_reason','')+')'}
  
      if o=='con':
         ck.out('')
         ck.out('Your results:')
         ck.out('')
  
         dstat=rr.get('last_stat_analysis',{}).get('dict_flat',{})
  
         x0=dstat.get("##characteristics#run#time_fwbw_ms#min",None)
         x0e=dstat.get("##characteristics#run#time_fwbw_ms#exp",None)
  
         if x0!=None:
              ck.out('* FWBW min: '+('%.0f'%x0)+' ms.')
         if x0e!=None:
              ck.out('* FWBW exp: '+('%.0f'%x0e)+' ms.')
  
         x1=dstat.get("##characteristics#run#time_fw_ms#min",None)
         x1e=dstat.get("##characteristics#run#time_fw_ms#exp",None)
  
         if x1!=None:
              ck.out('* FW   min: '+('%.0f'%x1)+' ms.')
         if x1e!=None:
              ck.out('* FW   exp: '+('%.0f'%x1e)+' ms.')
  
         x2=dstat.get("##characteristics#run#time_bw_ms#min",None)
         x2e=dstat.get("##characteristics#run#time_bw_ms#exp",None)
  
         if x2!=None:
              ck.out('* BW   min: '+('%.0f'%x2)+' ms.')
         if x2e!=None:
              ck.out('* BW   exp: '+('%.0f'%x2e)+' ms.')
  
      return {'return':0}
  