content: import argparse
  import matplotlib
  matplotlib.use('agg')
  import csv
  import json
  import multiprocessing as mp
  import os
  import random
  import re
  import sys
  from functools import partial
  from operator import attrgetter, itemgetter
  import networkx as nx
  import numpy as np
  import pandas as pd
  import time
  from sofa_aisi import *
  from sofa_common import *
  from sofa_config import *
  from sofa_print import *
  from matplotlib import pyplot as plt
  import grpc
  import potato_pb2
  import potato_pb2_grpc
  import socket
  import random
  import subprocess
  from sofa_ml import hsg_v2
  
  def random_generate_color():
      rand = lambda: random.randint(0, 255)
      return '#%02X%02X%02X' % (64, rand(), rand())
  
  def get_top_k_events(cfg, df, topk):
      topk_events=[]
      gby = df.groupby(['name'])
      df_agg = gby.aggregate(np.sum)
      df_agg_sorted = df_agg.sort_values(by=['duration'],ascending=False)
      #memcpy = ['copyKind_1_','copyKind_2_','copyKind_8_']
      if cfg.verbose:
          print("Top %d Events: "%topk)
          print(df_agg_sorted[['duration']][0:topk])
      eventName = df_agg_sorted[df_agg_sorted.columns[0:0]].head(topk).index.values.tolist()
      return eventName
  
  
  # input: pfv(performance feature vector), Pandas.DataFrame
  # output: hint, docker_image  
  def get_hint(potato_server, features):
      
      if len(features) > 0:
          pfv = potato_pb2.PerformanceFeatureVector() 
          for i in range(len(features)):
              name = features.iloc[i]['name']
              value = features.iloc[i]['value']
              #print('%s%s%s' % (str(i).ljust(10), name.ljust(30), ('%.3lf'%value).ljust(20)))
              pfv.name.append(name)
              pfv.value.append(value)
  			
          #print('Wait for response from POTATO server...')
          myhostname = socket.gethostname()
          channel = grpc.insecure_channel(potato_server)
          stub = potato_pb2_grpc.HintStub(channel)
          request = potato_pb2.HintRequest( hostname = myhostname,
                                            pfv = pfv) 
          response = stub.Hint(request)
          hint = response.hint 
          docker_image = response.docker_image
      else:
          hint = 'There is no pfv to get hints.' 
          docker_image = 'NA' 
  
      return hint, docker_image 
  
  def concurrency_breakdown(logdir, cfg, df_mpstat, df_cpu, df_gpu, df_nvsmi, df_bandwidth, features):
      if cfg.verbose:
          print_title('Concurrency Breakdown Analysis')
  
      total_elapsed_time = {'usr':0, 'sys':0, 'gpu':0, 'iow':0, 'idl':0}
      elapsed_time_ratio = {'usr':0, 'sys':0, 'gpu':0, 'iow':0, 'idl':0}
      total_interval_vector = []
      total_performace_vector = []
      
   
      if len(df_mpstat) == 0:
          print_warning(cfg, 'no mpstat and perf traces!')
          return features
  
      t_begin = df_mpstat.iloc[0]['timestamp']
      t_end = df_mpstat.iloc[-1]['timestamp']
      t = t_begin
      sample_time = (1 / float(cfg.sys_mon_rate))
      while t < t_end:
          t = t + sample_time
          if cfg.roi_end > 0 and (t < cfg.roi_begin or t > cfg.roi_end):
              continue
          
          window_begin = t - sample_time 
          window_end = t
         
          if len(df_cpu) > 0: 
              if df_cpu.iloc[0].timestamp > window_end:
                  continue
              cond1 = (df_cpu['timestamp'] > window_begin)
              cond2 = (df_cpu['timestamp'] <= window_end)
              df_cpu_interval = df_cpu[ cond1 & cond2 ]
          
          num_gpus = len(list(set(df_nvsmi['deviceId'])))
          cond1 = (df_nvsmi['timestamp'] > window_begin)
          cond2 = (df_nvsmi['timestamp'] <= window_end)
          sm = df_nvsmi['event'] == int(0)
          df_nvsmi_interval = df_nvsmi[ cond1 & cond2 & sm ]
          
          cond1 = (df_mpstat['timestamp'] > window_begin)
          cond2 = (df_mpstat['timestamp'] <= window_end)
          df_mpstat_interval = df_mpstat[ cond1 & cond2 ]
           
          cond1 = (df_bandwidth['timestamp'] > window_begin)
          cond2 = (df_bandwidth['timestamp'] <= window_end)
          tx = df_bandwidth['event'] == float(0)
          rx = df_bandwidth['event'] == float(1)
          df_tx_interval = df_bandwidth[ cond1 & cond2 & tx ]
          df_rx_interval = df_bandwidth[ cond1 & cond2 & rx ]
  
          mp_usr = []
          mp_sys = []
          mp_idl = []
          mp_iow = []
  
          usr = []
          sys = []
          irq = []     
          
          cpu_max = 0
          cpu_min = 100
          for i in range(len(df_mpstat_interval)):
              ratios = df_mpstat_interval.iloc[i]['name'].split(':')[1].split('|')
              
              #print(ratios)
              mp_usr.append(sample_time*int(ratios[1])/100.0)
              mp_sys.append(sample_time*int(ratios[2])/100.0)
              mp_idl.append(sample_time*int(ratios[3])/100.0)
              mp_iow.append(sample_time*int(ratios[4])/100.0)
  
              usr.append(int(ratios[1]))
              sys.append(int(ratios[2]))
              irq.append(int(ratios[5]))
                       
              cpu_tmp = int(ratios[1]) + int(ratios[2]) + int(ratios[5])
              if cpu_tmp > cpu_max:
                  cpu_max = cpu_tmp
              if cpu_tmp < cpu_min:
                  cpu_min = cpu_tmp
          mp_usr = np.asarray(mp_usr)
          mp_sys = np.asarray(mp_sys)
          mp_idl = np.asarray(mp_idl)
          mp_iow = np.asarray(mp_iow)
  
          usr = np.asarray(usr)
          sys = np.asarray(sys)
          irq = np.asarray(irq)
  
          elapsed_time = {'usr':0, 'sys':0, 'gpu':0, 'iow':0, 'idl':0} 
  
          if len(df_mpstat_interval) > 0:
              elapsed_time['usr'] = mp_usr.max()
              elapsed_time['sys'] = mp_sys.max()
              elapsed_time['gpu'] = df_nvsmi_interval['duration'].max() * 0.01 * sample_time
              elapsed_time['iow'] = mp_iow.max()
              #print('gput,usrt = ', elapsed_time['gpu'], elapsed_time['usr']) 
              dominator = max(elapsed_time, key=elapsed_time.get)
              #if elapsed_time['gpu'] > 0.1 :
              #    dominator = 'gpu'
              if elapsed_time[dominator] > sample_time * int(cfg.is_idle_threshold)/100:
                  total_elapsed_time[dominator] = total_elapsed_time[dominator] + sample_time
              else:
                  total_elapsed_time['idl'] += sample_time
  
              if num_gpus > 0:
                  time_gpu_avg = df_nvsmi_interval['duration'].sum() * 0.01 * sample_time / num_gpus
              else:
                  time_gpu_avg = 0 
  
              interval_vector = [mp_usr.max(),
                                 mp_sys.max(),
                                 mp_iow.max(),
                                 mp_idl.max(),
                                 time_gpu_avg,
                                 df_tx_interval['bandwidth'].sum(),
                                 df_rx_interval['bandwidth'].sum()]                             
              total_interval_vector.append(tuple(interval_vector)) 
              if num_gpus > 0:
                  sm_avg = df_nvsmi_interval['duration'].sum() / int(len(list(set(df_nvsmi_interval['deviceId']))))
              else:
                  sm_avg = 0
              performace_vector = [window_end,
                                   df_nvsmi_interval['duration'].max(), 
                                   sm_avg, 
                                   df_nvsmi_interval['duration'].min(), 
                                   round((usr.mean() + sys.mean() + irq.mean()), 0),
                                   cpu_max,
                                   cpu_min]
              total_performace_vector.append(tuple(performace_vector))
                                   
      total_all_elapsed_time = sum(total_elapsed_time.values())
      if total_all_elapsed_time > 0 :
          elapsed_time_ratio['usr'] = 100 * total_elapsed_time['usr'] / total_all_elapsed_time 
          elapsed_time_ratio['sys'] = 100 * total_elapsed_time['sys'] / total_all_elapsed_time 
          elapsed_time_ratio['gpu'] = 100 * total_elapsed_time['gpu'] / total_all_elapsed_time 
          elapsed_time_ratio['idl'] = 100 * total_elapsed_time['idl'] / total_all_elapsed_time
          elapsed_time_ratio['iow'] = 100 * total_elapsed_time['iow'] / total_all_elapsed_time 
          if cfg.verbose:
              print('Elapsed Time = %.1lf ' % total_all_elapsed_time)
              print('USR = %.1lf %%' % elapsed_time_ratio['usr'])
              print('SYS = %.1lf %%' % elapsed_time_ratio['sys'])
              if num_gpus > 0:
                  print('GPU = %.1lf %%' % elapsed_time_ratio['gpu'])
              print('IDL = %.1lf %%' % elapsed_time_ratio['idl'])
              print('IOW = %.1lf %%' % elapsed_time_ratio['iow'])
          if cfg.spotlight_gpu:
              elapsed_hotspot_time = cfg.roi_end - cfg.roi_begin 
          else:
              elapsed_hotspot_time = 0 
      
          df = pd.DataFrame({ 'name':['elapsed_usr_time_ratio', 'elapsed_sys_time_ratio', 'elapsed_gpu_time_ratio', 
                              'elapsed_iow_time_ratio', 'elapsed_hotspot_time'], 
                              'value':[elapsed_time_ratio['usr'], elapsed_time_ratio['sys'], elapsed_time_ratio['gpu'], 
                              elapsed_time_ratio['iow'], elapsed_hotspot_time ] }, 
                              columns=['name','value'])
      
          features = pd.concat([features, df])
      
      if len(total_performace_vector) > 0:
          performance_table = pd.DataFrame(total_performace_vector, columns = ['time', 'max_gpu_util', 'avg_gpu_util', 'min_gpu_util', 'cpu_util', 'cpu_max', 'cpu_min'])
          performance_table.to_csv('%s/performance.csv' % logdir)
          vector_table = pd.DataFrame(total_interval_vector, columns = ['usr' , 'sys', 'iow', 'idl','gpu', 'net_tx', 'net_rx'])
          pearson = vector_table.corr(method ='pearson').round(2)
          if cfg.verbose:
              print('Correlation Table :')
              print(pearson)
          df = pd.DataFrame({ 'name':['corr_gpu_usr', 'corr_gpu_sys', 'corr_gpu_iow', 'corr_gpu_ntx', 'corr_gpu_nrx'], 'value':[pearson['gpu'].usr, pearson['gpu'].sys, pearson['gpu'].iow, pearson['gpu'].net_tx, pearson['gpu'].net_rx]}, columns=['name','value'])
          features = pd.concat([features, df])
      return features
  
  def payload_sum(df):
      print((len(df)))
  
  class Event:
  
      def __init__(self, name, ttype, timestamp, duration):
          self.name = name
          self.ttype = ttype  # 0 for begin, 1 for end
          self.timestamp = timestamp
          self.duration = duration
  
      def __repr__(self):
          return repr((self.name, self.ttype, self.timestamp, self.duration))
  
  def nvsmi_profile(logdir, cfg, df_nvsmi, features):
      if not cfg.cluster_ip and cfg.verbose:
          print_title('SM & MEM & ENCODE/DECODE Profiling')
     
      if cfg.spotlight_gpu:
          if cfg.roi_end == 0 :
              print_warning(cfg, 'spotlight_gpu has no effects.')
          else:
              cond1 = (df_nvsmi['timestamp'] > cfg.roi_begin)
              cond2 = (df_nvsmi['timestamp'] <= cfg.roi_end)
              df_nvsmi = df_nvsmi[ cond1 & cond2 ]
  
      sm_start = df_nvsmi.iloc[0].timestamp 
      sm_end = df_nvsmi.iloc[-1].timestamp
      SM_time = sm_end - sm_start
      result = df_nvsmi.groupby(['deviceId','event'])['duration'].mean()
      result = result.astype(int)
      
      gpu_sm_util = df_nvsmi.groupby(['event'])['duration'].mean()[0]
      gpu_mem_util = df_nvsmi.groupby(['event'])['duration'].mean()[1]
      if cfg.nvsmi_data:
          gpu_enc_util = df_nvsmi.groupby(['event'])['duration'].mean()[2]
          gpu_dec_util = df_nvsmi.groupby(['event'])['duration'].mean()[3]
      else:
          gpu_enc_util = 0
          gpu_dec_util = 0
      
      sm = df_nvsmi['event'] == int(0)
      mem = df_nvsmi['event'] == int(1)
      enc = df_nvsmi['event'] == int(2)
      dec = df_nvsmi['event'] == int(3)
      gpunum = list(set(df_nvsmi['deviceId']))
      res = pd.DataFrame([], columns=['sm', 'mem', 'enc', 'dec'])
      sm_q = pd.DataFrame([], columns=['Q1', 'Q2', 'Q3', 'Avg'])
      mem_q = pd.DataFrame([], columns=['Q1', 'Q2', 'Q3', 'Avg'])
      for i in gpunum:
          gpuid = df_nvsmi['deviceId'] == int(i)
          gpudata = [round(df_nvsmi[sm & gpuid]['duration'].mean(), 2),
                     round(df_nvsmi[mem & gpuid]['duration'].mean(), 2),
                     round(df_nvsmi[enc & gpuid]['duration'].mean(), 2),
                     round(df_nvsmi[dec & gpuid]['duration'].mean(), 2)]
          smdata = [round(df_nvsmi[sm & gpuid]['duration'].quantile(0.25), 2),
                    round(df_nvsmi[sm & gpuid]['duration'].quantile(0.5), 2),
                    round(df_nvsmi[sm & gpuid]['duration'].quantile(0.75), 2),
                    round(df_nvsmi[sm & gpuid]['duration'].mean(), 2)]
          memdata = [round(df_nvsmi[mem & gpuid]['duration'].quantile(0.25), 2),
                     round(df_nvsmi[mem & gpuid]['duration'].quantile(0.5), 2),
                     round(df_nvsmi[mem & gpuid]['duration'].quantile(0.75), 2),
                     round(df_nvsmi[mem & gpuid]['duration'].mean(), 2)]
          gpu_tmp = pd.DataFrame([gpudata], columns=['sm', 'mem', 'enc', 'dec'], index=[i])
          sm_tmp = pd.DataFrame([smdata], columns=['Q1', 'Q2', 'Q3', 'Avg'], index=[i])
          mem_tmp = pd.DataFrame([memdata], columns=['Q1', 'Q2', 'Q3', 'Avg'], index=[i])
          res = pd.concat([res, gpu_tmp])
          sm_q = pd.concat([sm_q, sm_tmp]) 
          mem_q = pd.concat([mem_q, mem_tmp])
      res.index.name = 'gpu_id'
      sm_q.index.name = 'gpu_id'
      mem_q.index.name = 'gpu_id'
  
      if not cfg.cluster_ip and cfg.verbose:
          print('GPU Utilization (%):')
          print(res)
          print('\nGPU SM Quartile (%):')
          print(sm_q)
          print('\nGPU MEM Quartile (%):')
          print(mem_q)
          print('Overall Average SM Utilization (%): ', int(gpu_sm_util))
          print('Overall Average MEM Utilization (%): ', int(gpu_mem_util))
          print('Overall Average ENC Utilization (%): ', int(gpu_enc_util))
          print('Overall Average DEC Utilization (%): ', int(gpu_dec_util))
          print('Overall Active GPU Time (s): %.3lf' % (SM_time * gpu_sm_util/100.0))
      df = pd.DataFrame({'name':['gpu_sm_util_q2', 'gpu_sm_util_q3', 'gpu_sm_util', 'gpu_mem_util_q2', 'gpu_mem_util_q3', 'gpu_mem_util'], 
                      'value':[df_nvsmi[sm & gpuid]['duration'].quantile(0.5),
                               df_nvsmi[sm & gpuid]['duration'].quantile(0.75),
                               int(gpu_sm_util),
                               df_nvsmi[mem & gpuid]['duration'].quantile(0.5),
                               df_nvsmi[mem & gpuid]['duration'].quantile(0.75),
                               int(gpu_mem_util),
                              ]}, 
                      columns=['name','value'])
      features = pd.concat([features, df])
  
      return features
  
  def gpu_profile(logdir, cfg, df_gpu, features):
      if cfg.verbose:
          print_title('GPU Profiling')
          print('Per-GPU time (s):')
      groups = df_gpu.groupby("deviceId")["duration"]
      gpu_time = 0
      for key, item in groups:
          gpuid = int(float(key))
          per_gpu_time = groups.get_group(key).sum()
          if cfg.verbose:
              print("[%d]: %lf" % (gpuid, per_gpu_time))
          gpu_time = gpu_time + per_gpu_time 
      num_gpus = len(groups)
     
      kernel_time = 0
      grouped_df = df_gpu.groupby("copyKind")["duration"]
      for key, item in grouped_df:
          if key == 0:
              kernel_time = grouped_df.get_group(key).sum()
  
      nccl_time = 0
      grouped_df = df_gpu.groupby("name")["duration"]
      for key, item in grouped_df:
          #print("[%s]: %lf" % (key, grouped_df.get_group(key).sum()))
          if key.find("nccl") != -1:
              nccl_time = nccl_time + grouped_df.get_group(key).sum()
  
      features = comm_profile(logdir, cfg, df_gpu, features)
  
      get_top_k_events(cfg, df_gpu, 10)
      df = pd.DataFrame({'name':['gpu_time', 'num_gpus', 'kernel_time', 'nccl_time'], 
                          'value':[gpu_time, num_gpus, kernel_time, nccl_time] }, 
                          columns=['name','value'])
      features = pd.concat([features, df])
      return features
  
  def strace_profile(logdir, cfg, df, features):
      print_title('STRACE Profiling:')
  
      return features
  
   
  def net_profile(logdir, cfg, df, features):
      if not cfg.cluster_ip:
          print_title("Network Profiling:")
      grouped_df = df.groupby("name")["duration"]
      net_time = 0
      n_packets = 0
      for key, item in grouped_df:
          #print("[%s]: %lf" % (key, grouped_df.get_group(key).sum()))
          if key.find("network:tcp:") != -1:
              net_time = net_time + grouped_df.get_group(key).sum()
              n_packets = n_packets + 1
      #print(("total network time (s) = %.3lf" % net_time))
      #print(("total amount of network packets  = %d" % n_packets))
      # total network packet
      packet_num_matrix = df.groupby(['pkt_src','pkt_dst','payload']).size().unstack(level=1, fill_value=0)
      
      # total network traffic
      packet_sum_matrix = df.groupby(['pkt_src','pkt_dst'])["payload"].sum().unstack(level=1, fill_value=0)
      
      # ================ change pandas table columns and index name ====
      rename_index = packet_sum_matrix.index.tolist()
      rename_index2 = packet_num_matrix.index.tolist()
      rename_columns = packet_sum_matrix.columns.tolist()
      rename_columns2 = packet_num_matrix.columns.tolist() 
  
      def zero(s):
          if s[0:2] == '00':
              s = s[2]
          elif (s[0] == '0') and (s[1] != '0'): 
              s = s[1:3]
          return(s)
  
      def check_str(rename_list):
          rename_list_new = []
          for j in rename_list:
              j = str(int(j))
              a = j[-9:-6]
              b = j[-6:-3]
              c = j[-3:]  
              j = j[:-9] + '.' + zero(a) + '.' + zero(b) + '.' + zero(c)
              rename_list_new.append(j)
          return(rename_list_new)
   
      def check_str2(rename_list):
          rename_columns_2 = []
          for i in rename_list:
              i = str(int(i[0]))
              a = i[-9:-6] 
              b = i[-6:-3]
              c = i[-3:]
              i = i[:-9] + '.' + zero(a) + '.' + zero(b) + '.' + zero(c)
              rename_columns_2.append(i)
          return(rename_columns_2)
      
  
  
      rename_index_new = check_str(rename_index)
      rename_index_new = dict(zip(rename_index, rename_index_new))
      
      rename_index2_new = check_str2(rename_index2)
      rename_index2_final = list(set(rename_index2_new))
      rename_index2_final.sort(key=rename_index2_new.index)
      
      rename_columns_new = check_str(rename_columns)
      rename_columns_new = dict(zip(rename_columns, rename_columns_new))
      
      rename_columns2_new = check_str(rename_columns2)
      rename_columns2_new = dict(zip(rename_columns2, rename_columns2_new))          
   
      # rename here
      packet_sum_matrix = packet_sum_matrix.rename(columns=rename_columns_new)
      packet_num_matrix = packet_num_matrix.rename(columns=rename_columns2_new)
      packet_sum_matrix = packet_sum_matrix.rename(index=rename_index_new)
      packet_num_matrix.index.set_levels(rename_index2_final , level = 0, inplace = True)
      
      if cfg.verbose:
          print("total amount of network traffic : ", convertbyte(df['payload'].sum()), '\n', packet_sum_matrix.to_string(), "\n")
          print("total amount of network packets = %d\n" % packet_num_matrix.sum().sum() ,packet_num_matrix.to_string(), "\n")
      
      network_value = []
      src = []
      dst = []
      final = []
  
      for index in packet_sum_matrix.index:
          for column in packet_sum_matrix.columns:
              src.append(index)
              dst.append(column)
              network_value.append(packet_sum_matrix[column][index])
      record = list(zip(src, dst, network_value))
      record.sort(key=lambda tup:tup[2], reverse=True)
      
      for src, dst, value in record:
          if value == 0:
              pass
          else:
              item = [src, dst, convertbyte(value), round(value / df['payload'].sum(), 2)]
              final.append(item)
      summary = pd.DataFrame(final, columns=['Source', 'Destination', 'Amount', 'Percentage of a Node'])
      summary.to_csv(logdir + 'netrank.csv',
                  mode='w',
                  header=True,
                  index=False)
  
      df = pd.DataFrame({'name':['net_time'],                                                 
                         'value':[net_time] },  
                         columns=['name','value'])
      features = pd.concat([features, df])
      return features
  
  def convertbyte(B):
      B = int(B)
      KB = float(1024)
      MB = float(KB ** 2) # 1,048,576
      GB = float(KB ** 3) # 1,073,741,824
      TB = float(KB ** 4) # 1,099,511,627,776
  
      if B < KB:
          return '{} Bytes'.format(B)
      elif KB <= B < MB:
          return '{0:.2f} KB'.format(B/KB)
      elif MB <= B < GB:
          return '{0:.2f} MB'.format(B/MB)
      elif GB <= B < TB:
          return '{0:.2f} GB'.format(B/GB)
      elif TB <= B:
          return '{0:.2f} TB'.format(B/TB)
  
  def convertbytes(B):
      B = float(B)
      KB = float(1024)
      MB = float(KB ** 2) # 1,048,576
      GB = float(KB ** 3) # 1,073,741,824
      TB = float(KB ** 4) # 1,099,511,627,776
  
      if B < KB:
          return '{0:.2f} B/s'.format(B)
      elif KB <= B < MB:
          return '{0:.2f} KB/s'.format(B/KB)
      elif MB <= B < GB:
          return '{0:.2f} MB/s'.format(B/MB)
      elif GB <= B < TB:
          return '{0:.2f} GB/s'.format(B/GB)
      elif TB <= B:
          return '{0:.2f} TB/s'.format(B/TB)
  
  def netbandwidth_profile(logdir, cfg, df, features):
      if not cfg.cluster_ip and cfg.verbose:
          print_title('Network Bandwidth Profiling:')
      tx = df['event'] == float(0)
      rx = df['event'] == float(1)
        
      bw_tx_q1 = df[tx]['bandwidth'].quantile(0.25)
      bw_tx_q2 = df[tx]['bandwidth'].quantile(0.5)
      bw_tx_q3 = df[tx]['bandwidth'].quantile(0.75)
      bw_tx_mean = int(df[tx]['bandwidth'].mean())
      bw_rx_q1 = df[rx]['bandwidth'].quantile(0.25)
      bw_rx_q2 = df[rx]['bandwidth'].quantile(0.5)
      bw_rx_q3 = df[rx]['bandwidth'].quantile(0.75)
      bw_rx_mean = int(df[rx]['bandwidth'].mean())
      with open('%s/netstat.txt' % logdir) as f:
          lines = f.readlines()  
          first_line = lines[0]  
          last_line = lines[-1]
          tx_begin = first_line.split(',')[1]
          rx_begin = first_line.split(',')[2]
          tx_end = last_line.split(',')[1]
          rx_end = last_line.split(',')[2]
          tx_amount = int(last_line.split(',')[1]) - int(first_line.split(',')[1])
          rx_amount = int(last_line.split(',')[2]) - int(first_line.split(',')[2])
      if not cfg.cluster_ip:
          bw_tx_q1 = df[tx]['bandwidth'].quantile(0.25)
          bw_tx_q2 = df[tx]['bandwidth'].quantile(0.5)
          bw_tx_q3 = df[tx]['bandwidth'].quantile(0.75)
          bw_tx_mean = int(df[tx]['bandwidth'].mean())
          bw_rx_q1 = df[rx]['bandwidth'].quantile(0.25)
          bw_rx_q2 = df[rx]['bandwidth'].quantile(0.5)
          bw_rx_q3 = df[rx]['bandwidth'].quantile(0.75)
          bw_rx_mean = int(df[rx]['bandwidth'].mean())
          if cfg.verbose:
              print('Amount of Network Traffic : %s' % (convertbyte(tx_amount + rx_amount)))
              print('Amount of tx : %s' % convertbyte(tx_amount))
              print('Amount of rx : %s' % convertbyte(rx_amount))
              print('Bandwidth Quartile :')
              print('Q1  tx : %s, rx : %s' % ( convertbytes(bw_tx_q1), convertbytes(bw_rx_q1)))
              print('Q2  tx : %s, rx : %s' % ( convertbytes(bw_tx_q2), convertbytes(bw_rx_q2)))
              print('Q3  tx : %s, rx : %s' % ( convertbytes(bw_tx_q3), convertbytes(bw_rx_q3)))
              print('Avg tx : %s, rx : %s'% ( convertbytes(bw_tx_mean), convertbytes(bw_rx_mean)))
  
      #network chart part
      all_time = df[tx]['timestamp'].tolist()
      all_tx = df[tx]['bandwidth'].tolist()
      all_rx = df[rx]['bandwidth'].tolist()
      fig = plt.figure(dpi=128, figsize=(16, 14))
      plt.plot(all_time, all_tx, c='red', alpha=0.5, label='tx')
      plt.plot(all_time, all_rx, c='blue', alpha=0.5, label='rx')
      plt.legend(loc='upper right')
      plt.title("Network Report", fontsize=18)
      plt.xlabel('Timestamp (s)', fontsize=16)
      plt.ylabel("Bandwidth (bytes)", fontsize=16)
      fig.savefig("%s/network_report.pdf" % logdir, bbox_inches='tight')
      if not cfg.cluster_ip and cfg.verbose:
          print('Network Bandwidth Chart is saved at %s/network_report.pdf' %logdir)
  
          df_feature = pd.DataFrame({ 'name':['bw_tx_q2', 'bw_tx_q3', 'bw_rx_q2', 'bw_rx_q3'], 
                          'value':[bw_tx_q2, bw_tx_q3, bw_rx_q2, bw_rx_q3] }, 
                          columns=['name','value'])
          features = pd.concat([features, df_feature])   
   
      return features
  
  def blktrace_latency_profile(logdir, cfg, df, features):
      with open('%s/btt.txt' % logdir) as f:
          lines = f.readlines()
          for i, line in enumerate(lines):
              if '==================== All Devices ====================' in line:
                  start = i
              if '==================== Device Merge Information ====================' in line:
                  end = i
                  break
          bttoutput_result = lines[start:end]
  
      df_offset = pd.read_table('%s/offset_all.txt' % logdir, delim_whitespace=True, names=('time', 'start', 'end'))
      time = df_offset['time'].tolist()
      start_b = df_offset['start'].tolist()
      end_b = df_offset['end'].tolist()
      fig = plt.figure(dpi=128, figsize=(16, 14))
      plt.plot(time, start_b, c='red', marker='o', alpha=0.3, label='Start block')
      plt.legend(loc='upper right')
      plt.title("Block Offset Report", fontsize=18)
      plt.xlabel('Timestamp (s)', fontsize=16)
      plt.ylabel("Block Number", fontsize=16)
      fig.savefig("%s/offset_of_device_report.pdf" % logdir, bbox_inches='tight')
      print('Offset of Device Report is saved at %s/offset_of_device_report.pdf' %logdir)
  
      if cfg.verbose:
          print_title('Storage Profiling:')
          print('Blktracae Latency (s):')
          for btt in bttoutput_result:
              print(btt[:-1])
  
      blktrace_latency = df['event'] == 'C'
      blktrace_latency_q1 = df[blktrace_latency]['duration'].quantile(0.25)
      blktrace_latency_q2 = df[blktrace_latency]['duration'].quantile(0.5)
      blktrace_latency_q3 = df[blktrace_latency]['duration'].quantile(0.75)
      blktrace_latency_mean = df[blktrace_latency]['duration'].mean()
      
      df_feature = pd.DataFrame({ 'name':['blktrace_latency_q1','blktrace_latency_q2','blktrace_latency_q3'], 
                          'value': [blktrace_latency_q1, blktrace_latency_q2, blktrace_latency_q3] }, 
                          columns=['name','value'])
      features = pd.concat([features, df_feature])
      
   
      return features
  
  def diskstat_profile(logdir, cfg, df, features):
      #diskstat_dev = list(set(df['dev']))
      diskstat_r_q1 = df.groupby('dev')['d_read'].quantile(0.25)
      diskstat_w_q1 = df.groupby('dev')['d_write'].quantile(0.25)
      diskstat_q1 = df.groupby('dev')['d_disk_total'].quantile(0.25)
      diskstat_r_q2 = df.groupby('dev')['d_read'].quantile(0.5)
      diskstat_w_q2 = df.groupby('dev')['d_write'].quantile(0.5)
      diskstat_q2 = df.groupby('dev')['d_disk_total'].quantile(0.5)
      diskstat_r_q3 = df.groupby('dev')['d_read'].quantile(0.75)
      diskstat_w_q3 = df.groupby('dev')['d_write'].quantile(0.75)
      diskstat_q3 = df.groupby('dev')['d_disk_total'].quantile(0.75)
      diskstat_r_avg = df.groupby('dev')['d_read'].mean()
      diskstat_w_avg = df.groupby('dev')['d_write'].mean()
      diskstat_avg = df.groupby('dev')['d_disk_total'].mean()
      diskstat_r_iops = df.groupby('dev')['r_iops'].mean()
      diskstat_w_iops = df.groupby('dev')['w_iops'].mean()
      diskstat_iops = df.groupby('dev')['iops'].mean()
      diskstat_wait = df.groupby('dev')['await_time'].mean()
      diskstat_table = pd.concat([diskstat_r_q1, diskstat_r_q2, diskstat_r_q3, diskstat_r_avg, 
                                  diskstat_w_q1, diskstat_w_q2, diskstat_w_q3, diskstat_w_avg,
                                  diskstat_q1, diskstat_q2, diskstat_q3, diskstat_avg,
                                  diskstat_r_iops, diskstat_w_iops, diskstat_iops,
                                  diskstat_wait], axis=1, sort=False)
      diskstat_columns = ['Q1 throughput(Read)', 'Q2 throughput(Read)', 'Q3 throughput(Read)', 'Avg throughput(Read)',
                          'Q1 throughput(Write)', 'Q2 throughput(Write)', 'Q3 throughput(Write)', 'Avg throughput(Write)',
                          'Q1 throughput(R+W)', 'Q2 throughput(R+W)', 'Q3 throughput(R+W)', 'Avg throughput(R+W)',
                          'Avg IOPS(Read)', 'Avg IOPS(Write)', 'Avg IOPS(R+W)', 'Avg Await time(ms)']
      diskstat_table.columns = diskstat_columns
      diskstat_dev = diskstat_table.index.format()
  
      final_table = pd.DataFrame(columns=diskstat_columns)
      for j, dev in enumerate(diskstat_dev):
          tmp_list = []
          for i in diskstat_columns[:-4]:
              tmp_list.append(convertbytes(diskstat_table.iloc[j][i]))
          for i in diskstat_columns[-4:-1]:
              tmp_list.append('%d' % int(diskstat_table.iloc[j][i]))
          tmp_list.append('%.3lf ms' % diskstat_table.iloc[j][-1])
          tmp_table = pd.DataFrame([tuple(tmp_list)],
                                   columns=diskstat_columns,
                                   index=[dev])
          final_table = pd.concat([final_table, tmp_table])
      if cfg.verbose:
          print_title('DISKSTAT Profiling:')
          print('Disk Throughput Quartile :')
          print(final_table.T)
      
      df_feature = pd.DataFrame({ 'name':['diskstat_q1','diskstat_q2','diskstat_q3'], 
                          'value': [diskstat_q1.mean(), diskstat_q2.mean(),  diskstat_q3.mean()] }, 
                          columns=['name','value'])
      features = pd.concat([features, df_feature])   
   
      return features
  
  def cpu_profile(logdir, cfg, df):
      if cfg.verbose:
          print_title('CPU Profiling:')
          print('elapsed_time (s) = %.6lf' % cfg.elapsed_time) 
      grouped_df = df.groupby("deviceId")["duration"]
      total_exec_time = 0
      for key, item in grouped_df:
          print(("[%d]: %lf" % (key, grouped_df.get_group(key).sum())))
          total_exec_time = total_exec_time + grouped_df.get_group(key).sum()
      
      print("total execution time (s) = %.3lf" % total_exec_time)
  
      cpu_detail_profile_df = df[['timestamp','duration','name']]
      cpu_detail_profile_df = cpu_detail_profile_df.sort_values(by=['duration'], ascending=False)
      cpu_detail_profile_df['ratio(%)'] = cpu_detail_profile_df['duration']/total_exec_time * 100
      cpu_detail_profile_df = cpu_detail_profile_df[['timestamp','ratio(%)','duration','name']]
      print(cpu_detail_profile_df[:20].to_string(index=False))
  
  def vmstat_profile(logdir, cfg, df, features):
      _,_,_,_,_,_,df['si'],df['so'],df['bi'],df['bo'],df['in'],df['cs'],_,_,_,_,_=df['name'].str.split('|').str
      for col_name in ('si','so','bi','bo','in','cs'):
          df[col_name] = df[col_name].str[3:]
      vmstat_traces = df[['si','so','bi','bo','in','cs']].astype(float)
      vm_bi = vmstat_traces['bi'].mean()
      vm_bo = vmstat_traces['bo'].mean()
      vm_cs = vmstat_traces['cs'].mean()
      vm_in = vmstat_traces['in'].mean()
      if cfg.verbose:
          print_title('VMSTAT Profiling:')
          print('average bi/s: %d' % int(vm_cs))
          print('average bo/s: %d' % int(vm_in))
          print('average cs/s: %d' % int(vm_bi))
          print('average in/s: %d' % int(vm_bo))
  
      df_feature = pd.DataFrame({ 'name':['vm_bi', 'vm_bo', 'vm_cs', 'vm_in' ], 
                          'value':[vm_bi, vm_bo, vm_cs, vm_in] }, 
                          columns=['name','value'])
      features = pd.concat([features, df_feature])   
  
      return features
  
  def mpstat_profile(logdir, cfg, df, features):
      if not cfg.cluster_ip and cfg.verbose:
          print_title('MPSTAT Profiling:')
      num_cores = int(df['deviceId'].max() + 1)
      df_summary = pd.DataFrame( np.zeros((num_cores,5)), columns=['USR','SYS','IDL','IOW','IRQ'])
      _,_,_,_,_,df['USR'],df['SYS'],df['IDL'],df['IOW'],df['IRQ'],_ = df["name"].str.split('|').str
      df[['USR','SYS','IDL','IOW','IRQ']] = df[['USR','SYS','IDL','IOW','IRQ']].astype(float)
      df["dt_all"] = np.where(df["IDL"]==100, 0.1, df["duration"]/((100-df["IDL"])/100.0))
      df["t_USR"] = df['dt_all'] * df['USR']/100.0
      df["t_SYS"] = df['dt_all'] * df['SYS']/100.0
      df["t_IDL"] = df['dt_all'] * df['IDL']/100.0
      df["t_IOW"] = df['dt_all'] * df['IOW']/100.0
      df["t_IRQ"] = df['dt_all'] * df['IRQ']/100.0
      dfs=[]
      for i in range(num_cores):
          dfs.append(df.loc[df['deviceId'] == float(i)])
      for index,dff in enumerate(dfs):
          df_summary.iloc[index]['USR'] = dff['t_USR'].sum()
          df_summary.iloc[index]['SYS'] = dff['t_SYS'].sum()
          df_summary.iloc[index]['IDL'] = dff['t_IDL'].sum()
          df_summary.iloc[index]['IRQ'] = dff['t_IRQ'].sum()
          df_summary.iloc[index]['IOW'] = dff['t_IOW'].sum()
      if not cfg.cluster_ip and cfg.verbose:
          print('CPU Utilization (%):')
          print('core\tUSR\tSYS\tIDL\tIOW\tIRQ')
      for i in range(len(df_summary)):
          t_sum = df_summary.iloc[i].sum() 
          if not cfg.cluster_ip and cfg.verbose:
              print('%3d\t%3d\t%3d\t%3d\t%3d\t%3d'%(i,int(100.0*df_summary.iloc[i]['USR']/t_sum),
                                                      int(100.0*df_summary.iloc[i]['SYS']/t_sum),
                                                      int(100.0*df_summary.iloc[i]['IDL']/t_sum),
                                                      int(100.0*df_summary.iloc[i]['IOW']/t_sum),
                                                      int(100.0*df_summary.iloc[i]['IRQ']/t_sum) ))
      if not cfg.cluster_ip and cfg.verbose:
          print('CPU Time (s):')
          print('core\tUSR\tSYS\tIDL\tIOW\tIRQ')
      for i in range(len(df_summary)):
          t_sum = df_summary.iloc[i].sum()
          if not cfg.cluster_ip and cfg.verbose:
              print('%3d\t%.2lf\t%.2lf\t%.2lf\t%.2lf\t%.2lf'%(i,
                                                      df_summary.iloc[i]['USR'],
                                                      df_summary.iloc[i]['SYS'],
                                                      df_summary.iloc[i]['IDL'],
                                                      df_summary.iloc[i]['IOW'],
                                                      df_summary.iloc[i]['IRQ'] ))
  
      total_cpu_time = df_summary[['USR','SYS','IRQ']].sum().sum()
      cpu_util = int(100*total_cpu_time / (num_cores*cfg.elapsed_time))
      if not cfg.cluster_ip and cfg.verbose:
          print('Active CPU Time (s): %.3lf' % total_cpu_time)
          print('Active CPU ratio (%%): %3d' % cpu_util)
      df_feature = pd.DataFrame({ 'name':['num_cores', 'cpu_util'], 
                          'value':[num_cores, cpu_util] }, 
                          columns=['name','value'])
      features = pd.concat([features, df_feature])   
      return features
  
  
  def sofa_analyze(cfg):
      print_main_progress('SOFA analyzing...')
      filein = []
      df_cpu = pd.DataFrame([], columns=cfg.columns)
      df_gpu = pd.DataFrame([], columns=cfg.columns)
      df_net = pd.DataFrame([], columns=cfg.columns)
      df_mpstat = pd.DataFrame([], columns=cfg.columns)
      df_vmstat = pd.DataFrame([], columns=cfg.columns)
      df_bandwidth = pd.DataFrame([], columns=cfg.columns)
      df_blktrace = pd.DataFrame([], columns=cfg.columns)
      df_diskstat = pd.DataFrame([], columns=cfg.columns)
      df_nvsmi = pd.DataFrame([], columns=cfg.columns)
      iter_summary = None
      logdir = cfg.logdir
  
      with open(logdir+'/misc.txt') as f:
          lines = f.readlines()
          elapsed_time = float(lines[0].split()[1])
          vcores = int(lines[2].split()[1])
          cfg.elapsed_time = float(lines[0].split()[1])
      
      filein_gpu = logdir + "gputrace.csv"
      filein_cpu = logdir + "cputrace.csv"
      filein_net = logdir + "nettrace.csv"
      filein_vmstat = logdir + "vmstat.csv"
      filein_mpstat = logdir + "mpstat.csv"
      filein_strace = logdir + "strace.csv"
      filein_nvsmi = logdir + "nvsmi_trace.csv"
      filein_bandwidth = logdir + "netstat.csv"
      filein_blktrace = logdir + "blktrace.csv"
      filein_diskstat = logdir + "diskstat_vector.csv"
  
      if os.path.isfile('%s/nvlink_topo.txt' % logdir):
  
          with open(logdir + 'nvlink_topo.txt') as f:
              lines = f.readlines()
              if len(lines) > 0:
                  title = lines[0]
                  num_gpus = 1
                  for word in title.split():
                      if re.match(r'GPU', word) != None :
                         num_gpus = num_gpus + 1
                  print_info(cfg,'# of GPUs: ' + str(num_gpus) )
                  edges = []
                  if len(lines) >= num_gpus+1:
                      for i in range(num_gpus):
                          connections = lines[1+i].split()
                          for j in range(len(connections)):
                              if connections[j] == 'NV1' or connections[j] == 'NV2':
                                  edges.append((i,j-1))
                                  #print('%d connects to %d' % (i, j-1))
  
                      ring_found = False
                      G = nx.DiGraph(edges)
                      # Try to find ring with its length of num_gpus
                      for cycle in nx.simple_cycles(G):
                          if len(cycle) == num_gpus:
                              if cfg.verbose:
                                  print('One of the recommended ring having length of %d' % len(cycle))
                              ring_found = True
                              os.system("mkdir -p sofalog/sofa_hints/")
                              xring_order = ','.join(map(str, cycle))
                              with open("sofalog/sofa_hints/xring_order.txt", "w") as f:
                                  f.write('export CUDA_VISIBLE_DEVICES=' + xring_order)
                              break
  
                      # Try to find ring with its length of num_gpus/2
                      if not ring_found:
                          for cycle in nx.simple_cycles(G):
                              if len(cycle) == num_gpus/2:
                                  print(("One of the recommended ring having length of %d" % len(cycle) ))
                                  ring_found = True
                                  os.system("mkdir -p sofalog/sofa_hints/")
                                  xring_order = ','.join(map(str, cycle))
                                  with open("sofalog/sofa_hints/xring_order.txt", "w") as f:
                                      f.write('export CUDA_VISIBLE_DEVICES=' + xring_order)
                                  break
      # Construct Performance Features
      features = pd.DataFrame({'name':['elapsed_time'], 'value':[cfg.elapsed_time]}, columns=['name','value'])
  
      try:
          df_nvsmi = pd.read_csv(filein_nvsmi) 
          if not df_nvsmi.empty and cfg.spotlight_gpu:
              state = 0 
              sm_high = 0
              trigger = 10
              for i in range(len(df_nvsmi)):
                  if df_nvsmi.iloc[i].event == 0 and df_nvsmi.iloc[i].deviceId == 0 :
                      if df_nvsmi.iloc[i].duration >= 50:
                          sm_high = min(trigger, sm_high + 1)
                      if df_nvsmi.iloc[i].duration < 10:
                          sm_high = max(0, sm_high - 1)
                      if state == 0 and sm_high == trigger:
                          state = 1 
                          cfg.roi_begin = df_nvsmi.iloc[i].timestamp
                      elif state == 1 and sm_high == 0:
                          state = 0 
                          cfg.roi_end = df_nvsmi.iloc[i].timestamp
                      #print('sm_high=%d state=%d' % (sm_high, state))
              if cfg.roi_end - cfg.roi_begin < 0:
                  cfg.roi_end = 0
                  cfg.roi_begin = 0
      except IOError:
          print_warning(cfg, "nvsmi_trace.csv is not found")
  
      try:
          df_cpu = pd.read_csv(filein_cpu)
          if not df_cpu.empty: 
              if cfg.verbose:
                  cpu_profile(logdir, cfg, df_cpu)
              if cfg.enable_swarms and len(df_cpu) > cfg.num_swarms:
                  df_cpu, swarms = hsg_v2(cfg, df_cpu)
      except IOError as e:
          df_cpu = pd.DataFrame([], columns=cfg.columns)
          print_warning(cfg, "%s is not found" % filein_cpu)
  
      try:
          df_strace = pd.read_csv(filein_strace)
          if not df_strace.empty: 
              features = strace_profile(logdir, cfg, df_strace, features)
      except IOError as e:
          df_strace = pd.DataFrame([], columns=cfg.columns)
          print_warning(cfg, "%s is not found" % filein_strace)
  
      try:
          df_net = pd.read_csv(filein_net)
          if not df_net.empty: 
              features = net_profile(logdir, cfg, df_net, features)
      except IOError as e:
          df_net = pd.DataFrame([], columns=cfg.columns)
          print_warning(cfg, "%s is not found" % filein_net)
  
      try:
          df_bandwidth = pd.read_csv(filein_bandwidth)
          if not df_bandwidth.empty: 
              features = netbandwidth_profile(logdir, cfg, df_bandwidth, features)
      except IOError as e:
          df_bandwidth = pd.DataFrame([], columns=cfg.columns)
          print_warning(cfg, "%s is not found" % filein_bandwidth)
  
      try:
          df_blktrace = pd.read_csv(filein_blktrace)
          if not df_blktrace.empty: 
              features = blktrace_latency_profile(logdir, cfg, df_blktrace, features)
      except IOError as e:
          df_blktrace = pd.DataFrame([], columns=cfg.columns)
          print_warning(cfg, "%s is not found" % filein_blktrace)
  
      try:
          df_diskstat = pd.read_csv(filein_diskstat)
          if not df_diskstat.empty:
              features = diskstat_profile(logdir, cfg, df_diskstat, features)
      except IOError as e:
          df_diskstat = pd.DataFrame([], columns=cfg.columns)
          print_warning(cfg, "%s is not found" % filein_diskstat)
  
      try:
          df_vmstat = pd.read_csv(filein_vmstat)
          if not df_vmstat.empty: 
              features = vmstat_profile(logdir, cfg, df_vmstat, features)
      except IOError as e:
          df_vmstat = pd.DataFrame([], columns=cfg.columns)
          print_warning(cfg, "%s is not found" % filein_vmstat)
  
      try:
          df_mpstat = pd.read_csv(filein_mpstat)
          if not df_mpstat.empty: 
              features = mpstat_profile(logdir, cfg, df_mpstat, features)
      except IOError as e:
          df_mpstat = pd.DataFrame([], columns=cfg.columns)
          print_warning(cfg, "%s is not found" % filein_mpstat)
  
      try:
          df_nvsmi = pd.read_csv(filein_nvsmi) 
          features = nvsmi_profile(logdir, cfg, df_nvsmi, features)
      except IOError:
          print_warning(cfg, "nvsmi_trace.csv is not found")
  
      try:
          df_gpu = pd.read_csv(filein_gpu)
          if not df_gpu.empty: 
              features = gpu_profile(logdir, cfg, df_gpu, features)
      except IOError:
          df_gpu = pd.DataFrame([], columns=cfg.columns)
          print_warning(cfg, "%s is not found. If there is no need to profile GPU, just ignore it." % filein_gpu)
  
      try:
          if len(df_mpstat)>0:
              df_nvsmi.append(df_mpstat.iloc[0])
              features = concurrency_breakdown(logdir, cfg, df_mpstat, df_cpu, df_gpu, df_nvsmi, df_bandwidth, features)
      except IOError as e:
          print_warning(cfg, "Some files are not found, which are needed for concurrency_breakdown analysis")
  
  
  
  
  
      if cfg.enable_aisi:
          selected_pattern, iter_summary, features = sofa_aisi(logdir, cfg, df_cpu, df_gpu, df_strace, df_mpstat, features)           
  
      if 'IS_SOFA_ON_HAIHUB' not in os.environ or os.environ['IS_SOFA_ON_HAIHUB'] == 'no': 
              print_title('Final Performance Features')
              print('%s%s%s%s' % ('ID'.ljust(10),'Feature'.ljust(30),'Value'.ljust(20),'Unit'.ljust(20)) )
              for i in range(len(features)):
                  name = features.iloc[i]['name']
                  value = features.iloc[i]['value']
                  print('%s%s%s' % (str(i).ljust(10), name.ljust(30), ('%.3lf'%value).ljust(20)))
  
      if cfg.spotlight_gpu:
          try:
              print('Elapsed hotspot time: %.3lf' % features[features.name=='elapsed_hotspot_time'].value)
          except:
              print_warning(cfg, 'elpased_hostspot_time is not defined.')
   
      if cfg.potato_server:
          if cfg.potato_server.find(':') == -1:
              cfg.potato_server = cfg.potato_server + ':50051'
          hint, docker_image = get_hint(cfg.potato_server, features)
          df_report = pd.read_json(hint, orient='table')
          file_potato_report = cfg.logdir + 'potato_report.html'
          
          # Export report to HTML file.
          df_report.to_html(file_potato_report )
          with open(file_potato_report, 'a') as f:
              f.write('<head><link rel=stylesheet type="text/css" href="potato_report.css"></head>')
  
          print_title('POTATO Feedback')
          print('%s%s%s%s' % ('ID'.ljust(5), 'Metric'.ljust(20), 'Value'.ljust(10), 'Reference-Value'.ljust(30) ) )
          for i in range(len(df_report)):
              metric = df_report.iloc[i]['Metric']
              if metric != 'hybrid_suggestion':
                  value = df_report.iloc[i]['Value']
                  ref_value = df_report.iloc[i]['ReferenceValue']
                  print('%s%s%s%s' % (str(i).ljust(5), metric.ljust(20), ('%.3lf'%value).ljust(20), str(ref_value).ljust(30)))
   
          print('\n')
          print_hint('General Suggestions:')
          for i in range(len(df_report)):
              metric = df_report.iloc[i]['Metric']
              if metric != 'hybrid_suggestion':
                  suggestion = df_report.iloc[i]['Suggestion']
                  print('%d. %s' % (i, suggestion))
          
          print('\n')
          print_hint('Framework-specific Optimization Suggestions:')
          for i in range(len(df_report)):
              metric = df_report.iloc[i]['Metric']
              if metric == 'hybrid_suggestion':
                  suggestion = df_report.iloc[i]['Suggestion']
                  print('%d. %s' % (i, suggestion))
          
          #print(df_report[['Metric', 'Value', 'Reference Value']])
          #print(df_report[['Suggestion']])
          #print('Tag of optimal image recommended from POTATO: ' + highlight(docker_image))
          print('\n')
          print_hint('Please re-launch KubeFlow Jupyter-notebook to have suggested images or resources if necessary.')
      
      sofa_home = os.path.dirname(os.path.realpath(__file__))
      subprocess.Popen(
          ['bash', '-c', 'cp %s/../sofaboard/* %s;' % (sofa_home, cfg.logdir)])
      subprocess.Popen(['sleep', '2'])
      print('\n\n')
      print('Complete!!')
  
  def cluster_analyze(cfg):
      if cfg.verbose:
          print_title('Cluster Network Profiling :')
      cluster = cfg.cluster_ip.split(',')
      summary_net = pd.DataFrame([], columns=['Source', 'Destination', 'Amount', 'Percentage of a Node'])
      summary_compute = pd.DataFrame([], columns=['gpu_sm_util','gpu_mem_util','cpu_util'])
      summary_band = pd.DataFrame([], columns=['Q1', 'Q2', 'Q3', 'Avg'])  
      all = []
      for i, ip in enumerate(cluster):
          features = pd.DataFrame({'name':['elapsed_time'],
                                   'value':[cfg.elapsed_time]},
                                   columns=['name','value'])
          
          node = 'node ' + str(i)
          if cfg.verbose:
              print('node ' + str(i) + ' is ' + ip)
          logdir = tmp_dir[0:-1] + '-' + ip + '/'
          filein_net = logdir + "nettrace.csv"
          filein_mpstat = logdir + "mpstat.csv"
          filein_nvsmi = logdir + "nvsmi_trace.csv"
          filein_bandwidth = logdir + "netstat.csv"
          with open(logdir+'/misc.txt') as f:
              lines = f.readlines()
              elapsed_time = float(lines[0].split()[1])
              vcores = int(lines[2].split()[1])
              cfg.elapsed_time = float(lines[0].split()[1])
          try:
              df_net = pd.read_csv(filein_net)
              features = net_profile(logdir, cfg, df_net, features)
          except IOError as e:
              df_net = pd.DataFrame([], columns=cfg.columns)
              print_warning(cfg, "%s is not found" % filein_net)
          try:
              df_mpstat = pd.read_csv(filein_mpstat)
              features = mpstat_profile(logdir, cfg, df_mpstat, features)
          except IOError as e:
              df_mpstat = pd.DataFrame([], columns=cfg.columns)
              print_warning(cfg, "%s is not found" % filein_mpstat)
          try:
              df_nvsmi = pd.read_csv(filein_nvsmi) 
              features = nvsmi_profile(logdir, cfg, df_nvsmi, features)
          except IOError:
              print_warning(cfg, "nvsmi_trace.csv is not found")
          try:
              df_bandwidth = pd.read_csv(filein_bandwidth)
              features = netbandwidth_profile(logdir, cfg, df_bandwidth, features)
          except IOError as e:
              df_bandwidth = pd.DataFrame([], columns=cfg.columns)
              print_warning(cfg, "%s is not found" % filein_bandwidth)
  
          sm = int(features[features['name'] == 'gpu_sm_util']['value'])
          mem = int(features[features['name'] == 'gpu_mem_util']['value'])
          cpu = int(features[features['name'] == 'cpu_util']['value'])
          sm_mem_cpu = [sm, mem, cpu]
     
          compute_tmp = pd.DataFrame([sm_mem_cpu], columns = ['gpu_sm_util', 'gpu_mem_util', 'cpu_util'])
          summary_compute = pd.concat([summary_compute, pd.concat([compute_tmp], keys=[node])]) 
          net_tmp = pd.read_csv(logdir + "netrank.csv")
          summary_net = pd.concat([summary_net, pd.concat([net_tmp], keys=[node])])
         
          # for bandwidth report
          tx = df_bandwidth['event'] == float(0)
          rx = df_bandwidth['event'] == float(1)
          tx_tmp = [convertbytes(df_bandwidth[tx]['bandwidth'].quantile(0.25)),                         
                      convertbytes(df_bandwidth[tx]['bandwidth'].quantile(0.5)),
                      convertbytes(df_bandwidth[tx]['bandwidth'].quantile(0.75)),
                      convertbytes(df_bandwidth[tx]['bandwidth'].mean())]
          rx_tmp = [convertbytes(df_bandwidth[rx]['bandwidth'].quantile(0.25)),                         
                      convertbytes(df_bandwidth[rx]['bandwidth'].quantile(0.5)),
                      convertbytes(df_bandwidth[rx]['bandwidth'].quantile(0.75)),
                      convertbytes(df_bandwidth[rx]['bandwidth'].mean())]
          band_tmp = pd.DataFrame([tx_tmp], columns = ['Q1', 'Q2', 'Q3', 'Avg'], index = ['tx'])
          rx_pd = pd.DataFrame([rx_tmp], columns = ['Q1', 'Q2', 'Q3', 'Avg'], index = ['rx'])
          band_tmp = pd.concat([band_tmp, rx_pd]) 
          summary_band = pd.concat([summary_band, pd.concat([band_tmp], keys=[node])]) 
      if cfg.verbose:
          with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also
              print('Ranked Network Traffic : \n', summary_net, '\n')
              print('Cluster Bandwidth Quartile: \n', summary_band)
          print_title('Cluster Computation Profiling:')
          print(summary_compute)
  