text: import sys
  import os
  import signal
  import re
  import struct
  import socket
  import random
  import inspect
  #from fabric.state import connections as fab_connections
  import test_v1
  import re
  from common.connections import ContrailConnections
  from common.device_connection import NetconfConnection
  import traffic_tests
  from common.contrail_test_init import *
  from common import isolated_creds
  from vn_test import *
  from vm_test import *
  from floating_ip import *
  from svc_instance_fixture import SvcInstanceFixture
  from svc_template_fixture import SvcTemplateFixture
  from common.servicechain.config import ConfigSvcChain
  from common.servicechain.verify import VerifySvcChain
  from common.servicechain.mirror.verify import VerifySvcMirror
  from common.servicechain.mirror.config import ConfigSvcMirror
  from fabric.api import local
  #from serial_scripts.perf.base import PerfBase
  from tcutils.commands import *
  import time
  import copy
  import uuid
  
  class PerfBase(test_v1.BaseTestCase_v1,VerifySvcMirror):
  
      @classmethod
      def setUpClass(cls):
          super(PerfBase, cls).setUpClass()
          cls.nova_h = cls.connections.nova_h
          cls.orch = cls.connections.orch
          cls.vnc_lib_fixture = cls.connections.vnc_lib_fixture
          cls.results = 'logs/results_%s'%random.randint(0,100000)
          cls.results_file = open(cls.results,"aw")
          cls.encap_type = ["MPLSoUDP","MPLSoGRE"]
          # GET THESE VARIABLES from test configuration  TODO
          cls.spirent_linux_user = 'root'
          cls.spirent_linux_passwd = 'n1keenA'
          cls.ixia_linux_user = 'root'
          cls.ixia_linux_passwd = 'contrail123'
          cls.ixia_linux_host = '10.87.132.179'
          cls.ixia_host = '10.87.132.18'
          cls.spirent_linux_host = '10.87.132.185'
          #cls.mx1_ip = '10.87.74.51'
          #cls.mx2_ip = '10.87.140.181'
          cls.mx1_ip = '10.87.64.246'
          cls.mx2_ip = '10.87.140.181'
          cls.mx_user = 'root'
          cls.mx_password = 'Embe1mpls'
          cls.rt = {'ixia': ['2000','3000'],'spirent':['1','2'] }
          cls.family = ''
          #cls.set_cpu_cores = 4
          #cls.set_si = 1
          cls.dpdk_svc_scaling = False
          cls.nova_flavor= { '2':'perf_flavor_2cpu','3':'perf_flavor_3cpu','4':'perf_flavor_4cpu', '8':'perf_flavor_8cpu'}
          #cls.image_flavor= { '1':'perf-ubuntu-1404', '2':'ubuntu-perf-multique-2', '3':'dpdk_l2fwd_sleep3','4':'ubuntu-perf-multique-4','5':'dpdk-l2-no-delay-new','6':'tiny_in-net','7':'dpdk-l3fwd-mq-2','8':'perf-ubuntu-1404-v6-2','9':'perf-ubuntu-netronome','9':'DPDK-l2fwd-virtio-new','10':'DPDK-l2fwd-SRIOV-new-1','11':'dpdk_l2fwd_sleep3-new'}
          cls.vrouter_mask_list = ['0xff','0x3f','0xff','0xf000f']
          #cls.mx1_ip = '10.87.64.246'
          cls.mx1_handle = NetconfConnection(host = cls.mx1_ip,username=cls.mx_user,password=cls.mx_password)
          cls.mx1_handle.connect()
          cls.mx2_handle = NetconfConnection(host = cls.mx2_ip,username=cls.mx_user,password=cls.mx_password)
          cls.mx2_handle.connect()
          cls.host_cpu = {}
          cls.host = None
          cls.cpu_intr_mask = '80'
          #cls.vm_on_numa_node = '0'
          #cls.vm_on_numa_node = '1' 
          cls.perf_si_fixtures = []
          #cls.zone = 'nova'
          #cls.creds = '--os-username=%s --os-password=%s --os-tenant-name=%s --os-auth-url=%s'%(cls.inputs.admin_username,cls.inputs.admin_password,cls.inputs.admin_tenant,cls.inputs.auth_url)
          cls.update_hosts()
          cls.create_availability_zone()
          cls.nova_flavor_key_delete()
  
      @classmethod
      def tearDownClass(cls):
          # TODO to be enabled later
          cls.results_file.close()  
          super(PerfBase, cls).tearDownClass()
          cls.del_availability_zone()
          #TODO to be taken care later
          cls.nova_flavor_key_delete()
  
      @classmethod
      def print_results(self,result):
          self.results_file.write(result)
  
      @classmethod
      def update_hosts(self):
          host_list = self.connections.orch.get_hosts()
          self.kvm_hosts = copy.deepcopy(host_list)
          self.dpdk_hosts = [self.inputs.host_data[host]['name'] for host in self.inputs.dpdk_ips] 
          self.netronome_hosts = []
          for host in host_list:
              if host in self.dpdk_hosts:
                  self.kvm_hosts.remove(host)
          return
  
      @classmethod
      def create_availability_zone(self):
          zones = self.connections.orch.get_zones()
          self.agg_id = {} 
          if 'kvm' not in zones: 
              self.agg_id['kvm_id'] = self.connections.orch.create_agg('kvm','kvm')
              self.connections.orch.add_host_to_agg(self.agg_id['kvm_id'],self.kvm_hosts)
          if 'dpdk' not in zones: 
              self.agg_id['dpdk_id']= self.connections.orch.create_agg('dpdk','dpdk')
              self.connections.orch.add_host_to_agg(self.agg_id['dpdk_id'],self.dpdk_hosts)
          return 
  
      @classmethod
      def del_availability_zone(self):
          if 'kvm_id' in self.agg_id:
              self.connections.orch.del_host_from_agg(self.agg_id['kvm_id'],self.kvm_hosts)
              self.connections.orch.delete_agg(self.agg_id['kvm_id'])
          if 'dpdk_id' in self.agg_id:
              self.connections.orch.del_host_from_agg(self.agg_id['dpdk_id'],self.dpdk_hosts)
              self.connections.orch.delete_agg(self.agg_id['dpdk_id'])
          return 
  
      def set_test_config(self,**kwargs):
          self.perf_conf = {} 
          self.perf_conf['profile_name'] = kwargs.get('profile_name') 
          self.perf_conf['proto']  = kwargs.get('proto') 
          self.perf_conf['cores']  = kwargs.get('cores') 
          self.perf_conf['family'] = kwargs.get('family') 
          self.perf_conf['si']     = kwargs.get('si') 
          self.perf_conf['image']  = kwargs.get('image') 
          self.perf_conf['encap']  = kwargs.get('encap','MPLSoGRE')
          self.perf_conf['zone'] = kwargs.get('zone','kvm')  
          self.perf_conf['multiq'] = kwargs.get('multiq',False) 
          self.perf_conf['traffic'] = kwargs.get('traffic','ixia') 
          self.perf_conf['flow'] = kwargs.get('flow',None) 
          if self.perf_conf['traffic'] is 'ixia':
              self.perf_conf['left-rt'] = self.rt['ixia'][0]
              self.perf_conf['right-rt'] = self.rt['ixia'][1]
          if self.perf_conf['traffic'] is 'spirent':
              self.perf_conf['left-rt'] = self.rt['spirent'][0]
              self.perf_conf['right-rt'] = self.rt['spirent'][1]
  
          self.logger.info("profile_name : %s"%self.perf_conf['profile_name'])
          self.logger.info("proto      : %s"%self.perf_conf['proto'])
          self.logger.info("cores      : %s"%self.perf_conf['cores'])
          self.logger.info("family     : %s"%self.perf_conf['family'])
          self.logger.info("number of svms : %s"%self.perf_conf['si'])
          self.logger.info("image      : %s"%self.perf_conf['image'])
          self.logger.info("encap      : %s"%self.perf_conf['encap'])
          self.logger.info("zone       : %s"%self.perf_conf['zone'])
          self.logger.info("multiq     : %s"%self.perf_conf['multiq'])
          self.logger.info("traffic    : %s"%self.perf_conf['traffic'])
          return  
  
      @classmethod
      def nova_flavor_key_delete(self):
          flavor_list = self.orch.get_flavor_list()
          for flavor in flavor_list:
              if 'perf_flavor' in flavor.name:
                  self.orch.delete_flavor(flavor)
          return True
  
      def nova_flavor_update(self,extra_specs):
          flavor_list = self.orch.get_flavor_list()
          for flavor in flavor_list:
              if 'perf_flavor' in flavor.name:
                  flavor.set_keys(extra_specs)
          return True
  
      def nova_image_update(self,properties):
          image = self.orch.find_image(self.perf_conf['image'])
          if image:
              image.update(properties=properties)
          return True
   
      def update_image_flavor_options(self):
          # setting numa nodes 
          extra_specs = { 'hw:numa_nodes':'1' }  
          self.nova_flavor_update(extra_specs)
          # if multiq is enabled 
          if self.perf_conf['multiq']:
              properties = {'hw_vif_multiqueue_enabled': True}
              self.nova_image_update(properties=properties)
          # if dpdk or netronome is enabled
  
          if 'dpdk' in self.perf_conf['zone']:
              extra_specs = {  'hw:mem_page_size': 'large' }  
              self.nova_flavor_update(extra_specs)
  
          # if hw acceleration is enabled
          if 'netronome' in self.perf_conf['zone']:
              properties = {'agilio.hw_acceleration_features':'SR-IOV'}
              self.nova_image_update(properties=properties)
          return True
  
  
      def launch_svc_vms(self,host, left_rt='2000',right_rt='3000'):
          # Create two virtual networks
          self.vn1_name='vn100-left'
          self.vn1_subnets=['12.0.0.0/8','2005::/64']
          self.vn1_fq_name = "default-domain:" + self.inputs.project_name + ":" + self.vn1_name
          self.vn2_name = 'vn100-right'
          self.vn2_subnets = ['13.0.0.0/8','2006::/64']
          self.vn2_fq_name = "default-domain:" + self.inputs.project_name + ":" + self.vn2_name
          self.vmlist = []
          self.vm_fixture = []
          self.version = '2'
          self.userdata = '/tmp/metadata_script.txt'
  
          self.inputs.set_af(self.perf_conf['family'])
          # TODO , later
          #self.create_startup_script_queues('2')
          if self.perf_conf['family'] == 'v6':
              self.vn1_fixture= self.useFixture(VNFixture(project_name= self.inputs.project_name, connections= self.connections,vn_name=self.vn1_name, inputs= self.inputs,router_asn=self.inputs.router_asn, rt_number=self.inputs.mx_rt))
          else:
              self.vn1_fixture= self.useFixture(VNFixture(project_name= self.inputs.project_name, connections= self.connections,vn_name=self.vn1_name, inputs= self.inputs, subnets= self.vn1_subnets,router_asn=self.inputs.router_asn, rt_number=left_rt))
              self.vn2_fixture= self.useFixture(VNFixture(project_name= self.inputs.project_name, connections= self.connections,vn_name=self.vn2_name, inputs= self.inputs, subnets= self.vn2_subnets,router_asn=self.inputs.router_asn, rt_number=right_rt))
  
          # Create service instance 
          self.svc_tmp_name = 'perf_svc_template'
          self.svc_int_prefix = 'perf_vm_'
          self.svc_policy_name = 'perf_svc_policy'
          if self.perf_conf['si']> 1:
              self.dpdk_svc_scaling = True
  
          if self.version == '2':
              # create service template
              if_details =  { 'left': {}, 'right': {} }
              self.perf_st_fixture = self.useFixture(SvcTemplateFixture(
                                 connections=self.connections,
                                 st_name=self.svc_tmp_name, svc_img_name=self.perf_conf['image'], service_type='firewall',
                                 if_details=if_details, service_mode='in-network', svc_scaling=self.dpdk_svc_scaling,
                                 flavor=self.nova_flavor[str(self.perf_conf['cores'])],
                                 version=2,availability_zone_enable=True))
              if_details = { 'left' : {'vn_name' : self.vn1_fq_name},
                             'right': {'vn_name' : self.vn2_fq_name}
                           }
              for i in range(0, 1):
                  si_name = self.svc_int_prefix + str(i + 1)
                  self.perf_si_fixture = self.useFixture(SvcInstanceFixture(
                      connections=self.connections,
                      si_name=si_name,
                      svc_template=self.perf_st_fixture.st_obj, if_details=if_details,
                       max_inst=self.perf_conf['si'], availability_zone=self.perf_conf['zone']))
                  self.logger.debug('Launching SVM')
                  for i in range(self.perf_conf['si']):
                      svm_name = get_random_name("pt_svm" + str(i))
                      pt_name = get_random_name("port_tuple" + str(i))
                      svm_fixture = self.config_and_verify_vm(
                                   svm_name, image_name=self.perf_conf['image'], vns=[self.vn1_fixture, self.vn2_fixture], 
                                   count=1, flavor=self.nova_flavor[str(self.perf_conf['cores'])],
                                   zone=self.perf_conf['zone'],node_name=host)
                      port_tuples_props = []
                      svm_pt_props = {}
                      svm_pt_props['left'] = svm_fixture.vmi_ids[self.vn1_fq_name]
                      svm_pt_props['right'] = svm_fixture.vmi_ids[self.vn2_fq_name]
                      svm_pt_props['name'] = get_random_name("port_tuple")
                      port_tuples_props.append(svm_pt_props)
                      self.perf_si_fixture.add_port_tuple(svm_pt_props)
                  #self.perf_si_fixture.verify_on_setup()
                  self.perf_si_fixtures.append(self.perf_si_fixture)
  
     #     action_list = self.chain_si(self.vm_num,self.svc_int_prefix,self.inputs.project_name)
          action_list = []
  
          for i in range(1):
              si_name = self.svc_int_prefix + str(i + 1)
              # chain services by appending to action list
              si_fq_name = 'default-domain' + ':' + self.inputs.project_name + ':' + si_name
              action_list.append(si_fq_name)
  
          self.project.set_sec_group_for_allow_all()
          # Create a policy 
          self.rules = [{'direction': '<>',
                   'protocol': 'any',
                   'source_network': self.vn1_name,
                   'src_ports': [0, -1],
                   'dest_network': self.vn2_name,
                   'dst_ports': [0, -1],
                   'simple_action': None,
                   'action_list': {'apply_service': action_list}
                   }]
  
          self.svc_policy_fixture = self.config_policy(self.svc_policy_name, self.rules)
          self.vn1_policy_fix = self.attach_policy_to_vn( self.svc_policy_fixture, self.vn1_fixture)
          self.vn2_policy_fix = self.attach_policy_to_vn( self.svc_policy_fixture, self.vn2_fixture)
  
          time.sleep(180)
  
          return True
  
      def update_pre_perf_tunings(self):
          cmd='for f in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor ; do echo performance > $f; cat $f; done'
          out = self.inputs.run_cmd_on_server(self.host, cmd)
          cmd='service  irqbalance stop'
          out = self.inputs.run_cmd_on_server(self.host, cmd) 
          nics = ','.join(self.host_numa[self.host]['nics'])
          cmd='for intf in {%s}; do for i in $(grep $intf /proc/interrupts | awk {\'print $1\'} | sed s/://g);  do echo "0,%s" > /proc/irq/"$i"/smp_affinity ; done ; done'%(nics,self.cpu_intr_mask)
          out = self.inputs.run_cmd_on_server(self.host, cmd) 
          return True
  
      def update_post_perf_tunings(self):
          # CPU pinning of VMs based on numa node
          if not self.set_process_cpu(self.host,'qemu'):
                  self.logger.error("Not able to pin cpu :%s "%'qemu')
  
          if 'kvm' in self.perf_conf['zone']:
              if not self.set_process_cpu(self.host,'vhost'):
                  self.logger.error("Not able to pin cpu :%s "%'vhost')
             # TODO later 
              if not self.update_txquelen(self.host):
                  self.logger.error("Not able to update tap")
          return True
  
      def update_txquelen(self,host):
          cmd = 'for i in $(ifconfig | grep tap | awk {\'print $1\'}) ; do ifconfig $i txqueuelen 1000000 ; done'
          res = self.inputs.run_cmd_on_server(host, cmd)
          return True
  
      def set_process_cpu(self,host,process):
          cmd = 'ps ax | pgrep %s'%process
          pids = self.inputs.run_cmd_on_server(host, cmd)
          pids = pids.replace("\r","").split("\n")
          cpus_to_pin = [] 
          numa_id = 'cpu_numa_%s'%self.host_numa[host]['numa'][0]
          self.logger.info("Numa Id : %s"%numa_id)
          for pid in pids:
              cpus_to_pin = [] 
              cores = 1 if process == 'vhost' else int(self.perf_conf['cores'])
              for core in range(cores):
                  core_id = self.get_cpu_core(self.host_cpu[host][numa_id])
                  if core_id is not None:
                      cpus_to_pin.append(str(core_id))
                  else: 
                      self.logger.error("Not able to get CPU core to pin %s"%process)
              cpus_to_pin_str  = ','.join(cpus_to_pin)
              self.logger.info("cpus to pin the process %s :pid : %s:  %s"%(process,pid,cpus_to_pin_str))
              cmd = 'taskset -a -pc %s %s'%(cpus_to_pin_str,pid)
              self.inputs.run_cmd_on_server(host, cmd)
          return True
  
      def get_cpu_core(self,cpu_id):
          if len(cpu_id):
              return cpu_id.pop(0) 
          else:
              return None 
  
      def get_hosts(self):
          host = []
          if 'kvm' in self.perf_conf['zone']:
              host = self.kvm_hosts[0]
          if 'dpdk' in self.perf_conf['zone']:
              host = self.dpdk_hosts[0]
          if 'netronome' in self.perf_conf['zone']:
              host = self.netronome_hosts[0]
          return host
  
      def get_numa_nodes_cpu_cores(self):
          host_numa = {}
          host_cpu = {}
          host_numa[self.host] = {}
          host_numa[self.host]['nics'],host_numa[self.host]['numa'] = self.get_nics_numa_node(self.host)
          #host_numa[self.host]['numa'] = self.get_nics_numa_node(self.host)[1]
          host_cpu = self.get_cpu_cores(self.host)
          self.logger.info("host numa : %s"%host_numa)
          self.logger.info("cpu cores : %s"%host_cpu)
          return host_numa , host_cpu
  
      def restart_control(self):
          for node in self.inputs.bgp_control_ips:
                  self.inputs.restart_service('contrail-control', [node],
                                              container='control')
          return 
  
      def get_nics_numa_node(self,host_name):
          numa_nodes = [] 
          nics = []
          os_name = self.inputs.get_os_version(host_name) 
          if 'kvm' in self.perf_conf['zone']:
              if 'ubuntu' in os_name: 
                  cmd = ''
                  cmd = "mac=$(ifconfig | grep vhost0 | awk {\'print $5\'}) ; ifconfig | grep $mac | awk {\'print $1\'}"
                  nics = self.inputs.run_cmd_on_server(host_name,cmd)
                  nics = nics.replace("\r",'').split("\n")
                  nics = [x for x in nics if x not in ['vhost0']]
                  nics = [x for x in nics if x not in ['bond0']]
              elif 'redhat' or 'centos' in os_name:
                  cmd = 'mac=$(ifconfig | grep vhost0 -A3 | grep ether | awk {\'print $2\'}) ; ifconfig | grep $mac  -B3 | grep UP | awk {\'print $1\'}'
                  nics = self.inputs.run_cmd_on_server(host_name,cmd)
                  nics = nics.replace("\r",'').replace(":",'').split("\n")
                  nics = [x for x in nics if x not in ['vhost0']]
                  nics = [x for x in nics if x not in ['bond0']]
  
          if 'dpdk' in self.perf_conf['zone']:
              cmd = 'cat /var/run/vrouter/bond0_bond'
              nics = self.inputs.run_cmd_on_server(host_name,cmd)
              nics = nics.replace("\r",'').split(" ")[2].split(',')
  
          if 'netronome' in self.perf_conf['zone']:
              #TODO
              self.logger.info("Netronome mapping of nics ")
         
          if not nics:
              self.logger.error("NICs not present in two NUMA NODES")
              return nics,numa_nodes
  
          for intf in nics:
              cmd = 'cat /sys/class/net/%s/device/numa_node'%intf
              numa_nodes.append(self.inputs.run_cmd_on_server(host_name,cmd))
  
          if 'dpdk' in self.perf_conf['zone']:
              #TODO , need to get the nodes properly
              numa_nodes =  ['0']
  
          numa_nodes = list(set(numa_nodes)) 
  
          if (len(numa_nodes) > 1 ):
              self.logger.info("NICs present in two NUMA NODES")
  
          self.logger.info("NICs present NUMA NODES %s"%numa_nodes)
  
          return nics,numa_nodes
  
      def get_cpu_cores(self,host):
          cpus_resvd = []
          cpus_intr_resvd = []
          host_cpu = {}
          host_cpu[host] = {}
          if 'dpdk' in self.perf_conf['zone']:
              cpus_resvd = self.hextobit(self.vrouter_mask_list[0])
          cpus_intr_resvd = self.hextobit(self.cpu_intr_mask)
          host_cpu[host]['cpu_numa_0'] = [ x for x in  self.get_cpus_from_host(0,host) if x not in cpus_resvd]
          host_cpu[host]['cpu_numa_1'] = [ x for x in  self.get_cpus_from_host(1,host) if x not in cpus_resvd]
          host_cpu[host]['cpu_numa_0'] = [ x for x in  host_cpu[host]['cpu_numa_0'] if x not in cpus_intr_resvd]
          host_cpu[host]['cpu_numa_1'] = [ x for x in  host_cpu[host]['cpu_numa_1'] if x not in cpus_intr_resvd]
          return host_cpu
  
      def hextobit(self,mask):
          cpus = []
          count = 0
          val = int(mask,16)
          while (val):
              if (val & 0x1):
                  cpus.append(count)
              val = val >> 1
              count += 1
          return cpus
  
      def get_cpus_from_host(self,numa,host):
          cpus = []
          cmd = "lscpu | grep NUMA | grep node%s |  grep CPU | awk {'print $4'}"%numa
          res = self.inputs.run_cmd_on_server(host,cmd)
          cpu_list = res.replace('\n',',').split(',')
          for cpu in cpu_list:
              for i in range(int(cpu.split('-')[0]),int(cpu.split('-')[1])+1):
                  cpus.append(i)
          return cpus
  
      def configure_mx(self,encap,handle):
          cmd = []
          if encap == 'MPLSoUDP':
              if handle.host  == self.mx1_ip: 
                  cmd.append('set groups ixia_flow routing-options dynamic-tunnels contrail udp')
                  cmd.append('set groups ixia_flow protocols bgp group contrail export test1-export')
              elif handle.host == self.mx2_ip: 
                  cmd.append('set groups __contrail__ routing-options dynamic-tunnels __contrail__ udp')
                  cmd.append('set groups __contrail__ protocols bgp group __contrail__ export test1-export') 
  
          elif encap == 'MPLSoGRE':
              if handle.host  == self.mx1_ip: 
                  cmd.append('set groups ixia_flow routing-options dynamic-tunnels contrail gre')
                  cmd.append('delete groups ixia_flow protocols bgp group contrail export test1-export')
              elif handle.host == self.mx2_ip: 
                  cmd.append('set groups __contrail__ routing-options dynamic-tunnels __contrail__ gre')
                  cmd.append('delete groups __contrail__ protocols bgp group __contrail__ export test1-export') 
  
          self.logger.info("MX configuration cmd executed %s"%cmd)
  
          cli_output = handle.config(stmts = cmd,ignore_errors=True,timeout = 120) 
          
          self.restart_control()
  
          return True
  
      def update_encap_mode(self,host,mode):
          cmd = '/usr/bin/python /opt/contrail/utils/provision_encap.py --api_server_ip 127.0.0.1 --api_server_port 8082 --encap_priority %s --vxlan_vn_id_mode "automatic" --oper add --admin_user "admin" --admin_password "contrail123"'%(mode)
  
          self.logger.info("MX configuration cmd executed %s"%cmd)
  
          host = self.inputs.bgp_control_ips[0]
   
          out = self.inputs.run_cmd_on_server(host, cmd , container='control')
  
          if not re.findall("Updated",out):
              self.logger.error("Encapsulation not updated :%s "%out)
              return False
          res = ('Encapsulation Type :%s\n'%mode)
          self.logger.info("Encapsulation Type :%s "%res)
          return True
  
  
      def run_perf_tests(self,**kwargs):
          self.logger.info("Executing Performance Tests")
  
          self.set_test_config(**kwargs)
  
          self.host = self.get_hosts()
  
          self.host_numa,self.host_cpu = self.get_numa_nodes_cpu_cores()
  
          self.update_pre_perf_tunings()
  
          #TODO , enable mx config later 
          self.configure_mx(self.perf_conf['encap'],self.mx1_handle)
          self.configure_mx(self.perf_conf['encap'],self.mx2_handle)
  
          self.update_image_flavor_options()
  
          self.test_name = inspect.stack()[1][3]
  
          # TODO , fix up later 
          
          if not self.host :
              self.logger.error("Not able to find host to launch VMs")
              return False
  
          if not self.launch_svc_vms(self.host, self.perf_conf['left-rt'],self.perf_conf['right-rt']):
               self.logger.error("Failed to launch VMs")
               return False
  
          self.update_post_perf_tunings()
  
          if self.perf_conf['traffic'] is 'ixia':
              if not self.run_ixia_rfc_tests(self.perf_conf['profile_name']):
                  self.logger.error("Failed to run ixia tests successfully")
                  return False
  
          if self.perf_conf['traffic'] is 'spirent':
              if not self.start_spirent_test(self.perf_conf['profile_name']):
                  self.logger.error("Failed to run ixia tests successfully")
                  return False
  
          #import pdb;pdb.set_trace()
  
          return True
  
      def run_ixia_rfc_tests(self,script_name):
          # TDOD enable these lines later to start ixia test
          cmd='(nohup python /root/scripts/qt.py  /root/scripts/profiles/%s  > /root/results 2>&1 &); sleep 5' %(script_name)
          result = self.inputs.run_cmd_on_server(self.ixia_linux_host, cmd,username=self.ixia_linux_user ,password=self.ixia_linux_passwd)
          self.logger.info("RFC Test Running: %s "%result)
          self.logger.info("RFC Test Running: ... ")
          cnt = 0
          while True:
              if self.verify_ixia_test(script_name):
                  self.logger.info("RFC Test FINISHED")
                  break
              time.sleep(60)
              cnt = cnt + 1
              self.logger.info("RFC Test Running")
              #if cnt  > 10 :
              ##    fab_connections.clear()
              #    cnt = 0
          cmd='cat /root/results | grep ResultPath'
          time.sleep(60)
          result = self.inputs.run_cmd_on_server(self.ixia_linux_host, cmd,username=self.ixia_linux_user,password=self.ixia_linux_passwd)
          self.logger.info("Result path %s"%result)
          self.update_results(result)
          return True
  
      def verify_ixia_test(self,script_name):
          try:
              cmd='cat /root/results | grep FINISHED'
              result = self.inputs.run_cmd_on_server(self.ixia_linux_host, cmd,username=self.ixia_linux_user,password=self.ixia_linux_passwd)
              if not re.findall('FINISHED',result):
                  return False
              cmd='cat /root/results'
              result = self.inputs.run_cmd_on_server(self.ixia_linux_host, cmd,username=self.ixia_linux_user,password=self.ixia_linux_passwd)
              self.logger.info("RFC Test FINISHED%s"%result)
              if re.findall('ERROR',result):
                  self.logger.error("RFC Test ERROR%s"%result)
          except Exception as e:
              self.logger.exception("Exception in calling ixia host %s"%(e))
          return True
  
      def update_results(self,result):
          result = result.replace("\r","")
          result = result.split("\n")
          res = str(result).strip('[]').strip('\'')
          #res=res.replace('\\','/').replace('ResultPath:  /root/C://Users//Administrator//Desktop','%s'%self.ixia_host)+'/AggregateResults.csv'
          res=res.replace('\\','/').replace('ResultPath:  C://Users//Administrator//Desktop','%s'%self.ixia_host)+'/AggregateResults.csv'
          self.logger.info("Result : %s "%res)
          out = '\ntest: %s \nencap : %s   \ncores : %s  \nfamily: %s  \ninstances : %s \n'%(self.test_name,
                                    self.perf_conf['encap'],self.perf_conf['cores'],self.perf_conf['family'],self.perf_conf['si'])
          self.print_results("test_profile: %s"%self.perf_conf['profile_name']) 
          self.print_results(out) 
          self.print_results(res) 
          self.print_results('\n') 
          cmd = 'wget -O /tmp/AggregateResults.csv %s'%res
          res = self.inputs.run_cmd_on_server(self.host,cmd)
          cmd = 'cat /tmp/AggregateResults.csv ; sleep 5 ; rm /tmp/AggregateResults.csv'
          res = self.inputs.run_cmd_on_server(self.host,cmd)
          self.logger.info(res)
          self.logger.info(res)
          self.print_results(res) 
          self.print_results("\n==================================================================\n")
          return True
  
      def start_spirent_test(self,script_name):
          self.restart_control()
          cmd = "rm -rf /root/spirent/tests/%s/results; source /root/.bash_profile ;(nohup tclsh /root/spirent/tests/%s/test.tcl > results 2>&1 &); sleep 5" %(script_name, script_name)
          result = self.inputs.run_cmd_on_server(self.spirent_linux_host, cmd, self.spirent_linux_user , self.spirent_linux_passwd )
          self.logger.info("Spirent Test Running: %s "%result)
          while True:
              if self.verify_spirent_test():
                 self.logger.info("Finished running")
                 break
              time.sleep(60)
              self.logger.info("Spirent Test Running")
  
          if not self.get_spirent_result(script_name):
              self.logger.error("Failed to collect test results")
              return False
          return True
  
      def verify_spirent_test(self):
          cmd='cat results'
          result = self.inputs.run_cmd_on_server(self.spirent_linux_host, cmd,self.spirent_linux_user,self.spirent_linux_passwd)
          if not re.findall('Finished running',result):
             return False
          elif re.findall('ERROR',result):
              self.logger.info("Spirent Test ERROR%s"%result)
              return True
          else:
              self.logger.info("Spirent Test FINISHED%s"%result)
              cmd='cat results | grep ResultPath'
              result = self.inputs.run_cmd_on_server(self.spirent_linux_host, cmd,self.spirent_linux_user,self.spirent_linux_passwd)
              return True
  
      def get_spirent_result (self, script_name):
          uuid_curr = str(uuid.uuid1())
          remote_dir = '/var/www/html/tests/results/%s/%s/'%(script_name,uuid_curr)
          cmd = "mkdir /var/www/html/tests/results/%s/;mkdir %s ;cp -r /root/spirent/tests/%s/results/ %s ;sleep 1" %(script_name,remote_dir,script_name,remote_dir)
          result = self.inputs.run_cmd_on_server(self.spirent_linux_host, cmd, self.spirent_linux_user, self.spirent_linux_passwd)
          logs = "\nLog Path (%s): %s" %(script_name, remote_dir )
          http_logs = "\nLog Path (%s): http://%s//tests/results/%s/%s/" %(script_name,self.spirent_linux_host,script_name, uuid_curr)
          self.logger.info("%s"%logs)
          self.logger.info("%s"%http_logs)
          out = '\ntest: %s \nencap : %s   \ncores : %s  \nfamily: %s  \ninstances : %s \ntest_profile: %s \n'%(self.test_name,
                                    self.perf_conf['encap'],self.perf_conf['cores'],self.perf_conf['family'],self.perf_conf['si'],self.perf_conf['profile_name'])
          self.print_results(out) 
          self.print_results(logs) 
          self.print_results(http_logs) 
          self.print_results("\n") 
          if self.perf_conf['flow'] is 'flow':
              cmd = ' python /root/performance_results/performance_analysis_tool.py -f  %s/results/merged/client/realtime.csv  -o test.html -j time_vs_conn_rate_flow_tests ; sleep 5'%remote_dir
  
          if self.perf_conf['flow'] is 'flowscale':
              cmd = ' python /root/performance_results/performance_analysis_tool.py -f  %s/results/merged/client/realtime.csv  -o test.html -j time_vs_conn_rate_flow_scale_tests ; sleep 5'%remote_dir
  
          if self.perf_conf['flow'] is 'throughput':
              cmd = ' python /root/performance_results/performance_analysis_tool.py -f  %s/results/merged/client/realtime.csv  -o test.html -j time_throughput ; sleep 5'%remote_dir
          result = self.inputs.run_cmd_on_server(self.spirent_linux_host, cmd, self.spirent_linux_user, self.spirent_linux_passwd)
          result = result.replace("\r","")
          self.print_results(result)
          self.print_results("\n==================================================================\n")
          return True
  
  