files: # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
  
  import argparse
  import base64
  import time
  import datetime
  import multiprocessing
  import tarfile
  import traceback
  import asyncio
  from subprocess import run, PIPE
  import os
  import glob
  import pathlib
  import sys
  import gzip
  import re
  import json
  import logging
  import hashlib
  import shutil
  import copy
  import importlib
  import textwrap
  import math
  import pandas
  import yaml
  import graphviz
  import time
  import uuid
  import shlex
  import platform
  import getpass
  import distro
  import netifaces
  import webbrowser
  import packaging.version
  import packaging.specifiers
  from jinja2 import Environment, FileSystemLoader
  from pathlib import Path
  from timeit import default_timer as timer
  from siliconcompiler.client import *
  from siliconcompiler.schema import *
  from siliconcompiler.scheduler import _deferstep
  from siliconcompiler import leflib
  from siliconcompiler import utils
  from siliconcompiler import _metadata
  import psutil
  
  class TaskStatus():
      # Could use Python 'enum' class here, but that doesn't work nicely with
      # schema.
      PENDING = 'pending'
      SUCCESS = 'success'
      ERROR = 'error'
  
  class Chip:
      """Object for configuring and executing hardware design flows.
  
      This is the main object used for configuration, data, and
      execution within the SiliconCompiler platform.
  
      Args:
          design (string): Name of the top level chip design module.
  
      Examples:
          >>> siliconcompiler.Chip(design="top")
          Creates a chip object with name "top".
      """
  
      ###########################################################################
      def __init__(self, design, loglevel=None):
          # version numbers
          self.scversion = _metadata.version
          self.schemaversion = SCHEMA_VERSION
  
          # Local variables
          self.scroot = os.path.dirname(os.path.abspath(__file__))
          self.cwd = os.getcwd()
          self.error = 0
          self.cfg = schema_cfg()
          # The 'status' dictionary can be used to store ephemeral config values.
          # Its contents will not be saved, and can be set by parent scripts
          # such as a web server or supervisor process. Currently supported keys:
          # * 'jobhash': A hash or UUID which can identify jobs in a larger system.
          # * 'remote_cfg': Dictionary containing remote server configurations
          #                 (address, credentials, etc.)
          # * 'slurm_account': User account ID in a connected slurm HPC cluster.
          # * 'slurm_partition': Name of the partition in which a task should run
          #                      on a connected slurm HPC cluster.
          # * 'watchdog': Activity-monitoring semaphore for jobs scheduled on an
          #               HPC cluster; expects a 'threading.Event'-like object.
          # * 'max_fs_bytes': A limit on how much disk space a job is allowed
          #                   to consume in a connected HPC cluster's storage.
          self.status = {}
  
          self.builtin = ['minimum','maximum',
                          'nop', 'mux', 'join', 'verify']
  
          # We set 'design' and 'loglevel' directly in the config dictionary
          # because of a chicken-and-egg problem: self.set() relies on the logger,
          # but the logger relies on these values.
          self.cfg['design']['value'] = design
          if loglevel:
              self.cfg['option']['loglevel']['value'] = loglevel
  
          self._init_logger()
  
          self._loaded_modules = {
              'flows': [],
              'pdks': [],
              'libs': [],
              'checklists': []
          }
  
      ###########################################################################
      @property
      def design(self):
          '''Design name of chip object.
  
          This is an immutable property.'''
          return self.get('design')
  
      ###########################################################################
      def _init_logger(self, step=None, index=None, in_run=False):
  
          self.logger = logging.getLogger(uuid.uuid4().hex)
  
          # Don't propagate log messages to "root" handler (we get duplicate
          # messages without this)
          # TODO: this prevents us from being able to capture logs with pytest:
          # we should revisit it
          self.logger.propagate = False
  
          loglevel = self.get('option', 'loglevel')
  
          if loglevel=='DEBUG':
              prefix = '| %(levelname)-7s | %(funcName)-10s | %(lineno)-4s'
          else:
              prefix = '| %(levelname)-7s'
  
          if in_run:
              flow = self.get('option', 'flow')
  
              # Figure out how wide to make step and index fields
              max_step_len = 2
              max_index_len = 2
              for future_step in self.getkeys('flowgraph', flow):
                  max_step_len = max(len(future_step) + 1, max_step_len)
                  for future_index in self.getkeys('flowgraph', flow, future_step):
                      max_index_len = max(len(future_index) + 1, max_index_len)
  
              jobname = self.get('option', 'jobname')
  
              if step is None:
                  step = '-' * max(max_step_len // 4, 1)
              if index is None:
                  index = '-' * max(max_index_len // 4, 1)
  
              run_info = f'%s  | %-{max_step_len}s | %-{max_index_len}s' % (jobname, step, index)
              logformat = ' | '.join([prefix, run_info, '%(message)s'])
          else:
              logformat = ' | '.join([prefix, '%(message)s'])
  
          handler = logging.StreamHandler()
          formatter = logging.Formatter(logformat)
  
          handler.setFormatter(formatter)
  
          # Clear any existing handlers so we don't end up with duplicate messages
          # if repeat calls to _init_logger are made
          if len(self.logger.handlers) > 0:
              self.logger.handlers.clear()
  
          self.logger.addHandler(handler)
          self.logger.setLevel(loglevel)
  
      ###########################################################################
      def _deinit_logger(self):
          self.logger = None
  
      ###########################################################################
      def _get_switches(self, *keypath):
          '''Helper function for parsing switches and metavars for a keypath.'''
          #Switch field fully describes switch format
          switch = self.get(*keypath, field='switch')
  
          if switch is None:
              switches = []
          elif isinstance(switch, list):
              switches = switch
          else:
              switches = [switch]
          switchstrs = []
  
          # parse out switch from metavar
          # TODO: should we validate that metavar matches for each switch?
          for switch in switches:
              switchmatch = re.match(r'(-[\w_]+)\s+(.*)', switch)
              gccmatch = re.match(r'(-[\w_]+)(.*)', switch)
              plusmatch = re.match(r'(\+[\w_\+]+)(.*)', switch)
  
              if switchmatch:
                  switchstr = switchmatch.group(1)
                  metavar = switchmatch.group(2)
              elif gccmatch:
                  switchstr = gccmatch.group(1)
                  metavar = gccmatch.group(2)
              elif plusmatch:
                  switchstr = plusmatch.group(1)
                  metavar = plusmatch.group(2)
              switchstrs.append(switchstr)
  
          return switchstrs, metavar
  
      ###########################################################################
      def create_cmdline(self, progname, description=None, switchlist=None, input_map=None):
          """Creates an SC command line interface.
  
          Exposes parameters in the SC schema as command line switches,
          simplifying creation of SC apps with a restricted set of schema
          parameters exposed at the command line. The order of command
          line switch settings parsed from the command line is as follows:
  
           1. loglevel
           2. fpga_partname
           3. load_target('target')
           4. read_manifest([cfg])
           5. all other switches
  
          The cmdline interface is implemented using the Python argparse package
          and the following use restrictions apply.
  
          * Help is accessed with the '-h' switch.
          * Arguments that include spaces must be enclosed with double quotes.
          * List parameters are entered individually. (ie. -y libdir1 -y libdir2)
          * For parameters with Boolean types, the switch implies "true".
          * Special characters (such as '-') must be enclosed in double quotes.
          * Compiler compatible switches include: -D, -I, -O{0,1,2,3}
          * Verilog legacy switch formats are supported: +libext+, +incdir+
  
          Args:
              progname (str): Name of program to be executed.
              description (str): Short program description.
              switchlist (list of str): List of SC parameter switches to expose
                  at the command line. By default all SC schema switches are
                  available. Parameter switches should be entered based on the
                  parameter 'switch' field in the schema. For parameters with
                  multiple switches, both will be accepted if any one is included
                  in this list.
              input_map (dict of str): Dictionary mapping file extensions to input
                  filetypes. This is used to automatically assign positional
                  source arguments to ['input', ...] keypaths based on their file
                  extension. If None, the CLI will not accept positional source
                  arguments.
  
          Examples:
              >>> chip.create_cmdline(progname='sc-show',switchlist=['-input','-cfg'])
              Creates a command line interface for 'sc-show' app.
  
              >>> chip.create_cmdline(progname='sc', input_map={'v': 'verilog'})
              All sources ending in .v will be stored in ['input', 'verilog']
          """
  
          # Argparse
          parser = argparse.ArgumentParser(prog=progname,
                                           prefix_chars='-+',
                                           formatter_class=argparse.RawDescriptionHelpFormatter,
                                           description=description)
  
          # Get all keys from global dictionary or override at command line
          allkeys = self.getkeys()
  
          # Iterate over all keys to add parser arguments
          for keypath in allkeys:
              #Fetch fields from leaf cell
              helpstr = self.get(*keypath, field='shorthelp')
              typestr = self.get(*keypath, field='type')
  
              # argparse 'dest' must be a string, so join keypath with commas
              dest = '_'.join(keypath)
  
              switchstrs, metavar = self._get_switches(*keypath)
  
              # Three switch types (bool, list, scalar)
              if not switchlist or any(switch in switchlist for switch in switchstrs):
                  if typestr == 'bool':
                      parser.add_argument(*switchstrs,
                                          nargs='?',
                                          metavar=metavar,
                                          dest=dest,
                                          const='true',
                                          help=helpstr,
                                          default=argparse.SUPPRESS)
                  #list type arguments
                  elif re.match(r'\[', typestr):
                      #all the rest
                      parser.add_argument(*switchstrs,
                                          metavar=metavar,
                                          dest=dest,
                                          action='append',
                                          help=helpstr,
                                          default=argparse.SUPPRESS)
                  else:
                      #all the rest
                      parser.add_argument(*switchstrs,
                                          metavar=metavar,
                                          dest=dest,
                                          help=helpstr,
                                          default=argparse.SUPPRESS)
  
          if input_map is not None:
              parser.add_argument('source',
                                  nargs='*',
                                  help='Input files with filetype inferred by extension')
  
          #Preprocess sys.argv to enable linux commandline switch formats
          #(gcc, verilator, etc)
          scargs = []
  
          # Iterate from index 1, otherwise we end up with script name as a
          # 'source' positional argument
          for item in sys.argv[1:]:
              #Split switches with one character and a number after (O0,O1,O2)
              opt = re.match(r'(\-\w)(\d+)', item)
              #Split assign switches (-DCFG_ASIC=1)
              assign = re.search(r'(\-\w)(\w+\=\w+)', item)
              #Split plusargs (+incdir+/path)
              plusarg = re.search(r'(\+\w+\+)(.*)', item)
              if opt:
                  scargs.append(opt.group(1))
                  scargs.append(opt.group(2))
              elif plusarg:
                  scargs.append(plusarg.group(1))
                  scargs.append(plusarg.group(2))
              elif assign:
                  scargs.append(assign.group(1))
                  scargs.append(assign.group(2))
              else:
                  scargs.append(item)
  
          parser.add_argument('-version', action='version', version=_metadata.version)
  
          #Grab argument from pre-process sysargs
          cmdargs = vars(parser.parse_args(scargs))
  
          # Print banner
          print(_metadata.banner)
          print("Authors:", ", ".join(_metadata.authors))
          print("Version:", _metadata.version, "\n")
          print("-"*80)
  
          os.environ["COLUMNS"] = '80'
  
          # 1. set loglevel if set at command line
          if 'option_loglevel' in cmdargs.keys():
              self.logger.setLevel(cmdargs['option_loglevel'])
  
          # 2. read in target if set
          if 'option_target' in cmdargs.keys():
              if 'arg_pdk' in cmdargs.keys():
                  raise NotImplementedError("NOT IMPLEMENTED: ['arg', 'pdk'] parameter with target")
              if 'arg_flow' in cmdargs.keys():
                  raise NotImplementedError("NOT IMPLEMENTED: ['arg', 'flow'] parameter with target")
              if 'fpga_partname' in cmdargs.keys():
                  self.set('fpga', 'partname', cmdargs['fpga_partname'], clobber=True)
              # running target command
              self.load_target(cmdargs['option_target'])
  
          # 4. read in all cfg files
          if 'option_cfg' in cmdargs.keys():
              for item in cmdargs['option_cfg']:
                  self.read_manifest(item, clobber=True, clear=True)
  
          # Map sources to ['input'] keypath.
          if 'source' in cmdargs:
              for source in cmdargs['source']:
                  _, ext = os.path.splitext(source)
                  ext = ext.lstrip('.')
                  if ext in input_map:
                      filetype = input_map[ext]
                      if self.valid('input', filetype, quiet=True):
                          self.add('input', filetype, source)
                      else:
                          self.set('input', filetype, source)
                      self.logger.info(f'Source {source} inferred as {filetype}')
                  else:
                      self.logger.warning('Unable to infer input type for '
                          f'{source} based on file extension, ignoring. Use the '
                          '-input flag to provide it explicitly.')
              # we don't want to handle this in the next loop
              del cmdargs['source']
  
          # 5. Cycle through all command args and write to manifest
          for dest, vals in cmdargs.items():
              keypath = dest.split('_')
  
              # Turn everything into a list for uniformity
              if not isinstance(vals, list):
                  vals = [vals]
  
              # Cycle through all items
              for item in vals:
                  # Hack to handle the fact that we want optmode stored with an 'O'
                  # prefix.
                  if keypath == ['option', 'optmode']:
                      item = 'O' + item
  
                  num_free_keys = keypath.count('default')
  
                  if len(item.split(' ')) < num_free_keys + 1:
                      # Error out if value provided doesn't have enough words to
                      # fill in 'default' keys.
                      switches, metavar = self._get_switches(*keypath)
                      switchstr = '/'.join(switches)
                      self.logger.error(f'Invalid value {item} for switch {switchstr}. Expected format {metavar}.')
                      raise SiliconCompilerError('Invalid CLI arguments')
  
                  # We replace 'default' in keypath with first N words in provided
                  # value. Remainder is the actual value we want to store in the
                  # parameter.
                  *free_keys, val = item.split(' ', num_free_keys)
                  args = [free_keys.pop(0) if key == 'default' else key for key in keypath]
  
                  # Storing in manifest
                  self.logger.info(f"Command line argument entered: {args} Value: {val}")
                  typestr = self.get(*keypath, field='type')
                  if typestr.startswith('['):
                      if self.valid(*args, quiet=True):
                          self.add(*args, val)
                      else:
                          self.set(*args, val, clobber=True)
                  else:
                      self.set(*args, val, clobber=True)
  
      #########################################################################
      def find_function(self, modulename, funcname, moduletype=None):
          '''
          Returns a function attribute from a module on disk.
  
          Searches the SC root directory and the 'scpath' parameter for the
          modulename provided and imports the module if found. If the funcname
          provided is found in the module, a callable function attribute is
          returned, otherwise None is returned.
  
          The function assumes the following directory structure:
  
          * tools/modulename/modulename.py
          * flows/modulename.py
          * pdks/modulname.py
  
          If the moduletype is None, the module paths are search in the
          order: 'targets'->'flows'->'tools'->'pdks'->'libs'->'checklists'):
  
  
          Supported functions include:
  
          * targets (make_docs, setup)
          * pdks (make_docs, setup)
          * flows (make_docs, setup)
          * tools (make_docs, setup, check_version, runtime_options,
            pre_process, post_process)
          * libs (make_docs, setup)
  
          Args:
              modulename (str): Name of module to import.
              funcname (str): Name of the function to find within the module.
              moduletype (str): Type of module (flows, pdks, libs, checklists, targets).
  
          Examples:
              >>> setup_pdk = chip.find_function('freepdk45', 'setup', 'pdks')
              >>> setup_pdk()
              Imports the freepdk45 module and runs the setup_pdk function
  
          '''
  
          # module search path depends on modtype
          if moduletype is None:
              for item in ('targets', 'flows', 'tools', 'pdks', 'libs', 'checklists'):
                  fullpath = self.find_function(modulename, funcname, module_type=item)
                  if fullpath:
                      break
                  self.logger.error(f"Could not find module {modulename}")
                  self.error = 1
                  return None
          elif moduletype in ('targets','flows', 'pdks', 'libs'):
              fullpath = self._find_sc_file(f"{moduletype}/{modulename}.py", missing_ok=True)
          elif moduletype in ('tools', 'checklists'):
              fullpath = self._find_sc_file(f"{moduletype}/{modulename}/{modulename}.py", missing_ok=True)
          else:
              self.logger.error(f"Illegal module type '{moduletype}'.")
              self.error = 1
              return
  
          # try loading module if found
          if fullpath:
              self.logger.debug(f"Loading function '{funcname}' from module '{modulename}'")
  
              try:
                  spec = importlib.util.spec_from_file_location(modulename, fullpath)
                  imported = importlib.util.module_from_spec(spec)
                  spec.loader.exec_module(imported)
  
                  if hasattr(imported, funcname):
                      function = getattr(imported, funcname)
                  else:
                      function = None
                  return function
              except:
                  traceback.print_exc()
                  self.logger.error(f"Module setup failed for '{modulename}'")
                  self.error = 1
  
      ##########################################################################
      def load_target(self, name):
          """
          Loads a target module and runs the setup() function.
  
          The function searches the $SCPATH for targets/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
              flow (str): Target flow to
  
          Examples:
              >>> chip.load_target('freepdk45_demo')
              Loads the 'freepdk45_demo' target
  
          """
  
          self.set('option', 'target', name)
  
          func = self.find_function(name, 'setup', 'targets')
          if func is not None:
              func(self)
          else:
              self.logger.error(f'Target module {name} not found in $SCPATH or siliconcompiler/targets/.')
              raise SiliconCompilerError(f'Target module {name} not found $SCPATH or siliconcompiler/targets/.')
  
      ##########################################################################
      def load_pdk(self, name):
          """
          Loads a PDK module and runs the setup() function.
  
          The function searches the $SCPATH for pdks/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
  
          Examples:
              >>> chip.load_pdk('freepdk45_pdk')
              Loads the 'freepdk45' pdk
  
          """
  
          func = self.find_function(name, 'setup', 'pdks')
          if func is not None:
              self.logger.info(f"Loading PDK '{name}'")
              self._loaded_modules['pdks'].append(name)
              func(self)
          else:
              self.logger.error(f'PDK module {name} not found in $SCPATH or siliconcompiler/pdks/.')
              raise SiliconCompilerError(f'PDK module {name} not found in $SCPATH or siliconcompiler/pdks/.')
  
      ##########################################################################
      def load_flow(self, name):
          """
          Loads a flow  module and runs the setup() function.
  
          The function searches the $SCPATH for flows/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
  
          Examples:
              >>> chip.load_flow('asicflow')
              Loads the 'asicflow' flow
  
          """
  
          func = self.find_function(name, 'setup', 'flows')
          if func is not None:
              self.logger.info(f"Loading flow '{name}'")
              self._loaded_modules['flows'].append(name)
              func(self)
          else:
              self.logger.error(f'Flow module {name} not found in $SCPATH or siliconcompiler/flows/.')
              raise SiliconCompilerError(f'Flow module {name} not found in $SCPATH or siliconcompiler/flows/.')
  
      ##########################################################################
      def load_lib(self, name):
          """
          Loads a library module and runs the setup() function.
  
          The function searches the $SCPATH for libs/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
  
          Examples:
              >>> chip.load_lib('nangate45')
              Loads the 'nangate45' library
  
          """
  
          func = self.find_function(name, 'setup', 'libs')
          if func is not None:
              self.logger.info(f"Loading library '{name}'")
              self._loaded_modules['libs'].append(name)
              func(self)
          else:
              self.logger.error(f'Library module {name} not found in $SCPATH or siliconcompiler/libs/.')
              raise SiliconCompilerError(f'Library module {name} not found in $SCPATH or siliconcompiler/libs/.')
  
      ##########################################################################
      def load_checklist(self, name):
          """
          Loads a checklist module and runs the setup() function.
  
          The function searches the $SCPATH for checklist/<name>/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
  
          Examples:
              >>> chip.load_checklist('oh_tapeout')
              Loads the 'oh_tapeout' checklist
  
          """
  
          func = self.find_function(name, 'setup', 'checklists')
          if func is not None:
              self.logger.info(f"Loading checklist '{name}'")
              self._loaded_modules['checklists'].append(name)
              func(self)
          else:
              self.logger.error(f'Checklist module {name} not found in $SCPATH or siliconcompiler/checklists/.')
              raise SiliconCompilerError(f'Checklist module {name} not found in $SCPATH or siliconcompiler/checklists/.')
  
  
      ###########################################################################
      def list_metrics(self):
          '''
          Returns a list of all metrics in the schema.
  
          '''
  
          return self.getkeys('metric','default','default')
  
      ###########################################################################
      def help(self, *keypath):
          """
          Returns a schema parameter description.
  
          Args:
              *keypath(str): Keypath to parameter.
  
          Returns:
              A formatted multi-line help paragraph for the parameter provided.
  
          Examples:
              >>> print(chip.help('asic','diearea'))
              Displays help information about the 'asic, diearea' parameter
  
          """
  
          self.logger.debug('Fetching help for %s', keypath)
  
          #Fetch Values
  
          description = self.get(*keypath, field='shorthelp')
          typestr = self.get(*keypath, field='type')
          switchstr = str(self.get(*keypath, field='switch'))
          defstr = str(self.get(*keypath, field='defvalue'))
          requirement = str(self.get(*keypath, field='require'))
          helpstr = self.get(*keypath, field='help')
          example = self.get(*keypath, field='example')
  
          examplestr = ("\nExamples:    " + example[0] + ''.join(
                       ["\n             " + ex for ex in example[1:]]))
  
          #Removing multiple spaces and newlines
          helpstr = helpstr.rstrip()
          helpstr = helpstr.replace("\n", "")
          helpstr = ' '.join(helpstr.split())
  
          for idx, item in enumerate(example):
              example[idx] = ' '.join(item.split())
              example[idx] = example[idx].replace(", ", ",")
  
          #Wrap text
          para = textwrap.TextWrapper(width=60)
          para_list = para.wrap(text=helpstr)
  
          #Full Doc String
          fullstr = ("-"*80 +
                     "\nDescription: " + description +
                     "\nSwitch:      " + switchstr +
                     "\nType:        " + typestr  +
                     "\nRequirement: " + requirement   +
                     "\nDefault:     " + defstr   +
                     examplestr +
                     "\nHelp:        " + para_list[0] + "\n")
          for line in para_list[1:]:
              fullstr = (fullstr +
                         " "*13 + line.lstrip() + "\n")
  
          return fullstr
  
  
      ###########################################################################
      def valid(self, *args, valid_keypaths=None, quiet=True, default_valid=False):
          """
          Checks validity of a keypath.
  
          Checks the validity of a parameter keypath and returns True if the
          keypath is valid and False if invalid.
  
          Args:
              keypath(list str): Variable length schema key list.
              valid_keypaths (list of list): List of valid keypaths as lists. If
                  None, check against all keypaths in the schema.
              quiet (bool): If True, don't display warnings for invalid keypaths.
  
          Returns:
              Boolean indicating validity of keypath.
  
          Examples:
              >>> check = chip.valid('design')
              Returns True.
              >>> check = chip.valid('blah')
              Returns False.
          """
  
          keypathstr = ','.join(args)
          keylist = list(args)
          if default_valid:
              default = 'default'
          else:
              default = None
  
          if valid_keypaths is None:
              valid_keypaths = self.getkeys()
  
          # Look for a full match with default playing wild card
          for valid_keypath in valid_keypaths:
              if len(keylist) != len(valid_keypath):
                  continue
  
              ok = True
              for i in range(len(keylist)):
                  if valid_keypath[i] not in (keylist[i], default):
                      ok = False
                      break
              if ok:
                  return True
  
          # Match not found
          if not quiet:
              self.logger.warning(f"Keypath [{keypathstr}] is not valid")
          return False
  
      ###########################################################################
      def get(self, *keypath, field='value', job=None, cfg=None):
          """
          Returns a schema parameter field.
  
          Returns a schema parameter field based on the keypath provided in the
          ``*keypath``. See the :ref:`Schema Reference Manual<SiliconCompiler
          Schema>` for documentation of all supported keypaths. The returned type
          is consistent with the type field of the parameter. Fetching parameters
          with empty or undefined value files returns None for scalar types and []
          (empty list) for list types.  Accessing a non-existent keypath produces
          a logger error message and raises the Chip object error flag.
  
          Args:
              keypath(list str): Variable length schema key list.
              field(str): Parameter field to fetch.
              job (str): Jobname to use for dictionary access in place of the
                  current active jobname.
              cfg(dict): Alternate dictionary to access in place of the default
                  chip object schema dictionary.
  
          Returns:
              Value found for the keypath and field provided.
  
          Examples:
              >>> foundry = chip.get('pdk', 'foundry')
              Returns the name of the foundry from the PDK.
  
          """
  
          if cfg is None:
              if job is not None:
                  cfg = self.cfg['history'][job]
              else:
                  cfg = self.cfg
  
          keypathstr = ','.join(keypath)
  
          self.logger.debug(f"Reading from [{keypathstr}]. Field = '{field}'")
          return self._search(cfg, keypathstr, *keypath, field=field, mode='get')
  
      ###########################################################################
      def getkeys(self, *keypath, cfg=None, job=None):
          """
          Returns a list of schema dictionary keys.
  
          Searches the schema for the keypath provided and returns a list of
          keys found, excluding the generic 'default' key. Accessing a
          non-existent keypath produces a logger error message and raises the
          Chip object error flag.
  
          Args:
              keypath (list str): Variable length ordered schema key list
              cfg (dict): Alternate dictionary to access in place of self.cfg
              job (str): Jobname to use for dictionary access in place of the
                  current active jobname.
  
          Returns:
              List of keys found for the keypath provided.
  
          Examples:
              >>> keylist = chip.getkeys('pdk')
              Returns all keys for the 'pdk' keypath.
              >>> keylist = chip.getkeys()
              Returns all list of all keypaths in the schema.
          """
  
          if cfg is None:
              if job is None:
                  cfg = self.cfg
              else:
                  cfg = self.cfg['history'][job]
  
          if len(list(keypath)) > 0:
              keypathstr = ','.join(keypath)
              self.logger.debug('Getting schema parameter keys for: %s', keypathstr)
              keys = list(self._search(cfg, keypathstr, *keypath, mode='getkeys'))
              if 'default' in keys:
                  keys.remove('default')
          else:
              self.logger.debug('Getting all schema parameter keys.')
              keys = list(self._allkeys(cfg))
  
          return keys
  
      ###########################################################################
      def getdict(self, *keypath, cfg=None):
          """
          Returns a schema dictionary.
  
          Searches the schema for the keypath provided and returns a complete
          dictionary. Accessing a non-existent keypath produces a logger error
          message and raises the Chip object error flag.
  
          Args:
              keypath(list str): Variable length ordered schema key list
              cfg(dict): Alternate dictionary to access in place of self.cfg
  
          Returns:
              A schema dictionary
  
          Examples:
              >>> pdk = chip.getdict('pdk')
              Returns the complete dictionary found for the keypath 'pdk'
          """
  
          if cfg is None:
              cfg = self.cfg
  
          if len(list(keypath)) > 0:
              keypathstr = ','.join(keypath)
              self.logger.debug('Getting cfg for: %s', keypathstr)
              localcfg = self._search(cfg, keypathstr, *keypath, mode='getcfg')
  
          return copy.deepcopy(localcfg)
  
      ###########################################################################
      def set(self, *args, field='value', clobber=True, cfg=None):
          '''
          Sets a schema parameter field.
  
          Sets a schema parameter field based on the keypath and value provided in
          the ``*args``. See the :ref:`Schema Reference Manual<SiliconCompiler
          Schema>` for documentation of all supported keypaths. New schema
          dictionaries are automatically created for keypaths that overlap with
          'default' dictionaries. The write action is ignored if the parameter
          value is non-empty and the clobber option is set to False.
  
          The value provided must agree with the dictionary parameter 'type'.
          Accessing a non-existent keypath or providing a value that disagrees
          with the parameter type produces a logger error message and raises the
          Chip object error flag.
  
          Args:
              args (list): Parameter keypath followed by a value to set.
              field (str): Parameter field to set.
              clobber (bool): Existing value is overwritten if True.
              cfg(dict): Alternate dictionary to access in place of self.cfg
  
          Examples:
              >>> chip.set('design', 'top')
              Sets the name of the design to 'top'
          '''
  
          if cfg is None:
              cfg = self.cfg
  
          # Verify that all keys are strings
          for key in args[:-1]:
              if not isinstance(key,str):
                  self.logger.error(f"Key [{key}] is not a string [{args}]")
  
          keypathstr = ','.join(args[:-1])
          all_args = list(args)
  
          # Special case to ensure loglevel is updated ASAP
          if len(args) == 3 and args[1] == 'loglevel' and field == 'value':
              self.logger.setLevel(args[2])
  
          self.logger.debug(f"Setting [{keypathstr}] to {args[-1]}")
          return self._search(cfg, keypathstr, *all_args, field=field, mode='set', clobber=clobber)
  
      ###########################################################################
      def add(self, *args, cfg=None, field='value'):
          '''
          Adds item(s) to a schema parameter list.
  
          Adds item(s) to schema parameter list based on the keypath and value
          provided in the ``*args``.  See the :ref:`Schema Reference
          Manual<SiliconCompiler Schema>` for documentation of all supported
          keypaths. New schema dictionaries are automatically created for keypaths
          that overlap with 'default' dictionaries.
  
          The value provided must agree with the dictionary parameter 'type'.
          Accessing a non-existent keypath, providing a value that disagrees
          with the parameter type, or using add with a scalar parameter produces
          a logger error message and raises the Chip object error flag.
  
          Args:
              args (list): Parameter keypath followed by a value to add.
              cfg(dict): Alternate dictionary to access in place of self.cfg
              field (str): Parameter field to set.
  
          Examples:
              >>> chip.add('source', 'hello.v')
              Adds the file 'hello.v' to the list of sources.
          '''
  
          if cfg is None:
              cfg = self.cfg
  
          # Verify that all keys are strings
          for key in args[:-1]:
              if not isinstance(key,str):
                  self.logger.error(f"Key [{key}] is not a string [{args}]")
  
          keypathstr = ','.join(args
  
  [... the rest is omitted ...]
files: siliconcompiler/core.py
files: # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
  
  import argparse
  import base64
  import time
  import datetime
  import multiprocessing
  import tarfile
  import traceback
  import asyncio
  from subprocess import run, PIPE
  import os
  import glob
  import pathlib
  import sys
  import gzip
  import re
  import json
  import logging
  import hashlib
  import shutil
  import copy
  import importlib
  import textwrap
  import math
  import pandas
  import yaml
  import graphviz
  import time
  import uuid
  import shlex
  import platform
  import getpass
  import distro
  import netifaces
  import webbrowser
  import packaging.version
  import packaging.specifiers
  from jinja2 import Environment, FileSystemLoader
  from pathlib import Path
  from timeit import default_timer as timer
  from siliconcompiler.client import *
  from siliconcompiler.schema import *
  from siliconcompiler.scheduler import _deferstep
  from siliconcompiler import leflib
  from siliconcompiler import utils
  from siliconcompiler import _metadata
  import psutil
  
  class TaskStatus():
      # Could use Python 'enum' class here, but that doesn't work nicely with
      # schema.
      PENDING = 'pending'
      SUCCESS = 'success'
      ERROR = 'error'
  
  class Chip:
      """Object for configuring and executing hardware design flows.
  
      This is the main object used for configuration, data, and
      execution within the SiliconCompiler platform.
  
      Args:
          design (string): Name of the top level chip design module.
  
      Examples:
          >>> siliconcompiler.Chip(design="top")
          Creates a chip object with name "top".
      """
  
      ###########################################################################
      def __init__(self, design, loglevel=None):
          # version numbers
          self.scversion = _metadata.version
          self.schemaversion = SCHEMA_VERSION
  
          # Local variables
          self.scroot = os.path.dirname(os.path.abspath(__file__))
          self.cwd = os.getcwd()
          self.error = 0
          self.cfg = schema_cfg()
          # The 'status' dictionary can be used to store ephemeral config values.
          # Its contents will not be saved, and can be set by parent scripts
          # such as a web server or supervisor process. Currently supported keys:
          # * 'jobhash': A hash or UUID which can identify jobs in a larger system.
          # * 'remote_cfg': Dictionary containing remote server configurations
          #                 (address, credentials, etc.)
          # * 'slurm_account': User account ID in a connected slurm HPC cluster.
          # * 'slurm_partition': Name of the partition in which a task should run
          #                      on a connected slurm HPC cluster.
          # * 'watchdog': Activity-monitoring semaphore for jobs scheduled on an
          #               HPC cluster; expects a 'threading.Event'-like object.
          # * 'max_fs_bytes': A limit on how much disk space a job is allowed
          #                   to consume in a connected HPC cluster's storage.
          self.status = {}
  
          self.builtin = ['minimum','maximum',
                          'nop', 'mux', 'join', 'verify']
  
          # We set 'design' and 'loglevel' directly in the config dictionary
          # because of a chicken-and-egg problem: self.set() relies on the logger,
          # but the logger relies on these values.
          self.cfg['design']['value'] = design
          if loglevel:
              self.cfg['option']['loglevel']['value'] = loglevel
  
          self._init_logger()
  
          self._loaded_modules = {
              'flows': [],
              'pdks': [],
              'libs': [],
              'checklists': []
          }
  
      ###########################################################################
      @property
      def design(self):
          '''Design name of chip object.
  
          This is an immutable property.'''
          return self.get('design')
  
      ###########################################################################
      def _init_logger(self, step=None, index=None, in_run=False):
  
          self.logger = logging.getLogger(uuid.uuid4().hex)
  
          # Don't propagate log messages to "root" handler (we get duplicate
          # messages without this)
          # TODO: this prevents us from being able to capture logs with pytest:
          # we should revisit it
          self.logger.propagate = False
  
          loglevel = self.get('option', 'loglevel')
  
          if loglevel=='DEBUG':
              prefix = '| %(levelname)-7s | %(funcName)-10s | %(lineno)-4s'
          else:
              prefix = '| %(levelname)-7s'
  
          if in_run:
              flow = self.get('option', 'flow')
  
              # Figure out how wide to make step and index fields
              max_step_len = 2
              max_index_len = 2
              for future_step in self.getkeys('flowgraph', flow):
                  max_step_len = max(len(future_step) + 1, max_step_len)
                  for future_index in self.getkeys('flowgraph', flow, future_step):
                      max_index_len = max(len(future_index) + 1, max_index_len)
  
              jobname = self.get('option', 'jobname')
  
              if step is None:
                  step = '-' * max(max_step_len // 4, 1)
              if index is None:
                  index = '-' * max(max_index_len // 4, 1)
  
              run_info = f'%s  | %-{max_step_len}s | %-{max_index_len}s' % (jobname, step, index)
              logformat = ' | '.join([prefix, run_info, '%(message)s'])
          else:
              logformat = ' | '.join([prefix, '%(message)s'])
  
          handler = logging.StreamHandler()
          formatter = logging.Formatter(logformat)
  
          handler.setFormatter(formatter)
  
          # Clear any existing handlers so we don't end up with duplicate messages
          # if repeat calls to _init_logger are made
          if len(self.logger.handlers) > 0:
              self.logger.handlers.clear()
  
          self.logger.addHandler(handler)
          self.logger.setLevel(loglevel)
  
      ###########################################################################
      def _deinit_logger(self):
          self.logger = None
  
      ###########################################################################
      def _get_switches(self, *keypath):
          '''Helper function for parsing switches and metavars for a keypath.'''
          #Switch field fully describes switch format
          switch = self.get(*keypath, field='switch')
  
          if switch is None:
              switches = []
          elif isinstance(switch, list):
              switches = switch
          else:
              switches = [switch]
          switchstrs = []
  
          # parse out switch from metavar
          # TODO: should we validate that metavar matches for each switch?
          for switch in switches:
              switchmatch = re.match(r'(-[\w_]+)\s+(.*)', switch)
              gccmatch = re.match(r'(-[\w_]+)(.*)', switch)
              plusmatch = re.match(r'(\+[\w_\+]+)(.*)', switch)
  
              if switchmatch:
                  switchstr = switchmatch.group(1)
                  metavar = switchmatch.group(2)
              elif gccmatch:
                  switchstr = gccmatch.group(1)
                  metavar = gccmatch.group(2)
              elif plusmatch:
                  switchstr = plusmatch.group(1)
                  metavar = plusmatch.group(2)
              switchstrs.append(switchstr)
  
          return switchstrs, metavar
  
      ###########################################################################
      def create_cmdline(self, progname, description=None, switchlist=None, input_map=None):
          """Creates an SC command line interface.
  
          Exposes parameters in the SC schema as command line switches,
          simplifying creation of SC apps with a restricted set of schema
          parameters exposed at the command line. The order of command
          line switch settings parsed from the command line is as follows:
  
           1. loglevel
           2. fpga_partname
           3. load_target('target')
           4. read_manifest([cfg])
           5. all other switches
  
          The cmdline interface is implemented using the Python argparse package
          and the following use restrictions apply.
  
          * Help is accessed with the '-h' switch.
          * Arguments that include spaces must be enclosed with double quotes.
          * List parameters are entered individually. (ie. -y libdir1 -y libdir2)
          * For parameters with Boolean types, the switch implies "true".
          * Special characters (such as '-') must be enclosed in double quotes.
          * Compiler compatible switches include: -D, -I, -O{0,1,2,3}
          * Verilog legacy switch formats are supported: +libext+, +incdir+
  
          Args:
              progname (str): Name of program to be executed.
              description (str): Short program description.
              switchlist (list of str): List of SC parameter switches to expose
                  at the command line. By default all SC schema switches are
                  available. Parameter switches should be entered based on the
                  parameter 'switch' field in the schema. For parameters with
                  multiple switches, both will be accepted if any one is included
                  in this list.
              input_map (dict of str): Dictionary mapping file extensions to input
                  filetypes. This is used to automatically assign positional
                  source arguments to ['input', ...] keypaths based on their file
                  extension. If None, the CLI will not accept positional source
                  arguments.
  
          Examples:
              >>> chip.create_cmdline(progname='sc-show',switchlist=['-input','-cfg'])
              Creates a command line interface for 'sc-show' app.
  
              >>> chip.create_cmdline(progname='sc', input_map={'v': 'verilog'})
              All sources ending in .v will be stored in ['input', 'verilog']
          """
  
          # Argparse
          parser = argparse.ArgumentParser(prog=progname,
                                           prefix_chars='-+',
                                           formatter_class=argparse.RawDescriptionHelpFormatter,
                                           description=description)
  
          # Get all keys from global dictionary or override at command line
          allkeys = self.getkeys()
  
          # Iterate over all keys to add parser arguments
          for keypath in allkeys:
              #Fetch fields from leaf cell
              helpstr = self.get(*keypath, field='shorthelp')
              typestr = self.get(*keypath, field='type')
  
              # argparse 'dest' must be a string, so join keypath with commas
              dest = '_'.join(keypath)
  
              switchstrs, metavar = self._get_switches(*keypath)
  
              # Three switch types (bool, list, scalar)
              if not switchlist or any(switch in switchlist for switch in switchstrs):
                  if typestr == 'bool':
                      parser.add_argument(*switchstrs,
                                          nargs='?',
                                          metavar=metavar,
                                          dest=dest,
                                          const='true',
                                          help=helpstr,
                                          default=argparse.SUPPRESS)
                  #list type arguments
                  elif re.match(r'\[', typestr):
                      #all the rest
                      parser.add_argument(*switchstrs,
                                          metavar=metavar,
                                          dest=dest,
                                          action='append',
                                          help=helpstr,
                                          default=argparse.SUPPRESS)
                  else:
                      #all the rest
                      parser.add_argument(*switchstrs,
                                          metavar=metavar,
                                          dest=dest,
                                          help=helpstr,
                                          default=argparse.SUPPRESS)
  
          if input_map is not None:
              parser.add_argument('source',
                                  nargs='*',
                                  help='Input files with filetype inferred by extension')
  
          #Preprocess sys.argv to enable linux commandline switch formats
          #(gcc, verilator, etc)
          scargs = []
  
          # Iterate from index 1, otherwise we end up with script name as a
          # 'source' positional argument
          for item in sys.argv[1:]:
              #Split switches with one character and a number after (O0,O1,O2)
              opt = re.match(r'(\-\w)(\d+)', item)
              #Split assign switches (-DCFG_ASIC=1)
              assign = re.search(r'(\-\w)(\w+\=\w+)', item)
              #Split plusargs (+incdir+/path)
              plusarg = re.search(r'(\+\w+\+)(.*)', item)
              if opt:
                  scargs.append(opt.group(1))
                  scargs.append(opt.group(2))
              elif plusarg:
                  scargs.append(plusarg.group(1))
                  scargs.append(plusarg.group(2))
              elif assign:
                  scargs.append(assign.group(1))
                  scargs.append(assign.group(2))
              else:
                  scargs.append(item)
  
          parser.add_argument('-version', action='version', version=_metadata.version)
  
          #Grab argument from pre-process sysargs
          cmdargs = vars(parser.parse_args(scargs))
  
          # Print banner
          print(_metadata.banner)
          print("Authors:", ", ".join(_metadata.authors))
          print("Version:", _metadata.version, "\n")
          print("-"*80)
  
          os.environ["COLUMNS"] = '80'
  
          # 1. set loglevel if set at command line
          if 'option_loglevel' in cmdargs.keys():
              self.logger.setLevel(cmdargs['option_loglevel'])
  
          # 2. read in target if set
          if 'option_target' in cmdargs.keys():
              if 'arg_pdk' in cmdargs.keys():
                  raise NotImplementedError("NOT IMPLEMENTED: ['arg', 'pdk'] parameter with target")
              if 'arg_flow' in cmdargs.keys():
                  raise NotImplementedError("NOT IMPLEMENTED: ['arg', 'flow'] parameter with target")
              if 'fpga_partname' in cmdargs.keys():
                  self.set('fpga', 'partname', cmdargs['fpga_partname'], clobber=True)
              # running target command
              self.load_target(cmdargs['option_target'])
  
          # 4. read in all cfg files
          if 'option_cfg' in cmdargs.keys():
              for item in cmdargs['option_cfg']:
                  self.read_manifest(item, clobber=True, clear=True)
  
          # Map sources to ['input'] keypath.
          if 'source' in cmdargs:
              for source in cmdargs['source']:
                  _, ext = os.path.splitext(source)
                  ext = ext.lstrip('.')
                  if ext in input_map:
                      filetype = input_map[ext]
                      if self.valid('input', filetype, quiet=True):
                          self.add('input', filetype, source)
                      else:
                          self.set('input', filetype, source)
                      self.logger.info(f'Source {source} inferred as {filetype}')
                  else:
                      self.logger.warning('Unable to infer input type for '
                          f'{source} based on file extension, ignoring. Use the '
                          '-input flag to provide it explicitly.')
              # we don't want to handle this in the next loop
              del cmdargs['source']
  
          # 5. Cycle through all command args and write to manifest
          for dest, vals in cmdargs.items():
              keypath = dest.split('_')
  
              # Turn everything into a list for uniformity
              if not isinstance(vals, list):
                  vals = [vals]
  
              # Cycle through all items
              for item in vals:
                  # Hack to handle the fact that we want optmode stored with an 'O'
                  # prefix.
                  if keypath == ['option', 'optmode']:
                      item = 'O' + item
  
                  num_free_keys = keypath.count('default')
  
                  if len(item.split(' ')) < num_free_keys + 1:
                      # Error out if value provided doesn't have enough words to
                      # fill in 'default' keys.
                      switches, metavar = self._get_switches(*keypath)
                      switchstr = '/'.join(switches)
                      self.logger.error(f'Invalid value {item} for switch {switchstr}. Expected format {metavar}.')
                      raise SiliconCompilerError('Invalid CLI arguments')
  
                  # We replace 'default' in keypath with first N words in provided
                  # value. Remainder is the actual value we want to store in the
                  # parameter.
                  *free_keys, val = item.split(' ', num_free_keys)
                  args = [free_keys.pop(0) if key == 'default' else key for key in keypath]
  
                  # Storing in manifest
                  self.logger.info(f"Command line argument entered: {args} Value: {val}")
                  typestr = self.get(*keypath, field='type')
                  if typestr.startswith('['):
                      if self.valid(*args, quiet=True):
                          self.add(*args, val)
                      else:
                          self.set(*args, val, clobber=True)
                  else:
                      self.set(*args, val, clobber=True)
  
      #########################################################################
      def find_function(self, modulename, funcname, moduletype=None):
          '''
          Returns a function attribute from a module on disk.
  
          Searches the SC root directory and the 'scpath' parameter for the
          modulename provided and imports the module if found. If the funcname
          provided is found in the module, a callable function attribute is
          returned, otherwise None is returned.
  
          The function assumes the following directory structure:
  
          * tools/modulename/modulename.py
          * flows/modulename.py
          * pdks/modulname.py
  
          If the moduletype is None, the module paths are search in the
          order: 'targets'->'flows'->'tools'->'pdks'->'libs'->'checklists'):
  
  
          Supported functions include:
  
          * targets (make_docs, setup)
          * pdks (make_docs, setup)
          * flows (make_docs, setup)
          * tools (make_docs, setup, check_version, runtime_options,
            pre_process, post_process)
          * libs (make_docs, setup)
  
          Args:
              modulename (str): Name of module to import.
              funcname (str): Name of the function to find within the module.
              moduletype (str): Type of module (flows, pdks, libs, checklists, targets).
  
          Examples:
              >>> setup_pdk = chip.find_function('freepdk45', 'setup', 'pdks')
              >>> setup_pdk()
              Imports the freepdk45 module and runs the setup_pdk function
  
          '''
  
          # module search path depends on modtype
          if moduletype is None:
              for item in ('targets', 'flows', 'tools', 'pdks', 'libs', 'checklists'):
                  fullpath = self.find_function(modulename, funcname, module_type=item)
                  if fullpath:
                      break
                  self.logger.error(f"Could not find module {modulename}")
                  self.error = 1
                  return None
          elif moduletype in ('targets','flows', 'pdks', 'libs'):
              fullpath = self._find_sc_file(f"{moduletype}/{modulename}.py", missing_ok=True)
          elif moduletype in ('tools', 'checklists'):
              fullpath = self._find_sc_file(f"{moduletype}/{modulename}/{modulename}.py", missing_ok=True)
          else:
              self.logger.error(f"Illegal module type '{moduletype}'.")
              self.error = 1
              return
  
          # try loading module if found
          if fullpath:
              self.logger.debug(f"Loading function '{funcname}' from module '{modulename}'")
  
              try:
                  spec = importlib.util.spec_from_file_location(modulename, fullpath)
                  imported = importlib.util.module_from_spec(spec)
                  spec.loader.exec_module(imported)
  
                  if hasattr(imported, funcname):
                      function = getattr(imported, funcname)
                  else:
                      function = None
                  return function
              except:
                  traceback.print_exc()
                  self.logger.error(f"Module setup failed for '{modulename}'")
                  self.error = 1
  
      ##########################################################################
      def load_target(self, name):
          """
          Loads a target module and runs the setup() function.
  
          The function searches the $SCPATH for targets/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
              flow (str): Target flow to
  
          Examples:
              >>> chip.load_target('freepdk45_demo')
              Loads the 'freepdk45_demo' target
  
          """
  
          self.set('option', 'target', name)
  
          func = self.find_function(name, 'setup', 'targets')
          if func is not None:
              func(self)
          else:
              self.logger.error(f'Target module {name} not found in $SCPATH or siliconcompiler/targets/.')
              raise SiliconCompilerError(f'Target module {name} not found $SCPATH or siliconcompiler/targets/.')
  
      ##########################################################################
      def load_pdk(self, name):
          """
          Loads a PDK module and runs the setup() function.
  
          The function searches the $SCPATH for pdks/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
  
          Examples:
              >>> chip.load_pdk('freepdk45_pdk')
              Loads the 'freepdk45' pdk
  
          """
  
          func = self.find_function(name, 'setup', 'pdks')
          if func is not None:
              self.logger.info(f"Loading PDK '{name}'")
              self._loaded_modules['pdks'].append(name)
              func(self)
          else:
              self.logger.error(f'PDK module {name} not found in $SCPATH or siliconcompiler/pdks/.')
              raise SiliconCompilerError(f'PDK module {name} not found in $SCPATH or siliconcompiler/pdks/.')
  
      ##########################################################################
      def load_flow(self, name):
          """
          Loads a flow  module and runs the setup() function.
  
          The function searches the $SCPATH for flows/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
  
          Examples:
              >>> chip.load_flow('asicflow')
              Loads the 'asicflow' flow
  
          """
  
          func = self.find_function(name, 'setup', 'flows')
          if func is not None:
              self.logger.info(f"Loading flow '{name}'")
              self._loaded_modules['flows'].append(name)
              func(self)
          else:
              self.logger.error(f'Flow module {name} not found in $SCPATH or siliconcompiler/flows/.')
              raise SiliconCompilerError(f'Flow module {name} not found in $SCPATH or siliconcompiler/flows/.')
  
      ##########################################################################
      def load_lib(self, name):
          """
          Loads a library module and runs the setup() function.
  
          The function searches the $SCPATH for libs/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
  
          Examples:
              >>> chip.load_lib('nangate45')
              Loads the 'nangate45' library
  
          """
  
          func = self.find_function(name, 'setup', 'libs')
          if func is not None:
              self.logger.info(f"Loading library '{name}'")
              self._loaded_modules['libs'].append(name)
              func(self)
          else:
              self.logger.error(f'Library module {name} not found in $SCPATH or siliconcompiler/libs/.')
              raise SiliconCompilerError(f'Library module {name} not found in $SCPATH or siliconcompiler/libs/.')
  
      ##########################################################################
      def load_checklist(self, name):
          """
          Loads a checklist module and runs the setup() function.
  
          The function searches the $SCPATH for checklist/<name>/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
  
          Examples:
              >>> chip.load_checklist('oh_tapeout')
              Loads the 'oh_tapeout' checklist
  
          """
  
          func = self.find_function(name, 'setup', 'checklists')
          if func is not None:
              self.logger.info(f"Loading checklist '{name}'")
              self._loaded_modules['checklists'].append(name)
              func(self)
          else:
              self.logger.error(f'Checklist module {name} not found in $SCPATH or siliconcompiler/checklists/.')
              raise SiliconCompilerError(f'Checklist module {name} not found in $SCPATH or siliconcompiler/checklists/.')
  
  
      ###########################################################################
      def list_metrics(self):
          '''
          Returns a list of all metrics in the schema.
  
          '''
  
          return self.getkeys('metric','default','default')
  
      ###########################################################################
      def help(self, *keypath):
          """
          Returns a schema parameter description.
  
          Args:
              *keypath(str): Keypath to parameter.
  
          Returns:
              A formatted multi-line help paragraph for the parameter provided.
  
          Examples:
              >>> print(chip.help('asic','diearea'))
              Displays help information about the 'asic, diearea' parameter
  
          """
  
          self.logger.debug('Fetching help for %s', keypath)
  
          #Fetch Values
  
          description = self.get(*keypath, field='shorthelp')
          typestr = self.get(*keypath, field='type')
          switchstr = str(self.get(*keypath, field='switch'))
          defstr = str(self.get(*keypath, field='defvalue'))
          requirement = str(self.get(*keypath, field='require'))
          helpstr = self.get(*keypath, field='help')
          example = self.get(*keypath, field='example')
  
          examplestr = ("\nExamples:    " + example[0] + ''.join(
                       ["\n             " + ex for ex in example[1:]]))
  
          #Removing multiple spaces and newlines
          helpstr = helpstr.rstrip()
          helpstr = helpstr.replace("\n", "")
          helpstr = ' '.join(helpstr.split())
  
          for idx, item in enumerate(example):
              example[idx] = ' '.join(item.split())
              example[idx] = example[idx].replace(", ", ",")
  
          #Wrap text
          para = textwrap.TextWrapper(width=60)
          para_list = para.wrap(text=helpstr)
  
          #Full Doc String
          fullstr = ("-"*80 +
                     "\nDescription: " + description +
                     "\nSwitch:      " + switchstr +
                     "\nType:        " + typestr  +
                     "\nRequirement: " + requirement   +
                     "\nDefault:     " + defstr   +
                     examplestr +
                     "\nHelp:        " + para_list[0] + "\n")
          for line in para_list[1:]:
              fullstr = (fullstr +
                         " "*13 + line.lstrip() + "\n")
  
          return fullstr
  
  
      ###########################################################################
      def valid(self, *args, valid_keypaths=None, quiet=True, default_valid=False):
          """
          Checks validity of a keypath.
  
          Checks the validity of a parameter keypath and returns True if the
          keypath is valid and False if invalid.
  
          Args:
              keypath(list str): Variable length schema key list.
              valid_keypaths (list of list): List of valid keypaths as lists. If
                  None, check against all keypaths in the schema.
              quiet (bool): If True, don't display warnings for invalid keypaths.
  
          Returns:
              Boolean indicating validity of keypath.
  
          Examples:
              >>> check = chip.valid('design')
              Returns True.
              >>> check = chip.valid('blah')
              Returns False.
          """
  
          keypathstr = ','.join(args)
          keylist = list(args)
          if default_valid:
              default = 'default'
          else:
              default = None
  
          if valid_keypaths is None:
              valid_keypaths = self.getkeys()
  
          # Look for a full match with default playing wild card
          for valid_keypath in valid_keypaths:
              if len(keylist) != len(valid_keypath):
                  continue
  
              ok = True
              for i in range(len(keylist)):
                  if valid_keypath[i] not in (keylist[i], default):
                      ok = False
                      break
              if ok:
                  return True
  
          # Match not found
          if not quiet:
              self.logger.warning(f"Keypath [{keypathstr}] is not valid")
          return False
  
      ###########################################################################
      def get(self, *keypath, field='value', job=None, cfg=None):
          """
          Returns a schema parameter field.
  
          Returns a schema parameter field based on the keypath provided in the
          ``*keypath``. See the :ref:`Schema Reference Manual<SiliconCompiler
          Schema>` for documentation of all supported keypaths. The returned type
          is consistent with the type field of the parameter. Fetching parameters
          with empty or undefined value files returns None for scalar types and []
          (empty list) for list types.  Accessing a non-existent keypath produces
          a logger error message and raises the Chip object error flag.
  
          Args:
              keypath(list str): Variable length schema key list.
              field(str): Parameter field to fetch.
              job (str): Jobname to use for dictionary access in place of the
                  current active jobname.
              cfg(dict): Alternate dictionary to access in place of the default
                  chip object schema dictionary.
  
          Returns:
              Value found for the keypath and field provided.
  
          Examples:
              >>> foundry = chip.get('pdk', 'foundry')
              Returns the name of the foundry from the PDK.
  
          """
  
          if cfg is None:
              if job is not None:
                  cfg = self.cfg['history'][job]
              else:
                  cfg = self.cfg
  
          keypathstr = ','.join(keypath)
  
          self.logger.debug(f"Reading from [{keypathstr}]. Field = '{field}'")
          return self._search(cfg, keypathstr, *keypath, field=field, mode='get')
  
      ###########################################################################
      def getkeys(self, *keypath, cfg=None, job=None):
          """
          Returns a list of schema dictionary keys.
  
          Searches the schema for the keypath provided and returns a list of
          keys found, excluding the generic 'default' key. Accessing a
          non-existent keypath produces a logger error message and raises the
          Chip object error flag.
  
          Args:
              keypath (list str): Variable length ordered schema key list
              cfg (dict): Alternate dictionary to access in place of self.cfg
              job (str): Jobname to use for dictionary access in place of the
                  current active jobname.
  
          Returns:
              List of keys found for the keypath provided.
  
          Examples:
              >>> keylist = chip.getkeys('pdk')
              Returns all keys for the 'pdk' keypath.
              >>> keylist = chip.getkeys()
              Returns all list of all keypaths in the schema.
          """
  
          if cfg is None:
              if job is None:
                  cfg = self.cfg
              else:
                  cfg = self.cfg['history'][job]
  
          if len(list(keypath)) > 0:
              keypathstr = ','.join(keypath)
              self.logger.debug('Getting schema parameter keys for: %s', keypathstr)
              keys = list(self._search(cfg, keypathstr, *keypath, mode='getkeys'))
              if 'default' in keys:
                  keys.remove('default')
          else:
              self.logger.debug('Getting all schema parameter keys.')
              keys = list(self._allkeys(cfg))
  
          return keys
  
      ###########################################################################
      def getdict(self, *keypath, cfg=None):
          """
          Returns a schema dictionary.
  
          Searches the schema for the keypath provided and returns a complete
          dictionary. Accessing a non-existent keypath produces a logger error
          message and raises the Chip object error flag.
  
          Args:
              keypath(list str): Variable length ordered schema key list
              cfg(dict): Alternate dictionary to access in place of self.cfg
  
          Returns:
              A schema dictionary
  
          Examples:
              >>> pdk = chip.getdict('pdk')
              Returns the complete dictionary found for the keypath 'pdk'
          """
  
          if cfg is None:
              cfg = self.cfg
  
          if len(list(keypath)) > 0:
              keypathstr = ','.join(keypath)
              self.logger.debug('Getting cfg for: %s', keypathstr)
              localcfg = self._search(cfg, keypathstr, *keypath, mode='getcfg')
  
          return copy.deepcopy(localcfg)
  
      ###########################################################################
      def set(self, *args, field='value', clobber=True, cfg=None):
          '''
          Sets a schema parameter field.
  
          Sets a schema parameter field based on the keypath and value provided in
          the ``*args``. See the :ref:`Schema Reference Manual<SiliconCompiler
          Schema>` for documentation of all supported keypaths. New schema
          dictionaries are automatically created for keypaths that overlap with
          'default' dictionaries. The write action is ignored if the parameter
          value is non-empty and the clobber option is set to False.
  
          The value provided must agree with the dictionary parameter 'type'.
          Accessing a non-existent keypath or providing a value that disagrees
          with the parameter type produces a logger error message and raises the
          Chip object error flag.
  
          Args:
              args (list): Parameter keypath followed by a value to set.
              field (str): Parameter field to set.
              clobber (bool): Existing value is overwritten if True.
              cfg(dict): Alternate dictionary to access in place of self.cfg
  
          Examples:
              >>> chip.set('design', 'top')
              Sets the name of the design to 'top'
          '''
  
          if cfg is None:
              cfg = self.cfg
  
          # Verify that all keys are strings
          for key in args[:-1]:
              if not isinstance(key,str):
                  self.logger.error(f"Key [{key}] is not a string [{args}]")
  
          keypathstr = ','.join(args[:-1])
          all_args = list(args)
  
          # Special case to ensure loglevel is updated ASAP
          if len(args) == 3 and args[1] == 'loglevel' and field == 'value':
              self.logger.setLevel(args[2])
  
          self.logger.debug(f"Setting [{keypathstr}] to {args[-1]}")
          return self._search(cfg, keypathstr, *all_args, field=field, mode='set', clobber=clobber)
  
      ###########################################################################
      def add(self, *args, cfg=None, field='value'):
          '''
          Adds item(s) to a schema parameter list.
  
          Adds item(s) to schema parameter list based on the keypath and value
          provided in the ``*args``.  See the :ref:`Schema Reference
          Manual<SiliconCompiler Schema>` for documentation of all supported
          keypaths. New schema dictionaries are automatically created for keypaths
          that overlap with 'default' dictionaries.
  
          The value provided must agree with the dictionary parameter 'type'.
          Accessing a non-existent keypath, providing a value that disagrees
          with the parameter type, or using add with a scalar parameter produces
          a logger error message and raises the Chip object error flag.
  
          Args:
              args (list): Parameter keypath followed by a value to add.
              cfg(dict): Alternate dictionary to access in place of self.cfg
              field (str): Parameter field to set.
  
          Examples:
              >>> chip.add('source', 'hello.v')
              Adds the file 'hello.v' to the list of sources.
          '''
  
          if cfg is None:
              cfg = self.cfg
  
          # Verify that all keys are strings
          for key in args[:-1]:
              if not isinstance(key,str):
                  self.logger.error(f"Key [{key}] is not a string [{args}]")
  
          keypathstr = ','.join(args
  
  [... the rest is omitted ...]
files: siliconcompiler/core.py
files: # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
  import pytest
  
  import siliconcompiler
  
  def test_write_manifest():
  
      chip = siliconcompiler.Chip('top')
      chip.add('input', 'sdc','top.sdc')
      chip.add('input', 'verilog', 'top.v')
      chip.add('input', 'verilog', 'a.v')
      chip.add('input', 'verilog', 'b.v')
      chip.add('input', 'verilog', 'c.v')
  
      chip.write_manifest('top.pkg.json')
      chip.write_manifest('top.csv')
      chip.write_manifest('top.tcl', prune=False)
      chip.write_manifest('top.yaml')
  
  def test_advanced_tcl(monkeypatch):
      # Tkinter module is part of Python standard library, but may not be
      # available depending on if the system has the python3-tk package installed.
      # This line will import tkinter if it's available, and skip the test
      # otherwise.
      tkinter = pytest.importorskip('tkinter')
  
      chip = siliconcompiler.Chip('top')
  
      # Test complex strings
      desc = '''This description is potentially problematic since it includes
  multiple lines, spaces, and TCL special characters. This package costs $5 {for real}!'''
      chip.set('package', 'description', desc)
  
      # Test tuples
      chip.add('asic', 'diearea', (0, 0))
      chip.add('asic', 'diearea', (30, 40))
  
      # Test bools
      chip.set('option', 'quiet', True)
  
      # Test envvars
      chip.set('input', 'verilog', 'rtl/$TOPMOD.v')
  
      chip.write_manifest('top.tcl')
  
      # Read from config in TCL as test
      tcl = tkinter.Tcl()
  
      # Set env var to test ['input', 'verilog']
      monkeypatch.setenv('TOPMOD', 'design')
  
      def tcl_eval(expr):
          script = f'''
          source top.tcl
          return {expr}'''
          return tcl.eval(script)
  
      # When the TCL shell displays a multiline string, it gets surrounded in {}.
      expected_desc = '{' + desc + '}'
      assert tcl_eval('[dict get $sc_cfg package description]') == expected_desc
  
      assert tcl_eval('[lindex [lindex [dict get $sc_cfg asic diearea] 1] 0]') == '30.0'
      assert tcl_eval('[dict get $sc_cfg option quiet]') == 'true'
      assert tcl_eval('[dict get $sc_cfg input verilog]') == 'rtl/design.v'
  
  #########################
  if __name__ == "__main__":
      test_write_manifest()
  
  
  [... the rest is omitted ...]
files: tests/core/test_write_manifest.py
files: # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
  import pytest
  
  import siliconcompiler
  
  def test_write_manifest():
  
      chip = siliconcompiler.Chip('top')
      chip.add('input', 'sdc','top.sdc')
      chip.add('input', 'verilog', 'top.v')
      chip.add('input', 'verilog', 'a.v')
      chip.add('input', 'verilog', 'b.v')
      chip.add('input', 'verilog', 'c.v')
  
      chip.write_manifest('top.pkg.json')
      chip.write_manifest('top.csv')
      chip.write_manifest('top.tcl', prune=False)
      chip.write_manifest('top.yaml')
  
  def test_advanced_tcl(monkeypatch):
      # Tkinter module is part of Python standard library, but may not be
      # available depending on if the system has the python3-tk installed. This
      # line will import tkinter if it's available, and skip the test otherwise.
      tkinter = pytest.importorskip('tkinter')
  
      chip = siliconcompiler.Chip('top')
  
      # Test complex strings
      desc = '''This description is potentially problematic since it includes
  multiple lines, spaces, and TCL special characters. This package costs $5 {for real}!'''
      chip.set('package', 'description', desc)
  
      # Test tuples
      chip.add('asic', 'diearea', (0, 0))
      chip.add('asic', 'diearea', (30, 40))
  
      # Test bools
      chip.set('option', 'quiet', True)
  
      # Test envvars
      chip.set('input', 'verilog', 'rtl/$TOPMOD.v')
  
      chip.write_manifest('top.tcl')
  
      # Read from config in TCL as test
      tcl = tkinter.Tcl()
  
      # Set env var to test ['input', 'verilog']
      monkeypatch.setenv('TOPMOD', 'design')
  
      def tcl_eval(expr):
          script = f'''
          source top.tcl
          return {expr}'''
          return tcl.eval(script)
  
      # When we call puts on a multiline string, it does get mangled a bit. It
      # stays surrounded by {}, and {} within the string are still escaped.
      # TODO: is this problematic? I think it is okay for now since we don't
      # really read these strings within TCL, they just need to not break sourcing
      # the manifest.
      expected_desc = '{' + desc.replace('{', '\\{').replace('}', '\\}') + '}'
      assert tcl_eval('[dict get $sc_cfg package description]') == expected_desc
  
      assert tcl_eval('[lindex [lindex [dict get $sc_cfg asic diearea] 1] 0]') == '30.0'
      assert tcl_eval('[dict get $sc_cfg option quiet]') == 'true'
      assert tcl_eval('[dict get $sc_cfg input verilog]') == 'rtl/design.v'
  
  #########################
  if __name__ == "__main__":
      test_write_manifest()
  
  
  [... the rest is omitted ...]
files: tests/core/test_write_manifest.py