files.new_contents: # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
  
  import argparse
  import base64
  import time
  import datetime
  import multiprocessing
  import tarfile
  import traceback
  import asyncio
  from subprocess import run, PIPE
  import os
  import glob
  import pathlib
  import sys
  import gzip
  import re
  import json
  import logging
  import hashlib
  import shutil
  import copy
  import importlib
  import textwrap
  import math
  import pandas
  import yaml
  import graphviz
  import time
  import uuid
  import shlex
  import platform
  import getpass
  import distro
  import netifaces
  import webbrowser
  import packaging.version
  import packaging.specifiers
  from jinja2 import Environment, FileSystemLoader
  from pathlib import Path
  from timeit import default_timer as timer
  from siliconcompiler.client import *
  from siliconcompiler.schema import *
  from siliconcompiler.scheduler import _deferstep
  from siliconcompiler import leflib
  from siliconcompiler import utils
  from siliconcompiler import _metadata
  import psutil
  
  class TaskStatus():
      # Could use Python 'enum' class here, but that doesn't work nicely with
      # schema.
      PENDING = 'pending'
      SUCCESS = 'success'
      ERROR = 'error'
  
  class Chip:
      """Object for configuring and executing hardware design flows.
  
      This is the main object used for configuration, data, and
      execution within the SiliconCompiler platform.
  
      Args:
          design (string): Name of the top level chip design module.
  
      Examples:
          >>> siliconcompiler.Chip(design="top")
          Creates a chip object with name "top".
      """
  
      ###########################################################################
      def __init__(self, design, loglevel=None):
          # version numbers
          self.scversion = _metadata.version
          self.schemaversion = SCHEMA_VERSION
  
          # Local variables
          self.scroot = os.path.dirname(os.path.abspath(__file__))
          self.cwd = os.getcwd()
          self.error = 0
          self.cfg = schema_cfg()
          # The 'status' dictionary can be used to store ephemeral config values.
          # Its contents will not be saved, and can be set by parent scripts
          # such as a web server or supervisor process. Currently supported keys:
          # * 'jobhash': A hash or UUID which can identify jobs in a larger system.
          # * 'remote_cfg': Dictionary containing remote server configurations
          #                 (address, credentials, etc.)
          # * 'slurm_account': User account ID in a connected slurm HPC cluster.
          # * 'slurm_partition': Name of the partition in which a task should run
          #                      on a connected slurm HPC cluster.
          # * 'watchdog': Activity-monitoring semaphore for jobs scheduled on an
          #               HPC cluster; expects a 'threading.Event'-like object.
          # * 'max_fs_bytes': A limit on how much disk space a job is allowed
          #                   to consume in a connected HPC cluster's storage.
          self.status = {}
  
          self.builtin = ['minimum','maximum',
                          'nop', 'mux', 'join', 'verify']
  
          # We set 'design' and 'loglevel' directly in the config dictionary
          # because of a chicken-and-egg problem: self.set() relies on the logger,
          # but the logger relies on these values.
          self.cfg['design']['value'] = design
          if loglevel:
              self.cfg['option']['loglevel']['value'] = loglevel
  
          self._init_logger()
  
          self._loaded_modules = {
              'flows': [],
              'pdks': [],
              'libs': [],
              'checklists': []
          }
  
      ###########################################################################
      @property
      def design(self):
          '''Design name of chip object.
  
          This is an immutable property.'''
          return self.get('design')
  
      ###########################################################################
      def _init_logger(self, step=None, index=None, in_run=False):
  
          self.logger = logging.getLogger(uuid.uuid4().hex)
  
          # Don't propagate log messages to "root" handler (we get duplicate
          # messages without this)
          # TODO: this prevents us from being able to capture logs with pytest:
          # we should revisit it
          self.logger.propagate = False
  
          loglevel = self.get('option', 'loglevel')
  
          if loglevel=='DEBUG':
              prefix = '| %(levelname)-7s | %(funcName)-10s | %(lineno)-4s'
          else:
              prefix = '| %(levelname)-7s'
  
          if in_run:
              flow = self.get('option', 'flow')
  
              # Figure out how wide to make step and index fields
              max_step_len = 2
              max_index_len = 2
              for future_step in self.getkeys('flowgraph', flow):
                  max_step_len = max(len(future_step) + 1, max_step_len)
                  for future_index in self.getkeys('flowgraph', flow, future_step):
                      max_index_len = max(len(future_index) + 1, max_index_len)
  
              jobname = self.get('option', 'jobname')
  
              if step is None:
                  step = '-' * max(max_step_len // 4, 1)
              if index is None:
                  index = '-' * max(max_index_len // 4, 1)
  
              run_info = f'%s  | %-{max_step_len}s | %-{max_index_len}s' % (jobname, step, index)
              logformat = ' | '.join([prefix, run_info, '%(message)s'])
          else:
              logformat = ' | '.join([prefix, '%(message)s'])
  
          handler = logging.StreamHandler()
          formatter = logging.Formatter(logformat)
  
          handler.setFormatter(formatter)
  
          # Clear any existing handlers so we don't end up with duplicate messages
          # if repeat calls to _init_logger are made
          if len(self.logger.handlers) > 0:
              self.logger.handlers.clear()
  
          self.logger.addHandler(handler)
          self.logger.setLevel(loglevel)
  
      ###########################################################################
      def _deinit_logger(self):
          self.logger = None
  
      ###########################################################################
      def _get_switches(self, *keypath):
          '''Helper function for parsing switches and metavars for a keypath.'''
          #Switch field fully describes switch format
          switch = self.get(*keypath, field='switch')
  
          if switch is None:
              switches = []
          elif isinstance(switch, list):
              switches = switch
          else:
              switches = [switch]
          switchstrs = []
  
          # parse out switch from metavar
          # TODO: should we validate that metavar matches for each switch?
          for switch in switches:
              switchmatch = re.match(r'(-[\w_]+)\s+(.*)', switch)
              gccmatch = re.match(r'(-[\w_]+)(.*)', switch)
              plusmatch = re.match(r'(\+[\w_\+]+)(.*)', switch)
  
              if switchmatch:
                  switchstr = switchmatch.group(1)
                  metavar = switchmatch.group(2)
              elif gccmatch:
                  switchstr = gccmatch.group(1)
                  metavar = gccmatch.group(2)
              elif plusmatch:
                  switchstr = plusmatch.group(1)
                  metavar = plusmatch.group(2)
              switchstrs.append(switchstr)
  
          return switchstrs, metavar
  
      ###########################################################################
      def create_cmdline(self, progname, description=None, switchlist=None, input_map=None):
          """Creates an SC command line interface.
  
          Exposes parameters in the SC schema as command line switches,
          simplifying creation of SC apps with a restricted set of schema
          parameters exposed at the command line. The order of command
          line switch settings parsed from the command line is as follows:
  
           1. loglevel
           2. fpga_partname
           3. load_target('target')
           4. read_manifest([cfg])
           5. all other switches
  
          The cmdline interface is implemented using the Python argparse package
          and the following use restrictions apply.
  
          * Help is accessed with the '-h' switch.
          * Arguments that include spaces must be enclosed with double quotes.
          * List parameters are entered individually. (ie. -y libdir1 -y libdir2)
          * For parameters with Boolean types, the switch implies "true".
          * Special characters (such as '-') must be enclosed in double quotes.
          * Compiler compatible switches include: -D, -I, -O{0,1,2,3}
          * Verilog legacy switch formats are supported: +libext+, +incdir+
  
          Args:
              progname (str): Name of program to be executed.
              description (str): Short program description.
              switchlist (list of str): List of SC parameter switches to expose
                  at the command line. By default all SC schema switches are
                  available. Parameter switches should be entered based on the
                  parameter 'switch' field in the schema. For parameters with
                  multiple switches, both will be accepted if any one is included
                  in this list.
              input_map (dict of str): Dictionary mapping file extensions to input
                  filetypes. This is used to automatically assign positional
                  source arguments to ['input', ...] keypaths based on their file
                  extension. If None, the CLI will not accept positional source
                  arguments.
  
          Examples:
              >>> chip.create_cmdline(progname='sc-show',switchlist=['-input','-cfg'])
              Creates a command line interface for 'sc-show' app.
  
              >>> chip.create_cmdline(progname='sc', input_map={'v': 'verilog'})
              All sources ending in .v will be stored in ['input', 'verilog']
          """
  
          # Argparse
          parser = argparse.ArgumentParser(prog=progname,
                                           prefix_chars='-+',
                                           formatter_class=argparse.RawDescriptionHelpFormatter,
                                           description=description)
  
          # Get all keys from global dictionary or override at command line
          allkeys = self.getkeys()
  
          # Iterate over all keys to add parser arguments
          for keypath in allkeys:
              #Fetch fields from leaf cell
              helpstr = self.get(*keypath, field='shorthelp')
              typestr = self.get(*keypath, field='type')
  
              # argparse 'dest' must be a string, so join keypath with commas
              dest = '_'.join(keypath)
  
              switchstrs, metavar = self._get_switches(*keypath)
  
              # Three switch types (bool, list, scalar)
              if not switchlist or any(switch in switchlist for switch in switchstrs):
                  if typestr == 'bool':
                      parser.add_argument(*switchstrs,
                                          nargs='?',
                                          metavar=metavar,
                                          dest=dest,
                                          const='true',
                                          help=helpstr,
                                          default=argparse.SUPPRESS)
                  #list type arguments
                  elif re.match(r'\[', typestr):
                      #all the rest
                      parser.add_argument(*switchstrs,
                                          metavar=metavar,
                                          dest=dest,
                                          action='append',
                                          help=helpstr,
                                          default=argparse.SUPPRESS)
                  else:
                      #all the rest
                      parser.add_argument(*switchstrs,
                                          metavar=metavar,
                                          dest=dest,
                                          help=helpstr,
                                          default=argparse.SUPPRESS)
  
          if input_map is not None:
              parser.add_argument('source',
                                  nargs='*',
                                  help='Input files with filetype inferred by extension')
  
          #Preprocess sys.argv to enable linux commandline switch formats
          #(gcc, verilator, etc)
          scargs = []
  
          # Iterate from index 1, otherwise we end up with script name as a
          # 'source' positional argument
          for item in sys.argv[1:]:
              #Split switches with one character and a number after (O0,O1,O2)
              opt = re.match(r'(\-\w)(\d+)', item)
              #Split assign switches (-DCFG_ASIC=1)
              assign = re.search(r'(\-\w)(\w+\=\w+)', item)
              #Split plusargs (+incdir+/path)
              plusarg = re.search(r'(\+\w+\+)(.*)', item)
              if opt:
                  scargs.append(opt.group(1))
                  scargs.append(opt.group(2))
              elif plusarg:
                  scargs.append(plusarg.group(1))
                  scargs.append(plusarg.group(2))
              elif assign:
                  scargs.append(assign.group(1))
                  scargs.append(assign.group(2))
              else:
                  scargs.append(item)
  
          parser.add_argument('-version', action='version', version=_metadata.version)
  
          #Grab argument from pre-process sysargs
          cmdargs = vars(parser.parse_args(scargs))
  
          # Print banner
          print(_metadata.banner)
          print("Authors:", ", ".join(_metadata.authors))
          print("Version:", _metadata.version, "\n")
          print("-"*80)
  
          os.environ["COLUMNS"] = '80'
  
          # 1. set loglevel if set at command line
          if 'option_loglevel' in cmdargs.keys():
              self.logger.setLevel(cmdargs['option_loglevel'])
  
          # 2. read in target if set
          if 'option_target' in cmdargs.keys():
              if 'arg_pdk' in cmdargs.keys():
                  raise NotImplementedError("NOT IMPLEMENTED: ['arg', 'pdk'] parameter with target")
              if 'arg_flow' in cmdargs.keys():
                  raise NotImplementedError("NOT IMPLEMENTED: ['arg', 'flow'] parameter with target")
              if 'fpga_partname' in cmdargs.keys():
                  self.set('fpga', 'partname', cmdargs['fpga_partname'], clobber=True)
              # running target command
              self.load_target(cmdargs['option_target'])
  
          # 4. read in all cfg files
          if 'option_cfg' in cmdargs.keys():
              for item in cmdargs['option_cfg']:
                  self.read_manifest(item, clobber=True, clear=True)
  
          # Map sources to ['input'] keypath.
          if 'source' in cmdargs:
              for source in cmdargs['source']:
                  _, ext = os.path.splitext(source)
                  ext = ext.lstrip('.')
                  if ext in input_map:
                      filetype = input_map[ext]
                      if self.valid('input', filetype, quiet=True):
                          self.add('input', filetype, source)
                      else:
                          self.set('input', filetype, source)
                      self.logger.info(f'Source {source} inferred as {filetype}')
                  else:
                      self.logger.warning('Unable to infer input type for '
                          f'{source} based on file extension, ignoring. Use the '
                          '-input flag to provide it explicitly.')
              # we don't want to handle this in the next loop
              del cmdargs['source']
  
          # 5. Cycle through all command args and write to manifest
          for dest, vals in cmdargs.items():
              keypath = dest.split('_')
  
              # Turn everything into a list for uniformity
              if not isinstance(vals, list):
                  vals = [vals]
  
              # Cycle through all items
              for item in vals:
                  # Hack to handle the fact that we want optmode stored with an 'O'
                  # prefix.
                  if keypath == ['option', 'optmode']:
                      item = 'O' + item
  
                  num_free_keys = keypath.count('default')
  
                  if len(item.split(' ')) < num_free_keys + 1:
                      # Error out if value provided doesn't have enough words to
                      # fill in 'default' keys.
                      switches, metavar = self._get_switches(*keypath)
                      switchstr = '/'.join(switches)
                      self.logger.error(f'Invalid value {item} for switch {switchstr}. Expected format {metavar}.')
                      raise SiliconCompilerError('Invalid CLI arguments')
  
                  # We replace 'default' in keypath with first N words in provided
                  # value. Remainder is the actual value we want to store in the
                  # parameter.
                  *free_keys, val = item.split(' ', num_free_keys)
                  args = [free_keys.pop(0) if key == 'default' else key for key in keypath]
  
                  # Storing in manifest
                  self.logger.info(f"Command line argument entered: {args} Value: {val}")
                  typestr = self.get(*keypath, field='type')
                  if typestr.startswith('['):
                      if self.valid(*args, quiet=True):
                          self.add(*args, val)
                      else:
                          self.set(*args, val, clobber=True)
                  else:
                      self.set(*args, val, clobber=True)
  
      #########################################################################
      def find_function(self, modulename, funcname, moduletype=None):
          '''
          Returns a function attribute from a module on disk.
  
          Searches the SC root directory and the 'scpath' parameter for the
          modulename provided and imports the module if found. If the funcname
          provided is found in the module, a callable function attribute is
          returned, otherwise None is returned.
  
          The function assumes the following directory structure:
  
          * tools/modulename/modulename.py
          * flows/modulename.py
          * pdks/modulname.py
  
          If the moduletype is None, the module paths are search in the
          order: 'targets'->'flows'->'tools'->'pdks'->'libs'->'checklists'):
  
  
          Supported functions include:
  
          * targets (make_docs, setup)
          * pdks (make_docs, setup)
          * flows (make_docs, setup)
          * tools (make_docs, setup, check_version, runtime_options,
            pre_process, post_process)
          * libs (make_docs, setup)
  
          Args:
              modulename (str): Name of module to import.
              funcname (str): Name of the function to find within the module.
              moduletype (str): Type of module (flows, pdks, libs, checklists, targets).
  
          Examples:
              >>> setup_pdk = chip.find_function('freepdk45', 'setup', 'pdks')
              >>> setup_pdk()
              Imports the freepdk45 module and runs the setup_pdk function
  
          '''
  
          # module search path depends on modtype
          if moduletype is None:
              for item in ('targets', 'flows', 'tools', 'pdks', 'libs', 'checklists'):
                  fullpath = self.find_function(modulename, funcname, module_type=item)
                  if fullpath:
                      break
                  self.logger.error(f"Could not find module {modulename}")
                  self.error = 1
                  return None
          elif moduletype in ('targets','flows', 'pdks', 'libs'):
              fullpath = self._find_sc_file(f"{moduletype}/{modulename}.py", missing_ok=True)
          elif moduletype in ('tools', 'checklists'):
              fullpath = self._find_sc_file(f"{moduletype}/{modulename}/{modulename}.py", missing_ok=True)
          else:
              self.logger.error(f"Illegal module type '{moduletype}'.")
              self.error = 1
              return
  
          # try loading module if found
          if fullpath:
              self.logger.debug(f"Loading function '{funcname}' from module '{modulename}'")
  
              try:
                  spec = importlib.util.spec_from_file_location(modulename, fullpath)
                  imported = importlib.util.module_from_spec(spec)
                  spec.loader.exec_module(imported)
  
                  if hasattr(imported, funcname):
                      function = getattr(imported, funcname)
                  else:
                      function = None
                  return function
              except:
                  traceback.print_exc()
                  self.logger.error(f"Module setup failed for '{modulename}'")
                  self.error = 1
  
      ##########################################################################
      def load_target(self, name):
          """
          Loads a target module and runs the setup() function.
  
          The function searches the $SCPATH for targets/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
              flow (str): Target flow to
  
          Examples:
              >>> chip.load_target('freepdk45_demo')
              Loads the 'freepdk45_demo' target
  
          """
  
          self.set('option', 'target', name)
  
          func = self.find_function(name, 'setup', 'targets')
          if func is not None:
              func(self)
          else:
              self.logger.error(f'Target module {name} not found in $SCPATH or siliconcompiler/targets/.')
              raise SiliconCompilerError(f'Target module {name} not found $SCPATH or siliconcompiler/targets/.')
  
      ##########################################################################
      def load_pdk(self, name):
          """
          Loads a PDK module and runs the setup() function.
  
          The function searches the $SCPATH for pdks/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
  
          Examples:
              >>> chip.load_pdk('freepdk45_pdk')
              Loads the 'freepdk45' pdk
  
          """
  
          func = self.find_function(name, 'setup', 'pdks')
          if func is not None:
              self.logger.info(f"Loading PDK '{name}'")
              self._loaded_modules['pdks'].append(name)
              func(self)
          else:
              self.logger.error(f'PDK module {name} not found in $SCPATH or siliconcompiler/pdks/.')
              raise SiliconCompilerError(f'PDK module {name} not found in $SCPATH or siliconcompiler/pdks/.')
  
      ##########################################################################
      def load_flow(self, name):
          """
          Loads a flow  module and runs the setup() function.
  
          The function searches the $SCPATH for flows/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
  
          Examples:
              >>> chip.load_flow('asicflow')
              Loads the 'asicflow' flow
  
          """
  
          func = self.find_function(name, 'setup', 'flows')
          if func is not None:
              self.logger.info(f"Loading flow '{name}'")
              self._loaded_modules['flows'].append(name)
              func(self)
          else:
              self.logger.error(f'Flow module {name} not found in $SCPATH or siliconcompiler/flows/.')
              raise SiliconCompilerError(f'Flow module {name} not found in $SCPATH or siliconcompiler/flows/.')
  
      ##########################################################################
      def load_lib(self, name):
          """
          Loads a library module and runs the setup() function.
  
          The function searches the $SCPATH for libs/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
  
          Examples:
              >>> chip.load_lib('nangate45')
              Loads the 'nangate45' library
  
          """
  
          func = self.find_function(name, 'setup', 'libs')
          if func is not None:
              self.logger.info(f"Loading library '{name}'")
              self._loaded_modules['libs'].append(name)
              func(self)
          else:
              self.logger.error(f'Library module {name} not found in $SCPATH or siliconcompiler/libs/.')
              raise SiliconCompilerError(f'Library module {name} not found in $SCPATH or siliconcompiler/libs/.')
  
      ##########################################################################
      def load_checklist(self, name):
          """
          Loads a checklist module and runs the setup() function.
  
          The function searches the $SCPATH for checklist/<name>/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
  
          Examples:
              >>> chip.load_checklist('oh_tapeout')
              Loads the 'oh_tapeout' checklist
  
          """
  
          func = self.find_function(name, 'setup', 'checklists')
          if func is not None:
              self.logger.info(f"Loading checklist '{name}'")
              self._loaded_modules['checklists'].append(name)
              func(self)
          else:
              self.logger.error(f'Checklist module {name} not found in $SCPATH or siliconcompiler/checklists/.')
              raise SiliconCompilerError(f'Checklist module {name} not found in $SCPATH or siliconcompiler/checklists/.')
  
  
      ###########################################################################
      def list_metrics(self):
          '''
          Returns a list of all metrics in the schema.
  
          '''
  
          return self.getkeys('metric','default','default')
  
      ###########################################################################
      def help(self, *keypath):
          """
          Returns a schema parameter description.
  
          Args:
              *keypath(str): Keypath to parameter.
  
          Returns:
              A formatted multi-line help paragraph for the parameter provided.
  
          Examples:
              >>> print(chip.help('asic','diearea'))
              Displays help information about the 'asic, diearea' parameter
  
          """
  
          self.logger.debug('Fetching help for %s', keypath)
  
          #Fetch Values
  
          description = self.get(*keypath, field='shorthelp')
          typestr = self.get(*keypath, field='type')
          switchstr = str(self.get(*keypath, field='switch'))
          defstr = str(self.get(*keypath, field='defvalue'))
          requirement = str(self.get(*keypath, field='require'))
          helpstr = self.get(*keypath, field='help')
          example = self.get(*keypath, field='example')
  
          examplestr = ("\nExamples:    " + example[0] + ''.join(
                       ["\n             " + ex for ex in example[1:]]))
  
          #Removing multiple spaces and newlines
          helpstr = helpstr.rstrip()
          helpstr = helpstr.replace("\n", "")
          helpstr = ' '.join(helpstr.split())
  
          for idx, item in enumerate(example):
              example[idx] = ' '.join(item.split())
              example[idx] = example[idx].replace(", ", ",")
  
          #Wrap text
          para = textwrap.TextWrapper(width=60)
          para_list = para.wrap(text=helpstr)
  
          #Full Doc String
          fullstr = ("-"*80 +
                     "\nDescription: " + description +
                     "\nSwitch:      " + switchstr +
                     "\nType:        " + typestr  +
                     "\nRequirement: " + requirement   +
                     "\nDefault:     " + defstr   +
                     examplestr +
                     "\nHelp:        " + para_list[0] + "\n")
          for line in para_list[1:]:
              fullstr = (fullstr +
                         " "*13 + line.lstrip() + "\n")
  
          return fullstr
  
  
      ###########################################################################
      def valid(self, *args, valid_keypaths=None, quiet=True, default_valid=False):
          """
          Checks validity of a keypath.
  
          Checks the validity of a parameter keypath and returns True if the
          keypath is valid and False if invalid.
  
          Args:
              keypath(list str): Variable length schema key list.
              valid_keypaths (list of list): List of valid keypaths as lists. If
                  None, check against all keypaths in the schema.
              quiet (bool): If True, don't display warnings for invalid keypaths.
  
          Returns:
              Boolean indicating validity of keypath.
  
          Examples:
              >>> check = chip.valid('design')
              Returns True.
              >>> check = chip.valid('blah')
              Returns False.
          """
  
          keypathstr = ','.join(args)
          keylist = list(args)
          if default_valid:
              default = 'default'
          else:
              default = None
  
          if valid_keypaths is None:
              valid_keypaths = self.getkeys()
  
          # Look for a full match with default playing wild card
          for valid_keypath in valid_keypaths:
              if len(keylist) != len(valid_keypath):
                  continue
  
              ok = True
              for i in range(len(keylist)):
                  if valid_keypath[i] not in (keylist[i], default):
                      ok = False
                      break
              if ok:
                  return True
  
          # Match not found
          if not quiet:
              self.logger.warning(f"Keypath [{keypathstr}] is not valid")
          return False
  
      ###########################################################################
      def get(self, *keypath, field='value', job=None, cfg=None):
          """
          Returns a schema parameter field.
  
          Returns a schema parameter field based on the keypath provided in the
          ``*keypath``. See the :ref:`Schema Reference Manual<SiliconCompiler
          Schema>` for documentation of all supported keypaths. The returned type
          is consistent with the type field of the parameter. Fetching parameters
          with empty or undefined value files returns None for scalar types and []
          (empty list) for list types.  Accessing a non-existent keypath produces
          a logger error message and raises the Chip object error flag.
  
          Args:
              keypath(list str): Variable length schema key list.
              field(str): Parameter field to fetch.
              job (str): Jobname to use for dictionary access in place of the
                  current active jobname.
              cfg(dict): Alternate dictionary to access in place of the default
                  chip object schema dictionary.
  
          Returns:
              Value found for the keypath and field provided.
  
          Examples:
              >>> foundry = chip.get('pdk', 'foundry')
              Returns the name of the foundry from the PDK.
  
          """
  
          if cfg is None:
              if job is not None:
                  cfg = self.cfg['history'][job]
              else:
                  cfg = self.cfg
  
          keypathstr = ','.join(keypath)
  
          self.logger.debug(f"Reading from [{keypathstr}]. Field = '{field}'")
          return self._search(cfg, keypathstr, *keypath, field=field, mode='get')
  
      ###########################################################################
      def getkeys(self, *keypath, cfg=None, job=None):
          """
          Returns a list of schema dictionary keys.
  
          Searches the schema for the keypath provided and returns a list of
          keys found, excluding the generic 'default' key. Accessing a
          non-existent keypath produces a logger error message and raises the
          Chip object error flag.
  
          Args:
              keypath (list str): Variable length ordered schema key list
              cfg (dict): Alternate dictionary to access in place of self.cfg
              job (str): Jobname to use for dictionary access in place of the
                  current active jobname.
  
          Returns:
              List of keys found for the keypath provided.
  
          Examples:
              >>> keylist = chip.getkeys('pdk')
              Returns all keys for the 'pdk' keypath.
              >>> keylist = chip.getkeys()
              Returns all list of all keypaths in the schema.
          """
  
          if cfg is None:
              if job is None:
                  cfg = self.cfg
              else:
                  cfg = self.cfg['history'][job]
  
          if len(list(keypath)) > 0:
              keypathstr = ','.join(keypath)
              self.logger.debug('Getting schema parameter keys for: %s', keypathstr)
              keys = list(self._search(cfg, keypathstr, *keypath, mode='getkeys'))
              if 'default' in keys:
                  keys.remove('default')
          else:
              self.logger.debug('Getting all schema parameter keys.')
              keys = list(self._allkeys(cfg))
  
          return keys
  
      ###########################################################################
      def getdict(self, *keypath, cfg=None):
          """
          Returns a schema dictionary.
  
          Searches the schema for the keypath provided and returns a complete
          dictionary. Accessing a non-existent keypath produces a logger error
          message and raises the Chip object error flag.
  
          Args:
              keypath(list str): Variable length ordered schema key list
              cfg(dict): Alternate dictionary to access in place of self.cfg
  
          Returns:
              A schema dictionary
  
          Examples:
              >>> pdk = chip.getdict('pdk')
              Returns the complete dictionary found for the keypath 'pdk'
          """
  
          if cfg is None:
              cfg = self.cfg
  
          if len(list(keypath)) > 0:
              keypathstr = ','.join(keypath)
              self.logger.debug('Getting cfg for: %s', keypathstr)
              localcfg = self._search(cfg, keypathstr, *keypath, mode='getcfg')
  
          return copy.deepcopy(localcfg)
  
      ###########################################################################
      def set(self, *args, field='value', clobber=True, cfg=None):
          '''
          Sets a schema parameter field.
  
          Sets a schema parameter field based on the keypath and value provided in
          the ``*args``. See the :ref:`Schema Reference Manual<SiliconCompiler
          Schema>` for documentation of all supported keypaths. New schema
          dictionaries are automatically created for keypaths that overlap with
          'default' dictionaries. The write action is ignored if the parameter
          value is non-empty and the clobber option is set to False.
  
          The value provided must agree with the dictionary parameter 'type'.
          Accessing a non-existent keypath or providing a value that disagrees
          with the parameter type produces a logger error message and raises the
          Chip object error flag.
  
          Args:
              args (list): Parameter keypath followed by a value to set.
              field (str): Parameter field to set.
              clobber (bool): Existing value is overwritten if True.
              cfg(dict): Alternate dictionary to access in place of self.cfg
  
          Examples:
              >>> chip.set('design', 'top')
              Sets the name of the design to 'top'
          '''
  
          if cfg is None:
              cfg = self.cfg
  
          # Verify that all keys are strings
          for key in args[:-1]:
              if not isinstance(key,str):
                  self.logger.error(f"Key [{key}] is not a string [{args}]")
  
          keypathstr = ','.join(args[:-1])
          all_args = list(args)
  
          # Special case to ensure loglevel is updated ASAP
          if len(args) == 3 and args[1] == 'loglevel' and field == 'value':
              self.logger.setLevel(args[2])
  
          self.logger.debug(f"Setting [{keypathstr}] to {args[-1]}")
          return self._search(cfg, keypathstr, *all_args, field=field, mode='set', clobber=clobber)
  
      ###########################################################################
      def add(self, *args, cfg=None, field='value'):
          '''
          Adds item(s) to a schema parameter list.
  
          Adds item(s) to schema parameter list based on the keypath and value
          provided in the ``*args``.  See the :ref:`Schema Reference
          Manual<SiliconCompiler Schema>` for documentation of all supported
          keypaths. New schema dictionaries are automatically created for keypaths
          that overlap with 'default' dictionaries.
  
          The value provided must agree with the dictionary parameter 'type'.
          Accessing a non-existent keypath, providing a value that disagrees
          with the parameter type, or using add with a scalar parameter produces
          a logger error message and raises the Chip object error flag.
  
          Args:
              args (list): Parameter keypath followed by a value to add.
              cfg(dict): Alternate dictionary to access in place of self.cfg
              field (str): Parameter field to set.
  
          Examples:
              >>> chip.add('source', 'hello.v')
              Adds the file 'hello.v' to the list of sources.
          '''
  
          if cfg is None:
              cfg = self.cfg
  
          # Verify that all keys are strings
          for key in args[:-1]:
              if not isinstance(key,str):
                  self.logger.error(f"Key [{key}] is not a string [{args}]")
  
          keypathstr = ','.join(args[:-1])
          all_args = list(args)
  
          self.logger.debug(f'Appending value {args[-1]} to [{keypathstr}]')
          return self._search(cfg, keypathstr, *all_args, field=field, mode='add')
  
  
      ###########################################################################
      def _allkeys(self, cfg, keys=None, keylist=None):
          '''
          Returns list of all keypaths in the schema.
          '''
  
          if keys is None:
              keylist = []
              keys = []
          for k in cfg:
              newkeys = keys.copy()
              newkeys.append(k)
              if 'defvalue' in cfg[k]:
                  keylist.append(newkeys)
              else:
                  self._allkeys(cfg[k], keys=newkeys, keylist=keylist)
          return keylist
  
      ###########################################################################
      def _search(self, cfg, keypath, *args, field='value', mode='get', clobber=True):
          '''
          Internal recursive function that searches the Chip schema for a
          match to the combination of *args and fields supplied. The function is
          used to set and get data within the dictionary.
  
          Args:
              cfg(dict): The cfg schema to search
              keypath (str): Concatenated keypath used for error logging.
              args (str): Keypath/value variable list used for access
              field(str): Leaf cell field to access.
              mode(str): Action (set/get/add/getkeys/getkeys)
              clobber(bool): Specifies to clobber (for set action)
  
          '''
  
          all_args = list(args)
          param = all_args[0]
          val = all_args[-1]
          empty = [None, 'null', [], 'false']
  
          #set/add leaf cell (all_args=(param,val))
          if (mode in ('set', 'add')) & (len(all_args) == 2):
              # clean error if key not found
              if (not param in cfg) & (not 'default' in cfg):
                  self.logger.error(f"Set/Add keypath [{keypath}] does not exist.")
                  self.error = 1
              else:
                  # making an 'instance' of default if not found
                  if (not param in cfg) & ('default' in cfg):
                      cfg[param] = copy.deepcopy(cfg['default'])
                  list_type =bool(re.match(r'\[', cfg[param]['type']))
                  # checking for illegal fields
                  if not field in cfg[param] and (field != 'value'):
                      self.logger.error(f"Field '{field}' for keypath [{keypath}]' is not a valid field.")
                      self.error = 1
                  # check legality of value
                  if field == 'value':
                      (type_ok,type_error) = self._typecheck(cfg[param], param, val)
                      if not type_ok:
                          self.logger.error("%s", type_error)
                          self.error = 1
                  # converting python True/False to lower case string
                  if (field == 'value') and (cfg[param]['type'] == 'bool'):
                      if val == True:
                          val = "true"
                      elif val == False:
                          val = "false"
                  # checking if value has been set
                  # TODO: fix clobber!!
                  selval = cfg[param]['value']
                  # updating values
                  if cfg[param]['lock'] == "true":
                      self.logger.debug("Ignoring {mode}{} to [{keypath}]. Lock bit is set.")
                  elif (mode == 'set'):
                      if (field != 'value') or (selval in empty) or clobber:
                          if field in ('copy', 'lock'):
                              # boolean fields
                              if val is True:
                                  cfg[param][field] = "true"
                              elif val is False:
                                  cfg[param][field] = "false"
                              else:
                                  self.logger.error(f'{field} must be set to boolean.')
                                  self.error = 1
                          elif field in ('hashalgo', 'scope', 'require', 'type', 'unit',
                                         'shorthelp', 'notes', 'switch', 'help'):
                              # awlays string scalars
                              cfg[param][field] = val
                          elif field in ('example'):
                              # list from default schema (already a list)
                              cfg[param][field] = val
                          elif field in ('signature', 'filehash', 'date', 'author'):
                              # convert to list if appropriate
                              if isinstance(val, list) | (not list_type):
                                  cfg[param][field] = val
                              else:
                                  cfg[param][field] = [val]
                          elif (not list_type) & (val is None):
                              # special case for None
                              cfg[param][field] = None
                          elif (not list_type) & (not isinstance(val, list)):
                              # convert to string for scalar value
                              cfg[param][field] = str(val)
                          elif list_type & (not isinstance(val, list)):
                              # convert to string for list value
                              cfg[param][field] = [str(val)]
                          elif list_type & isinstance(val, list):
                              # converting tuples to strings
                              if re.search(r'\(', cfg[param]['type']):
                                  cfg[param][field] = list(map(str,val))
                              else:
                                  cfg[param][field] = val
                          else:
                              self.logger.error(f"Assigning list to scalar for [{keypath}]")
                              self.error = 1
                      else:
                          self.logger.debug(f"Ignoring set() to [{keypath}], value already set. Use clobber=true to override.")
                  elif (mode == 'add'):
                      if field in ('filehash', 'date', 'author', 'signature'):
                          cfg[param][field].append(str(val))
                      elif field in ('copy', 'lock'):
                          self.logger.error(f"Illegal use of add() for scalar field {field}.")
                          self.error = 1
                      elif list_type & (not isinstance(val, list)):
                          cfg[param][field].append(str(val))
                      elif list_type & isinstance(val, list):
                          cfg[param][field].extend(val)
                      else:
                          self.logger.error(f"Illegal use of add() for scalar parameter [{keypath}].")
                          self.error = 1
                  return cfg[param][field]
          #get leaf cell (all_args=param)
          elif len(all_args) == 1:
              if not param in cfg:
                  self.error = 1
                  self.logger.error(f"Get keypath [{keypath}] does not exist.")
              elif mode == 'getcfg':
                  return cfg[param]
              elif mode == 'getkeys':
                  return cfg[param].keys()
              else:
                  if not (field in cfg[param]) and (field!='value'):
                      self.error = 1
                      self.logger.error(f"Field '{field}' not found for keypath [{keypath}]")
                  elif field == 'value':
                      #Select default if no value has been set
                      if field not in cfg[param]:
                          selval = cfg[param]['defvalue']
                      else:
                          selval =  cfg[param]['value']
                      #check for list
                      if bool(re.match(r'\[', cfg[param]['type'])):
                          sctype = re.sub(r'[\[\]]', '', cfg[param]['type'])
                          return_list = []
                          if selval is None:
                              return None
                          for item in selval:
                              if sctype == 'int':
                                  return_list.append(int(item))
                              elif sctype == 'float':
                                  return_list.append(float(item))
                              elif sctype.startswith('(str,'):
                                  if isinstance(item,tuple):
                                      return_list.append(item)
                                  else:
                                      tuplestr = re.sub(r'[\(\)\'\s]','',item)
                                      return_list.append(tuple(tuplestr.split(',')))
                              elif sctype.startswith('(float,'):
                                  if isinstance(item,tuple):
                                      return_list.append(item)
                                  else:
                                      tuplestr = re.sub(r'[\(\)\s]','',item)
                                      return_list.append(tuple(map(float, tuplestr.split(','))))
                              else:
                                  return_list.append(item)
                          return return_list
                      else:
                          if selval is None:
                              # Unset scalar of any type
                              scalar = None
                          elif cfg[param]['type'] == "int":
                              #print(selval, type(selval))
                              scalar = int(float(selval))
                          elif cfg[param]['type'] == "float":
                              scalar = float(selval)
                          elif cfg[param]['type'] == "bool":
                              scalar = (selval == 'true')
                          elif re.match(r'\(', cfg[param]['type']):
                              tuplestr = re.sub(r'[\(\)\s]','',selval)
                              scalar = tuple(map(float, tuplestr.split(',')))
                          else:
                              scalar = selval
                          return scalar
                  #all non-value fields are strings (or lists of strings)
                  else:
                      if cfg[param][field] == 'true':
                          return True
                      elif cfg[param][field] == 'false':
                          return False
                      else:
                          return cfg[param][field]
          #if not leaf cell descend tree
          else:
              ##copying in default tree for dynamic trees
              if not param in cfg and 'default' in cfg:
                  cfg[param] = copy.deepcopy(cfg['default'])
              elif not param in cfg:
                  self.error = 1
                  self.logger.error(f"Get keypath [{keypath}] does not exist.")
                  return None
              all_args.pop(0)
              return self._search(cfg[param], keypath, *all_args, field=field, mode=mode, clobber=clobber)
  
      ###########################################################################
      def _prune(self, cfg, top=True, keeplists=False):
          '''
          Internal recursive function that creates a local copy of the Chip
          schema (cfg) with only essential non-empty parameters retained.
  
          '''
  
          # create a local copy of dict
          if top:
              localcfg = copy.deepcopy(cfg)
          else:
              localcfg = cfg
  
          #10 should be enough for anyone...
          maxdepth = 10
          i = 0
  
          #Prune when the default & value are set to the following
          if keeplists:
              empty = ("null", None)
          else:
              empty = ("null", None, [])
  
          # When at top of tree loop maxdepth times to make sure all stale
          # branches have been removed, not elegant, but stupid-simple
          # "good enough"
          while i < maxdepth:
              #Loop through all keys starting at the top
              for k in list(localcfg.keys()):
                  #removing all default/template keys
                  # reached a default subgraph, delete it
                  if k == 'default':
                      del localcfg[k]
                  # reached leaf-cell
                  elif 'help' in localcfg[k].keys():
                      del localcfg[k]['help']
                  elif 'example' in localcfg[k].keys():
                      del localcfg[k]['example']
                  elif 'defvalue' in localcfg[k].keys():
                      if localcfg[k]['defvalue'] in empty:
                          if 'value' in localcfg[k].keys():
                              if localcfg[k]['value'] in empty:
                                  del localcfg[k]
                          else:
                              del localcfg[k]
                  #removing stale branches
                  elif not localcfg[k]:
                      localcfg.pop(k)
                  #keep traversing tree
                  else:
                      self._prune(cfg=localcfg[k], top=False, keeplists=keeplists)
              if top:
                  i += 1
              else:
                  break
  
          return localcfg
  
      ###########################################################################
      def _find_sc_file(self, filename, missing_ok=False):
          """
          Returns the absolute path for the filename provided.
  
          Searches the SC root directory and the 'scpath' parameter for the
          filename provided and returns the absolute path. If no valid absolute
          path is found during the search, None is returned.
  
          Shell variables ('$' followed by strings consisting of numbers,
          underscores, and digits) are replaced with the variable value.
  
          Args:
              filename (str): Relative or absolute filename.
  
          Returns:
              Returns absolute path of 'filename' if found, otherwise returns
              None.
  
          Examples:
              >>> chip._find_sc_file('flows/asicflow.py')
             Returns the absolute path based on the sc installation directory.
  
          """
  
          # Replacing environment variables
          filename = self._resolve_env_vars(filename)
  
          # If we have a path relative to our cwd or an abs path, pass-through here
          if os.path.exists(os.path.abspath(filename)):
              return os.path.abspath(filename)
  
          # Otherwise, search relative to scpaths
          scpaths = [self.scroot, self.cwd]
          scpaths.extend(self.get('option', 'scpath'))
          if 'SCPATH' in os.environ:
              scpaths.extend(os.environ['SCPATH'].split(os.pathsep))
  
          searchdirs = ', '.join(scpaths)
          self.logger.debug(f"Searching for file {filename} in {searchdirs}")
  
          result = None
          for searchdir in scpaths:
              if not os.path.isabs(searchdir):
                  searchdir = os.path.join(self.cwd, searchdir)
  
              abspath = os.path.abspath(os.path.join(searchdir, filename))
              if os.path.exists(abspath):
                  result = abspath
                  break
  
          if result is None and not missing_ok:
              self.error = 1
              self.logger.error(f"File {filename} was not found")
  
          return result
  
      ###########################################################################
      def find_files(self, *keypath, cfg=None, missing_ok=False, job=None):
          """
          Returns absolute paths to files or directories based on the keypath
          provided.
  
          By default, this function first checks if the keypath provided has its
          `copy` parameter set to True. If so, it returns paths to the files in
          the build directory. Otherwise, it resolves these files based on the
          current working directory and SC path.
  
          The keypath provided must point to a schema parameter of type file, dir,
          or lists of either. Otherwise, it will trigger an error.
  
          Args:
              keypath (list str): Variable length schema key list.
              cfg (dict): Alternate dictionary to access in place of the default
                  chip object schema dictionary.
              missing_ok (bool): If True, silently return None when files aren't
                  found. If False, print an error and set the error flag.
              job (str): Jobname to use for dictionary access in place of the
                  current active jobname.
  
          Returns:
              If keys points to a scalar entry, returns an absolute path to that
              file/directory, or None if not found. It keys points to a list
              entry, returns a list of either the absolute paths or None for each
              entry, depending on whether it is found.
  
          Examples:
              >>> chip.find_files('source')
              Returns a list of absolute paths to source files, as specified in
              the schema.
  
          """
          if cfg is None:
              cfg = self.cfg
  
          copyall = self.get('option', 'copyall', cfg=cfg, job=job)
          paramtype = self.get(*keypath, field='type', cfg=cfg, job=job)
  
          if 'file' in paramtype:
              copy = self.get(*keypath, field='copy', cfg=cfg, job=job)
          else:
              copy = False
  
          if 'file' not in paramtype and 'dir' not in paramtype:
              self.logger.error('Can only call find_files on file or dir types')
              self.error = 1
              return None
  
          is_list = bool(re.match(r'\[', paramtype))
  
          paths = self.get(*keypath, cfg=cfg, job=job)
          # Convert to list if we have scalar
          if not is_list:
              paths = [paths]
  
          result = []
  
          # Special cases for various ['eda', ...] files that may be implicitly
          # under the workdir (or refdir in the case of scripts).
          # TODO: it may be cleaner to have a file resolution scope flag in schema
          # (e.g. 'scpath', 'workdir', 'refdir'), rather than harcoding special
          # cases.
          if keypath[0] == 'tool' and keypath[2] in ('input', 'output', 'report'):
              step = keypath[3]
              index = keypath[4]
              if keypath[2] == 'report':
                  io = ""
              else:
                  io = keypath[2] + 's'
              iodir = os.path.join(self._getworkdir(jobname=job, step=step, index=index), io)
              for path in paths:
                  abspath = os.path.join(iodir, path)
                  if os.path.isfile(abspath):
                      result.append(abspath)
              return result
          elif keypath[0] == 'tool' and keypath[2] == 'script':
              tool = keypath[1]
              step = keypath[3]
              index = keypath[4]
              refdirs = self.find_files('tool', tool, 'refdir', step, index)
              for path in paths:
                  for refdir in refdirs:
                      abspath = os.path.join(refdir, path)
                      if os.path.isfile(abspath):
                          result.append(abspath)
                          break
  
              return result
  
          for path in paths:
              if (copyall or copy) and ('file' in paramtype):
                  name = self._get_imported_filename(path)
                  abspath = os.path.join(self._getworkdir(jobname=job, step='import'), 'outputs', name)
                  if os.path.isfile(abspath):
                      # if copy is True and file is found in import outputs,
                      # continue. Otherwise, fall through to _find_sc_file (the
                      # file may not have been gathered in imports yet)
                      result.append(abspath)
                      continue
              result.append(self._find_sc_file(path, missing_ok=missing_ok))
          # Convert back to scalar if that was original type
          if not is_list:
              return result[0]
  
          return result
  
      ###########################################################################
      def find_result(self, filetype, step, jobname=None, index='0'):
          """
          Returns the absolute path of a compilation result.
  
          Utility function that returns the absolute path to a results
          file based on the provided arguments. The result directory
          structure is:
  
          <dir>/<design>/<jobname>/<step>/<index>/outputs/<design>.filetype
  
          Args:
              filetype (str): File extension (.v, .def, etc)
              step (str): Task step name ('syn', 'place', etc)
              jobname (str): Jobid directory name
              index (str): Task index
  
          Returns:
              Returns absolute path to file.
  
          Examples:
              >>> manifest_filepath = chip.find_result('.vg', 'syn')
             Returns the absolute path to the manifest.
          """
          if jobname is None:
              jobname = self.get('option', 'jobname')
  
          workdir = self._getworkdir(jobname, step, index)
          design = self.get('design')
          filename = f"{workdir}/outputs/{design}.{filetype}"
  
          self.logger.debug("Finding result %s", filename)
  
          if os.path.isfile(filename):
              return filename
          else:
              return None
  
      ###########################################################################
      def _abspath(self, cfg):
          '''
          Internal function that goes through provided dictionary and resolves all
          relative paths where required.
          '''
  
          for keypath in self.getkeys(cfg=cfg):
              paramtype = self.get(*keypath, cfg=cfg, field='type')
              value = self.get(*keypath, cfg=cfg)
              if value:
                  #only do something if type is file or dir
                  if 'file' in paramtype or 'dir' in paramtype:
                      abspaths = self.find_files(*keypath, cfg=cfg, missing_ok=True)
                      self.set(*keypath, abspaths, cfg=cfg)
  
      ###########################################################################
      def _print_csv(self, cfg, fout=None):
          allkeys = self.getkeys(cfg=cfg)
          for key in allkeys:
              keypath = f'"{",".join(key)}"'
              value = self.get(*key, cfg=cfg)
              if isinstance(value,list):
                  for item in value:
                      fout.write(f"{keypath},{item}")
              else:
                  fout.write(f"{keypath},{value}")
  
      ###########################################################################
      def _escape_val_tcl(self, val, typestr):
          '''Recursive helper function for converting Python values to safe TCL
          values, based on the SC type string.'''
          if val is None:
              return ''
          elif typestr.startswith('('):
              # Recurse into each item of tuple
              subtypes = typestr.strip('()').split(',')
              valstr = ' '.join(self._escape_val_tcl(v, subtype.strip())
                                for v, subtype in zip(val, subtypes))
              return f'[list {valstr}]'
          elif typestr.startswith('['):
              # Recurse into each item of list
              subtype = typestr.strip('[]')
              valstr = ' '.join(self._escape_val_tcl(v, subtype) for v in val)
              return f'[list {valstr}]'
          elif typestr == 'bool':
              return 'true' if val else 'false'
          elif typestr == 'str':
              # Escape string by surrounding it with "" and escaping the few
              # special characters that still get considered inside "". We don't
              # use {}, since this requires adding permanent backslashes to any
              # curly braces inside the string.
              # Source: https://www.tcl.tk/man/tcl8.4/TclCmd/Tcl.html (section [4] on)
              escaped_val = (val.replace('\\', '\\\\') # escape '\' to avoid backslash substition (do this first, since other replaces insert '\')
                                .replace('[', '\\[')   # escape '[' to avoid command substition
                                .replace('$', '\\$')   # escape '$' to avoid variable substition
                                .replace('"', '\\"'))  # escape '"' to avoid string terminating early
              return '"' + escaped_val + '"'
          elif typestr in ('file', 'dir'):
              # Replace $VAR with $env(VAR) for tcl
              val = re.sub(r'\$(\w+)', r'$env(\1)', val)
              # Same escapes as applied to string, minus $ (since we want to resolve env vars).
              escaped_val = (val.replace('\\', '\\\\') # escape '\' to avoid backslash substition (do this first, since other replaces insert '\')
                                .replace('[', '\\[')   # escape '[' to avoid command substition
                                .replace('"', '\\"'))  # escape '"' to avoid string terminating early
              return '"' +  escaped_val + '"'
          else:
              # floats/ints just become strings
              return str(val)
  
      ###########################################################################
      def _print_tcl(self, cfg, fout=None, prefix=""):
          '''
          Prints out schema as TCL dictionary
          '''
  
          fout.write("#############################################")
          fout.write("#!!!! AUTO-GENERATED FILE. DO NOT EDIT!!!!!!")
          fout.write("#############################################\n")
  
          allkeys = self.getkeys(cfg=cfg)
  
          for key in allkeys:
              typestr = self.get(*key, cfg=cfg, field='type')
              value = self.get(*key, cfg=cfg)
  
              #create a TCL dict
              keystr = ' '.join(key)
  
              valstr = self._escape_val_tcl(value, typestr)
  
              if not (typestr.startswith('[') or typestr.startswith('(')):
                  # treat scalars as lists as well
                  valstr = f'[list {valstr}]'
  
              outstr = f"{prefix} {keystr} {valstr}\n"
  
              #print out all non default values
              if 'default' not in key:
                  fout.write(outstr)
  
      ###########################################################################
  
      def merge_manifest(self, cfg, job=None, clobber=True, clear=True, check=False):
          """
          Merges an external manifest with the current compilation manifest.
  
          All value fields in the provided schema dictionary are merged into the
          current chip object. Dictionaries with non-existent keypath produces a
          logger error message and raises the Chip object error flag.
  
          Args:
              job (str): Specifies non-default job to merge into
              clear (bool): If True, disables append operations for list type
              clobber (bool): If True, overwrites existing parameter value
              check (bool): If True, checks the validity of each key
              partial (bool): If True, perform a partial merge, only merging
                  keypaths that may have been updated during run().
  
          Examples:
              >>> chip.merge_manifest('my.pkg.json')
             Merges all parameters in my.pk.json into the Chip object
  
          """
          self._merge_manifest(cfg, job, clobber, clear, check)
  
      def _key_may_be_updated(self, keypath):
          '''Helper that returns whether `keypath` can be updated mid-run.'''
          # TODO: cleaner way to manage this?
          if keypath[0] in ('metric', 'record'):
              return True
          if keypath[0] == 'flowgraph' and keypath[4] in ('select', 'status'):
              return True
          return False
  
      ###########################################################################
      def _merge_manifest(self, cfg, job=None, clobber=True, clear=True, check=False, partial=False):
          """
          Internal merge_manifest() implementation with `partial` arg.
  
          partial (bool): If True, perform a partial merge, only merging keypaths
          that may have been updated during run().
          """
          if job is not None:
              # fill ith default schema before populating
              self.cfg['history'][job] = schema_cfg()
              dst = self.cfg['history'][job]
          else:
              dst = self.cfg
  
          for keylist in self.getkeys(cfg=cfg):
              if partial and not self._key_may_be_updated(keylist):
                  continue
              if keylist[0] in ('history', 'library'):
                  continue
              #only read in valid keypaths without 'default'
              key_valid = True
              if check:
                  key_valid = self.valid(*keylist, quiet=False, default_valid=True)
              if key_valid and 'default' not in keylist:
                  # update value, handling scalars vs. lists
                  typestr = self.get(*keylist, cfg=cfg, field='type')
                  val = self.get(*keylist, cfg=cfg)
                  arg = keylist.copy()
                  arg.append(val)
                  if bool(re.match(r'\[', typestr)) & bool(not clear):
                      self.add(*arg, cfg=dst)
                  else:
                      self.set(*arg, cfg=dst, clobber=clobber)
  
                  # update other fields that a user might modify
                  for field in self.getdict(*keylist, cfg=cfg).keys():
                      if field in ('value', 'switch', 'type', 'require', 'defvalue',
                                   'shorthelp', 'example', 'help'):
                          # skip these fields (value handled above, others are static)
                          continue
                      v = self.get(*keylist, cfg=cfg, field=field)
                      self.set(*keylist, v, cfg=dst, field=field)
  
      ###########################################################################
      def _keypath_empty(self, key):
          '''
          Utility function to check key for an empty list.
          '''
  
          emptylist = ("null", None, [])
  
          value = self.get(*key)
          defvalue = self.get(*key, field='defvalue')
          value_empty = (defvalue in emptylist) and (value in emptylist)
  
          return value_empty
  
      ###########################################################################
      def _check_files(self):
          allowed_paths = [os.path.join(self.cwd, self.get('option', 'builddir'))]
          allowed_paths.extend(os.environ['SC_VALID_PATHS'].split(os.pathsep))
  
          for keypath in self.getkeys():
              if 'default' in keypath:
                  continue
  
              paramtype = self.get(*keypath, field='type')
              #only do something if type is file or dir
              if ('history' not in keypath and 'library' not in keypath) and ('file' in paramtype or 'dir' in paramtype):
  
                  if self.get(*keypath) is None:
                      # skip unset values (some directories are None by default)
                      continue
  
                  abspaths = self.find_files(*keypath, missing_ok=True)
                  if not isinstance(abspaths, list):
                      abspaths = [abspaths]
  
                  for abspath in abspaths:
                      ok = False
  
                      if abspath is not None:
                          for allowed_path in allowed_paths:
                              if os.path.commonpath([abspath, allowed_path]) == allowed_path:
                                  ok = True
                                  continue
  
                      if not ok:
                          self.logger.error(f'Keypath {keypath} contains path(s) '
                              'that do not exist or resolve to files outside of '
                              'allowed directories.')
                          return False
  
          return True
  
      ###########################################################################
      def check_filepaths(self):
          '''
          Verifies that paths to all files in manifest are valid.
          '''
  
          allkeys = self.getkeys()
          for keypath in allkeys:
              allpaths = []
              paramtype = self.get(*keypath, field='type')
              if 'file' in paramtype or 'dir' in paramtype:
                  if 'dir' not in keypath and self.get(*keypath):
                      allpaths = list(self.get(*keypath))
                  for path in allpaths:
                      #check for env var
                      m = re.match(r'\$(\w+)(.*)', path)
                      if m:
                          prefix_path = os.environ[m.group(1)]
                          path = prefix_path + m.group(2)
                      file_error = 'file' in paramtype and not os.path.isfile(path)
                      dir_error = 'dir' in paramtype and not os.path.isdir(path)
                      if file_error or dir_error:
                          self.logger.error(f"Paramater {keypath} path {path} is invalid")
                          self.error = 1
  
      ###########################################################################
      def _check_manifest_dynamic(self, step, index):
          '''Runtime checks called from _runtask().
  
          - Make sure expected inputs exist.
          - Make sure all required filepaths resolve correctly.
          '''
          flow = self.get('option', 'flow')
          tool = self.get('flowgraph', flow, step, index, 'tool')
          if self.valid('tool', tool, 'input', step, index):
              required_inputs = self.get('tool', tool, 'input', step, index)
          else:
              required_inputs = []
          input_dir = os.path.join(self._getworkdir(step=step, index=index), 'inputs')
          for filename in required_inputs:
              path = os.path.join(input_dir, filename)
              if not os.path.isfile(path):
                  self.logger.error(f'Required input {filename} not received for {step}{index}.')
                  self.error = 1
  
          if (not tool in self.builtin) and self.valid('tool', tool, 'require', step, index):
              all_required = self.get('tool', tool, 'require', step, index)
              for item in all_required:
                  keypath = item.split(',')
                  paramtype = self.get(*keypath, field='type')
                  if ('file' in paramtype) or ('dir' in paramtype):
                      abspath = self.find_files(*keypath)
                      if abspath is None or (isinstance(abspath, list) and None in abspath):
                          self.logger.error(f"Required file keypath {keypath} can't be resolved.")
                          self.error = 1
  
          # Need to run this check here since file resolution can change in
          # _runtask().
          if 'SC_VALID_PATHS' in os.environ:
              if not self._check_files():
                  self.error = 1
  
          return self.error
  
      ###########################################################################
      def check_manifest(self):
          '''
          Verifies the integrity of the pre-run compilation manifest.
  
          Checks the validity of the current schema manifest in
          memory to ensure that the design has been properly set up prior
          to running compilation. The function is called inside the run()
          function but can also be called separately. Checks performed by the
          check_manifest() function include:
  
          * Has a flowgraph been defined?
          * Does the manifest satisfy the schema requirement field settings?
          * Are all flowgraph input names legal step/index pairs?
          * Are the tool parameter setting requirements met?
  
          Returns:
              Returns True if the manifest is valid, else returns False.
  
          Examples:
              >>> manifest_ok = chip.check_manifest()
              Returns True of the Chip object dictionary checks out.
  
          '''
          # Dynamic checks
          # We only perform these if arg, step and arg, index are set.
          # We don't check inputs for skip all
          # TODO: Need to add skip step
  
          cur_step = self.get('arg', 'step')
          cur_index = self.get('arg', 'index')
          if cur_step and cur_index and not self.get('option', 'skipall'):
              return self._check_manifest_dynamic(cur_step, cur_index)
  
          design = self.get('design')
          flow = self.get('option', 'flow')
          jobname = self.get('option', 'jobname')
          steplist = self.get('option', 'steplist')
          if not steplist:
              steplist = self.list_steps()
  
          #1. Checking that flowgraph and steplist are legal
          if flow not in self.getkeys('flowgraph'):
              self.error = 1
              self.logger.error(f"flowgraph {flow} not defined.")
          legal_steps = self.getkeys('flowgraph',flow)
  
          if 'import' not in legal_steps:
              self.error = 1
              self.logger.error("Flowgraph doesn't contain import step.")
  
          indexlist = {}
          #TODO: refactor
          for step in steplist:
              if self.get('option', 'indexlist'):
                  indexlist[step] = self.get('option', 'indexlist')
              else:
                  indexlist[step] = self.getkeys('flowgraph', flow, step)
  
          for step in steplist:
              for index in indexlist[step]:
                  in_job = None
                  if (step in self.getkeys('option', 'jobinput') and
                      index in self.getkeys('option', 'jobinput', step)):
                      in_job = self.get('option', 'jobinput', step, index)
  
                  for in_step, in_index in self.get('flowgraph', flow, step, index, 'input'):
                      if in_job is not None:
                          workdir = self._getworkdir(jobname=in_job, step=in_step, index=in_index)
                          cfg = os.path.join(workdir, 'outputs', f'{design}.pkg.json')
                          if not os.path.isfile(cfg):
                              self.logger.error(f'{step}{index} relies on {in_step}{in_index} from job {in_job}, '
                                  'but this task has not been run.')
                              self.error = 1
                          continue
                      if in_step in steplist and in_index in indexlist[in_step]:
                          # we're gonna run this step, OK
                          continue
                      if self.get('flowgraph', flow, in_step, in_index, 'status') == TaskStatus.SUCCESS:
                          # this task has already completed successfully, OK
                         
  
  [... the rest is omitted ...]
files.new_path: siliconcompiler/core.py
files.old_contents: # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
  
  import argparse
  import base64
  import time
  import datetime
  import multiprocessing
  import tarfile
  import traceback
  import asyncio
  from subprocess import run, PIPE
  import os
  import glob
  import pathlib
  import sys
  import gzip
  import re
  import json
  import logging
  import hashlib
  import shutil
  import copy
  import importlib
  import textwrap
  import math
  import pandas
  import yaml
  import graphviz
  import time
  import uuid
  import shlex
  import platform
  import getpass
  import distro
  import netifaces
  import webbrowser
  import packaging.version
  import packaging.specifiers
  from jinja2 import Environment, FileSystemLoader
  from pathlib import Path
  from timeit import default_timer as timer
  from siliconcompiler.client import *
  from siliconcompiler.schema import *
  from siliconcompiler.scheduler import _deferstep
  from siliconcompiler import leflib
  from siliconcompiler import utils
  from siliconcompiler import _metadata
  import psutil
  
  class TaskStatus():
      # Could use Python 'enum' class here, but that doesn't work nicely with
      # schema.
      PENDING = 'pending'
      SUCCESS = 'success'
      ERROR = 'error'
  
  class Chip:
      """Object for configuring and executing hardware design flows.
  
      This is the main object used for configuration, data, and
      execution within the SiliconCompiler platform.
  
      Args:
          design (string): Name of the top level chip design module.
  
      Examples:
          >>> siliconcompiler.Chip(design="top")
          Creates a chip object with name "top".
      """
  
      ###########################################################################
      def __init__(self, design, loglevel=None):
          # version numbers
          self.scversion = _metadata.version
          self.schemaversion = SCHEMA_VERSION
  
          # Local variables
          self.scroot = os.path.dirname(os.path.abspath(__file__))
          self.cwd = os.getcwd()
          self.error = 0
          self.cfg = schema_cfg()
          # The 'status' dictionary can be used to store ephemeral config values.
          # Its contents will not be saved, and can be set by parent scripts
          # such as a web server or supervisor process. Currently supported keys:
          # * 'jobhash': A hash or UUID which can identify jobs in a larger system.
          # * 'remote_cfg': Dictionary containing remote server configurations
          #                 (address, credentials, etc.)
          # * 'slurm_account': User account ID in a connected slurm HPC cluster.
          # * 'slurm_partition': Name of the partition in which a task should run
          #                      on a connected slurm HPC cluster.
          # * 'watchdog': Activity-monitoring semaphore for jobs scheduled on an
          #               HPC cluster; expects a 'threading.Event'-like object.
          # * 'max_fs_bytes': A limit on how much disk space a job is allowed
          #                   to consume in a connected HPC cluster's storage.
          self.status = {}
  
          self.builtin = ['minimum','maximum',
                          'nop', 'mux', 'join', 'verify']
  
          # We set 'design' and 'loglevel' directly in the config dictionary
          # because of a chicken-and-egg problem: self.set() relies on the logger,
          # but the logger relies on these values.
          self.cfg['design']['value'] = design
          if loglevel:
              self.cfg['option']['loglevel']['value'] = loglevel
  
          self._init_logger()
  
          self._loaded_modules = {
              'flows': [],
              'pdks': [],
              'libs': [],
              'checklists': []
          }
  
      ###########################################################################
      @property
      def design(self):
          '''Design name of chip object.
  
          This is an immutable property.'''
          return self.get('design')
  
      ###########################################################################
      def _init_logger(self, step=None, index=None, in_run=False):
  
          self.logger = logging.getLogger(uuid.uuid4().hex)
  
          # Don't propagate log messages to "root" handler (we get duplicate
          # messages without this)
          # TODO: this prevents us from being able to capture logs with pytest:
          # we should revisit it
          self.logger.propagate = False
  
          loglevel = self.get('option', 'loglevel')
  
          if loglevel=='DEBUG':
              prefix = '| %(levelname)-7s | %(funcName)-10s | %(lineno)-4s'
          else:
              prefix = '| %(levelname)-7s'
  
          if in_run:
              flow = self.get('option', 'flow')
  
              # Figure out how wide to make step and index fields
              max_step_len = 2
              max_index_len = 2
              for future_step in self.getkeys('flowgraph', flow):
                  max_step_len = max(len(future_step) + 1, max_step_len)
                  for future_index in self.getkeys('flowgraph', flow, future_step):
                      max_index_len = max(len(future_index) + 1, max_index_len)
  
              jobname = self.get('option', 'jobname')
  
              if step is None:
                  step = '-' * max(max_step_len // 4, 1)
              if index is None:
                  index = '-' * max(max_index_len // 4, 1)
  
              run_info = f'%s  | %-{max_step_len}s | %-{max_index_len}s' % (jobname, step, index)
              logformat = ' | '.join([prefix, run_info, '%(message)s'])
          else:
              logformat = ' | '.join([prefix, '%(message)s'])
  
          handler = logging.StreamHandler()
          formatter = logging.Formatter(logformat)
  
          handler.setFormatter(formatter)
  
          # Clear any existing handlers so we don't end up with duplicate messages
          # if repeat calls to _init_logger are made
          if len(self.logger.handlers) > 0:
              self.logger.handlers.clear()
  
          self.logger.addHandler(handler)
          self.logger.setLevel(loglevel)
  
      ###########################################################################
      def _deinit_logger(self):
          self.logger = None
  
      ###########################################################################
      def _get_switches(self, *keypath):
          '''Helper function for parsing switches and metavars for a keypath.'''
          #Switch field fully describes switch format
          switch = self.get(*keypath, field='switch')
  
          if switch is None:
              switches = []
          elif isinstance(switch, list):
              switches = switch
          else:
              switches = [switch]
          switchstrs = []
  
          # parse out switch from metavar
          # TODO: should we validate that metavar matches for each switch?
          for switch in switches:
              switchmatch = re.match(r'(-[\w_]+)\s+(.*)', switch)
              gccmatch = re.match(r'(-[\w_]+)(.*)', switch)
              plusmatch = re.match(r'(\+[\w_\+]+)(.*)', switch)
  
              if switchmatch:
                  switchstr = switchmatch.group(1)
                  metavar = switchmatch.group(2)
              elif gccmatch:
                  switchstr = gccmatch.group(1)
                  metavar = gccmatch.group(2)
              elif plusmatch:
                  switchstr = plusmatch.group(1)
                  metavar = plusmatch.group(2)
              switchstrs.append(switchstr)
  
          return switchstrs, metavar
  
      ###########################################################################
      def create_cmdline(self, progname, description=None, switchlist=None, input_map=None):
          """Creates an SC command line interface.
  
          Exposes parameters in the SC schema as command line switches,
          simplifying creation of SC apps with a restricted set of schema
          parameters exposed at the command line. The order of command
          line switch settings parsed from the command line is as follows:
  
           1. loglevel
           2. fpga_partname
           3. load_target('target')
           4. read_manifest([cfg])
           5. all other switches
  
          The cmdline interface is implemented using the Python argparse package
          and the following use restrictions apply.
  
          * Help is accessed with the '-h' switch.
          * Arguments that include spaces must be enclosed with double quotes.
          * List parameters are entered individually. (ie. -y libdir1 -y libdir2)
          * For parameters with Boolean types, the switch implies "true".
          * Special characters (such as '-') must be enclosed in double quotes.
          * Compiler compatible switches include: -D, -I, -O{0,1,2,3}
          * Verilog legacy switch formats are supported: +libext+, +incdir+
  
          Args:
              progname (str): Name of program to be executed.
              description (str): Short program description.
              switchlist (list of str): List of SC parameter switches to expose
                  at the command line. By default all SC schema switches are
                  available. Parameter switches should be entered based on the
                  parameter 'switch' field in the schema. For parameters with
                  multiple switches, both will be accepted if any one is included
                  in this list.
              input_map (dict of str): Dictionary mapping file extensions to input
                  filetypes. This is used to automatically assign positional
                  source arguments to ['input', ...] keypaths based on their file
                  extension. If None, the CLI will not accept positional source
                  arguments.
  
          Examples:
              >>> chip.create_cmdline(progname='sc-show',switchlist=['-input','-cfg'])
              Creates a command line interface for 'sc-show' app.
  
              >>> chip.create_cmdline(progname='sc', input_map={'v': 'verilog'})
              All sources ending in .v will be stored in ['input', 'verilog']
          """
  
          # Argparse
          parser = argparse.ArgumentParser(prog=progname,
                                           prefix_chars='-+',
                                           formatter_class=argparse.RawDescriptionHelpFormatter,
                                           description=description)
  
          # Get all keys from global dictionary or override at command line
          allkeys = self.getkeys()
  
          # Iterate over all keys to add parser arguments
          for keypath in allkeys:
              #Fetch fields from leaf cell
              helpstr = self.get(*keypath, field='shorthelp')
              typestr = self.get(*keypath, field='type')
  
              # argparse 'dest' must be a string, so join keypath with commas
              dest = '_'.join(keypath)
  
              switchstrs, metavar = self._get_switches(*keypath)
  
              # Three switch types (bool, list, scalar)
              if not switchlist or any(switch in switchlist for switch in switchstrs):
                  if typestr == 'bool':
                      parser.add_argument(*switchstrs,
                                          nargs='?',
                                          metavar=metavar,
                                          dest=dest,
                                          const='true',
                                          help=helpstr,
                                          default=argparse.SUPPRESS)
                  #list type arguments
                  elif re.match(r'\[', typestr):
                      #all the rest
                      parser.add_argument(*switchstrs,
                                          metavar=metavar,
                                          dest=dest,
                                          action='append',
                                          help=helpstr,
                                          default=argparse.SUPPRESS)
                  else:
                      #all the rest
                      parser.add_argument(*switchstrs,
                                          metavar=metavar,
                                          dest=dest,
                                          help=helpstr,
                                          default=argparse.SUPPRESS)
  
          if input_map is not None:
              parser.add_argument('source',
                                  nargs='*',
                                  help='Input files with filetype inferred by extension')
  
          #Preprocess sys.argv to enable linux commandline switch formats
          #(gcc, verilator, etc)
          scargs = []
  
          # Iterate from index 1, otherwise we end up with script name as a
          # 'source' positional argument
          for item in sys.argv[1:]:
              #Split switches with one character and a number after (O0,O1,O2)
              opt = re.match(r'(\-\w)(\d+)', item)
              #Split assign switches (-DCFG_ASIC=1)
              assign = re.search(r'(\-\w)(\w+\=\w+)', item)
              #Split plusargs (+incdir+/path)
              plusarg = re.search(r'(\+\w+\+)(.*)', item)
              if opt:
                  scargs.append(opt.group(1))
                  scargs.append(opt.group(2))
              elif plusarg:
                  scargs.append(plusarg.group(1))
                  scargs.append(plusarg.group(2))
              elif assign:
                  scargs.append(assign.group(1))
                  scargs.append(assign.group(2))
              else:
                  scargs.append(item)
  
          parser.add_argument('-version', action='version', version=_metadata.version)
  
          #Grab argument from pre-process sysargs
          cmdargs = vars(parser.parse_args(scargs))
  
          # Print banner
          print(_metadata.banner)
          print("Authors:", ", ".join(_metadata.authors))
          print("Version:", _metadata.version, "\n")
          print("-"*80)
  
          os.environ["COLUMNS"] = '80'
  
          # 1. set loglevel if set at command line
          if 'option_loglevel' in cmdargs.keys():
              self.logger.setLevel(cmdargs['option_loglevel'])
  
          # 2. read in target if set
          if 'option_target' in cmdargs.keys():
              if 'arg_pdk' in cmdargs.keys():
                  raise NotImplementedError("NOT IMPLEMENTED: ['arg', 'pdk'] parameter with target")
              if 'arg_flow' in cmdargs.keys():
                  raise NotImplementedError("NOT IMPLEMENTED: ['arg', 'flow'] parameter with target")
              if 'fpga_partname' in cmdargs.keys():
                  self.set('fpga', 'partname', cmdargs['fpga_partname'], clobber=True)
              # running target command
              self.load_target(cmdargs['option_target'])
  
          # 4. read in all cfg files
          if 'option_cfg' in cmdargs.keys():
              for item in cmdargs['option_cfg']:
                  self.read_manifest(item, clobber=True, clear=True)
  
          # Map sources to ['input'] keypath.
          if 'source' in cmdargs:
              for source in cmdargs['source']:
                  _, ext = os.path.splitext(source)
                  ext = ext.lstrip('.')
                  if ext in input_map:
                      filetype = input_map[ext]
                      if self.valid('input', filetype, quiet=True):
                          self.add('input', filetype, source)
                      else:
                          self.set('input', filetype, source)
                      self.logger.info(f'Source {source} inferred as {filetype}')
                  else:
                      self.logger.warning('Unable to infer input type for '
                          f'{source} based on file extension, ignoring. Use the '
                          '-input flag to provide it explicitly.')
              # we don't want to handle this in the next loop
              del cmdargs['source']
  
          # 5. Cycle through all command args and write to manifest
          for dest, vals in cmdargs.items():
              keypath = dest.split('_')
  
              # Turn everything into a list for uniformity
              if not isinstance(vals, list):
                  vals = [vals]
  
              # Cycle through all items
              for item in vals:
                  # Hack to handle the fact that we want optmode stored with an 'O'
                  # prefix.
                  if keypath == ['option', 'optmode']:
                      item = 'O' + item
  
                  num_free_keys = keypath.count('default')
  
                  if len(item.split(' ')) < num_free_keys + 1:
                      # Error out if value provided doesn't have enough words to
                      # fill in 'default' keys.
                      switches, metavar = self._get_switches(*keypath)
                      switchstr = '/'.join(switches)
                      self.logger.error(f'Invalid value {item} for switch {switchstr}. Expected format {metavar}.')
                      raise SiliconCompilerError('Invalid CLI arguments')
  
                  # We replace 'default' in keypath with first N words in provided
                  # value. Remainder is the actual value we want to store in the
                  # parameter.
                  *free_keys, val = item.split(' ', num_free_keys)
                  args = [free_keys.pop(0) if key == 'default' else key for key in keypath]
  
                  # Storing in manifest
                  self.logger.info(f"Command line argument entered: {args} Value: {val}")
                  typestr = self.get(*keypath, field='type')
                  if typestr.startswith('['):
                      if self.valid(*args, quiet=True):
                          self.add(*args, val)
                      else:
                          self.set(*args, val, clobber=True)
                  else:
                      self.set(*args, val, clobber=True)
  
      #########################################################################
      def find_function(self, modulename, funcname, moduletype=None):
          '''
          Returns a function attribute from a module on disk.
  
          Searches the SC root directory and the 'scpath' parameter for the
          modulename provided and imports the module if found. If the funcname
          provided is found in the module, a callable function attribute is
          returned, otherwise None is returned.
  
          The function assumes the following directory structure:
  
          * tools/modulename/modulename.py
          * flows/modulename.py
          * pdks/modulname.py
  
          If the moduletype is None, the module paths are search in the
          order: 'targets'->'flows'->'tools'->'pdks'->'libs'->'checklists'):
  
  
          Supported functions include:
  
          * targets (make_docs, setup)
          * pdks (make_docs, setup)
          * flows (make_docs, setup)
          * tools (make_docs, setup, check_version, runtime_options,
            pre_process, post_process)
          * libs (make_docs, setup)
  
          Args:
              modulename (str): Name of module to import.
              funcname (str): Name of the function to find within the module.
              moduletype (str): Type of module (flows, pdks, libs, checklists, targets).
  
          Examples:
              >>> setup_pdk = chip.find_function('freepdk45', 'setup', 'pdks')
              >>> setup_pdk()
              Imports the freepdk45 module and runs the setup_pdk function
  
          '''
  
          # module search path depends on modtype
          if moduletype is None:
              for item in ('targets', 'flows', 'tools', 'pdks', 'libs', 'checklists'):
                  fullpath = self.find_function(modulename, funcname, module_type=item)
                  if fullpath:
                      break
                  self.logger.error(f"Could not find module {modulename}")
                  self.error = 1
                  return None
          elif moduletype in ('targets','flows', 'pdks', 'libs'):
              fullpath = self._find_sc_file(f"{moduletype}/{modulename}.py", missing_ok=True)
          elif moduletype in ('tools', 'checklists'):
              fullpath = self._find_sc_file(f"{moduletype}/{modulename}/{modulename}.py", missing_ok=True)
          else:
              self.logger.error(f"Illegal module type '{moduletype}'.")
              self.error = 1
              return
  
          # try loading module if found
          if fullpath:
              self.logger.debug(f"Loading function '{funcname}' from module '{modulename}'")
  
              try:
                  spec = importlib.util.spec_from_file_location(modulename, fullpath)
                  imported = importlib.util.module_from_spec(spec)
                  spec.loader.exec_module(imported)
  
                  if hasattr(imported, funcname):
                      function = getattr(imported, funcname)
                  else:
                      function = None
                  return function
              except:
                  traceback.print_exc()
                  self.logger.error(f"Module setup failed for '{modulename}'")
                  self.error = 1
  
      ##########################################################################
      def load_target(self, name):
          """
          Loads a target module and runs the setup() function.
  
          The function searches the $SCPATH for targets/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
              flow (str): Target flow to
  
          Examples:
              >>> chip.load_target('freepdk45_demo')
              Loads the 'freepdk45_demo' target
  
          """
  
          self.set('option', 'target', name)
  
          func = self.find_function(name, 'setup', 'targets')
          if func is not None:
              func(self)
          else:
              self.logger.error(f'Target module {name} not found in $SCPATH or siliconcompiler/targets/.')
              raise SiliconCompilerError(f'Target module {name} not found $SCPATH or siliconcompiler/targets/.')
  
      ##########################################################################
      def load_pdk(self, name):
          """
          Loads a PDK module and runs the setup() function.
  
          The function searches the $SCPATH for pdks/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
  
          Examples:
              >>> chip.load_pdk('freepdk45_pdk')
              Loads the 'freepdk45' pdk
  
          """
  
          func = self.find_function(name, 'setup', 'pdks')
          if func is not None:
              self.logger.info(f"Loading PDK '{name}'")
              self._loaded_modules['pdks'].append(name)
              func(self)
          else:
              self.logger.error(f'PDK module {name} not found in $SCPATH or siliconcompiler/pdks/.')
              raise SiliconCompilerError(f'PDK module {name} not found in $SCPATH or siliconcompiler/pdks/.')
  
      ##########################################################################
      def load_flow(self, name):
          """
          Loads a flow  module and runs the setup() function.
  
          The function searches the $SCPATH for flows/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
  
          Examples:
              >>> chip.load_flow('asicflow')
              Loads the 'asicflow' flow
  
          """
  
          func = self.find_function(name, 'setup', 'flows')
          if func is not None:
              self.logger.info(f"Loading flow '{name}'")
              self._loaded_modules['flows'].append(name)
              func(self)
          else:
              self.logger.error(f'Flow module {name} not found in $SCPATH or siliconcompiler/flows/.')
              raise SiliconCompilerError(f'Flow module {name} not found in $SCPATH or siliconcompiler/flows/.')
  
      ##########################################################################
      def load_lib(self, name):
          """
          Loads a library module and runs the setup() function.
  
          The function searches the $SCPATH for libs/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
  
          Examples:
              >>> chip.load_lib('nangate45')
              Loads the 'nangate45' library
  
          """
  
          func = self.find_function(name, 'setup', 'libs')
          if func is not None:
              self.logger.info(f"Loading library '{name}'")
              self._loaded_modules['libs'].append(name)
              func(self)
          else:
              self.logger.error(f'Library module {name} not found in $SCPATH or siliconcompiler/libs/.')
              raise SiliconCompilerError(f'Library module {name} not found in $SCPATH or siliconcompiler/libs/.')
  
      ##########################################################################
      def load_checklist(self, name):
          """
          Loads a checklist module and runs the setup() function.
  
          The function searches the $SCPATH for checklist/<name>/<name>.py and runs
          the setup function in that module if found.
  
          Args:
              name (str): Module name
  
          Examples:
              >>> chip.load_checklist('oh_tapeout')
              Loads the 'oh_tapeout' checklist
  
          """
  
          func = self.find_function(name, 'setup', 'checklists')
          if func is not None:
              self.logger.info(f"Loading checklist '{name}'")
              self._loaded_modules['checklists'].append(name)
              func(self)
          else:
              self.logger.error(f'Checklist module {name} not found in $SCPATH or siliconcompiler/checklists/.')
              raise SiliconCompilerError(f'Checklist module {name} not found in $SCPATH or siliconcompiler/checklists/.')
  
  
      ###########################################################################
      def list_metrics(self):
          '''
          Returns a list of all metrics in the schema.
  
          '''
  
          return self.getkeys('metric','default','default')
  
      ###########################################################################
      def help(self, *keypath):
          """
          Returns a schema parameter description.
  
          Args:
              *keypath(str): Keypath to parameter.
  
          Returns:
              A formatted multi-line help paragraph for the parameter provided.
  
          Examples:
              >>> print(chip.help('asic','diearea'))
              Displays help information about the 'asic, diearea' parameter
  
          """
  
          self.logger.debug('Fetching help for %s', keypath)
  
          #Fetch Values
  
          description = self.get(*keypath, field='shorthelp')
          typestr = self.get(*keypath, field='type')
          switchstr = str(self.get(*keypath, field='switch'))
          defstr = str(self.get(*keypath, field='defvalue'))
          requirement = str(self.get(*keypath, field='require'))
          helpstr = self.get(*keypath, field='help')
          example = self.get(*keypath, field='example')
  
          examplestr = ("\nExamples:    " + example[0] + ''.join(
                       ["\n             " + ex for ex in example[1:]]))
  
          #Removing multiple spaces and newlines
          helpstr = helpstr.rstrip()
          helpstr = helpstr.replace("\n", "")
          helpstr = ' '.join(helpstr.split())
  
          for idx, item in enumerate(example):
              example[idx] = ' '.join(item.split())
              example[idx] = example[idx].replace(", ", ",")
  
          #Wrap text
          para = textwrap.TextWrapper(width=60)
          para_list = para.wrap(text=helpstr)
  
          #Full Doc String
          fullstr = ("-"*80 +
                     "\nDescription: " + description +
                     "\nSwitch:      " + switchstr +
                     "\nType:        " + typestr  +
                     "\nRequirement: " + requirement   +
                     "\nDefault:     " + defstr   +
                     examplestr +
                     "\nHelp:        " + para_list[0] + "\n")
          for line in para_list[1:]:
              fullstr = (fullstr +
                         " "*13 + line.lstrip() + "\n")
  
          return fullstr
  
  
      ###########################################################################
      def valid(self, *args, valid_keypaths=None, quiet=True, default_valid=False):
          """
          Checks validity of a keypath.
  
          Checks the validity of a parameter keypath and returns True if the
          keypath is valid and False if invalid.
  
          Args:
              keypath(list str): Variable length schema key list.
              valid_keypaths (list of list): List of valid keypaths as lists. If
                  None, check against all keypaths in the schema.
              quiet (bool): If True, don't display warnings for invalid keypaths.
  
          Returns:
              Boolean indicating validity of keypath.
  
          Examples:
              >>> check = chip.valid('design')
              Returns True.
              >>> check = chip.valid('blah')
              Returns False.
          """
  
          keypathstr = ','.join(args)
          keylist = list(args)
          if default_valid:
              default = 'default'
          else:
              default = None
  
          if valid_keypaths is None:
              valid_keypaths = self.getkeys()
  
          # Look for a full match with default playing wild card
          for valid_keypath in valid_keypaths:
              if len(keylist) != len(valid_keypath):
                  continue
  
              ok = True
              for i in range(len(keylist)):
                  if valid_keypath[i] not in (keylist[i], default):
                      ok = False
                      break
              if ok:
                  return True
  
          # Match not found
          if not quiet:
              self.logger.warning(f"Keypath [{keypathstr}] is not valid")
          return False
  
      ###########################################################################
      def get(self, *keypath, field='value', job=None, cfg=None):
          """
          Returns a schema parameter field.
  
          Returns a schema parameter field based on the keypath provided in the
          ``*keypath``. See the :ref:`Schema Reference Manual<SiliconCompiler
          Schema>` for documentation of all supported keypaths. The returned type
          is consistent with the type field of the parameter. Fetching parameters
          with empty or undefined value files returns None for scalar types and []
          (empty list) for list types.  Accessing a non-existent keypath produces
          a logger error message and raises the Chip object error flag.
  
          Args:
              keypath(list str): Variable length schema key list.
              field(str): Parameter field to fetch.
              job (str): Jobname to use for dictionary access in place of the
                  current active jobname.
              cfg(dict): Alternate dictionary to access in place of the default
                  chip object schema dictionary.
  
          Returns:
              Value found for the keypath and field provided.
  
          Examples:
              >>> foundry = chip.get('pdk', 'foundry')
              Returns the name of the foundry from the PDK.
  
          """
  
          if cfg is None:
              if job is not None:
                  cfg = self.cfg['history'][job]
              else:
                  cfg = self.cfg
  
          keypathstr = ','.join(keypath)
  
          self.logger.debug(f"Reading from [{keypathstr}]. Field = '{field}'")
          return self._search(cfg, keypathstr, *keypath, field=field, mode='get')
  
      ###########################################################################
      def getkeys(self, *keypath, cfg=None, job=None):
          """
          Returns a list of schema dictionary keys.
  
          Searches the schema for the keypath provided and returns a list of
          keys found, excluding the generic 'default' key. Accessing a
          non-existent keypath produces a logger error message and raises the
          Chip object error flag.
  
          Args:
              keypath (list str): Variable length ordered schema key list
              cfg (dict): Alternate dictionary to access in place of self.cfg
              job (str): Jobname to use for dictionary access in place of the
                  current active jobname.
  
          Returns:
              List of keys found for the keypath provided.
  
          Examples:
              >>> keylist = chip.getkeys('pdk')
              Returns all keys for the 'pdk' keypath.
              >>> keylist = chip.getkeys()
              Returns all list of all keypaths in the schema.
          """
  
          if cfg is None:
              if job is None:
                  cfg = self.cfg
              else:
                  cfg = self.cfg['history'][job]
  
          if len(list(keypath)) > 0:
              keypathstr = ','.join(keypath)
              self.logger.debug('Getting schema parameter keys for: %s', keypathstr)
              keys = list(self._search(cfg, keypathstr, *keypath, mode='getkeys'))
              if 'default' in keys:
                  keys.remove('default')
          else:
              self.logger.debug('Getting all schema parameter keys.')
              keys = list(self._allkeys(cfg))
  
          return keys
  
      ###########################################################################
      def getdict(self, *keypath, cfg=None):
          """
          Returns a schema dictionary.
  
          Searches the schema for the keypath provided and returns a complete
          dictionary. Accessing a non-existent keypath produces a logger error
          message and raises the Chip object error flag.
  
          Args:
              keypath(list str): Variable length ordered schema key list
              cfg(dict): Alternate dictionary to access in place of self.cfg
  
          Returns:
              A schema dictionary
  
          Examples:
              >>> pdk = chip.getdict('pdk')
              Returns the complete dictionary found for the keypath 'pdk'
          """
  
          if cfg is None:
              cfg = self.cfg
  
          if len(list(keypath)) > 0:
              keypathstr = ','.join(keypath)
              self.logger.debug('Getting cfg for: %s', keypathstr)
              localcfg = self._search(cfg, keypathstr, *keypath, mode='getcfg')
  
          return copy.deepcopy(localcfg)
  
      ###########################################################################
      def set(self, *args, field='value', clobber=True, cfg=None):
          '''
          Sets a schema parameter field.
  
          Sets a schema parameter field based on the keypath and value provided in
          the ``*args``. See the :ref:`Schema Reference Manual<SiliconCompiler
          Schema>` for documentation of all supported keypaths. New schema
          dictionaries are automatically created for keypaths that overlap with
          'default' dictionaries. The write action is ignored if the parameter
          value is non-empty and the clobber option is set to False.
  
          The value provided must agree with the dictionary parameter 'type'.
          Accessing a non-existent keypath or providing a value that disagrees
          with the parameter type produces a logger error message and raises the
          Chip object error flag.
  
          Args:
              args (list): Parameter keypath followed by a value to set.
              field (str): Parameter field to set.
              clobber (bool): Existing value is overwritten if True.
              cfg(dict): Alternate dictionary to access in place of self.cfg
  
          Examples:
              >>> chip.set('design', 'top')
              Sets the name of the design to 'top'
          '''
  
          if cfg is None:
              cfg = self.cfg
  
          # Verify that all keys are strings
          for key in args[:-1]:
              if not isinstance(key,str):
                  self.logger.error(f"Key [{key}] is not a string [{args}]")
  
          keypathstr = ','.join(args[:-1])
          all_args = list(args)
  
          # Special case to ensure loglevel is updated ASAP
          if len(args) == 3 and args[1] == 'loglevel' and field == 'value':
              self.logger.setLevel(args[2])
  
          self.logger.debug(f"Setting [{keypathstr}] to {args[-1]}")
          return self._search(cfg, keypathstr, *all_args, field=field, mode='set', clobber=clobber)
  
      ###########################################################################
      def add(self, *args, cfg=None, field='value'):
          '''
          Adds item(s) to a schema parameter list.
  
          Adds item(s) to schema parameter list based on the keypath and value
          provided in the ``*args``.  See the :ref:`Schema Reference
          Manual<SiliconCompiler Schema>` for documentation of all supported
          keypaths. New schema dictionaries are automatically created for keypaths
          that overlap with 'default' dictionaries.
  
          The value provided must agree with the dictionary parameter 'type'.
          Accessing a non-existent keypath, providing a value that disagrees
          with the parameter type, or using add with a scalar parameter produces
          a logger error message and raises the Chip object error flag.
  
          Args:
              args (list): Parameter keypath followed by a value to add.
              cfg(dict): Alternate dictionary to access in place of self.cfg
              field (str): Parameter field to set.
  
          Examples:
              >>> chip.add('source', 'hello.v')
              Adds the file 'hello.v' to the list of sources.
          '''
  
          if cfg is None:
              cfg = self.cfg
  
          # Verify that all keys are strings
          for key in args[:-1]:
              if not isinstance(key,str):
                  self.logger.error(f"Key [{key}] is not a string [{args}]")
  
          keypathstr = ','.join(args[:-1])
          all_args = list(args)
  
          self.logger.debug(f'Appending value {args[-1]} to [{keypathstr}]')
          return self._search(cfg, keypathstr, *all_args, field=field, mode='add')
  
  
      ###########################################################################
      def _allkeys(self, cfg, keys=None, keylist=None):
          '''
          Returns list of all keypaths in the schema.
          '''
  
          if keys is None:
              keylist = []
              keys = []
          for k in cfg:
              newkeys = keys.copy()
              newkeys.append(k)
              if 'defvalue' in cfg[k]:
                  keylist.append(newkeys)
              else:
                  self._allkeys(cfg[k], keys=newkeys, keylist=keylist)
          return keylist
  
      ###########################################################################
      def _search(self, cfg, keypath, *args, field='value', mode='get', clobber=True):
          '''
          Internal recursive function that searches the Chip schema for a
          match to the combination of *args and fields supplied. The function is
          used to set and get data within the dictionary.
  
          Args:
              cfg(dict): The cfg schema to search
              keypath (str): Concatenated keypath used for error logging.
              args (str): Keypath/value variable list used for access
              field(str): Leaf cell field to access.
              mode(str): Action (set/get/add/getkeys/getkeys)
              clobber(bool): Specifies to clobber (for set action)
  
          '''
  
          all_args = list(args)
          param = all_args[0]
          val = all_args[-1]
          empty = [None, 'null', [], 'false']
  
          #set/add leaf cell (all_args=(param,val))
          if (mode in ('set', 'add')) & (len(all_args) == 2):
              # clean error if key not found
              if (not param in cfg) & (not 'default' in cfg):
                  self.logger.error(f"Set/Add keypath [{keypath}] does not exist.")
                  self.error = 1
              else:
                  # making an 'instance' of default if not found
                  if (not param in cfg) & ('default' in cfg):
                      cfg[param] = copy.deepcopy(cfg['default'])
                  list_type =bool(re.match(r'\[', cfg[param]['type']))
                  # checking for illegal fields
                  if not field in cfg[param] and (field != 'value'):
                      self.logger.error(f"Field '{field}' for keypath [{keypath}]' is not a valid field.")
                      self.error = 1
                  # check legality of value
                  if field == 'value':
                      (type_ok,type_error) = self._typecheck(cfg[param], param, val)
                      if not type_ok:
                          self.logger.error("%s", type_error)
                          self.error = 1
                  # converting python True/False to lower case string
                  if (field == 'value') and (cfg[param]['type'] == 'bool'):
                      if val == True:
                          val = "true"
                      elif val == False:
                          val = "false"
                  # checking if value has been set
                  # TODO: fix clobber!!
                  selval = cfg[param]['value']
                  # updating values
                  if cfg[param]['lock'] == "true":
                      self.logger.debug("Ignoring {mode}{} to [{keypath}]. Lock bit is set.")
                  elif (mode == 'set'):
                      if (field != 'value') or (selval in empty) or clobber:
                          if field in ('copy', 'lock'):
                              # boolean fields
                              if val is True:
                                  cfg[param][field] = "true"
                              elif val is False:
                                  cfg[param][field] = "false"
                              else:
                                  self.logger.error(f'{field} must be set to boolean.')
                                  self.error = 1
                          elif field in ('hashalgo', 'scope', 'require', 'type', 'unit',
                                         'shorthelp', 'notes', 'switch', 'help'):
                              # awlays string scalars
                              cfg[param][field] = val
                          elif field in ('example'):
                              # list from default schema (already a list)
                              cfg[param][field] = val
                          elif field in ('signature', 'filehash', 'date', 'author'):
                              # convert to list if appropriate
                              if isinstance(val, list) | (not list_type):
                                  cfg[param][field] = val
                              else:
                                  cfg[param][field] = [val]
                          elif (not list_type) & (val is None):
                              # special case for None
                              cfg[param][field] = None
                          elif (not list_type) & (not isinstance(val, list)):
                              # convert to string for scalar value
                              cfg[param][field] = str(val)
                          elif list_type & (not isinstance(val, list)):
                              # convert to string for list value
                              cfg[param][field] = [str(val)]
                          elif list_type & isinstance(val, list):
                              # converting tuples to strings
                              if re.search(r'\(', cfg[param]['type']):
                                  cfg[param][field] = list(map(str,val))
                              else:
                                  cfg[param][field] = val
                          else:
                              self.logger.error(f"Assigning list to scalar for [{keypath}]")
                              self.error = 1
                      else:
                          self.logger.debug(f"Ignoring set() to [{keypath}], value already set. Use clobber=true to override.")
                  elif (mode == 'add'):
                      if field in ('filehash', 'date', 'author', 'signature'):
                          cfg[param][field].append(str(val))
                      elif field in ('copy', 'lock'):
                          self.logger.error(f"Illegal use of add() for scalar field {field}.")
                          self.error = 1
                      elif list_type & (not isinstance(val, list)):
                          cfg[param][field].append(str(val))
                      elif list_type & isinstance(val, list):
                          cfg[param][field].extend(val)
                      else:
                          self.logger.error(f"Illegal use of add() for scalar parameter [{keypath}].")
                          self.error = 1
                  return cfg[param][field]
          #get leaf cell (all_args=param)
          elif len(all_args) == 1:
              if not param in cfg:
                  self.error = 1
                  self.logger.error(f"Get keypath [{keypath}] does not exist.")
              elif mode == 'getcfg':
                  return cfg[param]
              elif mode == 'getkeys':
                  return cfg[param].keys()
              else:
                  if not (field in cfg[param]) and (field!='value'):
                      self.error = 1
                      self.logger.error(f"Field '{field}' not found for keypath [{keypath}]")
                  elif field == 'value':
                      #Select default if no value has been set
                      if field not in cfg[param]:
                          selval = cfg[param]['defvalue']
                      else:
                          selval =  cfg[param]['value']
                      #check for list
                      if bool(re.match(r'\[', cfg[param]['type'])):
                          sctype = re.sub(r'[\[\]]', '', cfg[param]['type'])
                          return_list = []
                          if selval is None:
                              return None
                          for item in selval:
                              if sctype == 'int':
                                  return_list.append(int(item))
                              elif sctype == 'float':
                                  return_list.append(float(item))
                              elif sctype.startswith('(str,'):
                                  if isinstance(item,tuple):
                                      return_list.append(item)
                                  else:
                                      tuplestr = re.sub(r'[\(\)\'\s]','',item)
                                      return_list.append(tuple(tuplestr.split(',')))
                              elif sctype.startswith('(float,'):
                                  if isinstance(item,tuple):
                                      return_list.append(item)
                                  else:
                                      tuplestr = re.sub(r'[\(\)\s]','',item)
                                      return_list.append(tuple(map(float, tuplestr.split(','))))
                              else:
                                  return_list.append(item)
                          return return_list
                      else:
                          if selval is None:
                              # Unset scalar of any type
                              scalar = None
                          elif cfg[param]['type'] == "int":
                              #print(selval, type(selval))
                              scalar = int(float(selval))
                          elif cfg[param]['type'] == "float":
                              scalar = float(selval)
                          elif cfg[param]['type'] == "bool":
                              scalar = (selval == 'true')
                          elif re.match(r'\(', cfg[param]['type']):
                              tuplestr = re.sub(r'[\(\)\s]','',selval)
                              scalar = tuple(map(float, tuplestr.split(',')))
                          else:
                              scalar = selval
                          return scalar
                  #all non-value fields are strings (or lists of strings)
                  else:
                      if cfg[param][field] == 'true':
                          return True
                      elif cfg[param][field] == 'false':
                          return False
                      else:
                          return cfg[param][field]
          #if not leaf cell descend tree
          else:
              ##copying in default tree for dynamic trees
              if not param in cfg and 'default' in cfg:
                  cfg[param] = copy.deepcopy(cfg['default'])
              elif not param in cfg:
                  self.error = 1
                  self.logger.error(f"Get keypath [{keypath}] does not exist.")
                  return None
              all_args.pop(0)
              return self._search(cfg[param], keypath, *all_args, field=field, mode=mode, clobber=clobber)
  
      ###########################################################################
      def _prune(self, cfg, top=True, keeplists=False):
          '''
          Internal recursive function that creates a local copy of the Chip
          schema (cfg) with only essential non-empty parameters retained.
  
          '''
  
          # create a local copy of dict
          if top:
              localcfg = copy.deepcopy(cfg)
          else:
              localcfg = cfg
  
          #10 should be enough for anyone...
          maxdepth = 10
          i = 0
  
          #Prune when the default & value are set to the following
          if keeplists:
              empty = ("null", None)
          else:
              empty = ("null", None, [])
  
          # When at top of tree loop maxdepth times to make sure all stale
          # branches have been removed, not elegant, but stupid-simple
          # "good enough"
          while i < maxdepth:
              #Loop through all keys starting at the top
              for k in list(localcfg.keys()):
                  #removing all default/template keys
                  # reached a default subgraph, delete it
                  if k == 'default':
                      del localcfg[k]
                  # reached leaf-cell
                  elif 'help' in localcfg[k].keys():
                      del localcfg[k]['help']
                  elif 'example' in localcfg[k].keys():
                      del localcfg[k]['example']
                  elif 'defvalue' in localcfg[k].keys():
                      if localcfg[k]['defvalue'] in empty:
                          if 'value' in localcfg[k].keys():
                              if localcfg[k]['value'] in empty:
                                  del localcfg[k]
                          else:
                              del localcfg[k]
                  #removing stale branches
                  elif not localcfg[k]:
                      localcfg.pop(k)
                  #keep traversing tree
                  else:
                      self._prune(cfg=localcfg[k], top=False, keeplists=keeplists)
              if top:
                  i += 1
              else:
                  break
  
          return localcfg
  
      ###########################################################################
      def _find_sc_file(self, filename, missing_ok=False):
          """
          Returns the absolute path for the filename provided.
  
          Searches the SC root directory and the 'scpath' parameter for the
          filename provided and returns the absolute path. If no valid absolute
          path is found during the search, None is returned.
  
          Shell variables ('$' followed by strings consisting of numbers,
          underscores, and digits) are replaced with the variable value.
  
          Args:
              filename (str): Relative or absolute filename.
  
          Returns:
              Returns absolute path of 'filename' if found, otherwise returns
              None.
  
          Examples:
              >>> chip._find_sc_file('flows/asicflow.py')
             Returns the absolute path based on the sc installation directory.
  
          """
  
          # Replacing environment variables
          filename = self._resolve_env_vars(filename)
  
          # If we have a path relative to our cwd or an abs path, pass-through here
          if os.path.exists(os.path.abspath(filename)):
              return os.path.abspath(filename)
  
          # Otherwise, search relative to scpaths
          scpaths = [self.scroot, self.cwd]
          scpaths.extend(self.get('option', 'scpath'))
          if 'SCPATH' in os.environ:
              scpaths.extend(os.environ['SCPATH'].split(os.pathsep))
  
          searchdirs = ', '.join(scpaths)
          self.logger.debug(f"Searching for file {filename} in {searchdirs}")
  
          result = None
          for searchdir in scpaths:
              if not os.path.isabs(searchdir):
                  searchdir = os.path.join(self.cwd, searchdir)
  
              abspath = os.path.abspath(os.path.join(searchdir, filename))
              if os.path.exists(abspath):
                  result = abspath
                  break
  
          if result is None and not missing_ok:
              self.error = 1
              self.logger.error(f"File {filename} was not found")
  
          return result
  
      ###########################################################################
      def find_files(self, *keypath, cfg=None, missing_ok=False, job=None):
          """
          Returns absolute paths to files or directories based on the keypath
          provided.
  
          By default, this function first checks if the keypath provided has its
          `copy` parameter set to True. If so, it returns paths to the files in
          the build directory. Otherwise, it resolves these files based on the
          current working directory and SC path.
  
          The keypath provided must point to a schema parameter of type file, dir,
          or lists of either. Otherwise, it will trigger an error.
  
          Args:
              keypath (list str): Variable length schema key list.
              cfg (dict): Alternate dictionary to access in place of the default
                  chip object schema dictionary.
              missing_ok (bool): If True, silently return None when files aren't
                  found. If False, print an error and set the error flag.
              job (str): Jobname to use for dictionary access in place of the
                  current active jobname.
  
          Returns:
              If keys points to a scalar entry, returns an absolute path to that
              file/directory, or None if not found. It keys points to a list
              entry, returns a list of either the absolute paths or None for each
              entry, depending on whether it is found.
  
          Examples:
              >>> chip.find_files('source')
              Returns a list of absolute paths to source files, as specified in
              the schema.
  
          """
          if cfg is None:
              cfg = self.cfg
  
          copyall = self.get('option', 'copyall', cfg=cfg, job=job)
          paramtype = self.get(*keypath, field='type', cfg=cfg, job=job)
  
          if 'file' in paramtype:
              copy = self.get(*keypath, field='copy', cfg=cfg, job=job)
          else:
              copy = False
  
          if 'file' not in paramtype and 'dir' not in paramtype:
              self.logger.error('Can only call find_files on file or dir types')
              self.error = 1
              return None
  
          is_list = bool(re.match(r'\[', paramtype))
  
          paths = self.get(*keypath, cfg=cfg, job=job)
          # Convert to list if we have scalar
          if not is_list:
              paths = [paths]
  
          result = []
  
          # Special cases for various ['eda', ...] files that may be implicitly
          # under the workdir (or refdir in the case of scripts).
          # TODO: it may be cleaner to have a file resolution scope flag in schema
          # (e.g. 'scpath', 'workdir', 'refdir'), rather than harcoding special
          # cases.
          if keypath[0] == 'tool' and keypath[2] in ('input', 'output', 'report'):
              step = keypath[3]
              index = keypath[4]
              if keypath[2] == 'report':
                  io = ""
              else:
                  io = keypath[2] + 's'
              iodir = os.path.join(self._getworkdir(jobname=job, step=step, index=index), io)
              for path in paths:
                  abspath = os.path.join(iodir, path)
                  if os.path.isfile(abspath):
                      result.append(abspath)
              return result
          elif keypath[0] == 'tool' and keypath[2] == 'script':
              tool = keypath[1]
              step = keypath[3]
              index = keypath[4]
              refdirs = self.find_files('tool', tool, 'refdir', step, index)
              for path in paths:
                  for refdir in refdirs:
                      abspath = os.path.join(refdir, path)
                      if os.path.isfile(abspath):
                          result.append(abspath)
                          break
  
              return result
  
          for path in paths:
              if (copyall or copy) and ('file' in paramtype):
                  name = self._get_imported_filename(path)
                  abspath = os.path.join(self._getworkdir(jobname=job, step='import'), 'outputs', name)
                  if os.path.isfile(abspath):
                      # if copy is True and file is found in import outputs,
                      # continue. Otherwise, fall through to _find_sc_file (the
                      # file may not have been gathered in imports yet)
                      result.append(abspath)
                      continue
              result.append(self._find_sc_file(path, missing_ok=missing_ok))
          # Convert back to scalar if that was original type
          if not is_list:
              return result[0]
  
          return result
  
      ###########################################################################
      def find_result(self, filetype, step, jobname=None, index='0'):
          """
          Returns the absolute path of a compilation result.
  
          Utility function that returns the absolute path to a results
          file based on the provided arguments. The result directory
          structure is:
  
          <dir>/<design>/<jobname>/<step>/<index>/outputs/<design>.filetype
  
          Args:
              filetype (str): File extension (.v, .def, etc)
              step (str): Task step name ('syn', 'place', etc)
              jobname (str): Jobid directory name
              index (str): Task index
  
          Returns:
              Returns absolute path to file.
  
          Examples:
              >>> manifest_filepath = chip.find_result('.vg', 'syn')
             Returns the absolute path to the manifest.
          """
          if jobname is None:
              jobname = self.get('option', 'jobname')
  
          workdir = self._getworkdir(jobname, step, index)
          design = self.get('design')
          filename = f"{workdir}/outputs/{design}.{filetype}"
  
          self.logger.debug("Finding result %s", filename)
  
          if os.path.isfile(filename):
              return filename
          else:
              return None
  
      ###########################################################################
      def _abspath(self, cfg):
          '''
          Internal function that goes through provided dictionary and resolves all
          relative paths where required.
          '''
  
          for keypath in self.getkeys(cfg=cfg):
              paramtype = self.get(*keypath, cfg=cfg, field='type')
              value = self.get(*keypath, cfg=cfg)
              if value:
                  #only do something if type is file or dir
                  if 'file' in paramtype or 'dir' in paramtype:
                      abspaths = self.find_files(*keypath, cfg=cfg, missing_ok=True)
                      self.set(*keypath, abspaths, cfg=cfg)
  
      ###########################################################################
      def _print_csv(self, cfg, fout=None):
          allkeys = self.getkeys(cfg=cfg)
          for key in allkeys:
              keypath = f'"{",".join(key)}"'
              value = self.get(*key, cfg=cfg)
              if isinstance(value,list):
                  for item in value:
                      fout.write(f"{keypath},{item}")
              else:
                  fout.write(f"{keypath},{value}")
  
      ###########################################################################
      def _escape_val_tcl(self, val, typestr):
          '''Recursive helper function for converting Python values to safe TCL
          values, based on the SC type string.'''
          if val is None:
              return ''
          elif typestr.startswith('('):
              subtypes = typestr.strip('()').split(',')
              valstr = ' '.join(self._escape_val_tcl(v, subtype.strip())
                                for v, subtype in zip(val, subtypes))
              return f'[list {valstr}]'
          elif typestr.startswith('['):
              subtype = typestr.strip('[]')
              valstr = ' '.join(self._escape_val_tcl(v, subtype) for v in val)
              return f'[list {valstr}]'
          elif typestr == 'bool':
              return 'true' if val else 'false'
          elif typestr == 'str':
              # Surrounding a string with '{}' ensures that special characters
              # inside the string are treated as-is.
              # Source: https://datacadamia.com/lang/tcl/special_character#about
              escaped_val = val.replace('{', '\\{').replace('}', '\\}')
              return '{' + escaped_val + '}'
          elif typestr in ('file', 'dir'):
              # Replace $VAR with $env(VAR) for tcl
              # Surround with quotes to escape whitespace. We don't want to use {}
              # here, since that will break envvar expansion.
              return '"' + re.sub(r'\$(\w+)', r'$env(\1)', val) + '"'
          else:
              # floats/ints just become strings
              return str(val)
  
      ###########################################################################
      def _print_tcl(self, cfg, fout=None, prefix=""):
          '''
          Prints out schema as TCL dictionary
          '''
  
          fout.write("#############################################")
          fout.write("#!!!! AUTO-GENERATED FILE. DO NOT EDIT!!!!!!")
          fout.write("#############################################\n")
  
          allkeys = self.getkeys(cfg=cfg)
  
          for key in allkeys:
              typestr = self.get(*key, cfg=cfg, field='type')
              value = self.get(*key, cfg=cfg)
  
              #create a TCL dict
              keystr = ' '.join(key)
  
              valstr = self._escape_val_tcl(value, typestr)
  
              if not (typestr.startswith('[') or typestr.startswith('(')):
                  # treat scalars as lists as well
                  valstr = f'[list {valstr}]'
  
              outstr = f"{prefix} {keystr} {valstr}\n"
  
              #print out all non default values
              if 'default' not in key:
                  fout.write(outstr)
  
      ###########################################################################
  
      def merge_manifest(self, cfg, job=None, clobber=True, clear=True, check=False):
          """
          Merges an external manifest with the current compilation manifest.
  
          All value fields in the provided schema dictionary are merged into the
          current chip object. Dictionaries with non-existent keypath produces a
          logger error message and raises the Chip object error flag.
  
          Args:
              job (str): Specifies non-default job to merge into
              clear (bool): If True, disables append operations for list type
              clobber (bool): If True, overwrites existing parameter value
              check (bool): If True, checks the validity of each key
              partial (bool): If True, perform a partial merge, only merging
                  keypaths that may have been updated during run().
  
          Examples:
              >>> chip.merge_manifest('my.pkg.json')
             Merges all parameters in my.pk.json into the Chip object
  
          """
          self._merge_manifest(cfg, job, clobber, clear, check)
  
      def _key_may_be_updated(self, keypath):
          '''Helper that returns whether `keypath` can be updated mid-run.'''
          # TODO: cleaner way to manage this?
          if keypath[0] in ('metric', 'record'):
              return True
          if keypath[0] == 'flowgraph' and keypath[4] in ('select', 'status'):
              return True
          return False
  
      ###########################################################################
      def _merge_manifest(self, cfg, job=None, clobber=True, clear=True, check=False, partial=False):
          """
          Internal merge_manifest() implementation with `partial` arg.
  
          partial (bool): If True, perform a partial merge, only merging keypaths
          that may have been updated during run().
          """
          if job is not None:
              # fill ith default schema before populating
              self.cfg['history'][job] = schema_cfg()
              dst = self.cfg['history'][job]
          else:
              dst = self.cfg
  
          for keylist in self.getkeys(cfg=cfg):
              if partial and not self._key_may_be_updated(keylist):
                  continue
              if keylist[0] in ('history', 'library'):
                  continue
              #only read in valid keypaths without 'default'
              key_valid = True
              if check:
                  key_valid = self.valid(*keylist, quiet=False, default_valid=True)
              if key_valid and 'default' not in keylist:
                  # update value, handling scalars vs. lists
                  typestr = self.get(*keylist, cfg=cfg, field='type')
                  val = self.get(*keylist, cfg=cfg)
                  arg = keylist.copy()
                  arg.append(val)
                  if bool(re.match(r'\[', typestr)) & bool(not clear):
                      self.add(*arg, cfg=dst)
                  else:
                      self.set(*arg, cfg=dst, clobber=clobber)
  
                  # update other fields that a user might modify
                  for field in self.getdict(*keylist, cfg=cfg).keys():
                      if field in ('value', 'switch', 'type', 'require', 'defvalue',
                                   'shorthelp', 'example', 'help'):
                          # skip these fields (value handled above, others are static)
                          continue
                      v = self.get(*keylist, cfg=cfg, field=field)
                      self.set(*keylist, v, cfg=dst, field=field)
  
      ###########################################################################
      def _keypath_empty(self, key):
          '''
          Utility function to check key for an empty list.
          '''
  
          emptylist = ("null", None, [])
  
          value = self.get(*key)
          defvalue = self.get(*key, field='defvalue')
          value_empty = (defvalue in emptylist) and (value in emptylist)
  
          return value_empty
  
      ###########################################################################
      def _check_files(self):
          allowed_paths = [os.path.join(self.cwd, self.get('option', 'builddir'))]
          allowed_paths.extend(os.environ['SC_VALID_PATHS'].split(os.pathsep))
  
          for keypath in self.getkeys():
              if 'default' in keypath:
                  continue
  
              paramtype = self.get(*keypath, field='type')
              #only do something if type is file or dir
              if ('history' not in keypath and 'library' not in keypath) and ('file' in paramtype or 'dir' in paramtype):
  
                  if self.get(*keypath) is None:
                      # skip unset values (some directories are None by default)
                      continue
  
                  abspaths = self.find_files(*keypath, missing_ok=True)
                  if not isinstance(abspaths, list):
                      abspaths = [abspaths]
  
                  for abspath in abspaths:
                      ok = False
  
                      if abspath is not None:
                          for allowed_path in allowed_paths:
                              if os.path.commonpath([abspath, allowed_path]) == allowed_path:
                                  ok = True
                                  continue
  
                      if not ok:
                          self.logger.error(f'Keypath {keypath} contains path(s) '
                              'that do not exist or resolve to files outside of '
                              'allowed directories.')
                          return False
  
          return True
  
      ###########################################################################
      def check_filepaths(self):
          '''
          Verifies that paths to all files in manifest are valid.
          '''
  
          allkeys = self.getkeys()
          for keypath in allkeys:
              allpaths = []
              paramtype = self.get(*keypath, field='type')
              if 'file' in paramtype or 'dir' in paramtype:
                  if 'dir' not in keypath and self.get(*keypath):
                      allpaths = list(self.get(*keypath))
                  for path in allpaths:
                      #check for env var
                      m = re.match(r'\$(\w+)(.*)', path)
                      if m:
                          prefix_path = os.environ[m.group(1)]
                          path = prefix_path + m.group(2)
                      file_error = 'file' in paramtype and not os.path.isfile(path)
                      dir_error = 'dir' in paramtype and not os.path.isdir(path)
                      if file_error or dir_error:
                          self.logger.error(f"Paramater {keypath} path {path} is invalid")
                          self.error = 1
  
      ###########################################################################
      def _check_manifest_dynamic(self, step, index):
          '''Runtime checks called from _runtask().
  
          - Make sure expected inputs exist.
          - Make sure all required filepaths resolve correctly.
          '''
          flow = self.get('option', 'flow')
          tool = self.get('flowgraph', flow, step, index, 'tool')
          if self.valid('tool', tool, 'input', step, index):
              required_inputs = self.get('tool', tool, 'input', step, index)
          else:
              required_inputs = []
          input_dir = os.path.join(self._getworkdir(step=step, index=index), 'inputs')
          for filename in required_inputs:
              path = os.path.join(input_dir, filename)
              if not os.path.isfile(path):
                  self.logger.error(f'Required input {filename} not received for {step}{index}.')
                  self.error = 1
  
          if (not tool in self.builtin) and self.valid('tool', tool, 'require', step, index):
              all_required = self.get('tool', tool, 'require', step, index)
              for item in all_required:
                  keypath = item.split(',')
                  paramtype = self.get(*keypath, field='type')
                  if ('file' in paramtype) or ('dir' in paramtype):
                      abspath = self.find_files(*keypath)
                      if abspath is None or (isinstance(abspath, list) and None in abspath):
                          self.logger.error(f"Required file keypath {keypath} can't be resolved.")
                          self.error = 1
  
          # Need to run this check here since file resolution can change in
          # _runtask().
          if 'SC_VALID_PATHS' in os.environ:
              if not self._check_files():
                  self.error = 1
  
          return self.error
  
      ###########################################################################
      def check_manifest(self):
          '''
          Verifies the integrity of the pre-run compilation manifest.
  
          Checks the validity of the current schema manifest in
          memory to ensure that the design has been properly set up prior
          to running compilation. The function is called inside the run()
          function but can also be called separately. Checks performed by the
          check_manifest() function include:
  
          * Has a flowgraph been defined?
          * Does the manifest satisfy the schema requirement field settings?
          * Are all flowgraph input names legal step/index pairs?
          * Are the tool parameter setting requirements met?
  
          Returns:
              Returns True if the manifest is valid, else returns False.
  
          Examples:
              >>> manifest_ok = chip.check_manifest()
              Returns True of the Chip object dictionary checks out.
  
          '''
          # Dynamic checks
          # We only perform these if arg, step and arg, index are set.
          # We don't check inputs for skip all
          # TODO: Need to add skip step
  
          cur_step = self.get('arg', 'step')
          cur_index = self.get('arg', 'index')
          if cur_step and cur_index and not self.get('option', 'skipall'):
              return self._check_manifest_dynamic(cur_step, cur_index)
  
          design = self.get('design')
          flow = self.get('option', 'flow')
          jobname = self.get('option', 'jobname')
          steplist = self.get('option', 'steplist')
          if not steplist:
              steplist = self.list_steps()
  
          #1. Checking that flowgraph and steplist are legal
          if flow not in self.getkeys('flowgraph'):
              self.error = 1
              self.logger.error(f"flowgraph {flow} not defined.")
          legal_steps = self.getkeys('flowgraph',flow)
  
          if 'import' not in legal_steps:
              self.error = 1
              self.logger.error("Flowgraph doesn't contain import step.")
  
          indexlist = {}
          #TODO: refactor
          for step in steplist:
              if self.get('option', 'indexlist'):
                  indexlist[step] = self.get('option', 'indexlist')
              else:
                  indexlist[step] = self.getkeys('flowgraph', flow, step)
  
          for step in steplist:
              for index in indexlist[step]:
                  in_job = None
                  if (step in self.getkeys('option', 'jobinput') and
                      index in self.getkeys('option', 'jobinput', step)):
                      in_job = self.get('option', 'jobinput', step, index)
  
                  for in_step, in_index in self.get('flowgraph', flow, step, index, 'input'):
                      if in_job is not None:
                          workdir = self._getworkdir(jobname=in_job, step=in_step, index=in_index)
                          cfg = os.path.join(workdir, 'outputs', f'{design}.pkg.json')
                          if not os.path.isfile(cfg):
                              self.logger.error(f'{step}{index} relies on {in_step}{in_index} from job {in_job}, '
                                  'but this task has not been run.')
                              self.error = 1
                          continue
                      if in_step in steplist and in_index in indexlist[in_step]:
                          # we're gonna run this step, OK
                          continue
                      if self.get('flowgraph', flow, in_step, in_index, 'status') == TaskStatus.SUCCESS:
                          # this task has already completed successfully, OK
                          continue
                      self.logger.error(f'{step}{index} relies on {in_step}{in_index}, '
                          'but this task has not been run and is not in the current steplist.')
                      self.error = 1
  
          #2. Check libary names
          for item in self.get('asic', 'logiclib'):
              if item not in self.getkeys('library'):
                  self.error = 1
                  self.logger.error(f"Target library {item} not found.")
  
          #3. Check requirements list
          allkeys = self.getkeys()
          for key in allkeys:
              keypath = ",".join(key)
              if 'default' not in key and 'history' not in key and 'library' not in key:
                  key_empty = self._keypath_empty(key)
                  requirement = self.get(*key, field='require')
                  if key_empty and (str(requirement) == 'all'):
                      self.error = 1
                      self
  
  [... the rest is omitted ...]
files.old_path: siliconcompiler/core.py
files.new_contents: # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
  import pytest
  
  import siliconcompiler
  
  def test_write_manifest():
  
      chip = siliconcompiler.Chip('top')
      chip.add('input', 'sdc','top.sdc')
      chip.add('input', 'verilog', 'top.v')
      chip.add('input', 'verilog', 'a.v')
      chip.add('input', 'verilog', 'b.v')
      chip.add('input', 'verilog', 'c.v')
  
      chip.write_manifest('top.pkg.json')
      chip.write_manifest('top.csv')
      chip.write_manifest('top.tcl', prune=False)
      chip.write_manifest('top.yaml')
  
  def test_advanced_tcl(monkeypatch):
      # Tkinter module is part of Python standard library, but may not be
      # available depending on if the system has the python3-tk package installed.
      # This line will import tkinter if it's available, and skip the test
      # otherwise.
      tkinter = pytest.importorskip('tkinter')
  
      chip = siliconcompiler.Chip('top')
  
      # Test complex strings
      desc = '''This description is potentially problematic since it includes
  multiple lines, spaces, and TCL special characters. This package costs $5 {for real}!'''
      chip.set('package', 'description', desc)
  
      # Test tuples
      chip.add('asic', 'diearea', (0, 0))
      chip.add('asic', 'diearea', (30, 40))
  
      # Test bools
      chip.set('option', 'quiet', True)
  
      # Test envvars
      chip.set('input', 'verilog', 'rtl/$TOPMOD.v')
  
      chip.write_manifest('top.tcl')
  
      # Read from config in TCL as test
      tcl = tkinter.Tcl()
  
      # Set env var to test ['input', 'verilog']
      monkeypatch.setenv('TOPMOD', 'design')
  
      def tcl_eval(expr):
          script = f'''
          source top.tcl
          return {expr}'''
          return tcl.eval(script)
  
      # When the TCL shell displays a multiline string, it gets surrounded in {}.
      expected_desc = '{' + desc + '}'
      assert tcl_eval('[dict get $sc_cfg package description]') == expected_desc
  
      assert tcl_eval('[lindex [lindex [dict get $sc_cfg asic diearea] 1] 0]') == '30.0'
      assert tcl_eval('[dict get $sc_cfg option quiet]') == 'true'
      assert tcl_eval('[dict get $sc_cfg input verilog]') == 'rtl/design.v'
  
  #########################
  if __name__ == "__main__":
      test_write_manifest()
  
  
  [... the rest is omitted ...]
files.new_path: tests/core/test_write_manifest.py
files.old_contents: # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
  import pytest
  
  import siliconcompiler
  
  def test_write_manifest():
  
      chip = siliconcompiler.Chip('top')
      chip.add('input', 'sdc','top.sdc')
      chip.add('input', 'verilog', 'top.v')
      chip.add('input', 'verilog', 'a.v')
      chip.add('input', 'verilog', 'b.v')
      chip.add('input', 'verilog', 'c.v')
  
      chip.write_manifest('top.pkg.json')
      chip.write_manifest('top.csv')
      chip.write_manifest('top.tcl', prune=False)
      chip.write_manifest('top.yaml')
  
  def test_advanced_tcl(monkeypatch):
      # Tkinter module is part of Python standard library, but may not be
      # available depending on if the system has the python3-tk installed. This
      # line will import tkinter if it's available, and skip the test otherwise.
      tkinter = pytest.importorskip('tkinter')
  
      chip = siliconcompiler.Chip('top')
  
      # Test complex strings
      desc = '''This description is potentially problematic since it includes
  multiple lines, spaces, and TCL special characters. This package costs $5 {for real}!'''
      chip.set('package', 'description', desc)
  
      # Test tuples
      chip.add('asic', 'diearea', (0, 0))
      chip.add('asic', 'diearea', (30, 40))
  
      # Test bools
      chip.set('option', 'quiet', True)
  
      # Test envvars
      chip.set('input', 'verilog', 'rtl/$TOPMOD.v')
  
      chip.write_manifest('top.tcl')
  
      # Read from config in TCL as test
      tcl = tkinter.Tcl()
  
      # Set env var to test ['input', 'verilog']
      monkeypatch.setenv('TOPMOD', 'design')
  
      def tcl_eval(expr):
          script = f'''
          source top.tcl
          return {expr}'''
          return tcl.eval(script)
  
      # When we call puts on a multiline string, it does get mangled a bit. It
      # stays surrounded by {}, and {} within the string are still escaped.
      # TODO: is this problematic? I think it is okay for now since we don't
      # really read these strings within TCL, they just need to not break sourcing
      # the manifest.
      expected_desc = '{' + desc.replace('{', '\\{').replace('}', '\\}') + '}'
      assert tcl_eval('[dict get $sc_cfg package description]') == expected_desc
  
      assert tcl_eval('[lindex [lindex [dict get $sc_cfg asic diearea] 1] 0]') == '30.0'
      assert tcl_eval('[dict get $sc_cfg option quiet]') == 'true'
      assert tcl_eval('[dict get $sc_cfg input verilog]') == 'rtl/design.v'
  
  #########################
  if __name__ == "__main__":
      test_write_manifest()
  
  
  [... the rest is omitted ...]
files.old_path: tests/core/test_write_manifest.py