files: # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
  
  import argparse
  import base64
  import time
  import datetime
  import multiprocessing
  import tarfile
  import traceback
  import asyncio
  from subprocess import run, PIPE
  import os
  import glob
  import pathlib
  import sys
  import gzip
  import re
  import json
  import logging
  import hashlib
  import shutil
  import copy
  import importlib
  import textwrap
  import math
  import pandas
  import yaml
  import graphviz
  import time
  import uuid
  import shlex
  import platform
  import getpass
  import distro
  import netifaces
  import webbrowser
  import packaging.version
  import packaging.specifiers
  from jinja2 import Environment, FileSystemLoader
  from pathlib import Path
  from timeit import default_timer as timer
  from siliconcompiler.client import *
  from siliconcompiler.schema import *
  from siliconcompiler.scheduler import _deferstep
  from siliconcompiler import leflib
  from siliconcompiler import utils
  from siliconcompiler import _metadata
  import psutil
  
  class TaskStatus():
      # Could use Python 'enum' class here, but that doesn't work nicely with
      # schema.
      PENDING = 'pending'
      SUCCESS = 'success'
      ERROR = 'error'
  
  class Chip:
      """Object for configuring and executing hardware design flows.
  
      This is the main object used for configuration, data, and
      execution within the SiliconCompiler platform.
  
      Args:
          design (string): Name of the top level chip design module.
  
      Examples:
          >>> siliconcompiler.Chip(design="top")
          Creates a chip object with name "top".
      """
  
      ###########################################################################
      def __init__(self, design, loglevel=None):
          # version numbers
          self.scversion = _metadata.version
          self.schemaversion = SCHEMA_VERSION
  
          # Local variables
          self.scroot = os.path.dirname(os.path.abspath(__file__))
          self.cwd = os.getcwd()
          self.error = 0
          self.cfg = schema_cfg()
          # The 'status' dictionary can be used to store ephemeral config values.
          # Its contents will not be saved, and can be set by parent scripts
          # such as a web server or supervisor process. Currently supported keys:
          # * 'jobhash': A hash or UUID which can identify jobs in a larger system.
          # * 'remote_cfg': Dictionary containing remote server configurations
          #                 (address, credentials, etc.)
          # * 'slurm_account': User account ID in a connected slurm HPC cluster.
          # * 'slurm_partition': Name of the partition in which a task should run
          #                      on a connected slurm HPC cluster.
          # * 'watchdog': Activity-monitoring semaphore for jobs scheduled on an
          #               HPC cluster; expects a 'threading.Event'-like object.
          # * 'max_fs_bytes': A limit on how much disk space a job is allowed
          #                   to consume in a connected HPC cluster's storage.
          self.status = {}
  
          self.builtin = ['minimum','maximum',
                          'nop', 'mux', 'join', 'verify']
  
          # We set 'design' and 'loglevel' directly in the config dictionary
          # because of a chicken-and-egg problem: self.set() relies on the logger,
          # but the logger relies on these values.
          self.cfg['design']['value'] = design
          if loglevel:
              self.cfg['option']['loglevel']['value'] = loglevel
  
          self._init_logger()
  
          self._loaded_modules = {
              'flows': [],
              'pdks': [],
              'libs': [],
              'checklists': []
          }
  
      ###########################################################################
      @property
      def design(self):
          '''Design name of chip object.
  
          This is an immutable property.'''
          return self.get('design')
  
      ###########################################################################
      def _init_logger(self, step=None, index=None, in_run=False):
  
          self.logger = logging.getLogger(uuid.uuid4().hex)
  
          # Don't propagate log messages to "root" handler (we get duplicate
          # messages without this)
          # TODO: this prevents us from being able to capture logs with pytest:
          # we should revisit it
          self.logger.propagate = False
  
          loglevel = self.get('option', 'loglevel')
  
          if loglevel=='DEBUG':
              prefix = '| %(levelname)-7s | %(funcName)-10s | %(lineno)-4s'
          else:
              prefix = '| %(levelname)-7s'
  
          if in_run:
              flow = self.get('option', 'flow')
  
              # Figure out how wide to make step and index fields
              max_step_len = 2
              max_index_len = 2
              for future_step in self.getkeys('flowgraph', flow):
                  max_step_len = max(len(future_step) + 1, max_step_len)
                  for future_index in self.getkeys('flowgraph', flow, future_step):
                      max_index_len = max(len(future_index) + 1, max_index_len)
  
              jobname = self.get('option', 'jobname')
  
              if step is None:
                  step = '-' * max(max_step_len // 4, 1)
              if index is None:
                  index = '-' * max(max_index_len // 4, 1)
  
              run_info = f'%s  | %-{max_step_len}s | %-{max_index_len}s' % (jobname, step, index)
              logformat = ' | '.join([prefix, run_info, '%(message)s'])
          else:
              logformat = ' | '.join([prefix, '%(message)s'])
  
          handler = logging.StreamHandler()
          formatter = logging.Formatter(logformat)
  
          handler.setFormatter(formatter)
  
          # Clear any existing handlers so we don't end up with duplicate messages
          # if repeat calls to _init_logger are made
          if len(self.logger.handlers) > 0:
              self.logger.handlers.clear()
  
          self.logger.addHandler(handler)
          self.logger.setLevel(loglevel)
  
      ###########################################################################
      def _deinit_logger(self):
          self.logger = None
  
      ###########################################################################
      def _get_switches(self, *keypath):
          '''Helper function for parsing switches and metavars for a keypath.'''
          #Switch field fully describes switch format
          switch = self.get(*keypath, field='switch')
  
          if switch is None:
              switches = []
          elif isinstance(switch, list):
              switches = switch
          else:
              switches = [switch]
          switchstrs = []
  
          # parse out switch from metavar
          # TODO: should we validate that metavar matches for each switch?
          for switch in switches:
              switchmatch = re.match(r'(-[\w_]+)\s+(.*)', switch)
              gccmatch = re.match(r'(-[\w_]+)(.*)', switch)
              plusmatch = re.match(r'(\+[\w_\+]+)(.*)', switch)
  
              if switchmatch:
                  switchstr = switchmatch.group(1)
                  metavar = switchmatch.group(2)
              elif gccmatch:
                  switchstr = gccmatch.group(1)
                  metavar = gccmatch.group(2)
              elif plusmatch:
                  switchstr = plusmatch.group(1)
                  metavar = plusmatch.group(2)
              switchstrs.append(switchstr)
  
          return switchstrs, metavar
  
      ###########################################################################
      def create_cmdline(self, progname, description=None, switchlist=None, input_map=None):
          """Creates an SC command line interface.
  
          Exposes parameters in the SC schema as command line switches,
          simplifying creation of SC apps with a restricted set of schema
          parameters exposed at the command line. The order of command
          line switch settings parsed from the command line is as follows:
  
           1. loglevel
           2. fpga_partname
           3. load_target('target')
           4. read_manifest([cfg])
           5. all other switches
  
          The cmdline interface is implemented using the Python argparse package
          and the following use restrictions apply.
  
          * Help is accessed with the '-h' switch.
          * Arguments that include spaces must be enclosed with double quotes.
          * List parameters are entered individually. (ie. -y libdir1 -y libdir2)
          * For parameters with Boolean types, the switch implies "true".
          * Special characters (such as '-') must be enclosed in double quotes.
          * Compiler compatible switches include: -D, -I, -O{0,1,2,3}
          * Verilog legacy switch formats are supported: +libext+, +incdir+
  
          Args:
              progname (str): Name of program to be executed.
              description (str): Short program description.
              switchlist (list of str): List of SC parameter switches to expose
                  at the command line. By default all SC schema switches are
                  available. Parameter switches should be entered based on the
                  parameter
  
  [... the rest is omitted ...]
files: siliconcompiler/core.py
files: # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
  
  import argparse
  import base64
  import time
  import datetime
  import multiprocessing
  import tarfile
  import traceback
  import asyncio
  from subprocess import run, PIPE
  import os
  import glob
  import pathlib
  import sys
  import gzip
  import re
  import json
  import logging
  import hashlib
  import shutil
  import copy
  import importlib
  import textwrap
  import math
  import pandas
  import yaml
  import graphviz
  import time
  import uuid
  import shlex
  import platform
  import getpass
  import distro
  import netifaces
  import webbrowser
  import packaging.version
  import packaging.specifiers
  from jinja2 import Environment, FileSystemLoader
  from pathlib import Path
  from timeit import default_timer as timer
  from siliconcompiler.client import *
  from siliconcompiler.schema import *
  from siliconcompiler.scheduler import _deferstep
  from siliconcompiler import leflib
  from siliconcompiler import utils
  from siliconcompiler import _metadata
  import psutil
  
  class TaskStatus():
      # Could use Python 'enum' class here, but that doesn't work nicely with
      # schema.
      PENDING = 'pending'
      SUCCESS = 'success'
      ERROR = 'error'
  
  class Chip:
      """Object for configuring and executing hardware design flows.
  
      This is the main object used for configuration, data, and
      execution within the SiliconCompiler platform.
  
      Args:
          design (string): Name of the top level chip design module.
  
      Examples:
          >>> siliconcompiler.Chip(design="top")
          Creates a chip object with name "top".
      """
  
      ###########################################################################
      def __init__(self, design, loglevel=None):
          # version numbers
          self.scversion = _metadata.version
          self.schemaversion = SCHEMA_VERSION
  
          # Local variables
          self.scroot = os.path.dirname(os.path.abspath(__file__))
          self.cwd = os.getcwd()
          self.error = 0
          self.cfg = schema_cfg()
          # The 'status' dictionary can be used to store ephemeral config values.
          # Its contents will not be saved, and can be set by parent scripts
          # such as a web server or supervisor process. Currently supported keys:
          # * 'jobhash': A hash or UUID which can identify jobs in a larger system.
          # * 'remote_cfg': Dictionary containing remote server configurations
          #                 (address, credentials, etc.)
          # * 'slurm_account': User account ID in a connected slurm HPC cluster.
          # * 'slurm_partition': Name of the partition in which a task should run
          #                      on a connected slurm HPC cluster.
          # * 'watchdog': Activity-monitoring semaphore for jobs scheduled on an
          #               HPC cluster; expects a 'threading.Event'-like object.
          # * 'max_fs_bytes': A limit on how much disk space a job is allowed
          #                   to consume in a connected HPC cluster's storage.
          self.status = {}
  
          self.builtin = ['minimum','maximum',
                          'nop', 'mux', 'join', 'verify']
  
          # We set 'design' and 'loglevel' directly in the config dictionary
          # because of a chicken-and-egg problem: self.set() relies on the logger,
          # but the logger relies on these values.
          self.cfg['design']['value'] = design
          if loglevel:
              self.cfg['option']['loglevel']['value'] = loglevel
  
          self._init_logger()
  
          self._loaded_modules = {
              'flows': [],
              'pdks': [],
              'libs': [],
              'checklists': []
          }
  
      ###########################################################################
      @property
      def design(self):
          '''Design name of chip object.
  
          This is an immutable property.'''
          return self.get('design')
  
      ###########################################################################
      def _init_logger(self, step=None, index=None, in_run=False):
  
          self.logger = logging.getLogger(uuid.uuid4().hex)
  
          # Don't propagate log messages to "root" handler (we get duplicate
          # messages without this)
          # TODO: this prevents us from being able to capture logs with pytest:
          # we should revisit it
          self.logger.propagate = False
  
          loglevel = self.get('option', 'loglevel')
  
          if loglevel=='DEBUG':
              prefix = '| %(levelname)-7s | %(funcName)-10s | %(lineno)-4s'
          else:
              prefix = '| %(levelname)-7s'
  
          if in_run:
              flow = self.get('option', 'flow')
  
              # Figure out how wide to make step and index fields
              max_step_len = 2
              max_index_len = 2
              for future_step in self.getkeys('flowgraph', flow):
                  max_step_len = max(len(future_step) + 1, max_step_len)
                  for future_index in self.getkeys('flowgraph', flow, future_step):
                      max_index_len = max(len(future_index) + 1, max_index_len)
  
              jobname = self.get('option', 'jobname')
  
              if step is None:
                  step = '-' * max(max_step_len // 4, 1)
              if index is None:
                  index = '-' * max(max_index_len // 4, 1)
  
              run_info = f'%s  | %-{max_step_len}s | %-{max_index_len}s' % (jobname, step, index)
              logformat = ' | '.join([prefix, run_info, '%(message)s'])
          else:
              logformat = ' | '.join([prefix, '%(message)s'])
  
          handler = logging.StreamHandler()
          formatter = logging.Formatter(logformat)
  
          handler.setFormatter(formatter)
  
          # Clear any existing handlers so we don't end up with duplicate messages
          # if repeat calls to _init_logger are made
          if len(self.logger.handlers) > 0:
              self.logger.handlers.clear()
  
          self.logger.addHandler(handler)
          self.logger.setLevel(loglevel)
  
      ###########################################################################
      def _deinit_logger(self):
          self.logger = None
  
      ###########################################################################
      def _get_switches(self, *keypath):
          '''Helper function for parsing switches and metavars for a keypath.'''
          #Switch field fully describes switch format
          switch = self.get(*keypath, field='switch')
  
          if switch is None:
              switches = []
          elif isinstance(switch, list):
              switches = switch
          else:
              switches = [switch]
          switchstrs = []
  
          # parse out switch from metavar
          # TODO: should we validate that metavar matches for each switch?
          for switch in switches:
              switchmatch = re.match(r'(-[\w_]+)\s+(.*)', switch)
              gccmatch = re.match(r'(-[\w_]+)(.*)', switch)
              plusmatch = re.match(r'(\+[\w_\+]+)(.*)', switch)
  
              if switchmatch:
                  switchstr = switchmatch.group(1)
                  metavar = switchmatch.group(2)
              elif gccmatch:
                  switchstr = gccmatch.group(1)
                  metavar = gccmatch.group(2)
              elif plusmatch:
                  switchstr = plusmatch.group(1)
                  metavar = plusmatch.group(2)
              switchstrs.append(switchstr)
  
          return switchstrs, metavar
  
      ###########################################################################
      def create_cmdline(self, progname, description=None, switchlist=None, input_map=None):
          """Creates an SC command line interface.
  
          Exposes parameters in the SC schema as command line switches,
          simplifying creation of SC apps with a restricted set of schema
          parameters exposed at the command line. The order of command
          line switch settings parsed from the command line is as follows:
  
           1. loglevel
           2. fpga_partname
           3. load_target('target')
           4. read_manifest([cfg])
           5. all other switches
  
          The cmdline interface is implemented using the Python argparse package
          and the following use restrictions apply.
  
          * Help is accessed with the '-h' switch.
          * Arguments that include spaces must be enclosed with double quotes.
          * List parameters are entered individually. (ie. -y libdir1 -y libdir2)
          * For parameters with Boolean types, the switch implies "true".
          * Special characters (such as '-') must be enclosed in double quotes.
          * Compiler compatible switches include: -D, -I, -O{0,1,2,3}
          * Verilog legacy switch formats are supported: +libext+, +incdir+
  
          Args:
              progname (str): Name of program to be executed.
              description (str): Short program description.
              switchlist (list of str): List of SC parameter switches to expose
                  at the command line. By default all SC schema switches are
                  available. Parameter switches should be entered based on the
                  parameter
  
  [... the rest is omitted ...]
files: siliconcompiler/core.py
files: # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
  import pytest
  
  import siliconcompiler
  
  def test_write_manifest():
  
      chip = siliconcompiler.Chip('top')
      chip.add('input', 'sdc','top.sdc')
      chip.add('input', 'verilog', 'top.v')
      chip.add('input', 'verilog', 'a.v')
      chip.add('input', 'verilog', 'b.v')
      chip.add('input', 'verilog', 'c.v')
  
      chip.write_manifest('top.pkg.json')
      chip.write_manifest('top.csv')
      chip.write_manifest('top.tcl', prune=False)
      chip.write_manifest('top.yaml')
  
  def test_advanced_tcl(monkeypatch):
      # Tkinter module is part of Python standard library, but may not be
      # available depending on if the system has the python3-tk installed. This
      # line will import tkinter if it's available, and skip the test otherwise.
      tkinter = pytest.importorskip('tkinter')
  
      chip = siliconcompiler.Chip('top')
  
      # Test complex strings
      desc = '''This description is potentially problematic since it includes
  multiple lines, spaces, and TCL special characters. This package costs $5 {for real}!'''
      chip.set('package', 'description', desc)
  
      # Test tuples
      chip.add('asic', 'diearea', (0, 0))
      chip.add('asic', 'diearea', (30, 40))
  
      # Test bools
      chip.set('option', 'quiet', True)
  
      # Test envvars
      chip.set('input', 'verilog', 'rtl/$TOPMOD.v')
  
      chip.write_manifest('top.tcl')
  
      # Read from config in TCL as test
      tcl = tkinter.Tcl()
  
      # Set env var to test ['input', 'verilog']
      monkeypatch.setenv('TOPMOD', 'design')
  
      def tcl_eval(expr):
          script = f'''
          source top.tcl
          return {expr}'''
          return tcl.eval(script)
  
      # When we call puts on a multiline string, it does get mangled a bit. It
      # stays surrounded by {}, and {} within the string are still escaped.
      # TODO: is this problematic? I think it is okay for now since we don't
      # really read these strings within TCL, they just need to not break sourcing
      # the manifest.
      expected_desc = '{' + desc.replace('{', '\\{').replace('}', '\\}') + '}'
      assert tcl_eval('[dict get $sc_cfg package description]') == expected_desc
  
      assert tcl_eval('[lindex [lindex [dict get $sc_cfg asic diearea] 1] 0]') == '30.0'
      assert tcl_eval('[dict get $sc_cfg option quiet]') == 'true'
      assert tcl_eval('[dict get $sc_cfg input verilog]') == 'rtl/design.v'
  
  #########################
  if __name__ == "__main__":
      test_write_manifest()
  
  
  [... the rest is omitted ...]
files: tests/core/test_write_manifest.py
files: # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
  import siliconcompiler
  
  def test_write_manifest():
  
      chip = siliconcompiler.Chip('top')
      chip.add('input', 'sdc','top.sdc')
      chip.add('input', 'verilog', 'top.v')
      chip.add('input', 'verilog', 'a.v')
      chip.add('input', 'verilog', 'b.v')
      chip.add('input', 'verilog', 'c.v')
  
      chip.write_manifest('top.pkg.json')
      chip.write_manifest('top.csv')
      chip.write_manifest('top.tcl', prune=False)
      chip.write_manifest('top.yaml')
  
  #########################
  if __name__ == "__main__":
      test_write_manifest()
  
  
  [... the rest is omitted ...]
files: tests/core/test_write_manifest.py