files: '''Sphinx extension that provides directives for automatically generating
  documentation for the three types of dynamically loaded modules used by SC:
  flows, foundries, and tools.
  '''
  
  from docutils import nodes
  from sphinx.util.nodes import nested_parse_with_titles
  from docutils.statemachine import ViewList
  from sphinx.util.docutils import SphinxDirective
  import docutils
  from docutils.parsers.rst import directives
  
  import copy
  import importlib
  import pkgutil
  import os
  import sys
  import subprocess
  
  from common import *
  
  from siliconcompiler import utils
  
  #############
  # Helpers
  #############
  
  # We need this in a few places, so just make it global
  SC_ROOT = os.path.abspath(f'{__file__}/../../../')
  
  def build_schema_value_table(schema, keypath_prefix=[], skip_zero_weight=False):
      '''Helper function for displaying values set in schema as a docutils table.'''
      table = [[strong('Keypath'), strong('Value')]]
      flat_cfg = flatten(schema)
      for keys, val in flat_cfg.items():
          full_keypath = list(keypath_prefix) + list(keys)
  
          if (skip_zero_weight and
              len(full_keypath) == 6 and full_keypath[0] == 'flowgraph' and full_keypath[-2] == 'weight' and
              'value' in val and val['value'] == '0'):
              continue
  
          keypath = ', '.join(full_keypath)
  
          if 'value' in val and val['value']:
              # Don't display false booleans
              if val['type'] == 'bool' and val['value'] == 'false':
                  continue
              if val['type'].startswith('['):
                  if len(val['value']) > 1:
                      val_node = build_list([code(v) for v in val['value']])
                  elif len(val['value']) > 0:
                      val_node = code(val['value'][0])
                  else:
                      val_node = para('')
              else:
                  val_node = code(val['value'])
  
              table.append([code(keypath), val_node])
  
      if len(table) > 1:
          return build_table(table)
      else:
          return None
  
  def build_config_recursive(schema, keypath_prefix=[], sec_key_prefix=[]):
      '''Helper function for displaying schema at each level as tables unde nested
      sections.
  
      For each item:
      - If it's a leaf, collect it into a table we will display at this
          level
      - Otherwise, recurse and collect sections of lower levels
      '''
      leaves = {}
      child_sections = []
      for key, val in schema.items():
          if key == 'default': continue
          if 'help' in val:
              if 'value' in val and val['value']:
                  leaves.update({key: val})
          else:
              children = build_config_recursive(val, keypath_prefix=keypath_prefix+[key], sec_key_prefix=sec_key_prefix)
              child_sections.extend(children)
  
      # If we've found leaves, create a new section where we'll display a
      # table plus all child sections.
      if len(leaves) > 0:
          keypath = ', '.join(keypath_prefix)
          section_key = '-'.join(sec_key_prefix + keypath_prefix)
          top = build_section(keypath, section_key)
          top += build_schema_value_table(leaves, keypath_prefix=keypath_prefix)
          top += child_sections
          return [top]
      else:
          # Otherwise, just pass on the child sections -- we don't want to
          # create an extra level of section hierarchy for levels of the
          # schema without leaves.
          return child_sections
  
  #############
  # Base class
  #############
  
  def flag_opt(argument):
      if argument is not None:
          raise ValueError('Flag should not have content')
      return True
  
  class DynamicGen(SphinxDirective):
      '''Base class for all three directives provided by this extension.
  
      Each child class implements a directive by overriding the display_config()
      method and setting a PATH member variable.
      '''
  
      option_spec = {'nobuiltins': flag_opt}
  
      def document_module(self, module, modname, path):
          '''Build section documenting given module and name.'''
          s = build_section_with_target(modname, f'{modname}-ref', self.state.document)
  
          if not hasattr(module, 'make_docs'):
              return None
  
          make_docs = getattr(module, 'make_docs')
  
          # raw docstrings have fun
  
  [... the rest is omitted ...]
files: docs/_ext/dynamicgen.py
files: '''Sphinx extension that provides directives for automatically generating
  documentation for the three types of dynamically loaded modules used by SC:
  flows, foundries, and tools.
  '''
  
  from docutils import nodes
  from sphinx.util.nodes import nested_parse_with_titles
  from docutils.statemachine import ViewList
  from sphinx.util.docutils import SphinxDirective
  import docutils
  from docutils.parsers.rst import directives
  
  import copy
  import importlib
  import pkgutil
  import os
  import sys
  import subprocess
  
  from common import *
  
  import siliconcompiler
  
  #############
  # Helpers
  #############
  
  # We need this in a few places, so just make it global
  SC_ROOT = os.path.abspath(f'{__file__}/../../../')
  
  def build_schema_value_table(schema, keypath_prefix=[], skip_zero_weight=False):
      '''Helper function for displaying values set in schema as a docutils table.'''
      table = [[strong('Keypath'), strong('Value')]]
      flat_cfg = flatten(schema)
      for keys, val in flat_cfg.items():
          full_keypath = list(keypath_prefix) + list(keys)
  
          if (skip_zero_weight and
              len(full_keypath) == 6 and full_keypath[0] == 'flowgraph' and full_keypath[-2] == 'weight' and
              'value' in val and val['value'] == '0'):
              continue
  
          keypath = ', '.join(full_keypath)
  
          if 'value' in val and val['value']:
              # Don't display false booleans
              if val['type'] == 'bool' and val['value'] == 'false':
                  continue
              if val['type'].startswith('['):
                  if len(val['value']) > 1:
                      val_node = build_list([code(v) for v in val['value']])
                  elif len(val['value']) > 0:
                      val_node = code(val['value'][0])
                  else:
                      val_node = para('')
              else:
                  val_node = code(val['value'])
  
              table.append([code(keypath), val_node])
  
      if len(table) > 1:
          return build_table(table)
      else:
          return None
  
  def build_config_recursive(schema, keypath_prefix=[], sec_key_prefix=[]):
      '''Helper function for displaying schema at each level as tables unde nested
      sections.
  
      For each item:
      - If it's a leaf, collect it into a table we will display at this
          level
      - Otherwise, recurse and collect sections of lower levels
      '''
      leaves = {}
      child_sections = []
      for key, val in schema.items():
          if key == 'default': continue
          if 'help' in val:
              if 'value' in val and val['value']:
                  leaves.update({key: val})
          else:
              children = build_config_recursive(val, keypath_prefix=keypath_prefix+[key], sec_key_prefix=sec_key_prefix)
              child_sections.extend(children)
  
      # If we've found leaves, create a new section where we'll display a
      # table plus all child sections.
      if len(leaves) > 0:
          keypath = ', '.join(keypath_prefix)
          section_key = '-'.join(sec_key_prefix + keypath_prefix)
          top = build_section(keypath, section_key)
          top += build_schema_value_table(leaves, keypath_prefix=keypath_prefix)
          top += child_sections
          return [top]
      else:
          # Otherwise, just pass on the child sections -- we don't want to
          # create an extra level of section hierarchy for levels of the
          # schema without leaves.
          return child_sections
  
  def trim(docstring):
      '''Helper function for cleaning up indentation of docstring.
  
      This is important for properly parsing complex RST in our docs.
  
      Source:
      https://www.python.org/dev/peps/pep-0257/#handling-docstring-indentation'''
      if not docstring:
          return ''
      # Convert tabs to spaces (following the normal Python rules)
      # and split into a list of lines:
      lines = docstring.expandtabs().splitlines()
      # Determine minimum indentation (first line doesn't count):
      indent = sys.maxsize
      for line in lines[1:]:
          stripped = line.lstrip()
          if stripped:
              indent = min(indent, len(line) - len(stripped))
      # Remove indentation (first line is special):
      trimmed = [lines[0].strip()]
      if indent
  
  [... the rest is omitted ...]
files: docs/_ext/dynamicgen.py
files: from docutils import nodes
  from sphinx.util.nodes import nested_parse_with_titles
  from docutils.statemachine import ViewList
  from sphinx.util.docutils import SphinxDirective
  from siliconcompiler.schema import schema_cfg
  
  from common import *
  
  # Main Sphinx plugin
  class SchemaGen(SphinxDirective):
  
      def run(self):
          self.env.note_dependency('../siliconcompiler/schema.py')
  
          schema = schema_cfg()
  
          # Split up schema into root and nested keys so that we can add a custom
          # header to the root entries.
          basic_schema = {}
          nested_schema = {}
          for key, val in schema.items():
              if is_leaf(val):
                  basic_schema[key] = val
              else:
                  nested_schema[key] = val
  
          basic_section = build_section('root', 'root')
          for n in self.process_schema(basic_schema):
              basic_section += n
  
          nested_sections = self.process_schema(nested_schema)
  
          return [basic_section] + nested_sections
  
      def process_schema(self, schema, parents=[]):
          if 'help' in schema:
              entries = [[strong('Description'),   para(schema['shorthelp'])],
                         [strong('Type'),          para(schema['type'])],
                         [strong('Default Value'), para(schema['defvalue'])],
                         [strong('CLI Switch'),    code(schema['switch'])]]
              for example in schema['example']:
                  name, ex = example.split(':', 1)
                  entries.append([strong(f'Example ({name.upper()})'), code(ex.strip())])
  
              table = build_table(entries)
              body = self.parse_rst(schema['help'])
  
              return [table, body]
          else:
              sections = []
              for key in schema.keys():
                  if key == 'default':
                      for n in self.process_schema(schema['default'], parents=parents):
                          sections.append(n)
                  else:
                      section_key = '-'.join(parents) + '-' + key
                      section = build_section(key, section_key)
                      for n in self.process_schema(schema[key], parents=parents+[key]):
                          section += n
                      sections.append(section)
  
              # Sort all sections alphabetically by title. We may also have nodes
              # in this list that aren't sections if  `schema` has a 'default'
              # entry that's a leaf. In this case, we sort this as an empty string
              # in order to put this node at the beginning of the list.
              return sorted(sections, key=lambda s: s[0][0] if isinstance(s, nodes.section) else '')
  
      def parse_rst(self, content):
          rst = ViewList()
          # use fake filename 'inline' for error # reporting
          for i, line in enumerate(content.split('\n')):
              rst.append(line, 'inline', i)
          body = nodes.paragraph()
          nested_parse_with_titles(self.state, rst, body)
  
          return body
  
  
  def setup(app):
      app.add_directive('schemagen', SchemaGen)
  
      return {
          'version': '0.1',
          'parallel_read_safe': True,
          'parallel_write_safe': True,
      }
  
  
  [... the rest is omitted ...]
files: docs/_ext/schemagen.py
files: from docutils import nodes
  from sphinx.util.nodes import nested_parse_with_titles
  from docutils.statemachine import ViewList
  from sphinx.util.docutils import SphinxDirective
  from siliconcompiler.schema import schema_cfg
  
  from common import *
  
  # Main Sphinx plugin
  class SchemaGen(SphinxDirective):
  
      def run(self):
          self.env.note_dependency('../siliconcompiler/schema.py')
  
          schema = schema_cfg()
  
          # Split up schema into root and nested keys so that we can add a custom
          # header to the root entries.
          basic_schema = {}
          nested_schema = {}
          for key, val in schema.items():
              if is_leaf(val):
                  basic_schema[key] = val
              else:
                  nested_schema[key] = val
  
          basic_section = build_section('root', 'root')
          for n in self.process_schema(basic_schema):
              basic_section += n
  
          nested_sections = self.process_schema(nested_schema)
  
          return [basic_section] + nested_sections
  
      def process_schema(self, schema, parents=[]):
          if 'help' in schema:
              entries = [[strong('Description'),   para(schema['shorthelp'])],
                         [strong('Type'),          para(schema['type'])],
                         [strong('Default Value'), para(schema['defvalue'])],
                         [strong('CLI Switch'),    code(schema['switch'])]]
              for example in schema['example']:
                  name, ex = example.split(':', 1)
                  entries.append([strong(f'Example ({name.upper()})'), code(ex.strip())])
  
              table = build_table(entries)
              body = self.parse_rst(schema['help'])
  
              return [table, body]
          else:
              sections = []
              for key in schema.keys():
                  if key == 'default':
                      for n in self.process_schema(schema['default'], parents=parents):
                          sections.append(n)
                  else:
                      section_key = '-'.join(parents) + '-' + key
                      section = build_section(key, section_key)
                      for n in self.process_schema(schema[key], parents=parents+[key]):
                          section += n
                      sections.append(section)
  
              # Sort all sections alphabetically by title. We may also have nodes
              # in this list that aren't sections if  `schema` has a 'default'
              # entry that's a leaf. In this case, we sort this as an empty string
              # in order to put this node at the beginning of the list.
              return sorted(sections, key=lambda s: s[0][0] if isinstance(s, nodes.section) else '')
  
      def parse_rst(self, content):
          rst = ViewList()
          # use fake filename 'inline' and fake line number '1' for error reporting
          rst.append(content, 'inline', 1)
          body = nodes.paragraph()
          nested_parse_with_titles(self.state, rst, body)
  
          return body
  
  
  def setup(app):
      app.add_directive('schemagen', SchemaGen)
  
      return {
          'version': '0.1',
          'parallel_read_safe': True,
          'parallel_write_safe': True,
      }
  
  
  [... the rest is omitted ...]
files: docs/_ext/schemagen.py
files: # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
  
  from siliconcompiler import utils
  
  import re
  import os
  import sys
  import copy
  import json
  
  #############################################################################
  # PARAM DEFINITION
  #############################################################################
  
  def scparam(cfg,
              keypath,
              sctype=None,
              require=None,
              defvalue=None,
              scope='global',
              copy='false',
              lock='false',
              hashalgo='sha256',
              signature=None,
              shorthelp=None,
              switch=None,
              example=None,
              schelp=None):
  
      # 1. decend keypath until done
      # 2. create key if missing
      # 3. populate leaf cell when keypath empty
      if keypath:
          key = keypath[0]
          keypath.pop(0)
          if key not in cfg.keys():
              cfg[key] = {}
          scparam(cfg[key],
                  keypath,
                  sctype=sctype,
                  scope=scope,
                  require=require,
                  defvalue=defvalue,
                  copy=copy,
                  lock=lock,
                  signature=signature,
                  hashalgo=hashalgo,
                  shorthelp=shorthelp,
                  switch=switch,
                  example=example,
                  schelp=schelp)
      else:
  
          # removing leading spaces as if schelp were a docstring
          schelp = utils.trim(schelp)
  
          # setting valus based on types
          # note (bools are never lists)
          if re.match(r'bool',sctype):
              require = 'all'
              defvalue = 'false'
          if re.match(r'\[',sctype) and signature is None:
              signature = []
          if re.match(r'\[',sctype) and defvalue is None:
              defvalue = []
  
          # mandatory for all
          cfg['type'] = sctype
          cfg['scope'] = scope
          cfg['require'] = require
          cfg['lock'] = lock
          cfg['switch'] = switch
          cfg['shorthelp'] = shorthelp
          cfg['example'] = example
          cfg['help'] = schelp
          cfg['defvalue'] = defvalue
          cfg['value'] = defvalue
          cfg['signature'] = signature
  
          # file only values
          if re.search(r'file',sctype):
              cfg['hashalgo'] = hashalgo
              cfg['copy'] = copy
              cfg['filehash'] = []
              cfg['date'] = []
              cfg['author'] = []
  
  
  #############################################################################
  # CHIP CONFIGURATION
  #############################################################################
  
  def schema_cfg():
      '''Method for defining Chip configuration schema
      All the keys defined in this dictionary are reserved words.
      '''
  
      # SC version number (bump on every non trivial change)
      # Version number following semver standard.
      # Software version syncs with SC releases (from _metadata)
  
      SCHEMA_VERSION = '0.8.0'
  
      # Basic schema setup
      cfg = {}
  
      # Version handling
      cfg = schema_version(cfg, SCHEMA_VERSION)
  
      # Runtime options
      cfg = schema_options(cfg)
      cfg = schema_arg(cfg)
  
      # Primary sources
      cfg = schema_design(cfg)
  
      # Constraints
      #cfg = schema_constraints(cfg)
  
      # Project configuration
      cfg = schema_package(cfg, 'package')
      cfg = schema_checklist(cfg)
      cfg = schema_design(cfg)
      cfg = schema_read(cfg)
      cfg = schema_fpga(cfg)
      cfg = schema_asic(cfg)
      cfg = schema_mcmm(cfg)
  
      # Flow Information
      cfg = schema_flowgraph(cfg)
      cfg = schema_flowstatus(cfg)
      cfg = schema_eda(cfg)
  
      # PDK
      cfg = schema_pdk(cfg)
  
      # Package management
      cfg = schema_libs(cfg)
      cfg = schema_package(cfg, 'library')
      cfg = schema_checklist(cfg, 'library')
  
      # Compilation records
      cfg = schema_metric(cfg)
      cfg = schema_record(cfg)
  
      return cfg
  
  ###############################################################################
  # Minimal setup FPGA
  ###############################################################################
  
  def schema_version(cfg, version):
  
      scparam(cfg,['version', 'schema'],
              sctype='str',
              defvalue=version,
              shorthelp="Schema version number",
              switch="-version_schema <str>",
              example=["cli: -version_schema",
                       "api: chip.get('version', 'schema')"],
              schelp="""SiliconCompiler schema version number."""
  
  [... the rest is omitted ...]
files: siliconcompiler/schema.py
files: # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
  
  import re
  import os
  import sys
  import copy
  import json
  
  #############################################################################
  # PARAM DEFINITION
  #############################################################################
  
  def scparam(cfg,
              keypath,
              sctype=None,
              require=None,
              defvalue=None,
              scope='global',
              copy='false',
              lock='false',
              hashalgo='sha256',
              signature=None,
              shorthelp=None,
              switch=None,
              example=None,
              schelp=None):
  
      # 1. decend keypath until done
      # 2. create key if missing
      # 3. populate leaf cell when keypath empty
      if keypath:
          key = keypath[0]
          keypath.pop(0)
          if key not in cfg.keys():
              cfg[key] = {}
          scparam(cfg[key],
                  keypath,
                  sctype=sctype,
                  scope=scope,
                  require=require,
                  defvalue=defvalue,
                  copy=copy,
                  lock=lock,
                  signature=signature,
                  hashalgo=hashalgo,
                  shorthelp=shorthelp,
                  switch=switch,
                  example=example,
                  schelp=schelp)
      else:
  
          # removing leading newline and space
          schelp = re.sub(r'\n\s*', " ", schelp)
          schelp = schelp.strip()
  
          # setting valus based on types
          # note (bools are never lists)
          if re.match(r'bool',sctype):
              require = 'all'
              defvalue = 'false'
          if re.match(r'\[',sctype) and signature is None:
              signature = []
          if re.match(r'\[',sctype) and defvalue is None:
              defvalue = []
  
          # mandatory for all
          cfg['type'] = sctype
          cfg['scope'] = scope
          cfg['require'] = require
          cfg['lock'] = lock
          cfg['switch'] = switch
          cfg['shorthelp'] = shorthelp
          cfg['example'] = example
          cfg['help'] = schelp
          cfg['defvalue'] = defvalue
          cfg['value'] = defvalue
          cfg['signature'] = signature
  
          # file only values
          if re.search(r'file',sctype):
              cfg['hashalgo'] = hashalgo
              cfg['copy'] = copy
              cfg['filehash'] = []
              cfg['date'] = []
              cfg['author'] = []
  
  
  #############################################################################
  # CHIP CONFIGURATION
  #############################################################################
  
  def schema_cfg():
      '''Method for defining Chip configuration schema
      All the keys defined in this dictionary are reserved words.
      '''
  
      # SC version number (bump on every non trivial change)
      # Version number following semver standard.
      # Software version syncs with SC releases (from _metadata)
  
      SCHEMA_VERSION = '0.8.0'
  
      # Basic schema setup
      cfg = {}
  
      # Version handling
      cfg = schema_version(cfg, SCHEMA_VERSION)
  
      # Runtime options
      cfg = schema_options(cfg)
      cfg = schema_arg(cfg)
  
      # Primary sources
      cfg = schema_design(cfg)
  
      # Constraints
      #cfg = schema_constraints(cfg)
  
      # Project configuration
      cfg = schema_package(cfg, 'package')
      cfg = schema_checklist(cfg)
      cfg = schema_design(cfg)
      cfg = schema_read(cfg)
      cfg = schema_fpga(cfg)
      cfg = schema_asic(cfg)
      cfg = schema_mcmm(cfg)
  
      # Flow Information
      cfg = schema_flowgraph(cfg)
      cfg = schema_flowstatus(cfg)
      cfg = schema_eda(cfg)
  
      # PDK
      cfg = schema_pdk(cfg)
  
      # Package management
      cfg = schema_libs(cfg)
      cfg = schema_package(cfg, 'library')
      cfg = schema_checklist(cfg, 'library')
  
      # Compilation records
      cfg = schema_metric(cfg)
      cfg = schema_record(cfg)
  
      return cfg
  
  ###############################################################################
  # Minimal setup FPGA
  ###############################################################################
  
  def schema_version(cfg, version):
  
      scparam(cfg,['version', 'schema'],
              sctype='str',
              defvalue=version,
              shorthelp="Schema version number",
              switch="-version_schema <str>",
              example=["cli: -version_schema",
                       "api: chip.get('version', 'schema')"],
              schelp="""SiliconCompiler schema
  
  [... the rest is omitted ...]
files: siliconcompiler/schema.py
files: import os
  import shutil
  import sys
  
  def copytree(src, dst, ignore=[], dirs_exist_ok=False, link=False):
      '''Simple implementation of shutil.copytree to give us a dirs_exist_ok
      option in Python < 3.8.
  
      If link is True, create hard links in dst pointing to files in src
      instead of copying them.
      '''
      os.makedirs(dst, exist_ok=dirs_exist_ok)
  
      for name in os.listdir(src):
          if name in ignore:
              continue
  
          srcfile = os.path.join(src, name)
          dstfile = os.path.join(dst, name)
  
          if os.path.isdir(srcfile):
              copytree(srcfile, dstfile, ignore=ignore, dirs_exist_ok=dirs_exist_ok)
          elif link:
              os.link(srcfile, dstfile)
          else:
              shutil.copy2(srcfile, dstfile)
  
  def trim(docstring):
      '''Helper function for cleaning up indentation of docstring.
  
      This is important for properly parsing complex RST in our docs.
  
      Source:
      https://www.python.org/dev/peps/pep-0257/#handling-docstring-indentation'''
      if not docstring:
          return ''
      # Convert tabs to spaces (following the normal Python rules)
      # and split into a list of lines:
      lines = docstring.expandtabs().splitlines()
      # Determine minimum indentation (first line doesn't count):
      indent = sys.maxsize
      for line in lines[1:]:
          stripped = line.lstrip()
          if stripped:
              indent = min(indent, len(line) - len(stripped))
      # Remove indentation (first line is special):
      trimmed = [lines[0].strip()]
      if indent < sys.maxsize:
          for line in lines[1:]:
              trimmed.append(line[indent:].rstrip())
      # Strip off trailing and leading blank lines:
      while trimmed and not trimmed[-1]:
          trimmed.pop()
      while trimmed and not trimmed[0]:
          trimmed.pop(0)
      # Return a single string:
      return '\n'.join(trimmed)
  
  
  [... the rest is omitted ...]
files: siliconcompiler/utils.py
files: import os
  import shutil
  
  def copytree(src, dst, ignore=[], dirs_exist_ok=False, link=False):
      '''Simple implementation of shutil.copytree to give us a dirs_exist_ok
      option in Python < 3.8.
  
      If link is True, create hard links in dst pointing to files in src
      instead of copying them.
      '''
      os.makedirs(dst, exist_ok=dirs_exist_ok)
  
      for name in os.listdir(src):
          if name in ignore:
              continue
  
          srcfile = os.path.join(src, name)
          dstfile = os.path.join(dst, name)
  
          if os.path.isdir(srcfile):
              copytree(srcfile, dstfile, ignore=ignore, dirs_exist_ok=dirs_exist_ok)
          elif link:
              os.link(srcfile, dstfile)
          else:
              shutil.copy2(srcfile, dstfile)
  
  
  [... the rest is omitted ...]
files: siliconcompiler/utils.py