text: """Attempt to generate templates for module reference with Sphinx
  
  XXX - we exclude extension modules
  
  To include extension modules, first identify them as valid in the
  ``_uri2path`` method, then handle them in the ``_parse_module`` script.
  
  We get functions and classes by parsing the text of .py files.
  Alternatively we could import the modules for discovery, and we'd have
  to do that for extension modules.  This would involve changing the
  ``_parse_module`` method to work via import and introspection, and
  might involve changing ``discover_modules`` (which determines which
  files are modules, and therefore which module URIs will be passed to
  ``_parse_module``).
  
  NOTE: this is a modified version of a script originally shipped with the
  PyMVPA project, which we've adapted for NIPY use.  PyMVPA is an MIT-licensed
  project."""
  
  from __future__ import print_function
  
  # Stdlib imports
  import ast
  import inspect
  import os
  import re
  
  class Obj(object):
      '''Namespace to hold arbitrary information.'''
      def __init__(self, **kwargs):
          for k, v in kwargs.items():
              setattr(self, k, v)
                                          
  class FuncClsScanner(ast.NodeVisitor):
      """Scan a module for top-level functions and classes.
      
      Skips objects with an @undoc decorator, or a name starting with '_'.
      """
      def __init__(self):
          ast.NodeVisitor.__init__(self)
          self.classes = []
          self.classes_seen = set()
          self.functions = []
      
      @staticmethod
      def has_undoc_decorator(node):
          return any(isinstance(d, ast.Name) and d.id == 'undoc' \
                                  for d in node.decorator_list)
  
      def visit_If(self, node):
          if isinstance(node.test, ast.Compare) \
                  and isinstance(node.test.left, ast.Name) \
                  and node.test.left.id == '__name__':
              return   # Ignore classes defined in "if __name__ == '__main__':"
  
          self.generic_visit(node)
      
      def visit_FunctionDef(self, node):
          if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
                  and node.name not in self.functions:
              self.functions.append(node.name)
      
      def visit_ClassDef(self, node):
          if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
                  and node.name not in self.classes_seen:
              cls = Obj(name=node.name)
              cls.has_init = any(isinstance(n, ast.FunctionDef) and \
                                  n.name=='__init__' for n in node.body)
              self.classes.append(cls)
              self.classes_seen.add(node.name)
      
      def scan(self, mod):
          self.visit(mod)
          return self.functions, self.classes
  
  # Functions and classes
  class ApiDocWriter(object):
      ''' Class for automatic detection and parsing of API docs
      to Sphinx-parsable reST format'''
  
      # only separating first two levels
      rst_section_levels = ['*', '=', '-', '~', '^']
  
      def __init__(self,
                   package_name,
                   rst_extension='.rst',
                   package_skip_patterns=None,
                   module_skip_patterns=None,
                   names_from__all__=None,
                   ):
          ''' Initialize package for parsing
  
          Parameters
          ----------
          package_name : string
              Name of the top-level package.  *package_name* must be the
              name of an importable package
          rst_extension : string, optional
              Extension for reST files, default '.rst'
          package_skip_patterns : None or sequence of {strings, regexps}
              Sequence of strings giving URIs of packages to be excluded
              Operates on the package path, starting at (including) the
              first dot in the package path, after *package_name* - so,
              if *package_name* is ``sphinx``, then ``sphinx.util`` will
              result in ``.util`` being passed for earching by these
              regexps.  If is None, gives default. Default is:
              ['\.tests$']
          module_skip_patterns : None or sequence
              Sequence of strings giving URIs of modules to be excluded
              Operates on the module name including preceding URI path,
              back to the first dot after *package_name*.  For example
              ``sphinx.util.console`` results in the string to search of
              ``.util.console``
              If is None, gives default. Default is:
              ['\.setup$', '\._']
          names_from__all__ : set, optional
              Modules listed in here will be scanned by doing ``from mod import *``,
              rather than finding function and class definitions by scanning the
              AST. This is intended for API modules which expose things defined in
              other files. Modules listed here must define ``__all__`` to avoid
              exposing everything they import.
          '''
          if package_skip_patterns is None:
              package_skip_patterns = ['\\.tests$']
          if module_skip_patterns is None:
              module_skip_patterns = ['\\.setup$', '\\._']
          self.package_name = package_name
          self.rst_extension = rst_extension
          self.package_skip_patterns = package_skip_patterns
          self.module_skip_patterns = module_skip_patterns
          self.names_from__all__ = names_from__all__ or set()
  
      def get_package_name(self):
          return self._package_name
  
      def set_package_name(self, package_name):
          ''' Set package_name
  
          >>> docwriter = ApiDocWriter('sphinx')
          >>> import sphinx
          >>> docwriter.root_path == sphinx.__path__[0]
          True
          >>> docwriter.package_name = 'docutils'
          >>> import docutils
          >>> docwriter.root_path == docutils.__path__[0]
          True
          '''
          # It's also possible to imagine caching the module parsing here
          self._package_name = package_name
          self.root_module = __import__(package_name)
          self.root_path = self.root_module.__path__[0]
          self.written_modules = None
  
      package_name = property(get_package_name, set_package_name, None,
                              'get/set package_name')
  
      def _uri2path(self, uri):
          ''' Convert uri to absolute filepath
  
          Parameters
          ----------
          uri : string
              URI of python module to return path for
  
          Returns
          -------
          path : None or string
              Returns None if there is no valid path for this URI
              Otherwise returns absolute file system path for URI
  
          Examples
          --------
          >>> docwriter = ApiDocWriter('sphinx')
          >>> import sphinx
          >>> modpath = sphinx.__path__[0]
          >>> res = docwriter._uri2path('sphinx.builder')
          >>> res == os.path.join(modpath, 'builder.py')
          True
          >>> res = docwriter._uri2path('sphinx')
          >>> res == os.path.join(modpath, '__init__.py')
          True
          >>> docwriter._uri2path('sphinx.does_not_exist')
  
          '''
          if uri == self.package_name:
              return os.path.join(self.root_path, '__init__.py')
          path = uri.replace('.', os.path.sep)
          path = path.replace(self.package_name + os.path.sep, '')
          path = os.path.join(self.root_path, path)
          # XXX maybe check for extensions as well?
          if os.path.exists(path + '.py'): # file
              path += '.py'
          elif os.path.exists(os.path.join(path, '__init__.py')):
              path = os.path.join(path, '__init__.py')
          else:
              return None
          return path
  
      def _path2uri(self, dirpath):
          ''' Convert directory path to uri '''
          relpath = dirpath.replace(self.root_path, self.package_name)
          if relpath.startswith(os.path.sep):
              relpath = relpath[1:]
          return relpath.replace(os.path.sep, '.')
  
      def _parse_module(self, uri):
          ''' Parse module defined in *uri* '''
          filename = self._uri2path(uri)
          if filename is None:
              # nothing that we could handle here.
              return ([],[])
          with open(filename, 'rb') as f:
              mod = ast.parse(f.read())
          return FuncClsScanner().scan(mod)
  
      def _import_funcs_classes(self, uri):
          """Import * from uri, and separate out functions and classes."""
          ns = {}
          exec('from %s import *' % uri, ns)
          funcs, classes = [], []
          for name, obj in ns.items():
              if inspect.isclass(obj):
                  cls = Obj(name=name, has_init='__init__' in obj.__dict__)
                  classes.append(cls)
              elif inspect.isfunction(obj):
                  funcs.append(name)
  
          return sorted(funcs), sorted(classes, key=lambda x: x.name)
  
      def find_funcs_classes(self, uri):
          """Find the functions and classes defined in the module ``uri``"""
          if uri in self.names_from__all__:
              # For API modules which expose things defined elsewhere, import them
              return self._import_funcs_classes(uri)
          else:
              # For other modules, scan their AST to see what they define
              return self._parse_module(uri)
  
      def generate_api_doc(self, uri):
          '''Make autodoc documentation template string for a module
  
          Parameters
          ----------
          uri : string
              python location of module - e.g 'sphinx.builder'
  
          Returns
          -------
          S : string
              Contents of API doc
          '''
          # get the names of all classes and functions
          functions, classes = self.find_funcs_classes(uri)
          if not len(functions) and not len(classes):
              #print ('WARNING: Empty -', uri)  # dbg
              return ''
  
          # Make a shorter version of the uri that omits the package name for
          # titles
          uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)
  
          ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'
  
          # Set the chapter title to read 'Module:' for all modules except for the
          # main packages
          if '.' in uri:
              chap_title = 'Module: :mod:`' + uri_short + '`'
          else:
              chap_title = ':mod:`' + uri_short + '`'
          ad += chap_title + '\n' + self.rst_section_levels[1] * len(chap_title)
  
          ad += '\n.. automodule:: ' + uri + '\n'
          ad += '\n.. currentmodule:: ' + uri + '\n'
          
          if classes:
              subhead = str(len(classes)) + (' Classes' if len(classes) > 1 else ' Class')
              ad += '\n'+ subhead + '\n' + \
                    self.rst_section_levels[2] * len(subhead) + '\n'
  
          for c in classes:
              ad += '\n.. autoclass:: ' + c.name + '\n'
              # must NOT exclude from index to keep cross-refs working
              ad += '  :members:\n' \
                    '  :show-inheritance:\n'
              if c.has_init:
                    ad += '\n  .. automethod:: __init__\n'
          
          if functions:
              subhead = str(len(functions)) + (' Functions' if len(functions) > 1 else ' Function')
              ad += '\n'+ subhead + '\n' + \
                    self.rst_section_levels[2] * len(subhead) + '\n'
          for f in functions:
              # must NOT exclude from index to keep cross-refs working
              ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
          return ad
  
      def _survives_exclude(self, matchstr, match_type):
          ''' Returns True if *matchstr* does not match patterns
  
          ``self.package_name`` removed from front of string if present
  
          Examples
          --------
          >>> dw = ApiDocWriter('sphinx')
          >>> dw._survives_exclude('sphinx.okpkg', 'package')
          True
          >>> dw.package_skip_patterns.append('^\\.badpkg$')
          >>> dw._survives_exclude('sphinx.badpkg', 'package')
          False
          >>> dw._survives_exclude('sphinx.badpkg', 'module')
          True
          >>> dw._survives_exclude('sphinx.badmod', 'module')
          True
          >>> dw.module_skip_patterns.append('^\\.badmod$')
          >>> dw._survives_exclude('sphinx.badmod', 'module')
          False
          '''
          if match_type == 'module':
              patterns = self.module_skip_patterns
          elif match_type == 'package':
              patterns = self.package_skip_patterns
          else:
              raise ValueError('Cannot interpret match type "%s"'
                               % match_type)
          # Match to URI without package name
          L = len(self.package_name)
          if matchstr[:L] == self.package_name:
              matchstr = matchstr[L:]
          for pat in patterns:
              try:
                  pat.search
              except AttributeError:
                  pat = re.compile(pat)
              if pat.search(matchstr):
                  return False
          return True
  
      def discover_modules(self):
          ''' Return module sequence discovered from ``self.package_name``
  
  
          Parameters
          ----------
          None
  
          Returns
          -------
          mods : sequence
              Sequence of module names within ``self.package_name``
  
          Examples
          --------
          >>> dw = ApiDocWriter('sphinx')
          >>> mods = dw.discover_modules()
          >>> 'sphinx.util' in mods
          True
          >>> dw.package_skip_patterns.append('\.util$')
          >>> 'sphinx.util' in dw.discover_modules()
          False
          >>>
          '''
          modules = [self.package_name]
          # raw directory parsing
          for dirpath, dirnames, filenames in os.walk(self.root_path):
              # Check directory names for packages
              root_uri = self._path2uri(os.path.join(self.root_path,
                                                     dirpath))
              for dirname in dirnames[:]: # copy list - we modify inplace
                  package_uri = '.'.join((root_uri, dirname))
                  if (self._uri2path(package_uri) and
                      self._survives_exclude(package_uri, 'package')):
                      modules.append(package_uri)
                  else:
                      dirnames.remove(dirname)
              # Check filenames for modules
              for filename in filenames:
                  module_name = filename[:-3]
                  module_uri = '.'.join((root_uri, module_name))
                  if (self._uri2path(module_uri) and
                      self._survives_exclude(module_uri, 'module')):
                      modules.append(module_uri)
          return sorted(modules)
  
      def write_modules_api(self, modules,outdir):
          # write the list
          written_modules = []
          for m in modules:
              api_str = self.generate_api_doc(m)
              if not api_str:
                  continue
              # write out to file
              outfile = os.path.join(outdir,
                                     m + self.rst_extension)
              fileobj = open(outfile, 'wt')
              fileobj.write(api_str)
              fileobj.close()
              written_modules.append(m)
          self.written_modules = written_modules
  
      def write_api_docs(self, outdir):
          """Generate API reST files.
  
          Parameters
          ----------
          outdir : string
              Directory name in which to store files
              We create automatic filenames for each module
  
          Returns
          -------
          None
  
          Notes
          -----
          Sets self.written_modules to list of written modules
          """
          if not os.path.exists(outdir):
              os.mkdir(outdir)
          # compose list of modules
          modules = self.discover_modules()
          self.write_modules_api(modules,outdir)
  
      def write_index(self, outdir, path='gen.rst', relative_to=None):
          """Make a reST API index file from written files
  
          Parameters
          ----------
          outdir : string
              Directory to which to write generated index file
          path : string
              Filename to write index to
          relative_to : string
              path to which written filenames are relative.  This
              component of the written file path will be removed from
              outdir, in the generated index.  Default is None, meaning,
              leave path as it is.
          """
          if self.written_modules is None:
              raise ValueError('No modules written')
          # Get full filename path
          path = os.path.join(outdir, path)
          # Path written into index is relative to rootpath
          if relative_to is not None:
              relpath = outdir.replace(relative_to + os.path.sep, '')
          else:
              relpath = outdir
          idx = open(path,'wt')
          w = idx.write
          w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
          w('.. autosummary::\n'
            '   :toctree: %s\n\n' % relpath)
          for mod in self.written_modules:
              w('   %s\n' % mod)
          idx.close()
  