text: # -*- test-case-name: twisted.test.test_modules -*-
  # Copyright (c) Twisted Matrix Laboratories.
  # See LICENSE for details.
  
  """
  This module aims to provide a unified, object-oriented view of Python's
  runtime hierarchy.
  
  Python is a very dynamic language with wide variety of introspection utilities.
  However, these utilities can be hard to use, because there is no consistent
  API.  The introspection API in python is made up of attributes (__name__,
  __module__, func_name, etc) on instances, modules, classes and functions which
  vary between those four types, utility modules such as 'inspect' which provide
  some functionality, the 'imp' module, the "compiler" module, the semantics of
  PEP 302 support, and setuptools, among other things.
  
  At the top, you have "PythonPath", an abstract representation of sys.path which
  includes methods to locate top-level modules, with or without loading them.
  The top-level exposed functions in this module for accessing the system path
  are "walkModules", "iterModules", and "getModule".
  
  From most to least specific, here are the objects provided::
  
                    PythonPath  # sys.path
                        |
                        v
                    PathEntry   # one entry on sys.path: an importer
                        |
                        v
                   PythonModule # a module or package that can be loaded
                        |
                        v
                   PythonAttribute # an attribute of a module (function or class)
                        |
                        v
                   PythonAttribute # an attribute of a function or class
                        |
                        v
                       ...
  
  Here's an example of idiomatic usage: this is what you would do to list all of
  the modules outside the standard library's python-files directory::
  
      import os
      stdlibdir = os.path.dirname(os.__file__)
  
      from twisted.python.modules import iterModules
  
      for modinfo in iterModules():
          if (modinfo.pathEntry.filePath.path != stdlibdir
              and not modinfo.isPackage()):
              print('unpackaged: %s: %s' % (
                  modinfo.name, modinfo.filePath.path))
  
  @var theSystemPath: The very top of the Python object space.
  @type theSystemPath: L{PythonPath}
  """
  
  
  import inspect
  import sys
  import warnings
  import zipimport
  
  # let's try to keep path imports to a minimum...
  from os.path import dirname, split as splitpath
  from typing import cast
  
  from zope.interface import Interface, implementer
  
  from twisted.python.compat import nativeString
  from twisted.python.components import registerAdapter
  from twisted.python.filepath import FilePath, UnlistableError
  from twisted.python.reflect import namedAny
  from twisted.python.zippath import ZipArchive
  
  _nothing = object()
  
  PYTHON_EXTENSIONS = [".py"]
  OPTIMIZED_MODE = __doc__ is None
  if OPTIMIZED_MODE:
      PYTHON_EXTENSIONS.append(".pyo")
  else:
      PYTHON_EXTENSIONS.append(".pyc")
  
  
  def _isPythonIdentifier(string):
      """
      cheezy fake test for proper identifier-ness.
  
      @param string: a L{str} which might or might not be a valid python
          identifier.
      @return: True or False
      """
      textString = nativeString(string)
      return " " not in textString and "." not in textString and "-" not in textString
  
  
  def _isPackagePath(fpath):
      # Determine if a FilePath-like object is a Python package.  TODO: deal with
      # __init__module.(so|dll|pyd)?
      extless = fpath.splitext()[0]
      basend = splitpath(extless)[1]
      return basend == "__init__"
  
  
  class _ModuleIteratorHelper:
      """
      This mixin provides common behavior between python module and path entries,
      since the mechanism for searching sys.path and __path__ attributes is
      remarkably similar.
      """
  
      def iterModules(self):
          """
          Loop over the modules present below this entry or package on PYTHONPATH.
  
          For modules which are not packages, this will yield nothing.
  
          For packages and path entries, this will only yield modules one level
          down; i.e. if there is a package a.b.c, iterModules on a will only
          return a.b.  If you want to descend deeply, use walkModules.
  
          @return: a generator which yields PythonModule instances that describe
          modules which can be, or have been, imported.
          """
          yielded = {}
          if not self.filePath.exists():
              return
  
          for placeToLook in self._packagePaths():
              try:
                  children = sorted(placeToLook.children())
              except UnlistableError:
                  continue
  
              for potentialTopLevel in children:
                  ext = potentialTopLevel.splitext()[1]
                  potentialBasename = potentialTopLevel.basename()[: -len(ext)]
                  if ext in PYTHON_EXTENSIONS:
                      # TODO: this should be a little choosier about which path entry
                      # it selects first, and it should do all the .so checking and
                      # crud
                      if not _isPythonIdentifier(potentialBasename):
                          continue
                      modname = self._subModuleName(potentialBasename)
                      if modname.split(".")[-1] == "__init__":
                          # This marks the directory as a package so it can't be
                          # a module.
                          continue
                      if modname not in yielded:
                          yielded[modname] = True
                          pm = PythonModule(modname, potentialTopLevel, self._getEntry())
                          assert pm != self
                          yield pm
                  else:
                      if (
                          ext
                          or not _isPythonIdentifier(potentialBasename)
                          or not potentialTopLevel.isdir()
                      ):
                          continue
                      modname = self._subModuleName(potentialTopLevel.basename())
                      for ext in PYTHON_EXTENSIONS:
                          initpy = potentialTopLevel.child("__init__" + ext)
                          if initpy.exists() and modname not in yielded:
                              yielded[modname] = True
                              pm = PythonModule(modname, initpy, self._getEntry())
                              assert pm != self
                              yield pm
                              break
  
      def walkModules(self, importPackages=False):
          """
          Similar to L{iterModules}, this yields self, and then every module in my
          package or entry, and every submodule in each package or entry.
  
          In other words, this is deep, and L{iterModules} is shallow.
          """
          yield self
          for package in self.iterModules():
              yield from package.walkModules(importPackages=importPackages)
  
      def _subModuleName(self, mn):
          """
          This is a hook to provide packages with the ability to specify their names
          as a prefix to submodules here.
          """
          return mn
  
      def _packagePaths(self):
          """
          Implement in subclasses to specify where to look for modules.
  
          @return: iterable of FilePath-like objects.
          """
          raise NotImplementedError()
  
      def _getEntry(self):
          """
          Implement in subclasses to specify what path entry submodules will come
          from.
  
          @return: a PathEntry instance.
          """
          raise NotImplementedError()
  
      def __getitem__(self, modname):
          """
          Retrieve a module from below this path or package.
  
          @param modname: a str naming a module to be loaded.  For entries, this
          is a top-level, undotted package name, and for packages it is the name
          of the module without the package prefix.  For example, if you have a
          PythonModule representing the 'twisted' package, you could use::
  
              twistedPackageObj['python']['modules']
  
          to retrieve this module.
  
          @raise KeyError: if the module is not found.
  
          @return: a PythonModule.
          """
          for module in self.iterModules():
              if module.name == self._subModuleName(modname):
                  return module
          raise KeyError(modname)
  
      def __iter__(self):
          """
          Implemented to raise NotImplementedError for clarity, so that attempting to
          loop over this object won't call __getitem__.
  
          Note: in the future there might be some sensible default for iteration,
          like 'walkEverything', so this is deliberately untested and undefined
          behavior.
          """
          raise NotImplementedError()
  
  
  class PythonAttribute:
      """
      I represent a function, class, or other object that is present.
  
      @ivar name: the fully-qualified python name of this attribute.
  
      @ivar onObject: a reference to a PythonModule or other PythonAttribute that
      is this attribute's logical parent.
  
      @ivar name: the fully qualified python name of the attribute represented by
      this class.
      """
  
      def __init__(self, name, onObject, loaded, pythonValue):
          """
          Create a PythonAttribute.  This is a private constructor.  Do not construct
          me directly, use PythonModule.iterAttributes.
  
          @param name: the FQPN
          @param onObject: see ivar
          @param loaded: always True, for now
          @param pythonValue: the value of the attribute we're pointing to.
          """
          self.name = name
          self.onObject = onObject
          self._loaded = loaded
          self.pythonValue = pythonValue
  
      def __repr__(self) -> str:
          return f"PythonAttribute<{self.name!r}>"
  
      def isLoaded(self):
          """
          Return a boolean describing whether the attribute this describes has
          actually been loaded into memory by importing its module.
  
          Note: this currently always returns true; there is no Python parser
          support in this module yet.
          """
          return self._loaded
  
      def load(self, default=_nothing):
          """
          Load the value associated with this attribute.
  
          @return: an arbitrary Python object, or 'default' if there is an error
          loading it.
          """
          return self.pythonValue
  
      def iterAttributes(self):
          for name, val in inspect.getmembers(self.load()):
              yield PythonAttribute(self.name + "." + name, self, True, val)
  
  
  class PythonModule(_ModuleIteratorHelper):
      """
      Representation of a module which could be imported from sys.path.
  
      @ivar name: the fully qualified python name of this module.
  
      @ivar filePath: a FilePath-like object which points to the location of this
      module.
  
      @ivar pathEntry: a L{PathEntry} instance which this module was located
      from.
      """
  
      def __init__(self, name, filePath, pathEntry):
          """
          Create a PythonModule.  Do not construct this directly, instead inspect a
          PythonPath or other PythonModule instances.
  
          @param name: see ivar
          @param filePath: see ivar
          @param pathEntry: see ivar
          """
          _name = nativeString(name)
          assert not _name.endswith(".__init__")
          self.name = _name
          self.filePath = filePath
          self.parentPath = filePath.parent()
          self.pathEntry = pathEntry
  
      def _getEntry(self):
          return self.pathEntry
  
      def __repr__(self) -> str:
          """
          Return a string representation including the module name.
          """
          return f"PythonModule<{self.name!r}>"
  
      def isLoaded(self):
          """
          Determine if the module is loaded into sys.modules.
  
          @return: a boolean: true if loaded, false if not.
          """
          return self.pathEntry.pythonPath.moduleDict.get(self.name) is not None
  
      def iterAttributes(self):
          """
          List all the attributes defined in this module.
  
          Note: Future work is planned here to make it possible to list python
          attributes on a module without loading the module by inspecting ASTs or
          bytecode, but currently any iteration of PythonModule objects insists
          they must be loaded, and will use inspect.getmodule.
  
          @raise NotImplementedError: if this module is not loaded.
  
          @return: a generator yielding PythonAttribute instances describing the
          attributes of this module.
          """
          if not self.isLoaded():
              raise NotImplementedError(
                  "You can't load attributes from non-loaded modules yet."
              )
          for name, val in inspect.getmembers(self.load()):
              yield PythonAttribute(self.name + "." + name, self, True, val)
  
      def isPackage(self):
          """
          Returns true if this module is also a package, and might yield something
          from iterModules.
          """
          return _isPackagePath(self.filePath)
  
      def load(self, default=_nothing):
          """
          Load this module.
  
          @param default: if specified, the value to return in case of an error.
  
          @return: a genuine python module.
  
          @raise Exception: Importing modules is a risky business;
          the erorrs of any code run at module scope may be raised from here, as
          well as ImportError if something bizarre happened to the system path
          between the discovery of this PythonModule object and the attempt to
          import it.  If you specify a default, the error will be swallowed
          entirely, and not logged.
  
          @rtype: types.ModuleType.
          """
          try:
              return self.pathEntry.pythonPath.moduleLoader(self.name)
          except BaseException:  # this needs more thought...
              if default is not _nothing:
                  return default
              raise
  
      def __eq__(self, other: object) -> bool:
          """
          PythonModules with the same name are equal.
          """
          if isinstance(other, PythonModule):
              return cast(bool, other.name == self.name)
          return NotImplemented
  
      def walkModules(self, importPackages=False):
          if importPackages and self.isPackage():
              self.load()
          return super().walkModules(importPackages=importPackages)
  
      def _subModuleName(self, mn):
          """
          submodules of this module are prefixed with our name.
          """
          return self.name + "." + mn
  
      def _packagePaths(self):
          """
          Yield a sequence of FilePath-like objects which represent path segments.
          """
          if not self.isPackage():
              return
          if self.isLoaded():
              load = self.load()
              if hasattr(load, "__path__"):
                  for fn in load.__path__:
                      if fn == self.parentPath.path:
                          # this should _really_ exist.
                          assert self.parentPath.exists()
                          yield self.parentPath
                      else:
                          smp = self.pathEntry.pythonPath._smartPath(fn)
                          if smp.exists():
                              yield smp
          else:
              yield self.parentPath
  
  
  class PathEntry(_ModuleIteratorHelper):
      """
      I am a proxy for a single entry on sys.path.
  
      @ivar filePath: a FilePath-like object pointing at the filesystem location
      or archive file where this path entry is stored.
  
      @ivar pythonPath: a PythonPath instance.
      """
  
      def __init__(self, filePath, pythonPath):
          """
          Create a PathEntry.  This is a private constructor.
          """
          self.filePath = filePath
          self.pythonPath = pythonPath
  
      def _getEntry(self):
          return self
  
      def __repr__(self) -> str:
          return f"PathEntry<{self.filePath!r}>"
  
      def _packagePaths(self):
          yield self.filePath
  
  
  class IPathImportMapper(Interface):
      """
      This is an internal interface, used to map importers to factories for
      FilePath-like objects.
      """
  
      def mapPath(pathLikeString):
          """
          Return a FilePath-like object.
  
          @param pathLikeString: a path-like string, like one that might be
          passed to an import hook.
  
          @return: a L{FilePath}, or something like it (currently only a
          L{ZipPath}, but more might be added later).
          """
  
  
  @implementer(IPathImportMapper)
  class _DefaultMapImpl:
      """Wrapper for the default importer, i.e. None."""
  
      def mapPath(self, fsPathString):
          return FilePath(fsPathString)
  
  
  _theDefaultMapper = _DefaultMapImpl()
  
  
  @implementer(IPathImportMapper)
  class _ZipMapImpl:
      """IPathImportMapper implementation for zipimport.ZipImporter."""
  
      def __init__(self, importer):
          self.importer = importer
  
      def mapPath(self, fsPathString):
          """
          Map the given FS path to a ZipPath, by looking at the ZipImporter's
          "archive" attribute and using it as our ZipArchive root, then walking
          down into the archive from there.
  
          @return: a L{zippath.ZipPath} or L{zippath.ZipArchive} instance.
          """
          za = ZipArchive(self.importer.archive)
          myPath = FilePath(self.importer.archive)
          itsPath = FilePath(fsPathString)
          if myPath == itsPath:
              return za
          # This is NOT a general-purpose rule for sys.path or __file__:
          # zipimport specifically uses regular OS path syntax in its
          # pathnames, even though zip files specify that slashes are always
          # the separator, regardless of platform.
          segs = itsPath.segmentsFrom(myPath)
          zp = za
          for seg in segs:
              zp = zp.child(seg)
          return zp
  
  
  registerAdapter(_ZipMapImpl, zipimport.zipimporter, IPathImportMapper)
  
  
  def _defaultSysPathFactory():
      """
      Provide the default behavior of PythonPath's sys.path factory, which is to
      return the current value of sys.path.
  
      @return: L{sys.path}
      """
      return sys.path
  
  
  class PythonPath:
      """
      I represent the very top of the Python object-space, the module list in
      C{sys.path} and the modules list in C{sys.modules}.
  
      @ivar _sysPath: A sequence of strings like C{sys.path}.  This attribute is
      read-only.
  
      @ivar sysPath: The current value of the module search path list.
      @type sysPath: C{list}
  
      @ivar moduleDict: A dictionary mapping string module names to module
      objects, like C{sys.modules}.
  
      @ivar sysPathHooks: A list of PEP-302 path hooks, like C{sys.path_hooks}.
  
      @ivar moduleLoader: A function that takes a fully-qualified python name and
      returns a module, like L{twisted.python.reflect.namedAny}.
      """
  
      def __init__(
          self,
          sysPath=None,
          moduleDict=sys.modules,
          sysPathHooks=sys.path_hooks,
          importerCache=sys.path_importer_cache,
          moduleLoader=namedAny,
          sysPathFactory=None,
      ):
          """
          Create a PythonPath.  You almost certainly want to use
          modules.theSystemPath, or its aliased methods, rather than creating a
          new instance yourself, though.
  
          All parameters are optional, and if unspecified, will use 'system'
          equivalents that makes this PythonPath like the global L{theSystemPath}
          instance.
  
          @param sysPath: a sys.path-like list to use for this PythonPath, to
          specify where to load modules from.
  
          @param moduleDict: a sys.modules-like dictionary to use for keeping
          track of what modules this PythonPath has loaded.
  
          @param sysPathHooks: sys.path_hooks-like list of PEP-302 path hooks to
          be used for this PythonPath, to determie which importers should be
          used.
  
          @param importerCache: a sys.path_importer_cache-like list of PEP-302
          importers.  This will be used in conjunction with the given
          sysPathHooks.
  
          @param moduleLoader: a module loader function which takes a string and
          returns a module.  That is to say, it is like L{namedAny} - *not* like
          L{__import__}.
  
          @param sysPathFactory: a 0-argument callable which returns the current
          value of a sys.path-like list of strings.  Specify either this, or
          sysPath, not both.  This alternative interface is provided because the
          way the Python import mechanism works, you can re-bind the 'sys.path'
          name and that is what is used for current imports, so it must be a
          factory rather than a value to deal with modification by rebinding
          rather than modification by mutation.  Note: it is not recommended to
          rebind sys.path.  Although this mechanism can deal with that, it is a
          subtle point which some tools that it is easy for tools which interact
          with sys.path to miss.
          """
          if sysPath is not None:
              sysPathFactory = lambda: sysPath
          elif sysPathFactory is None:
              sysPathFactory = _defaultSysPathFactory
          self._sysPathFactory = sysPathFactory
          self._sysPath = sysPath
          self.moduleDict = moduleDict
          self.sysPathHooks = sysPathHooks
          self.importerCache = importerCache
          self.moduleLoader = moduleLoader
  
      @property
      def sysPath(self):
          """
          Retrieve the current value of the module search path list.
          """
          return self._sysPathFactory()
  
      def _findEntryPathString(self, modobj):
          """
          Determine where a given Python module object came from by looking at path
          entries.
          """
          topPackageObj = modobj
          while "." in topPackageObj.__name__:
              topPackageObj = self.moduleDict[
                  ".".join(topPackageObj.__name__.split(".")[:-1])
              ]
          if _isPackagePath(FilePath(topPackageObj.__file__)):
              # if package 'foo' is on sys.path at /a/b/foo, package 'foo's
              # __file__ will be /a/b/foo/__init__.py, and we are looking for
              # /a/b here, the path-entry; so go up two steps.
              rval = dirname(dirname(topPackageObj.__file__))
          else:
              # the module is completely top-level, not within any packages.  The
              # path entry it's on is just its dirname.
              rval = dirname(topPackageObj.__file__)
  
          # There are probably some awful tricks that an importer could pull
          # which would break this, so let's just make sure... it's a loaded
          # module after all, which means that its path MUST be in
          # path_importer_cache according to PEP 302 -glyph
          if rval not in self.importerCache:
              warnings.warn(
                  "%s (for module %s) not in path importer cache "
                  "(PEP 302 violation - check your local configuration)."
                  % (rval, modobj.__name__),
                  stacklevel=3,
              )
  
          return rval
  
      def _smartPath(self, pathName):
          """
          Given a path entry from sys.path which may refer to an importer,
          return the appropriate FilePath-like instance.
  
          @param pathName: a str describing the path.
  
          @return: a FilePath-like object.
          """
          importr = self.importerCache.get(pathName, _nothing)
          if importr is _nothing:
              for hook in self.sysPathHooks:
                  try:
                      importr = hook(pathName)
                  except ImportError:
                      pass
              if importr is _nothing:  # still
                  importr = None
          return IPathImportMapper(importr, _theDefaultMapper).mapPath(pathName)
  
      def iterEntries(self):
          """
          Iterate the entries on my sysPath.
  
          @return: a generator yielding PathEntry objects
          """
          for pathName in self.sysPath:
              fp = self._smartPath(pathName)
              yield PathEntry(fp, self)
  
      def __getitem__(self, modname):
          """
          Get a python module by its given fully-qualified name.
  
          @param modname: The fully-qualified Python module name to load.
  
          @type modname: C{str}
  
          @return: an object representing the module identified by C{modname}
  
          @rtype: L{PythonModule}
  
          @raise KeyError: if the module name is not a valid module name, or no
              such module can be identified as loadable.
          """
          # See if the module is already somewhere in Python-land.
          moduleObject = self.moduleDict.get(modname)
          if moduleObject is not None:
              # we need 2 paths; one of the path entry and one for the module.
              pe = PathEntry(
                  self._smartPath(self._findEntryPathString(moduleObject)), self
              )
              mp = self._smartPath(moduleObject.__file__)
              return PythonModule(modname, mp, pe)
  
          # Recurse if we're trying to get a submodule.
          if "." in modname:
              pkg = self
              for name in modname.split("."):
                  pkg = pkg[name]
              return pkg
  
          # Finally do the slowest possible thing and iterate
          for module in self.iterModules():
              if module.name == modname:
                  return module
          raise KeyError(modname)
  
      def __contains__(self, module):
          """
          Check to see whether or not a module exists on my import path.
  
          @param module: The name of the module to look for on my import path.
          @type module: C{str}
          """
          try:
              self.__getitem__(module)
              return True
          except KeyError:
              return False
  
      def __repr__(self) -> str:
          """
          Display my sysPath and moduleDict in a string representation.
          """
          return f"PythonPath({self.sysPath!r},{self.moduleDict!r})"
  
      def iterModules(self):
          """
          Yield all top-level modules on my sysPath.
          """
          for entry in self.iterEntries():
              yield from entry.iterModules()
  
      def walkModules(self, importPackages=False):
          """
          Similar to L{iterModules}, this yields every module on the path, then every
          submodule in each package or entry.
          """
          for package in self.iterModules():
              yield from package.walkModules(importPackages=False)
  
  
  theSystemPath = PythonPath()
  
  
  def walkModules(importPackages=False):
      """
      Deeply iterate all modules on the global python path.
  
      @param importPackages: Import packages as they are seen.
      """
      return theSystemPath.walkModules(importPackages=importPackages)
  
  
  def iterModules():
      """
      Iterate all modules and top-level packages on the global Python path, but
      do not descend into packages.
      """
      return theSystemPath.iterModules()
  
  
  def getModule(moduleName):
      """
      Retrieve a module from the system path.
      """
      return theSystemPath[moduleName]
  