<NME> workspace.py
<BEF> from .config import GlobalConfig
from .fs import FileSystem, LocalFileSystem, FileException
from .imports import get_imports
from .fetch import fetch_dependency
from .requirements_parser import parse_requirements, get_version_specifier_for_pkg
from typing import Dict, Set, List

import logging
import sys
import os
import os.path
import shutil
import threading
import opentracing
import jedi._compatibility


log = logging.getLogger(__name__)


class DummyFile:
    def __init__(self, contents):
        self.contents = contents

    def read(self):
        return self.contents

    def close(self):
        pass


class Module:
    def __init__(self,
                 name: str,
                 qualified_name: str,
                 path: str,
                 is_package: bool=False,
                 is_external: bool=False,
                 is_stdlib: bool=False,
                 is_native: bool=False,
                 is_namespace_package: bool=False):
        self.name = name
        self.qualified_name = qualified_name
        self.path = path
        self.is_package = is_package
        self.is_external = is_external
        self.is_stdlib = is_stdlib
        self.is_native = is_native
        self.is_namespace_package = is_namespace_package

    def __repr__(self):
        return "PythonModule({}, {})".format(self.name, self.path)


class Workspace:

    def __init__(self, fs: FileSystem, project_root: str,
                 original_root_path: str= "", pip_args: List[str]=[]):

        self.pip_args = pip_args
        self.project_packages = set()
        self.PROJECT_ROOT = project_root
        self.repo = None
        self.hash = None

        if original_root_path.startswith(
                "git://") and "?" in original_root_path:
            repo_and_hash = original_root_path.split("?")
            self.repo = repo_and_hash[0]
            original_root_path = self.repo
            self.hash = repo_and_hash[1]

        self.is_stdlib = (self.repo == GlobalConfig.STDLIB_REPO_URL)

        if original_root_path.startswith("git://"):
            original_root_path = original_root_path[6:]

        # turn the original root path into something that can be used as a
        # file/path name or cache key
        self.key = original_root_path.replace("/", ".").replace("\\", ".")
        if self.hash:
            self.key = ".".join((self.key, self.hash))

        # TODO: allow different Python versions per project/workspace
        self.PYTHON_PATH = GlobalConfig.PYTHON_PATH
        self.PACKAGES_PATH = os.path.join(
            GlobalConfig.PACKAGES_PARENT, self.key)
        log.debug("Setting Python path to %s", self.PYTHON_PATH)
        log.debug("Setting package path to %s", self.PACKAGES_PATH)

        self.fs = fs
        self.local_fs = LocalFileSystem()
        self.source_paths = {path for path in self.fs.walk(
            self.PROJECT_ROOT) if path.endswith(".py")}
        self.project = {}
        self.stdlib = {}
        self.dependencies = {}
        self.module_paths = {}
        # keep track of which package folders have been indexed, since we fetch
        # and index new folders on-demand
                continue

    def index_project(self):
        all_paths = list(self.fs.walk(self.PROJECT_ROOT))

        package_paths = {}
        for path in all_paths:
            folder, filename = os.path.split(path)
            if filename == "__init__.py":
                package_paths[folder] = True

        for path in all_paths:
            folder, filename = os.path.split(path)
            basename, ext = os.path.splitext(filename)
        # go ahead and index them, otherwise just create the folder and let
        # them be fetched on-demand
        if os.path.exists(self.PACKAGES_PATH):
            self.index_external_modules()
        else:
            else:
                continue
            qualified_name_components = [this]
            # qualified name should only contain folder names that are packages, not all folder names
            while parent in package_paths:
                parent, this = os.path.split(parent)
                qualified_name_components.append(this)
                           breadcrumb: str=None):
        """Given a root library path (e.g., the Python root path or the dist-
        packages root path), this method traverses it recursively and indexes
        all the packages and modules contained therein. It constructs a mapping
        from the fully qualified module name to a Module object containing the
        metadata that Jedi needs.

        :param index: the dictionary that should be used to store this index (will be modified)
        :param library_path: the root path containing the modules and packages to be indexed
        :param is_stdlib: flag indicating whether this invocation is indexing the standard library
        :param breadcrumb: should be omitted by the caller; this method uses it to keep track of
        the fully qualified module name
        """
        parent, this = os.path.split(library_path)
        basename, extension = os.path.splitext(this)
        if Workspace.is_package(
                library_path) or extension == ".py" and this != "__init__.py":
            qualified_name = ".".join(
                (breadcrumb, basename)) if breadcrumb else basename
        elif extension == ".so":
            basename = basename.split(".")[0]
            qualified_name = ".".join(
                (breadcrumb, basename)) if breadcrumb else basename
        else:
            qualified_name = breadcrumb

        if os.path.isdir(library_path):
            # don't index third-party packages installed in our python path
            if library_path == os.path.join(self.PYTHON_PATH, "site-packages"):
                return

            # recursively index this folder
            for child in os.listdir(library_path):
                self.index_dependencies(index, os.path.join(
                    library_path, child), is_stdlib, qualified_name)
        elif this == "__init__.py":
            # we're already inside a package
            module_name = os.path.basename(parent)
            the_module = Module(module_name,
                                qualified_name,
                                library_path,
                                True,
                                True,
                                is_stdlib)
            index[qualified_name] = the_module
            self.module_paths[os.path.abspath(the_module.path)] = the_module

        elif extension == ".py":
            # just a regular non-package module
            the_module = Module(basename,
                                qualified_name,
                                library_path,
                                False,
                                True,
                                is_stdlib)
            index[qualified_name] = the_module
            self.module_paths[os.path.abspath(the_module.path)] = the_module

        elif extension == ".so":
            # native module -- mark it as such and report a warning or
            # something
            the_module = Module(basename,
                                qualified_name,
                                "",
                                False,
                                True,
                                is_stdlib,
                                True)
            index[qualified_name] = the_module
            self.module_paths[os.path.abspath(the_module.path)] = the_module

    def index_project(self):
        """This method traverses all the project files (starting with
        self.PROJECT_ROOT) and indexes all the packages and modules contained
        therein.

        It constructs a mapping from the fully qualified module name to
        a Module object containing the metadata that Jedi needs. Because
        it only has a flat list of paths/uris to work with (as opposed
        to being able to walk the file tree top-down), it does some
        extra work to figure out the qualified names of each module.
        """
        all_paths = list(self.fs.walk(self.PROJECT_ROOT))

        # TODO: maybe try to exec setup.py with a sandboxed global env and builtins dict or
        # something pre-compute the set of all packages in this project -- this will be useful
        # when trying to figure out each module's qualified name, as well as the packages that
        # are exported by this project
        package_paths = {}
        top_level_modules = set()
        for path in all_paths:
            folder, filename = os.path.split(path)
            if filename == "__init__.py":
                package_paths[folder] = True
            if folder == "/"\
                    and filename.endswith(".py")\
                    and filename not in {"__init__.py", "setup.py", "tests.py", "test.py"}:
                basename, extension = os.path.splitext(filename)
                top_level_modules.add(basename)

        # figure out this project's exports (for xpackages)
        for path in package_paths:
            if path.startswith("/"):
                path = path[1:]
            else:
                continue
            path_components = path.split("/")
            if len(path_components) > 1:
                continue
            else:
                self.project_packages.add(path_components[0])

        # if this is the case, then the exports must be in top-level Python files
        if not self.project_packages:
            for m in top_level_modules:
                self.project_packages.add(m)

        # now index all modules and packages, taking care to compute their qualified names
        # correctly (can be tricky depending on how the folders are nested, and whether they have
        # '__init__.py's or not
        for path in all_paths:
            folder, filename = os.path.split(path)
            basename, ext = os.path.splitext(filename)
            if filename == "__init__.py":
                parent, this = os.path.split(folder)
            elif filename.endswith(".py"):
                parent = folder
                this = basename
            else:
                continue
            qualified_name_components = [this]
            # A module's qualified name should only contain the names of its enclosing folders that
            # are packages (i.e., that contain an '__init__.py'), not the names of *all* its
            # enclosing folders. Hence, the following loop only accumulates qualified name
            # components until it encounters a folder that isn't in the pre-computed
            # set of packages.
            while parent and parent != "/" and parent in package_paths:
                parent, this = os.path.split(parent)
                qualified_name_components.append(this)
            qualified_name_components.reverse()
            qualified_name = ".".join(qualified_name_components)
            if filename == "__init__.py":
                module_name = os.path.basename(folder)
                the_module = Module(module_name, qualified_name, path, True)
                self.project[qualified_name] = the_module
                self.module_paths[the_module.path] = the_module
            elif ext == ".py" and not basename.startswith("__") and not basename.endswith("__"):
                the_module = Module(basename, qualified_name, path)
                self.project[qualified_name] = the_module
                self.module_paths[the_module.path] = the_module

    def find_stdlib_module(self, qualified_name: str) -> Module:
        return self.stdlib.get(qualified_name, None)

    def find_project_module(self, qualified_name: str) -> Module:
        return self.project.get(qualified_name, None)

    def find_external_module(self, qualified_name: str) -> Module:
        package_name = qualified_name.split(".")[0]
        if package_name not in self.fetched:
            self.indexing_lock.acquire()
            self.fetched.add(package_name)
            specifier = self.get_ext_pkg_version_specifier(package_name)
            fetch_dependency(package_name, specifier, self.PACKAGES_PATH, self.pip_args)
            self.index_external_modules()
            self.indexing_lock.release()
        the_module = self.dependencies.get(qualified_name, None)
        if the_module and the_module.is_native:
            raise NotImplementedError("Unable to analyze native modules")
        else:
            return the_module

    def get_ext_pkg_version_specifier(self, package_name):
        """Gets the version specifier to use after parsing the project's
        requirements file.

        (See limitations and caveats in .requirements_parser.parse_requirements()
        and .requirements_parser.get_version_specifier_for_pkg()).

        If a requirements file isn't found at the root of the repo, or if there was an error
        parsing it, a string representing that any version is allowed is returned.
        """
        pkg_specifiers_map = {}
        try:
            pkg_specifiers_map = parse_requirements(
                "/requirements.txt", self.fs)
        except (FileException, FileNotFoundError) as e:
            log.warning(
                "error parsing requirements file for {}, err: {}".format(
                    self.PROJECT_ROOT, e))

        return get_version_specifier_for_pkg(package_name, pkg_specifiers_map)

    def index_external_modules(self):
        for path in os.listdir(self.PACKAGES_PATH):
            if path not in self.indexed_folders:
                self.index_dependencies(
                    self.dependencies, os.path.join(self.PACKAGES_PATH, path))
                self.indexed_folders.add(path)

    def open_module_file(self, the_module: Module,
                         parent_span: opentracing.Span):
        if the_module.path not in self.source_paths:
            return None
        elif the_module.is_external:
            return DummyFile(self.local_fs.open(the_module.path, parent_span))
        else:
            return DummyFile(self.fs.open(the_module.path, parent_span))

    def get_module_by_path(self, path: str) -> Module:
        return self.module_paths.get(path, None)

    def get_modules(self, qualified_name: str) -> List[Module]:
        project_module = self.project.get(qualified_name, None)
        external_module = self.find_external_module(qualified_name)
        stdlib_module = self.stdlib.get(qualified_name, None)
        return list(
            filter(None, [project_module, external_module, stdlib_module]))

    def get_dependencies(self, parent_span: opentracing.Span) -> list:
        top_level_stdlib = {p.split(".")[0] for p in self.stdlib}
        top_level_imports = get_imports(
            self.fs, self.PROJECT_ROOT, parent_span)
        stdlib_imports = top_level_imports & top_level_stdlib
        external_imports = top_level_imports - top_level_stdlib - self.project_packages
        dependencies = [{"attributes": {"name": n}} for n in external_imports]
        if stdlib_imports:
            dependencies.append({
                "attributes": {
                    "name": "cpython",
                    "repoURL": "git://github.com/python/cpython"
                }
            })
        return dependencies

    def get_package_information(self, parent_span: opentracing.Span) -> list:
        if self.is_stdlib:
            return [{
                "package": {"name": "cpython"},
                "dependencies": []
            }]
        else:
            return [
                {
                    "package": {"name": p},
                    # multiple packages in the project share the same
                    # dependencies
                    "dependencies": self.get_dependencies(parent_span)
                } for p in self.project_packages
            ]

    # finds a project module using the newer, more dynamic import rules detailed in PEP 420
    # (see https://www.python.org/dev/peps/pep-0420/)
    def find_internal_module(
            self, name: str, qualified_name: str, dirs: List[str]):
        module_paths = []
        for parent in dirs:
            if os.path.join(parent, name, "__init__.py") in self.source_paths:
                # there's a folder at this level that implements a package with
                # the name we're looking for
                module_path = os.path.join(parent, name, "__init__.py")
                module_file = DummyFile(self.fs.open(module_path))
                return module_file, module_path, True
            elif (os.path.basename(parent) == name and
                  os.path.join(parent, "__init__.py") in self.source_paths):
                # we're already in a package with the name we're looking for
                module_path = os.path.join(parent, "__init__.py")
                module_file = DummyFile(self.fs.open(module_path))
                return module_file, module_path, True
            elif os.path.join(parent, name + ".py") in self.source_paths:
                # there's a file at this level that implements a module with
                # the name we're looking for
                module_path = os.path.join(parent, name + ".py")
                module_file = DummyFile(self.fs.open(module_path))
                return module_file, module_path, False
            elif self.folder_exists(os.path.join(parent, name)):
                # there's a folder at this level that implements a namespace
                # package with the name we're looking for
                module_paths.append(os.path.join(parent, name))
            elif self.folder_exists(parent) and os.path.basename(parent) == name:
                # we're already in a namespace package with the name we're
                # looking for
                module_paths.append(parent)
        if not module_paths:
            return None, None, None
        return None, jedi._compatibility.ImplicitNSInfo(
            qualified_name, module_paths), False

    def folder_exists(self, name):
        for path in self.source_paths:
            if os.path.commonpath((name, path)) == name:
                return True
        return False

    @staticmethod
    def is_package(path: str) -> bool:
        return os.path.isdir(path) and "__init__.py" in os.listdir(path)

    @staticmethod
    def get_top_level_package_names(index: Dict[str, Module]) -> Set[str]:
        return {name.split(".")[0] for name in index}

<MSG> more detailed comments

<DFF> @@ -101,14 +101,25 @@ class Workspace:
                 continue
 
     def index_project(self):
+        """
+        This method traverses all the project files (starting with self.PROJECT_ROOT) and indexes all the packages and
+        modules contained therein. It constructs a mapping from the fully qualified module name to a Module object
+        containing the metadata that Jedi needs. Because it only has a flat list of paths/uris to work with (as
+        opposed to being able to walk the file tree top-down), it does some extra work to figure out the qualified
+        names of each module.
+        """
         all_paths = list(self.fs.walk(self.PROJECT_ROOT))
 
+        # pre-compute the set of all packages in this project -- this will be useful a bit later, when trying to
+        # figure out each module's qualified name
         package_paths = {}
         for path in all_paths:
             folder, filename = os.path.split(path)
             if filename == "__init__.py":
                 package_paths[folder] = True
 
+        # now index all modules and packages, taking care to compute their qualified names correctly (can be tricky
+        # depending on how the folders are nested, and whether they have '__init__.py's or not
         for path in all_paths:
             folder, filename = os.path.split(path)
             basename, ext = os.path.splitext(filename)
@@ -120,7 +131,10 @@ class Workspace:
             else:
                 continue
             qualified_name_components = [this]
-            # qualified name should only contain folder names that are packages, not all folder names
+            # A module's qualified name should only contain the names of its enclosing folders that are packages (i.e.,
+            # that contain an '__init__.py'), not the names of *all* its enclosing folders. Hence, the following loop
+            # only accumulates qualified name components until it encounters a folder that isn't in the pre-computed
+            # set of packages.
             while parent in package_paths:
                 parent, this = os.path.split(parent)
                 qualified_name_components.append(this)

{'commit': 'diff --git a/langserver/workspace.py b/langserver/workspace.py\nindex 653dfe1..7a55c9a 100644\n--- a/langserver/workspace.py\n+++ b/langserver/workspace.py\n@@ -101,14 +101,25 @@ class Workspace:\n                 continue\n \n     def index_project(self):\n+        """\n+        This method traverses all the project files (starting with self.PROJECT_ROOT) and indexes all the packages and\n+        modules contained therein. It constructs a mapping from the fully qualified module name to a Module object\n+        containing the metadata that Jedi needs. Because it only has a flat list of paths/uris to work with (as\n+        opposed to being able to walk the file tree top-down), it does some extra work to figure out the qualified\n+        names of each module.\n+        """\n         all_paths = list(self.fs.walk(self.PROJECT_ROOT))\n \n+        # pre-compute the set of all packages in this project -- this will be useful a bit later, when trying to\n+        # figure out each module\'s qualified name\n         package_paths = {}\n         for path in all_paths:\n             folder, filename = os.path.split(path)\n             if filename == "__init__.py":\n                 package_paths[folder] = True\n \n+        # now index all modules and packages, taking care to compute their qualified names correctly (can be tricky\n+        # depending on how the folders are nested, and whether they have \'__init__.py\'s or not\n         for path in all_paths:\n             folder, filename = os.path.split(path)\n             basename, ext = os.path.splitext(filename)\n@@ -120,7 +131,10 @@ class Workspace:\n             else:\n                 continue\n             qualified_name_components = [this]\n-            # qualified name should only contain folder names that are packages, not all folder names\n+            # A module\'s qualified name should only contain the names of its enclosing folders that are packages (i.e.,\n+            # that contain an \'__init__.py\'), not the names of *all* its enclosing folders. Hence, the following loop\n+            # only accumulates qualified name components until it encounters a folder that isn\'t in the pre-computed\n+            # set of packages.\n             while parent in package_paths:\n                 parent, this = os.path.split(parent)\n                 qualified_name_components.append(this)\n', 'commit_subject': 'more detailed comments', 'repo_name': 'sourcegraph/python-langserver', 'committer': {'name': 'Aaron Leung', 'email': '5954f79e2a5212476aadbec74d7bc0f708506e94@sourcegraph.com', 'time_sec': '1494903280', 'tz_offset': '-420', 'date': {'seconds': '1494903280'}}, 'commit_tree': 'bf90cfcedb3334644a0a65af383a271bc1779fb4', 'commit_message': 'more detailed comments\n', 'language_name': 'Python', 'license': 'mit', 'hunks': '@@ -101,14 +101,25 @@ class Workspace:\n                 continue\n \n     def index_project(self):\n+        """\n+        This method traverses all the project files (starting with self.PROJECT_ROOT) and indexes all the packages and\n+        modules contained therein. It constructs a mapping from the fully qualified module name to a Module object\n+        containing the metadata that Jedi needs. Because it only has a flat list of paths/uris to work with (as\n+        opposed to being able to walk the file tree top-down), it does some extra work to figure out the qualified\n+        names of each module.\n+        """\n         all_paths = list(self.fs.walk(self.PROJECT_ROOT))\n \n+        # pre-compute the set of all packages in this project -- this will be useful a bit later, when trying to\n+        # figure out each module\'s qualified name\n         package_paths = {}\n         for path in all_paths:\n             folder, filename = os.path.split(path)\n             if filename == "__init__.py":\n                 package_paths[folder] = True\n \n+        # now index all modules and packages, taking care to compute their qualified names correctly (can be tricky\n+        # depending on how the folders are nested, and whether they have \'__init__.py\'s or not\n         for path in all_paths:\n             folder, filename = os.path.split(path)\n             basename, ext = os.path.splitext(filename)\n@@ -120,7 +131,10 @@ class Workspace:\n             else:\n                 continue\n             qualified_name_components = [this]\n-            # qualified name should only contain folder names that are packages, not all folder names\n+            # A module\'s qualified name should only contain the names of its enclosing folders that are packages (i.e.,\n+            # that contain an \'__init__.py\'), not the names of *all* its enclosing folders. Hence, the following loop\n+            # only accumulates qualified name components until it encounters a folder that isn\'t in the pre-computed\n+            # set of packages.\n             while parent in package_paths:\n                 parent, this = os.path.split(parent)\n                 qualified_name_components.append(this)\n', 'addition_count': 15, 'deletion_count': 1, 'src_file': 'a/langserver/workspace.py', 'tgt_file': 'b/langserver/workspace.py', 'file_extension': '.py', 'before_file': 'from .config import GlobalConfig\nfrom .fs import FileSystem, LocalFileSystem, FileException\nfrom .imports import get_imports\nfrom .fetch import fetch_dependency\nfrom .requirements_parser import parse_requirements, get_version_specifier_for_pkg\nfrom typing import Dict, Set, List\n\nimport logging\nimport sys\nimport os\nimport os.path\nimport shutil\nimport threading\nimport opentracing\nimport jedi._compatibility\n\n\nlog = logging.getLogger(__name__)\n\n\nclass DummyFile:\n    def __init__(self, contents):\n        self.contents = contents\n\n    def read(self):\n        return self.contents\n\n    def close(self):\n        pass\n\n\nclass Module:\n    def __init__(self,\n                 name: str,\n                 qualified_name: str,\n                 path: str,\n                 is_package: bool=False,\n                 is_external: bool=False,\n                 is_stdlib: bool=False,\n                 is_native: bool=False,\n                 is_namespace_package: bool=False):\n        self.name = name\n        self.qualified_name = qualified_name\n        self.path = path\n        self.is_package = is_package\n        self.is_external = is_external\n        self.is_stdlib = is_stdlib\n        self.is_native = is_native\n        self.is_namespace_package = is_namespace_package\n\n    def __repr__(self):\n        return "PythonModule({}, {})".format(self.name, self.path)\n\n\nclass Workspace:\n\n    def __init__(self, fs: FileSystem, project_root: str,\n                 original_root_path: str= "", pip_args: List[str]=[]):\n\n        self.pip_args = pip_args\n        self.project_packages = set()\n        self.PROJECT_ROOT = project_root\n        self.repo = None\n        self.hash = None\n\n        if original_root_path.startswith(\n                "git://") and "?" in original_root_path:\n            repo_and_hash = original_root_path.split("?")\n            self.repo = repo_and_hash[0]\n            original_root_path = self.repo\n            self.hash = repo_and_hash[1]\n\n        self.is_stdlib = (self.repo == GlobalConfig.STDLIB_REPO_URL)\n\n        if original_root_path.startswith("git://"):\n            original_root_path = original_root_path[6:]\n\n        # turn the original root path into something that can be used as a\n        # file/path name or cache key\n        self.key = original_root_path.replace("/", ".").replace("\\\\", ".")\n        if self.hash:\n            self.key = ".".join((self.key, self.hash))\n\n        # TODO: allow different Python versions per project/workspace\n        self.PYTHON_PATH = GlobalConfig.PYTHON_PATH\n        self.PACKAGES_PATH = os.path.join(\n            GlobalConfig.PACKAGES_PARENT, self.key)\n        log.debug("Setting Python path to %s", self.PYTHON_PATH)\n        log.debug("Setting package path to %s", self.PACKAGES_PATH)\n\n        self.fs = fs\n        self.local_fs = LocalFileSystem()\n        self.source_paths = {path for path in self.fs.walk(\n            self.PROJECT_ROOT) if path.endswith(".py")}\n        self.project = {}\n        self.stdlib = {}\n        self.dependencies = {}\n        self.module_paths = {}\n        # keep track of which package folders have been indexed, since we fetch\n        # and index new folders on-demand\n                continue\n\n    def index_project(self):\n        all_paths = list(self.fs.walk(self.PROJECT_ROOT))\n\n        package_paths = {}\n        for path in all_paths:\n            folder, filename = os.path.split(path)\n            if filename == "__init__.py":\n                package_paths[folder] = True\n\n        for path in all_paths:\n            folder, filename = os.path.split(path)\n            basename, ext = os.path.splitext(filename)\n        # go ahead and index them, otherwise just create the folder and let\n        # them be fetched on-demand\n        if os.path.exists(self.PACKAGES_PATH):\n            self.index_external_modules()\n        else:\n            else:\n                continue\n            qualified_name_components = [this]\n            # qualified name should only contain folder names that are packages, not all folder names\n            while parent in package_paths:\n                parent, this = os.path.split(parent)\n                qualified_name_components.append(this)\n                           breadcrumb: str=None):\n        """Given a root library path (e.g., the Python root path or the dist-\n        packages root path), this method traverses it recursively and indexes\n        all the packages and modules contained therein. It constructs a mapping\n        from the fully qualified module name to a Module object containing the\n        metadata that Jedi needs.\n\n        :param index: the dictionary that should be used to store this index (will be modified)\n        :param library_path: the root path containing the modules and packages to be indexed\n        :param is_stdlib: flag indicating whether this invocation is indexing the standard library\n        :param breadcrumb: should be omitted by the caller; this method uses it to keep track of\n        the fully qualified module name\n        """\n        parent, this = os.path.split(library_path)\n        basename, extension = os.path.splitext(this)\n        if Workspace.is_package(\n                library_path) or extension == ".py" and this != "__init__.py":\n            qualified_name = ".".join(\n                (breadcrumb, basename)) if breadcrumb else basename\n        elif extension == ".so":\n            basename = basename.split(".")[0]\n            qualified_name = ".".join(\n                (breadcrumb, basename)) if breadcrumb else basename\n        else:\n            qualified_name = breadcrumb\n\n        if os.path.isdir(library_path):\n            # don\'t index third-party packages installed in our python path\n            if library_path == os.path.join(self.PYTHON_PATH, "site-packages"):\n                return\n\n            # recursively index this folder\n            for child in os.listdir(library_path):\n                self.index_dependencies(index, os.path.join(\n                    library_path, child), is_stdlib, qualified_name)\n        elif this == "__init__.py":\n            # we\'re already inside a package\n            module_name = os.path.basename(parent)\n            the_module = Module(module_name,\n                                qualified_name,\n                                library_path,\n                                True,\n                                True,\n                                is_stdlib)\n            index[qualified_name] = the_module\n            self.module_paths[os.path.abspath(the_module.path)] = the_module\n\n        elif extension == ".py":\n            # just a regular non-package module\n            the_module = Module(basename,\n                                qualified_name,\n                                library_path,\n                                False,\n                                True,\n                                is_stdlib)\n            index[qualified_name] = the_module\n            self.module_paths[os.path.abspath(the_module.path)] = the_module\n\n        elif extension == ".so":\n            # native module -- mark it as such and report a warning or\n            # something\n            the_module = Module(basename,\n                                qualified_name,\n                                "",\n                                False,\n                                True,\n                                is_stdlib,\n                                True)\n            index[qualified_name] = the_module\n            self.module_paths[os.path.abspath(the_module.path)] = the_module\n\n    def index_project(self):\n        """This method traverses all the project files (starting with\n        self.PROJECT_ROOT) and indexes all the packages and modules contained\n        therein.\n\n        It constructs a mapping from the fully qualified module name to\n        a Module object containing the metadata that Jedi needs. Because\n        it only has a flat list of paths/uris to work with (as opposed\n        to being able to walk the file tree top-down), it does some\n        extra work to figure out the qualified names of each module.\n        """\n        all_paths = list(self.fs.walk(self.PROJECT_ROOT))\n\n        # TODO: maybe try to exec setup.py with a sandboxed global env and builtins dict or\n        # something pre-compute the set of all packages in this project -- this will be useful\n        # when trying to figure out each module\'s qualified name, as well as the packages that\n        # are exported by this project\n        package_paths = {}\n        top_level_modules = set()\n        for path in all_paths:\n            folder, filename = os.path.split(path)\n            if filename == "__init__.py":\n                package_paths[folder] = True\n            if folder == "/"\\\n                    and filename.endswith(".py")\\\n                    and filename not in {"__init__.py", "setup.py", "tests.py", "test.py"}:\n                basename, extension = os.path.splitext(filename)\n                top_level_modules.add(basename)\n\n        # figure out this project\'s exports (for xpackages)\n        for path in package_paths:\n            if path.startswith("/"):\n                path = path[1:]\n            else:\n                continue\n            path_components = path.split("/")\n            if len(path_components) > 1:\n                continue\n            else:\n                self.project_packages.add(path_components[0])\n\n        # if this is the case, then the exports must be in top-level Python files\n        if not self.project_packages:\n            for m in top_level_modules:\n                self.project_packages.add(m)\n\n        # now index all modules and packages, taking care to compute their qualified names\n        # correctly (can be tricky depending on how the folders are nested, and whether they have\n        # \'__init__.py\'s or not\n        for path in all_paths:\n            folder, filename = os.path.split(path)\n            basename, ext = os.path.splitext(filename)\n            if filename == "__init__.py":\n                parent, this = os.path.split(folder)\n            elif filename.endswith(".py"):\n                parent = folder\n                this = basename\n            else:\n                continue\n            qualified_name_components = [this]\n            # A module\'s qualified name should only contain the names of its enclosing folders that\n            # are packages (i.e., that contain an \'__init__.py\'), not the names of *all* its\n            # enclosing folders. Hence, the following loop only accumulates qualified name\n            # components until it encounters a folder that isn\'t in the pre-computed\n            # set of packages.\n            while parent and parent != "/" and parent in package_paths:\n                parent, this = os.path.split(parent)\n                qualified_name_components.append(this)\n            qualified_name_components.reverse()\n            qualified_name = ".".join(qualified_name_components)\n            if filename == "__init__.py":\n                module_name = os.path.basename(folder)\n                the_module = Module(module_name, qualified_name, path, True)\n                self.project[qualified_name] = the_module\n                self.module_paths[the_module.path] = the_module\n            elif ext == ".py" and not basename.startswith("__") and not basename.endswith("__"):\n                the_module = Module(basename, qualified_name, path)\n                self.project[qualified_name] = the_module\n                self.module_paths[the_module.path] = the_module\n\n    def find_stdlib_module(self, qualified_name: str) -> Module:\n        return self.stdlib.get(qualified_name, None)\n\n    def find_project_module(self, qualified_name: str) -> Module:\n        return self.project.get(qualified_name, None)\n\n    def find_external_module(self, qualified_name: str) -> Module:\n        package_name = qualified_name.split(".")[0]\n        if package_name not in self.fetched:\n            self.indexing_lock.acquire()\n            self.fetched.add(package_name)\n            specifier = self.get_ext_pkg_version_specifier(package_name)\n            fetch_dependency(package_name, specifier, self.PACKAGES_PATH, self.pip_args)\n            self.index_external_modules()\n            self.indexing_lock.release()\n        the_module = self.dependencies.get(qualified_name, None)\n        if the_module and the_module.is_native:\n            raise NotImplementedError("Unable to analyze native modules")\n        else:\n            return the_module\n\n    def get_ext_pkg_version_specifier(self, package_name):\n        """Gets the version specifier to use after parsing the project\'s\n        requirements file.\n\n        (See limitations and caveats in .requirements_parser.parse_requirements()\n        and .requirements_parser.get_version_specifier_for_pkg()).\n\n        If a requirements file isn\'t found at the root of the repo, or if there was an error\n        parsing it, a string representing that any version is allowed is returned.\n        """\n        pkg_specifiers_map = {}\n        try:\n            pkg_specifiers_map = parse_requirements(\n                "/requirements.txt", self.fs)\n        except (FileException, FileNotFoundError) as e:\n            log.warning(\n                "error parsing requirements file for {}, err: {}".format(\n                    self.PROJECT_ROOT, e))\n\n        return get_version_specifier_for_pkg(package_name, pkg_specifiers_map)\n\n    def index_external_modules(self):\n        for path in os.listdir(self.PACKAGES_PATH):\n            if path not in self.indexed_folders:\n                self.index_dependencies(\n                    self.dependencies, os.path.join(self.PACKAGES_PATH, path))\n                self.indexed_folders.add(path)\n\n    def open_module_file(self, the_module: Module,\n                         parent_span: opentracing.Span):\n        if the_module.path not in self.source_paths:\n            return None\n        elif the_module.is_external:\n            return DummyFile(self.local_fs.open(the_module.path, parent_span))\n        else:\n            return DummyFile(self.fs.open(the_module.path, parent_span))\n\n    def get_module_by_path(self, path: str) -> Module:\n        return self.module_paths.get(path, None)\n\n    def get_modules(self, qualified_name: str) -> List[Module]:\n        project_module = self.project.get(qualified_name, None)\n        external_module = self.find_external_module(qualified_name)\n        stdlib_module = self.stdlib.get(qualified_name, None)\n        return list(\n            filter(None, [project_module, external_module, stdlib_module]))\n\n    def get_dependencies(self, parent_span: opentracing.Span) -> list:\n        top_level_stdlib = {p.split(".")[0] for p in self.stdlib}\n        top_level_imports = get_imports(\n            self.fs, self.PROJECT_ROOT, parent_span)\n        stdlib_imports = top_level_imports & top_level_stdlib\n        external_imports = top_level_imports - top_level_stdlib - self.project_packages\n        dependencies = [{"attributes": {"name": n}} for n in external_imports]\n        if stdlib_imports:\n            dependencies.append({\n                "attributes": {\n                    "name": "cpython",\n                    "repoURL": "git://github.com/python/cpython"\n                }\n            })\n        return dependencies\n\n    def get_package_information(self, parent_span: opentracing.Span) -> list:\n        if self.is_stdlib:\n            return [{\n                "package": {"name": "cpython"},\n                "dependencies": []\n            }]\n        else:\n            return [\n                {\n                    "package": {"name": p},\n                    # multiple packages in the project share the same\n                    # dependencies\n                    "dependencies": self.get_dependencies(parent_span)\n                } for p in self.project_packages\n            ]\n\n    # finds a project module using the newer, more dynamic import rules detailed in PEP 420\n    # (see https://www.python.org/dev/peps/pep-0420/)\n    def find_internal_module(\n            self, name: str, qualified_name: str, dirs: List[str]):\n        module_paths = []\n        for parent in dirs:\n            if os.path.join(parent, name, "__init__.py") in self.source_paths:\n                # there\'s a folder at this level that implements a package with\n                # the name we\'re looking for\n                module_path = os.path.join(parent, name, "__init__.py")\n                module_file = DummyFile(self.fs.open(module_path))\n                return module_file, module_path, True\n            elif (os.path.basename(parent) == name and\n                  os.path.join(parent, "__init__.py") in self.source_paths):\n                # we\'re already in a package with the name we\'re looking for\n                module_path = os.path.join(parent, "__init__.py")\n                module_file = DummyFile(self.fs.open(module_path))\n                return module_file, module_path, True\n            elif os.path.join(parent, name + ".py") in self.source_paths:\n                # there\'s a file at this level that implements a module with\n                # the name we\'re looking for\n                module_path = os.path.join(parent, name + ".py")\n                module_file = DummyFile(self.fs.open(module_path))\n                return module_file, module_path, False\n            elif self.folder_exists(os.path.join(parent, name)):\n                # there\'s a folder at this level that implements a namespace\n                # package with the name we\'re looking for\n                module_paths.append(os.path.join(parent, name))\n            elif self.folder_exists(parent) and os.path.basename(parent) == name:\n                # we\'re already in a namespace package with the name we\'re\n                # looking for\n                module_paths.append(parent)\n        if not module_paths:\n            return None, None, None\n        return None, jedi._compatibility.ImplicitNSInfo(\n            qualified_name, module_paths), False\n\n    def folder_exists(self, name):\n        for path in self.source_paths:\n            if os.path.commonpath((name, path)) == name:\n                return True\n        return False\n\n    @staticmethod\n    def is_package(path: str) -> bool:\n        return os.path.isdir(path) and "__init__.py" in os.listdir(path)\n\n    @staticmethod\n    def get_top_level_package_names(index: Dict[str, Module]) -> Set[str]:\n        return {name.split(".")[0] for name in index}\n', 'message': 'more detailed comments\n', 'author_name': 'Aaron Leung', 'lang': 'py'}