We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Telemetry: track `extensions` and `html_theme` Sphinx variables
While working on https://github.com/sphinx-doc/sphinx/issues/10070, @AA-Turner ask for the content of `extensions` and `html_theme` and I think it could be useful to store in the Telemetry database somehow. Considering these values may be dynamically populated, we should find a way to execute this code and grab the final value of them.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `readthedocs/telemetry/collectors.py`
Content:
```
1 """Data collectors."""
2 
3 import json
4 
5 import dparse
6 import structlog
7 
8 from readthedocs.config.models import PythonInstallRequirements
9 
10 log = structlog.get_logger(__name__)
11 
12 
13 class BuildDataCollector:
14 
15     """
16     Build data collector.
17 
18     Collect data from a runnig build.
19     """
20 
21     def __init__(self, environment):
22         self.environment = environment
23         self.build = self.environment.build
24         self.project = self.environment.project
25         self.version = self.environment.version
26         self.config = self.environment.config
27         self.checkout_path = self.project.checkout_path(self.version.slug)
28 
29         log.bind(
30             build_id=self.build["id"],
31             project_slug=self.project.slug,
32             version_slug=self.version.slug,
33         )
34 
35     @staticmethod
36     def _safe_json_loads(content, default=None):
37         def lowercase(d):  # pylint: disable=invalid-name
38             """Convert all dictionary keys to lowercase."""
39             return {k.lower(): i for k, i in d.items()}
40 
41         # pylint: disable=broad-except
42         try:
43             # Use ``object_hook`` parameter to lowercase all the keys of the dictionary.
44             # This helps us to have our data normalized and improve queries.
45             return json.loads(content, object_hook=lowercase)
46         except Exception:
47             log.info(
48                 "Error while loading JSON content.",
49                 exc_info=True,
50             )
51             return default
52 
53     def run(self, *args, **kwargs):
54         build_cmd = self.environment.run(*args, record=False, demux=True, **kwargs)
55         return build_cmd.exit_code, build_cmd.output, build_cmd.error
56 
57     def collect(self):
58         """
59         Collect all relevant data from the runnig build.
60 
61         Data that can be extracted from the database (project/organization)
62         isn't collected here.
63         """
64 
65         # NOTE: we could run each command inside a try/except block to have a
66         # more granular protection and be able to save data from those commands
67         # that didn't fail. Otherwise, if one command fails, all the data for
68         # this Build is lost.
69 
70         data = {}
71         data["config"] = {"user": self.config.source_config}
72         data["os"] = self._get_operating_system()
73         data["python"] = self._get_python_version()
74 
75         user_apt_packages, all_apt_packages = self._get_apt_packages()
76         conda_packages = (
77             self._get_all_conda_packages() if self.config.is_using_conda else {}
78         )
79         data["packages"] = {
80             "pip": {
81                 "user": self._get_user_pip_packages(),
82                 "all": self._get_all_pip_packages(),
83             },
84             "conda": {
85                 "all": conda_packages,
86             },
87             "apt": {
88                 "user": user_apt_packages,
89                 "all": all_apt_packages,
90             },
91         }
92         return data
93 
94     def _get_all_conda_packages(self):
95         """
96         Get all the packages installed by the user using conda.
97 
98         This includes top level and transitive dependencies.
99         The output of ``conda list`` is in the form of::
100 
101             [
102                 {
103                     "base_url": "https://conda.anaconda.org/conda-forge",
104                     "build_number": 0,
105                     "build_string": "py_0",
106                     "channel": "conda-forge",
107                     "dist_name": "alabaster-0.7.12-py_0",
108                     "name": "alabaster",
109                     "platform": "noarch",
110                     "version": "0.7.12"
111                 },
112                 {
113                     "base_url": "https://conda.anaconda.org/conda-forge",
114                     "build_number": 0,
115                     "build_string": "pyh9f0ad1d_0",
116                     "channel": "conda-forge",
117                     "dist_name": "asn1crypto-1.4.0-pyh9f0ad1d_0",
118                     "name": "asn1crypto",
119                     "platform": "noarch",
120                     "version": "1.4.0"
121                 }
122             ]
123         """
124         code, stdout, _ = self.run(
125             "conda", "list", "--json", "--name", self.version.slug
126         )
127         if code == 0 and stdout:
128             packages = self._safe_json_loads(stdout, [])
129             packages = [
130                 {
131                     "name": package["name"],
132                     "channel": package["channel"],
133                     "version": package["version"],
134                 }
135                 for package in packages
136             ]
137             return packages
138         return []
139 
140     def _get_user_pip_packages(self):
141         """
142         Get all the packages to be installed defined by the user.
143 
144         It parses all the requirements files specified in the config file by
145         the user (python.install.requirements) using ``dparse`` --a 3rd party
146         package.
147 
148         If the version of the package is explicit (==) it saves that particular
149         version. Otherwise, if it's not defined, it saves ``undefined`` and if
150         it's a non deterministic operation (like >=, <= or ~=) it saves
151         ``unknown`` in the version.
152 
153         """
154         results = []
155         # pylint: disable=too-many-nested-blocks
156         for install in self.config.python.install:
157             if isinstance(install, PythonInstallRequirements):
158                 if install.requirements:
159                     cmd = ["cat", install.requirements]
160                     _, stdout, _ = self.run(*cmd, cwd=self.checkout_path)
161                     # pylint: disable=invalid-name
162                     df = dparse.parse(
163                         stdout, file_type=dparse.filetypes.requirements_txt
164                     ).serialize()
165                     dependencies = df.get("dependencies", [])
166                     for requirement in dependencies:
167                         name = requirement.get("name", "").lower()
168                         if not name:
169                             continue
170 
171                         # If the user defines a specific version in the
172                         # requirements file, we save it Otherwise, we don't
173                         # because we don't know which version will be
174                         # installed.
175                         version = "undefined"
176                         specs = str(requirement.get("specs", ""))
177                         if specs:
178                             if specs.startswith("=="):
179                                 version = specs.replace("==", "", 1)
180                             else:
181                                 version = "unknown"
182 
183                         results.append(
184                             {
185                                 "name": name,
186                                 "version": version,
187                             }
188                         )
189         return results
190 
191     def _get_all_pip_packages(self):
192         """
193         Get all the packages installed by pip.
194 
195         This includes top level and transitive dependencies.
196         The output of ``pip list`` is in the form of::
197 
198             [
199                 {
200                     "name": "requests-mock",
201                     "version": "1.8.0"
202                 },
203                 {
204                     "name": "requests-toolbelt",
205                     "version": "0.9.1"
206                 },
207                 {
208                     "name": "rstcheck",
209                     "version": "3.3.1"
210                 },
211                 {
212                     "name": "selectolax",
213                     "version": "0.2.10"
214                 },
215                 {
216                     "name": "slumber",
217                     "version": "0.7.1"
218                 }
219             ]
220         """
221         cmd = [
222             "python",
223             "-m",
224             "pip",
225             "list",
226             "--pre",
227             "--local",
228             "--format",
229             "json",
230             "--not-required",
231         ]
232         code, stdout, _ = self.run(*cmd)
233         if code == 0 and stdout:
234             return self._safe_json_loads(stdout, [])
235         return []
236 
237     def _get_operating_system(self):
238         """
239         Get the current operating system.
240 
241         The output of ``lsb_release --description`` is in the form of::
242 
243             Description:	Ubuntu 20.04.3 LTS
244         """
245         code, stdout, _ = self.run("lsb_release", "--description")
246         stdout = stdout.strip()
247         if code == 0 and stdout:
248             parts = stdout.split("\t")
249             if len(parts) == 2:
250                 return parts[1]
251         return ""
252 
253     def _get_apt_packages(self):
254         """
255         Get the list of installed apt packages (global and from the user).
256 
257         The current source of user installed packages is the config file,
258         but we have only the name, so we take the version from the list of all
259         installed packages.
260         """
261         all_apt_packages = self._get_all_apt_packages()
262         all_apt_packages_dict = {
263             package["name"]: package["version"] for package in all_apt_packages
264         }
265         user_apt_packages = self._get_user_apt_packages()
266         for package in user_apt_packages:
267             package["version"] = all_apt_packages_dict.get(package["name"], "")
268         return user_apt_packages, all_apt_packages
269 
270     def _get_all_apt_packages(self):
271         """
272         Get all installed apt packages and their versions.
273 
274         The output of ``dpkg-query --show`` is the form of::
275 
276             adduser 3.116ubuntu1
277             apt 1.6.14
278             base-files 10.1ubuntu2.11
279             base-passwd 3.5.44
280             bash 4.4.18-2ubuntu1.2
281             bsdutils 1:2.31.1-0.4ubuntu3.7
282             bzip2 1.0.6-8.1ubuntu0.2
283             coreutils 8.28-1ubuntu1
284             dash 0.5.8-2.10
285             debconf 1.5.66ubuntu1
286             debianutils 4.8.4
287             diffutils 1:3.6-1
288             dpkg 1.19.0.5ubuntu2.3
289             e2fsprogs 1.44.1-1ubuntu1.3
290             fdisk 2.31.1-0.4ubuntu3.7
291             findutils 4.6.0+git+20170828-2
292             gcc-8-base 8.4.0-1ubuntu1~18.04
293             gpgv 2.2.4-1ubuntu1.4
294             grep 3.1-2build1
295             gzip 1.6-5ubuntu1.2
296             hostname 3.20
297         """
298         code, stdout, _ = self.run(
299             "dpkg-query", "--showformat", "${package} ${version}\\n", "--show"
300         )
301         stdout = stdout.strip()
302         packages = []
303         if code != 0 or not stdout:
304             return packages
305 
306         for line in stdout.split("\n"):
307             parts = line.split()
308             if len(parts) == 2:
309                 package, version = parts
310                 packages.append(
311                     {
312                         "name": package.lower(),
313                         "version": version,
314                     }
315                 )
316 
317         return packages
318 
319     def _get_user_apt_packages(self):
320         return [
321             {"name": package.lower(), "version": ""}
322             for package in self.config.build.apt_packages
323         ]
324 
325     def _get_python_version(self):
326         """
327         Get the python version currently used.
328 
329         The output of ``python --version`` is in the form of::
330 
331             Python 3.8.12
332         """
333         code, stdout, _ = self.run("python", "--version")
334         stdout = stdout.strip()
335         if code == 0 and stdout:
336             parts = stdout.split()
337             if len(parts) == 2:
338                 return parts[1]
339         return ""
340
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/readthedocs/telemetry/collectors.py b/readthedocs/telemetry/collectors.py
--- a/readthedocs/telemetry/collectors.py
+++ b/readthedocs/telemetry/collectors.py
@@ -1,6 +1,7 @@
 """Data collectors."""
 
 import json
+import os
 
 import dparse
 import structlog
@@ -89,6 +90,37 @@
                 "all": all_apt_packages,
             },
         }
+        data["doctool"] = self._get_doctool()
+        return data
+
+    def _get_doctool_name(self):
+        if self.version.is_sphinx_type:
+            return "sphinx"
+
+        if self.version.is_mkdocs_type:
+            return "mkdocs"
+
+        return "generic"
+
+    def _get_doctool(self):
+        data = {
+            "name": self._get_doctool_name(),
+            "extensions": [],
+            "html_theme": "",
+        }
+
+        if self._get_doctool_name() != "sphinx":
+            return data
+
+        conf_py_dir = os.path.join(
+            self.checkout_path,
+            os.path.dirname(self.config.sphinx.configuration),
+        )
+        filepath = os.path.join(conf_py_dir, "_build", "json", "telemetry.json")
+        if os.path.exists(filepath):
+            with open(filepath, "r") as json_file:
+                content = json_file.read()
+            data.update(self._safe_json_loads(content, {}))
         return data
 
     def _get_all_conda_packages(self):

{"golden_diff": "diff --git a/readthedocs/telemetry/collectors.py b/readthedocs/telemetry/collectors.py\n--- a/readthedocs/telemetry/collectors.py\n+++ b/readthedocs/telemetry/collectors.py\n@@ -1,6 +1,7 @@\n \"\"\"Data collectors.\"\"\"\n \n import json\n+import os\n \n import dparse\n import structlog\n@@ -89,6 +90,37 @@\n                 \"all\": all_apt_packages,\n             },\n         }\n+        data[\"doctool\"] = self._get_doctool()\n+        return data\n+\n+    def _get_doctool_name(self):\n+        if self.version.is_sphinx_type:\n+            return \"sphinx\"\n+\n+        if self.version.is_mkdocs_type:\n+            return \"mkdocs\"\n+\n+        return \"generic\"\n+\n+    def _get_doctool(self):\n+        data = {\n+            \"name\": self._get_doctool_name(),\n+            \"extensions\": [],\n+            \"html_theme\": \"\",\n+        }\n+\n+        if self._get_doctool_name() != \"sphinx\":\n+            return data\n+\n+        conf_py_dir = os.path.join(\n+            self.checkout_path,\n+            os.path.dirname(self.config.sphinx.configuration),\n+        )\n+        filepath = os.path.join(conf_py_dir, \"_build\", \"json\", \"telemetry.json\")\n+        if os.path.exists(filepath):\n+            with open(filepath, \"r\") as json_file:\n+                content = json_file.read()\n+            data.update(self._safe_json_loads(content, {}))\n         return data\n \n     def _get_all_conda_packages(self):\n", "issue": "Telemetry: track `extensions` and `html_theme` Sphinx variables\nWhile working on https://github.com/sphinx-doc/sphinx/issues/10070, @AA-Turner ask for the content of `extensions` and `html_theme` and I think it could be useful to store in the Telemetry database somehow. Considering these values may be dynamically populated, we should find a way to execute this code and grab the final value of them.\n", "before_files": [{"content": "\"\"\"Data collectors.\"\"\"\n\nimport json\n\nimport dparse\nimport structlog\n\nfrom readthedocs.config.models import PythonInstallRequirements\n\nlog = structlog.get_logger(__name__)\n\n\nclass BuildDataCollector:\n\n    \"\"\"\n    Build data collector.\n\n    Collect data from a runnig build.\n    \"\"\"\n\n    def __init__(self, environment):\n        self.environment = environment\n        self.build = self.environment.build\n        self.project = self.environment.project\n        self.version = self.environment.version\n        self.config = self.environment.config\n        self.checkout_path = self.project.checkout_path(self.version.slug)\n\n        log.bind(\n            build_id=self.build[\"id\"],\n            project_slug=self.project.slug,\n            version_slug=self.version.slug,\n        )\n\n    @staticmethod\n    def _safe_json_loads(content, default=None):\n        def lowercase(d):  # pylint: disable=invalid-name\n            \"\"\"Convert all dictionary keys to lowercase.\"\"\"\n            return {k.lower(): i for k, i in d.items()}\n\n        # pylint: disable=broad-except\n        try:\n            # Use ``object_hook`` parameter to lowercase all the keys of the dictionary.\n            # This helps us to have our data normalized and improve queries.\n            return json.loads(content, object_hook=lowercase)\n        except Exception:\n            log.info(\n                \"Error while loading JSON content.\",\n                exc_info=True,\n            )\n            return default\n\n    def run(self, *args, **kwargs):\n        build_cmd = self.environment.run(*args, record=False, demux=True, **kwargs)\n        return build_cmd.exit_code, build_cmd.output, build_cmd.error\n\n    def collect(self):\n        \"\"\"\n        Collect all relevant data from the runnig build.\n\n        Data that can be extracted from the database (project/organization)\n        isn't collected here.\n        \"\"\"\n\n        # NOTE: we could run each command inside a try/except block to have a\n        # more granular protection and be able to save data from those commands\n        # that didn't fail. Otherwise, if one command fails, all the data for\n        # this Build is lost.\n\n        data = {}\n        data[\"config\"] = {\"user\": self.config.source_config}\n        data[\"os\"] = self._get_operating_system()\n        data[\"python\"] = self._get_python_version()\n\n        user_apt_packages, all_apt_packages = self._get_apt_packages()\n        conda_packages = (\n            self._get_all_conda_packages() if self.config.is_using_conda else {}\n        )\n        data[\"packages\"] = {\n            \"pip\": {\n                \"user\": self._get_user_pip_packages(),\n                \"all\": self._get_all_pip_packages(),\n            },\n            \"conda\": {\n                \"all\": conda_packages,\n            },\n            \"apt\": {\n                \"user\": user_apt_packages,\n                \"all\": all_apt_packages,\n            },\n        }\n        return data\n\n    def _get_all_conda_packages(self):\n        \"\"\"\n        Get all the packages installed by the user using conda.\n\n        This includes top level and transitive dependencies.\n        The output of ``conda list`` is in the form of::\n\n            [\n                {\n                    \"base_url\": \"https://conda.anaconda.org/conda-forge\",\n                    \"build_number\": 0,\n                    \"build_string\": \"py_0\",\n                    \"channel\": \"conda-forge\",\n                    \"dist_name\": \"alabaster-0.7.12-py_0\",\n                    \"name\": \"alabaster\",\n                    \"platform\": \"noarch\",\n                    \"version\": \"0.7.12\"\n                },\n                {\n                    \"base_url\": \"https://conda.anaconda.org/conda-forge\",\n                    \"build_number\": 0,\n                    \"build_string\": \"pyh9f0ad1d_0\",\n                    \"channel\": \"conda-forge\",\n                    \"dist_name\": \"asn1crypto-1.4.0-pyh9f0ad1d_0\",\n                    \"name\": \"asn1crypto\",\n                    \"platform\": \"noarch\",\n                    \"version\": \"1.4.0\"\n                }\n            ]\n        \"\"\"\n        code, stdout, _ = self.run(\n            \"conda\", \"list\", \"--json\", \"--name\", self.version.slug\n        )\n        if code == 0 and stdout:\n            packages = self._safe_json_loads(stdout, [])\n            packages = [\n                {\n                    \"name\": package[\"name\"],\n                    \"channel\": package[\"channel\"],\n                    \"version\": package[\"version\"],\n                }\n                for package in packages\n            ]\n            return packages\n        return []\n\n    def _get_user_pip_packages(self):\n        \"\"\"\n        Get all the packages to be installed defined by the user.\n\n        It parses all the requirements files specified in the config file by\n        the user (python.install.requirements) using ``dparse`` --a 3rd party\n        package.\n\n        If the version of the package is explicit (==) it saves that particular\n        version. Otherwise, if it's not defined, it saves ``undefined`` and if\n        it's a non deterministic operation (like >=, <= or ~=) it saves\n        ``unknown`` in the version.\n\n        \"\"\"\n        results = []\n        # pylint: disable=too-many-nested-blocks\n        for install in self.config.python.install:\n            if isinstance(install, PythonInstallRequirements):\n                if install.requirements:\n                    cmd = [\"cat\", install.requirements]\n                    _, stdout, _ = self.run(*cmd, cwd=self.checkout_path)\n                    # pylint: disable=invalid-name\n                    df = dparse.parse(\n                        stdout, file_type=dparse.filetypes.requirements_txt\n                    ).serialize()\n                    dependencies = df.get(\"dependencies\", [])\n                    for requirement in dependencies:\n                        name = requirement.get(\"name\", \"\").lower()\n                        if not name:\n                            continue\n\n                        # If the user defines a specific version in the\n                        # requirements file, we save it Otherwise, we don't\n                        # because we don't know which version will be\n                        # installed.\n                        version = \"undefined\"\n                        specs = str(requirement.get(\"specs\", \"\"))\n                        if specs:\n                            if specs.startswith(\"==\"):\n                                version = specs.replace(\"==\", \"\", 1)\n                            else:\n                                version = \"unknown\"\n\n                        results.append(\n                            {\n                                \"name\": name,\n                                \"version\": version,\n                            }\n                        )\n        return results\n\n    def _get_all_pip_packages(self):\n        \"\"\"\n        Get all the packages installed by pip.\n\n        This includes top level and transitive dependencies.\n        The output of ``pip list`` is in the form of::\n\n            [\n                {\n                    \"name\": \"requests-mock\",\n                    \"version\": \"1.8.0\"\n                },\n                {\n                    \"name\": \"requests-toolbelt\",\n                    \"version\": \"0.9.1\"\n                },\n                {\n                    \"name\": \"rstcheck\",\n                    \"version\": \"3.3.1\"\n                },\n                {\n                    \"name\": \"selectolax\",\n                    \"version\": \"0.2.10\"\n                },\n                {\n                    \"name\": \"slumber\",\n                    \"version\": \"0.7.1\"\n                }\n            ]\n        \"\"\"\n        cmd = [\n            \"python\",\n            \"-m\",\n            \"pip\",\n            \"list\",\n            \"--pre\",\n            \"--local\",\n            \"--format\",\n            \"json\",\n            \"--not-required\",\n        ]\n        code, stdout, _ = self.run(*cmd)\n        if code == 0 and stdout:\n            return self._safe_json_loads(stdout, [])\n        return []\n\n    def _get_operating_system(self):\n        \"\"\"\n        Get the current operating system.\n\n        The output of ``lsb_release --description`` is in the form of::\n\n            Description:\tUbuntu 20.04.3 LTS\n        \"\"\"\n        code, stdout, _ = self.run(\"lsb_release\", \"--description\")\n        stdout = stdout.strip()\n        if code == 0 and stdout:\n            parts = stdout.split(\"\\t\")\n            if len(parts) == 2:\n                return parts[1]\n        return \"\"\n\n    def _get_apt_packages(self):\n        \"\"\"\n        Get the list of installed apt packages (global and from the user).\n\n        The current source of user installed packages is the config file,\n        but we have only the name, so we take the version from the list of all\n        installed packages.\n        \"\"\"\n        all_apt_packages = self._get_all_apt_packages()\n        all_apt_packages_dict = {\n            package[\"name\"]: package[\"version\"] for package in all_apt_packages\n        }\n        user_apt_packages = self._get_user_apt_packages()\n        for package in user_apt_packages:\n            package[\"version\"] = all_apt_packages_dict.get(package[\"name\"], \"\")\n        return user_apt_packages, all_apt_packages\n\n    def _get_all_apt_packages(self):\n        \"\"\"\n        Get all installed apt packages and their versions.\n\n        The output of ``dpkg-query --show`` is the form of::\n\n            adduser 3.116ubuntu1\n            apt 1.6.14\n            base-files 10.1ubuntu2.11\n            base-passwd 3.5.44\n            bash 4.4.18-2ubuntu1.2\n            bsdutils 1:2.31.1-0.4ubuntu3.7\n            bzip2 1.0.6-8.1ubuntu0.2\n            coreutils 8.28-1ubuntu1\n            dash 0.5.8-2.10\n            debconf 1.5.66ubuntu1\n            debianutils 4.8.4\n            diffutils 1:3.6-1\n            dpkg 1.19.0.5ubuntu2.3\n            e2fsprogs 1.44.1-1ubuntu1.3\n            fdisk 2.31.1-0.4ubuntu3.7\n            findutils 4.6.0+git+20170828-2\n            gcc-8-base 8.4.0-1ubuntu1~18.04\n            gpgv 2.2.4-1ubuntu1.4\n            grep 3.1-2build1\n            gzip 1.6-5ubuntu1.2\n            hostname 3.20\n        \"\"\"\n        code, stdout, _ = self.run(\n            \"dpkg-query\", \"--showformat\", \"${package} ${version}\\\\n\", \"--show\"\n        )\n        stdout = stdout.strip()\n        packages = []\n        if code != 0 or not stdout:\n            return packages\n\n        for line in stdout.split(\"\\n\"):\n            parts = line.split()\n            if len(parts) == 2:\n                package, version = parts\n                packages.append(\n                    {\n                        \"name\": package.lower(),\n                        \"version\": version,\n                    }\n                )\n\n        return packages\n\n    def _get_user_apt_packages(self):\n        return [\n            {\"name\": package.lower(), \"version\": \"\"}\n            for package in self.config.build.apt_packages\n        ]\n\n    def _get_python_version(self):\n        \"\"\"\n        Get the python version currently used.\n\n        The output of ``python --version`` is in the form of::\n\n            Python 3.8.12\n        \"\"\"\n        code, stdout, _ = self.run(\"python\", \"--version\")\n        stdout = stdout.strip()\n        if code == 0 and stdout:\n            parts = stdout.split()\n            if len(parts) == 2:\n                return parts[1]\n        return \"\"\n", "path": "readthedocs/telemetry/collectors.py"}], "after_files": [{"content": "\"\"\"Data collectors.\"\"\"\n\nimport json\nimport os\n\nimport dparse\nimport structlog\n\nfrom readthedocs.config.models import PythonInstallRequirements\n\nlog = structlog.get_logger(__name__)\n\n\nclass BuildDataCollector:\n\n    \"\"\"\n    Build data collector.\n\n    Collect data from a runnig build.\n    \"\"\"\n\n    def __init__(self, environment):\n        self.environment = environment\n        self.build = self.environment.build\n        self.project = self.environment.project\n        self.version = self.environment.version\n        self.config = self.environment.config\n        self.checkout_path = self.project.checkout_path(self.version.slug)\n\n        log.bind(\n            build_id=self.build[\"id\"],\n            project_slug=self.project.slug,\n            version_slug=self.version.slug,\n        )\n\n    @staticmethod\n    def _safe_json_loads(content, default=None):\n        def lowercase(d):  # pylint: disable=invalid-name\n            \"\"\"Convert all dictionary keys to lowercase.\"\"\"\n            return {k.lower(): i for k, i in d.items()}\n\n        # pylint: disable=broad-except\n        try:\n            # Use ``object_hook`` parameter to lowercase all the keys of the dictionary.\n            # This helps us to have our data normalized and improve queries.\n            return json.loads(content, object_hook=lowercase)\n        except Exception:\n            log.info(\n                \"Error while loading JSON content.\",\n                exc_info=True,\n            )\n            return default\n\n    def run(self, *args, **kwargs):\n        build_cmd = self.environment.run(*args, record=False, demux=True, **kwargs)\n        return build_cmd.exit_code, build_cmd.output, build_cmd.error\n\n    def collect(self):\n        \"\"\"\n        Collect all relevant data from the runnig build.\n\n        Data that can be extracted from the database (project/organization)\n        isn't collected here.\n        \"\"\"\n\n        # NOTE: we could run each command inside a try/except block to have a\n        # more granular protection and be able to save data from those commands\n        # that didn't fail. Otherwise, if one command fails, all the data for\n        # this Build is lost.\n\n        data = {}\n        data[\"config\"] = {\"user\": self.config.source_config}\n        data[\"os\"] = self._get_operating_system()\n        data[\"python\"] = self._get_python_version()\n\n        user_apt_packages, all_apt_packages = self._get_apt_packages()\n        conda_packages = (\n            self._get_all_conda_packages() if self.config.is_using_conda else {}\n        )\n        data[\"packages\"] = {\n            \"pip\": {\n                \"user\": self._get_user_pip_packages(),\n                \"all\": self._get_all_pip_packages(),\n            },\n            \"conda\": {\n                \"all\": conda_packages,\n            },\n            \"apt\": {\n                \"user\": user_apt_packages,\n                \"all\": all_apt_packages,\n            },\n        }\n        data[\"doctool\"] = self._get_doctool()\n        return data\n\n    def _get_doctool_name(self):\n        if self.version.is_sphinx_type:\n            return \"sphinx\"\n\n        if self.version.is_mkdocs_type:\n            return \"mkdocs\"\n\n        return \"generic\"\n\n    def _get_doctool(self):\n        data = {\n            \"name\": self._get_doctool_name(),\n            \"extensions\": [],\n            \"html_theme\": \"\",\n        }\n\n        if self._get_doctool_name() != \"sphinx\":\n            return data\n\n        conf_py_dir = os.path.join(\n            self.checkout_path,\n            os.path.dirname(self.config.sphinx.configuration),\n        )\n        filepath = os.path.join(conf_py_dir, \"_build\", \"json\", \"telemetry.json\")\n        if os.path.exists(filepath):\n            with open(filepath, \"r\") as json_file:\n                content = json_file.read()\n            data.update(self._safe_json_loads(content, {}))\n        return data\n\n    def _get_all_conda_packages(self):\n        \"\"\"\n        Get all the packages installed by the user using conda.\n\n        This includes top level and transitive dependencies.\n        The output of ``conda list`` is in the form of::\n\n            [\n                {\n                    \"base_url\": \"https://conda.anaconda.org/conda-forge\",\n                    \"build_number\": 0,\n                    \"build_string\": \"py_0\",\n                    \"channel\": \"conda-forge\",\n                    \"dist_name\": \"alabaster-0.7.12-py_0\",\n                    \"name\": \"alabaster\",\n                    \"platform\": \"noarch\",\n                    \"version\": \"0.7.12\"\n                },\n                {\n                    \"base_url\": \"https://conda.anaconda.org/conda-forge\",\n                    \"build_number\": 0,\n                    \"build_string\": \"pyh9f0ad1d_0\",\n                    \"channel\": \"conda-forge\",\n                    \"dist_name\": \"asn1crypto-1.4.0-pyh9f0ad1d_0\",\n                    \"name\": \"asn1crypto\",\n                    \"platform\": \"noarch\",\n                    \"version\": \"1.4.0\"\n                }\n            ]\n        \"\"\"\n        code, stdout, _ = self.run(\n            \"conda\", \"list\", \"--json\", \"--name\", self.version.slug\n        )\n        if code == 0 and stdout:\n            packages = self._safe_json_loads(stdout, [])\n            packages = [\n                {\n                    \"name\": package[\"name\"],\n                    \"channel\": package[\"channel\"],\n                    \"version\": package[\"version\"],\n                }\n                for package in packages\n            ]\n            return packages\n        return []\n\n    def _get_user_pip_packages(self):\n        \"\"\"\n        Get all the packages to be installed defined by the user.\n\n        It parses all the requirements files specified in the config file by\n        the user (python.install.requirements) using ``dparse`` --a 3rd party\n        package.\n\n        If the version of the package is explicit (==) it saves that particular\n        version. Otherwise, if it's not defined, it saves ``undefined`` and if\n        it's a non deterministic operation (like >=, <= or ~=) it saves\n        ``unknown`` in the version.\n\n        \"\"\"\n        results = []\n        # pylint: disable=too-many-nested-blocks\n        for install in self.config.python.install:\n            if isinstance(install, PythonInstallRequirements):\n                if install.requirements:\n                    cmd = [\"cat\", install.requirements]\n                    _, stdout, _ = self.run(*cmd, cwd=self.checkout_path)\n                    # pylint: disable=invalid-name\n                    df = dparse.parse(\n                        stdout, file_type=dparse.filetypes.requirements_txt\n                    ).serialize()\n                    dependencies = df.get(\"dependencies\", [])\n                    for requirement in dependencies:\n                        name = requirement.get(\"name\", \"\").lower()\n                        if not name:\n                            continue\n\n                        # If the user defines a specific version in the\n                        # requirements file, we save it Otherwise, we don't\n                        # because we don't know which version will be\n                        # installed.\n                        version = \"undefined\"\n                        specs = str(requirement.get(\"specs\", \"\"))\n                        if specs:\n                            if specs.startswith(\"==\"):\n                                version = specs.replace(\"==\", \"\", 1)\n                            else:\n                                version = \"unknown\"\n\n                        results.append(\n                            {\n                                \"name\": name,\n                                \"version\": version,\n                            }\n                        )\n        return results\n\n    def _get_all_pip_packages(self):\n        \"\"\"\n        Get all the packages installed by pip.\n\n        This includes top level and transitive dependencies.\n        The output of ``pip list`` is in the form of::\n\n            [\n                {\n                    \"name\": \"requests-mock\",\n                    \"version\": \"1.8.0\"\n                },\n                {\n                    \"name\": \"requests-toolbelt\",\n                    \"version\": \"0.9.1\"\n                },\n                {\n                    \"name\": \"rstcheck\",\n                    \"version\": \"3.3.1\"\n                },\n                {\n                    \"name\": \"selectolax\",\n                    \"version\": \"0.2.10\"\n                },\n                {\n                    \"name\": \"slumber\",\n                    \"version\": \"0.7.1\"\n                }\n            ]\n        \"\"\"\n        cmd = [\n            \"python\",\n            \"-m\",\n            \"pip\",\n            \"list\",\n            \"--pre\",\n            \"--local\",\n            \"--format\",\n            \"json\",\n            \"--not-required\",\n        ]\n        code, stdout, _ = self.run(*cmd)\n        if code == 0 and stdout:\n            return self._safe_json_loads(stdout, [])\n        return []\n\n    def _get_operating_system(self):\n        \"\"\"\n        Get the current operating system.\n\n        The output of ``lsb_release --description`` is in the form of::\n\n            Description:\tUbuntu 20.04.3 LTS\n        \"\"\"\n        code, stdout, _ = self.run(\"lsb_release\", \"--description\")\n        stdout = stdout.strip()\n        if code == 0 and stdout:\n            parts = stdout.split(\"\\t\")\n            if len(parts) == 2:\n                return parts[1]\n        return \"\"\n\n    def _get_apt_packages(self):\n        \"\"\"\n        Get the list of installed apt packages (global and from the user).\n\n        The current source of user installed packages is the config file,\n        but we have only the name, so we take the version from the list of all\n        installed packages.\n        \"\"\"\n        all_apt_packages = self._get_all_apt_packages()\n        all_apt_packages_dict = {\n            package[\"name\"]: package[\"version\"] for package in all_apt_packages\n        }\n        user_apt_packages = self._get_user_apt_packages()\n        for package in user_apt_packages:\n            package[\"version\"] = all_apt_packages_dict.get(package[\"name\"], \"\")\n        return user_apt_packages, all_apt_packages\n\n    def _get_all_apt_packages(self):\n        \"\"\"\n        Get all installed apt packages and their versions.\n\n        The output of ``dpkg-query --show`` is the form of::\n\n            adduser 3.116ubuntu1\n            apt 1.6.14\n            base-files 10.1ubuntu2.11\n            base-passwd 3.5.44\n            bash 4.4.18-2ubuntu1.2\n            bsdutils 1:2.31.1-0.4ubuntu3.7\n            bzip2 1.0.6-8.1ubuntu0.2\n            coreutils 8.28-1ubuntu1\n            dash 0.5.8-2.10\n            debconf 1.5.66ubuntu1\n            debianutils 4.8.4\n            diffutils 1:3.6-1\n            dpkg 1.19.0.5ubuntu2.3\n            e2fsprogs 1.44.1-1ubuntu1.3\n            fdisk 2.31.1-0.4ubuntu3.7\n            findutils 4.6.0+git+20170828-2\n            gcc-8-base 8.4.0-1ubuntu1~18.04\n            gpgv 2.2.4-1ubuntu1.4\n            grep 3.1-2build1\n            gzip 1.6-5ubuntu1.2\n            hostname 3.20\n        \"\"\"\n        code, stdout, _ = self.run(\n            \"dpkg-query\", \"--showformat\", \"${package} ${version}\\\\n\", \"--show\"\n        )\n        stdout = stdout.strip()\n        packages = []\n        if code != 0 or not stdout:\n            return packages\n\n        for line in stdout.split(\"\\n\"):\n            parts = line.split()\n            if len(parts) == 2:\n                package, version = parts\n                packages.append(\n                    {\n                        \"name\": package.lower(),\n                        \"version\": version,\n                    }\n                )\n\n        return packages\n\n    def _get_user_apt_packages(self):\n        return [\n            {\"name\": package.lower(), \"version\": \"\"}\n            for package in self.config.build.apt_packages\n        ]\n\n    def _get_python_version(self):\n        \"\"\"\n        Get the python version currently used.\n\n        The output of ``python --version`` is in the form of::\n\n            Python 3.8.12\n        \"\"\"\n        code, stdout, _ = self.run(\"python\", \"--version\")\n        stdout = stdout.strip()\n        if code == 0 and stdout:\n            parts = stdout.split()\n            if len(parts) == 2:\n                return parts[1]\n        return \"\"\n", "path": "readthedocs/telemetry/collectors.py"}]}