We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
jobs get-output-files should split downloaded files into subdirectories
> ```
> ll /home/cleber/avocado/job-results/job-2020-06-04T15.17-9bd5b5e
> total 708
> -rw-rw-r--. 1 cleber cleber     41 Jun  4 15:17 id
> drwxrwxr-x. 2 cleber cleber   4096 Jun  4 15:17 jobdata
> -rw-rw-r--. 1 cleber cleber  15818 Jun  4 15:17 job.log
> -rw-rw-r--. 1 cleber cleber 668480 Jun  4 15:17 results.html
> -rw-rw-r--. 1 cleber cleber   1029 Jun  4 15:17 results.json
> -rw-rw-r--. 1 cleber cleber     36 Jun  4 15:17 results.tap
> -rw-rw-r--. 1 cleber cleber    345 Jun  4 15:17 results.xml
> drwxrwxr-x. 5 cleber cleber   4096 Jun  4 15:17 sysinfo
> -rw-rw-r--. 1 cleber cleber     16 Jun  4 15:18 test2_created
> -rw-rw-r--. 1 cleber cleber     16 Jun  4 15:18 test_created
> drwxrwxr-x. 4 cleber cleber   4096 Jun  4 15:17 test-results
> ```
> 
> I had the expectation that, `test_created` would to to `test-results/1-_tmp_t.sh/` and `test2_created` would go to `test-results/2-_tmp_t2.sh/`. Is this expected now? Yes, no, in the future?

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `avocado/plugins/jobs.py`
Content:
```
1 # This program is free software; you can redistribute it and/or modify
2 # it under the terms of the GNU General Public License as published by
3 # the Free Software Foundation; either version 2 of the License, or
4 # (at your option) any later version.
5 #
6 # This program is distributed in the hope that it will be useful,
7 # but WITHOUT ANY WARRANTY; without even the implied warranty of
8 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
9 #
10 # See LICENSE for more details.
11 #
12 # Copyright: Red Hat Inc. 2020
13 # Authors: Beraldo Leal <bleal@redhat.com>
14 
15 """
16 Jobs subcommand
17 """
18 import json
19 import os
20 
21 from datetime import datetime
22 from glob import glob
23 
24 from avocado.core import exit_codes
25 from avocado.core import output
26 from avocado.core.data_dir import get_job_results_dir, get_logs_dir
27 from avocado.core.future.settings import settings
28 from avocado.core.output import LOG_UI
29 from avocado.core.plugin_interfaces import CLICmd
30 from avocado.core.spawners.exceptions import SpawnerException
31 from avocado.core.spawners.podman import PodmanSpawner
32 from avocado.core.spawners.process import ProcessSpawner
33 from avocado.utils import astring
34 
35 
36 class Jobs(CLICmd):
37     """
38     Implements the avocado 'jobs' subcommand
39     """
40     name = 'jobs'
41     description = 'Manage Avocado jobs'
42 
43     def _get_data_from_file(self, filename):
44         if not filename or not os.path.isfile(filename):
45             raise FileNotFoundError('File not found {}'.format(filename))
46 
47         with open(filename, 'r') as fp:
48             return json.load(fp)
49 
50     def _print_job_details(self, details):
51         for key, value in details.items():
52             LOG_UI.info("%-12s: %s", key, value)
53 
54     def _print_job_tests(self, tests):
55         test_matrix = []
56         date_fmt = "%Y/%m/%d %H:%M:%S"
57         for test in tests:
58             status = test.get('status')
59             decorator = output.TEST_STATUS_DECORATOR_MAPPING.get(status)
60             end = datetime.fromtimestamp(test.get('end'))
61             test_matrix.append((test.get('id'),
62                                 end.strftime(date_fmt),
63                                 "%5f" % float(test.get('time')),
64                                 decorator(status, '')))
65         header = (output.TERM_SUPPORT.header_str('Test ID'),
66                   output.TERM_SUPPORT.header_str('End Time'),
67                   output.TERM_SUPPORT.header_str('Run Time'),
68                   output.TERM_SUPPORT.header_str('Status'))
69         for line in astring.iter_tabular_output(test_matrix,
70                                                 header=header,
71                                                 strip=True):
72             LOG_UI.debug(line)
73 
74     def _save_stream_to_file(self, stream, filename):
75         """Save stream to a file.
76 
77         Directory must exists before calling this function.
78         """
79         dirname = os.path.dirname(filename)
80         if not os.path.isdir(dirname):
81             LOG_UI.error("%s does not exist. Exiting...", dirname)
82             return exit_codes.AVOCADO_GENERIC_CRASH
83 
84         with open(filename, 'ab') as output_file:
85             output_file.write(stream)
86 
87     def configure(self, parser):
88         """
89         Add the subparser for the assets action.
90 
91         :param parser: The Avocado command line application parser
92         :type parser: :class:`avocado.core.parser.ArgumentParser`
93         """
94         parser = super(Jobs, self).configure(parser)
95 
96         subcommands = parser.add_subparsers(dest='jobs_subcommand',
97                                             metavar='sub-command')
98         subcommands.required = True
99 
100         help_msg = 'List all known jobs by Avocado'
101         subcommands.add_parser('list', help=help_msg)
102 
103         help_msg = ('Show details about a specific job. When passing a Job '
104                     'ID, you can use any Job Reference (job_id, "latest", '
105                     'or job results path).')
106         show_parser = subcommands.add_parser('show', help=help_msg)
107         settings.register_option(section='jobs.show',
108                                  key='job_id',
109                                  help_msg='JOB id',
110                                  metavar='JOBID',
111                                  default='latest',
112                                  nargs='?',
113                                  positional_arg=True,
114                                  parser=show_parser)
115         help_msg = ('Download output files generated by tests on '
116                     'AVOCADO_TEST_OUTPUT_DIR')
117         output_files_parser = subcommands.add_parser('get-output-files',
118                                                      help=help_msg)
119         settings.register_option(section='jobs.get.output_files',
120                                  key='job_id',
121                                  help_msg='JOB id',
122                                  metavar='JOBID',
123                                  default=None,
124                                  positional_arg=True,
125                                  parser=output_files_parser)
126 
127         settings.register_option(section='jobs.get.output_files',
128                                  key='destination',
129                                  help_msg='Destination path',
130                                  metavar='DESTINATION',
131                                  default=None,
132                                  positional_arg=True,
133                                  parser=output_files_parser)
134 
135     def handle_list_command(self, jobs_results):
136         """Called when 'avocado jobs list' command is executed."""
137 
138         for filename in jobs_results.values():
139             with open(filename, 'r') as fp:
140                 job = json.load(fp)
141                 try:
142                     started_ts = job['tests'][0]['start']
143                     started = datetime.fromtimestamp(started_ts)
144                 except IndexError:
145                     continue
146                 LOG_UI.info("%-40s %-26s %3s (%s/%s/%s/%s)",
147                             job['job_id'],
148                             str(started),
149                             job['total'],
150                             job['pass'],
151                             job['skip'],
152                             job['errors'],
153                             job['failures'])
154 
155         return exit_codes.AVOCADO_ALL_OK
156 
157     def _download_tests(self, tests, destination, job_id, spawner):
158         for test in tests:
159             test_id = test.get('id')
160             LOG_UI.info("Downloading files for test %s", test_id)
161             try:
162                 files_buffers = spawner().stream_output(job_id, test_id)
163                 for filename, stream in files_buffers:
164                     dest = os.path.join(destination, filename)
165                     self._save_stream_to_file(stream, dest)
166             except SpawnerException as ex:
167                 LOG_UI.error("Error: Failed to download: %s. Exiting...", ex)
168                 return exit_codes.AVOCADO_GENERIC_CRASH
169         return exit_codes.AVOCADO_ALL_OK
170 
171     def handle_output_files_command(self, config):
172         """Called when 'avocado jobs get-output-files' command is executed."""
173 
174         job_id = config.get('jobs.get.output_files.job_id')
175         destination = config.get('jobs.get.output_files.destination')
176 
177         results_dir = get_job_results_dir(job_id)
178         results_file = os.path.join(results_dir, 'results.json')
179         config_file = os.path.join(results_dir, 'jobdata/args.json')
180 
181         try:
182             config_data = self._get_data_from_file(config_file)
183             results_data = self._get_data_from_file(results_file)
184         except FileNotFoundError as ex:
185             LOG_UI.error("Could not get job information: %s", ex)
186             return exit_codes.AVOCADO_GENERIC_CRASH
187 
188         spawners = {'process': ProcessSpawner,
189                     'podman': PodmanSpawner}
190 
191         spawner_name = config_data.get('nrun.spawner')
192         spawner = spawners.get(spawner_name)
193 
194         if spawner is None:
195             msg = ("Could not find the spawner for job %s. This command is "
196                    "experimental and only supported when job executed with "
197                    "the Spawner architecture.")
198             LOG_UI.error(msg, job_id)
199             return exit_codes.AVOCADO_GENERIC_CRASH
200 
201         return self._download_tests(results_data.get('tests'),
202                                     destination,
203                                     job_id,
204                                     spawner)
205 
206     def handle_show_command(self, config):
207         """Called when 'avocado jobs show' command is executed."""
208 
209         job_id = config.get('jobs.show.job_id')
210         results_dir = get_job_results_dir(job_id)
211         if results_dir is None:
212             LOG_UI.error("Error: Job %s not found", job_id)
213             return exit_codes.AVOCADO_GENERIC_CRASH
214 
215         results_file = os.path.join(results_dir, 'results.json')
216         config_file = os.path.join(results_dir, 'jobdata/args.json')
217         try:
218             results_data = self._get_data_from_file(results_file)
219         except FileNotFoundError as ex:
220             # Results data are important and should exit if not found
221             LOG_UI.error(ex)
222             return exit_codes.AVOCADO_GENERIC_CRASH
223 
224         try:
225             config_data = self._get_data_from_file(config_file)
226         except FileNotFoundError:
227             pass
228 
229         data = {'JOB ID': job_id,
230                 'JOB LOG': results_data.get('debuglog'),
231                 'SPAWNER': config_data.get('nrun.spawner', 'unknown')}
232 
233         # We could improve this soon with more data and colors
234         self._print_job_details(data)
235         LOG_UI.info("")
236         self._print_job_tests(results_data.get('tests'))
237         results = ('PASS %d | ERROR %d | FAIL %d | SKIP %d |'
238                    'WARN %d | INTERRUPT %s | CANCEL %s')
239         results %= (results_data.get('pass', 0),
240                     results_data.get('error', 0),
241                     results_data.get('failures', 0),
242                     results_data.get('skip', 0),
243                     results_data.get('warn', 0),
244                     results_data.get('interrupt', 0),
245                     results_data.get('cancel', 0))
246         self._print_job_details({'RESULTS': results})
247         return exit_codes.AVOCADO_ALL_OK
248 
249     def run(self, config):
250         results = {}
251 
252         jobs_dir = get_logs_dir()
253         for result in glob(os.path.join(jobs_dir, '*/results.json')):
254             with open(result, 'r') as fp:
255                 job = json.load(fp)
256                 results[job['job_id']] = result
257 
258         subcommand = config.get('jobs_subcommand')
259         if subcommand == 'list':
260             return self.handle_list_command(results)
261         elif subcommand == 'show':
262             return self.handle_show_command(config)
263         elif subcommand == 'get-output-files':
264             return self.handle_output_files_command(config)
265         return exit_codes.AVOCADO_ALL_OK
266
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/avocado/plugins/jobs.py b/avocado/plugins/jobs.py
--- a/avocado/plugins/jobs.py
+++ b/avocado/plugins/jobs.py
@@ -77,9 +77,7 @@
         Directory must exists before calling this function.
         """
         dirname = os.path.dirname(filename)
-        if not os.path.isdir(dirname):
-            LOG_UI.error("%s does not exist. Exiting...", dirname)
-            return exit_codes.AVOCADO_GENERIC_CRASH
+        os.makedirs(dirname, exist_ok=True)
 
         with open(filename, 'ab') as output_file:
             output_file.write(stream)
@@ -161,7 +159,9 @@
             try:
                 files_buffers = spawner().stream_output(job_id, test_id)
                 for filename, stream in files_buffers:
-                    dest = os.path.join(destination, filename)
+                    dest = os.path.join(destination,
+                                        test_id.replace('/', '_'),
+                                        filename)
                     self._save_stream_to_file(stream, dest)
             except SpawnerException as ex:
                 LOG_UI.error("Error: Failed to download: %s. Exiting...", ex)

{"golden_diff": "diff --git a/avocado/plugins/jobs.py b/avocado/plugins/jobs.py\n--- a/avocado/plugins/jobs.py\n+++ b/avocado/plugins/jobs.py\n@@ -77,9 +77,7 @@\n         Directory must exists before calling this function.\n         \"\"\"\n         dirname = os.path.dirname(filename)\n-        if not os.path.isdir(dirname):\n-            LOG_UI.error(\"%s does not exist. Exiting...\", dirname)\n-            return exit_codes.AVOCADO_GENERIC_CRASH\n+        os.makedirs(dirname, exist_ok=True)\n \n         with open(filename, 'ab') as output_file:\n             output_file.write(stream)\n@@ -161,7 +159,9 @@\n             try:\n                 files_buffers = spawner().stream_output(job_id, test_id)\n                 for filename, stream in files_buffers:\n-                    dest = os.path.join(destination, filename)\n+                    dest = os.path.join(destination,\n+                                        test_id.replace('/', '_'),\n+                                        filename)\n                     self._save_stream_to_file(stream, dest)\n             except SpawnerException as ex:\n                 LOG_UI.error(\"Error: Failed to download: %s. Exiting...\", ex)\n", "issue": "jobs get-output-files should split downloaded files into subdirectories\n> ```\r\n> ll /home/cleber/avocado/job-results/job-2020-06-04T15.17-9bd5b5e\r\n> total 708\r\n> -rw-rw-r--. 1 cleber cleber     41 Jun  4 15:17 id\r\n> drwxrwxr-x. 2 cleber cleber   4096 Jun  4 15:17 jobdata\r\n> -rw-rw-r--. 1 cleber cleber  15818 Jun  4 15:17 job.log\r\n> -rw-rw-r--. 1 cleber cleber 668480 Jun  4 15:17 results.html\r\n> -rw-rw-r--. 1 cleber cleber   1029 Jun  4 15:17 results.json\r\n> -rw-rw-r--. 1 cleber cleber     36 Jun  4 15:17 results.tap\r\n> -rw-rw-r--. 1 cleber cleber    345 Jun  4 15:17 results.xml\r\n> drwxrwxr-x. 5 cleber cleber   4096 Jun  4 15:17 sysinfo\r\n> -rw-rw-r--. 1 cleber cleber     16 Jun  4 15:18 test2_created\r\n> -rw-rw-r--. 1 cleber cleber     16 Jun  4 15:18 test_created\r\n> drwxrwxr-x. 4 cleber cleber   4096 Jun  4 15:17 test-results\r\n> ```\r\n> \r\n> I had the expectation that, `test_created` would to to `test-results/1-_tmp_t.sh/` and `test2_created` would go to `test-results/2-_tmp_t2.sh/`. Is this expected now? Yes, no, in the future?\n", "before_files": [{"content": "# This program is free software; you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation; either version 2 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n#\n# See LICENSE for more details.\n#\n# Copyright: Red Hat Inc. 2020\n# Authors: Beraldo Leal <bleal@redhat.com>\n\n\"\"\"\nJobs subcommand\n\"\"\"\nimport json\nimport os\n\nfrom datetime import datetime\nfrom glob import glob\n\nfrom avocado.core import exit_codes\nfrom avocado.core import output\nfrom avocado.core.data_dir import get_job_results_dir, get_logs_dir\nfrom avocado.core.future.settings import settings\nfrom avocado.core.output import LOG_UI\nfrom avocado.core.plugin_interfaces import CLICmd\nfrom avocado.core.spawners.exceptions import SpawnerException\nfrom avocado.core.spawners.podman import PodmanSpawner\nfrom avocado.core.spawners.process import ProcessSpawner\nfrom avocado.utils import astring\n\n\nclass Jobs(CLICmd):\n    \"\"\"\n    Implements the avocado 'jobs' subcommand\n    \"\"\"\n    name = 'jobs'\n    description = 'Manage Avocado jobs'\n\n    def _get_data_from_file(self, filename):\n        if not filename or not os.path.isfile(filename):\n            raise FileNotFoundError('File not found {}'.format(filename))\n\n        with open(filename, 'r') as fp:\n            return json.load(fp)\n\n    def _print_job_details(self, details):\n        for key, value in details.items():\n            LOG_UI.info(\"%-12s: %s\", key, value)\n\n    def _print_job_tests(self, tests):\n        test_matrix = []\n        date_fmt = \"%Y/%m/%d %H:%M:%S\"\n        for test in tests:\n            status = test.get('status')\n            decorator = output.TEST_STATUS_DECORATOR_MAPPING.get(status)\n            end = datetime.fromtimestamp(test.get('end'))\n            test_matrix.append((test.get('id'),\n                                end.strftime(date_fmt),\n                                \"%5f\" % float(test.get('time')),\n                                decorator(status, '')))\n        header = (output.TERM_SUPPORT.header_str('Test ID'),\n                  output.TERM_SUPPORT.header_str('End Time'),\n                  output.TERM_SUPPORT.header_str('Run Time'),\n                  output.TERM_SUPPORT.header_str('Status'))\n        for line in astring.iter_tabular_output(test_matrix,\n                                                header=header,\n                                                strip=True):\n            LOG_UI.debug(line)\n\n    def _save_stream_to_file(self, stream, filename):\n        \"\"\"Save stream to a file.\n\n        Directory must exists before calling this function.\n        \"\"\"\n        dirname = os.path.dirname(filename)\n        if not os.path.isdir(dirname):\n            LOG_UI.error(\"%s does not exist. Exiting...\", dirname)\n            return exit_codes.AVOCADO_GENERIC_CRASH\n\n        with open(filename, 'ab') as output_file:\n            output_file.write(stream)\n\n    def configure(self, parser):\n        \"\"\"\n        Add the subparser for the assets action.\n\n        :param parser: The Avocado command line application parser\n        :type parser: :class:`avocado.core.parser.ArgumentParser`\n        \"\"\"\n        parser = super(Jobs, self).configure(parser)\n\n        subcommands = parser.add_subparsers(dest='jobs_subcommand',\n                                            metavar='sub-command')\n        subcommands.required = True\n\n        help_msg = 'List all known jobs by Avocado'\n        subcommands.add_parser('list', help=help_msg)\n\n        help_msg = ('Show details about a specific job. When passing a Job '\n                    'ID, you can use any Job Reference (job_id, \"latest\", '\n                    'or job results path).')\n        show_parser = subcommands.add_parser('show', help=help_msg)\n        settings.register_option(section='jobs.show',\n                                 key='job_id',\n                                 help_msg='JOB id',\n                                 metavar='JOBID',\n                                 default='latest',\n                                 nargs='?',\n                                 positional_arg=True,\n                                 parser=show_parser)\n        help_msg = ('Download output files generated by tests on '\n                    'AVOCADO_TEST_OUTPUT_DIR')\n        output_files_parser = subcommands.add_parser('get-output-files',\n                                                     help=help_msg)\n        settings.register_option(section='jobs.get.output_files',\n                                 key='job_id',\n                                 help_msg='JOB id',\n                                 metavar='JOBID',\n                                 default=None,\n                                 positional_arg=True,\n                                 parser=output_files_parser)\n\n        settings.register_option(section='jobs.get.output_files',\n                                 key='destination',\n                                 help_msg='Destination path',\n                                 metavar='DESTINATION',\n                                 default=None,\n                                 positional_arg=True,\n                                 parser=output_files_parser)\n\n    def handle_list_command(self, jobs_results):\n        \"\"\"Called when 'avocado jobs list' command is executed.\"\"\"\n\n        for filename in jobs_results.values():\n            with open(filename, 'r') as fp:\n                job = json.load(fp)\n                try:\n                    started_ts = job['tests'][0]['start']\n                    started = datetime.fromtimestamp(started_ts)\n                except IndexError:\n                    continue\n                LOG_UI.info(\"%-40s %-26s %3s (%s/%s/%s/%s)\",\n                            job['job_id'],\n                            str(started),\n                            job['total'],\n                            job['pass'],\n                            job['skip'],\n                            job['errors'],\n                            job['failures'])\n\n        return exit_codes.AVOCADO_ALL_OK\n\n    def _download_tests(self, tests, destination, job_id, spawner):\n        for test in tests:\n            test_id = test.get('id')\n            LOG_UI.info(\"Downloading files for test %s\", test_id)\n            try:\n                files_buffers = spawner().stream_output(job_id, test_id)\n                for filename, stream in files_buffers:\n                    dest = os.path.join(destination, filename)\n                    self._save_stream_to_file(stream, dest)\n            except SpawnerException as ex:\n                LOG_UI.error(\"Error: Failed to download: %s. Exiting...\", ex)\n                return exit_codes.AVOCADO_GENERIC_CRASH\n        return exit_codes.AVOCADO_ALL_OK\n\n    def handle_output_files_command(self, config):\n        \"\"\"Called when 'avocado jobs get-output-files' command is executed.\"\"\"\n\n        job_id = config.get('jobs.get.output_files.job_id')\n        destination = config.get('jobs.get.output_files.destination')\n\n        results_dir = get_job_results_dir(job_id)\n        results_file = os.path.join(results_dir, 'results.json')\n        config_file = os.path.join(results_dir, 'jobdata/args.json')\n\n        try:\n            config_data = self._get_data_from_file(config_file)\n            results_data = self._get_data_from_file(results_file)\n        except FileNotFoundError as ex:\n            LOG_UI.error(\"Could not get job information: %s\", ex)\n            return exit_codes.AVOCADO_GENERIC_CRASH\n\n        spawners = {'process': ProcessSpawner,\n                    'podman': PodmanSpawner}\n\n        spawner_name = config_data.get('nrun.spawner')\n        spawner = spawners.get(spawner_name)\n\n        if spawner is None:\n            msg = (\"Could not find the spawner for job %s. This command is \"\n                   \"experimental and only supported when job executed with \"\n                   \"the Spawner architecture.\")\n            LOG_UI.error(msg, job_id)\n            return exit_codes.AVOCADO_GENERIC_CRASH\n\n        return self._download_tests(results_data.get('tests'),\n                                    destination,\n                                    job_id,\n                                    spawner)\n\n    def handle_show_command(self, config):\n        \"\"\"Called when 'avocado jobs show' command is executed.\"\"\"\n\n        job_id = config.get('jobs.show.job_id')\n        results_dir = get_job_results_dir(job_id)\n        if results_dir is None:\n            LOG_UI.error(\"Error: Job %s not found\", job_id)\n            return exit_codes.AVOCADO_GENERIC_CRASH\n\n        results_file = os.path.join(results_dir, 'results.json')\n        config_file = os.path.join(results_dir, 'jobdata/args.json')\n        try:\n            results_data = self._get_data_from_file(results_file)\n        except FileNotFoundError as ex:\n            # Results data are important and should exit if not found\n            LOG_UI.error(ex)\n            return exit_codes.AVOCADO_GENERIC_CRASH\n\n        try:\n            config_data = self._get_data_from_file(config_file)\n        except FileNotFoundError:\n            pass\n\n        data = {'JOB ID': job_id,\n                'JOB LOG': results_data.get('debuglog'),\n                'SPAWNER': config_data.get('nrun.spawner', 'unknown')}\n\n        # We could improve this soon with more data and colors\n        self._print_job_details(data)\n        LOG_UI.info(\"\")\n        self._print_job_tests(results_data.get('tests'))\n        results = ('PASS %d | ERROR %d | FAIL %d | SKIP %d |'\n                   'WARN %d | INTERRUPT %s | CANCEL %s')\n        results %= (results_data.get('pass', 0),\n                    results_data.get('error', 0),\n                    results_data.get('failures', 0),\n                    results_data.get('skip', 0),\n                    results_data.get('warn', 0),\n                    results_data.get('interrupt', 0),\n                    results_data.get('cancel', 0))\n        self._print_job_details({'RESULTS': results})\n        return exit_codes.AVOCADO_ALL_OK\n\n    def run(self, config):\n        results = {}\n\n        jobs_dir = get_logs_dir()\n        for result in glob(os.path.join(jobs_dir, '*/results.json')):\n            with open(result, 'r') as fp:\n                job = json.load(fp)\n                results[job['job_id']] = result\n\n        subcommand = config.get('jobs_subcommand')\n        if subcommand == 'list':\n            return self.handle_list_command(results)\n        elif subcommand == 'show':\n            return self.handle_show_command(config)\n        elif subcommand == 'get-output-files':\n            return self.handle_output_files_command(config)\n        return exit_codes.AVOCADO_ALL_OK\n", "path": "avocado/plugins/jobs.py"}], "after_files": [{"content": "# This program is free software; you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation; either version 2 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n#\n# See LICENSE for more details.\n#\n# Copyright: Red Hat Inc. 2020\n# Authors: Beraldo Leal <bleal@redhat.com>\n\n\"\"\"\nJobs subcommand\n\"\"\"\nimport json\nimport os\n\nfrom datetime import datetime\nfrom glob import glob\n\nfrom avocado.core import exit_codes\nfrom avocado.core import output\nfrom avocado.core.data_dir import get_job_results_dir, get_logs_dir\nfrom avocado.core.future.settings import settings\nfrom avocado.core.output import LOG_UI\nfrom avocado.core.plugin_interfaces import CLICmd\nfrom avocado.core.spawners.exceptions import SpawnerException\nfrom avocado.core.spawners.podman import PodmanSpawner\nfrom avocado.core.spawners.process import ProcessSpawner\nfrom avocado.utils import astring\n\n\nclass Jobs(CLICmd):\n    \"\"\"\n    Implements the avocado 'jobs' subcommand\n    \"\"\"\n    name = 'jobs'\n    description = 'Manage Avocado jobs'\n\n    def _get_data_from_file(self, filename):\n        if not filename or not os.path.isfile(filename):\n            raise FileNotFoundError('File not found {}'.format(filename))\n\n        with open(filename, 'r') as fp:\n            return json.load(fp)\n\n    def _print_job_details(self, details):\n        for key, value in details.items():\n            LOG_UI.info(\"%-12s: %s\", key, value)\n\n    def _print_job_tests(self, tests):\n        test_matrix = []\n        date_fmt = \"%Y/%m/%d %H:%M:%S\"\n        for test in tests:\n            status = test.get('status')\n            decorator = output.TEST_STATUS_DECORATOR_MAPPING.get(status)\n            end = datetime.fromtimestamp(test.get('end'))\n            test_matrix.append((test.get('id'),\n                                end.strftime(date_fmt),\n                                \"%5f\" % float(test.get('time')),\n                                decorator(status, '')))\n        header = (output.TERM_SUPPORT.header_str('Test ID'),\n                  output.TERM_SUPPORT.header_str('End Time'),\n                  output.TERM_SUPPORT.header_str('Run Time'),\n                  output.TERM_SUPPORT.header_str('Status'))\n        for line in astring.iter_tabular_output(test_matrix,\n                                                header=header,\n                                                strip=True):\n            LOG_UI.debug(line)\n\n    def _save_stream_to_file(self, stream, filename):\n        \"\"\"Save stream to a file.\n\n        Directory must exists before calling this function.\n        \"\"\"\n        dirname = os.path.dirname(filename)\n        os.makedirs(dirname, exist_ok=True)\n\n        with open(filename, 'ab') as output_file:\n            output_file.write(stream)\n\n    def configure(self, parser):\n        \"\"\"\n        Add the subparser for the assets action.\n\n        :param parser: The Avocado command line application parser\n        :type parser: :class:`avocado.core.parser.ArgumentParser`\n        \"\"\"\n        parser = super(Jobs, self).configure(parser)\n\n        subcommands = parser.add_subparsers(dest='jobs_subcommand',\n                                            metavar='sub-command')\n        subcommands.required = True\n\n        help_msg = 'List all known jobs by Avocado'\n        subcommands.add_parser('list', help=help_msg)\n\n        help_msg = ('Show details about a specific job. When passing a Job '\n                    'ID, you can use any Job Reference (job_id, \"latest\", '\n                    'or job results path).')\n        show_parser = subcommands.add_parser('show', help=help_msg)\n        settings.register_option(section='jobs.show',\n                                 key='job_id',\n                                 help_msg='JOB id',\n                                 metavar='JOBID',\n                                 default='latest',\n                                 nargs='?',\n                                 positional_arg=True,\n                                 parser=show_parser)\n        help_msg = ('Download output files generated by tests on '\n                    'AVOCADO_TEST_OUTPUT_DIR')\n        output_files_parser = subcommands.add_parser('get-output-files',\n                                                     help=help_msg)\n        settings.register_option(section='jobs.get.output_files',\n                                 key='job_id',\n                                 help_msg='JOB id',\n                                 metavar='JOBID',\n                                 default=None,\n                                 positional_arg=True,\n                                 parser=output_files_parser)\n\n        settings.register_option(section='jobs.get.output_files',\n                                 key='destination',\n                                 help_msg='Destination path',\n                                 metavar='DESTINATION',\n                                 default=None,\n                                 positional_arg=True,\n                                 parser=output_files_parser)\n\n    def handle_list_command(self, jobs_results):\n        \"\"\"Called when 'avocado jobs list' command is executed.\"\"\"\n\n        for filename in jobs_results.values():\n            with open(filename, 'r') as fp:\n                job = json.load(fp)\n                try:\n                    started_ts = job['tests'][0]['start']\n                    started = datetime.fromtimestamp(started_ts)\n                except IndexError:\n                    continue\n                LOG_UI.info(\"%-40s %-26s %3s (%s/%s/%s/%s)\",\n                            job['job_id'],\n                            str(started),\n                            job['total'],\n                            job['pass'],\n                            job['skip'],\n                            job['errors'],\n                            job['failures'])\n\n        return exit_codes.AVOCADO_ALL_OK\n\n    def _download_tests(self, tests, destination, job_id, spawner):\n        for test in tests:\n            test_id = test.get('id')\n            LOG_UI.info(\"Downloading files for test %s\", test_id)\n            try:\n                files_buffers = spawner().stream_output(job_id, test_id)\n                for filename, stream in files_buffers:\n                    dest = os.path.join(destination,\n                                        test_id.replace('/', '_'),\n                                        filename)\n                    self._save_stream_to_file(stream, dest)\n            except SpawnerException as ex:\n                LOG_UI.error(\"Error: Failed to download: %s. Exiting...\", ex)\n                return exit_codes.AVOCADO_GENERIC_CRASH\n        return exit_codes.AVOCADO_ALL_OK\n\n    def handle_output_files_command(self, config):\n        \"\"\"Called when 'avocado jobs get-output-files' command is executed.\"\"\"\n\n        job_id = config.get('jobs.get.output_files.job_id')\n        destination = config.get('jobs.get.output_files.destination')\n\n        results_dir = get_job_results_dir(job_id)\n        results_file = os.path.join(results_dir, 'results.json')\n        config_file = os.path.join(results_dir, 'jobdata/args.json')\n\n        try:\n            config_data = self._get_data_from_file(config_file)\n            results_data = self._get_data_from_file(results_file)\n        except FileNotFoundError as ex:\n            LOG_UI.error(\"Could not get job information: %s\", ex)\n            return exit_codes.AVOCADO_GENERIC_CRASH\n\n        spawners = {'process': ProcessSpawner,\n                    'podman': PodmanSpawner}\n\n        spawner_name = config_data.get('nrun.spawner')\n        spawner = spawners.get(spawner_name)\n\n        if spawner is None:\n            msg = (\"Could not find the spawner for job %s. This command is \"\n                   \"experimental and only supported when job executed with \"\n                   \"the Spawner architecture.\")\n            LOG_UI.error(msg, job_id)\n            return exit_codes.AVOCADO_GENERIC_CRASH\n\n        return self._download_tests(results_data.get('tests'),\n                                    destination,\n                                    job_id,\n                                    spawner)\n\n    def handle_show_command(self, config):\n        \"\"\"Called when 'avocado jobs show' command is executed.\"\"\"\n\n        job_id = config.get('jobs.show.job_id')\n        results_dir = get_job_results_dir(job_id)\n        if results_dir is None:\n            LOG_UI.error(\"Error: Job %s not found\", job_id)\n            return exit_codes.AVOCADO_GENERIC_CRASH\n\n        results_file = os.path.join(results_dir, 'results.json')\n        config_file = os.path.join(results_dir, 'jobdata/args.json')\n        try:\n            results_data = self._get_data_from_file(results_file)\n        except FileNotFoundError as ex:\n            # Results data are important and should exit if not found\n            LOG_UI.error(ex)\n            return exit_codes.AVOCADO_GENERIC_CRASH\n\n        try:\n            config_data = self._get_data_from_file(config_file)\n        except FileNotFoundError:\n            pass\n\n        data = {'JOB ID': job_id,\n                'JOB LOG': results_data.get('debuglog'),\n                'SPAWNER': config_data.get('nrun.spawner', 'unknown')}\n\n        # We could improve this soon with more data and colors\n        self._print_job_details(data)\n        LOG_UI.info(\"\")\n        self._print_job_tests(results_data.get('tests'))\n        results = ('PASS %d | ERROR %d | FAIL %d | SKIP %d |'\n                   'WARN %d | INTERRUPT %s | CANCEL %s')\n        results %= (results_data.get('pass', 0),\n                    results_data.get('error', 0),\n                    results_data.get('failures', 0),\n                    results_data.get('skip', 0),\n                    results_data.get('warn', 0),\n                    results_data.get('interrupt', 0),\n                    results_data.get('cancel', 0))\n        self._print_job_details({'RESULTS': results})\n        return exit_codes.AVOCADO_ALL_OK\n\n    def run(self, config):\n        results = {}\n\n        jobs_dir = get_logs_dir()\n        for result in glob(os.path.join(jobs_dir, '*/results.json')):\n            with open(result, 'r') as fp:\n                job = json.load(fp)\n                results[job['job_id']] = result\n\n        subcommand = config.get('jobs_subcommand')\n        if subcommand == 'list':\n            return self.handle_list_command(results)\n        elif subcommand == 'show':\n            return self.handle_show_command(config)\n        elif subcommand == 'get-output-files':\n            return self.handle_output_files_command(config)\n        return exit_codes.AVOCADO_ALL_OK\n", "path": "avocado/plugins/jobs.py"}]}