prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  move hydra generated files into an output directory relative to the job dir
  automatically generated files like config.yaml, overrides.yaml, and others can go here.
  Also additional debug information about the hydra config composition order.
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `hydra/plugins/common/utils.py`
  Content:
  ```
  1 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
  2 
  3 import copy
  4 import logging
  5 import os
  6 import re
  7 import sys
  8 from time import strftime, localtime
  9 
  10 import six
  11 from omegaconf import OmegaConf, DictConfig, ListConfig
  12 
  13 # pylint: disable=C0103
  14 log = logging.getLogger(__name__)
  15 
  16 
  17 def configure_log(log_config, verbose_config):
  18     assert isinstance(verbose_config, (bool, str, ListConfig))
  19     if log_config is not None:
  20         conf = log_config.to_container(resolve=True)
  21         logging.config.dictConfig(conf)
  22     else:
  23         # default logging to stdout
  24         root = logging.getLogger()
  25         root.setLevel(logging.INFO)
  26         handler = logging.StreamHandler(sys.stdout)
  27         formatter = logging.Formatter(
  28             "[%(asctime)s][%(name)s][%(levelname)s] - %(message)s"
  29         )
  30         handler.setFormatter(formatter)
  31         root.addHandler(handler)
  32     if isinstance(verbose_config, bool):
  33         if verbose_config:
  34             logging.getLogger().setLevel(logging.DEBUG)
  35     else:
  36         if isinstance(verbose_config, str):
  37             verbose_config = OmegaConf.create([verbose_config])
  38         for logger in verbose_config:
  39             logging.getLogger(logger).setLevel(logging.DEBUG)
  40 
  41 
  42 def save_config(cfg, filename):
  43     with open(os.path.join(filename), "w") as file:
  44         file.write(cfg.pretty())
  45 
  46 
  47 def get_overrides_dirname(lst, exclude_keys=[]):
  48     lst = [x for x in lst if x not in exclude_keys]
  49     lst.sort()
  50     return re.sub(pattern="[=]", repl="=", string=",".join(lst))
  51 
  52 
  53 def filter_overrides(overrides):
  54     """
  55     :param overrides: overrides list
  56     :return: returning a new overrides list with all the keys starting with hydra. fitlered.
  57     """
  58     return [x for x in overrides if not x.startswith("hydra.")]
  59 
  60 
  61 def run_job(config, task_function, job_dir_key, job_subdir_key):
  62     old_cwd = os.getcwd()
  63     working_dir = str(config.select(job_dir_key))
  64     if job_subdir_key is not None:
  65         # evaluate job_subdir_key lazily.
  66         # this is running on the client side in sweep and contains things such as job:id which
  67         # are only available there.
  68         subdir = str(config.select(job_subdir_key))
  69         working_dir = os.path.join(working_dir, subdir)
  70     try:
  71         ret = JobReturn()
  72         ret.working_dir = working_dir
  73         task_cfg = copy.deepcopy(config)
  74         # TODO: update this after https://github.com/omry/omegaconf/issues/42 is resolved
  75         hydra_cfg = OmegaConf.create({"hydra": task_cfg["hydra"]})
  76         del task_cfg["hydra"]
  77         ret.cfg = task_cfg
  78         ret.hydra_cfg = copy.deepcopy(HydraConfig())
  79         ret.overrides = config.hydra.overrides.task.to_container()
  80         if not os.path.exists(working_dir):
  81             os.makedirs(working_dir)
  82         os.chdir(working_dir)
  83         configure_log(hydra_cfg.hydra.job_logging, hydra_cfg.hydra.verbose)
  84 
  85         save_config(task_cfg, "config.yaml")
  86         save_config(hydra_cfg, "hydra.yaml")
  87         save_config(config.hydra.overrides.task, "overrides.yaml")
  88         ret.return_value = task_function(task_cfg)
  89         ret.task_name = JobRuntime().get("name")
  90         return ret
  91     finally:
  92         os.chdir(old_cwd)
  93 
  94 
  95 def get_valid_filename(s):
  96     s = str(s).strip().replace(" ", "_")
  97     return re.sub(r"(?u)[^-\w.]", "", s)
  98 
  99 
  100 def setup_globals():
  101     try:
  102         OmegaConf.register_resolver(
  103             "now", lambda pattern: strftime(pattern, localtime())
  104         )
  105 
  106         def job_error(x):
  107             raise Exception(
  108                 "job:{} is no longer available. use hydra.job.{}".format(x, x)
  109             )
  110 
  111         OmegaConf.register_resolver("job", job_error)
  112 
  113     except AssertionError:
  114         # calling it again in no_workers mode will throw. safe to ignore.
  115         pass
  116 
  117 
  118 class JobReturn:
  119     def __init__(self):
  120         self.overrides = None
  121         self.return_value = None
  122         self.cfg = None
  123         self.hydra_cfg = None
  124         self.working_dir = None
  125         self.task_name = None
  126 
  127 
  128 class Singleton(type):
  129     _instances = {}
  130 
  131     def __call__(cls, *args, **kwargs):
  132         if cls not in cls._instances:
  133             cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
  134         return cls._instances[cls]
  135 
  136     @staticmethod
  137     def get_state():
  138         return Singleton._instances
  139 
  140     @staticmethod
  141     def set_state(instances):
  142         Singleton._instances = instances
  143 
  144 
  145 @six.add_metaclass(Singleton)
  146 class JobRuntime:
  147     def __init__(self):
  148         self.conf = OmegaConf.create()
  149         self.set("name", "UNKNOWN_NAME")
  150 
  151     def get(self, key):
  152         ret = self.conf.select(key)
  153         if ret is None:
  154             raise KeyError("Key not found in {}: {}".format(type(self).__name__, key))
  155         return ret
  156 
  157     def set(self, key, value):
  158         log.debug("Setting {}:{}={}".format(type(self).__name__, key, value))
  159         self.conf[key] = value
  160 
  161 
  162 @six.add_metaclass(Singleton)
  163 class HydraConfig(DictConfig):
  164     def __init__(self):
  165         super(HydraConfig, self).__init__(content={})
  166         self.hydra = None
  167 
  168     def set_config(self, cfg):
  169         try:
  170             OmegaConf.set_readonly(self, False)
  171             self.hydra = copy.deepcopy(cfg.hydra)
  172         finally:
  173             OmegaConf.set_readonly(self, True)
  174
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/hydra/plugins/common/utils.py b/hydra/plugins/common/utils.py
  --- a/hydra/plugins/common/utils.py
  +++ b/hydra/plugins/common/utils.py
  @@ -5,6 +5,7 @@
   import os
   import re
   import sys
  +from hydra._internal.pathlib import Path
   from time import strftime, localtime
   
   import six
  @@ -39,8 +40,9 @@
               logging.getLogger(logger).setLevel(logging.DEBUG)
   
   
  -def save_config(cfg, filename):
  -    with open(os.path.join(filename), "w") as file:
  +def _save_config(cfg, filename, output_dir):
  +    Path(str(output_dir)).mkdir(parents=True, exist_ok=True)
  +    with open(str(output_dir / filename), "w") as file:
           file.write(cfg.pretty())
   
   
  @@ -77,14 +79,15 @@
           ret.cfg = task_cfg
           ret.hydra_cfg = copy.deepcopy(HydraConfig())
           ret.overrides = config.hydra.overrides.task.to_container()
  -        if not os.path.exists(working_dir):
  -            os.makedirs(working_dir)
  +        # handle output directories here
  +        Path(str(working_dir)).mkdir(parents=True, exist_ok=True)
           os.chdir(working_dir)
  -        configure_log(hydra_cfg.hydra.job_logging, hydra_cfg.hydra.verbose)
  +        hydra_output = Path(hydra_cfg.hydra.output_subdir)
   
  -        save_config(task_cfg, "config.yaml")
  -        save_config(hydra_cfg, "hydra.yaml")
  -        save_config(config.hydra.overrides.task, "overrides.yaml")
  +        configure_log(hydra_cfg.hydra.job_logging, hydra_cfg.hydra.verbose)
  +        _save_config(task_cfg, "config.yaml", hydra_output)
  +        _save_config(hydra_cfg, "hydra.yaml", hydra_output)
  +        _save_config(config.hydra.overrides.task, "overrides.yaml", hydra_output)
           ret.return_value = task_function(task_cfg)
           ret.task_name = JobRuntime().get("name")
           return ret
  
verification_info: {"golden_diff": "diff --git a/hydra/plugins/common/utils.py b/hydra/plugins/common/utils.py\n--- a/hydra/plugins/common/utils.py\n+++ b/hydra/plugins/common/utils.py\n@@ -5,6 +5,7 @@\n import os\n import re\n import sys\n+from hydra._internal.pathlib import Path\n from time import strftime, localtime\n \n import six\n@@ -39,8 +40,9 @@\n             logging.getLogger(logger).setLevel(logging.DEBUG)\n \n \n-def save_config(cfg, filename):\n-    with open(os.path.join(filename), \"w\") as file:\n+def _save_config(cfg, filename, output_dir):\n+    Path(str(output_dir)).mkdir(parents=True, exist_ok=True)\n+    with open(str(output_dir / filename), \"w\") as file:\n         file.write(cfg.pretty())\n \n \n@@ -77,14 +79,15 @@\n         ret.cfg = task_cfg\n         ret.hydra_cfg = copy.deepcopy(HydraConfig())\n         ret.overrides = config.hydra.overrides.task.to_container()\n-        if not os.path.exists(working_dir):\n-            os.makedirs(working_dir)\n+        # handle output directories here\n+        Path(str(working_dir)).mkdir(parents=True, exist_ok=True)\n         os.chdir(working_dir)\n-        configure_log(hydra_cfg.hydra.job_logging, hydra_cfg.hydra.verbose)\n+        hydra_output = Path(hydra_cfg.hydra.output_subdir)\n \n-        save_config(task_cfg, \"config.yaml\")\n-        save_config(hydra_cfg, \"hydra.yaml\")\n-        save_config(config.hydra.overrides.task, \"overrides.yaml\")\n+        configure_log(hydra_cfg.hydra.job_logging, hydra_cfg.hydra.verbose)\n+        _save_config(task_cfg, \"config.yaml\", hydra_output)\n+        _save_config(hydra_cfg, \"hydra.yaml\", hydra_output)\n+        _save_config(config.hydra.overrides.task, \"overrides.yaml\", hydra_output)\n         ret.return_value = task_function(task_cfg)\n         ret.task_name = JobRuntime().get(\"name\")\n         return ret\n", "issue": "move hydra generated files into an output directory relative to the job dir\nautomatically generated files like config.yaml, overrides.yaml, and others can go here.\r\nAlso additional debug information about the hydra config composition order.\n", "before_files": [{"content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\n\nimport copy\nimport logging\nimport os\nimport re\nimport sys\nfrom time import strftime, localtime\n\nimport six\nfrom omegaconf import OmegaConf, DictConfig, ListConfig\n\n# pylint: disable=C0103\nlog = logging.getLogger(__name__)\n\n\ndef configure_log(log_config, verbose_config):\n    assert isinstance(verbose_config, (bool, str, ListConfig))\n    if log_config is not None:\n        conf = log_config.to_container(resolve=True)\n        logging.config.dictConfig(conf)\n    else:\n        # default logging to stdout\n        root = logging.getLogger()\n        root.setLevel(logging.INFO)\n        handler = logging.StreamHandler(sys.stdout)\n        formatter = logging.Formatter(\n            \"[%(asctime)s][%(name)s][%(levelname)s] - %(message)s\"\n        )\n        handler.setFormatter(formatter)\n        root.addHandler(handler)\n    if isinstance(verbose_config, bool):\n        if verbose_config:\n            logging.getLogger().setLevel(logging.DEBUG)\n    else:\n        if isinstance(verbose_config, str):\n            verbose_config = OmegaConf.create([verbose_config])\n        for logger in verbose_config:\n            logging.getLogger(logger).setLevel(logging.DEBUG)\n\n\ndef save_config(cfg, filename):\n    with open(os.path.join(filename), \"w\") as file:\n        file.write(cfg.pretty())\n\n\ndef get_overrides_dirname(lst, exclude_keys=[]):\n    lst = [x for x in lst if x not in exclude_keys]\n    lst.sort()\n    return re.sub(pattern=\"[=]\", repl=\"=\", string=\",\".join(lst))\n\n\ndef filter_overrides(overrides):\n    \"\"\"\n    :param overrides: overrides list\n    :return: returning a new overrides list with all the keys starting with hydra. fitlered.\n    \"\"\"\n    return [x for x in overrides if not x.startswith(\"hydra.\")]\n\n\ndef run_job(config, task_function, job_dir_key, job_subdir_key):\n    old_cwd = os.getcwd()\n    working_dir = str(config.select(job_dir_key))\n    if job_subdir_key is not None:\n        # evaluate job_subdir_key lazily.\n        # this is running on the client side in sweep and contains things such as job:id which\n        # are only available there.\n        subdir = str(config.select(job_subdir_key))\n        working_dir = os.path.join(working_dir, subdir)\n    try:\n        ret = JobReturn()\n        ret.working_dir = working_dir\n        task_cfg = copy.deepcopy(config)\n        # TODO: update this after https://github.com/omry/omegaconf/issues/42 is resolved\n        hydra_cfg = OmegaConf.create({\"hydra\": task_cfg[\"hydra\"]})\n        del task_cfg[\"hydra\"]\n        ret.cfg = task_cfg\n        ret.hydra_cfg = copy.deepcopy(HydraConfig())\n        ret.overrides = config.hydra.overrides.task.to_container()\n        if not os.path.exists(working_dir):\n            os.makedirs(working_dir)\n        os.chdir(working_dir)\n        configure_log(hydra_cfg.hydra.job_logging, hydra_cfg.hydra.verbose)\n\n        save_config(task_cfg, \"config.yaml\")\n        save_config(hydra_cfg, \"hydra.yaml\")\n        save_config(config.hydra.overrides.task, \"overrides.yaml\")\n        ret.return_value = task_function(task_cfg)\n        ret.task_name = JobRuntime().get(\"name\")\n        return ret\n    finally:\n        os.chdir(old_cwd)\n\n\ndef get_valid_filename(s):\n    s = str(s).strip().replace(\" \", \"_\")\n    return re.sub(r\"(?u)[^-\\w.]\", \"\", s)\n\n\ndef setup_globals():\n    try:\n        OmegaConf.register_resolver(\n            \"now\", lambda pattern: strftime(pattern, localtime())\n        )\n\n        def job_error(x):\n            raise Exception(\n                \"job:{} is no longer available. use hydra.job.{}\".format(x, x)\n            )\n\n        OmegaConf.register_resolver(\"job\", job_error)\n\n    except AssertionError:\n        # calling it again in no_workers mode will throw. safe to ignore.\n        pass\n\n\nclass JobReturn:\n    def __init__(self):\n        self.overrides = None\n        self.return_value = None\n        self.cfg = None\n        self.hydra_cfg = None\n        self.working_dir = None\n        self.task_name = None\n\n\nclass Singleton(type):\n    _instances = {}\n\n    def __call__(cls, *args, **kwargs):\n        if cls not in cls._instances:\n            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)\n        return cls._instances[cls]\n\n    @staticmethod\n    def get_state():\n        return Singleton._instances\n\n    @staticmethod\n    def set_state(instances):\n        Singleton._instances = instances\n\n\n@six.add_metaclass(Singleton)\nclass JobRuntime:\n    def __init__(self):\n        self.conf = OmegaConf.create()\n        self.set(\"name\", \"UNKNOWN_NAME\")\n\n    def get(self, key):\n        ret = self.conf.select(key)\n        if ret is None:\n            raise KeyError(\"Key not found in {}: {}\".format(type(self).__name__, key))\n        return ret\n\n    def set(self, key, value):\n        log.debug(\"Setting {}:{}={}\".format(type(self).__name__, key, value))\n        self.conf[key] = value\n\n\n@six.add_metaclass(Singleton)\nclass HydraConfig(DictConfig):\n    def __init__(self):\n        super(HydraConfig, self).__init__(content={})\n        self.hydra = None\n\n    def set_config(self, cfg):\n        try:\n            OmegaConf.set_readonly(self, False)\n            self.hydra = copy.deepcopy(cfg.hydra)\n        finally:\n            OmegaConf.set_readonly(self, True)\n", "path": "hydra/plugins/common/utils.py"}], "after_files": [{"content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\n\nimport copy\nimport logging\nimport os\nimport re\nimport sys\nfrom hydra._internal.pathlib import Path\nfrom time import strftime, localtime\n\nimport six\nfrom omegaconf import OmegaConf, DictConfig, ListConfig\n\n# pylint: disable=C0103\nlog = logging.getLogger(__name__)\n\n\ndef configure_log(log_config, verbose_config):\n    assert isinstance(verbose_config, (bool, str, ListConfig))\n    if log_config is not None:\n        conf = log_config.to_container(resolve=True)\n        logging.config.dictConfig(conf)\n    else:\n        # default logging to stdout\n        root = logging.getLogger()\n        root.setLevel(logging.INFO)\n        handler = logging.StreamHandler(sys.stdout)\n        formatter = logging.Formatter(\n            \"[%(asctime)s][%(name)s][%(levelname)s] - %(message)s\"\n        )\n        handler.setFormatter(formatter)\n        root.addHandler(handler)\n    if isinstance(verbose_config, bool):\n        if verbose_config:\n            logging.getLogger().setLevel(logging.DEBUG)\n    else:\n        if isinstance(verbose_config, str):\n            verbose_config = OmegaConf.create([verbose_config])\n        for logger in verbose_config:\n            logging.getLogger(logger).setLevel(logging.DEBUG)\n\n\ndef _save_config(cfg, filename, output_dir):\n    Path(str(output_dir)).mkdir(parents=True, exist_ok=True)\n    with open(str(output_dir / filename), \"w\") as file:\n        file.write(cfg.pretty())\n\n\ndef get_overrides_dirname(lst, exclude_keys=[]):\n    lst = [x for x in lst if x not in exclude_keys]\n    lst.sort()\n    return re.sub(pattern=\"[=]\", repl=\"=\", string=\",\".join(lst))\n\n\ndef filter_overrides(overrides):\n    \"\"\"\n    :param overrides: overrides list\n    :return: returning a new overrides list with all the keys starting with hydra. fitlered.\n    \"\"\"\n    return [x for x in overrides if not x.startswith(\"hydra.\")]\n\n\ndef run_job(config, task_function, job_dir_key, job_subdir_key):\n    old_cwd = os.getcwd()\n    working_dir = str(config.select(job_dir_key))\n    if job_subdir_key is not None:\n        # evaluate job_subdir_key lazily.\n        # this is running on the client side in sweep and contains things such as job:id which\n        # are only available there.\n        subdir = str(config.select(job_subdir_key))\n        working_dir = os.path.join(working_dir, subdir)\n    try:\n        ret = JobReturn()\n        ret.working_dir = working_dir\n        task_cfg = copy.deepcopy(config)\n        # TODO: update this after https://github.com/omry/omegaconf/issues/42 is resolved\n        hydra_cfg = OmegaConf.create({\"hydra\": task_cfg[\"hydra\"]})\n        del task_cfg[\"hydra\"]\n        ret.cfg = task_cfg\n        ret.hydra_cfg = copy.deepcopy(HydraConfig())\n        ret.overrides = config.hydra.overrides.task.to_container()\n        # handle output directories here\n        Path(str(working_dir)).mkdir(parents=True, exist_ok=True)\n        os.chdir(working_dir)\n        hydra_output = Path(hydra_cfg.hydra.output_subdir)\n\n        configure_log(hydra_cfg.hydra.job_logging, hydra_cfg.hydra.verbose)\n        _save_config(task_cfg, \"config.yaml\", hydra_output)\n        _save_config(hydra_cfg, \"hydra.yaml\", hydra_output)\n        _save_config(config.hydra.overrides.task, \"overrides.yaml\", hydra_output)\n        ret.return_value = task_function(task_cfg)\n        ret.task_name = JobRuntime().get(\"name\")\n        return ret\n    finally:\n        os.chdir(old_cwd)\n\n\ndef get_valid_filename(s):\n    s = str(s).strip().replace(\" \", \"_\")\n    return re.sub(r\"(?u)[^-\\w.]\", \"\", s)\n\n\ndef setup_globals():\n    try:\n        OmegaConf.register_resolver(\n            \"now\", lambda pattern: strftime(pattern, localtime())\n        )\n\n        def job_error(x):\n            raise Exception(\n                \"job:{} is no longer available. use hydra.job.{}\".format(x, x)\n            )\n\n        OmegaConf.register_resolver(\"job\", job_error)\n\n    except AssertionError:\n        # calling it again in no_workers mode will throw. safe to ignore.\n        pass\n\n\nclass JobReturn:\n    def __init__(self):\n        self.overrides = None\n        self.return_value = None\n        self.cfg = None\n        self.hydra_cfg = None\n        self.working_dir = None\n        self.task_name = None\n\n\nclass Singleton(type):\n    _instances = {}\n\n    def __call__(cls, *args, **kwargs):\n        if cls not in cls._instances:\n            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)\n        return cls._instances[cls]\n\n    @staticmethod\n    def get_state():\n        return Singleton._instances\n\n    @staticmethod\n    def set_state(instances):\n        Singleton._instances = instances\n\n\n@six.add_metaclass(Singleton)\nclass JobRuntime:\n    def __init__(self):\n        self.conf = OmegaConf.create()\n        self.set(\"name\", \"UNKNOWN_NAME\")\n\n    def get(self, key):\n        ret = self.conf.select(key)\n        if ret is None:\n            raise KeyError(\"Key not found in {}: {}\".format(type(self).__name__, key))\n        return ret\n\n    def set(self, key, value):\n        log.debug(\"Setting {}:{}={}\".format(type(self).__name__, key, value))\n        self.conf[key] = value\n\n\n@six.add_metaclass(Singleton)\nclass HydraConfig(DictConfig):\n    def __init__(self):\n        super(HydraConfig, self).__init__(content={})\n        self.hydra = None\n\n    def set_config(self, cfg):\n        try:\n            OmegaConf.set_readonly(self, False)\n            self.hydra = copy.deepcopy(cfg.hydra)\n        finally:\n            OmegaConf.set_readonly(self, True)\n", "path": "hydra/plugins/common/utils.py"}]}