prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  AllenNLP integration: Can't load configurations that expect ext_vars from environment variables
  Optuna's AllenNLP integration evaluates the jsonnet config that you give it, passing in the trial parameters as the `ext_vars`. AllenNLP's default behavior when loading a configuration is a little different, in that it also passes through environment variables: 
  
  https://github.com/allenai/allennlp/blob/4de68a42a0df89df01a999ee48f361f24c8c19d4/allennlp/common/params.py#L488
  
  So if you rely on specifying e.g. a path to a file as an environment variable, that configuration won't load as expected when using the integration
  
  ## Expected behavior
  
  Optuna should pass environment variables through when the jsonnet is interpreted. Ideally, it'd just use the AllenNLP config loading code directly so that all behavior is identical, just overriding whatever the trial needs to override.
  
  ## Environment
  
  - Optuna version: 1.5.0
  - Python version: 3.6
  - OS: Ubuntu 18.04.4
  - (Optional) Other libraries and their versions: AllenNLP 1.0
  
  ## Error messages, stack traces, or logs
  
  You'll get `RuntimeError: undefined external variable` during the interpretation of the jsonnet here:
  
  https://github.com/optuna/optuna/blob/9f5c5cf5c36df1628567f0b4a9873892d148ad40/optuna/integration/allennlp.py#L108
  
  ## Steps to reproduce
  
  1. Use a AllenNLP configuration that expects to be able to use an environment variable to fill an extVar
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `optuna/integration/allennlp.py`
  Content:
  ```
  1 import json
  2 import os
  3 from typing import Any
  4 from typing import Dict
  5 from typing import List
  6 from typing import Optional
  7 from typing import Union
  8 import warnings
  9 
  10 import optuna
  11 from optuna._experimental import experimental
  12 from optuna._imports import try_import
  13 
  14 with try_import() as _imports:
  15     import allennlp.commands
  16     import allennlp.common.util
  17 
  18 if _imports.is_successful():
  19     import _jsonnet
  20 
  21 
  22 def dump_best_config(input_config_file: str, output_config_file: str, study: optuna.Study) -> None:
  23     """Save JSON config file after updating with parameters from the best trial in the study.
  24 
  25     Args:
  26         input_config_file:
  27             Input Jsonnet config file used with
  28             :class:`~optuna.integration.AllenNLPExecutor`.
  29         output_config_file:
  30             Output JSON config file.
  31         study:
  32             Instance of :class:`~optuna.study.Study`.
  33             Note that :func:`~optuna.study.Study.optimize` must have been called.
  34 
  35     """
  36     _imports.check()
  37 
  38     best_params = study.best_params
  39     for key, value in best_params.items():
  40         best_params[key] = str(value)
  41     best_config = json.loads(_jsonnet.evaluate_file(input_config_file, ext_vars=best_params))
  42     best_config = allennlp.common.params.infer_and_cast(best_config)
  43 
  44     with open(output_config_file, "w") as f:
  45         json.dump(best_config, f, indent=4)
  46 
  47 
  48 @experimental("1.4.0")
  49 class AllenNLPExecutor(object):
  50     """AllenNLP extension to use optuna with Jsonnet config file.
  51 
  52     This feature is experimental since AllenNLP major release will come soon.
  53     The interface may change without prior notice to correspond to the update.
  54 
  55     See the examples of `objective function <https://github.com/optuna/optuna/blob/
  56     master/examples/allennlp/allennlp_jsonnet.py>`_ and
  57     `config file <https://github.com/optuna/optuna/blob/master/
  58     examples/allennlp/classifier.jsonnet>`_.
  59 
  60     Args:
  61         trial:
  62             A :class:`~optuna.trial.Trial` corresponding to the current evaluation
  63             of the objective function.
  64         config_file:
  65             Config file for AllenNLP.
  66             Hyperparameters should be masked with ``std.extVar``.
  67             Please refer to `the config example <https://github.com/allenai/allentune/blob/
  68             master/examples/classifier.jsonnet>`_.
  69         serialization_dir:
  70             A path which model weights and logs are saved.
  71         metrics:
  72             An evaluation metric for the result of ``objective``.
  73         include_package:
  74             Additional packages to include.
  75             For more information, please see
  76             `AllenNLP documentation <https://docs.allennlp.org/master/api/commands/train/>`_.
  77 
  78     """
  79 
  80     def __init__(
  81         self,
  82         trial: optuna.Trial,
  83         config_file: str,
  84         serialization_dir: str,
  85         metrics: str = "best_validation_accuracy",
  86         *,
  87         include_package: Optional[Union[str, List[str]]] = None
  88     ):
  89         _imports.check()
  90 
  91         self._params = trial.params
  92         self._config_file = config_file
  93         self._serialization_dir = serialization_dir
  94         self._metrics = metrics
  95         if include_package is None:
  96             include_package = []
  97         if isinstance(include_package, str):
  98             self._include_package = [include_package]
  99         else:
  100             self._include_package = include_package
  101 
  102     def _build_params(self) -> Dict[str, Any]:
  103         """Create a dict of params for AllenNLP."""
  104         # _build_params is based on allentune's train_func.
  105         # https://github.com/allenai/allentune/blob/master/allentune/modules/allennlp_runner.py#L34-L65
  106         for key, value in self._params.items():
  107             self._params[key] = str(value)
  108         _params = json.loads(_jsonnet.evaluate_file(self._config_file, ext_vars=self._params))
  109 
  110         # _params contains a list of string or string as value values.
  111         # Some params couldn't be casted correctly and
  112         # infer_and_cast converts them into desired values.
  113         return allennlp.common.params.infer_and_cast(_params)
  114 
  115     def run(self) -> float:
  116         """Train a model using AllenNLP."""
  117         try:
  118             import_func = allennlp.common.util.import_submodules
  119         except AttributeError:
  120             import_func = allennlp.common.util.import_module_and_submodules
  121             warnings.warn("AllenNLP>0.9 has not been supported officially yet.")
  122 
  123         for package_name in self._include_package:
  124             import_func(package_name)
  125 
  126         params = allennlp.common.params.Params(self._build_params())
  127         allennlp.commands.train.train_model(params, self._serialization_dir)
  128 
  129         metrics = json.load(open(os.path.join(self._serialization_dir, "metrics.json")))
  130         return metrics[self._metrics]
  131
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/optuna/integration/allennlp.py b/optuna/integration/allennlp.py
  --- a/optuna/integration/allennlp.py
  +++ b/optuna/integration/allennlp.py
  @@ -57,6 +57,13 @@
       `config file <https://github.com/optuna/optuna/blob/master/
       examples/allennlp/classifier.jsonnet>`_.
   
  +    .. note::
  +        In :class:`~optuna.integration.AllenNLPExecutor`,
  +        you can pass parameters to AllenNLP by either defining a search space using
  +        Optuna suggest methods or setting environment variables just like AllenNLP CLI.
  +        If a value is set in both a search space in Optuna and the environment variables,
  +        the executor will use the value specified in the search space in Optuna.
  +
       Args:
           trial:
               A :class:`~optuna.trial.Trial` corresponding to the current evaluation
  @@ -100,17 +107,29 @@
               self._include_package = include_package
   
       def _build_params(self) -> Dict[str, Any]:
  -        """Create a dict of params for AllenNLP."""
  -        # _build_params is based on allentune's train_func.
  -        # https://github.com/allenai/allentune/blob/master/allentune/modules/allennlp_runner.py#L34-L65
  -        for key, value in self._params.items():
  -            self._params[key] = str(value)
  -        _params = json.loads(_jsonnet.evaluate_file(self._config_file, ext_vars=self._params))
  -
  -        # _params contains a list of string or string as value values.
  +        """Create a dict of params for AllenNLP.
  +
  +        _build_params is based on allentune's train_func.
  +        For more detail, please refer to
  +        https://github.com/allenai/allentune/blob/master/allentune/modules/allennlp_runner.py#L34-L65
  +
  +        """
  +        params = self._environment_variables()
  +        params.update({key: str(value) for key, value in self._params.items()})
  +
  +        allennlp_params = json.loads(_jsonnet.evaluate_file(self._config_file, ext_vars=params))
  +        # allennlp_params contains a list of string or string as value values.
           # Some params couldn't be casted correctly and
           # infer_and_cast converts them into desired values.
  -        return allennlp.common.params.infer_and_cast(_params)
  +        return allennlp.common.params.infer_and_cast(allennlp_params)
  +
  +    @staticmethod
  +    def _is_encodable(value: str) -> bool:
  +        # https://github.com/allenai/allennlp/blob/master/allennlp/common/params.py#L77-L85
  +        return (value == "") or (value.encode("utf-8", "ignore") != b"")
  +
  +    def _environment_variables(self) -> Dict[str, str]:
  +        return {key: value for key, value in os.environ.items() if self._is_encodable(value)}
   
       def run(self) -> float:
           """Train a model using AllenNLP."""
  
verification_info: {"golden_diff": "diff --git a/optuna/integration/allennlp.py b/optuna/integration/allennlp.py\n--- a/optuna/integration/allennlp.py\n+++ b/optuna/integration/allennlp.py\n@@ -57,6 +57,13 @@\n     `config file <https://github.com/optuna/optuna/blob/master/\n     examples/allennlp/classifier.jsonnet>`_.\n \n+    .. note::\n+        In :class:`~optuna.integration.AllenNLPExecutor`,\n+        you can pass parameters to AllenNLP by either defining a search space using\n+        Optuna suggest methods or setting environment variables just like AllenNLP CLI.\n+        If a value is set in both a search space in Optuna and the environment variables,\n+        the executor will use the value specified in the search space in Optuna.\n+\n     Args:\n         trial:\n             A :class:`~optuna.trial.Trial` corresponding to the current evaluation\n@@ -100,17 +107,29 @@\n             self._include_package = include_package\n \n     def _build_params(self) -> Dict[str, Any]:\n-        \"\"\"Create a dict of params for AllenNLP.\"\"\"\n-        # _build_params is based on allentune's train_func.\n-        # https://github.com/allenai/allentune/blob/master/allentune/modules/allennlp_runner.py#L34-L65\n-        for key, value in self._params.items():\n-            self._params[key] = str(value)\n-        _params = json.loads(_jsonnet.evaluate_file(self._config_file, ext_vars=self._params))\n-\n-        # _params contains a list of string or string as value values.\n+        \"\"\"Create a dict of params for AllenNLP.\n+\n+        _build_params is based on allentune's train_func.\n+        For more detail, please refer to\n+        https://github.com/allenai/allentune/blob/master/allentune/modules/allennlp_runner.py#L34-L65\n+\n+        \"\"\"\n+        params = self._environment_variables()\n+        params.update({key: str(value) for key, value in self._params.items()})\n+\n+        allennlp_params = json.loads(_jsonnet.evaluate_file(self._config_file, ext_vars=params))\n+        # allennlp_params contains a list of string or string as value values.\n         # Some params couldn't be casted correctly and\n         # infer_and_cast converts them into desired values.\n-        return allennlp.common.params.infer_and_cast(_params)\n+        return allennlp.common.params.infer_and_cast(allennlp_params)\n+\n+    @staticmethod\n+    def _is_encodable(value: str) -> bool:\n+        # https://github.com/allenai/allennlp/blob/master/allennlp/common/params.py#L77-L85\n+        return (value == \"\") or (value.encode(\"utf-8\", \"ignore\") != b\"\")\n+\n+    def _environment_variables(self) -> Dict[str, str]:\n+        return {key: value for key, value in os.environ.items() if self._is_encodable(value)}\n \n     def run(self) -> float:\n         \"\"\"Train a model using AllenNLP.\"\"\"\n", "issue": "AllenNLP integration: Can't load configurations that expect ext_vars from environment variables\nOptuna's AllenNLP integration evaluates the jsonnet config that you give it, passing in the trial parameters as the `ext_vars`. AllenNLP's default behavior when loading a configuration is a little different, in that it also passes through environment variables: \r\n\r\nhttps://github.com/allenai/allennlp/blob/4de68a42a0df89df01a999ee48f361f24c8c19d4/allennlp/common/params.py#L488\r\n\r\nSo if you rely on specifying e.g. a path to a file as an environment variable, that configuration won't load as expected when using the integration\r\n\r\n## Expected behavior\r\n\r\nOptuna should pass environment variables through when the jsonnet is interpreted. Ideally, it'd just use the AllenNLP config loading code directly so that all behavior is identical, just overriding whatever the trial needs to override.\r\n\r\n## Environment\r\n\r\n- Optuna version: 1.5.0\r\n- Python version: 3.6\r\n- OS: Ubuntu 18.04.4\r\n- (Optional) Other libraries and their versions: AllenNLP 1.0\r\n\r\n## Error messages, stack traces, or logs\r\n\r\nYou'll get `RuntimeError: undefined external variable` during the interpretation of the jsonnet here:\r\n\r\nhttps://github.com/optuna/optuna/blob/9f5c5cf5c36df1628567f0b4a9873892d148ad40/optuna/integration/allennlp.py#L108\r\n\r\n## Steps to reproduce\r\n\r\n1. Use a AllenNLP configuration that expects to be able to use an environment variable to fill an extVar\n", "before_files": [{"content": "import json\nimport os\nfrom typing import Any\nfrom typing import Dict\nfrom typing import List\nfrom typing import Optional\nfrom typing import Union\nimport warnings\n\nimport optuna\nfrom optuna._experimental import experimental\nfrom optuna._imports import try_import\n\nwith try_import() as _imports:\n    import allennlp.commands\n    import allennlp.common.util\n\nif _imports.is_successful():\n    import _jsonnet\n\n\ndef dump_best_config(input_config_file: str, output_config_file: str, study: optuna.Study) -> None:\n    \"\"\"Save JSON config file after updating with parameters from the best trial in the study.\n\n    Args:\n        input_config_file:\n            Input Jsonnet config file used with\n            :class:`~optuna.integration.AllenNLPExecutor`.\n        output_config_file:\n            Output JSON config file.\n        study:\n            Instance of :class:`~optuna.study.Study`.\n            Note that :func:`~optuna.study.Study.optimize` must have been called.\n\n    \"\"\"\n    _imports.check()\n\n    best_params = study.best_params\n    for key, value in best_params.items():\n        best_params[key] = str(value)\n    best_config = json.loads(_jsonnet.evaluate_file(input_config_file, ext_vars=best_params))\n    best_config = allennlp.common.params.infer_and_cast(best_config)\n\n    with open(output_config_file, \"w\") as f:\n        json.dump(best_config, f, indent=4)\n\n\n@experimental(\"1.4.0\")\nclass AllenNLPExecutor(object):\n    \"\"\"AllenNLP extension to use optuna with Jsonnet config file.\n\n    This feature is experimental since AllenNLP major release will come soon.\n    The interface may change without prior notice to correspond to the update.\n\n    See the examples of `objective function <https://github.com/optuna/optuna/blob/\n    master/examples/allennlp/allennlp_jsonnet.py>`_ and\n    `config file <https://github.com/optuna/optuna/blob/master/\n    examples/allennlp/classifier.jsonnet>`_.\n\n    Args:\n        trial:\n            A :class:`~optuna.trial.Trial` corresponding to the current evaluation\n            of the objective function.\n        config_file:\n            Config file for AllenNLP.\n            Hyperparameters should be masked with ``std.extVar``.\n            Please refer to `the config example <https://github.com/allenai/allentune/blob/\n            master/examples/classifier.jsonnet>`_.\n        serialization_dir:\n            A path which model weights and logs are saved.\n        metrics:\n            An evaluation metric for the result of ``objective``.\n        include_package:\n            Additional packages to include.\n            For more information, please see\n            `AllenNLP documentation <https://docs.allennlp.org/master/api/commands/train/>`_.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        trial: optuna.Trial,\n        config_file: str,\n        serialization_dir: str,\n        metrics: str = \"best_validation_accuracy\",\n        *,\n        include_package: Optional[Union[str, List[str]]] = None\n    ):\n        _imports.check()\n\n        self._params = trial.params\n        self._config_file = config_file\n        self._serialization_dir = serialization_dir\n        self._metrics = metrics\n        if include_package is None:\n            include_package = []\n        if isinstance(include_package, str):\n            self._include_package = [include_package]\n        else:\n            self._include_package = include_package\n\n    def _build_params(self) -> Dict[str, Any]:\n        \"\"\"Create a dict of params for AllenNLP.\"\"\"\n        # _build_params is based on allentune's train_func.\n        # https://github.com/allenai/allentune/blob/master/allentune/modules/allennlp_runner.py#L34-L65\n        for key, value in self._params.items():\n            self._params[key] = str(value)\n        _params = json.loads(_jsonnet.evaluate_file(self._config_file, ext_vars=self._params))\n\n        # _params contains a list of string or string as value values.\n        # Some params couldn't be casted correctly and\n        # infer_and_cast converts them into desired values.\n        return allennlp.common.params.infer_and_cast(_params)\n\n    def run(self) -> float:\n        \"\"\"Train a model using AllenNLP.\"\"\"\n        try:\n            import_func = allennlp.common.util.import_submodules\n        except AttributeError:\n            import_func = allennlp.common.util.import_module_and_submodules\n            warnings.warn(\"AllenNLP>0.9 has not been supported officially yet.\")\n\n        for package_name in self._include_package:\n            import_func(package_name)\n\n        params = allennlp.common.params.Params(self._build_params())\n        allennlp.commands.train.train_model(params, self._serialization_dir)\n\n        metrics = json.load(open(os.path.join(self._serialization_dir, \"metrics.json\")))\n        return metrics[self._metrics]\n", "path": "optuna/integration/allennlp.py"}], "after_files": [{"content": "import json\nimport os\nfrom typing import Any\nfrom typing import Dict\nfrom typing import List\nfrom typing import Optional\nfrom typing import Union\nimport warnings\n\nimport optuna\nfrom optuna._experimental import experimental\nfrom optuna._imports import try_import\n\nwith try_import() as _imports:\n    import allennlp.commands\n    import allennlp.common.util\n\nif _imports.is_successful():\n    import _jsonnet\n\n\ndef dump_best_config(input_config_file: str, output_config_file: str, study: optuna.Study) -> None:\n    \"\"\"Save JSON config file after updating with parameters from the best trial in the study.\n\n    Args:\n        input_config_file:\n            Input Jsonnet config file used with\n            :class:`~optuna.integration.AllenNLPExecutor`.\n        output_config_file:\n            Output JSON config file.\n        study:\n            Instance of :class:`~optuna.study.Study`.\n            Note that :func:`~optuna.study.Study.optimize` must have been called.\n\n    \"\"\"\n    _imports.check()\n\n    best_params = study.best_params\n    for key, value in best_params.items():\n        best_params[key] = str(value)\n    best_config = json.loads(_jsonnet.evaluate_file(input_config_file, ext_vars=best_params))\n    best_config = allennlp.common.params.infer_and_cast(best_config)\n\n    with open(output_config_file, \"w\") as f:\n        json.dump(best_config, f, indent=4)\n\n\n@experimental(\"1.4.0\")\nclass AllenNLPExecutor(object):\n    \"\"\"AllenNLP extension to use optuna with Jsonnet config file.\n\n    This feature is experimental since AllenNLP major release will come soon.\n    The interface may change without prior notice to correspond to the update.\n\n    See the examples of `objective function <https://github.com/optuna/optuna/blob/\n    master/examples/allennlp/allennlp_jsonnet.py>`_ and\n    `config file <https://github.com/optuna/optuna/blob/master/\n    examples/allennlp/classifier.jsonnet>`_.\n\n    .. note::\n        In :class:`~optuna.integration.AllenNLPExecutor`,\n        you can pass parameters to AllenNLP by either defining a search space using\n        Optuna suggest methods or setting environment variables just like AllenNLP CLI.\n        If a value is set in both a search space in Optuna and the environment variables,\n        the executor will use the value specified in the search space in Optuna.\n\n    Args:\n        trial:\n            A :class:`~optuna.trial.Trial` corresponding to the current evaluation\n            of the objective function.\n        config_file:\n            Config file for AllenNLP.\n            Hyperparameters should be masked with ``std.extVar``.\n            Please refer to `the config example <https://github.com/allenai/allentune/blob/\n            master/examples/classifier.jsonnet>`_.\n        serialization_dir:\n            A path which model weights and logs are saved.\n        metrics:\n            An evaluation metric for the result of ``objective``.\n        include_package:\n            Additional packages to include.\n            For more information, please see\n            `AllenNLP documentation <https://docs.allennlp.org/master/api/commands/train/>`_.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        trial: optuna.Trial,\n        config_file: str,\n        serialization_dir: str,\n        metrics: str = \"best_validation_accuracy\",\n        *,\n        include_package: Optional[Union[str, List[str]]] = None\n    ):\n        _imports.check()\n\n        self._params = trial.params\n        self._config_file = config_file\n        self._serialization_dir = serialization_dir\n        self._metrics = metrics\n        if include_package is None:\n            include_package = []\n        if isinstance(include_package, str):\n            self._include_package = [include_package]\n        else:\n            self._include_package = include_package\n\n    def _build_params(self) -> Dict[str, Any]:\n        \"\"\"Create a dict of params for AllenNLP.\n\n        _build_params is based on allentune's train_func.\n        For more detail, please refer to\n        https://github.com/allenai/allentune/blob/master/allentune/modules/allennlp_runner.py#L34-L65\n\n        \"\"\"\n        params = self._environment_variables()\n        params.update({key: str(value) for key, value in self._params.items()})\n\n        allennlp_params = json.loads(_jsonnet.evaluate_file(self._config_file, ext_vars=params))\n        # allennlp_params contains a list of string or string as value values.\n        # Some params couldn't be casted correctly and\n        # infer_and_cast converts them into desired values.\n        return allennlp.common.params.infer_and_cast(allennlp_params)\n\n    @staticmethod\n    def _is_encodable(value: str) -> bool:\n        # https://github.com/allenai/allennlp/blob/master/allennlp/common/params.py#L77-L85\n        return (value == \"\") or (value.encode(\"utf-8\", \"ignore\") != b\"\")\n\n    def _environment_variables(self) -> Dict[str, str]:\n        return {key: value for key, value in os.environ.items() if self._is_encodable(value)}\n\n    def run(self) -> float:\n        \"\"\"Train a model using AllenNLP.\"\"\"\n        try:\n            import_func = allennlp.common.util.import_submodules\n        except AttributeError:\n            import_func = allennlp.common.util.import_module_and_submodules\n            warnings.warn(\"AllenNLP>0.9 has not been supported officially yet.\")\n\n        for package_name in self._include_package:\n            import_func(package_name)\n\n        params = allennlp.common.params.Params(self._build_params())\n        allennlp.commands.train.train_model(params, self._serialization_dir)\n\n        metrics = json.load(open(os.path.join(self._serialization_dir, \"metrics.json\")))\n        return metrics[self._metrics]\n", "path": "optuna/integration/allennlp.py"}]}