We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
How use multiple gpu?
### Feature Description
I want to use a single machine with multiple gpu for training, but it seems to have no actual effect### Code Example

```python
with strategy.scope():
```

### Reason
Speed up the calculation of toxins

### Solution
<!---
Please tell us how to implement the feature,
if you have one in mind.
-->


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `autokeras/graph.py`
Content:
```
1 # Copyright 2020 The AutoKeras Authors.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 import kerastuner
16 import tensorflow as tf
17 from tensorflow.python.util import nest
18 
19 from autokeras import blocks as blocks_module
20 from autokeras import nodes as nodes_module
21 from autokeras.engine import head as head_module
22 from autokeras.engine import serializable
23 from autokeras.utils import utils
24 
25 
26 def feature_encoding_input(block):
27     """Fetch the column_types and column_names.
28 
29     The values are fetched for FeatureEncoding from StructuredDataInput.
30     """
31     if not isinstance(block.inputs[0], nodes_module.StructuredDataInput):
32         raise TypeError(
33             "CategoricalToNumerical can only be used with StructuredDataInput."
34         )
35     block.column_types = block.inputs[0].column_types
36     block.column_names = block.inputs[0].column_names
37 
38 
39 # Compile the graph.
40 COMPILE_FUNCTIONS = {
41     blocks_module.StructuredDataBlock: [feature_encoding_input],
42     blocks_module.CategoricalToNumerical: [feature_encoding_input],
43 }
44 
45 
46 def load_graph(filepath, custom_objects=None):
47     if custom_objects is None:
48         custom_objects = {}
49     with tf.keras.utils.custom_object_scope(custom_objects):
50         return Graph.from_config(utils.load_json(filepath))
51 
52 
53 class Graph(kerastuner.HyperModel, serializable.Serializable):
54     """A graph consists of connected Blocks, or Heads.
55 
56     # Arguments
57         inputs: A list of input node(s) for the Graph.
58         outputs: A list of output node(s) for the Graph.
59         override_hps: A list of HyperParameters. The predefined HyperParameters that
60             will override the space of the Hyperparameters defined in the Hypermodels
61             with the same names.
62     """
63 
64     def __init__(self, inputs=None, outputs=None, override_hps=None):
65         super().__init__()
66         self.inputs = nest.flatten(inputs)
67         self.outputs = nest.flatten(outputs)
68         self._node_to_id = {}
69         self._nodes = []
70         self.blocks = []
71         self._block_to_id = {}
72         if inputs and outputs:
73             self._build_network()
74         self.override_hps = override_hps or []
75 
76     def compile(self):
77         """Share the information between blocks."""
78         for block in self.blocks:
79             for func in COMPILE_FUNCTIONS.get(block.__class__, []):
80                 func(block)
81 
82     def _register_hps(self, hp):
83         """Register the override HyperParameters for current HyperParameters."""
84         for single_hp in self.override_hps:
85             name = single_hp.name
86             if name not in hp.values:
87                 hp._register(single_hp)
88                 hp.values[name] = single_hp.default
89 
90     def _build_network(self):
91         self._node_to_id = {}
92 
93         # Recursively find all the interested nodes.
94         for input_node in self.inputs:
95             self._search_network(input_node, self.outputs, set(), set())
96         self._nodes = sorted(
97             list(self._node_to_id.keys()), key=lambda x: self._node_to_id[x]
98         )
99 
100         for node in self.inputs + self.outputs:
101             if node not in self._node_to_id:
102                 raise ValueError("Inputs and outputs not connected.")
103 
104         # Find the blocks.
105         blocks = []
106         for input_node in self._nodes:
107             for block in input_node.out_blocks:
108                 if (
109                     any(
110                         [
111                             output_node in self._node_to_id
112                             for output_node in block.outputs
113                         ]
114                     )
115                     and block not in blocks
116                 ):
117                     blocks.append(block)
118 
119         # Check if all the inputs of the blocks are set as inputs.
120         for block in blocks:
121             for input_node in block.inputs:
122                 if input_node not in self._node_to_id:
123                     raise ValueError(
124                         "A required input is missing for HyperModel "
125                         "{name}.".format(name=block.name)
126                     )
127 
128         # Calculate the in degree of all the nodes
129         in_degree = [0] * len(self._nodes)
130         for node_id, node in enumerate(self._nodes):
131             in_degree[node_id] = len(
132                 [block for block in node.in_blocks if block in blocks]
133             )
134 
135         # Add the blocks in topological order.
136         self.blocks = []
137         self._block_to_id = {}
138         while len(blocks) != 0:
139             new_added = []
140 
141             # Collect blocks with in degree 0.
142             for block in blocks:
143                 if any([in_degree[self._node_to_id[node]] for node in block.inputs]):
144                     continue
145                 new_added.append(block)
146 
147             # Remove the collected blocks from blocks.
148             for block in new_added:
149                 blocks.remove(block)
150 
151             for block in new_added:
152                 # Add the collected blocks to the Graph.
153                 self._add_block(block)
154 
155                 # Decrease the in degree of the output nodes.
156                 for output_node in block.outputs:
157                     output_node_id = self._node_to_id[output_node]
158                     in_degree[output_node_id] -= 1
159 
160     def _search_network(self, input_node, outputs, in_stack_nodes, visited_nodes):
161         visited_nodes.add(input_node)
162         in_stack_nodes.add(input_node)
163 
164         outputs_reached = False
165         if input_node in outputs:
166             outputs_reached = True
167 
168         for block in input_node.out_blocks:
169             for output_node in block.outputs:
170                 if output_node in in_stack_nodes:
171                     raise ValueError("The network has a cycle.")
172                 if output_node not in visited_nodes:
173                     self._search_network(
174                         output_node, outputs, in_stack_nodes, visited_nodes
175                     )
176                 if output_node in self._node_to_id.keys():
177                     outputs_reached = True
178 
179         if outputs_reached:
180             self._add_node(input_node)
181 
182         in_stack_nodes.remove(input_node)
183 
184     def _add_block(self, block):
185         if block not in self.blocks:
186             block_id = len(self.blocks)
187             self._block_to_id[block] = block_id
188             self.blocks.append(block)
189 
190     def _add_node(self, input_node):
191         if input_node not in self._node_to_id:
192             self._node_to_id[input_node] = len(self._node_to_id)
193 
194     def get_config(self):
195         blocks = [blocks_module.serialize(block) for block in self.blocks]
196         nodes = {
197             str(self._node_to_id[node]): nodes_module.serialize(node)
198             for node in self.inputs
199         }
200         override_hps = [
201             kerastuner.engine.hyperparameters.serialize(hp)
202             for hp in self.override_hps
203         ]
204         block_inputs = {
205             str(block_id): [self._node_to_id[node] for node in block.inputs]
206             for block_id, block in enumerate(self.blocks)
207         }
208         block_outputs = {
209             str(block_id): [self._node_to_id[node] for node in block.outputs]
210             for block_id, block in enumerate(self.blocks)
211         }
212 
213         outputs = [self._node_to_id[node] for node in self.outputs]
214 
215         return {
216             "override_hps": override_hps,  # List [serialized].
217             "blocks": blocks,  # Dict {id: serialized}.
218             "nodes": nodes,  # Dict {id: serialized}.
219             "outputs": outputs,  # List of node_ids.
220             "block_inputs": block_inputs,  # Dict {id: List of node_ids}.
221             "block_outputs": block_outputs,  # Dict {id: List of node_ids}.
222         }
223 
224     @classmethod
225     def from_config(cls, config):
226         blocks = [blocks_module.deserialize(block) for block in config["blocks"]]
227         nodes = {
228             int(node_id): nodes_module.deserialize(node)
229             for node_id, node in config["nodes"].items()
230         }
231         override_hps = [
232             kerastuner.engine.hyperparameters.deserialize(config)
233             for config in config["override_hps"]
234         ]
235 
236         inputs = [nodes[node_id] for node_id in nodes]
237         for block_id, block in enumerate(blocks):
238             input_nodes = [
239                 nodes[node_id] for node_id in config["block_inputs"][str(block_id)]
240             ]
241             output_nodes = nest.flatten(block(input_nodes))
242             for output_node, node_id in zip(
243                 output_nodes, config["block_outputs"][str(block_id)]
244             ):
245                 nodes[node_id] = output_node
246 
247         outputs = [nodes[node_id] for node_id in config["outputs"]]
248         return cls(inputs=inputs, outputs=outputs, override_hps=override_hps)
249 
250     def build(self, hp):
251         """Build the HyperModel into a Keras Model."""
252         tf.keras.backend.clear_session()
253         self._register_hps(hp)
254         self.compile()
255         real_nodes = {}
256         for input_node in self.inputs:
257             node_id = self._node_to_id[input_node]
258             real_nodes[node_id] = input_node.build()
259         for block in self.blocks:
260             temp_inputs = [
261                 real_nodes[self._node_to_id[input_node]]
262                 for input_node in block.inputs
263             ]
264             outputs = block.build(hp, inputs=temp_inputs)
265             outputs = nest.flatten(outputs)
266             for output_node, real_output_node in zip(block.outputs, outputs):
267                 real_nodes[self._node_to_id[output_node]] = real_output_node
268         model = tf.keras.Model(
269             [real_nodes[self._node_to_id[input_node]] for input_node in self.inputs],
270             [
271                 real_nodes[self._node_to_id[output_node]]
272                 for output_node in self.outputs
273             ],
274         )
275 
276         return self._compile_keras_model(hp, model)
277 
278     def _get_metrics(self):
279         metrics = {}
280         for output_node in self.outputs:
281             block = output_node.in_blocks[0]
282             if isinstance(block, head_module.Head):
283                 metrics[block.name] = block.metrics
284         return metrics
285 
286     def _get_loss(self):
287         loss = {}
288         for output_node in self.outputs:
289             block = output_node.in_blocks[0]
290             if isinstance(block, head_module.Head):
291                 loss[block.name] = block.loss
292         return loss
293 
294     def _compile_keras_model(self, hp, model):
295         # Specify hyperparameters from compile(...)
296         optimizer_name = hp.Choice(
297             "optimizer", ["adam", "adadelta", "sgd"], default="adam"
298         )
299         learning_rate = hp.Choice(
300             "learning_rate", [1e-1, 1e-2, 1e-3, 1e-4, 1e-5], default=1e-3
301         )
302 
303         if optimizer_name == "adam":
304             optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
305         elif optimizer_name == "adadelta":
306             optimizer = tf.keras.optimizers.Adadelta(learning_rate=learning_rate)
307         elif optimizer_name == "sgd":
308             optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate)
309 
310         model.compile(
311             optimizer=optimizer, metrics=self._get_metrics(), loss=self._get_loss()
312         )
313 
314         return model
315 
316     def save(self, filepath):
317         utils.save_json(filepath, self.get_config())
318
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/autokeras/graph.py b/autokeras/graph.py
--- a/autokeras/graph.py
+++ b/autokeras/graph.py
@@ -249,7 +249,6 @@
 
     def build(self, hp):
         """Build the HyperModel into a Keras Model."""
-        tf.keras.backend.clear_session()
         self._register_hps(hp)
         self.compile()
         real_nodes = {}

{"golden_diff": "diff --git a/autokeras/graph.py b/autokeras/graph.py\n--- a/autokeras/graph.py\n+++ b/autokeras/graph.py\n@@ -249,7 +249,6 @@\n \n     def build(self, hp):\n         \"\"\"Build the HyperModel into a Keras Model.\"\"\"\n-        tf.keras.backend.clear_session()\n         self._register_hps(hp)\n         self.compile()\n         real_nodes = {}\n", "issue": "How use multiple gpu?\n### Feature Description\r\nI want to use a single machine with multiple gpu for training, but it seems to have no actual effect### Code Example\r\n\r\n```python\r\nwith strategy.scope():\r\n```\r\n\r\n### Reason\r\nSpeed up the calculation of toxins\r\n\r\n### Solution\r\n<!---\r\nPlease tell us how to implement the feature,\r\nif you have one in mind.\r\n-->\r\n\n", "before_files": [{"content": "# Copyright 2020 The AutoKeras Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport kerastuner\nimport tensorflow as tf\nfrom tensorflow.python.util import nest\n\nfrom autokeras import blocks as blocks_module\nfrom autokeras import nodes as nodes_module\nfrom autokeras.engine import head as head_module\nfrom autokeras.engine import serializable\nfrom autokeras.utils import utils\n\n\ndef feature_encoding_input(block):\n    \"\"\"Fetch the column_types and column_names.\n\n    The values are fetched for FeatureEncoding from StructuredDataInput.\n    \"\"\"\n    if not isinstance(block.inputs[0], nodes_module.StructuredDataInput):\n        raise TypeError(\n            \"CategoricalToNumerical can only be used with StructuredDataInput.\"\n        )\n    block.column_types = block.inputs[0].column_types\n    block.column_names = block.inputs[0].column_names\n\n\n# Compile the graph.\nCOMPILE_FUNCTIONS = {\n    blocks_module.StructuredDataBlock: [feature_encoding_input],\n    blocks_module.CategoricalToNumerical: [feature_encoding_input],\n}\n\n\ndef load_graph(filepath, custom_objects=None):\n    if custom_objects is None:\n        custom_objects = {}\n    with tf.keras.utils.custom_object_scope(custom_objects):\n        return Graph.from_config(utils.load_json(filepath))\n\n\nclass Graph(kerastuner.HyperModel, serializable.Serializable):\n    \"\"\"A graph consists of connected Blocks, or Heads.\n\n    # Arguments\n        inputs: A list of input node(s) for the Graph.\n        outputs: A list of output node(s) for the Graph.\n        override_hps: A list of HyperParameters. The predefined HyperParameters that\n            will override the space of the Hyperparameters defined in the Hypermodels\n            with the same names.\n    \"\"\"\n\n    def __init__(self, inputs=None, outputs=None, override_hps=None):\n        super().__init__()\n        self.inputs = nest.flatten(inputs)\n        self.outputs = nest.flatten(outputs)\n        self._node_to_id = {}\n        self._nodes = []\n        self.blocks = []\n        self._block_to_id = {}\n        if inputs and outputs:\n            self._build_network()\n        self.override_hps = override_hps or []\n\n    def compile(self):\n        \"\"\"Share the information between blocks.\"\"\"\n        for block in self.blocks:\n            for func in COMPILE_FUNCTIONS.get(block.__class__, []):\n                func(block)\n\n    def _register_hps(self, hp):\n        \"\"\"Register the override HyperParameters for current HyperParameters.\"\"\"\n        for single_hp in self.override_hps:\n            name = single_hp.name\n            if name not in hp.values:\n                hp._register(single_hp)\n                hp.values[name] = single_hp.default\n\n    def _build_network(self):\n        self._node_to_id = {}\n\n        # Recursively find all the interested nodes.\n        for input_node in self.inputs:\n            self._search_network(input_node, self.outputs, set(), set())\n        self._nodes = sorted(\n            list(self._node_to_id.keys()), key=lambda x: self._node_to_id[x]\n        )\n\n        for node in self.inputs + self.outputs:\n            if node not in self._node_to_id:\n                raise ValueError(\"Inputs and outputs not connected.\")\n\n        # Find the blocks.\n        blocks = []\n        for input_node in self._nodes:\n            for block in input_node.out_blocks:\n                if (\n                    any(\n                        [\n                            output_node in self._node_to_id\n                            for output_node in block.outputs\n                        ]\n                    )\n                    and block not in blocks\n                ):\n                    blocks.append(block)\n\n        # Check if all the inputs of the blocks are set as inputs.\n        for block in blocks:\n            for input_node in block.inputs:\n                if input_node not in self._node_to_id:\n                    raise ValueError(\n                        \"A required input is missing for HyperModel \"\n                        \"{name}.\".format(name=block.name)\n                    )\n\n        # Calculate the in degree of all the nodes\n        in_degree = [0] * len(self._nodes)\n        for node_id, node in enumerate(self._nodes):\n            in_degree[node_id] = len(\n                [block for block in node.in_blocks if block in blocks]\n            )\n\n        # Add the blocks in topological order.\n        self.blocks = []\n        self._block_to_id = {}\n        while len(blocks) != 0:\n            new_added = []\n\n            # Collect blocks with in degree 0.\n            for block in blocks:\n                if any([in_degree[self._node_to_id[node]] for node in block.inputs]):\n                    continue\n                new_added.append(block)\n\n            # Remove the collected blocks from blocks.\n            for block in new_added:\n                blocks.remove(block)\n\n            for block in new_added:\n                # Add the collected blocks to the Graph.\n                self._add_block(block)\n\n                # Decrease the in degree of the output nodes.\n                for output_node in block.outputs:\n                    output_node_id = self._node_to_id[output_node]\n                    in_degree[output_node_id] -= 1\n\n    def _search_network(self, input_node, outputs, in_stack_nodes, visited_nodes):\n        visited_nodes.add(input_node)\n        in_stack_nodes.add(input_node)\n\n        outputs_reached = False\n        if input_node in outputs:\n            outputs_reached = True\n\n        for block in input_node.out_blocks:\n            for output_node in block.outputs:\n                if output_node in in_stack_nodes:\n                    raise ValueError(\"The network has a cycle.\")\n                if output_node not in visited_nodes:\n                    self._search_network(\n                        output_node, outputs, in_stack_nodes, visited_nodes\n                    )\n                if output_node in self._node_to_id.keys():\n                    outputs_reached = True\n\n        if outputs_reached:\n            self._add_node(input_node)\n\n        in_stack_nodes.remove(input_node)\n\n    def _add_block(self, block):\n        if block not in self.blocks:\n            block_id = len(self.blocks)\n            self._block_to_id[block] = block_id\n            self.blocks.append(block)\n\n    def _add_node(self, input_node):\n        if input_node not in self._node_to_id:\n            self._node_to_id[input_node] = len(self._node_to_id)\n\n    def get_config(self):\n        blocks = [blocks_module.serialize(block) for block in self.blocks]\n        nodes = {\n            str(self._node_to_id[node]): nodes_module.serialize(node)\n            for node in self.inputs\n        }\n        override_hps = [\n            kerastuner.engine.hyperparameters.serialize(hp)\n            for hp in self.override_hps\n        ]\n        block_inputs = {\n            str(block_id): [self._node_to_id[node] for node in block.inputs]\n            for block_id, block in enumerate(self.blocks)\n        }\n        block_outputs = {\n            str(block_id): [self._node_to_id[node] for node in block.outputs]\n            for block_id, block in enumerate(self.blocks)\n        }\n\n        outputs = [self._node_to_id[node] for node in self.outputs]\n\n        return {\n            \"override_hps\": override_hps,  # List [serialized].\n            \"blocks\": blocks,  # Dict {id: serialized}.\n            \"nodes\": nodes,  # Dict {id: serialized}.\n            \"outputs\": outputs,  # List of node_ids.\n            \"block_inputs\": block_inputs,  # Dict {id: List of node_ids}.\n            \"block_outputs\": block_outputs,  # Dict {id: List of node_ids}.\n        }\n\n    @classmethod\n    def from_config(cls, config):\n        blocks = [blocks_module.deserialize(block) for block in config[\"blocks\"]]\n        nodes = {\n            int(node_id): nodes_module.deserialize(node)\n            for node_id, node in config[\"nodes\"].items()\n        }\n        override_hps = [\n            kerastuner.engine.hyperparameters.deserialize(config)\n            for config in config[\"override_hps\"]\n        ]\n\n        inputs = [nodes[node_id] for node_id in nodes]\n        for block_id, block in enumerate(blocks):\n            input_nodes = [\n                nodes[node_id] for node_id in config[\"block_inputs\"][str(block_id)]\n            ]\n            output_nodes = nest.flatten(block(input_nodes))\n            for output_node, node_id in zip(\n                output_nodes, config[\"block_outputs\"][str(block_id)]\n            ):\n                nodes[node_id] = output_node\n\n        outputs = [nodes[node_id] for node_id in config[\"outputs\"]]\n        return cls(inputs=inputs, outputs=outputs, override_hps=override_hps)\n\n    def build(self, hp):\n        \"\"\"Build the HyperModel into a Keras Model.\"\"\"\n        tf.keras.backend.clear_session()\n        self._register_hps(hp)\n        self.compile()\n        real_nodes = {}\n        for input_node in self.inputs:\n            node_id = self._node_to_id[input_node]\n            real_nodes[node_id] = input_node.build()\n        for block in self.blocks:\n            temp_inputs = [\n                real_nodes[self._node_to_id[input_node]]\n                for input_node in block.inputs\n            ]\n            outputs = block.build(hp, inputs=temp_inputs)\n            outputs = nest.flatten(outputs)\n            for output_node, real_output_node in zip(block.outputs, outputs):\n                real_nodes[self._node_to_id[output_node]] = real_output_node\n        model = tf.keras.Model(\n            [real_nodes[self._node_to_id[input_node]] for input_node in self.inputs],\n            [\n                real_nodes[self._node_to_id[output_node]]\n                for output_node in self.outputs\n            ],\n        )\n\n        return self._compile_keras_model(hp, model)\n\n    def _get_metrics(self):\n        metrics = {}\n        for output_node in self.outputs:\n            block = output_node.in_blocks[0]\n            if isinstance(block, head_module.Head):\n                metrics[block.name] = block.metrics\n        return metrics\n\n    def _get_loss(self):\n        loss = {}\n        for output_node in self.outputs:\n            block = output_node.in_blocks[0]\n            if isinstance(block, head_module.Head):\n                loss[block.name] = block.loss\n        return loss\n\n    def _compile_keras_model(self, hp, model):\n        # Specify hyperparameters from compile(...)\n        optimizer_name = hp.Choice(\n            \"optimizer\", [\"adam\", \"adadelta\", \"sgd\"], default=\"adam\"\n        )\n        learning_rate = hp.Choice(\n            \"learning_rate\", [1e-1, 1e-2, 1e-3, 1e-4, 1e-5], default=1e-3\n        )\n\n        if optimizer_name == \"adam\":\n            optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n        elif optimizer_name == \"adadelta\":\n            optimizer = tf.keras.optimizers.Adadelta(learning_rate=learning_rate)\n        elif optimizer_name == \"sgd\":\n            optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate)\n\n        model.compile(\n            optimizer=optimizer, metrics=self._get_metrics(), loss=self._get_loss()\n        )\n\n        return model\n\n    def save(self, filepath):\n        utils.save_json(filepath, self.get_config())\n", "path": "autokeras/graph.py"}], "after_files": [{"content": "# Copyright 2020 The AutoKeras Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport kerastuner\nimport tensorflow as tf\nfrom tensorflow.python.util import nest\n\nfrom autokeras import blocks as blocks_module\nfrom autokeras import nodes as nodes_module\nfrom autokeras.engine import head as head_module\nfrom autokeras.engine import serializable\nfrom autokeras.utils import utils\n\n\ndef feature_encoding_input(block):\n    \"\"\"Fetch the column_types and column_names.\n\n    The values are fetched for FeatureEncoding from StructuredDataInput.\n    \"\"\"\n    if not isinstance(block.inputs[0], nodes_module.StructuredDataInput):\n        raise TypeError(\n            \"CategoricalToNumerical can only be used with StructuredDataInput.\"\n        )\n    block.column_types = block.inputs[0].column_types\n    block.column_names = block.inputs[0].column_names\n\n\n# Compile the graph.\nCOMPILE_FUNCTIONS = {\n    blocks_module.StructuredDataBlock: [feature_encoding_input],\n    blocks_module.CategoricalToNumerical: [feature_encoding_input],\n}\n\n\ndef load_graph(filepath, custom_objects=None):\n    if custom_objects is None:\n        custom_objects = {}\n    with tf.keras.utils.custom_object_scope(custom_objects):\n        return Graph.from_config(utils.load_json(filepath))\n\n\nclass Graph(kerastuner.HyperModel, serializable.Serializable):\n    \"\"\"A graph consists of connected Blocks, or Heads.\n\n    # Arguments\n        inputs: A list of input node(s) for the Graph.\n        outputs: A list of output node(s) for the Graph.\n        override_hps: A list of HyperParameters. The predefined HyperParameters that\n            will override the space of the Hyperparameters defined in the Hypermodels\n            with the same names.\n    \"\"\"\n\n    def __init__(self, inputs=None, outputs=None, override_hps=None):\n        super().__init__()\n        self.inputs = nest.flatten(inputs)\n        self.outputs = nest.flatten(outputs)\n        self._node_to_id = {}\n        self._nodes = []\n        self.blocks = []\n        self._block_to_id = {}\n        if inputs and outputs:\n            self._build_network()\n        self.override_hps = override_hps or []\n\n    def compile(self):\n        \"\"\"Share the information between blocks.\"\"\"\n        for block in self.blocks:\n            for func in COMPILE_FUNCTIONS.get(block.__class__, []):\n                func(block)\n\n    def _register_hps(self, hp):\n        \"\"\"Register the override HyperParameters for current HyperParameters.\"\"\"\n        for single_hp in self.override_hps:\n            name = single_hp.name\n            if name not in hp.values:\n                hp._register(single_hp)\n                hp.values[name] = single_hp.default\n\n    def _build_network(self):\n        self._node_to_id = {}\n\n        # Recursively find all the interested nodes.\n        for input_node in self.inputs:\n            self._search_network(input_node, self.outputs, set(), set())\n        self._nodes = sorted(\n            list(self._node_to_id.keys()), key=lambda x: self._node_to_id[x]\n        )\n\n        for node in self.inputs + self.outputs:\n            if node not in self._node_to_id:\n                raise ValueError(\"Inputs and outputs not connected.\")\n\n        # Find the blocks.\n        blocks = []\n        for input_node in self._nodes:\n            for block in input_node.out_blocks:\n                if (\n                    any(\n                        [\n                            output_node in self._node_to_id\n                            for output_node in block.outputs\n                        ]\n                    )\n                    and block not in blocks\n                ):\n                    blocks.append(block)\n\n        # Check if all the inputs of the blocks are set as inputs.\n        for block in blocks:\n            for input_node in block.inputs:\n                if input_node not in self._node_to_id:\n                    raise ValueError(\n                        \"A required input is missing for HyperModel \"\n                        \"{name}.\".format(name=block.name)\n                    )\n\n        # Calculate the in degree of all the nodes\n        in_degree = [0] * len(self._nodes)\n        for node_id, node in enumerate(self._nodes):\n            in_degree[node_id] = len(\n                [block for block in node.in_blocks if block in blocks]\n            )\n\n        # Add the blocks in topological order.\n        self.blocks = []\n        self._block_to_id = {}\n        while len(blocks) != 0:\n            new_added = []\n\n            # Collect blocks with in degree 0.\n            for block in blocks:\n                if any([in_degree[self._node_to_id[node]] for node in block.inputs]):\n                    continue\n                new_added.append(block)\n\n            # Remove the collected blocks from blocks.\n            for block in new_added:\n                blocks.remove(block)\n\n            for block in new_added:\n                # Add the collected blocks to the Graph.\n                self._add_block(block)\n\n                # Decrease the in degree of the output nodes.\n                for output_node in block.outputs:\n                    output_node_id = self._node_to_id[output_node]\n                    in_degree[output_node_id] -= 1\n\n    def _search_network(self, input_node, outputs, in_stack_nodes, visited_nodes):\n        visited_nodes.add(input_node)\n        in_stack_nodes.add(input_node)\n\n        outputs_reached = False\n        if input_node in outputs:\n            outputs_reached = True\n\n        for block in input_node.out_blocks:\n            for output_node in block.outputs:\n                if output_node in in_stack_nodes:\n                    raise ValueError(\"The network has a cycle.\")\n                if output_node not in visited_nodes:\n                    self._search_network(\n                        output_node, outputs, in_stack_nodes, visited_nodes\n                    )\n                if output_node in self._node_to_id.keys():\n                    outputs_reached = True\n\n        if outputs_reached:\n            self._add_node(input_node)\n\n        in_stack_nodes.remove(input_node)\n\n    def _add_block(self, block):\n        if block not in self.blocks:\n            block_id = len(self.blocks)\n            self._block_to_id[block] = block_id\n            self.blocks.append(block)\n\n    def _add_node(self, input_node):\n        if input_node not in self._node_to_id:\n            self._node_to_id[input_node] = len(self._node_to_id)\n\n    def get_config(self):\n        blocks = [blocks_module.serialize(block) for block in self.blocks]\n        nodes = {\n            str(self._node_to_id[node]): nodes_module.serialize(node)\n            for node in self.inputs\n        }\n        override_hps = [\n            kerastuner.engine.hyperparameters.serialize(hp)\n            for hp in self.override_hps\n        ]\n        block_inputs = {\n            str(block_id): [self._node_to_id[node] for node in block.inputs]\n            for block_id, block in enumerate(self.blocks)\n        }\n        block_outputs = {\n            str(block_id): [self._node_to_id[node] for node in block.outputs]\n            for block_id, block in enumerate(self.blocks)\n        }\n\n        outputs = [self._node_to_id[node] for node in self.outputs]\n\n        return {\n            \"override_hps\": override_hps,  # List [serialized].\n            \"blocks\": blocks,  # Dict {id: serialized}.\n            \"nodes\": nodes,  # Dict {id: serialized}.\n            \"outputs\": outputs,  # List of node_ids.\n            \"block_inputs\": block_inputs,  # Dict {id: List of node_ids}.\n            \"block_outputs\": block_outputs,  # Dict {id: List of node_ids}.\n        }\n\n    @classmethod\n    def from_config(cls, config):\n        blocks = [blocks_module.deserialize(block) for block in config[\"blocks\"]]\n        nodes = {\n            int(node_id): nodes_module.deserialize(node)\n            for node_id, node in config[\"nodes\"].items()\n        }\n        override_hps = [\n            kerastuner.engine.hyperparameters.deserialize(config)\n            for config in config[\"override_hps\"]\n        ]\n\n        inputs = [nodes[node_id] for node_id in nodes]\n        for block_id, block in enumerate(blocks):\n            input_nodes = [\n                nodes[node_id] for node_id in config[\"block_inputs\"][str(block_id)]\n            ]\n            output_nodes = nest.flatten(block(input_nodes))\n            for output_node, node_id in zip(\n                output_nodes, config[\"block_outputs\"][str(block_id)]\n            ):\n                nodes[node_id] = output_node\n\n        outputs = [nodes[node_id] for node_id in config[\"outputs\"]]\n        return cls(inputs=inputs, outputs=outputs, override_hps=override_hps)\n\n    def build(self, hp):\n        \"\"\"Build the HyperModel into a Keras Model.\"\"\"\n        self._register_hps(hp)\n        self.compile()\n        real_nodes = {}\n        for input_node in self.inputs:\n            node_id = self._node_to_id[input_node]\n            real_nodes[node_id] = input_node.build()\n        for block in self.blocks:\n            temp_inputs = [\n                real_nodes[self._node_to_id[input_node]]\n                for input_node in block.inputs\n            ]\n            outputs = block.build(hp, inputs=temp_inputs)\n            outputs = nest.flatten(outputs)\n            for output_node, real_output_node in zip(block.outputs, outputs):\n                real_nodes[self._node_to_id[output_node]] = real_output_node\n        model = tf.keras.Model(\n            [real_nodes[self._node_to_id[input_node]] for input_node in self.inputs],\n            [\n                real_nodes[self._node_to_id[output_node]]\n                for output_node in self.outputs\n            ],\n        )\n\n        return self._compile_keras_model(hp, model)\n\n    def _get_metrics(self):\n        metrics = {}\n        for output_node in self.outputs:\n            block = output_node.in_blocks[0]\n            if isinstance(block, head_module.Head):\n                metrics[block.name] = block.metrics\n        return metrics\n\n    def _get_loss(self):\n        loss = {}\n        for output_node in self.outputs:\n            block = output_node.in_blocks[0]\n            if isinstance(block, head_module.Head):\n                loss[block.name] = block.loss\n        return loss\n\n    def _compile_keras_model(self, hp, model):\n        # Specify hyperparameters from compile(...)\n        optimizer_name = hp.Choice(\n            \"optimizer\", [\"adam\", \"adadelta\", \"sgd\"], default=\"adam\"\n        )\n        learning_rate = hp.Choice(\n            \"learning_rate\", [1e-1, 1e-2, 1e-3, 1e-4, 1e-5], default=1e-3\n        )\n\n        if optimizer_name == \"adam\":\n            optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n        elif optimizer_name == \"adadelta\":\n            optimizer = tf.keras.optimizers.Adadelta(learning_rate=learning_rate)\n        elif optimizer_name == \"sgd\":\n            optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate)\n\n        model.compile(\n            optimizer=optimizer, metrics=self._get_metrics(), loss=self._get_loss()\n        )\n\n        return model\n\n    def save(self, filepath):\n        utils.save_json(filepath, self.get_config())\n", "path": "autokeras/graph.py"}]}