prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Evaluation criteria for MLP
  Are there any evaluation criterias for MLP module in Autokeras? 
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `autokeras/net_module.py`
  Content:
  ```
  1 from functools import reduce
  2 
  3 import torch
  4 import numpy as np
  5 
  6 import os
  7 import time
  8 
  9 from autokeras.constant import Constant
  10 from autokeras.search import BayesianSearcher, train
  11 
  12 from autokeras.utils import pickle_to_file, rand_temp_folder_generator, ensure_dir
  13 from autokeras.nn.generator import CnnGenerator, MlpGenerator, ResNetGenerator, DenseNetGenerator
  14 
  15 
  16 class NetworkModule:
  17     """ Class to create a network module.
  18 
  19     Attributes:
  20         loss: A function taking two parameters, the predictions and the ground truth.
  21         metric: An instance of the Metric subclasses.
  22         searcher_args: A dictionary containing the parameters for the searcher's __init__ function.
  23         searcher: An instance of the Searcher class.
  24         path: A string. The path to the directory to save the searcher.
  25         verbose: A boolean. Setting it to true prints to stdout.
  26         generators: A list of instances of the NetworkGenerator class or its subclasses.
  27         search_type: A constant denoting the type of hyperparameter search algorithm that must be used.
  28     """
  29 
  30     def __init__(self, loss, metric, searcher_args=None, path=None, verbose=False, search_type=BayesianSearcher):
  31         self.searcher_args = searcher_args if searcher_args is not None else {}
  32         self.searcher = None
  33         self.path = path if path is not None else rand_temp_folder_generator()
  34         ensure_dir(self.path)
  35         if verbose:
  36             print('Saving Directory:', self.path)
  37         self.verbose = verbose
  38         self.loss = loss
  39         self.metric = metric
  40         self.generators = []
  41         self.search_type = search_type
  42 
  43     def fit(self, n_output_node, input_shape, train_data, test_data, time_limit=24 * 60 * 60):
  44         """ Search the best network.
  45 
  46         Args:
  47             n_output_node: A integer value represent the number of output node in the final layer.
  48             input_shape: A tuple to express the shape of every train entry. For example,
  49                 MNIST dataset would be (28,28,1).
  50             train_data: A PyTorch DataLoader instance representing the training data.
  51             test_data: A PyTorch DataLoader instance representing the testing data.
  52             time_limit: A integer value represents the time limit on searching for models.
  53         """
  54         # Create the searcher and save on disk
  55 
  56         if not self.searcher:
  57             input_shape = input_shape[1:]
  58             self.searcher_args['n_output_node'] = n_output_node
  59             self.searcher_args['input_shape'] = input_shape
  60             self.searcher_args['path'] = self.path
  61             self.searcher_args['metric'] = self.metric
  62             self.searcher_args['loss'] = self.loss
  63             self.searcher_args['generators'] = self.generators
  64             self.searcher_args['verbose'] = self.verbose
  65             pickle_to_file(self, os.path.join(self.path, 'module'))
  66             self.searcher = self.search_type(**self.searcher_args)
  67 
  68         start_time = time.time()
  69         time_remain = time_limit
  70         try:
  71             while time_remain > 0:
  72                 self.searcher.search(train_data, test_data, int(time_remain))
  73                 pickle_to_file(self, os.path.join(self.path, 'module'))
  74                 if len(self.searcher.history) >= Constant.MAX_MODEL_NUM:
  75                     break
  76                 time_elapsed = time.time() - start_time
  77                 time_remain = time_limit - time_elapsed
  78             # if no search executed during the time_limit, then raise an error
  79             if time_remain <= 0:
  80                 raise TimeoutError
  81         except TimeoutError:
  82             if len(self.searcher.history) == 0:
  83                 raise TimeoutError("Search Time too short. No model was found during the search time.")
  84             elif self.verbose:
  85                 print('Time is out.')
  86 
  87     def final_fit(self, train_data, test_data, trainer_args=None, retrain=False):
  88         """Final training after found the best architecture.
  89 
  90         Args:
  91             train_data: A DataLoader instance representing the training data.
  92             test_data: A DataLoader instance representing the testing data.
  93             trainer_args: A dictionary containing the parameters of the ModelTrainer constructor.
  94             retrain: A boolean of whether reinitialize the weights of the model.
  95         """
  96         graph = self.searcher.load_best_model()
  97 
  98         if retrain:
  99             graph.weighted = False
  100         _, _1, graph = train(None, graph,
  101                              train_data,
  102                              test_data,
  103                              trainer_args,
  104                              self.metric,
  105                              self.loss,
  106                              self.verbose,
  107                              self.path)
  108         self.searcher.replace_model(graph, self.searcher.get_best_model_id())
  109         pickle_to_file(self, os.path.join(self.path, 'module'))
  110 
  111     @property
  112     def best_model(self):
  113         return self.searcher.load_best_model()
  114 
  115     def predict(self, test_loader):
  116         model = self.best_model.produce_model()
  117         model.eval()
  118 
  119         outputs = []
  120         with torch.no_grad():
  121             for index, inputs in enumerate(test_loader):
  122                 outputs.append(model(inputs).numpy())
  123         output = reduce(lambda x, y: np.concatenate((x, y)), outputs)
  124         return output
  125 
  126 
  127 class CnnModule(NetworkModule):
  128     """ Class to create a CNN module."""
  129 
  130     def __init__(self, loss, metric, searcher_args=None, path=None, verbose=False,
  131                  search_type=BayesianSearcher):
  132         super(CnnModule, self).__init__(loss, metric, searcher_args, path, verbose, search_type)
  133         self.generators.append(CnnGenerator)
  134         self.generators.append(ResNetGenerator)
  135         self.generators.append(DenseNetGenerator)
  136 
  137 
  138 class MlpModule(NetworkModule):
  139     """ Class to create an MLP module."""
  140 
  141     def __init__(self, loss, metric, searcher_args=None, path=None, verbose=False):
  142         super(MlpModule, self).__init__(loss, metric, searcher_args, path, verbose)
  143         self.generators.extend([MlpGenerator] * 2)
  144
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/autokeras/net_module.py b/autokeras/net_module.py
  --- a/autokeras/net_module.py
  +++ b/autokeras/net_module.py
  @@ -11,6 +11,7 @@
   
   from autokeras.utils import pickle_to_file, rand_temp_folder_generator, ensure_dir
   from autokeras.nn.generator import CnnGenerator, MlpGenerator, ResNetGenerator, DenseNetGenerator
  +from autokeras.utils import get_device
   
   
   class NetworkModule:
  @@ -123,6 +124,24 @@
           output = reduce(lambda x, y: np.concatenate((x, y)), outputs)
           return output
   
  +    def evaluate(self, test_data):
  +        """Evaluate the performance of the best architecture in terms of the loss.
  +
  +        Args:
  +            test_data: A DataLoader instance representing the testing data.
  +        """
  +        model = self.best_model.produce_model()
  +        model.eval()
  +        device = get_device()
  +        target, prediction = [], []
  +
  +        with torch.no_grad():
  +            for _, (x, y) in enumerate(test_data):
  +                x, y = x.to(device), y.to(device)
  +                prediction.append(model(x))
  +                target.append(y)
  +        return self.metric().compute(prediction, target)
  +
   
   class CnnModule(NetworkModule):
       """ Class to create a CNN module."""
  
verification_info: {"golden_diff": "diff --git a/autokeras/net_module.py b/autokeras/net_module.py\n--- a/autokeras/net_module.py\n+++ b/autokeras/net_module.py\n@@ -11,6 +11,7 @@\n \n from autokeras.utils import pickle_to_file, rand_temp_folder_generator, ensure_dir\n from autokeras.nn.generator import CnnGenerator, MlpGenerator, ResNetGenerator, DenseNetGenerator\n+from autokeras.utils import get_device\n \n \n class NetworkModule:\n@@ -123,6 +124,24 @@\n         output = reduce(lambda x, y: np.concatenate((x, y)), outputs)\n         return output\n \n+    def evaluate(self, test_data):\n+        \"\"\"Evaluate the performance of the best architecture in terms of the loss.\n+\n+        Args:\n+            test_data: A DataLoader instance representing the testing data.\n+        \"\"\"\n+        model = self.best_model.produce_model()\n+        model.eval()\n+        device = get_device()\n+        target, prediction = [], []\n+\n+        with torch.no_grad():\n+            for _, (x, y) in enumerate(test_data):\n+                x, y = x.to(device), y.to(device)\n+                prediction.append(model(x))\n+                target.append(y)\n+        return self.metric().compute(prediction, target)\n+\n \n class CnnModule(NetworkModule):\n     \"\"\" Class to create a CNN module.\"\"\"\n", "issue": "Evaluation criteria for MLP\nAre there any evaluation criterias for MLP module in Autokeras? \r\n\n", "before_files": [{"content": "from functools import reduce\n\nimport torch\nimport numpy as np\n\nimport os\nimport time\n\nfrom autokeras.constant import Constant\nfrom autokeras.search import BayesianSearcher, train\n\nfrom autokeras.utils import pickle_to_file, rand_temp_folder_generator, ensure_dir\nfrom autokeras.nn.generator import CnnGenerator, MlpGenerator, ResNetGenerator, DenseNetGenerator\n\n\nclass NetworkModule:\n    \"\"\" Class to create a network module.\n\n    Attributes:\n        loss: A function taking two parameters, the predictions and the ground truth.\n        metric: An instance of the Metric subclasses.\n        searcher_args: A dictionary containing the parameters for the searcher's __init__ function.\n        searcher: An instance of the Searcher class.\n        path: A string. The path to the directory to save the searcher.\n        verbose: A boolean. Setting it to true prints to stdout.\n        generators: A list of instances of the NetworkGenerator class or its subclasses.\n        search_type: A constant denoting the type of hyperparameter search algorithm that must be used.\n    \"\"\"\n\n    def __init__(self, loss, metric, searcher_args=None, path=None, verbose=False, search_type=BayesianSearcher):\n        self.searcher_args = searcher_args if searcher_args is not None else {}\n        self.searcher = None\n        self.path = path if path is not None else rand_temp_folder_generator()\n        ensure_dir(self.path)\n        if verbose:\n            print('Saving Directory:', self.path)\n        self.verbose = verbose\n        self.loss = loss\n        self.metric = metric\n        self.generators = []\n        self.search_type = search_type\n\n    def fit(self, n_output_node, input_shape, train_data, test_data, time_limit=24 * 60 * 60):\n        \"\"\" Search the best network.\n\n        Args:\n            n_output_node: A integer value represent the number of output node in the final layer.\n            input_shape: A tuple to express the shape of every train entry. For example,\n                MNIST dataset would be (28,28,1).\n            train_data: A PyTorch DataLoader instance representing the training data.\n            test_data: A PyTorch DataLoader instance representing the testing data.\n            time_limit: A integer value represents the time limit on searching for models.\n        \"\"\"\n        # Create the searcher and save on disk\n\n        if not self.searcher:\n            input_shape = input_shape[1:]\n            self.searcher_args['n_output_node'] = n_output_node\n            self.searcher_args['input_shape'] = input_shape\n            self.searcher_args['path'] = self.path\n            self.searcher_args['metric'] = self.metric\n            self.searcher_args['loss'] = self.loss\n            self.searcher_args['generators'] = self.generators\n            self.searcher_args['verbose'] = self.verbose\n            pickle_to_file(self, os.path.join(self.path, 'module'))\n            self.searcher = self.search_type(**self.searcher_args)\n\n        start_time = time.time()\n        time_remain = time_limit\n        try:\n            while time_remain > 0:\n                self.searcher.search(train_data, test_data, int(time_remain))\n                pickle_to_file(self, os.path.join(self.path, 'module'))\n                if len(self.searcher.history) >= Constant.MAX_MODEL_NUM:\n                    break\n                time_elapsed = time.time() - start_time\n                time_remain = time_limit - time_elapsed\n            # if no search executed during the time_limit, then raise an error\n            if time_remain <= 0:\n                raise TimeoutError\n        except TimeoutError:\n            if len(self.searcher.history) == 0:\n                raise TimeoutError(\"Search Time too short. No model was found during the search time.\")\n            elif self.verbose:\n                print('Time is out.')\n\n    def final_fit(self, train_data, test_data, trainer_args=None, retrain=False):\n        \"\"\"Final training after found the best architecture.\n\n        Args:\n            train_data: A DataLoader instance representing the training data.\n            test_data: A DataLoader instance representing the testing data.\n            trainer_args: A dictionary containing the parameters of the ModelTrainer constructor.\n            retrain: A boolean of whether reinitialize the weights of the model.\n        \"\"\"\n        graph = self.searcher.load_best_model()\n\n        if retrain:\n            graph.weighted = False\n        _, _1, graph = train(None, graph,\n                             train_data,\n                             test_data,\n                             trainer_args,\n                             self.metric,\n                             self.loss,\n                             self.verbose,\n                             self.path)\n        self.searcher.replace_model(graph, self.searcher.get_best_model_id())\n        pickle_to_file(self, os.path.join(self.path, 'module'))\n\n    @property\n    def best_model(self):\n        return self.searcher.load_best_model()\n\n    def predict(self, test_loader):\n        model = self.best_model.produce_model()\n        model.eval()\n\n        outputs = []\n        with torch.no_grad():\n            for index, inputs in enumerate(test_loader):\n                outputs.append(model(inputs).numpy())\n        output = reduce(lambda x, y: np.concatenate((x, y)), outputs)\n        return output\n\n\nclass CnnModule(NetworkModule):\n    \"\"\" Class to create a CNN module.\"\"\"\n\n    def __init__(self, loss, metric, searcher_args=None, path=None, verbose=False,\n                 search_type=BayesianSearcher):\n        super(CnnModule, self).__init__(loss, metric, searcher_args, path, verbose, search_type)\n        self.generators.append(CnnGenerator)\n        self.generators.append(ResNetGenerator)\n        self.generators.append(DenseNetGenerator)\n\n\nclass MlpModule(NetworkModule):\n    \"\"\" Class to create an MLP module.\"\"\"\n\n    def __init__(self, loss, metric, searcher_args=None, path=None, verbose=False):\n        super(MlpModule, self).__init__(loss, metric, searcher_args, path, verbose)\n        self.generators.extend([MlpGenerator] * 2)\n", "path": "autokeras/net_module.py"}], "after_files": [{"content": "from functools import reduce\n\nimport torch\nimport numpy as np\n\nimport os\nimport time\n\nfrom autokeras.constant import Constant\nfrom autokeras.search import BayesianSearcher, train\n\nfrom autokeras.utils import pickle_to_file, rand_temp_folder_generator, ensure_dir\nfrom autokeras.nn.generator import CnnGenerator, MlpGenerator, ResNetGenerator, DenseNetGenerator\nfrom autokeras.utils import get_device\n\n\nclass NetworkModule:\n    \"\"\" Class to create a network module.\n\n    Attributes:\n        loss: A function taking two parameters, the predictions and the ground truth.\n        metric: An instance of the Metric subclasses.\n        searcher_args: A dictionary containing the parameters for the searcher's __init__ function.\n        searcher: An instance of the Searcher class.\n        path: A string. The path to the directory to save the searcher.\n        verbose: A boolean. Setting it to true prints to stdout.\n        generators: A list of instances of the NetworkGenerator class or its subclasses.\n        search_type: A constant denoting the type of hyperparameter search algorithm that must be used.\n    \"\"\"\n\n    def __init__(self, loss, metric, searcher_args=None, path=None, verbose=False, search_type=BayesianSearcher):\n        self.searcher_args = searcher_args if searcher_args is not None else {}\n        self.searcher = None\n        self.path = path if path is not None else rand_temp_folder_generator()\n        ensure_dir(self.path)\n        if verbose:\n            print('Saving Directory:', self.path)\n        self.verbose = verbose\n        self.loss = loss\n        self.metric = metric\n        self.generators = []\n        self.search_type = search_type\n\n    def fit(self, n_output_node, input_shape, train_data, test_data, time_limit=24 * 60 * 60):\n        \"\"\" Search the best network.\n\n        Args:\n            n_output_node: A integer value represent the number of output node in the final layer.\n            input_shape: A tuple to express the shape of every train entry. For example,\n                MNIST dataset would be (28,28,1).\n            train_data: A PyTorch DataLoader instance representing the training data.\n            test_data: A PyTorch DataLoader instance representing the testing data.\n            time_limit: A integer value represents the time limit on searching for models.\n        \"\"\"\n        # Create the searcher and save on disk\n\n        if not self.searcher:\n            input_shape = input_shape[1:]\n            self.searcher_args['n_output_node'] = n_output_node\n            self.searcher_args['input_shape'] = input_shape\n            self.searcher_args['path'] = self.path\n            self.searcher_args['metric'] = self.metric\n            self.searcher_args['loss'] = self.loss\n            self.searcher_args['generators'] = self.generators\n            self.searcher_args['verbose'] = self.verbose\n            pickle_to_file(self, os.path.join(self.path, 'module'))\n            self.searcher = self.search_type(**self.searcher_args)\n\n        start_time = time.time()\n        time_remain = time_limit\n        try:\n            while time_remain > 0:\n                self.searcher.search(train_data, test_data, int(time_remain))\n                pickle_to_file(self, os.path.join(self.path, 'module'))\n                if len(self.searcher.history) >= Constant.MAX_MODEL_NUM:\n                    break\n                time_elapsed = time.time() - start_time\n                time_remain = time_limit - time_elapsed\n            # if no search executed during the time_limit, then raise an error\n            if time_remain <= 0:\n                raise TimeoutError\n        except TimeoutError:\n            if len(self.searcher.history) == 0:\n                raise TimeoutError(\"Search Time too short. No model was found during the search time.\")\n            elif self.verbose:\n                print('Time is out.')\n\n    def final_fit(self, train_data, test_data, trainer_args=None, retrain=False):\n        \"\"\"Final training after found the best architecture.\n\n        Args:\n            train_data: A DataLoader instance representing the training data.\n            test_data: A DataLoader instance representing the testing data.\n            trainer_args: A dictionary containing the parameters of the ModelTrainer constructor.\n            retrain: A boolean of whether reinitialize the weights of the model.\n        \"\"\"\n        graph = self.searcher.load_best_model()\n\n        if retrain:\n            graph.weighted = False\n        _, _1, graph = train(None, graph,\n                             train_data,\n                             test_data,\n                             trainer_args,\n                             self.metric,\n                             self.loss,\n                             self.verbose,\n                             self.path)\n        self.searcher.replace_model(graph, self.searcher.get_best_model_id())\n        pickle_to_file(self, os.path.join(self.path, 'module'))\n\n    @property\n    def best_model(self):\n        return self.searcher.load_best_model()\n\n    def predict(self, test_loader):\n        model = self.best_model.produce_model()\n        model.eval()\n\n        outputs = []\n        with torch.no_grad():\n            for index, inputs in enumerate(test_loader):\n                outputs.append(model(inputs).numpy())\n        output = reduce(lambda x, y: np.concatenate((x, y)), outputs)\n        return output\n\n    def evaluate(self, test_data):\n        \"\"\"Evaluate the performance of the best architecture in terms of the loss.\n\n        Args:\n            test_data: A DataLoader instance representing the testing data.\n        \"\"\"\n        model = self.best_model.produce_model()\n        model.eval()\n        device = get_device()\n        target, prediction = [], []\n\n        with torch.no_grad():\n            for _, (x, y) in enumerate(test_data):\n                x, y = x.to(device), y.to(device)\n                prediction.append(model(x))\n                target.append(y)\n        return self.metric().compute(prediction, target)\n\n\nclass CnnModule(NetworkModule):\n    \"\"\" Class to create a CNN module.\"\"\"\n\n    def __init__(self, loss, metric, searcher_args=None, path=None, verbose=False,\n                 search_type=BayesianSearcher):\n        super(CnnModule, self).__init__(loss, metric, searcher_args, path, verbose, search_type)\n        self.generators.append(CnnGenerator)\n        self.generators.append(ResNetGenerator)\n        self.generators.append(DenseNetGenerator)\n\n\nclass MlpModule(NetworkModule):\n    \"\"\" Class to create an MLP module.\"\"\"\n\n    def __init__(self, loss, metric, searcher_args=None, path=None, verbose=False):\n        super(MlpModule, self).__init__(loss, metric, searcher_args, path, verbose)\n        self.generators.extend([MlpGenerator] * 2)\n", "path": "autokeras/net_module.py"}]}