prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Model evaluation on test set yields different results on the same model
  <!---
  **If you are reporting a bug:**
  *  Verify that your issue is not being currently addressed by other issues or pull requests.
  *  Please note that Auto-Keras is only compatible with **Python 3.6**.
  *  Tag the issue with the `bug report` tag.
  -->
  
  ### Bug Description
  Test set evaluation yields different results if re-run.
  
  ### Reproducing Steps
  Steps to reproduce the behavior:
  re-run multiple times:
  ```python
  y_pred = clf.evaluate(X_test, y_test)
  print(y_pred)
  ```
  will yield different results:
  ```
  0.0992
  0.1032
  0.101
  0.0989
  ```
  
  Further, using manual evaluation:
  ```python
  y_prediction = clf.predict(x_test=X_test)
  from sklearn.metrics import accuracy_score
  accuracy_score(y_pred=y_prediction, y_true=y_test)
  ```
  leads to different results as well.  It looks like the model either uses some random function (AFAIK all: ```random_states=42``` in the source code) or there is some major error in the pipeline that causes different predictions of the test set all the time.
  
  ### Expected Behavior
  Final evaluation on a test set should not lead to different results using the same model on the same data.
  
  ### Setup Details
  Include the details about the versions of:
   - OS type and version: Linux
   - Python: 3.6.5
   - autokeras: 0.2.11
   - scikit-learn:0.19.1
   - numpy:1.14.5
   - keras: 2.2.2
   - scipy:1.1.0
   - tensorflow: 1.10.0
   - pytorch:0.4.1
  
  ### Additional context
  This error is verified on MNIST and Fashion-MNIST.
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `autokeras/preprocessor.py`
  Content:
  ```
  1 import torch
  2 
  3 import numpy as np
  4 from torch.utils.data import Dataset, DataLoader
  5 from torchvision.transforms import ToPILImage, RandomCrop, RandomHorizontalFlip, ToTensor, Normalize, Compose
  6 
  7 from autokeras.constant import Constant
  8 
  9 
  10 class OneHotEncoder:
  11     """A class that can format data.
  12 
  13     This class provides ways to transform data's classification label into vector.
  14 
  15     Attributes:
  16           data: The input data
  17           n_classes: The number of classes in the classification problem.
  18           labels: The number of labels.
  19           label_to_vec: Mapping from label to vector.
  20           int_to_label: Mapping from int to label.
  21     """
  22 
  23     def __init__(self):
  24         """Initialize a OneHotEncoder"""
  25         self.data = None
  26         self.n_classes = 0
  27         self.labels = None
  28         self.label_to_vec = {}
  29         self.int_to_label = {}
  30 
  31     def fit(self, data):
  32         """Create mapping from label to vector, and vector to label."""
  33         data = np.array(data).flatten()
  34         self.labels = set(data)
  35         self.n_classes = len(self.labels)
  36         for index, label in enumerate(self.labels):
  37             vec = np.array([0] * self.n_classes)
  38             vec[index] = 1
  39             self.label_to_vec[label] = vec
  40             self.int_to_label[index] = label
  41 
  42     def transform(self, data):
  43         """Get vector for every element in the data array."""
  44         data = np.array(data)
  45         if len(data.shape) > 1:
  46             data = data.flatten()
  47         return np.array(list(map(lambda x: self.label_to_vec[x], data)))
  48 
  49     def inverse_transform(self, data):
  50         """Get label for every element in data."""
  51         return np.array(list(map(lambda x: self.int_to_label[x], np.argmax(np.array(data), axis=1))))
  52 
  53 
  54 class DataTransformer:
  55     def __init__(self, data, augment=Constant.DATA_AUGMENTATION):
  56         self.max_val = data.max()
  57         data = data / self.max_val
  58         self.mean = np.mean(data, axis=(0, 1, 2), keepdims=True).flatten()
  59         self.std = np.std(data, axis=(0, 1, 2), keepdims=True).flatten()
  60         self.augment = augment
  61 
  62     def transform_train(self, data, targets=None, batch_size=None):
  63         if not self.augment:
  64             augment_list = []
  65         else:
  66             augment_list = [ToPILImage(),
  67                             RandomCrop(data.shape[1:3], padding=4),
  68                             RandomHorizontalFlip(),
  69                             ToTensor()
  70                             ]
  71         common_list = [Normalize(torch.Tensor(self.mean), torch.Tensor(self.std))]
  72         compose_list = augment_list + common_list
  73 
  74         return self._transform(batch_size, compose_list, data, targets)
  75 
  76     def transform_test(self, data, targets=None, batch_size=None):
  77         common_list = [Normalize(torch.Tensor(self.mean), torch.Tensor(self.std))]
  78         compose_list = common_list
  79 
  80         return self._transform(batch_size, compose_list, data, targets)
  81 
  82     def _transform(self, batch_size, compose_list, data, targets):
  83         if batch_size is None:
  84             batch_size = Constant.MAX_BATCH_SIZE
  85         batch_size = min(len(data), batch_size)
  86         data = data / self.max_val
  87         data = torch.Tensor(data.transpose(0, 3, 1, 2))
  88         data_transforms = Compose(compose_list)
  89         dataset = MultiTransformDataset(data, targets, data_transforms)
  90         return DataLoader(dataset, batch_size=batch_size, shuffle=True)
  91 
  92 
  93 class MultiTransformDataset(Dataset):
  94     def __init__(self, dataset, target, compose):
  95         self.dataset = dataset
  96         self.target = target
  97         self.compose = compose
  98 
  99     def __getitem__(self, index):
  100         feature = self.dataset[index]
  101         if self.target is None:
  102             return self.compose(feature)
  103         return self.compose(feature), self.target[index]
  104 
  105     def __len__(self):
  106         return len(self.dataset)
  107
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/autokeras/preprocessor.py b/autokeras/preprocessor.py
  --- a/autokeras/preprocessor.py
  +++ b/autokeras/preprocessor.py
  @@ -71,23 +71,31 @@
           common_list = [Normalize(torch.Tensor(self.mean), torch.Tensor(self.std))]
           compose_list = augment_list + common_list
   
  -        return self._transform(batch_size, compose_list, data, targets)
  +        dataset = self._transform(compose_list, data, targets)
  +
  +        if batch_size is None:
  +            batch_size = Constant.MAX_BATCH_SIZE
  +        batch_size = min(len(data), batch_size)
  +
  +        return DataLoader(dataset, batch_size=batch_size, shuffle=True)
   
       def transform_test(self, data, targets=None, batch_size=None):
           common_list = [Normalize(torch.Tensor(self.mean), torch.Tensor(self.std))]
           compose_list = common_list
   
  -        return self._transform(batch_size, compose_list, data, targets)
  +        dataset = self._transform(compose_list, data, targets)
   
  -    def _transform(self, batch_size, compose_list, data, targets):
           if batch_size is None:
               batch_size = Constant.MAX_BATCH_SIZE
           batch_size = min(len(data), batch_size)
  +
  +        return DataLoader(dataset, batch_size=batch_size, shuffle=False)
  +
  +    def _transform(self, compose_list, data, targets):
           data = data / self.max_val
           data = torch.Tensor(data.transpose(0, 3, 1, 2))
           data_transforms = Compose(compose_list)
  -        dataset = MultiTransformDataset(data, targets, data_transforms)
  -        return DataLoader(dataset, batch_size=batch_size, shuffle=True)
  +        return MultiTransformDataset(data, targets, data_transforms)
   
   
   class MultiTransformDataset(Dataset):
  
verification_info: {"golden_diff": "diff --git a/autokeras/preprocessor.py b/autokeras/preprocessor.py\n--- a/autokeras/preprocessor.py\n+++ b/autokeras/preprocessor.py\n@@ -71,23 +71,31 @@\n         common_list = [Normalize(torch.Tensor(self.mean), torch.Tensor(self.std))]\n         compose_list = augment_list + common_list\n \n-        return self._transform(batch_size, compose_list, data, targets)\n+        dataset = self._transform(compose_list, data, targets)\n+\n+        if batch_size is None:\n+            batch_size = Constant.MAX_BATCH_SIZE\n+        batch_size = min(len(data), batch_size)\n+\n+        return DataLoader(dataset, batch_size=batch_size, shuffle=True)\n \n     def transform_test(self, data, targets=None, batch_size=None):\n         common_list = [Normalize(torch.Tensor(self.mean), torch.Tensor(self.std))]\n         compose_list = common_list\n \n-        return self._transform(batch_size, compose_list, data, targets)\n+        dataset = self._transform(compose_list, data, targets)\n \n-    def _transform(self, batch_size, compose_list, data, targets):\n         if batch_size is None:\n             batch_size = Constant.MAX_BATCH_SIZE\n         batch_size = min(len(data), batch_size)\n+\n+        return DataLoader(dataset, batch_size=batch_size, shuffle=False)\n+\n+    def _transform(self, compose_list, data, targets):\n         data = data / self.max_val\n         data = torch.Tensor(data.transpose(0, 3, 1, 2))\n         data_transforms = Compose(compose_list)\n-        dataset = MultiTransformDataset(data, targets, data_transforms)\n-        return DataLoader(dataset, batch_size=batch_size, shuffle=True)\n+        return MultiTransformDataset(data, targets, data_transforms)\n \n \n class MultiTransformDataset(Dataset):\n", "issue": "Model evaluation on test set yields different results on the same model\n<!---\r\n**If you are reporting a bug:**\r\n*  Verify that your issue is not being currently addressed by other issues or pull requests.\r\n*  Please note that Auto-Keras is only compatible with **Python 3.6**.\r\n*  Tag the issue with the `bug report` tag.\r\n-->\r\n\r\n### Bug Description\r\nTest set evaluation yields different results if re-run.\r\n\r\n### Reproducing Steps\r\nSteps to reproduce the behavior:\r\nre-run multiple times:\r\n```python\r\ny_pred = clf.evaluate(X_test, y_test)\r\nprint(y_pred)\r\n```\r\nwill yield different results:\r\n```\r\n0.0992\r\n0.1032\r\n0.101\r\n0.0989\r\n```\r\n\r\nFurther, using manual evaluation:\r\n```python\r\ny_prediction = clf.predict(x_test=X_test)\r\nfrom sklearn.metrics import accuracy_score\r\naccuracy_score(y_pred=y_prediction, y_true=y_test)\r\n```\r\nleads to different results as well.  It looks like the model either uses some random function (AFAIK all: ```random_states=42``` in the source code) or there is some major error in the pipeline that causes different predictions of the test set all the time.\r\n\r\n### Expected Behavior\r\nFinal evaluation on a test set should not lead to different results using the same model on the same data.\r\n\r\n### Setup Details\r\nInclude the details about the versions of:\r\n - OS type and version: Linux\r\n - Python: 3.6.5\r\n - autokeras: 0.2.11\r\n - scikit-learn:0.19.1\r\n - numpy:1.14.5\r\n - keras: 2.2.2\r\n - scipy:1.1.0\r\n - tensorflow: 1.10.0\r\n - pytorch:0.4.1\r\n\r\n### Additional context\r\nThis error is verified on MNIST and Fashion-MNIST.\n", "before_files": [{"content": "import torch\n\nimport numpy as np\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import ToPILImage, RandomCrop, RandomHorizontalFlip, ToTensor, Normalize, Compose\n\nfrom autokeras.constant import Constant\n\n\nclass OneHotEncoder:\n    \"\"\"A class that can format data.\n\n    This class provides ways to transform data's classification label into vector.\n\n    Attributes:\n          data: The input data\n          n_classes: The number of classes in the classification problem.\n          labels: The number of labels.\n          label_to_vec: Mapping from label to vector.\n          int_to_label: Mapping from int to label.\n    \"\"\"\n\n    def __init__(self):\n        \"\"\"Initialize a OneHotEncoder\"\"\"\n        self.data = None\n        self.n_classes = 0\n        self.labels = None\n        self.label_to_vec = {}\n        self.int_to_label = {}\n\n    def fit(self, data):\n        \"\"\"Create mapping from label to vector, and vector to label.\"\"\"\n        data = np.array(data).flatten()\n        self.labels = set(data)\n        self.n_classes = len(self.labels)\n        for index, label in enumerate(self.labels):\n            vec = np.array([0] * self.n_classes)\n            vec[index] = 1\n            self.label_to_vec[label] = vec\n            self.int_to_label[index] = label\n\n    def transform(self, data):\n        \"\"\"Get vector for every element in the data array.\"\"\"\n        data = np.array(data)\n        if len(data.shape) > 1:\n            data = data.flatten()\n        return np.array(list(map(lambda x: self.label_to_vec[x], data)))\n\n    def inverse_transform(self, data):\n        \"\"\"Get label for every element in data.\"\"\"\n        return np.array(list(map(lambda x: self.int_to_label[x], np.argmax(np.array(data), axis=1))))\n\n\nclass DataTransformer:\n    def __init__(self, data, augment=Constant.DATA_AUGMENTATION):\n        self.max_val = data.max()\n        data = data / self.max_val\n        self.mean = np.mean(data, axis=(0, 1, 2), keepdims=True).flatten()\n        self.std = np.std(data, axis=(0, 1, 2), keepdims=True).flatten()\n        self.augment = augment\n\n    def transform_train(self, data, targets=None, batch_size=None):\n        if not self.augment:\n            augment_list = []\n        else:\n            augment_list = [ToPILImage(),\n                            RandomCrop(data.shape[1:3], padding=4),\n                            RandomHorizontalFlip(),\n                            ToTensor()\n                            ]\n        common_list = [Normalize(torch.Tensor(self.mean), torch.Tensor(self.std))]\n        compose_list = augment_list + common_list\n\n        return self._transform(batch_size, compose_list, data, targets)\n\n    def transform_test(self, data, targets=None, batch_size=None):\n        common_list = [Normalize(torch.Tensor(self.mean), torch.Tensor(self.std))]\n        compose_list = common_list\n\n        return self._transform(batch_size, compose_list, data, targets)\n\n    def _transform(self, batch_size, compose_list, data, targets):\n        if batch_size is None:\n            batch_size = Constant.MAX_BATCH_SIZE\n        batch_size = min(len(data), batch_size)\n        data = data / self.max_val\n        data = torch.Tensor(data.transpose(0, 3, 1, 2))\n        data_transforms = Compose(compose_list)\n        dataset = MultiTransformDataset(data, targets, data_transforms)\n        return DataLoader(dataset, batch_size=batch_size, shuffle=True)\n\n\nclass MultiTransformDataset(Dataset):\n    def __init__(self, dataset, target, compose):\n        self.dataset = dataset\n        self.target = target\n        self.compose = compose\n\n    def __getitem__(self, index):\n        feature = self.dataset[index]\n        if self.target is None:\n            return self.compose(feature)\n        return self.compose(feature), self.target[index]\n\n    def __len__(self):\n        return len(self.dataset)\n", "path": "autokeras/preprocessor.py"}], "after_files": [{"content": "import torch\n\nimport numpy as np\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import ToPILImage, RandomCrop, RandomHorizontalFlip, ToTensor, Normalize, Compose\n\nfrom autokeras.constant import Constant\n\n\nclass OneHotEncoder:\n    \"\"\"A class that can format data.\n\n    This class provides ways to transform data's classification label into vector.\n\n    Attributes:\n          data: The input data\n          n_classes: The number of classes in the classification problem.\n          labels: The number of labels.\n          label_to_vec: Mapping from label to vector.\n          int_to_label: Mapping from int to label.\n    \"\"\"\n\n    def __init__(self):\n        \"\"\"Initialize a OneHotEncoder\"\"\"\n        self.data = None\n        self.n_classes = 0\n        self.labels = None\n        self.label_to_vec = {}\n        self.int_to_label = {}\n\n    def fit(self, data):\n        \"\"\"Create mapping from label to vector, and vector to label.\"\"\"\n        data = np.array(data).flatten()\n        self.labels = set(data)\n        self.n_classes = len(self.labels)\n        for index, label in enumerate(self.labels):\n            vec = np.array([0] * self.n_classes)\n            vec[index] = 1\n            self.label_to_vec[label] = vec\n            self.int_to_label[index] = label\n\n    def transform(self, data):\n        \"\"\"Get vector for every element in the data array.\"\"\"\n        data = np.array(data)\n        if len(data.shape) > 1:\n            data = data.flatten()\n        return np.array(list(map(lambda x: self.label_to_vec[x], data)))\n\n    def inverse_transform(self, data):\n        \"\"\"Get label for every element in data.\"\"\"\n        return np.array(list(map(lambda x: self.int_to_label[x], np.argmax(np.array(data), axis=1))))\n\n\nclass DataTransformer:\n    def __init__(self, data, augment=Constant.DATA_AUGMENTATION):\n        self.max_val = data.max()\n        data = data / self.max_val\n        self.mean = np.mean(data, axis=(0, 1, 2), keepdims=True).flatten()\n        self.std = np.std(data, axis=(0, 1, 2), keepdims=True).flatten()\n        self.augment = augment\n\n    def transform_train(self, data, targets=None, batch_size=None):\n        if not self.augment:\n            augment_list = []\n        else:\n            augment_list = [ToPILImage(),\n                            RandomCrop(data.shape[1:3], padding=4),\n                            RandomHorizontalFlip(),\n                            ToTensor()\n                            ]\n        common_list = [Normalize(torch.Tensor(self.mean), torch.Tensor(self.std))]\n        compose_list = augment_list + common_list\n\n        dataset = self._transform(compose_list, data, targets)\n\n        if batch_size is None:\n            batch_size = Constant.MAX_BATCH_SIZE\n        batch_size = min(len(data), batch_size)\n\n        return DataLoader(dataset, batch_size=batch_size, shuffle=True)\n\n    def transform_test(self, data, targets=None, batch_size=None):\n        common_list = [Normalize(torch.Tensor(self.mean), torch.Tensor(self.std))]\n        compose_list = common_list\n\n        dataset = self._transform(compose_list, data, targets)\n\n        if batch_size is None:\n            batch_size = Constant.MAX_BATCH_SIZE\n        batch_size = min(len(data), batch_size)\n\n        return DataLoader(dataset, batch_size=batch_size, shuffle=False)\n\n    def _transform(self, compose_list, data, targets):\n        data = data / self.max_val\n        data = torch.Tensor(data.transpose(0, 3, 1, 2))\n        data_transforms = Compose(compose_list)\n        return MultiTransformDataset(data, targets, data_transforms)\n\n\nclass MultiTransformDataset(Dataset):\n    def __init__(self, dataset, target, compose):\n        self.dataset = dataset\n        self.target = target\n        self.compose = compose\n\n    def __getitem__(self, index):\n        feature = self.dataset[index]\n        if self.target is None:\n            return self.compose(feature)\n        return self.compose(feature), self.target[index]\n\n    def __len__(self):\n        return len(self.dataset)\n", "path": "autokeras/preprocessor.py"}]}