prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Make the HyperparameterScheduler compatible with the CaseOptimizer
  ### Feature motivation
  The HyperparameterScheduler is not compatible with the CaseOptimizer since the hyperparameters are attributes of the optimizers inside the CaseOptimizer. 
  
  ### Feature description
  I propose one of the two possible solutions: 
  Either we could give HyperparameterScheduler the optimizer as an argument. It could be called via ```  HyperparameterScheduler(schedule, hyperparameter, optimizer, verbose=0) ``` and the right optimizer inside the CaseOptimizer can be addressed. (@koenhelwegen) 
  
  My second proposal would be to search the CaseOptimizer for optimizers that have the hyperparameter as attribute. Then the schedule can be applied to this optimizer only. The downside of this would be that in case there are two optimizers inside the CaseOptimizer that have a hyperparameter with the same name the schedule would be applied to both of them. I do not think this would happen very often but it could definitively be an issue. See code below for my second proposal. 
  ### Feature implementation
  ``` python 
  class HyperparameterScheduler(tf.keras.callbacks.Callback):
      """Generic hyperparameter scheduler.
      # Arguments
      schedule: a function that takes an epoch index as input
          (integer, indexed from 0) and returns a new hyperparameter as output.
      hyperparameter: str. the name of the hyperparameter to be scheduled.
      verbose: int. 0: quiet, 1: update messages.
      """
  
      def __init__(self, schedule, hyperparameter, verbose=0):
          super(HyperparameterScheduler, self).__init__()
          self.schedule = schedule
          self.hyperparameter = hyperparameter
          self.verbose = verbose
  
      def on_epoch_begin(self, epoch, logs=None):
          for op in self.model.optimizer.optimizers:
              if hasattr(op, self.hyperparameter):
  
                  hp = getattr(op, self.hyperparameter)
                  try:  # new API
                      hyperparameter_val = tf.keras.backend.get_value(hp)
                      hyperparameter_val = self.schedule(epoch, hyperparameter_val)
                  except TypeError:  # Support for old API for backward compatibility
                      hyperparameter_val = self.schedule(epoch)
  
                  tf.keras.backend.set_value(hp, hyperparameter_val)
  
                  if self.verbose > 0:
                      print(
                          f"Epoch {epoch + 1}: {self.hyperparameter} changning to {tf.keras.backend.get_value(hp)}."
                      )
  
      def on_epoch_end(self, epoch, logs=None):
          logs = logs or {}
          for op in self.model.optimizer.optimizers:
              if hasattr(op, self.hyperparameter):
                  hp = getattr(op, self.hyperparameter)
                  logs[self.hyperparameter] = tf.keras.backend.get_value(hp)
  ```
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `larq/callbacks.py`
  Content:
  ```
  1 import tensorflow as tf
  2 
  3 
  4 class HyperparameterScheduler(tf.keras.callbacks.Callback):
  5     """Generic hyperparameter scheduler.
  6 
  7     # Arguments
  8     schedule: a function that takes an epoch index as input
  9         (integer, indexed from 0) and returns a new hyperparameter as output.
  10     hyperparameter: str. the name of the hyperparameter to be scheduled.
  11     verbose: int. 0: quiet, 1: update messages.
  12     """
  13 
  14     def __init__(self, schedule, hyperparameter, verbose=0):
  15         super(HyperparameterScheduler, self).__init__()
  16         self.schedule = schedule
  17         self.hyperparameter = hyperparameter
  18         self.verbose = verbose
  19 
  20     def on_epoch_begin(self, epoch, logs=None):
  21         if not hasattr(self.model.optimizer, self.hyperparameter):
  22             raise ValueError(
  23                 f'Optimizer must have a "{self.hyperparameter}" attribute.'
  24             )
  25 
  26         hp = getattr(self.model.optimizer, self.hyperparameter)
  27         try:  # new API
  28             hyperparameter_val = tf.keras.backend.get_value(hp)
  29             hyperparameter_val = self.schedule(epoch, hyperparameter_val)
  30         except TypeError:  # Support for old API for backward compatibility
  31             hyperparameter_val = self.schedule(epoch)
  32 
  33         tf.keras.backend.set_value(hp, hyperparameter_val)
  34 
  35         if self.verbose > 0:
  36             print(
  37                 f"Epoch {epoch + 1}: {self.hyperparameter} changning to {tf.keras.backend.get_value(hp)}."
  38             )
  39 
  40     def on_epoch_end(self, epoch, logs=None):
  41         logs = logs or {}
  42         hp = getattr(self.model.optimizer, self.hyperparameter)
  43         logs[self.hyperparameter] = tf.keras.backend.get_value(hp)
  44
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/larq/callbacks.py b/larq/callbacks.py
  --- a/larq/callbacks.py
  +++ b/larq/callbacks.py
  @@ -4,26 +4,40 @@
   class HyperparameterScheduler(tf.keras.callbacks.Callback):
       """Generic hyperparameter scheduler.
   
  +    !!! example
  +        ```python
  +        bop = lq.optimizers.Bop(threshold=1e-6, gamma=1e-3)
  +        adam = tf.keras.optimizers.Adam(0.01)
  +        optimizer = lq.optimizers.CaseOptimizer(
  +            (lq.optimizers.Bop.is_binary_variable, bop), default_optimizer=adam,
  +        )
  +        callbacks = [
  +            HyperparameterScheduler(lambda x: 0.001 * (0.1 ** (x // 30)), "gamma", bop)
  +        ]
  +        ```
       # Arguments
  +    optimizer: the optimizer that contains the hyperparameter that will be scheduled.
  +        Defaults to `self.model.optimizer` if `optimizer == None`.
       schedule: a function that takes an epoch index as input
           (integer, indexed from 0) and returns a new hyperparameter as output.
       hyperparameter: str. the name of the hyperparameter to be scheduled.
       verbose: int. 0: quiet, 1: update messages.
       """
   
  -    def __init__(self, schedule, hyperparameter, verbose=0):
  +    def __init__(self, schedule, hyperparameter, optimizer=None, verbose=0):
           super(HyperparameterScheduler, self).__init__()
  +        self.optimizer = optimizer if optimizer else self.model.optimizer
           self.schedule = schedule
           self.hyperparameter = hyperparameter
           self.verbose = verbose
   
       def on_epoch_begin(self, epoch, logs=None):
  -        if not hasattr(self.model.optimizer, self.hyperparameter):
  +        if not hasattr(self.optimizer, self.hyperparameter):
               raise ValueError(
                   f'Optimizer must have a "{self.hyperparameter}" attribute.'
               )
   
  -        hp = getattr(self.model.optimizer, self.hyperparameter)
  +        hp = getattr(self.optimizer, self.hyperparameter)
           try:  # new API
               hyperparameter_val = tf.keras.backend.get_value(hp)
               hyperparameter_val = self.schedule(epoch, hyperparameter_val)
  @@ -34,10 +48,10 @@
   
           if self.verbose > 0:
               print(
  -                f"Epoch {epoch + 1}: {self.hyperparameter} changning to {tf.keras.backend.get_value(hp)}."
  +                f"Epoch {epoch + 1}: {self.hyperparameter} changing to {tf.keras.backend.get_value(hp)}."
               )
   
       def on_epoch_end(self, epoch, logs=None):
           logs = logs or {}
  -        hp = getattr(self.model.optimizer, self.hyperparameter)
  +        hp = getattr(self.optimizer, self.hyperparameter)
           logs[self.hyperparameter] = tf.keras.backend.get_value(hp)
  
verification_info: {"golden_diff": "diff --git a/larq/callbacks.py b/larq/callbacks.py\n--- a/larq/callbacks.py\n+++ b/larq/callbacks.py\n@@ -4,26 +4,40 @@\n class HyperparameterScheduler(tf.keras.callbacks.Callback):\n     \"\"\"Generic hyperparameter scheduler.\n \n+    !!! example\n+        ```python\n+        bop = lq.optimizers.Bop(threshold=1e-6, gamma=1e-3)\n+        adam = tf.keras.optimizers.Adam(0.01)\n+        optimizer = lq.optimizers.CaseOptimizer(\n+            (lq.optimizers.Bop.is_binary_variable, bop), default_optimizer=adam,\n+        )\n+        callbacks = [\n+            HyperparameterScheduler(lambda x: 0.001 * (0.1 ** (x // 30)), \"gamma\", bop)\n+        ]\n+        ```\n     # Arguments\n+    optimizer: the optimizer that contains the hyperparameter that will be scheduled.\n+        Defaults to `self.model.optimizer` if `optimizer == None`.\n     schedule: a function that takes an epoch index as input\n         (integer, indexed from 0) and returns a new hyperparameter as output.\n     hyperparameter: str. the name of the hyperparameter to be scheduled.\n     verbose: int. 0: quiet, 1: update messages.\n     \"\"\"\n \n-    def __init__(self, schedule, hyperparameter, verbose=0):\n+    def __init__(self, schedule, hyperparameter, optimizer=None, verbose=0):\n         super(HyperparameterScheduler, self).__init__()\n+        self.optimizer = optimizer if optimizer else self.model.optimizer\n         self.schedule = schedule\n         self.hyperparameter = hyperparameter\n         self.verbose = verbose\n \n     def on_epoch_begin(self, epoch, logs=None):\n-        if not hasattr(self.model.optimizer, self.hyperparameter):\n+        if not hasattr(self.optimizer, self.hyperparameter):\n             raise ValueError(\n                 f'Optimizer must have a \"{self.hyperparameter}\" attribute.'\n             )\n \n-        hp = getattr(self.model.optimizer, self.hyperparameter)\n+        hp = getattr(self.optimizer, self.hyperparameter)\n         try:  # new API\n             hyperparameter_val = tf.keras.backend.get_value(hp)\n             hyperparameter_val = self.schedule(epoch, hyperparameter_val)\n@@ -34,10 +48,10 @@\n \n         if self.verbose > 0:\n             print(\n-                f\"Epoch {epoch + 1}: {self.hyperparameter} changning to {tf.keras.backend.get_value(hp)}.\"\n+                f\"Epoch {epoch + 1}: {self.hyperparameter} changing to {tf.keras.backend.get_value(hp)}.\"\n             )\n \n     def on_epoch_end(self, epoch, logs=None):\n         logs = logs or {}\n-        hp = getattr(self.model.optimizer, self.hyperparameter)\n+        hp = getattr(self.optimizer, self.hyperparameter)\n         logs[self.hyperparameter] = tf.keras.backend.get_value(hp)\n", "issue": "Make the HyperparameterScheduler compatible with the CaseOptimizer\n### Feature motivation\r\nThe HyperparameterScheduler is not compatible with the CaseOptimizer since the hyperparameters are attributes of the optimizers inside the CaseOptimizer. \r\n\r\n### Feature description\r\nI propose one of the two possible solutions: \r\nEither we could give HyperparameterScheduler the optimizer as an argument. It could be called via ```  HyperparameterScheduler(schedule, hyperparameter, optimizer, verbose=0) ``` and the right optimizer inside the CaseOptimizer can be addressed. (@koenhelwegen) \r\n\r\nMy second proposal would be to search the CaseOptimizer for optimizers that have the hyperparameter as attribute. Then the schedule can be applied to this optimizer only. The downside of this would be that in case there are two optimizers inside the CaseOptimizer that have a hyperparameter with the same name the schedule would be applied to both of them. I do not think this would happen very often but it could definitively be an issue. See code below for my second proposal. \r\n### Feature implementation\r\n``` python \r\nclass HyperparameterScheduler(tf.keras.callbacks.Callback):\r\n    \"\"\"Generic hyperparameter scheduler.\r\n    # Arguments\r\n    schedule: a function that takes an epoch index as input\r\n        (integer, indexed from 0) and returns a new hyperparameter as output.\r\n    hyperparameter: str. the name of the hyperparameter to be scheduled.\r\n    verbose: int. 0: quiet, 1: update messages.\r\n    \"\"\"\r\n\r\n    def __init__(self, schedule, hyperparameter, verbose=0):\r\n        super(HyperparameterScheduler, self).__init__()\r\n        self.schedule = schedule\r\n        self.hyperparameter = hyperparameter\r\n        self.verbose = verbose\r\n\r\n    def on_epoch_begin(self, epoch, logs=None):\r\n        for op in self.model.optimizer.optimizers:\r\n            if hasattr(op, self.hyperparameter):\r\n\r\n                hp = getattr(op, self.hyperparameter)\r\n                try:  # new API\r\n                    hyperparameter_val = tf.keras.backend.get_value(hp)\r\n                    hyperparameter_val = self.schedule(epoch, hyperparameter_val)\r\n                except TypeError:  # Support for old API for backward compatibility\r\n                    hyperparameter_val = self.schedule(epoch)\r\n\r\n                tf.keras.backend.set_value(hp, hyperparameter_val)\r\n\r\n                if self.verbose > 0:\r\n                    print(\r\n                        f\"Epoch {epoch + 1}: {self.hyperparameter} changning to {tf.keras.backend.get_value(hp)}.\"\r\n                    )\r\n\r\n    def on_epoch_end(self, epoch, logs=None):\r\n        logs = logs or {}\r\n        for op in self.model.optimizer.optimizers:\r\n            if hasattr(op, self.hyperparameter):\r\n                hp = getattr(op, self.hyperparameter)\r\n                logs[self.hyperparameter] = tf.keras.backend.get_value(hp)\r\n```\n", "before_files": [{"content": "import tensorflow as tf\n\n\nclass HyperparameterScheduler(tf.keras.callbacks.Callback):\n    \"\"\"Generic hyperparameter scheduler.\n\n    # Arguments\n    schedule: a function that takes an epoch index as input\n        (integer, indexed from 0) and returns a new hyperparameter as output.\n    hyperparameter: str. the name of the hyperparameter to be scheduled.\n    verbose: int. 0: quiet, 1: update messages.\n    \"\"\"\n\n    def __init__(self, schedule, hyperparameter, verbose=0):\n        super(HyperparameterScheduler, self).__init__()\n        self.schedule = schedule\n        self.hyperparameter = hyperparameter\n        self.verbose = verbose\n\n    def on_epoch_begin(self, epoch, logs=None):\n        if not hasattr(self.model.optimizer, self.hyperparameter):\n            raise ValueError(\n                f'Optimizer must have a \"{self.hyperparameter}\" attribute.'\n            )\n\n        hp = getattr(self.model.optimizer, self.hyperparameter)\n        try:  # new API\n            hyperparameter_val = tf.keras.backend.get_value(hp)\n            hyperparameter_val = self.schedule(epoch, hyperparameter_val)\n        except TypeError:  # Support for old API for backward compatibility\n            hyperparameter_val = self.schedule(epoch)\n\n        tf.keras.backend.set_value(hp, hyperparameter_val)\n\n        if self.verbose > 0:\n            print(\n                f\"Epoch {epoch + 1}: {self.hyperparameter} changning to {tf.keras.backend.get_value(hp)}.\"\n            )\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        hp = getattr(self.model.optimizer, self.hyperparameter)\n        logs[self.hyperparameter] = tf.keras.backend.get_value(hp)\n", "path": "larq/callbacks.py"}], "after_files": [{"content": "import tensorflow as tf\n\n\nclass HyperparameterScheduler(tf.keras.callbacks.Callback):\n    \"\"\"Generic hyperparameter scheduler.\n\n    !!! example\n        ```python\n        bop = lq.optimizers.Bop(threshold=1e-6, gamma=1e-3)\n        adam = tf.keras.optimizers.Adam(0.01)\n        optimizer = lq.optimizers.CaseOptimizer(\n            (lq.optimizers.Bop.is_binary_variable, bop), default_optimizer=adam,\n        )\n        callbacks = [\n            HyperparameterScheduler(lambda x: 0.001 * (0.1 ** (x // 30)), \"gamma\", bop)\n        ]\n        ```\n    # Arguments\n    optimizer: the optimizer that contains the hyperparameter that will be scheduled.\n        Defaults to `self.model.optimizer` if `optimizer == None`.\n    schedule: a function that takes an epoch index as input\n        (integer, indexed from 0) and returns a new hyperparameter as output.\n    hyperparameter: str. the name of the hyperparameter to be scheduled.\n    verbose: int. 0: quiet, 1: update messages.\n    \"\"\"\n\n    def __init__(self, schedule, hyperparameter, optimizer=None, verbose=0):\n        super(HyperparameterScheduler, self).__init__()\n        self.optimizer = optimizer if optimizer else self.model.optimizer\n        self.schedule = schedule\n        self.hyperparameter = hyperparameter\n        self.verbose = verbose\n\n    def on_epoch_begin(self, epoch, logs=None):\n        if not hasattr(self.optimizer, self.hyperparameter):\n            raise ValueError(\n                f'Optimizer must have a \"{self.hyperparameter}\" attribute.'\n            )\n\n        hp = getattr(self.optimizer, self.hyperparameter)\n        try:  # new API\n            hyperparameter_val = tf.keras.backend.get_value(hp)\n            hyperparameter_val = self.schedule(epoch, hyperparameter_val)\n        except TypeError:  # Support for old API for backward compatibility\n            hyperparameter_val = self.schedule(epoch)\n\n        tf.keras.backend.set_value(hp, hyperparameter_val)\n\n        if self.verbose > 0:\n            print(\n                f\"Epoch {epoch + 1}: {self.hyperparameter} changing to {tf.keras.backend.get_value(hp)}.\"\n            )\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        hp = getattr(self.optimizer, self.hyperparameter)\n        logs[self.hyperparameter] = tf.keras.backend.get_value(hp)\n", "path": "larq/callbacks.py"}]}