We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
CaseOptimizer should throw if optimizer matches no variable
### Feature motivation
We are changing the default behaviour of Bop in #442. If users are not careful this could lead to hard to debug errors when no variable matches any optimizer.

### Feature description
We should throw if one optimizer in used by `CaseOptimizer` doesn't receive any variables.

### Feature implementation
We should throw if any of the [`len(opt_grads_and_vars) == 0`](https://github.com/larq/larq/blob/c44ed976812edc2cc7d60c98155513942fc3af98/larq/optimizers.py#L140).

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `larq/optimizers.py`
Content:
```
1 """Neural networks with extremely low-precision weights and activations, such as
2 Binarized Neural Networks (BNNs), usually contain a mix of low-precision weights (e.g.
3 1-bit) and  higher-precision weights (e.g. 8-bit, 16-bit, or 32-bit). Examples of this
4 include the first and last layers of image classificiation models, which have
5 higher-precision weights in most BNN architectures from the literature.
6 
7 Training a BNN, then, consists of optimizing both low-precision and higher-precision
8 weights. In `larq`, we provide a mechanism to target different bit-precision variables
9 with different optimizers using the `CaseOptimizer` class. Modeled after the
10 [`tf.case`](https://www.tensorflow.org/api_docs/python/tf/case) signature,
11 `CaseOptimizer` accepts pairs of predicates and optimizers. A predicate, given a
12 variable, decides whether its optimizer should train that variable.
13 
14 A `CaseOptimizer` behaves much like any other
15 [Keras optimizer](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers), and
16 once you instantiate it you can pass it to your `model.compile()` as usual. To
17 instantiate a `CaseOptimzer`, pass one or a list of `(predicate, optimizer)` tuples,
18 along with a `default` optimizer which trains any variables not claimed by another
19 optimizer. A variable may not be claimed by more than one optimizer's predicate.
20 
21 !!! example
22     ```python
23     no_op_quantizer = lq.quantizers.NoOpQuantizer(precision=1)
24     layer = lq.layers.QuantDense(16, kernel_quantizer=no_op_quantizer)
25 
26     case_optimizer = lq.optimizers.CaseOptimizer(
27         (
28             lq.optimizers.Bop.is_binary_variable,  # predicate
29             lq.optimizers.Bop(threshold=1e-6, gamma=1e-3),  # optimizer
30         ),
31         default_optimizer=tf.keras.optimizers.Adam(0.01),
32     )
33     ```
34 """
35 
36 
37 import warnings
38 from copy import deepcopy
39 from typing import Callable, Optional, Tuple
40 
41 import tensorflow as tf
42 
43 import larq as lq
44 from larq import utils
45 
46 __all__ = ["Bop", "CaseOptimizer"]
47 
48 
49 @utils.register_keras_custom_object
50 class CaseOptimizer(tf.keras.optimizers.Optimizer):
51     """An optmizer wrapper that applies different optimizers to a subset of variables.
52 
53     An optimizer is used to train a variable iff its accompanying predicate evaluates to
54     `True`.
55 
56     For each variable, at most one optimizer's predicate may evaluate to `True`. If no
57     optimizer's predicate evaluates to `True` for a variable, it is trained with the
58     `default_optimizer`. If a variable is claimed by no optimizers and
59     `default_optimizer == None`, the variable is not trained.
60 
61     # Arguments
62     predicate_optimizer_pairs: One or more `(pred, tf.keras.optimizers.Optimizer)` pairs,
63         where `pred`  takes one `tf.Variable` as argument and returns `True` if the
64         optimizer should be used for that variable, e.g. `pred(var) == True`.
65     default_optimizer: A `tf.keras.optimizers.Optimizer` to be applied to any variable
66         not claimed by any other optimizer. (Must be passed as keyword argument.)
67     """
68 
69     def __init__(
70         self,
71         *predicate_optimizer_pairs: Tuple[
72             Callable[[tf.Variable], bool], tf.keras.optimizers.Optimizer
73         ],
74         default_optimizer: Optional[tf.keras.optimizers.Optimizer] = None,
75         name: str = "optimizer_case",
76     ):
77         super().__init__(name=name)
78 
79         # Type checks for (predicate, optimizer) pairs
80         for i, (predicate, optimizer) in enumerate(predicate_optimizer_pairs):
81             if not callable(predicate):
82                 raise TypeError(
83                     f"Expected callable predicate at `predicate_optimizer_pairs[{i}][0]` but got `{type(predicate)}`."
84                 )
85             if not isinstance(optimizer, tf.keras.optimizers.Optimizer):
86                 raise TypeError(
87                     f"Expected `tf.keras.optimizers.Optimizer` at `predicate_optimizer_pairs[{i}][1]` but got `{type(optimizer)}`."
88                 )
89 
90         # Type check for default optimizers
91         if default_optimizer is not None and not isinstance(
92             default_optimizer, tf.keras.optimizers.Optimizer
93         ):
94             raise TypeError(
95                 f"Expected `tf.keras.optimizers.Optimizer` for `default_optimizer` but got `{type(default_optimizer)}`."
96             )
97 
98         self.pred_opt_pairs = predicate_optimizer_pairs
99         self.default = default_optimizer
100 
101         self.var_opt_mapping = None
102 
103         # List of optimizers ending in `default_optimizer`, for easier internal access
104         self.optimizers = [opt for (_, opt) in self.pred_opt_pairs]
105 
106         if self.default:
107             self.optimizers.append(self.default)
108             self.DEFAULT_OPT_INDEX = len(self.pred_opt_pairs)
109 
110         # Track optimizers to support reloading via tf.train.Checkpoint
111         for i, optimizer in enumerate(self.optimizers):
112             self._track_trackable(optimizer, name=f"optimizer_{i}")
113 
114     @property
115     def weights(self):
116         weights = []
117         for optimizer in self.optimizers:
118             weights.extend(optimizer.weights)
119         return weights
120 
121     def apply_gradients(self, grads_and_vars, name: Optional[str] = None):
122         """Apply gradients to variables for each optimizer.
123 
124         On the first call to `apply_gradients()`, compute the mapping from variables to
125         optimizers and cache it in the `self.var_opt_mapping` dict for serialization and
126         faster access.
127         """
128 
129         if self.var_opt_mapping is None:
130             # Convert `grads_and_vars` to list so we can iterate multiple times over it
131             grads_and_vars = list(grads_and_vars)
132             self._compute_var_opt_mapping(grads_and_vars)
133 
134         # Split gradients and variables into a separate list for each optimizer
135         grad_var_lists = [[] for _ in range(len(self.pred_opt_pairs) + 1)]
136         for grad, var in grads_and_vars:
137             if var.name in self.var_opt_mapping:
138                 grad_var_lists[self.var_opt_mapping[var.name]].append((grad, var))
139 
140         # Apply gradients to each optimizer
141         train_ops = [
142             optimizer.apply_gradients(opt_grads_and_vars)
143             for optimizer, opt_grads_and_vars in zip(self.optimizers, grad_var_lists)
144         ]
145 
146         return tf.group(*train_ops, name="train_with_group")
147 
148     def get_config(self):
149         optimizer_configs = [opt.get_config() for (_, opt) in self.pred_opt_pairs]
150         default_config = self.default.get_config()
151 
152         config = {
153             "optimizer_configs": [
154                 {"class_name": optimizer_config["name"], "config": optimizer_config}
155                 for optimizer_config in optimizer_configs
156             ],
157             "default_config": {
158                 "class_name": default_config["name"],
159                 "config": default_config,
160             },
161             "var_opt_mapping": self.var_opt_mapping,  # serialized instead of `pred`s
162         }
163         return {**super().get_config(), **config}
164 
165     @classmethod
166     def from_config(cls, original_config, custom_objects=None):
167         config = deepcopy(original_config)
168 
169         case_optimizer = cls(
170             *[  # `(pred, opt)` tuples
171                 (
172                     lambda _: False,  # placeholder callable (`pred` is not serialized)
173                     tf.keras.optimizers.deserialize(  # optimizer `opt`
174                         opt_config, custom_objects=custom_objects
175                     ),
176                 )
177                 for opt_config in config["optimizer_configs"]
178             ],
179             default_optimizer=tf.keras.optimizers.deserialize(
180                 config["default_config"], custom_objects=custom_objects
181             ),
182         )
183 
184         # Since we no longer have the `pred`s, we set the mapping explicitly
185         case_optimizer.var_opt_mapping = config["var_opt_mapping"]
186 
187         return case_optimizer
188 
189     def _compute_var_opt_mapping(self, grads_and_vars):
190         """Compute a unique mapping from variables to optimizer indices."""
191 
192         self.var_opt_mapping = {}
193 
194         for grad, var in grads_and_vars:
195             num_optimizers = 0
196 
197             # Find the optimizer(s) that want to claim this variable
198             for optimizer_index, (predicate, _) in enumerate(self.pred_opt_pairs):
199                 if predicate(var):
200                     self.var_opt_mapping[var.name] = optimizer_index
201                     num_optimizers += 1
202 
203             if num_optimizers > 1:
204                 raise ValueError(f"Variable `{var}` claimed by multiple optimizers.")
205             if num_optimizers == 0:
206                 if self.default is not None:
207                     self.var_opt_mapping[var.name] = self.DEFAULT_OPT_INDEX
208                 else:
209                     warnings.warn(
210                         f"No `default_optimizer` provided to train variable `{var}`."
211                     )
212 
213 
214 @utils.register_keras_custom_object
215 class Bop(tf.keras.optimizers.Optimizer):
216     """Binary optimizer (Bop).
217 
218     Bop is a latent-free optimizer for Binarized Neural Networks (BNNs) and
219     Binary Weight Networks (BWN).
220 
221     Bop maintains an exponential moving average of the gradients controlled by
222     `gamma`. If this average exceeds the `threshold`, a weight is flipped.
223 
224     The hyperparameter `gamma` is somewhat analogues to the learning rate in
225     SGD methods: a high `gamma` results in rapid convergence but also makes
226     training more noisy.
227 
228     Note that the default `threshold` is not optimal for all situations.
229     Setting the threshold too high results in little learning, while setting it
230     too low results in overly noisy behaviour.
231 
232     !!! warning
233         The `is_binary_variable` check of this optimizer will only target variables that
234         have been explicitly marked as being binary using `NoOpQuantizer(precision=1)`.
235 
236     !!! example
237         ```python
238         no_op_quantizer = lq.quantizers.NoOpQuantizer(precision=1)
239         layer = lq.layers.QuantDense(16, kernel_quantizer=no_op_quantizer)
240 
241         optimizer = lq.optimizers.CaseOptimizer(
242             (
243                 lq.optimizers.Bop.is_binary_variable,
244                 lq.optimizers.Bop(),
245             ),
246             default_optimizer=tf.keras.optimizers.Adam(0.01),  # for FP weights
247         )
248         ```
249 
250     # Arguments
251     threshold: magnitude of average gradient signal required to flip a weight.
252     gamma: the adaptivity rate.
253     name: name of the optimizer.
254 
255     # References
256     - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://papers.nips.cc/paper/8971-latent-weights-do-not-exist-rethinking-binarized-neural-network-optimization)
257     """
258 
259     def __init__(
260         self, threshold: float = 1e-8, gamma: float = 1e-4, name: str = "Bop", **kwargs
261     ):
262         super().__init__(name=name, **kwargs)
263 
264         self._set_hyper("threshold", threshold)
265         self._set_hyper("gamma", gamma)
266 
267     def _create_slots(self, var_list):
268         for var in var_list:
269             self.add_slot(var, "m")
270 
271     def _get_decayed_hyper(self, name: str, var_dtype):
272         hyper = self._get_hyper(name, var_dtype)
273         if isinstance(hyper, tf.keras.optimizers.schedules.LearningRateSchedule):
274             local_step = tf.cast(self.iterations, var_dtype)
275             hyper = tf.cast(hyper(local_step), var_dtype)
276         return hyper
277 
278     def _resource_apply_dense(self, grad, var):
279         var_dtype = var.dtype.base_dtype
280         gamma = self._get_decayed_hyper("gamma", var_dtype)
281         threshold = self._get_decayed_hyper("threshold", var_dtype)
282         m = self.get_slot(var, "m")
283 
284         m_t = m.assign_add(gamma * (grad - m))
285         var_t = lq.math.sign(-tf.sign(var * m_t - threshold) * var)
286         return var.assign(var_t).op
287 
288     def _resource_apply_sparse(self, grad, var, indices):
289         raise NotImplementedError()
290 
291     def get_config(self):
292         config = {
293             "threshold": self._serialize_hyperparameter("threshold"),
294             "gamma": self._serialize_hyperparameter("gamma"),
295         }
296         return {**super().get_config(), **config}
297 
298     @classmethod
299     def from_config(cls, config, custom_objects=None):
300         for hyper in ("gamma", "threshold"):
301             if hyper in config and isinstance(config[hyper], dict):
302                 config[hyper] = tf.keras.optimizers.schedules.deserialize(
303                     config[hyper], custom_objects=custom_objects
304                 )
305         return cls(**config)
306 
307     @staticmethod
308     def is_binary_variable(var: tf.Variable) -> bool:
309         """Returns `True` for variables with `var.precision == 1`.
310 
311         This is an example of a predictate that can be used by the `CaseOptimizer`.
312 
313         # Arguments
314         var: a `tf.Variable`.
315         """
316         return getattr(var, "precision", 32) == 1
317
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/larq/optimizers.py b/larq/optimizers.py
--- a/larq/optimizers.py
+++ b/larq/optimizers.py
@@ -210,6 +210,13 @@
                         f"No `default_optimizer` provided to train variable `{var}`."
                     )
 
+        # Make sure that each optimizer touches at least one variable
+        for optimizer_index, (_, optimizer) in enumerate(self.pred_opt_pairs):
+            if optimizer_index not in self.var_opt_mapping.values():
+                raise ValueError(
+                    f"Optimizer `{optimizer}` did not claim any variables."
+                )
+
 
 @utils.register_keras_custom_object
 class Bop(tf.keras.optimizers.Optimizer):

{"golden_diff": "diff --git a/larq/optimizers.py b/larq/optimizers.py\n--- a/larq/optimizers.py\n+++ b/larq/optimizers.py\n@@ -210,6 +210,13 @@\n                         f\"No `default_optimizer` provided to train variable `{var}`.\"\n                     )\n \n+        # Make sure that each optimizer touches at least one variable\n+        for optimizer_index, (_, optimizer) in enumerate(self.pred_opt_pairs):\n+            if optimizer_index not in self.var_opt_mapping.values():\n+                raise ValueError(\n+                    f\"Optimizer `{optimizer}` did not claim any variables.\"\n+                )\n+\n \n @utils.register_keras_custom_object\n class Bop(tf.keras.optimizers.Optimizer):\n", "issue": "CaseOptimizer should throw if optimizer matches no variable\n### Feature motivation\r\nWe are changing the default behaviour of Bop in #442. If users are not careful this could lead to hard to debug errors when no variable matches any optimizer.\r\n\r\n### Feature description\r\nWe should throw if one optimizer in used by `CaseOptimizer` doesn't receive any variables.\r\n\r\n### Feature implementation\r\nWe should throw if any of the [`len(opt_grads_and_vars) == 0`](https://github.com/larq/larq/blob/c44ed976812edc2cc7d60c98155513942fc3af98/larq/optimizers.py#L140).\n", "before_files": [{"content": "\"\"\"Neural networks with extremely low-precision weights and activations, such as\nBinarized Neural Networks (BNNs), usually contain a mix of low-precision weights (e.g.\n1-bit) and  higher-precision weights (e.g. 8-bit, 16-bit, or 32-bit). Examples of this\ninclude the first and last layers of image classificiation models, which have\nhigher-precision weights in most BNN architectures from the literature.\n\nTraining a BNN, then, consists of optimizing both low-precision and higher-precision\nweights. In `larq`, we provide a mechanism to target different bit-precision variables\nwith different optimizers using the `CaseOptimizer` class. Modeled after the\n[`tf.case`](https://www.tensorflow.org/api_docs/python/tf/case) signature,\n`CaseOptimizer` accepts pairs of predicates and optimizers. A predicate, given a\nvariable, decides whether its optimizer should train that variable.\n\nA `CaseOptimizer` behaves much like any other\n[Keras optimizer](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers), and\nonce you instantiate it you can pass it to your `model.compile()` as usual. To\ninstantiate a `CaseOptimzer`, pass one or a list of `(predicate, optimizer)` tuples,\nalong with a `default` optimizer which trains any variables not claimed by another\noptimizer. A variable may not be claimed by more than one optimizer's predicate.\n\n!!! example\n    ```python\n    no_op_quantizer = lq.quantizers.NoOpQuantizer(precision=1)\n    layer = lq.layers.QuantDense(16, kernel_quantizer=no_op_quantizer)\n\n    case_optimizer = lq.optimizers.CaseOptimizer(\n        (\n            lq.optimizers.Bop.is_binary_variable,  # predicate\n            lq.optimizers.Bop(threshold=1e-6, gamma=1e-3),  # optimizer\n        ),\n        default_optimizer=tf.keras.optimizers.Adam(0.01),\n    )\n    ```\n\"\"\"\n\n\nimport warnings\nfrom copy import deepcopy\nfrom typing import Callable, Optional, Tuple\n\nimport tensorflow as tf\n\nimport larq as lq\nfrom larq import utils\n\n__all__ = [\"Bop\", \"CaseOptimizer\"]\n\n\n@utils.register_keras_custom_object\nclass CaseOptimizer(tf.keras.optimizers.Optimizer):\n    \"\"\"An optmizer wrapper that applies different optimizers to a subset of variables.\n\n    An optimizer is used to train a variable iff its accompanying predicate evaluates to\n    `True`.\n\n    For each variable, at most one optimizer's predicate may evaluate to `True`. If no\n    optimizer's predicate evaluates to `True` for a variable, it is trained with the\n    `default_optimizer`. If a variable is claimed by no optimizers and\n    `default_optimizer == None`, the variable is not trained.\n\n    # Arguments\n    predicate_optimizer_pairs: One or more `(pred, tf.keras.optimizers.Optimizer)` pairs,\n        where `pred`  takes one `tf.Variable` as argument and returns `True` if the\n        optimizer should be used for that variable, e.g. `pred(var) == True`.\n    default_optimizer: A `tf.keras.optimizers.Optimizer` to be applied to any variable\n        not claimed by any other optimizer. (Must be passed as keyword argument.)\n    \"\"\"\n\n    def __init__(\n        self,\n        *predicate_optimizer_pairs: Tuple[\n            Callable[[tf.Variable], bool], tf.keras.optimizers.Optimizer\n        ],\n        default_optimizer: Optional[tf.keras.optimizers.Optimizer] = None,\n        name: str = \"optimizer_case\",\n    ):\n        super().__init__(name=name)\n\n        # Type checks for (predicate, optimizer) pairs\n        for i, (predicate, optimizer) in enumerate(predicate_optimizer_pairs):\n            if not callable(predicate):\n                raise TypeError(\n                    f\"Expected callable predicate at `predicate_optimizer_pairs[{i}][0]` but got `{type(predicate)}`.\"\n                )\n            if not isinstance(optimizer, tf.keras.optimizers.Optimizer):\n                raise TypeError(\n                    f\"Expected `tf.keras.optimizers.Optimizer` at `predicate_optimizer_pairs[{i}][1]` but got `{type(optimizer)}`.\"\n                )\n\n        # Type check for default optimizers\n        if default_optimizer is not None and not isinstance(\n            default_optimizer, tf.keras.optimizers.Optimizer\n        ):\n            raise TypeError(\n                f\"Expected `tf.keras.optimizers.Optimizer` for `default_optimizer` but got `{type(default_optimizer)}`.\"\n            )\n\n        self.pred_opt_pairs = predicate_optimizer_pairs\n        self.default = default_optimizer\n\n        self.var_opt_mapping = None\n\n        # List of optimizers ending in `default_optimizer`, for easier internal access\n        self.optimizers = [opt for (_, opt) in self.pred_opt_pairs]\n\n        if self.default:\n            self.optimizers.append(self.default)\n            self.DEFAULT_OPT_INDEX = len(self.pred_opt_pairs)\n\n        # Track optimizers to support reloading via tf.train.Checkpoint\n        for i, optimizer in enumerate(self.optimizers):\n            self._track_trackable(optimizer, name=f\"optimizer_{i}\")\n\n    @property\n    def weights(self):\n        weights = []\n        for optimizer in self.optimizers:\n            weights.extend(optimizer.weights)\n        return weights\n\n    def apply_gradients(self, grads_and_vars, name: Optional[str] = None):\n        \"\"\"Apply gradients to variables for each optimizer.\n\n        On the first call to `apply_gradients()`, compute the mapping from variables to\n        optimizers and cache it in the `self.var_opt_mapping` dict for serialization and\n        faster access.\n        \"\"\"\n\n        if self.var_opt_mapping is None:\n            # Convert `grads_and_vars` to list so we can iterate multiple times over it\n            grads_and_vars = list(grads_and_vars)\n            self._compute_var_opt_mapping(grads_and_vars)\n\n        # Split gradients and variables into a separate list for each optimizer\n        grad_var_lists = [[] for _ in range(len(self.pred_opt_pairs) + 1)]\n        for grad, var in grads_and_vars:\n            if var.name in self.var_opt_mapping:\n                grad_var_lists[self.var_opt_mapping[var.name]].append((grad, var))\n\n        # Apply gradients to each optimizer\n        train_ops = [\n            optimizer.apply_gradients(opt_grads_and_vars)\n            for optimizer, opt_grads_and_vars in zip(self.optimizers, grad_var_lists)\n        ]\n\n        return tf.group(*train_ops, name=\"train_with_group\")\n\n    def get_config(self):\n        optimizer_configs = [opt.get_config() for (_, opt) in self.pred_opt_pairs]\n        default_config = self.default.get_config()\n\n        config = {\n            \"optimizer_configs\": [\n                {\"class_name\": optimizer_config[\"name\"], \"config\": optimizer_config}\n                for optimizer_config in optimizer_configs\n            ],\n            \"default_config\": {\n                \"class_name\": default_config[\"name\"],\n                \"config\": default_config,\n            },\n            \"var_opt_mapping\": self.var_opt_mapping,  # serialized instead of `pred`s\n        }\n        return {**super().get_config(), **config}\n\n    @classmethod\n    def from_config(cls, original_config, custom_objects=None):\n        config = deepcopy(original_config)\n\n        case_optimizer = cls(\n            *[  # `(pred, opt)` tuples\n                (\n                    lambda _: False,  # placeholder callable (`pred` is not serialized)\n                    tf.keras.optimizers.deserialize(  # optimizer `opt`\n                        opt_config, custom_objects=custom_objects\n                    ),\n                )\n                for opt_config in config[\"optimizer_configs\"]\n            ],\n            default_optimizer=tf.keras.optimizers.deserialize(\n                config[\"default_config\"], custom_objects=custom_objects\n            ),\n        )\n\n        # Since we no longer have the `pred`s, we set the mapping explicitly\n        case_optimizer.var_opt_mapping = config[\"var_opt_mapping\"]\n\n        return case_optimizer\n\n    def _compute_var_opt_mapping(self, grads_and_vars):\n        \"\"\"Compute a unique mapping from variables to optimizer indices.\"\"\"\n\n        self.var_opt_mapping = {}\n\n        for grad, var in grads_and_vars:\n            num_optimizers = 0\n\n            # Find the optimizer(s) that want to claim this variable\n            for optimizer_index, (predicate, _) in enumerate(self.pred_opt_pairs):\n                if predicate(var):\n                    self.var_opt_mapping[var.name] = optimizer_index\n                    num_optimizers += 1\n\n            if num_optimizers > 1:\n                raise ValueError(f\"Variable `{var}` claimed by multiple optimizers.\")\n            if num_optimizers == 0:\n                if self.default is not None:\n                    self.var_opt_mapping[var.name] = self.DEFAULT_OPT_INDEX\n                else:\n                    warnings.warn(\n                        f\"No `default_optimizer` provided to train variable `{var}`.\"\n                    )\n\n\n@utils.register_keras_custom_object\nclass Bop(tf.keras.optimizers.Optimizer):\n    \"\"\"Binary optimizer (Bop).\n\n    Bop is a latent-free optimizer for Binarized Neural Networks (BNNs) and\n    Binary Weight Networks (BWN).\n\n    Bop maintains an exponential moving average of the gradients controlled by\n    `gamma`. If this average exceeds the `threshold`, a weight is flipped.\n\n    The hyperparameter `gamma` is somewhat analogues to the learning rate in\n    SGD methods: a high `gamma` results in rapid convergence but also makes\n    training more noisy.\n\n    Note that the default `threshold` is not optimal for all situations.\n    Setting the threshold too high results in little learning, while setting it\n    too low results in overly noisy behaviour.\n\n    !!! warning\n        The `is_binary_variable` check of this optimizer will only target variables that\n        have been explicitly marked as being binary using `NoOpQuantizer(precision=1)`.\n\n    !!! example\n        ```python\n        no_op_quantizer = lq.quantizers.NoOpQuantizer(precision=1)\n        layer = lq.layers.QuantDense(16, kernel_quantizer=no_op_quantizer)\n\n        optimizer = lq.optimizers.CaseOptimizer(\n            (\n                lq.optimizers.Bop.is_binary_variable,\n                lq.optimizers.Bop(),\n            ),\n            default_optimizer=tf.keras.optimizers.Adam(0.01),  # for FP weights\n        )\n        ```\n\n    # Arguments\n    threshold: magnitude of average gradient signal required to flip a weight.\n    gamma: the adaptivity rate.\n    name: name of the optimizer.\n\n    # References\n    - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://papers.nips.cc/paper/8971-latent-weights-do-not-exist-rethinking-binarized-neural-network-optimization)\n    \"\"\"\n\n    def __init__(\n        self, threshold: float = 1e-8, gamma: float = 1e-4, name: str = \"Bop\", **kwargs\n    ):\n        super().__init__(name=name, **kwargs)\n\n        self._set_hyper(\"threshold\", threshold)\n        self._set_hyper(\"gamma\", gamma)\n\n    def _create_slots(self, var_list):\n        for var in var_list:\n            self.add_slot(var, \"m\")\n\n    def _get_decayed_hyper(self, name: str, var_dtype):\n        hyper = self._get_hyper(name, var_dtype)\n        if isinstance(hyper, tf.keras.optimizers.schedules.LearningRateSchedule):\n            local_step = tf.cast(self.iterations, var_dtype)\n            hyper = tf.cast(hyper(local_step), var_dtype)\n        return hyper\n\n    def _resource_apply_dense(self, grad, var):\n        var_dtype = var.dtype.base_dtype\n        gamma = self._get_decayed_hyper(\"gamma\", var_dtype)\n        threshold = self._get_decayed_hyper(\"threshold\", var_dtype)\n        m = self.get_slot(var, \"m\")\n\n        m_t = m.assign_add(gamma * (grad - m))\n        var_t = lq.math.sign(-tf.sign(var * m_t - threshold) * var)\n        return var.assign(var_t).op\n\n    def _resource_apply_sparse(self, grad, var, indices):\n        raise NotImplementedError()\n\n    def get_config(self):\n        config = {\n            \"threshold\": self._serialize_hyperparameter(\"threshold\"),\n            \"gamma\": self._serialize_hyperparameter(\"gamma\"),\n        }\n        return {**super().get_config(), **config}\n\n    @classmethod\n    def from_config(cls, config, custom_objects=None):\n        for hyper in (\"gamma\", \"threshold\"):\n            if hyper in config and isinstance(config[hyper], dict):\n                config[hyper] = tf.keras.optimizers.schedules.deserialize(\n                    config[hyper], custom_objects=custom_objects\n                )\n        return cls(**config)\n\n    @staticmethod\n    def is_binary_variable(var: tf.Variable) -> bool:\n        \"\"\"Returns `True` for variables with `var.precision == 1`.\n\n        This is an example of a predictate that can be used by the `CaseOptimizer`.\n\n        # Arguments\n        var: a `tf.Variable`.\n        \"\"\"\n        return getattr(var, \"precision\", 32) == 1\n", "path": "larq/optimizers.py"}], "after_files": [{"content": "\"\"\"Neural networks with extremely low-precision weights and activations, such as\nBinarized Neural Networks (BNNs), usually contain a mix of low-precision weights (e.g.\n1-bit) and  higher-precision weights (e.g. 8-bit, 16-bit, or 32-bit). Examples of this\ninclude the first and last layers of image classificiation models, which have\nhigher-precision weights in most BNN architectures from the literature.\n\nTraining a BNN, then, consists of optimizing both low-precision and higher-precision\nweights. In `larq`, we provide a mechanism to target different bit-precision variables\nwith different optimizers using the `CaseOptimizer` class. Modeled after the\n[`tf.case`](https://www.tensorflow.org/api_docs/python/tf/case) signature,\n`CaseOptimizer` accepts pairs of predicates and optimizers. A predicate, given a\nvariable, decides whether its optimizer should train that variable.\n\nA `CaseOptimizer` behaves much like any other\n[Keras optimizer](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers), and\nonce you instantiate it you can pass it to your `model.compile()` as usual. To\ninstantiate a `CaseOptimzer`, pass one or a list of `(predicate, optimizer)` tuples,\nalong with a `default` optimizer which trains any variables not claimed by another\noptimizer. A variable may not be claimed by more than one optimizer's predicate.\n\n!!! example\n    ```python\n    no_op_quantizer = lq.quantizers.NoOpQuantizer(precision=1)\n    layer = lq.layers.QuantDense(16, kernel_quantizer=no_op_quantizer)\n\n    case_optimizer = lq.optimizers.CaseOptimizer(\n        (\n            lq.optimizers.Bop.is_binary_variable,  # predicate\n            lq.optimizers.Bop(threshold=1e-6, gamma=1e-3),  # optimizer\n        ),\n        default_optimizer=tf.keras.optimizers.Adam(0.01),\n    )\n    ```\n\"\"\"\n\n\nimport warnings\nfrom copy import deepcopy\nfrom typing import Callable, Optional, Tuple\n\nimport tensorflow as tf\n\nimport larq as lq\nfrom larq import utils\n\n__all__ = [\"Bop\", \"CaseOptimizer\"]\n\n\n@utils.register_keras_custom_object\nclass CaseOptimizer(tf.keras.optimizers.Optimizer):\n    \"\"\"An optmizer wrapper that applies different optimizers to a subset of variables.\n\n    An optimizer is used to train a variable iff its accompanying predicate evaluates to\n    `True`.\n\n    For each variable, at most one optimizer's predicate may evaluate to `True`. If no\n    optimizer's predicate evaluates to `True` for a variable, it is trained with the\n    `default_optimizer`. If a variable is claimed by no optimizers and\n    `default_optimizer == None`, the variable is not trained.\n\n    # Arguments\n    predicate_optimizer_pairs: One or more `(pred, tf.keras.optimizers.Optimizer)` pairs,\n        where `pred`  takes one `tf.Variable` as argument and returns `True` if the\n        optimizer should be used for that variable, e.g. `pred(var) == True`.\n    default_optimizer: A `tf.keras.optimizers.Optimizer` to be applied to any variable\n        not claimed by any other optimizer. (Must be passed as keyword argument.)\n    \"\"\"\n\n    def __init__(\n        self,\n        *predicate_optimizer_pairs: Tuple[\n            Callable[[tf.Variable], bool], tf.keras.optimizers.Optimizer\n        ],\n        default_optimizer: Optional[tf.keras.optimizers.Optimizer] = None,\n        name: str = \"optimizer_case\",\n    ):\n        super().__init__(name=name)\n\n        # Type checks for (predicate, optimizer) pairs\n        for i, (predicate, optimizer) in enumerate(predicate_optimizer_pairs):\n            if not callable(predicate):\n                raise TypeError(\n                    f\"Expected callable predicate at `predicate_optimizer_pairs[{i}][0]` but got `{type(predicate)}`.\"\n                )\n            if not isinstance(optimizer, tf.keras.optimizers.Optimizer):\n                raise TypeError(\n                    f\"Expected `tf.keras.optimizers.Optimizer` at `predicate_optimizer_pairs[{i}][1]` but got `{type(optimizer)}`.\"\n                )\n\n        # Type check for default optimizers\n        if default_optimizer is not None and not isinstance(\n            default_optimizer, tf.keras.optimizers.Optimizer\n        ):\n            raise TypeError(\n                f\"Expected `tf.keras.optimizers.Optimizer` for `default_optimizer` but got `{type(default_optimizer)}`.\"\n            )\n\n        self.pred_opt_pairs = predicate_optimizer_pairs\n        self.default = default_optimizer\n\n        self.var_opt_mapping = None\n\n        # List of optimizers ending in `default_optimizer`, for easier internal access\n        self.optimizers = [opt for (_, opt) in self.pred_opt_pairs]\n\n        if self.default:\n            self.optimizers.append(self.default)\n            self.DEFAULT_OPT_INDEX = len(self.pred_opt_pairs)\n\n        # Track optimizers to support reloading via tf.train.Checkpoint\n        for i, optimizer in enumerate(self.optimizers):\n            self._track_trackable(optimizer, name=f\"optimizer_{i}\")\n\n    @property\n    def weights(self):\n        weights = []\n        for optimizer in self.optimizers:\n            weights.extend(optimizer.weights)\n        return weights\n\n    def apply_gradients(self, grads_and_vars, name: Optional[str] = None):\n        \"\"\"Apply gradients to variables for each optimizer.\n\n        On the first call to `apply_gradients()`, compute the mapping from variables to\n        optimizers and cache it in the `self.var_opt_mapping` dict for serialization and\n        faster access.\n        \"\"\"\n\n        if self.var_opt_mapping is None:\n            # Convert `grads_and_vars` to list so we can iterate multiple times over it\n            grads_and_vars = list(grads_and_vars)\n            self._compute_var_opt_mapping(grads_and_vars)\n\n        # Split gradients and variables into a separate list for each optimizer\n        grad_var_lists = [[] for _ in range(len(self.pred_opt_pairs) + 1)]\n        for grad, var in grads_and_vars:\n            if var.name in self.var_opt_mapping:\n                grad_var_lists[self.var_opt_mapping[var.name]].append((grad, var))\n\n        # Apply gradients to each optimizer\n        train_ops = [\n            optimizer.apply_gradients(opt_grads_and_vars)\n            for optimizer, opt_grads_and_vars in zip(self.optimizers, grad_var_lists)\n        ]\n\n        return tf.group(*train_ops, name=\"train_with_group\")\n\n    def get_config(self):\n        optimizer_configs = [opt.get_config() for (_, opt) in self.pred_opt_pairs]\n        default_config = self.default.get_config()\n\n        config = {\n            \"optimizer_configs\": [\n                {\"class_name\": optimizer_config[\"name\"], \"config\": optimizer_config}\n                for optimizer_config in optimizer_configs\n            ],\n            \"default_config\": {\n                \"class_name\": default_config[\"name\"],\n                \"config\": default_config,\n            },\n            \"var_opt_mapping\": self.var_opt_mapping,  # serialized instead of `pred`s\n        }\n        return {**super().get_config(), **config}\n\n    @classmethod\n    def from_config(cls, original_config, custom_objects=None):\n        config = deepcopy(original_config)\n\n        case_optimizer = cls(\n            *[  # `(pred, opt)` tuples\n                (\n                    lambda _: False,  # placeholder callable (`pred` is not serialized)\n                    tf.keras.optimizers.deserialize(  # optimizer `opt`\n                        opt_config, custom_objects=custom_objects\n                    ),\n                )\n                for opt_config in config[\"optimizer_configs\"]\n            ],\n            default_optimizer=tf.keras.optimizers.deserialize(\n                config[\"default_config\"], custom_objects=custom_objects\n            ),\n        )\n\n        # Since we no longer have the `pred`s, we set the mapping explicitly\n        case_optimizer.var_opt_mapping = config[\"var_opt_mapping\"]\n\n        return case_optimizer\n\n    def _compute_var_opt_mapping(self, grads_and_vars):\n        \"\"\"Compute a unique mapping from variables to optimizer indices.\"\"\"\n\n        self.var_opt_mapping = {}\n\n        for grad, var in grads_and_vars:\n            num_optimizers = 0\n\n            # Find the optimizer(s) that want to claim this variable\n            for optimizer_index, (predicate, _) in enumerate(self.pred_opt_pairs):\n                if predicate(var):\n                    self.var_opt_mapping[var.name] = optimizer_index\n                    num_optimizers += 1\n\n            if num_optimizers > 1:\n                raise ValueError(f\"Variable `{var}` claimed by multiple optimizers.\")\n            if num_optimizers == 0:\n                if self.default is not None:\n                    self.var_opt_mapping[var.name] = self.DEFAULT_OPT_INDEX\n                else:\n                    warnings.warn(\n                        f\"No `default_optimizer` provided to train variable `{var}`.\"\n                    )\n\n        # Make sure that each optimizer touches at least one variable\n        for optimizer_index, (_, optimizer) in enumerate(self.pred_opt_pairs):\n            if optimizer_index not in self.var_opt_mapping.values():\n                raise ValueError(\n                    f\"Optimizer `{optimizer}` did not claim any variables.\"\n                )\n\n\n@utils.register_keras_custom_object\nclass Bop(tf.keras.optimizers.Optimizer):\n    \"\"\"Binary optimizer (Bop).\n\n    Bop is a latent-free optimizer for Binarized Neural Networks (BNNs) and\n    Binary Weight Networks (BWN).\n\n    Bop maintains an exponential moving average of the gradients controlled by\n    `gamma`. If this average exceeds the `threshold`, a weight is flipped.\n\n    The hyperparameter `gamma` is somewhat analogues to the learning rate in\n    SGD methods: a high `gamma` results in rapid convergence but also makes\n    training more noisy.\n\n    Note that the default `threshold` is not optimal for all situations.\n    Setting the threshold too high results in little learning, while setting it\n    too low results in overly noisy behaviour.\n\n    !!! warning\n        The `is_binary_variable` check of this optimizer will only target variables that\n        have been explicitly marked as being binary using `NoOpQuantizer(precision=1)`.\n\n    !!! example\n        ```python\n        no_op_quantizer = lq.quantizers.NoOpQuantizer(precision=1)\n        layer = lq.layers.QuantDense(16, kernel_quantizer=no_op_quantizer)\n\n        optimizer = lq.optimizers.CaseOptimizer(\n            (\n                lq.optimizers.Bop.is_binary_variable,\n                lq.optimizers.Bop(),\n            ),\n            default_optimizer=tf.keras.optimizers.Adam(0.01),  # for FP weights\n        )\n        ```\n\n    # Arguments\n    threshold: magnitude of average gradient signal required to flip a weight.\n    gamma: the adaptivity rate.\n    name: name of the optimizer.\n\n    # References\n    - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://papers.nips.cc/paper/8971-latent-weights-do-not-exist-rethinking-binarized-neural-network-optimization)\n    \"\"\"\n\n    def __init__(\n        self, threshold: float = 1e-8, gamma: float = 1e-4, name: str = \"Bop\", **kwargs\n    ):\n        super().__init__(name=name, **kwargs)\n\n        self._set_hyper(\"threshold\", threshold)\n        self._set_hyper(\"gamma\", gamma)\n\n    def _create_slots(self, var_list):\n        for var in var_list:\n            self.add_slot(var, \"m\")\n\n    def _get_decayed_hyper(self, name: str, var_dtype):\n        hyper = self._get_hyper(name, var_dtype)\n        if isinstance(hyper, tf.keras.optimizers.schedules.LearningRateSchedule):\n            local_step = tf.cast(self.iterations, var_dtype)\n            hyper = tf.cast(hyper(local_step), var_dtype)\n        return hyper\n\n    def _resource_apply_dense(self, grad, var):\n        var_dtype = var.dtype.base_dtype\n        gamma = self._get_decayed_hyper(\"gamma\", var_dtype)\n        threshold = self._get_decayed_hyper(\"threshold\", var_dtype)\n        m = self.get_slot(var, \"m\")\n\n        m_t = m.assign_add(gamma * (grad - m))\n        var_t = lq.math.sign(-tf.sign(var * m_t - threshold) * var)\n        return var.assign(var_t).op\n\n    def _resource_apply_sparse(self, grad, var, indices):\n        raise NotImplementedError()\n\n    def get_config(self):\n        config = {\n            \"threshold\": self._serialize_hyperparameter(\"threshold\"),\n            \"gamma\": self._serialize_hyperparameter(\"gamma\"),\n        }\n        return {**super().get_config(), **config}\n\n    @classmethod\n    def from_config(cls, config, custom_objects=None):\n        for hyper in (\"gamma\", \"threshold\"):\n            if hyper in config and isinstance(config[hyper], dict):\n                config[hyper] = tf.keras.optimizers.schedules.deserialize(\n                    config[hyper], custom_objects=custom_objects\n                )\n        return cls(**config)\n\n    @staticmethod\n    def is_binary_variable(var: tf.Variable) -> bool:\n        \"\"\"Returns `True` for variables with `var.precision == 1`.\n\n        This is an example of a predictate that can be used by the `CaseOptimizer`.\n\n        # Arguments\n        var: a `tf.Variable`.\n        \"\"\"\n        return getattr(var, \"precision\", 32) == 1\n", "path": "larq/optimizers.py"}]}