We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Change default hyperparameters of Bop
The default hyperparameters for Bop are not really optimal (i.e. gamma is probably too high for most task): `Bop(threshold=1e-07, gamma=0.01, name='Bop', **kwargs)`

For example in our [paper](https://papers.nips.cc/paper/8971-latent-weights-do-not-exist-rethinking-binarized-neural-network-optimization.pdf) we used a gamma decayed from 1e-4 to 1e-6 and a threshold of 1e-8 for the ImageNet experiments.

I think we should update the default parameters to a more sensible choice. @MariaHeuss What do you think?

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `larq/optimizers.py`
Content:
```
1 """Neural networks with extremely low-precision weights and activations, such as
2 Binarized Neural Networks (BNNs), usually contain a mix of low-precision weights (e.g.
3 1-bit) and  higher-precision weights (e.g. 8-bit, 16-bit, or 32-bit). Examples of this
4 include the first and last layers of image classificiation models, which have
5 higher-precision weights in most BNN architectures from the literature.
6 
7 Training a BNN, then, consists of optimizing both low-precision and higher-precision
8 weights. In `larq`, we provide a mechanism to target different bit-precision variables
9 with different optimizers using the `CaseOptimizer` class. Modeled after the
10 [`tf.case`](https://www.tensorflow.org/api_docs/python/tf/case) signature,
11 `CaseOptimizer` accepts pairs of predicates and optimizers. A predicate, given a
12 variable, decides whether its optimizer should train that variable.
13 
14 A `CaseOptimizer` behaves much like any other
15 [Keras optimizer](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers), and
16 once you instantiate it you can pass it to your `model.compile()` as usual. To
17 instantiate a `CaseOptimzer`, pass one or a list of `(predicate, optimizer)` tuples,
18 along with a `default` optimizer which trains any variables not claimed by another
19 optimizer. A variable may not be claimed by more than one optimizer's predicate.
20 
21 !!! example
22     ```python
23     case_optimizer = lq.optimizers.CaseOptimizer(
24         (
25             lq.optimizers.Bop.is_binary_variable,  # predicate
26             lq.optimizers.Bop(threshold=1e-6, gamma=1e-3),  # optimizer
27         ),
28         default_optimizer=tf.keras.optimizers.Adam(0.01),
29     )
30     ```
31 """
32 
33 
34 import warnings
35 from copy import deepcopy
36 
37 import tensorflow as tf
38 
39 import larq as lq
40 from larq import utils
41 
42 __all__ = ["Bop", "CaseOptimizer"]
43 
44 
45 @utils.register_keras_custom_object
46 class CaseOptimizer(tf.keras.optimizers.Optimizer):
47     """An optmizer wrapper that applies different optimizers to a subset of variables.
48 
49     An optimizer is used to train a variable iff its accompanying predicate evaluates to
50     `True`.
51 
52     For each variable, at most one optimizer's predicate may evaluate to `True`. If no
53     optimizer's predicate evaluates to `True` for a variable, it is trained with the
54     `default_optimizer`. If a variable is claimed by no optimizers and
55     `default_optimizer == None`, the variable is not trained.
56 
57     # Arguments
58     predicate_optimizer_pairs: One or more `(pred, tf.keras.optimizers.Optimzer)` pairs,
59         where `pred`  takes one `tf.Variable` as argument and returns `True` if the
60         optimizer should be used for that variable, e.g. `pred(var) == True`.
61     default_optimizer: A `tf.keras.optimizers.Optimizer` to be applied to any variable
62         not claimed by any other optimizer. (Must be passed as keyword argument.)
63     """
64 
65     def __init__(
66         self, *predicate_optimizer_pairs, default_optimizer=None, name="optimizer_case"
67     ):
68         super().__init__(name=name)
69 
70         # Type checks for (predicate, optimizer) pairs
71         for i, (predicate, optimizer) in enumerate(predicate_optimizer_pairs):
72             if not callable(predicate):
73                 raise TypeError(
74                     f"Expected callable predicate at `predicate_optimizer_pairs[{i}][0]` but got `{type(predicate)}`."
75                 )
76             if not isinstance(optimizer, tf.keras.optimizers.Optimizer):
77                 raise TypeError(
78                     f"Expected `tf.keras.optimizers.Optimizer` at `predicate_optimizer_pairs[{i}][1]` but got `{type(optimizer)}`."
79                 )
80 
81         # Type check for default optimizers
82         if default_optimizer is not None and not isinstance(
83             default_optimizer, tf.keras.optimizers.Optimizer
84         ):
85             raise TypeError(
86                 f"Expected `tf.keras.optimizers.Optimizer` for `default_optimizer` but got `{type(default_optimizer)}`."
87             )
88 
89         self.pred_opt_pairs = predicate_optimizer_pairs
90         self.default = default_optimizer
91 
92         self.var_opt_mapping = None
93 
94         # List of optimizers ending in `default_optimizer`, for easier internal access
95         self.optimizers = [opt for (_, opt) in self.pred_opt_pairs]
96 
97         if self.default:
98             self.optimizers.append(self.default)
99             self.DEFAULT_OPT_INDEX = len(self.pred_opt_pairs)
100 
101     @property
102     def weights(self):
103         weights = []
104         for optimizer in self.optimizers:
105             weights.extend(optimizer.weights)
106         return weights
107 
108     def apply_gradients(self, grads_and_vars, name=None):
109         """Apply gradients to variables for each optimizer.
110 
111         On the first call to `apply_gradients()`, compute the mapping from variables to
112         optimizers and cache it in the `self.var_opt_mapping` dict for serialization and
113         faster access.
114         """
115 
116         if self.var_opt_mapping is None:
117             # Convert `grads_and_vars` to list so we can iterate multiple times over it
118             grads_and_vars = list(grads_and_vars)
119             self._compute_var_opt_mapping(grads_and_vars)
120 
121         # Split gradients and variables into a separate list for each optimizer
122         grad_var_lists = [[] for _ in range(len(self.pred_opt_pairs) + 1)]
123         for grad, var in grads_and_vars:
124             if var.name in self.var_opt_mapping:
125                 grad_var_lists[self.var_opt_mapping[var.name]].append((grad, var))
126 
127         # Apply gradients to each optimizer
128         train_ops = [
129             optimizer.apply_gradients(opt_grads_and_vars)
130             for optimizer, opt_grads_and_vars in zip(self.optimizers, grad_var_lists)
131         ]
132 
133         return tf.group(*train_ops, name="train_with_group")
134 
135     def get_config(self):
136         optimizer_configs = [opt.get_config() for (_, opt) in self.pred_opt_pairs]
137         default_config = self.default.get_config()
138 
139         config = {
140             "optimizer_configs": [
141                 {"class_name": optimizer_config["name"], "config": optimizer_config}
142                 for optimizer_config in optimizer_configs
143             ],
144             "default_config": {
145                 "class_name": default_config["name"],
146                 "config": default_config,
147             },
148             "var_opt_mapping": self.var_opt_mapping,  # serialized instead of `pred`s
149         }
150         return {**super().get_config(), **config}
151 
152     @classmethod
153     def from_config(cls, original_config, custom_objects=None):
154         config = deepcopy(original_config)
155 
156         case_optimizer = cls(
157             *[  # `(pred, opt)` tuples
158                 (
159                     lambda _: False,  # placeholder callable (`pred` is not serialized)
160                     tf.keras.optimizers.deserialize(  # optimizer `opt`
161                         opt_config, custom_objects=custom_objects
162                     ),
163                 )
164                 for opt_config in config["optimizer_configs"]
165             ],
166             default_optimizer=tf.keras.optimizers.deserialize(
167                 config["default_config"], custom_objects=custom_objects
168             ),
169         )
170 
171         # Since we no longer have the `pred`s, we set the mapping explicitly
172         case_optimizer.var_opt_mapping = config["var_opt_mapping"]
173 
174         return case_optimizer
175 
176     def _compute_var_opt_mapping(self, grads_and_vars):
177         """Compute a unique mapping from variables to optimizer indices."""
178 
179         self.var_opt_mapping = {}
180 
181         for grad, var in grads_and_vars:
182             num_optimizers = 0
183 
184             # Find the optimizer(s) that want to claim this variable
185             for optimizer_index, (predicate, _) in enumerate(self.pred_opt_pairs):
186                 if predicate(var):
187                     self.var_opt_mapping[var.name] = optimizer_index
188                     num_optimizers += 1
189 
190             if num_optimizers > 1:
191                 raise ValueError(f"Variable `{var}` claimed by multiple optimizers.")
192             if num_optimizers == 0:
193                 if self.default is not None:
194                     self.var_opt_mapping[var.name] = self.DEFAULT_OPT_INDEX
195                 else:
196                     warnings.warn(
197                         f"No `default_optimizer` provided to train variable `{var}`."
198                     )
199 
200 
201 @utils.register_keras_custom_object
202 class Bop(tf.keras.optimizers.Optimizer):
203     """Binary optimizer (Bop).
204 
205     Bop is a latent-free optimizer for Binarized Neural Networks (BNNs) and
206     Binary Weight Networks (BWN).
207 
208     Bop maintains an exponential moving average of the gradients controlled by
209     `gamma`. If this average exceeds the `threshold`, a weight is flipped.
210     Additionally, Bop accepts a regular optimizer that is applied to the
211     non-binary weights in the network.
212 
213     The hyperparameter `gamma` is somewhat analogues to the learning rate in
214     SGD methods: a high `gamma` results in rapid convergence but also makes
215     training more noisy.
216 
217     Note that the default `threshold` is not optimal for all situations.
218     Setting the threshold too high results in little learning, while setting it
219     too low results in overly noisy behaviour.
220 
221     !!! example
222         ```python
223         optimizer = lq.optimizers.CaseOptimizer(
224             (
225                 lq.optimizers.Bop.is_binary_variable,
226                 lq.optimizers.Bop(),
227             ),
228             default_optimizer=tf.keras.optimizers.Adam(0.01),  # for FP weights
229         )
230         ```
231 
232     # Arguments
233     threshold: determines to whether to flip each weight.
234     gamma: the adaptivity rate.
235     name: name of the optimizer.
236 
237     # References
238     - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://papers.nips.cc/paper/8971-latent-weights-do-not-exist-rethinking-binarized-neural-network-optimization)
239     """
240 
241     def __init__(self, threshold=1e-7, gamma=1e-2, name="Bop", **kwargs):
242         super().__init__(name=name, **kwargs)
243 
244         self._set_hyper("threshold", threshold)
245         self._set_hyper("gamma", gamma)
246 
247     def _create_slots(self, var_list):
248         for var in var_list:
249             self.add_slot(var, "m")
250 
251     def _get_decayed_hyper(self, name, var_dtype):
252         hyper = self._get_hyper(name, var_dtype)
253         if isinstance(hyper, tf.keras.optimizers.schedules.LearningRateSchedule):
254             local_step = tf.cast(self.iterations, var_dtype)
255             hyper = tf.cast(hyper(local_step), var_dtype)
256         return hyper
257 
258     def _resource_apply_dense(self, grad, var):
259         var_dtype = var.dtype.base_dtype
260         gamma = self._get_decayed_hyper("gamma", var_dtype)
261         threshold = self._get_decayed_hyper("threshold", var_dtype)
262         m = self.get_slot(var, "m")
263 
264         m_t = m.assign_add(gamma * (grad - m))
265         var_t = lq.math.sign(-tf.sign(var * m_t - threshold) * var)
266         return var.assign(var_t).op
267 
268     def _resource_apply_sparse(self, grad, var, indices):
269         raise NotImplementedError()
270 
271     def get_config(self):
272         config = {
273             "threshold": self._serialize_hyperparameter("threshold"),
274             "gamma": self._serialize_hyperparameter("gamma"),
275         }
276         return {**super().get_config(), **config}
277 
278     @classmethod
279     def from_config(cls, config, custom_objects=None):
280         for hyper in ("gamma", "threshold"):
281             if hyper in config and isinstance(config[hyper], dict):
282                 config[hyper] = tf.keras.optimizers.schedules.deserialize(
283                     config[hyper], custom_objects=custom_objects
284                 )
285         return cls(**config)
286 
287     @staticmethod
288     def is_binary_variable(var):
289         """Returns True for binary variables named using the Larq Zoo naming scheme.
290 
291         This is an example of a predictate that can be used by the `CaseOptimizer`.
292 
293         # Arguments
294         var: a `tf.Variable`.
295         """
296         return "/kernel" in var.name and "quant_" in var.name
297
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/larq/optimizers.py b/larq/optimizers.py
--- a/larq/optimizers.py
+++ b/larq/optimizers.py
@@ -238,7 +238,7 @@
     - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://papers.nips.cc/paper/8971-latent-weights-do-not-exist-rethinking-binarized-neural-network-optimization)
     """
 
-    def __init__(self, threshold=1e-7, gamma=1e-2, name="Bop", **kwargs):
+    def __init__(self, threshold=1e-8, gamma=1e-4, name="Bop", **kwargs):
         super().__init__(name=name, **kwargs)
 
         self._set_hyper("threshold", threshold)

{"golden_diff": "diff --git a/larq/optimizers.py b/larq/optimizers.py\n--- a/larq/optimizers.py\n+++ b/larq/optimizers.py\n@@ -238,7 +238,7 @@\n     - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://papers.nips.cc/paper/8971-latent-weights-do-not-exist-rethinking-binarized-neural-network-optimization)\n     \"\"\"\n \n-    def __init__(self, threshold=1e-7, gamma=1e-2, name=\"Bop\", **kwargs):\n+    def __init__(self, threshold=1e-8, gamma=1e-4, name=\"Bop\", **kwargs):\n         super().__init__(name=name, **kwargs)\n \n         self._set_hyper(\"threshold\", threshold)\n", "issue": "Change default hyperparameters of Bop\nThe default hyperparameters for Bop are not really optimal (i.e. gamma is probably too high for most task): `Bop(threshold=1e-07, gamma=0.01, name='Bop', **kwargs)`\r\n\r\nFor example in our [paper](https://papers.nips.cc/paper/8971-latent-weights-do-not-exist-rethinking-binarized-neural-network-optimization.pdf) we used a gamma decayed from 1e-4 to 1e-6 and a threshold of 1e-8 for the ImageNet experiments.\r\n\r\nI think we should update the default parameters to a more sensible choice. @MariaHeuss What do you think?\n", "before_files": [{"content": "\"\"\"Neural networks with extremely low-precision weights and activations, such as\nBinarized Neural Networks (BNNs), usually contain a mix of low-precision weights (e.g.\n1-bit) and  higher-precision weights (e.g. 8-bit, 16-bit, or 32-bit). Examples of this\ninclude the first and last layers of image classificiation models, which have\nhigher-precision weights in most BNN architectures from the literature.\n\nTraining a BNN, then, consists of optimizing both low-precision and higher-precision\nweights. In `larq`, we provide a mechanism to target different bit-precision variables\nwith different optimizers using the `CaseOptimizer` class. Modeled after the\n[`tf.case`](https://www.tensorflow.org/api_docs/python/tf/case) signature,\n`CaseOptimizer` accepts pairs of predicates and optimizers. A predicate, given a\nvariable, decides whether its optimizer should train that variable.\n\nA `CaseOptimizer` behaves much like any other\n[Keras optimizer](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers), and\nonce you instantiate it you can pass it to your `model.compile()` as usual. To\ninstantiate a `CaseOptimzer`, pass one or a list of `(predicate, optimizer)` tuples,\nalong with a `default` optimizer which trains any variables not claimed by another\noptimizer. A variable may not be claimed by more than one optimizer's predicate.\n\n!!! example\n    ```python\n    case_optimizer = lq.optimizers.CaseOptimizer(\n        (\n            lq.optimizers.Bop.is_binary_variable,  # predicate\n            lq.optimizers.Bop(threshold=1e-6, gamma=1e-3),  # optimizer\n        ),\n        default_optimizer=tf.keras.optimizers.Adam(0.01),\n    )\n    ```\n\"\"\"\n\n\nimport warnings\nfrom copy import deepcopy\n\nimport tensorflow as tf\n\nimport larq as lq\nfrom larq import utils\n\n__all__ = [\"Bop\", \"CaseOptimizer\"]\n\n\n@utils.register_keras_custom_object\nclass CaseOptimizer(tf.keras.optimizers.Optimizer):\n    \"\"\"An optmizer wrapper that applies different optimizers to a subset of variables.\n\n    An optimizer is used to train a variable iff its accompanying predicate evaluates to\n    `True`.\n\n    For each variable, at most one optimizer's predicate may evaluate to `True`. If no\n    optimizer's predicate evaluates to `True` for a variable, it is trained with the\n    `default_optimizer`. If a variable is claimed by no optimizers and\n    `default_optimizer == None`, the variable is not trained.\n\n    # Arguments\n    predicate_optimizer_pairs: One or more `(pred, tf.keras.optimizers.Optimzer)` pairs,\n        where `pred`  takes one `tf.Variable` as argument and returns `True` if the\n        optimizer should be used for that variable, e.g. `pred(var) == True`.\n    default_optimizer: A `tf.keras.optimizers.Optimizer` to be applied to any variable\n        not claimed by any other optimizer. (Must be passed as keyword argument.)\n    \"\"\"\n\n    def __init__(\n        self, *predicate_optimizer_pairs, default_optimizer=None, name=\"optimizer_case\"\n    ):\n        super().__init__(name=name)\n\n        # Type checks for (predicate, optimizer) pairs\n        for i, (predicate, optimizer) in enumerate(predicate_optimizer_pairs):\n            if not callable(predicate):\n                raise TypeError(\n                    f\"Expected callable predicate at `predicate_optimizer_pairs[{i}][0]` but got `{type(predicate)}`.\"\n                )\n            if not isinstance(optimizer, tf.keras.optimizers.Optimizer):\n                raise TypeError(\n                    f\"Expected `tf.keras.optimizers.Optimizer` at `predicate_optimizer_pairs[{i}][1]` but got `{type(optimizer)}`.\"\n                )\n\n        # Type check for default optimizers\n        if default_optimizer is not None and not isinstance(\n            default_optimizer, tf.keras.optimizers.Optimizer\n        ):\n            raise TypeError(\n                f\"Expected `tf.keras.optimizers.Optimizer` for `default_optimizer` but got `{type(default_optimizer)}`.\"\n            )\n\n        self.pred_opt_pairs = predicate_optimizer_pairs\n        self.default = default_optimizer\n\n        self.var_opt_mapping = None\n\n        # List of optimizers ending in `default_optimizer`, for easier internal access\n        self.optimizers = [opt for (_, opt) in self.pred_opt_pairs]\n\n        if self.default:\n            self.optimizers.append(self.default)\n            self.DEFAULT_OPT_INDEX = len(self.pred_opt_pairs)\n\n    @property\n    def weights(self):\n        weights = []\n        for optimizer in self.optimizers:\n            weights.extend(optimizer.weights)\n        return weights\n\n    def apply_gradients(self, grads_and_vars, name=None):\n        \"\"\"Apply gradients to variables for each optimizer.\n\n        On the first call to `apply_gradients()`, compute the mapping from variables to\n        optimizers and cache it in the `self.var_opt_mapping` dict for serialization and\n        faster access.\n        \"\"\"\n\n        if self.var_opt_mapping is None:\n            # Convert `grads_and_vars` to list so we can iterate multiple times over it\n            grads_and_vars = list(grads_and_vars)\n            self._compute_var_opt_mapping(grads_and_vars)\n\n        # Split gradients and variables into a separate list for each optimizer\n        grad_var_lists = [[] for _ in range(len(self.pred_opt_pairs) + 1)]\n        for grad, var in grads_and_vars:\n            if var.name in self.var_opt_mapping:\n                grad_var_lists[self.var_opt_mapping[var.name]].append((grad, var))\n\n        # Apply gradients to each optimizer\n        train_ops = [\n            optimizer.apply_gradients(opt_grads_and_vars)\n            for optimizer, opt_grads_and_vars in zip(self.optimizers, grad_var_lists)\n        ]\n\n        return tf.group(*train_ops, name=\"train_with_group\")\n\n    def get_config(self):\n        optimizer_configs = [opt.get_config() for (_, opt) in self.pred_opt_pairs]\n        default_config = self.default.get_config()\n\n        config = {\n            \"optimizer_configs\": [\n                {\"class_name\": optimizer_config[\"name\"], \"config\": optimizer_config}\n                for optimizer_config in optimizer_configs\n            ],\n            \"default_config\": {\n                \"class_name\": default_config[\"name\"],\n                \"config\": default_config,\n            },\n            \"var_opt_mapping\": self.var_opt_mapping,  # serialized instead of `pred`s\n        }\n        return {**super().get_config(), **config}\n\n    @classmethod\n    def from_config(cls, original_config, custom_objects=None):\n        config = deepcopy(original_config)\n\n        case_optimizer = cls(\n            *[  # `(pred, opt)` tuples\n                (\n                    lambda _: False,  # placeholder callable (`pred` is not serialized)\n                    tf.keras.optimizers.deserialize(  # optimizer `opt`\n                        opt_config, custom_objects=custom_objects\n                    ),\n                )\n                for opt_config in config[\"optimizer_configs\"]\n            ],\n            default_optimizer=tf.keras.optimizers.deserialize(\n                config[\"default_config\"], custom_objects=custom_objects\n            ),\n        )\n\n        # Since we no longer have the `pred`s, we set the mapping explicitly\n        case_optimizer.var_opt_mapping = config[\"var_opt_mapping\"]\n\n        return case_optimizer\n\n    def _compute_var_opt_mapping(self, grads_and_vars):\n        \"\"\"Compute a unique mapping from variables to optimizer indices.\"\"\"\n\n        self.var_opt_mapping = {}\n\n        for grad, var in grads_and_vars:\n            num_optimizers = 0\n\n            # Find the optimizer(s) that want to claim this variable\n            for optimizer_index, (predicate, _) in enumerate(self.pred_opt_pairs):\n                if predicate(var):\n                    self.var_opt_mapping[var.name] = optimizer_index\n                    num_optimizers += 1\n\n            if num_optimizers > 1:\n                raise ValueError(f\"Variable `{var}` claimed by multiple optimizers.\")\n            if num_optimizers == 0:\n                if self.default is not None:\n                    self.var_opt_mapping[var.name] = self.DEFAULT_OPT_INDEX\n                else:\n                    warnings.warn(\n                        f\"No `default_optimizer` provided to train variable `{var}`.\"\n                    )\n\n\n@utils.register_keras_custom_object\nclass Bop(tf.keras.optimizers.Optimizer):\n    \"\"\"Binary optimizer (Bop).\n\n    Bop is a latent-free optimizer for Binarized Neural Networks (BNNs) and\n    Binary Weight Networks (BWN).\n\n    Bop maintains an exponential moving average of the gradients controlled by\n    `gamma`. If this average exceeds the `threshold`, a weight is flipped.\n    Additionally, Bop accepts a regular optimizer that is applied to the\n    non-binary weights in the network.\n\n    The hyperparameter `gamma` is somewhat analogues to the learning rate in\n    SGD methods: a high `gamma` results in rapid convergence but also makes\n    training more noisy.\n\n    Note that the default `threshold` is not optimal for all situations.\n    Setting the threshold too high results in little learning, while setting it\n    too low results in overly noisy behaviour.\n\n    !!! example\n        ```python\n        optimizer = lq.optimizers.CaseOptimizer(\n            (\n                lq.optimizers.Bop.is_binary_variable,\n                lq.optimizers.Bop(),\n            ),\n            default_optimizer=tf.keras.optimizers.Adam(0.01),  # for FP weights\n        )\n        ```\n\n    # Arguments\n    threshold: determines to whether to flip each weight.\n    gamma: the adaptivity rate.\n    name: name of the optimizer.\n\n    # References\n    - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://papers.nips.cc/paper/8971-latent-weights-do-not-exist-rethinking-binarized-neural-network-optimization)\n    \"\"\"\n\n    def __init__(self, threshold=1e-7, gamma=1e-2, name=\"Bop\", **kwargs):\n        super().__init__(name=name, **kwargs)\n\n        self._set_hyper(\"threshold\", threshold)\n        self._set_hyper(\"gamma\", gamma)\n\n    def _create_slots(self, var_list):\n        for var in var_list:\n            self.add_slot(var, \"m\")\n\n    def _get_decayed_hyper(self, name, var_dtype):\n        hyper = self._get_hyper(name, var_dtype)\n        if isinstance(hyper, tf.keras.optimizers.schedules.LearningRateSchedule):\n            local_step = tf.cast(self.iterations, var_dtype)\n            hyper = tf.cast(hyper(local_step), var_dtype)\n        return hyper\n\n    def _resource_apply_dense(self, grad, var):\n        var_dtype = var.dtype.base_dtype\n        gamma = self._get_decayed_hyper(\"gamma\", var_dtype)\n        threshold = self._get_decayed_hyper(\"threshold\", var_dtype)\n        m = self.get_slot(var, \"m\")\n\n        m_t = m.assign_add(gamma * (grad - m))\n        var_t = lq.math.sign(-tf.sign(var * m_t - threshold) * var)\n        return var.assign(var_t).op\n\n    def _resource_apply_sparse(self, grad, var, indices):\n        raise NotImplementedError()\n\n    def get_config(self):\n        config = {\n            \"threshold\": self._serialize_hyperparameter(\"threshold\"),\n            \"gamma\": self._serialize_hyperparameter(\"gamma\"),\n        }\n        return {**super().get_config(), **config}\n\n    @classmethod\n    def from_config(cls, config, custom_objects=None):\n        for hyper in (\"gamma\", \"threshold\"):\n            if hyper in config and isinstance(config[hyper], dict):\n                config[hyper] = tf.keras.optimizers.schedules.deserialize(\n                    config[hyper], custom_objects=custom_objects\n                )\n        return cls(**config)\n\n    @staticmethod\n    def is_binary_variable(var):\n        \"\"\"Returns True for binary variables named using the Larq Zoo naming scheme.\n\n        This is an example of a predictate that can be used by the `CaseOptimizer`.\n\n        # Arguments\n        var: a `tf.Variable`.\n        \"\"\"\n        return \"/kernel\" in var.name and \"quant_\" in var.name\n", "path": "larq/optimizers.py"}], "after_files": [{"content": "\"\"\"Neural networks with extremely low-precision weights and activations, such as\nBinarized Neural Networks (BNNs), usually contain a mix of low-precision weights (e.g.\n1-bit) and  higher-precision weights (e.g. 8-bit, 16-bit, or 32-bit). Examples of this\ninclude the first and last layers of image classificiation models, which have\nhigher-precision weights in most BNN architectures from the literature.\n\nTraining a BNN, then, consists of optimizing both low-precision and higher-precision\nweights. In `larq`, we provide a mechanism to target different bit-precision variables\nwith different optimizers using the `CaseOptimizer` class. Modeled after the\n[`tf.case`](https://www.tensorflow.org/api_docs/python/tf/case) signature,\n`CaseOptimizer` accepts pairs of predicates and optimizers. A predicate, given a\nvariable, decides whether its optimizer should train that variable.\n\nA `CaseOptimizer` behaves much like any other\n[Keras optimizer](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers), and\nonce you instantiate it you can pass it to your `model.compile()` as usual. To\ninstantiate a `CaseOptimzer`, pass one or a list of `(predicate, optimizer)` tuples,\nalong with a `default` optimizer which trains any variables not claimed by another\noptimizer. A variable may not be claimed by more than one optimizer's predicate.\n\n!!! example\n    ```python\n    case_optimizer = lq.optimizers.CaseOptimizer(\n        (\n            lq.optimizers.Bop.is_binary_variable,  # predicate\n            lq.optimizers.Bop(threshold=1e-6, gamma=1e-3),  # optimizer\n        ),\n        default_optimizer=tf.keras.optimizers.Adam(0.01),\n    )\n    ```\n\"\"\"\n\n\nimport warnings\nfrom copy import deepcopy\n\nimport tensorflow as tf\n\nimport larq as lq\nfrom larq import utils\n\n__all__ = [\"Bop\", \"CaseOptimizer\"]\n\n\n@utils.register_keras_custom_object\nclass CaseOptimizer(tf.keras.optimizers.Optimizer):\n    \"\"\"An optmizer wrapper that applies different optimizers to a subset of variables.\n\n    An optimizer is used to train a variable iff its accompanying predicate evaluates to\n    `True`.\n\n    For each variable, at most one optimizer's predicate may evaluate to `True`. If no\n    optimizer's predicate evaluates to `True` for a variable, it is trained with the\n    `default_optimizer`. If a variable is claimed by no optimizers and\n    `default_optimizer == None`, the variable is not trained.\n\n    # Arguments\n    predicate_optimizer_pairs: One or more `(pred, tf.keras.optimizers.Optimzer)` pairs,\n        where `pred`  takes one `tf.Variable` as argument and returns `True` if the\n        optimizer should be used for that variable, e.g. `pred(var) == True`.\n    default_optimizer: A `tf.keras.optimizers.Optimizer` to be applied to any variable\n        not claimed by any other optimizer. (Must be passed as keyword argument.)\n    \"\"\"\n\n    def __init__(\n        self, *predicate_optimizer_pairs, default_optimizer=None, name=\"optimizer_case\"\n    ):\n        super().__init__(name=name)\n\n        # Type checks for (predicate, optimizer) pairs\n        for i, (predicate, optimizer) in enumerate(predicate_optimizer_pairs):\n            if not callable(predicate):\n                raise TypeError(\n                    f\"Expected callable predicate at `predicate_optimizer_pairs[{i}][0]` but got `{type(predicate)}`.\"\n                )\n            if not isinstance(optimizer, tf.keras.optimizers.Optimizer):\n                raise TypeError(\n                    f\"Expected `tf.keras.optimizers.Optimizer` at `predicate_optimizer_pairs[{i}][1]` but got `{type(optimizer)}`.\"\n                )\n\n        # Type check for default optimizers\n        if default_optimizer is not None and not isinstance(\n            default_optimizer, tf.keras.optimizers.Optimizer\n        ):\n            raise TypeError(\n                f\"Expected `tf.keras.optimizers.Optimizer` for `default_optimizer` but got `{type(default_optimizer)}`.\"\n            )\n\n        self.pred_opt_pairs = predicate_optimizer_pairs\n        self.default = default_optimizer\n\n        self.var_opt_mapping = None\n\n        # List of optimizers ending in `default_optimizer`, for easier internal access\n        self.optimizers = [opt for (_, opt) in self.pred_opt_pairs]\n\n        if self.default:\n            self.optimizers.append(self.default)\n            self.DEFAULT_OPT_INDEX = len(self.pred_opt_pairs)\n\n    @property\n    def weights(self):\n        weights = []\n        for optimizer in self.optimizers:\n            weights.extend(optimizer.weights)\n        return weights\n\n    def apply_gradients(self, grads_and_vars, name=None):\n        \"\"\"Apply gradients to variables for each optimizer.\n\n        On the first call to `apply_gradients()`, compute the mapping from variables to\n        optimizers and cache it in the `self.var_opt_mapping` dict for serialization and\n        faster access.\n        \"\"\"\n\n        if self.var_opt_mapping is None:\n            # Convert `grads_and_vars` to list so we can iterate multiple times over it\n            grads_and_vars = list(grads_and_vars)\n            self._compute_var_opt_mapping(grads_and_vars)\n\n        # Split gradients and variables into a separate list for each optimizer\n        grad_var_lists = [[] for _ in range(len(self.pred_opt_pairs) + 1)]\n        for grad, var in grads_and_vars:\n            if var.name in self.var_opt_mapping:\n                grad_var_lists[self.var_opt_mapping[var.name]].append((grad, var))\n\n        # Apply gradients to each optimizer\n        train_ops = [\n            optimizer.apply_gradients(opt_grads_and_vars)\n            for optimizer, opt_grads_and_vars in zip(self.optimizers, grad_var_lists)\n        ]\n\n        return tf.group(*train_ops, name=\"train_with_group\")\n\n    def get_config(self):\n        optimizer_configs = [opt.get_config() for (_, opt) in self.pred_opt_pairs]\n        default_config = self.default.get_config()\n\n        config = {\n            \"optimizer_configs\": [\n                {\"class_name\": optimizer_config[\"name\"], \"config\": optimizer_config}\n                for optimizer_config in optimizer_configs\n            ],\n            \"default_config\": {\n                \"class_name\": default_config[\"name\"],\n                \"config\": default_config,\n            },\n            \"var_opt_mapping\": self.var_opt_mapping,  # serialized instead of `pred`s\n        }\n        return {**super().get_config(), **config}\n\n    @classmethod\n    def from_config(cls, original_config, custom_objects=None):\n        config = deepcopy(original_config)\n\n        case_optimizer = cls(\n            *[  # `(pred, opt)` tuples\n                (\n                    lambda _: False,  # placeholder callable (`pred` is not serialized)\n                    tf.keras.optimizers.deserialize(  # optimizer `opt`\n                        opt_config, custom_objects=custom_objects\n                    ),\n                )\n                for opt_config in config[\"optimizer_configs\"]\n            ],\n            default_optimizer=tf.keras.optimizers.deserialize(\n                config[\"default_config\"], custom_objects=custom_objects\n            ),\n        )\n\n        # Since we no longer have the `pred`s, we set the mapping explicitly\n        case_optimizer.var_opt_mapping = config[\"var_opt_mapping\"]\n\n        return case_optimizer\n\n    def _compute_var_opt_mapping(self, grads_and_vars):\n        \"\"\"Compute a unique mapping from variables to optimizer indices.\"\"\"\n\n        self.var_opt_mapping = {}\n\n        for grad, var in grads_and_vars:\n            num_optimizers = 0\n\n            # Find the optimizer(s) that want to claim this variable\n            for optimizer_index, (predicate, _) in enumerate(self.pred_opt_pairs):\n                if predicate(var):\n                    self.var_opt_mapping[var.name] = optimizer_index\n                    num_optimizers += 1\n\n            if num_optimizers > 1:\n                raise ValueError(f\"Variable `{var}` claimed by multiple optimizers.\")\n            if num_optimizers == 0:\n                if self.default is not None:\n                    self.var_opt_mapping[var.name] = self.DEFAULT_OPT_INDEX\n                else:\n                    warnings.warn(\n                        f\"No `default_optimizer` provided to train variable `{var}`.\"\n                    )\n\n\n@utils.register_keras_custom_object\nclass Bop(tf.keras.optimizers.Optimizer):\n    \"\"\"Binary optimizer (Bop).\n\n    Bop is a latent-free optimizer for Binarized Neural Networks (BNNs) and\n    Binary Weight Networks (BWN).\n\n    Bop maintains an exponential moving average of the gradients controlled by\n    `gamma`. If this average exceeds the `threshold`, a weight is flipped.\n    Additionally, Bop accepts a regular optimizer that is applied to the\n    non-binary weights in the network.\n\n    The hyperparameter `gamma` is somewhat analogues to the learning rate in\n    SGD methods: a high `gamma` results in rapid convergence but also makes\n    training more noisy.\n\n    Note that the default `threshold` is not optimal for all situations.\n    Setting the threshold too high results in little learning, while setting it\n    too low results in overly noisy behaviour.\n\n    !!! example\n        ```python\n        optimizer = lq.optimizers.CaseOptimizer(\n            (\n                lq.optimizers.Bop.is_binary_variable,\n                lq.optimizers.Bop(),\n            ),\n            default_optimizer=tf.keras.optimizers.Adam(0.01),  # for FP weights\n        )\n        ```\n\n    # Arguments\n    threshold: determines to whether to flip each weight.\n    gamma: the adaptivity rate.\n    name: name of the optimizer.\n\n    # References\n    - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://papers.nips.cc/paper/8971-latent-weights-do-not-exist-rethinking-binarized-neural-network-optimization)\n    \"\"\"\n\n    def __init__(self, threshold=1e-8, gamma=1e-4, name=\"Bop\", **kwargs):\n        super().__init__(name=name, **kwargs)\n\n        self._set_hyper(\"threshold\", threshold)\n        self._set_hyper(\"gamma\", gamma)\n\n    def _create_slots(self, var_list):\n        for var in var_list:\n            self.add_slot(var, \"m\")\n\n    def _get_decayed_hyper(self, name, var_dtype):\n        hyper = self._get_hyper(name, var_dtype)\n        if isinstance(hyper, tf.keras.optimizers.schedules.LearningRateSchedule):\n            local_step = tf.cast(self.iterations, var_dtype)\n            hyper = tf.cast(hyper(local_step), var_dtype)\n        return hyper\n\n    def _resource_apply_dense(self, grad, var):\n        var_dtype = var.dtype.base_dtype\n        gamma = self._get_decayed_hyper(\"gamma\", var_dtype)\n        threshold = self._get_decayed_hyper(\"threshold\", var_dtype)\n        m = self.get_slot(var, \"m\")\n\n        m_t = m.assign_add(gamma * (grad - m))\n        var_t = lq.math.sign(-tf.sign(var * m_t - threshold) * var)\n        return var.assign(var_t).op\n\n    def _resource_apply_sparse(self, grad, var, indices):\n        raise NotImplementedError()\n\n    def get_config(self):\n        config = {\n            \"threshold\": self._serialize_hyperparameter(\"threshold\"),\n            \"gamma\": self._serialize_hyperparameter(\"gamma\"),\n        }\n        return {**super().get_config(), **config}\n\n    @classmethod\n    def from_config(cls, config, custom_objects=None):\n        for hyper in (\"gamma\", \"threshold\"):\n            if hyper in config and isinstance(config[hyper], dict):\n                config[hyper] = tf.keras.optimizers.schedules.deserialize(\n                    config[hyper], custom_objects=custom_objects\n                )\n        return cls(**config)\n\n    @staticmethod\n    def is_binary_variable(var):\n        \"\"\"Returns True for binary variables named using the Larq Zoo naming scheme.\n\n        This is an example of a predictate that can be used by the `CaseOptimizer`.\n\n        # Arguments\n        var: a `tf.Variable`.\n        \"\"\"\n        return \"/kernel\" in var.name and \"quant_\" in var.name\n", "path": "larq/optimizers.py"}]}