We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Adding binary optimizer (Bop) documentation 
- [ ] enhancing Bop documentation in code
- [ ] adding Bop documentation/tutorial to [Larq official documentation page](https://plumerai.github.io/larq/)

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `larq/optimizers_v2.py`
Content:
```
1 import tensorflow as tf
2 import larq as lq
3 
4 from larq import utils
5 from copy import deepcopy
6 
7 
8 @utils.register_keras_custom_object
9 class Bop(tf.keras.optimizers.Optimizer):
10     """Binary optimizer (Bop).
11 
12     Bop is a latent-free optimizer for Binarized Neural Networks (BNNs).
13 
14     !!! example
15         ```python
16         optimizer = lq.optimizers.Bop(fp_optimizer=tf.keras.optimizers.Adam(0.01))
17         ```
18 
19     # Arguments
20     fp_optimizer: a `tf.keras.optimizers.Optimizer`.
21     threshold: determines to whether to flip each weight.
22     gamma: the adaptivity rate.
23     name: name of the optimizer.
24 
25     # References
26     - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://arxiv.org/abs/1906.02107)
27     """
28 
29     def __init__(self, fp_optimizer, threshold=1e-5, gamma=1e-2, name="Bop", **kwargs):
30         super().__init__(name=name, **kwargs)
31 
32         if not isinstance(fp_optimizer, tf.keras.optimizers.Optimizer):
33             raise TypeError(
34                 f"Expected tf.keras.optimizers.Optimizer, received {type(fp_optimizer)}."
35             )
36 
37         self.fp_optimizer = fp_optimizer
38         self._set_hyper("threshold", threshold)
39         self._set_hyper("gamma", gamma)
40 
41     def _create_slots(self, var_list):
42         for var in var_list:
43             if self.is_binary(var):
44                 self.add_slot(var, "m")
45 
46     def apply_gradients(self, grads_and_vars, name=None):
47         bin_grads_and_vars = [(g, v) for g, v in grads_and_vars if self.is_binary(v)]
48         fp_grads_and_vars = [(g, v) for g, v in grads_and_vars if not self.is_binary(v)]
49 
50         bin_train_op = super().apply_gradients(bin_grads_and_vars, name=name)
51         fp_train_op = self.fp_optimizer.apply_gradients(fp_grads_and_vars, name=name)
52 
53         return tf.group(bin_train_op, fp_train_op, name="train_with_bop")
54 
55     def _resource_apply_sparse(self, grad, var, indices):
56         raise NotImplementedError()
57 
58     def __getattr__(self, name):
59         if name == "lr":
60             return self.fp_optimizer.lr
61         return super().__getattr__(name)
62 
63     def _get_decayed_hyper(self, name, var_dtype):
64         hyper = self._get_hyper(name, var_dtype)
65         if isinstance(hyper, tf.keras.optimizers.schedules.LearningRateSchedule):
66             local_step = tf.cast(self.iterations, var_dtype)
67             hyper = tf.cast(hyper(local_step), var_dtype)
68         return hyper
69 
70     def _resource_apply_dense(self, grad, var):
71         var_dtype = var.dtype.base_dtype
72         gamma = self._get_decayed_hyper("gamma", var_dtype)
73         threshold = self._get_decayed_hyper("threshold", var_dtype)
74         m = self.get_slot(var, "m")
75 
76         m_t = tf.compat.v1.assign(
77             m, (1 - gamma) * m + gamma * grad, use_locking=self._use_locking
78         )
79         var_t = lq.math.sign(-tf.sign(var * m_t - threshold) * var)
80         return tf.compat.v1.assign(var, var_t, use_locking=self._use_locking).op
81 
82     @staticmethod
83     def is_binary(var):
84         return "/kernel" in var.name and "quant_" in var.name
85 
86     def get_config(self):
87         fp_optimizer_config = self.fp_optimizer.get_config()
88         config = {
89             "threshold": self._serialize_hyperparameter("threshold"),
90             "gamma": self._serialize_hyperparameter("gamma"),
91             "fp_optimizer": {
92                 "class_name": fp_optimizer_config["name"],
93                 "config": fp_optimizer_config,
94             },
95         }
96         return {**super().get_config(), **config}
97 
98     @classmethod
99     def from_config(cls, config, custom_objects=None):
100         new_config = deepcopy(config)
101         fp_optimizer = tf.keras.optimizers.deserialize(
102             new_config["fp_optimizer"], custom_objects=custom_objects
103         )
104         new_config.pop("fp_optimizer", None)
105         return cls(fp_optimizer, **new_config)
106
```

Path: `larq/optimizers_v1.py`
Content:
```
1 import tensorflow as tf
2 import numpy as np
3 import larq as lq
4 
5 from larq import utils
6 from copy import deepcopy
7 
8 
9 @utils.register_keras_custom_object
10 class XavierLearningRateScaling(tf.keras.optimizers.Optimizer):
11     """Optimizer wrapper for Xavier Learning Rate Scaling
12 
13     Scale the weights learning rates respectively with the weights initialization
14 
15     !!! note ""
16         This is a wrapper and does not implement any optimization algorithm.
17 
18     !!! example
19         ```python
20         optimizer = lq.optimizers.XavierLearningRateScaling(
21             tf.keras.optimizers.Adam(0.01), model
22         )
23         ```
24 
25     # Arguments
26     optimizer: A `tf.keras.optimizers.Optimizer`
27     model: A `tf.keras.Model`
28 
29     # References
30     - [BinaryConnect: Training Deep Neural Networks with binary weights during
31       propagations](https://arxiv.org/abs/1511.00363)
32     """
33 
34     def __init__(self, optimizer, model):
35         if int(tf.__version__[0]) == 2:
36             raise NotImplementedError(
37                 "XavierLearningRateScaling is not supported by Tensorflow 2.0."
38             )
39 
40         if not isinstance(optimizer, tf.keras.optimizers.Optimizer):
41             raise ValueError(
42                 f"Expected tf.keras.optimizers.Optimizer, received {type(optimizer)}."
43             )
44         self.optimizer = optimizer
45 
46         if isinstance(model, tf.keras.Model):
47             self.multipliers = {}
48             for layer in model.layers:
49                 if hasattr(layer, "quantized_latent_weights"):
50                     for weight in layer.quantized_latent_weights:
51                         self.multipliers[weight.name] = self.get_lr_multiplier(weight)
52         elif isinstance(model, dict):
53             self.multipliers = model
54         else:
55             raise ValueError(f"Expected tf.keras.Model or dict, received {type(model)}")
56 
57     def get_lr_multiplier(self, weight):
58         shape = weight.get_shape().as_list()
59         n_input = shape[-2]
60         n_output = shape[-1]
61         if len(shape) == 4:
62             kernelsize = np.prod(shape[:-2])
63             coeff = 1.0 / np.sqrt(1.5 / ((kernelsize * (n_input + n_output))))
64         elif len(shape) == 2:
65             coeff = 1.0 / np.sqrt(1.5 / ((1.0 * (n_input + n_output))))
66         else:
67             raise NotImplementedError(
68                 "Xavier Learning rate scaling not implimented for this kernelsize"
69             )
70         return coeff
71 
72     def get_updates(self, loss, params):
73         mult_lr_params = [p for p in params if p.name in self.multipliers]
74         base_lr_params = [p for p in params if p.name not in self.multipliers]
75 
76         updates = []
77         base_lr = self.optimizer.lr
78         for param in mult_lr_params:
79             self.optimizer.lr = base_lr * self.multipliers[param.name]
80             updates.extend(self.optimizer.get_updates(loss, [param]))
81 
82         self.optimizer.lr = base_lr
83         updates.extend(self.optimizer.get_updates(loss, base_lr_params))
84 
85         return updates
86 
87     def __getattr__(self, name):
88         return getattr(self.optimizer, name)
89 
90     def get_config(self):
91         return {
92             "optimizer": {
93                 "class_name": self.optimizer.__class__.__name__,
94                 "config": self.optimizer.get_config(),
95             },
96             "multipliers": self.multipliers,
97         }
98 
99     @classmethod
100     def from_config(cls, config, custom_objects=None):
101         optimizer = tf.keras.optimizers.deserialize(
102             config.pop("optimizer"), custom_objects=custom_objects
103         )
104         return cls(optimizer, config["multipliers"])
105 
106 
107 @utils.register_keras_custom_object
108 class Bop(tf.keras.optimizers.Optimizer):
109     """Binary optimizer (Bop).
110 
111     Bop is a latent-free optimizer for Binarized Neural Networks (BNNs).
112 
113     !!! example
114         ```python
115         optimizer = lq.optimizers.Bop(fp_optimizer=tf.keras.optimizers.Adam(0.01))
116 
117         ```
118 
119     # Arguments
120     fp_optimizer: a `tf.keras.optimizers.Optimizer`.
121     threshold: determines to whether to flip each weight.
122     gamma: the adaptivity rate.
123     name: name of the optimizer.
124 
125     # References
126     - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://arxiv.org/abs/1906.02107)
127     """
128 
129     def __init__(self, fp_optimizer, threshold=1e-5, gamma=1e-2, name="Bop", **kwargs):
130         super().__init__(**kwargs)
131 
132         if not isinstance(fp_optimizer, tf.keras.optimizers.Optimizer):
133             raise TypeError(
134                 f"Expected tf.keras.optimizers.Optimizer, received {type(fp_optimizer)}."
135             )
136 
137         with tf.keras.backend.name_scope(self.__class__.__name__):
138 
139             self.fp_optimizer = fp_optimizer
140             self.threshold = tf.keras.backend.variable(threshold, name="threshold")
141             self.gamma = tf.keras.backend.variable(gamma, name="gamma")
142 
143     def get_updates(self, loss, params):
144         grads = self.get_gradients(loss, params)
145         self.updates = []
146 
147         ms = [
148             tf.keras.backend.zeros(
149                 tf.keras.backend.int_shape(p), dtype=tf.keras.backend.dtype(p)
150             )
151             for p in params
152         ]
153 
154         fp_params = []
155 
156         for p, g, m in zip(params, grads, ms):
157             if self.is_binary(p):
158                 m_t = (1 - self.gamma) * m + self.gamma * g
159 
160                 self.updates.append(tf.assign(m, m_t))
161                 self.updates.append(
162                     tf.assign(p, lq.math.sign(-p * tf.sign(p * m_t - self.threshold)))
163                 )
164 
165             else:
166                 fp_params.append(p)
167 
168         return self.updates + self.fp_optimizer.get_updates(loss, fp_params)
169 
170     @staticmethod
171     def is_binary(var):
172         return "/kernel" in var.name and "quant_" in var.name
173 
174     def __getattr__(self, name):
175         if name == "lr":
176             return self.fp_optimizer.lr
177         return super().__getattr__(name)
178 
179     def get_config(self):
180         fp_optimizer_config = self.fp_optimizer.get_config()
181         config = {
182             "threshold": float(tf.keras.backend.get_value(self.threshold)),
183             "gamma": float(tf.keras.backend.get_value(self.gamma)),
184             "fp_optimizer": {
185                 "class_name": self.fp_optimizer.__class__.__name__,
186                 "config": fp_optimizer_config,
187             },
188         }
189         return {**super().get_config(), **config}
190 
191     @classmethod
192     def from_config(cls, config, custom_objects=None):
193         new_config = deepcopy(config)
194         fp_optimizer = tf.keras.optimizers.deserialize(
195             new_config["fp_optimizer"], custom_objects=custom_objects
196         )
197         new_config.pop("fp_optimizer", None)
198         return cls(fp_optimizer, **new_config)
199
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/larq/optimizers_v1.py b/larq/optimizers_v1.py
--- a/larq/optimizers_v1.py
+++ b/larq/optimizers_v1.py
@@ -108,12 +108,25 @@
 class Bop(tf.keras.optimizers.Optimizer):
     """Binary optimizer (Bop).
 
-    Bop is a latent-free optimizer for Binarized Neural Networks (BNNs).
+    Bop is a latent-free optimizer for Binarized Neural Networks (BNNs) and
+    Binary Weight Networks (BWN).
+
+    Bop maintains an exponential moving average of the gradients controlled by
+    `gamma`. If this average exceeds the `threshold`, a weight is flipped.
+    Additionally, Bop accepts a regular optimizer that is applied to the
+    non-binary weights in the network.
+
+    The hyperparameter `gamma` is somewhat analogues to the learning rate in
+    SGD methods: a high `gamma` results in rapid convergence but also makes
+    training more noisy.
+
+    Note that the default `threshold` is not optimal for all situations.
+    Setting the threshold too high results in little learning, while setting it
+    too low results in overly noisy behaviour.
 
     !!! example
         ```python
         optimizer = lq.optimizers.Bop(fp_optimizer=tf.keras.optimizers.Adam(0.01))
-
         ```
 
     # Arguments
@@ -126,7 +139,7 @@
     - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://arxiv.org/abs/1906.02107)
     """
 
-    def __init__(self, fp_optimizer, threshold=1e-5, gamma=1e-2, name="Bop", **kwargs):
+    def __init__(self, fp_optimizer, threshold=1e-7, gamma=1e-2, name="Bop", **kwargs):
         super().__init__(**kwargs)
 
         if not isinstance(fp_optimizer, tf.keras.optimizers.Optimizer):
diff --git a/larq/optimizers_v2.py b/larq/optimizers_v2.py
--- a/larq/optimizers_v2.py
+++ b/larq/optimizers_v2.py
@@ -9,7 +9,21 @@
 class Bop(tf.keras.optimizers.Optimizer):
     """Binary optimizer (Bop).
 
-    Bop is a latent-free optimizer for Binarized Neural Networks (BNNs).
+    Bop is a latent-free optimizer for Binarized Neural Networks (BNNs) and
+    Binary Weight Networks (BWN).
+
+    Bop maintains an exponential moving average of the gradients controlled by
+    `gamma`. If this average exceeds the `threshold`, a weight is flipped.
+    Additionally, Bop accepts a regular optimizer that is applied to the
+    non-binary weights in the network.
+
+    The hyperparameter `gamma` is somewhat analogues to the learning rate in
+    SGD methods: a high `gamma` results in rapid convergence but also makes
+    training more noisy.
+
+    Note that the default `threshold` is not optimal for all situations.
+    Setting the threshold too high results in little learning, while setting it
+    too low results in overly noisy behaviour.
 
     !!! example
         ```python
@@ -26,7 +40,7 @@
     - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://arxiv.org/abs/1906.02107)
     """
 
-    def __init__(self, fp_optimizer, threshold=1e-5, gamma=1e-2, name="Bop", **kwargs):
+    def __init__(self, fp_optimizer, threshold=1e-7, gamma=1e-2, name="Bop", **kwargs):
         super().__init__(name=name, **kwargs)
 
         if not isinstance(fp_optimizer, tf.keras.optimizers.Optimizer):

{"golden_diff": "diff --git a/larq/optimizers_v1.py b/larq/optimizers_v1.py\n--- a/larq/optimizers_v1.py\n+++ b/larq/optimizers_v1.py\n@@ -108,12 +108,25 @@\n class Bop(tf.keras.optimizers.Optimizer):\n     \"\"\"Binary optimizer (Bop).\n \n-    Bop is a latent-free optimizer for Binarized Neural Networks (BNNs).\n+    Bop is a latent-free optimizer for Binarized Neural Networks (BNNs) and\n+    Binary Weight Networks (BWN).\n+\n+    Bop maintains an exponential moving average of the gradients controlled by\n+    `gamma`. If this average exceeds the `threshold`, a weight is flipped.\n+    Additionally, Bop accepts a regular optimizer that is applied to the\n+    non-binary weights in the network.\n+\n+    The hyperparameter `gamma` is somewhat analogues to the learning rate in\n+    SGD methods: a high `gamma` results in rapid convergence but also makes\n+    training more noisy.\n+\n+    Note that the default `threshold` is not optimal for all situations.\n+    Setting the threshold too high results in little learning, while setting it\n+    too low results in overly noisy behaviour.\n \n     !!! example\n         ```python\n         optimizer = lq.optimizers.Bop(fp_optimizer=tf.keras.optimizers.Adam(0.01))\n-\n         ```\n \n     # Arguments\n@@ -126,7 +139,7 @@\n     - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://arxiv.org/abs/1906.02107)\n     \"\"\"\n \n-    def __init__(self, fp_optimizer, threshold=1e-5, gamma=1e-2, name=\"Bop\", **kwargs):\n+    def __init__(self, fp_optimizer, threshold=1e-7, gamma=1e-2, name=\"Bop\", **kwargs):\n         super().__init__(**kwargs)\n \n         if not isinstance(fp_optimizer, tf.keras.optimizers.Optimizer):\ndiff --git a/larq/optimizers_v2.py b/larq/optimizers_v2.py\n--- a/larq/optimizers_v2.py\n+++ b/larq/optimizers_v2.py\n@@ -9,7 +9,21 @@\n class Bop(tf.keras.optimizers.Optimizer):\n     \"\"\"Binary optimizer (Bop).\n \n-    Bop is a latent-free optimizer for Binarized Neural Networks (BNNs).\n+    Bop is a latent-free optimizer for Binarized Neural Networks (BNNs) and\n+    Binary Weight Networks (BWN).\n+\n+    Bop maintains an exponential moving average of the gradients controlled by\n+    `gamma`. If this average exceeds the `threshold`, a weight is flipped.\n+    Additionally, Bop accepts a regular optimizer that is applied to the\n+    non-binary weights in the network.\n+\n+    The hyperparameter `gamma` is somewhat analogues to the learning rate in\n+    SGD methods: a high `gamma` results in rapid convergence but also makes\n+    training more noisy.\n+\n+    Note that the default `threshold` is not optimal for all situations.\n+    Setting the threshold too high results in little learning, while setting it\n+    too low results in overly noisy behaviour.\n \n     !!! example\n         ```python\n@@ -26,7 +40,7 @@\n     - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://arxiv.org/abs/1906.02107)\n     \"\"\"\n \n-    def __init__(self, fp_optimizer, threshold=1e-5, gamma=1e-2, name=\"Bop\", **kwargs):\n+    def __init__(self, fp_optimizer, threshold=1e-7, gamma=1e-2, name=\"Bop\", **kwargs):\n         super().__init__(name=name, **kwargs)\n \n         if not isinstance(fp_optimizer, tf.keras.optimizers.Optimizer):\n", "issue": "Adding binary optimizer (Bop) documentation \n- [ ] enhancing Bop documentation in code\r\n- [ ] adding Bop documentation/tutorial to [Larq official documentation page](https://plumerai.github.io/larq/)\n", "before_files": [{"content": "import tensorflow as tf\nimport larq as lq\n\nfrom larq import utils\nfrom copy import deepcopy\n\n\n@utils.register_keras_custom_object\nclass Bop(tf.keras.optimizers.Optimizer):\n    \"\"\"Binary optimizer (Bop).\n\n    Bop is a latent-free optimizer for Binarized Neural Networks (BNNs).\n\n    !!! example\n        ```python\n        optimizer = lq.optimizers.Bop(fp_optimizer=tf.keras.optimizers.Adam(0.01))\n        ```\n\n    # Arguments\n    fp_optimizer: a `tf.keras.optimizers.Optimizer`.\n    threshold: determines to whether to flip each weight.\n    gamma: the adaptivity rate.\n    name: name of the optimizer.\n\n    # References\n    - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://arxiv.org/abs/1906.02107)\n    \"\"\"\n\n    def __init__(self, fp_optimizer, threshold=1e-5, gamma=1e-2, name=\"Bop\", **kwargs):\n        super().__init__(name=name, **kwargs)\n\n        if not isinstance(fp_optimizer, tf.keras.optimizers.Optimizer):\n            raise TypeError(\n                f\"Expected tf.keras.optimizers.Optimizer, received {type(fp_optimizer)}.\"\n            )\n\n        self.fp_optimizer = fp_optimizer\n        self._set_hyper(\"threshold\", threshold)\n        self._set_hyper(\"gamma\", gamma)\n\n    def _create_slots(self, var_list):\n        for var in var_list:\n            if self.is_binary(var):\n                self.add_slot(var, \"m\")\n\n    def apply_gradients(self, grads_and_vars, name=None):\n        bin_grads_and_vars = [(g, v) for g, v in grads_and_vars if self.is_binary(v)]\n        fp_grads_and_vars = [(g, v) for g, v in grads_and_vars if not self.is_binary(v)]\n\n        bin_train_op = super().apply_gradients(bin_grads_and_vars, name=name)\n        fp_train_op = self.fp_optimizer.apply_gradients(fp_grads_and_vars, name=name)\n\n        return tf.group(bin_train_op, fp_train_op, name=\"train_with_bop\")\n\n    def _resource_apply_sparse(self, grad, var, indices):\n        raise NotImplementedError()\n\n    def __getattr__(self, name):\n        if name == \"lr\":\n            return self.fp_optimizer.lr\n        return super().__getattr__(name)\n\n    def _get_decayed_hyper(self, name, var_dtype):\n        hyper = self._get_hyper(name, var_dtype)\n        if isinstance(hyper, tf.keras.optimizers.schedules.LearningRateSchedule):\n            local_step = tf.cast(self.iterations, var_dtype)\n            hyper = tf.cast(hyper(local_step), var_dtype)\n        return hyper\n\n    def _resource_apply_dense(self, grad, var):\n        var_dtype = var.dtype.base_dtype\n        gamma = self._get_decayed_hyper(\"gamma\", var_dtype)\n        threshold = self._get_decayed_hyper(\"threshold\", var_dtype)\n        m = self.get_slot(var, \"m\")\n\n        m_t = tf.compat.v1.assign(\n            m, (1 - gamma) * m + gamma * grad, use_locking=self._use_locking\n        )\n        var_t = lq.math.sign(-tf.sign(var * m_t - threshold) * var)\n        return tf.compat.v1.assign(var, var_t, use_locking=self._use_locking).op\n\n    @staticmethod\n    def is_binary(var):\n        return \"/kernel\" in var.name and \"quant_\" in var.name\n\n    def get_config(self):\n        fp_optimizer_config = self.fp_optimizer.get_config()\n        config = {\n            \"threshold\": self._serialize_hyperparameter(\"threshold\"),\n            \"gamma\": self._serialize_hyperparameter(\"gamma\"),\n            \"fp_optimizer\": {\n                \"class_name\": fp_optimizer_config[\"name\"],\n                \"config\": fp_optimizer_config,\n            },\n        }\n        return {**super().get_config(), **config}\n\n    @classmethod\n    def from_config(cls, config, custom_objects=None):\n        new_config = deepcopy(config)\n        fp_optimizer = tf.keras.optimizers.deserialize(\n            new_config[\"fp_optimizer\"], custom_objects=custom_objects\n        )\n        new_config.pop(\"fp_optimizer\", None)\n        return cls(fp_optimizer, **new_config)\n", "path": "larq/optimizers_v2.py"}, {"content": "import tensorflow as tf\nimport numpy as np\nimport larq as lq\n\nfrom larq import utils\nfrom copy import deepcopy\n\n\n@utils.register_keras_custom_object\nclass XavierLearningRateScaling(tf.keras.optimizers.Optimizer):\n    \"\"\"Optimizer wrapper for Xavier Learning Rate Scaling\n\n    Scale the weights learning rates respectively with the weights initialization\n\n    !!! note \"\"\n        This is a wrapper and does not implement any optimization algorithm.\n\n    !!! example\n        ```python\n        optimizer = lq.optimizers.XavierLearningRateScaling(\n            tf.keras.optimizers.Adam(0.01), model\n        )\n        ```\n\n    # Arguments\n    optimizer: A `tf.keras.optimizers.Optimizer`\n    model: A `tf.keras.Model`\n\n    # References\n    - [BinaryConnect: Training Deep Neural Networks with binary weights during\n      propagations](https://arxiv.org/abs/1511.00363)\n    \"\"\"\n\n    def __init__(self, optimizer, model):\n        if int(tf.__version__[0]) == 2:\n            raise NotImplementedError(\n                \"XavierLearningRateScaling is not supported by Tensorflow 2.0.\"\n            )\n\n        if not isinstance(optimizer, tf.keras.optimizers.Optimizer):\n            raise ValueError(\n                f\"Expected tf.keras.optimizers.Optimizer, received {type(optimizer)}.\"\n            )\n        self.optimizer = optimizer\n\n        if isinstance(model, tf.keras.Model):\n            self.multipliers = {}\n            for layer in model.layers:\n                if hasattr(layer, \"quantized_latent_weights\"):\n                    for weight in layer.quantized_latent_weights:\n                        self.multipliers[weight.name] = self.get_lr_multiplier(weight)\n        elif isinstance(model, dict):\n            self.multipliers = model\n        else:\n            raise ValueError(f\"Expected tf.keras.Model or dict, received {type(model)}\")\n\n    def get_lr_multiplier(self, weight):\n        shape = weight.get_shape().as_list()\n        n_input = shape[-2]\n        n_output = shape[-1]\n        if len(shape) == 4:\n            kernelsize = np.prod(shape[:-2])\n            coeff = 1.0 / np.sqrt(1.5 / ((kernelsize * (n_input + n_output))))\n        elif len(shape) == 2:\n            coeff = 1.0 / np.sqrt(1.5 / ((1.0 * (n_input + n_output))))\n        else:\n            raise NotImplementedError(\n                \"Xavier Learning rate scaling not implimented for this kernelsize\"\n            )\n        return coeff\n\n    def get_updates(self, loss, params):\n        mult_lr_params = [p for p in params if p.name in self.multipliers]\n        base_lr_params = [p for p in params if p.name not in self.multipliers]\n\n        updates = []\n        base_lr = self.optimizer.lr\n        for param in mult_lr_params:\n            self.optimizer.lr = base_lr * self.multipliers[param.name]\n            updates.extend(self.optimizer.get_updates(loss, [param]))\n\n        self.optimizer.lr = base_lr\n        updates.extend(self.optimizer.get_updates(loss, base_lr_params))\n\n        return updates\n\n    def __getattr__(self, name):\n        return getattr(self.optimizer, name)\n\n    def get_config(self):\n        return {\n            \"optimizer\": {\n                \"class_name\": self.optimizer.__class__.__name__,\n                \"config\": self.optimizer.get_config(),\n            },\n            \"multipliers\": self.multipliers,\n        }\n\n    @classmethod\n    def from_config(cls, config, custom_objects=None):\n        optimizer = tf.keras.optimizers.deserialize(\n            config.pop(\"optimizer\"), custom_objects=custom_objects\n        )\n        return cls(optimizer, config[\"multipliers\"])\n\n\n@utils.register_keras_custom_object\nclass Bop(tf.keras.optimizers.Optimizer):\n    \"\"\"Binary optimizer (Bop).\n\n    Bop is a latent-free optimizer for Binarized Neural Networks (BNNs).\n\n    !!! example\n        ```python\n        optimizer = lq.optimizers.Bop(fp_optimizer=tf.keras.optimizers.Adam(0.01))\n\n        ```\n\n    # Arguments\n    fp_optimizer: a `tf.keras.optimizers.Optimizer`.\n    threshold: determines to whether to flip each weight.\n    gamma: the adaptivity rate.\n    name: name of the optimizer.\n\n    # References\n    - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://arxiv.org/abs/1906.02107)\n    \"\"\"\n\n    def __init__(self, fp_optimizer, threshold=1e-5, gamma=1e-2, name=\"Bop\", **kwargs):\n        super().__init__(**kwargs)\n\n        if not isinstance(fp_optimizer, tf.keras.optimizers.Optimizer):\n            raise TypeError(\n                f\"Expected tf.keras.optimizers.Optimizer, received {type(fp_optimizer)}.\"\n            )\n\n        with tf.keras.backend.name_scope(self.__class__.__name__):\n\n            self.fp_optimizer = fp_optimizer\n            self.threshold = tf.keras.backend.variable(threshold, name=\"threshold\")\n            self.gamma = tf.keras.backend.variable(gamma, name=\"gamma\")\n\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = []\n\n        ms = [\n            tf.keras.backend.zeros(\n                tf.keras.backend.int_shape(p), dtype=tf.keras.backend.dtype(p)\n            )\n            for p in params\n        ]\n\n        fp_params = []\n\n        for p, g, m in zip(params, grads, ms):\n            if self.is_binary(p):\n                m_t = (1 - self.gamma) * m + self.gamma * g\n\n                self.updates.append(tf.assign(m, m_t))\n                self.updates.append(\n                    tf.assign(p, lq.math.sign(-p * tf.sign(p * m_t - self.threshold)))\n                )\n\n            else:\n                fp_params.append(p)\n\n        return self.updates + self.fp_optimizer.get_updates(loss, fp_params)\n\n    @staticmethod\n    def is_binary(var):\n        return \"/kernel\" in var.name and \"quant_\" in var.name\n\n    def __getattr__(self, name):\n        if name == \"lr\":\n            return self.fp_optimizer.lr\n        return super().__getattr__(name)\n\n    def get_config(self):\n        fp_optimizer_config = self.fp_optimizer.get_config()\n        config = {\n            \"threshold\": float(tf.keras.backend.get_value(self.threshold)),\n            \"gamma\": float(tf.keras.backend.get_value(self.gamma)),\n            \"fp_optimizer\": {\n                \"class_name\": self.fp_optimizer.__class__.__name__,\n                \"config\": fp_optimizer_config,\n            },\n        }\n        return {**super().get_config(), **config}\n\n    @classmethod\n    def from_config(cls, config, custom_objects=None):\n        new_config = deepcopy(config)\n        fp_optimizer = tf.keras.optimizers.deserialize(\n            new_config[\"fp_optimizer\"], custom_objects=custom_objects\n        )\n        new_config.pop(\"fp_optimizer\", None)\n        return cls(fp_optimizer, **new_config)\n", "path": "larq/optimizers_v1.py"}], "after_files": [{"content": "import tensorflow as tf\nimport larq as lq\n\nfrom larq import utils\nfrom copy import deepcopy\n\n\n@utils.register_keras_custom_object\nclass Bop(tf.keras.optimizers.Optimizer):\n    \"\"\"Binary optimizer (Bop).\n\n    Bop is a latent-free optimizer for Binarized Neural Networks (BNNs) and\n    Binary Weight Networks (BWN).\n\n    Bop maintains an exponential moving average of the gradients controlled by\n    `gamma`. If this average exceeds the `threshold`, a weight is flipped.\n    Additionally, Bop accepts a regular optimizer that is applied to the\n    non-binary weights in the network.\n\n    The hyperparameter `gamma` is somewhat analogues to the learning rate in\n    SGD methods: a high `gamma` results in rapid convergence but also makes\n    training more noisy.\n\n    Note that the default `threshold` is not optimal for all situations.\n    Setting the threshold too high results in little learning, while setting it\n    too low results in overly noisy behaviour.\n\n    !!! example\n        ```python\n        optimizer = lq.optimizers.Bop(fp_optimizer=tf.keras.optimizers.Adam(0.01))\n        ```\n\n    # Arguments\n    fp_optimizer: a `tf.keras.optimizers.Optimizer`.\n    threshold: determines to whether to flip each weight.\n    gamma: the adaptivity rate.\n    name: name of the optimizer.\n\n    # References\n    - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://arxiv.org/abs/1906.02107)\n    \"\"\"\n\n    def __init__(self, fp_optimizer, threshold=1e-7, gamma=1e-2, name=\"Bop\", **kwargs):\n        super().__init__(name=name, **kwargs)\n\n        if not isinstance(fp_optimizer, tf.keras.optimizers.Optimizer):\n            raise TypeError(\n                f\"Expected tf.keras.optimizers.Optimizer, received {type(fp_optimizer)}.\"\n            )\n\n        self.fp_optimizer = fp_optimizer\n        self._set_hyper(\"threshold\", threshold)\n        self._set_hyper(\"gamma\", gamma)\n\n    def _create_slots(self, var_list):\n        for var in var_list:\n            if self.is_binary(var):\n                self.add_slot(var, \"m\")\n\n    def apply_gradients(self, grads_and_vars, name=None):\n        bin_grads_and_vars = [(g, v) for g, v in grads_and_vars if self.is_binary(v)]\n        fp_grads_and_vars = [(g, v) for g, v in grads_and_vars if not self.is_binary(v)]\n\n        bin_train_op = super().apply_gradients(bin_grads_and_vars, name=name)\n        fp_train_op = self.fp_optimizer.apply_gradients(fp_grads_and_vars, name=name)\n\n        return tf.group(bin_train_op, fp_train_op, name=\"train_with_bop\")\n\n    def _resource_apply_sparse(self, grad, var, indices):\n        raise NotImplementedError()\n\n    def __getattr__(self, name):\n        if name == \"lr\":\n            return self.fp_optimizer.lr\n        return super().__getattr__(name)\n\n    def _get_decayed_hyper(self, name, var_dtype):\n        hyper = self._get_hyper(name, var_dtype)\n        if isinstance(hyper, tf.keras.optimizers.schedules.LearningRateSchedule):\n            local_step = tf.cast(self.iterations, var_dtype)\n            hyper = tf.cast(hyper(local_step), var_dtype)\n        return hyper\n\n    def _resource_apply_dense(self, grad, var):\n        var_dtype = var.dtype.base_dtype\n        gamma = self._get_decayed_hyper(\"gamma\", var_dtype)\n        threshold = self._get_decayed_hyper(\"threshold\", var_dtype)\n        m = self.get_slot(var, \"m\")\n\n        m_t = tf.compat.v1.assign(\n            m, (1 - gamma) * m + gamma * grad, use_locking=self._use_locking\n        )\n        var_t = lq.math.sign(-tf.sign(var * m_t - threshold) * var)\n        return tf.compat.v1.assign(var, var_t, use_locking=self._use_locking).op\n\n    @staticmethod\n    def is_binary(var):\n        return \"/kernel\" in var.name and \"quant_\" in var.name\n\n    def get_config(self):\n        fp_optimizer_config = self.fp_optimizer.get_config()\n        config = {\n            \"threshold\": self._serialize_hyperparameter(\"threshold\"),\n            \"gamma\": self._serialize_hyperparameter(\"gamma\"),\n            \"fp_optimizer\": {\n                \"class_name\": fp_optimizer_config[\"name\"],\n                \"config\": fp_optimizer_config,\n            },\n        }\n        return {**super().get_config(), **config}\n\n    @classmethod\n    def from_config(cls, config, custom_objects=None):\n        new_config = deepcopy(config)\n        fp_optimizer = tf.keras.optimizers.deserialize(\n            new_config[\"fp_optimizer\"], custom_objects=custom_objects\n        )\n        new_config.pop(\"fp_optimizer\", None)\n        return cls(fp_optimizer, **new_config)\n", "path": "larq/optimizers_v2.py"}, {"content": "import tensorflow as tf\nimport numpy as np\nimport larq as lq\n\nfrom larq import utils\nfrom copy import deepcopy\n\n\n@utils.register_keras_custom_object\nclass XavierLearningRateScaling(tf.keras.optimizers.Optimizer):\n    \"\"\"Optimizer wrapper for Xavier Learning Rate Scaling\n\n    Scale the weights learning rates respectively with the weights initialization\n\n    !!! note \"\"\n        This is a wrapper and does not implement any optimization algorithm.\n\n    !!! example\n        ```python\n        optimizer = lq.optimizers.XavierLearningRateScaling(\n            tf.keras.optimizers.Adam(0.01), model\n        )\n        ```\n\n    # Arguments\n    optimizer: A `tf.keras.optimizers.Optimizer`\n    model: A `tf.keras.Model`\n\n    # References\n    - [BinaryConnect: Training Deep Neural Networks with binary weights during\n      propagations](https://arxiv.org/abs/1511.00363)\n    \"\"\"\n\n    def __init__(self, optimizer, model):\n        if int(tf.__version__[0]) == 2:\n            raise NotImplementedError(\n                \"XavierLearningRateScaling is not supported by Tensorflow 2.0.\"\n            )\n\n        if not isinstance(optimizer, tf.keras.optimizers.Optimizer):\n            raise ValueError(\n                f\"Expected tf.keras.optimizers.Optimizer, received {type(optimizer)}.\"\n            )\n        self.optimizer = optimizer\n\n        if isinstance(model, tf.keras.Model):\n            self.multipliers = {}\n            for layer in model.layers:\n                if hasattr(layer, \"quantized_latent_weights\"):\n                    for weight in layer.quantized_latent_weights:\n                        self.multipliers[weight.name] = self.get_lr_multiplier(weight)\n        elif isinstance(model, dict):\n            self.multipliers = model\n        else:\n            raise ValueError(f\"Expected tf.keras.Model or dict, received {type(model)}\")\n\n    def get_lr_multiplier(self, weight):\n        shape = weight.get_shape().as_list()\n        n_input = shape[-2]\n        n_output = shape[-1]\n        if len(shape) == 4:\n            kernelsize = np.prod(shape[:-2])\n            coeff = 1.0 / np.sqrt(1.5 / ((kernelsize * (n_input + n_output))))\n        elif len(shape) == 2:\n            coeff = 1.0 / np.sqrt(1.5 / ((1.0 * (n_input + n_output))))\n        else:\n            raise NotImplementedError(\n                \"Xavier Learning rate scaling not implimented for this kernelsize\"\n            )\n        return coeff\n\n    def get_updates(self, loss, params):\n        mult_lr_params = [p for p in params if p.name in self.multipliers]\n        base_lr_params = [p for p in params if p.name not in self.multipliers]\n\n        updates = []\n        base_lr = self.optimizer.lr\n        for param in mult_lr_params:\n            self.optimizer.lr = base_lr * self.multipliers[param.name]\n            updates.extend(self.optimizer.get_updates(loss, [param]))\n\n        self.optimizer.lr = base_lr\n        updates.extend(self.optimizer.get_updates(loss, base_lr_params))\n\n        return updates\n\n    def __getattr__(self, name):\n        return getattr(self.optimizer, name)\n\n    def get_config(self):\n        return {\n            \"optimizer\": {\n                \"class_name\": self.optimizer.__class__.__name__,\n                \"config\": self.optimizer.get_config(),\n            },\n            \"multipliers\": self.multipliers,\n        }\n\n    @classmethod\n    def from_config(cls, config, custom_objects=None):\n        optimizer = tf.keras.optimizers.deserialize(\n            config.pop(\"optimizer\"), custom_objects=custom_objects\n        )\n        return cls(optimizer, config[\"multipliers\"])\n\n\n@utils.register_keras_custom_object\nclass Bop(tf.keras.optimizers.Optimizer):\n    \"\"\"Binary optimizer (Bop).\n\n    Bop is a latent-free optimizer for Binarized Neural Networks (BNNs) and\n    Binary Weight Networks (BWN).\n\n    Bop maintains an exponential moving average of the gradients controlled by\n    `gamma`. If this average exceeds the `threshold`, a weight is flipped.\n    Additionally, Bop accepts a regular optimizer that is applied to the\n    non-binary weights in the network.\n\n    The hyperparameter `gamma` is somewhat analogues to the learning rate in\n    SGD methods: a high `gamma` results in rapid convergence but also makes\n    training more noisy.\n\n    Note that the default `threshold` is not optimal for all situations.\n    Setting the threshold too high results in little learning, while setting it\n    too low results in overly noisy behaviour.\n\n    !!! example\n        ```python\n        optimizer = lq.optimizers.Bop(fp_optimizer=tf.keras.optimizers.Adam(0.01))\n        ```\n\n    # Arguments\n    fp_optimizer: a `tf.keras.optimizers.Optimizer`.\n    threshold: determines to whether to flip each weight.\n    gamma: the adaptivity rate.\n    name: name of the optimizer.\n\n    # References\n    - [Latent Weights Do Not Exist: Rethinking Binarized Neural Network Optimization](https://arxiv.org/abs/1906.02107)\n    \"\"\"\n\n    def __init__(self, fp_optimizer, threshold=1e-7, gamma=1e-2, name=\"Bop\", **kwargs):\n        super().__init__(**kwargs)\n\n        if not isinstance(fp_optimizer, tf.keras.optimizers.Optimizer):\n            raise TypeError(\n                f\"Expected tf.keras.optimizers.Optimizer, received {type(fp_optimizer)}.\"\n            )\n\n        with tf.keras.backend.name_scope(self.__class__.__name__):\n\n            self.fp_optimizer = fp_optimizer\n            self.threshold = tf.keras.backend.variable(threshold, name=\"threshold\")\n            self.gamma = tf.keras.backend.variable(gamma, name=\"gamma\")\n\n    def get_updates(self, loss, params):\n        grads = self.get_gradients(loss, params)\n        self.updates = []\n\n        ms = [\n            tf.keras.backend.zeros(\n                tf.keras.backend.int_shape(p), dtype=tf.keras.backend.dtype(p)\n            )\n            for p in params\n        ]\n\n        fp_params = []\n\n        for p, g, m in zip(params, grads, ms):\n            if self.is_binary(p):\n                m_t = (1 - self.gamma) * m + self.gamma * g\n\n                self.updates.append(tf.assign(m, m_t))\n                self.updates.append(\n                    tf.assign(p, lq.math.sign(-p * tf.sign(p * m_t - self.threshold)))\n                )\n\n            else:\n                fp_params.append(p)\n\n        return self.updates + self.fp_optimizer.get_updates(loss, fp_params)\n\n    @staticmethod\n    def is_binary(var):\n        return \"/kernel\" in var.name and \"quant_\" in var.name\n\n    def __getattr__(self, name):\n        if name == \"lr\":\n            return self.fp_optimizer.lr\n        return super().__getattr__(name)\n\n    def get_config(self):\n        fp_optimizer_config = self.fp_optimizer.get_config()\n        config = {\n            \"threshold\": float(tf.keras.backend.get_value(self.threshold)),\n            \"gamma\": float(tf.keras.backend.get_value(self.gamma)),\n            \"fp_optimizer\": {\n                \"class_name\": self.fp_optimizer.__class__.__name__,\n                \"config\": fp_optimizer_config,\n            },\n        }\n        return {**super().get_config(), **config}\n\n    @classmethod\n    def from_config(cls, config, custom_objects=None):\n        new_config = deepcopy(config)\n        fp_optimizer = tf.keras.optimizers.deserialize(\n            new_config[\"fp_optimizer\"], custom_objects=custom_objects\n        )\n        new_config.pop(\"fp_optimizer\", None)\n        return cls(fp_optimizer, **new_config)\n", "path": "larq/optimizers_v1.py"}]}