gobeetle: store redis gc stats in redis and display them in the config server web UI
MODIFY
Makefile
Makefile
@@ -47,7 +47,7 @@ GO_MODULES = $(patsubst %,$(GO_SRC)/%, client.go server.go datatypes.go server_s
go get github.com/gobuffalo/packr/packr
touch .packr
-$(GO_SRC)/a_main-packr.go: $(GO_SRC)/templates/index.html .packr
+$(GO_SRC)/a_main-packr.go: $(wildcard $(GO_SRC)/templates/*.html) .packr
cd $(GO_SRC) && packr
beetle: $(GO_SRC)/beetle.go $(GO_MODULES) $(GO_SRC)/a_main-packr.go

MODIFY
go/a_main-packr.go
go/a_main-packr.go
@@ -7,5 +7,6 @@ import "github.com/gobuffalo/packr"
// You can use the "packr clean" command to clean up this,
// and any other packr generated files.
func init() {
- packr.PackJSONBytes("./templates", "index.html", "\"PCFkb2N0eXBlIGh0bWw+CjxodG1sPgogIDxoZWFkPgogICAgPHRpdGxlPkJlZXRsZSBDb25maWd1cmF0aW9uIFNlcnZlciBTdGF0dXM8L3RpdGxlPgogICAgPHN0eWxlIG1lZGlhPSJzY3JlZW4iIHR5cGU9InRleHQvY3NzIj4KICAgICAgaHRtbCB7IGZvbnQ6IDEuMjVlbS8xLjUgYXJpYWwsIHNhbnMtc2VyaWY7fQogICAgICBib2R5IHsgbWFyZ2luOiAxZW07IH0KICAgICAgdGQgeyBwYWRkaW5nOiAwLjFlbSAwLjJlbTsgdmVydGljYWwtYWxpZ246IHRvcDsgfQogICAgICB1bCB7IGxpc3Qtc3R5bGUtdHlwZTogbm9uZTsgbWFyZ2luOiAwOyBwYWRkaW5nOiAwO30KICAgICAgbGkgeyB9CiAgICAgIGgxLmF2YWlsYWJsZSB7IGNvbG9yOiAjNTc4MGIyOyBtYXJnaW4tYm90dG9tOiAwLjJlbTt9CiAgICAgIGgxLnVuYXZhaWxhYmxlIHsgY29sb3I6ICNBNTJBMkE7IG1hcmdpbi1ib3R0b206IDAuMmVtO30KICAgICAgYTpsaW5rLCBhOnZpc2l0ZWQge3RleHQtZGVjb3JhdGlvbjpub25lOyBjb2xvcjojQTUyQTJBO30KICAgICAgYTpob3ZlciwgYTphY3RpdmUge3RleHQtZGVjb3JhdGlvbjpub25lOyBjb2xvcjojRkYwMDAwO30KICAgICAgYSB7CiAgICAgICAgICBwYWRkaW5nOiAxMHB4OyBiYWNrZ3JvdW5kOiAjY2RjZGNkOwogICAgICAgICAgLW1vei1ib3JkZXItcmFkaXVzOiA1cHg7CiAgICAgICAgICBib3JkZXItcmFkaXVzOiA1cHg7CiAgICAgICAgICAtbW96LWJveC1zaGFkb3c6IDJweCAycHggMnB4ICNiYmI7CiAgICAgICAgICAtd2Via2l0LWJveC1zaGFkb3c6IDJweCAycHggMnB4ICNiYmI7CiAgICAgICAgICBib3gtc2hhZG93OiAycHggMnB4IDJweCAjYmJiOwogICAgICB9CiAgICAgIGZvcm0geyBmb250LXNpemU6IDFlbTsgbWFyZ2luLWJvdHRvbTogMWVtOyB9CiAgICA8L3N0eWxlPgogIDwvaGVhZD4KICA8Ym9keT4KICAgIHt7IHJhbmdlIC5TeXN0ZW1zIH19CiAgICA8aDEgY2xhc3M9Int7IGlmIC5SZWRpc01hc3RlckF2YWlsYWJsZSB9fWF2YWlsYWJsZXt7IGVsc2UgfX11bmF2YWlsYWJsZXt7IGVuZCB9fSI+QmVldGxlIENvbmZpZ3VyYXRpb24gU2VydmVyIFN0YXR1czwvaDE+CiAgICB7eyBpZiBub3QgLlJlZGlzTWFzdGVyQXZhaWxhYmxlIH19CiAgICA8Zm9ybSBuYW1lPSdtYXN0ZXJzd2l0Y2gnIG1ldGhvZD0ncG9zdCcgYWN0aW9uPScvaW5pdGlhdGVfbWFzdGVyX3N3aXRjaCZhbXA7c3lzdGVtX25hbWU9e3sgLlN5c3RlbU5hbWUgfX0nPgogICAgICBNYXN0ZXIgZG93biEKICAgICAgPGEgaHJlZj0namF2YXNjcmlwdDogZG9jdW1lbnQubWFzdGVyc3dpdGNoLnN1Ym1pdCgpOyc+SW5pdGlhdGUgbWFzdGVyIHN3aXRjaDwvYT4KICAgICAgb3Igd2FpdCB1bnRpbCBpdCBpcyBwZXJmb3JtZWQgaXQgYXV0b21hdGljYWxseS4KICAgIDwvZm9ybT4KICAgIHt7IGVuZCB9fQogICAgPHRhYmxlIGNlbGxzcGFjaW5nPTA+CiAgICAgIDx0cj48dGQ+c3lzdGVtX25hbWU8L3RkPjx0ZD57eyAuU3lzdGVtTmFtZX19PC90ZD48L3RyPgogICAgICA8dHI+PHRkPnN3aXRjaF9pbl9wcm9ncmVzczwvdGQ+PHRkPnt7IC5Td2l0Y2hJblByb2dyZXNzfX08L3RkPjwvdHI+CiAgICAgIDx0cj48dGQ+cmVkaXNfbWFzdGVyX2F2YWlsYWJsZTwvdGQ+PHRkPjx1bD57eyAuUmVkaXNNYXN0ZXJBdmFpbGFibGUgfX08L3RkPjwvdHI+CiAgICAgIDx0cj48dGQ+cmVkaXNfbWFzdGVyPC90ZD48dGQ+e3sgLlJlZGlzTWFzdGVyfX08L3RkPjwvdHI+CiAgICAgIDx0cj48dGQ+cmVkaXNfc2xhdmVzX2F2YWlsYWJsZTwvdGQ+PHRkPjx1bD57eyBpZiBub3QgLlJlZGlzU2xhdmVzQXZhaWxhYmxlIH19bm9uZXt7IGVsc2UgfX17eyByYW5nZSAuUmVkaXNTbGF2ZXNBdmFpbGFibGUgfX08bGk+e3sgLiB9fTwvbGk+e3sgZW5kIH19e3sgZW5kIH19PC91bD48L3RkPjwvdHI+CiAgICAgIDx0cj48dGQ+Y29uZmlndXJlZF9yZWRpc19zZXJ2ZXJzPC90ZD48dGQ+PHVsPnt7IHJhbmdlIC5Db25maWd1cmVkUmVkaXNTZXJ2ZXJzIH19PGxpPnt7IC4gfX08L2xpPnt7IGVuZCB9fTwvdWw+PC90ZD48L3RyPgogICAgPC90YWJsZT4KICAgIHt7IGVuZCB9fQogICAgPGgxIGNsYXNzPSJhdmFpbGFibGUiPkdsb2JhbCBDb25maWd1cmF0aW9uPC9oMj4KICAgIDx0YWJsZSBjZWxsc3BhY2luZz0wPgogICAgICA8dHI+PHRkPmJlZXRsZV92ZXJzaW9uPC90ZD48dGQ+e3sgLkJlZXRsZVZlcnNpb259fTwvdGQ+PC90cj4KICAgICAgPHRyPjx0ZD51bnNlZW5fY2xpZW50X2lkczwvdGQ+PHRkPjx1bD57eyBpZiBub3QgLlVuc2VlbkNsaWVudElkcyB9fW5vbmV7eyBlbHNlIH19e3sgcmFuZ2UgLlVuc2VlbkNsaWVudElkcyB9fTxsaT57eyAuIH19PC9saT57eyBlbmQgfX17eyBlbmQgfX08L3VsPjwvdGQ+PC90cj4KICAgICAgPHRyPjx0ZD51bnJlc3BvbnNpdmVfY2xpZW50czwvdGQ+PHRkPjx1bD57eyBpZiBub3QgLlVucmVzcG9uc2l2ZUNsaWVudHMgfX1ub25le3sgZWxzZSB9fXt7IHJhbmdlIC5VbnJlc3BvbnNpdmVDbGllbnRzIH19PGxpPnt7IC4gfX08L2xpPnt7IGVuZCB9fXt7IGVuZCB9fTwvdWw+PC90ZD48L3RyPgogICAgICA8dHI+PHRkPnVua25vd25fY2xpZW50X2lkczwvdGQ+PHRkPjx1bD57eyBpZiBub3QgLlVua25vd25DbGllbnRJZHMgfX1ub25le3sgZWxzZSB9fXt7IHJhbmdlIC5Vbmtub3duQ2xpZW50SWRzIH19PGxpPnt7IC4gfX08L2xpPnt7IGVuZCB9fXt7IGVuZCB9fTwvdWw+PC90ZD48L3RyPgogICAgICA8dHI+PHRkPmNvbmZpZ3VyZWRfY2xpZW50X2lkczwvdGQ+PHRkPjx1bD57eyByYW5nZSAuQ29uZmlndXJlZENsaWVudElkcyB9fTxsaT57eyAuIH19PC9saT57eyBlbmQgfX08L3VsPjwvdGQ+PC90cj4KICAgIDwvdGFibGU+CiAgPC9ib2R5Pgo8L2h0bWw+Cg==\"")
+ packr.PackJSONBytes("./templates", "gcstats.html", "\"PCFkb2N0eXBlIGh0bWw+CjxodG1sPgogIDxoZWFkPgogICAgPHRpdGxlPkdDIFN0YXRpc3RpY3M8L3RpdGxlPgogICAgPHN0eWxlIG1lZGlhPSJzY3JlZW4iIHR5cGU9InRleHQvY3NzIj4KICAgICAgaHRtbCB7IGZvbnQ6IDEuMWVtLzEuMiBhcmlhbCwgc2Fucy1zZXJpZjt9CiAgICAgIGJvZHkgeyBtYXJnaW46IDFlbTsgfQogICAgICB0aCB7IHBhZGRpbmc6IDNweCAzcHg7IHZlcnRpY2FsLWFsaWduOiB0b3A7IGZvbnQtd2VpZ2h0OiBub3JtYWw7IH0KICAgICAgdGQgeyBwYWRkaW5nOiAxcHggMXB4OyB2ZXJ0aWNhbC1hbGlnbjogdG9wOyB0ZXh0LWFsaWduOiByaWdodDsgfQogICAgICBkbCB7IG1hcmdpbi1sZWZ0OiAwcHg7IHBhZGRpbmc6IDFweDt9CiAgICAgIGRkIHsgbWFyZ2luLWlubGluZS1zdGFydDogMHB4OyBwYWRkaW5nOiAxcHg7fQogICAgICBkdCB7IGNvbG9yOiAjNTc4MGIyOyBmb250LXdlaWdodDogYm9sZDsgZm9udC1zaXplOiAxMTAlO30KICAgICAgaDIgeyBjb2xvcjogIzU3ODBiMjsgfQogICAgICB0YWJsZSB7IG1hcmdpbi1sZWZ0OiAwOyBwYWRkaW5nOiAxcHg7IH0KICAgIDwvc3R5bGU+CiAgPC9oZWFkPgogIDxib2R5PgogICAge3sgaWYgLiB9fQogICAgPGgyPkxhc3QgR0MgY29tcGxldGVkIGF0IHt7IC5UaW1lc3RhbXBIdW1hbiB9fTwvaDI+CiAgICA8cD5BY3RpdmUgZGVkdXAgc3RvcmUgZW50cmllcyBieSBxdWV1ZSwgd2l0aCBhbiBob3VyIGJhc2UgaGlzdG9ncmFtIG9mIGZ1dHVyZSBleHBpcmllcy48L3A+CiAgICA8ZGw+CiAgICAgIHt7IHJhbmdlIC5RdWV1ZXMgfX0KICAgICAgPGR0Pnt7IC5RdWV1ZSB9fToge3sgLlRvdGFsRXhwaXJpZXMgfX08L2R0PgogICAgICA8ZGQ+CiAgICAgICAgPHRhYmxlPgogICAgICAgICAgPGNvbGdyb3VwPgogICAgICAgICAgICA8Y29sIHNwYW49IjEiIHN0eWxlPSJ3aWR0aDogMmVtOyI+CiAgICAgICAgICAgIDxjb2wgc3Bhbj0iMSIgc3R5bGU9IndpZHRoOiA0ZW07Ij4KICAgICAgICAgIDwvY29sZ3JvdXA+CiAgICAgICAgICB7eyByYW5nZSAuRXhwaXJpZXMgfX0KICAgICAgICAgIDx0cj48dGQ+e3sgLkhvdXIgfX1oPC90ZD48dGQ+e3sgLkNvdW50ICB9fTwvdGQ+PC90cj4KICAgICAgICAgIHt7IGVuZCB9fQogICAgICAgIDwvdGFibGU+CiAgICAgIDwvZGQ+CiAgICAgIHt7IGVuZCB9fQogICAgPC9kbD4KICAgIHt7IGVsc2UgfX0KICAgIDxoMj5ObyBHQyBpbmZvcm1hdGlvbiBhdmFpbGFibGU8L2gyPgogICAge3sgZW5kIH19CiAgPC9ib2R5Pgo8L2h0bWw+Cg==\"")
+ packr.PackJSONBytes("./templates", "index.html", "\"PCFkb2N0eXBlIGh0bWw+CjxodG1sPgogIDxoZWFkPgogICAgPHRpdGxlPkJlZXRsZSBDb25maWd1cmF0aW9uIFNlcnZlciBTdGF0dXM8L3RpdGxlPgogICAgPHN0eWxlIG1lZGlhPSJzY3JlZW4iIHR5cGU9InRleHQvY3NzIj4KICAgICAgaHRtbCB7IGZvbnQ6IDEuMjVlbS8xLjUgYXJpYWwsIHNhbnMtc2VyaWY7fQogICAgICBib2R5IHsgbWFyZ2luOiAxZW07IH0KICAgICAgdGQgeyBwYWRkaW5nOiAwLjFlbSAwLjJlbTsgdmVydGljYWwtYWxpZ246IHRvcDsgfQogICAgICB1bCB7IGxpc3Qtc3R5bGUtdHlwZTogbm9uZTsgbWFyZ2luOiAwOyBwYWRkaW5nOiAwO30KICAgICAgbGkgeyB9CiAgICAgIGgxLmF2YWlsYWJsZSB7IGNvbG9yOiAjNTc4MGIyOyBtYXJnaW4tYm90dG9tOiAwLjJlbTt9CiAgICAgIGgxLnVuYXZhaWxhYmxlIHsgY29sb3I6ICNBNTJBMkE7IG1hcmdpbi1ib3R0b206IDAuMmVtO30KICAgICAgYTpsaW5rLCBhOnZpc2l0ZWQge3RleHQtZGVjb3JhdGlvbjpub25lOyBjb2xvcjojNTc4MGIyO30KICAgICAgYTpob3ZlciwgYTphY3RpdmUge3RleHQtZGVjb3JhdGlvbjpub25lOyBjb2xvcjpibHVlO30KICAgICAgYSB7IHRleHQtZGVjb3JhdGlvbjpub25lOwogICAgICAgICAgcGFkZGluZzogNXB4OyBiYWNrZ3JvdW5kOiAjY2RjZGNkOwogICAgICAgICAgLW1vei1ib3JkZXItcmFkaXVzOiA0cHg7CiAgICAgICAgICBib3JkZXItcmFkaXVzOiA0cHg7CiAgICAgICAgICAtbW96LWJveC1zaGFkb3c6IDFweCAxcHggMXB4ICNiYmI7CiAgICAgICAgICAtd2Via2l0LWJveC1zaGFkb3c6IDFweCAxcHggMXB4ICNiYmI7CiAgICAgICAgICBib3gtc2hhZG93OiAxcHggMXB4IDFweCAjYmJiOwogICAgICB9CiAgICAgIGZvcm0gYTpsaW5rLCBmb3JtIGE6dmlzaXRlZCB7dGV4dC1kZWNvcmF0aW9uOm5vbmU7IGNvbG9yOiNBNTJBMkE7fQogICAgICBmb3JtIGE6aG92ZXIsIGZvcm0gYTphY3RpdmUge3RleHQtZGVjb3JhdGlvbjpub25lOyBjb2xvcjojRkYwMDAwO30KICAgICAgZm9ybSBhIHsKICAgICAgICAgIHBhZGRpbmc6IDEwcHg7IGJhY2tncm91bmQ6ICNjZGNkY2Q7CiAgICAgICAgICAtbW96LWJvcmRlci1yYWRpdXM6IDVweDsKICAgICAgICAgIGJvcmRlci1yYWRpdXM6IDVweDsKICAgICAgICAgIC1tb3otYm94LXNoYWRvdzogMnB4IDJweCAycHggI2JiYjsKICAgICAgICAgIC13ZWJraXQtYm94LXNoYWRvdzogMnB4IDJweCAycHggI2JiYjsKICAgICAgICAgIGJveC1zaGFkb3c6IDJweCAycHggMnB4ICNiYmI7CiAgICAgIH0KICAgICAgZm9ybSB7IGZvbnQtc2l6ZTogMWVtOyBtYXJnaW4tYm90dG9tOiAxZW07IH0KICAgIDwvc3R5bGU+CiAgPC9oZWFkPgogIDxib2R5PgogICAge3sgcmFuZ2UgLlN5c3RlbXMgfX0KICAgIDxoMSBjbGFzcz0ie3sgaWYgLlJlZGlzTWFzdGVyQXZhaWxhYmxlIH19YXZhaWxhYmxle3sgZWxzZSB9fXVuYXZhaWxhYmxle3sgZW5kIH19Ij5CZWV0bGUgQ29uZmlndXJhdGlvbiBTZXJ2ZXIgU3RhdHVzPC9oMT4KICAgIHt7IGlmIG5vdCAuUmVkaXNNYXN0ZXJBdmFpbGFibGUgfX0KICAgIDxmb3JtIG5hbWU9J21hc3RlcnN3aXRjaCcgbWV0aG9kPSdwb3N0JyBhY3Rpb249Jy9pbml0aWF0ZV9tYXN0ZXJfc3dpdGNoJmFtcDtzeXN0ZW1fbmFtZT17eyAuU3lzdGVtTmFtZSB9fSc+CiAgICAgIE1hc3RlciBkb3duIQogICAgICA8YSBocmVmPSdqYXZhc2NyaXB0OiBkb2N1bWVudC5tYXN0ZXJzd2l0Y2guc3VibWl0KCk7Jz5Jbml0aWF0ZSBtYXN0ZXIgc3dpdGNoPC9hPgogICAgICBvciB3YWl0IHVudGlsIGl0IGlzIHBlcmZvcm1lZCBpdCBhdXRvbWF0aWNhbGx5LgogICAgPC9mb3JtPgogICAge3sgZW5kIH19CiAgICA8dGFibGUgY2VsbHNwYWNpbmc9MD4KICAgICAgPHRyPjx0ZD5zeXN0ZW1fbmFtZTwvdGQ+PHRkPnt7IC5TeXN0ZW1OYW1lfX08L3RkPjwvdHI+CiAgICAgIDx0cj48dGQ+c3dpdGNoX2luX3Byb2dyZXNzPC90ZD48dGQ+e3sgLlN3aXRjaEluUHJvZ3Jlc3N9fTwvdGQ+PC90cj4KICAgICAgPHRyPjx0ZD5yZWRpc19tYXN0ZXJfYXZhaWxhYmxlPC90ZD48dGQ+PHVsPnt7IC5SZWRpc01hc3RlckF2YWlsYWJsZSB9fTwvdGQ+PC90cj4KICAgICAgPHRyPjx0ZD5yZWRpc19tYXN0ZXI8L3RkPjx0ZD57eyAuUmVkaXNNYXN0ZXJ9fTwvdGQ+PC90cj4KICAgICAgPHRyPjx0ZD5yZWRpc19zbGF2ZXNfYXZhaWxhYmxlPC90ZD48dGQ+PHVsPnt7IGlmIG5vdCAuUmVkaXNTbGF2ZXNBdmFpbGFibGUgfX1ub25le3sgZWxzZSB9fXt7IHJhbmdlIC5SZWRpc1NsYXZlc0F2YWlsYWJsZSB9fTxsaT57eyAuIH19PC9saT57eyBlbmQgfX17eyBlbmQgfX08L3VsPjwvdGQ+PC90cj4KICAgICAgPHRyPjx0ZD5jb25maWd1cmVkX3JlZGlzX3NlcnZlcnM8L3RkPjx0ZD48dWw+e3sgcmFuZ2UgLkNvbmZpZ3VyZWRSZWRpc1NlcnZlcnMgfX08bGk+e3sgLiB9fTwvbGk+e3sgZW5kIH19PC91bD48L3RkPjwvdHI+CiAgICAgIDx0cj48dGQ+bGFzdF9nYzwvdGQ+PHRkPnt7IGlmIC5HQ0luZm8gfX08YSBocmVmPS9nY3N0YXRzP3N5c3RlbT17eyAuU3lzdGVtTmFtZSB9fT57eyAuR0NJbmZvLlRpbWVzdGFtcEh1bWFuIH19PC9hPnt7IGVsc2UgfX11bmtub3due3sgZW5kIH19PC90ZD48L3RyPgogICAgPC90YWJsZT4KICAgIHt7IGVuZCB9fQogICAgPGgxIGNsYXNzPSJhdmFpbGFibGUiPkdsb2JhbCBDb25maWd1cmF0aW9uPC9oMj4KICAgIDx0YWJsZSBjZWxsc3BhY2luZz0wPgogICAgICA8dHI+PHRkPmJlZXRsZV92ZXJzaW9uPC90ZD48dGQ+e3sgLkJlZXRsZVZlcnNpb259fTwvdGQ+PC90cj4KICAgICAgPHRyPjx0ZD51bnNlZW5fY2xpZW50X2lkczwvdGQ+PHRkPjx1bD57eyBpZiBub3QgLlVuc2VlbkNsaWVudElkcyB9fW5vbmV7eyBlbHNlIH19e3sgcmFuZ2UgLlVuc2VlbkNsaWVudElkcyB9fTxsaT57eyAuIH19PC9saT57eyBlbmQgfX17eyBlbmQgfX08L3VsPjwvdGQ+PC90cj4KICAgICAgPHRyPjx0ZD51bnJlc3BvbnNpdmVfY2xpZW50czwvdGQ+PHRkPjx1bD57eyBpZiBub3QgLlVucmVzcG9uc2l2ZUNsaWVudHMgfX1ub25le3sgZWxzZSB9fXt7IHJhbmdlIC5VbnJlc3BvbnNpdmVDbGllbnRzIH19PGxpPnt7IC4gfX08L2xpPnt7IGVuZCB9fXt7IGVuZCB9fTwvdWw+PC90ZD48L3RyPgogICAgICA8dHI+PHRkPnVua25vd25fY2xpZW50X2lkczwvdGQ+PHRkPjx1bD57eyBpZiBub3QgLlVua25vd25DbGllbnRJZHMgfX1ub25le3sgZWxzZSB9fXt7IHJhbmdlIC5Vbmtub3duQ2xpZW50SWRzIH19PGxpPnt7IC4gfX08L2xpPnt7IGVuZCB9fXt7IGVuZCB9fTwvdWw+PC90ZD48L3RyPgogICAgICA8dHI+PHRkPmNvbmZpZ3VyZWRfY2xpZW50X2lkczwvdGQ+PHRkPjx1bD57eyByYW5nZSAuQ29uZmlndXJlZENsaWVudElkcyB9fTxsaT57eyAuIH19PC9saT57eyBlbmQgfX08L3VsPjwvdGQ+PC90cj4KICAgIDwvdGFibGU+CiAgPC9ib2R5Pgo8L2h0bWw+Cg==\"")
}

MODIFY
go/failover_state.go
go/failover_state.go
package main
import (
+ "encoding/json"
"fmt"
"math"
"os"
@@ -25,6 +26,7 @@ type FailoverState struct {
retries int // Count down for checking a master to come back after it has become unreachable.
system string // The name of the failover set.
server *ServerState // Backpointer to embedding server.
+ gcInfo *GCInfo // Information on last garbage collection.
}
// GetConfig returns the server state in a thread safe manner.
@@ -37,6 +39,25 @@ func (s *FailoverState) ClientTimeout() time.Duration {
return s.server.ClientTimeout()
}
+// SetGCInfo retrieves info about the last garbage collection from the urrent master and remembers it.
+func (s *FailoverState) SetGCInfo() {
+ if s.currentMaster == nil {
+ return
+ }
+ data, err := s.currentMaster.redis.Get("beetle:lastgc").Result()
+ if err != nil {
+ logInfo("could not retrieve last GC info for system '%s': %s", s.system, err)
+ return
+ }
+ var info GCInfo
+ err = json.Unmarshal([]byte(data), &info)
+ if err != nil {
+ logInfo("could not unmarshal last GC info for system '%s': %s", s.system, err)
+ return
+ }
+ s.gcInfo = &info
+}
+
// SendToWebSockets sends a message to all registered clients channels.
func (s *FailoverState) SendToWebSockets(msg *MsgBody) (err error) {
return s.server.SendToWebSockets(msg)
@@ -143,6 +164,7 @@ func (s *FailoverState) DetermineInitialMaster(mastersFromFile map[string]string
} else {
s.currentMaster = s.redis.AutoDetectMaster()
}
+ s.SetGCInfo()
}
// DetermineNewMaster uses the cached redis information to either select a new
@@ -313,6 +335,7 @@ func (s *FailoverState) CheckRedisAvailability() {
}
s.StartWatcher()
s.MasterAvailable()
+ s.SetGCInfo()
} else {
retriesLeft := s.GetConfig().RedisMasterRetries - (s.retries + 1)
logWarn("Redis master not available! (Retries left: %d)", retriesLeft)

MODIFY
go/garbage_collect_keys.go
go/garbage_collect_keys.go
@@ -2,6 +2,7 @@ package main
import (
"bufio"
+ "encoding/json"
"fmt"
"math"
"os"
@@ -85,63 +86,93 @@ func (s *GCState) resetExpiries() {
s.expiries = make(map[string]map[int]int)
}
-type queueInfo struct {
- queue string
- totalExpiries int
+type QueueInfo struct {
+ Queue string `json:"queue"`
+ TotalExpiries int `json:"total_expiries"`
+ Expiries HourInfos `json:"expiries"`
}
-type queueInfos []queueInfo
-func (s queueInfos) Len() int {
+type QueueInfos []QueueInfo
+
+func (s QueueInfos) Len() int {
return len(s)
}
-func (s queueInfos) Less(i, j int) bool {
- return s[i].totalExpiries > s[j].totalExpiries
+func (s QueueInfos) Less(i, j int) bool {
+ return s[i].TotalExpiries > s[j].TotalExpiries
}
-func (s queueInfos) Swap(i, j int) {
+func (s QueueInfos) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}
-type hourInfo struct {
- hour int
- count int
+type HourInfo struct {
+ Hour int `json:"hour"`
+ Count int `json:"count"`
}
-type hourInfos []hourInfo
+type HourInfos []HourInfo
-func (s hourInfos) Len() int {
+func (s HourInfos) Len() int {
return len(s)
}
-func (s hourInfos) Less(i, j int) bool {
- return s[i].hour > s[j].hour
+func (s HourInfos) Less(i, j int) bool {
+ return s[i].Hour > s[j].Hour
}
-func (s hourInfos) Swap(i, j int) {
+func (s HourInfos) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}
-func (s *GCState) dumpExpiries() {
- l := make(queueInfos, 0, len(s.expiries))
+func (s *GCState) getQueueInfos() QueueInfos {
+ l := make(QueueInfos, 0, len(s.expiries))
for q, m := range s.expiries {
- i := queueInfo{queue: q}
- for _, v := range m {
- i.totalExpiries += v
+ i := QueueInfo{Queue: q, Expiries: make(HourInfos, 0, len(m))}
+ for h, c := range m {
+ i.TotalExpiries += c
+ i.Expiries = append(i.Expiries, HourInfo{Hour: h, Count: c})
}
+ sort.Sort(i.Expiries)
l = append(l, i)
}
sort.Sort(l)
- for _, i := range l {
+ return l
+}
+
+type GCInfo struct {
+ Timestamp int64 `json:"timestamp"`
+ Queues QueueInfos `json:"queues"`
+}
+
+func (i *GCInfo) TimestampHuman() string {
+ if i == nil {
+ return "unknown"
+ }
+ return time.Unix(i.Timestamp, 0).Format(time.RFC1123)
+}
+
+func (s *GCState) storeQueueInfos(infos QueueInfos) {
+ gcInfo := GCInfo{Timestamp: time.Now().Unix(), Queues: infos}
+ data, err := json.Marshal(gcInfo)
+ if err != nil {
+ logError("could not encode gc info as json: %s", err)
+ return
+ }
+ _, err = s.redis.Set("beetle:lastgc", data, 0).Result()
+ if err != nil {
+ logError("could not store GC information in redis: %s", err)
+ }
+ logInfo("updated GC information in dedup store")
+}
+
+func (s *GCState) dumpQueueInfos(infos QueueInfos) {
+ logInfo("active keys in dedup store by queue")
+ for _, i := range infos {
fmt.Println("--------------------------------------------------------------")
- fmt.Printf("%s: %d\n", i.queue, i.totalExpiries)
+ fmt.Printf("%s: %d\n", i.Queue, i.TotalExpiries)
fmt.Println("--------------------------------------------------------------")
- h := make(hourInfos, 0, len(s.expiries[i.queue]))
- for k, v := range s.expiries[i.queue] {
- h = append(h, hourInfo{hour: k, count: v})
- }
- sort.Sort(h)
- for _, hi := range h {
- fmt.Printf("%3dh: %5d\n", hi.hour, hi.count)
+ for _, hi := range i.Expiries {
+ fmt.Printf("%3dh: %5d\n", hi.Hour, hi.Count)
}
}
}
@@ -177,16 +208,15 @@ func (s *GCState) gcKey(key string, threshold uint64) (bool, error) {
return err == nil, err
}
-func (s *GCState) garbageCollectKeys(db int) {
+func (s *GCState) garbageCollectKeys(db int) bool {
var total, expired int
var cursor uint64
- defer func() { s.dumpExpiries() }()
defer func() { logInfo("expired %d keys out of %d in db %d", expired, total, db) }()
ticker := time.NewTicker(1 * time.Second)
-collecting:
+ threshold := time.Now().Unix() + int64(s.opts.GcThreshold)
for range ticker.C {
if interrupted {
- return
+ return false
}
if s.getMaster(db) {
if cursor == 0 {
@@ -198,35 +228,29 @@ collecting:
keys, cursor, err = s.redis.Scan(cursor, "msgid:*:expires", 10000).Result()
if err != nil {
logError("starting over: %v", err)
- cursor = 0
- total = 0
- expired = 0
- continue collecting
+ return true
}
logDebug("retrieved %d keys from db %d", len(keys), db)
total += len(keys)
- threshold := time.Now().Unix() + int64(s.opts.GcThreshold)
for _, key := range keys {
if interrupted {
- return
+ return false
}
collected, err := s.gcKey(key, uint64(threshold))
if err != nil {
logError("starting over: %v", err)
- cursor = 0
- total = 0
- expired = 0
- goto collecting
+ return true
}
if collected {
expired++
}
}
if cursor == 0 {
- return
+ return false
}
}
}
+ return false
}
func (s *GCState) garbageCollectKeysFromFile(db int, filePath string) {
@@ -292,11 +316,13 @@ func (s *GCState) getMaster(db int) bool {
// RunGarbageCollectKeys runs a garbage collection on the redis master using the
// redis SCAN operation. Restarts from the beginning, should the master change
// while running the scan. Terminates as soon as a full scan has been performed
-// on all databases which need GC.
+// successfully on all databases which need GC.
func RunGarbageCollectKeys(opts GCOptions) error {
logDebug("garbage collecting keys with options: %+v", opts)
state := &GCState{opts: opts}
state.keySuffixes = []string{"status", "ack_count", "timeout", "delay", "attempts", "exceptions", "mutex", "expires"}
+restart:
+ state.resetExpiries()
for _, s := range strings.Split(opts.GcDatabases, ",") {
if interrupted {
break
@@ -307,16 +333,21 @@ func RunGarbageCollectKeys(opts GCOptions) error {
logError("%v", err)
continue
}
- state.resetExpiries()
if opts.GcKeyFile == "" {
- state.garbageCollectKeys(db)
+ restart := state.garbageCollectKeys(db)
+ if restart {
+ goto restart
+ }
} else {
state.garbageCollectKeysFromFile(db, opts.GcKeyFile)
}
}
}
+ state.getMaster(0)
if state.redis != nil {
- return state.redis.Close()
+ infos := state.getQueueInfos()
+ state.storeQueueInfos(infos)
+ state.dumpQueueInfos(infos)
}
return nil
}

MODIFY
go/server.go
go/server.go
@@ -25,7 +25,10 @@ func waitForWaitGroupWithTimeout(wg *sync.WaitGroup, timeout time.Duration) bool
}
}
-var htmlTemplate string
+var (
+ htmlTemplate string
+ gcStatsTemplate string
+)
// RunConfigurationServer implements the main server loop.
func RunConfigurationServer(o ServerOptions) error {
@@ -47,7 +50,7 @@ func RunConfigurationServer(o ServerOptions) error {
state.configChanges = make(chan consul.Env)
}
- // load html template
+ // load html templates
box := packr.NewBox("./templates")
html, err := box.FindString("index.html")
if err != nil {
@@ -55,6 +58,12 @@ func RunConfigurationServer(o ServerOptions) error {
os.Exit(1)
}
htmlTemplate = html
+ html, err = box.FindString("gcstats.html")
+ if err != nil {
+ logError("could not load gcstats.html")
+ os.Exit(1)
+ }
+ gcStatsTemplate = html
state.clientHandler(state.GetConfig().Port)
logInfo("shutting down")

MODIFY
go/server_state.go
go/server_state.go
@@ -136,6 +136,7 @@ type FailoverStatus struct {
RedisMasterAvailable bool `json:"redis_master_available"`
RedisSlavesAvailable []string `json:"redis_slaves_available"`
SwitchInProgress bool `json:"switch_in_progress"`
+ GCInfo *GCInfo `json:"lastgc"`
}
// ServerStatus is used to faciliate JSON conversion of parts of the server state.
@@ -148,7 +149,16 @@ type ServerStatus struct {
Systems []FailoverStatus `json:"redis_systems"`
}
-// GetStatus creates a ServerStatus from the curretn server state.
+func (s *ServerStatus) GetFailoverStatus(system string) *FailoverStatus {
+ for _, fs := range s.Systems {
+ if fs.SystemName == system {
+ return &fs
+ }
+ }
+ return nil
+}
+
+// GetStatus creates a ServerStatus from the current server state.
func (s *ServerState) GetStatus() *ServerStatus {
failoverStats := []FailoverStatus{}
@@ -168,6 +178,7 @@ func (s *ServerState) GetStatus() *ServerStatus {
RedisMasterAvailable: rs.MasterIsAvailable(),
RedisSlavesAvailable: rs.redis.Slaves().Servers(),
SwitchInProgress: rs.WatcherPaused(),
+ GCInfo: rs.gcInfo,
})
}
@@ -540,6 +551,32 @@ func (s *ServerState) serveWs(w http.ResponseWriter, r *http.Request) {
s.wsReader(ws)
}
+func (s *ServerState) serveGCStats(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/html")
+ keys, ok := r.URL.Query()["system"]
+ if !ok || len(keys[0]) < 1 {
+ logError("Url parameter 'system' is missing")
+ w.WriteHeader(400)
+ return
+ }
+ system := keys[0]
+ tmpl, err := template.New("gcstats.html").Parse(gcStatsTemplate)
+ if err != nil {
+ w.WriteHeader(500)
+ return
+ }
+ status := s.GetStatusFromDispatcher()
+ fs := status.GetFailoverStatus(system)
+ if fs == nil {
+ w.WriteHeader(404)
+ return
+ }
+ err = tmpl.Execute(w, fs.GCInfo)
+ if err != nil {
+ logError("template execution failed: %s", err)
+ }
+}
+
func (s *ServerState) dispatchRequest(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/", "/.html":
@@ -574,6 +611,8 @@ func (s *ServerState) dispatchRequest(w http.ResponseWriter, r *http.Request) {
s.serveWs(w, r)
case "/notifications":
s.serveNotifications(w, r)
+ case "/gcstats":
+ s.serveGCStats(w, r)
default:
http.NotFound(w, r)
}

ADD
go/templates/gcstats.html
+<!doctype html>
+<html>
+ <head>
+ <title>GC Statistics</title>
+ <style media="screen" type="text/css">
+ html { font: 1.1em/1.2 arial, sans-serif;}
+ body { margin: 1em; }
+ th { padding: 3px 3px; vertical-align: top; font-weight: normal; }
+ td { padding: 1px 1px; vertical-align: top; text-align: right; }
+ dl { margin-left: 0px; padding: 1px;}
+ dd { margin-inline-start: 0px; padding: 1px;}
+ dt { color: #5780b2; font-weight: bold; font-size: 110%;}
+ h2 { color: #5780b2; }
+ table { margin-left: 0; padding: 1px; }
+ </style>
+ </head>
+ <body>
+ {{ if . }}
+ <h2>Last GC completed at {{ .TimestampHuman }}</h2>
+ <p>Active dedup store entries by queue, with an hour base histogram of future expiries.</p>
+ <dl>
+ {{ range .Queues }}
+ <dt>{{ .Queue }}: {{ .TotalExpiries }}</dt>
+ <dd>
+ <table>
+ <colgroup>
+ <col span="1" style="width: 2em;">
+ <col span="1" style="width: 4em;">
+ </colgroup>
+ {{ range .Expiries }}
+ <tr><td>{{ .Hour }}h</td><td>{{ .Count }}</td></tr>
+ {{ end }}
+ </table>
+ </dd>
+ {{ end }}
+ </dl>
+ {{ else }}
+ <h2>No GC information available</h2>
+ {{ end }}
+ </body>
+</html>

MODIFY
go/templates/index.html
go/templates/index.html
li { }
h1.available { color: #5780b2; margin-bottom: 0.2em;}
h1.unavailable { color: #A52A2A; margin-bottom: 0.2em;}
- a:link, a:visited {text-decoration:none; color:#A52A2A;}
- a:hover, a:active {text-decoration:none; color:#FF0000;}
- a {
+ a:link, a:visited {text-decoration:none; color:#5780b2;}
+ a:hover, a:active {text-decoration:none; color:blue;}
+ a { text-decoration:none;
+ padding: 5px; background: #cdcdcd;
+ -moz-border-radius: 4px;
+ border-radius: 4px;
+ -moz-box-shadow: 1px 1px 1px #bbb;
+ -webkit-box-shadow: 1px 1px 1px #bbb;
+ box-shadow: 1px 1px 1px #bbb;
+ }
+ form a:link, form a:visited {text-decoration:none; color:#A52A2A;}
+ form a:hover, form a:active {text-decoration:none; color:#FF0000;}
+ form a {
padding: 10px; background: #cdcdcd;
-moz-border-radius: 5px;
border-radius: 5px;
<tr><td>redis_master</td><td>{{ .RedisMaster}}</td></tr>
<tr><td>redis_slaves_available</td><td><ul>{{ if not .RedisSlavesAvailable }}none{{ else }}{{ range .RedisSlavesAvailable }}<li>{{ . }}</li>{{ end }}{{ end }}</ul></td></tr>
<tr><td>configured_redis_servers</td><td><ul>{{ range .ConfiguredRedisServers }}<li>{{ . }}</li>{{ end }}</ul></td></tr>
+ <tr><td>last_gc</td><td>{{ if .GCInfo }}<a href=/gcstats?system={{ .SystemName }}>{{ .GCInfo.TimestampHuman }}</a>{{ else }}unknown{{ end }}</td></tr>
</table>
{{ end }}
<h1 class="available">Global Configuration</h2>

gobeetle: store redis gc stats in redis and display them in the config server web UI