stat/card: add sketch serialisation
MODIFY
stat/card/card_test.go
stat/card/card_test.go
package card
import (
+ "encoding"
"fmt"
"hash/fnv"
"io"
@@ -195,6 +196,96 @@ func TestResetCounters(t *testing.T) {
}
}
+type counterEncoder interface {
+ counter
+ encoding.BinaryMarshaler
+ encoding.BinaryUnmarshaler
+}
+
+var counterEncoderTests = []struct {
+ name string
+ count int
+ src, dst func() counterEncoder
+}{
+ {
+ name: "HyperLogLog32-4-4-FNV-1a", count: 1e3,
+ src: func() counterEncoder { return mustCounterEncoder(NewHyperLogLog32(4, fnv.New32a())) },
+ dst: func() counterEncoder { return mustCounterEncoder(NewHyperLogLog32(4, fnv.New32a())) },
+ },
+ {
+ name: "HyperLogLog32-4-8-FNV-1a", count: 1e3,
+ src: func() counterEncoder { return mustCounterEncoder(NewHyperLogLog32(4, fnv.New32a())) },
+ dst: func() counterEncoder { return mustCounterEncoder(NewHyperLogLog32(8, fnv.New32a())) },
+ },
+ {
+ name: "HyperLogLog32-8-4-FNV-1a", count: 1e3,
+ src: func() counterEncoder { return mustCounterEncoder(NewHyperLogLog32(8, fnv.New32a())) },
+ dst: func() counterEncoder { return mustCounterEncoder(NewHyperLogLog32(4, fnv.New32a())) },
+ },
+ {
+ name: "HyperLogLog64-4-4-FNV-1a", count: 1e3,
+ src: func() counterEncoder { return mustCounterEncoder(NewHyperLogLog64(4, fnv.New64a())) },
+ dst: func() counterEncoder { return mustCounterEncoder(NewHyperLogLog64(4, fnv.New64a())) },
+ },
+ {
+ name: "HyperLogLog64-4-8-FNV-1a", count: 1e3,
+ src: func() counterEncoder { return mustCounterEncoder(NewHyperLogLog64(4, fnv.New64a())) },
+ dst: func() counterEncoder { return mustCounterEncoder(NewHyperLogLog64(8, fnv.New64a())) },
+ },
+ {
+ name: "HyperLogLog64-8-4-FNV-1a", count: 1e3,
+ src: func() counterEncoder { return mustCounterEncoder(NewHyperLogLog64(8, fnv.New64a())) },
+ dst: func() counterEncoder { return mustCounterEncoder(NewHyperLogLog64(4, fnv.New64a())) },
+ },
+}
+
+func mustCounterEncoder(c counterEncoder, err error) counterEncoder {
+ if err != nil {
+ panic(fmt.Sprintf("bad test: %v", err))
+ }
+ return c
+}
+
+func TestBinaryEncoding(t *testing.T) {
+ for _, test := range counterEncoderTests {
+ rnd := rand.New(rand.NewSource(1))
+ src := test.src()
+ for i := 0; i < int(test.count); i++ {
+ buf := strconv.AppendUint(nil, rnd.Uint64(), 16)
+ buf = append(buf, '-')
+ buf = strconv.AppendUint(buf, uint64(i), 16)
+ n, err := src.Write(buf)
+ if n != len(buf) {
+ t.Errorf("unexpected number of bytes written for %s: got:%d want:%d",
+ test.name, n, len(buf))
+ break
+ }
+ if err != nil {
+ t.Errorf("unexpected error for %s: %v", test.name, err)
+ break
+ }
+ }
+
+ buf, err := src.MarshalBinary()
+ if err != nil {
+ t.Errorf("unexpected error marshaling binary for %s: %v", test.name, err)
+ continue
+ }
+ dst := test.dst()
+ err = dst.UnmarshalBinary(buf)
+ if err != nil {
+ t.Errorf("unexpected error unmarshaling binary for %s: %v", test.name, err)
+ continue
+ }
+ gotSrc := src.Count()
+ gotDst := dst.Count()
+
+ if gotSrc != gotDst {
+ t.Errorf("unexpected count for %s: got:%.0f want:%.0f", test.name, gotDst, gotSrc)
+ }
+ }
+}
+
var rhoQTests = []struct {
bits uint
q uint8

MODIFY
stat/card/hll32.go
stat/card/hll32.go
package card
import (
+ "bytes"
+ "encoding/gob"
"errors"
+ "fmt"
"hash"
"math"
"math/bits"
+ "path"
"reflect"
)
@@ -142,3 +146,82 @@ func (h *HyperLogLog32) Reset() {
h.register[i] = 0
}
}
+
+// MarshalBinary marshals the sketch in the receiver. It encodes the
+// name of the hash function, the precision of the sketch and the
+// sketch data. The receiver must have a non-nil hash function.
+func (h *HyperLogLog32) MarshalBinary() ([]byte, error) {
+ if h.hash == nil {
+ return nil, errors.New("card: hash function not set")
+ }
+ t := reflect.TypeOf(h.hash)
+ var prefix string
+ if t.Kind() == reflect.Ptr {
+ t = t.Elem()
+ prefix = "*"
+ }
+ var buf bytes.Buffer
+ enc := gob.NewEncoder(&buf)
+ err := enc.Encode(uint8(w32))
+ if err != nil {
+ return nil, err
+ }
+ err = enc.Encode(prefix + path.Join(t.PkgPath(), t.Name()))
+ if err != nil {
+ return nil, err
+ }
+ err = enc.Encode(h.p)
+ if err != nil {
+ return nil, err
+ }
+ err = enc.Encode(h.register)
+ if err != nil {
+ return nil, err
+ }
+ return buf.Bytes(), nil
+}
+
+// UnmarshalBinary unmarshals the binary representation of a sketch
+// into the receiver. The precision of the receiver will be set after
+// return. The receiver must have a non-nil hash function value that is
+// the same type as the one that was stored in the binary data.
+func (h *HyperLogLog32) UnmarshalBinary(b []byte) error {
+ if h.hash == nil {
+ return errors.New("card: hash function not set")
+ }
+ t := reflect.TypeOf(h.hash)
+ var prefix string
+ if t.Kind() == reflect.Ptr {
+ t = t.Elem()
+ prefix = "*"
+ }
+ dec := gob.NewDecoder(bytes.NewReader(b))
+ var size uint8
+ err := dec.Decode(&size)
+ if err != nil {
+ return err
+ }
+ if size != w32 {
+ return fmt.Errorf("card: mismatched hash function size: dst=%d src=%d", w32, size)
+ }
+ var srcHash string
+ err = dec.Decode(&srcHash)
+ if err != nil {
+ return err
+ }
+ dstHash := prefix + path.Join(t.PkgPath(), t.Name())
+ if dstHash != srcHash {
+ return fmt.Errorf("card: mismatched hash function: dst=%s src=%s", dstHash, srcHash)
+ }
+ err = dec.Decode(&h.p)
+ if err != nil {
+ return err
+ }
+ h.m = uint32(1) << h.p
+ h.register = h.register[:0]
+ err = dec.Decode(&h.register)
+ if err != nil {
+ return err
+ }
+ return nil
+}

MODIFY
stat/card/hll64.go
stat/card/hll64.go
package card
import (
+ "bytes"
+ "encoding/gob"
"errors"
+ "fmt"
"hash"
"math"
"math/bits"
+ "path"
"reflect"
)
@@ -144,3 +148,82 @@ func (h *HyperLogLog64) Reset() {
h.register[i] = 0
}
}
+
+// MarshalBinary marshals the sketch in the receiver. It encodes the
+// name of the hash function, the precision of the sketch and the
+// sketch data. The receiver must have a non-nil hash function.
+func (h *HyperLogLog64) MarshalBinary() ([]byte, error) {
+ if h.hash == nil {
+ return nil, errors.New("card: hash function not set")
+ }
+ t := reflect.TypeOf(h.hash)
+ var prefix string
+ if t.Kind() == reflect.Ptr {
+ t = t.Elem()
+ prefix = "*"
+ }
+ var buf bytes.Buffer
+ enc := gob.NewEncoder(&buf)
+ err := enc.Encode(uint8(w64))
+ if err != nil {
+ return nil, err
+ }
+ err = enc.Encode(prefix + path.Join(t.PkgPath(), t.Name()))
+ if err != nil {
+ return nil, err
+ }
+ err = enc.Encode(h.p)
+ if err != nil {
+ return nil, err
+ }
+ err = enc.Encode(h.register)
+ if err != nil {
+ return nil, err
+ }
+ return buf.Bytes(), nil
+}
+
+// UnmarshalBinary unmarshals the binary representation of a sketch
+// into the receiver. The precision of the receiver will be set after
+// return. The receiver must have a non-nil hash function value that is
+// the same type as the one that was stored in the binary data.
+func (h *HyperLogLog64) UnmarshalBinary(b []byte) error {
+ if h.hash == nil {
+ return errors.New("card: hash function not set")
+ }
+ t := reflect.TypeOf(h.hash)
+ var prefix string
+ if t.Kind() == reflect.Ptr {
+ t = t.Elem()
+ prefix = "*"
+ }
+ dec := gob.NewDecoder(bytes.NewReader(b))
+ var size uint8
+ err := dec.Decode(&size)
+ if err != nil {
+ return err
+ }
+ if size != w64 {
+ return fmt.Errorf("card: mismatched hash function size: dst=%d src=%d", w64, size)
+ }
+ var srcHash string
+ err = dec.Decode(&srcHash)
+ if err != nil {
+ return err
+ }
+ dstHash := prefix + path.Join(t.PkgPath(), t.Name())
+ if dstHash != srcHash {
+ return fmt.Errorf("card: mismatched hash function: dst=%s src=%s", dstHash, srcHash)
+ }
+ err = dec.Decode(&h.p)
+ if err != nil {
+ return err
+ }
+ h.m = uint64(1) << h.p
+ h.register = h.register[:0]
+ err = dec.Decode(&h.register)
+ if err != nil {
+ return err
+ }
+ return nil
+}

stat/card: add sketch serialisation