config server: fixed race condition when accessing server state
HTTP requests run in threads separate from the server dispatcher
thread and thus can cause race conditions/crashes when accessing
server state. Solved this by adding a closure evaluator to the
dispatcher.
MODIFY
go/go.sum
go/go.sum
@@ -266,6 +266,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.0.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.1.0 h1:g0fH8RicVgNl+zVZDCDfbdWxAWoAEJyI7I3TZYXFiig=
@@ -300,6 +301,7 @@ github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/unrolled/secure v0.0.0-20180918153822-f340ee86eb8b/go.mod h1:mnPT77IAdsi/kV7+Es7y+pXALeV3h7G6dQF6mNYjcLA=

MODIFY
go/server.go
go/server.go
@@ -89,6 +89,19 @@ type ServerState struct {
failoverConfidenceLevel float64 // Failover confidence level, normalized to the interval [0,1.0]
systemNames []string // All system names. Firste on is used for saving server state.
failovers map[string]*FailoverState // Maps system name to failover state.
+ cmdChannel chan command // Channel for messages to perform state access/changing in the dispatcher thread, passed as closures.
+}
+
+type command struct {
+ closure func()
+ reply chan struct{}
+}
+
+// Evaluate sends a command (as a closure) to the dispatcher thread and waits for completion.
+func (s *ServerState) Evaluate(closure func()) {
+ replyChannel := make(chan struct{}, 1)
+ s.cmdChannel <- command{closure: closure, reply: replyChannel}
+ <-replyChannel
}
// GetConfig returns the server state in a thread safe manner.
@@ -168,6 +181,15 @@ func (s *ServerState) GetStatus() *ServerStatus {
}
}
+// GetStatusFromDispatcher retrieves the status from the dispatcher thread.
+func (s *ServerState) GetStatusFromDispatcher() *ServerStatus {
+ var res *ServerStatus
+ s.Evaluate(func() {
+ res = s.GetStatus()
+ })
+ return res
+}
+
// pst and psta facilitate sorting of (string, time.Duration pairs) by duration,
// then string value.
type pst struct {
@@ -420,6 +442,9 @@ func (s *ServerState) dispatcher() {
}
for !interrupted {
select {
+ case cmd := <-s.cmdChannel:
+ cmd.closure()
+ cmd.reply <- struct{}{}
case msg := <-s.wsChannel:
s.handleWebSocketMsg(msg)
case system := <-s.timerChannel:
@@ -483,6 +508,7 @@ func NewServerState(o ServerOptions) *ServerState {
CheckOrigin: func(r *http.Request) bool { return true },
}
s.wsChannel = make(chan *WsMsg, 10000)
+ s.cmdChannel = make(chan command, 1000)
s.clientIds = make(StringSet)
for _, id := range strings.Split(s.GetConfig().ClientIds, ",") {
if id != "" {
@@ -702,13 +728,13 @@ func (s *ServerState) dispatchRequest(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(500)
return
}
- err = tmpl.Execute(w, s.GetStatus())
+ err = tmpl.Execute(w, s.GetStatusFromDispatcher())
if err != nil {
logError("template execution failed: %s", err)
}
case "/.json":
w.Header().Set("Content-Type", "application/json")
- b, err := json.Marshal(s.GetStatus())
+ b, err := json.Marshal(s.GetStatusFromDispatcher())
if err != nil {
w.WriteHeader(500)
return
@@ -745,7 +771,9 @@ func (s *ServerState) initiateMasterSwitch(w http.ResponseWriter, r *http.Reques
fmt.Fprintf(w, "Master switch not possible for unknown system: '%s'\n", system)
return
}
- if fs.InitiateMasterSwitch() {
+ var initiated bool
+ s.Evaluate(func() { initiated = fs.InitiateMasterSwitch() })
+ if initiated {
w.WriteHeader(201)
fmt.Fprintf(w, "Master switch initiated\n")
} else {

config server: fixed race condition when accessing server state

HTTP requests run in threads separate from the server dispatcher
thread and thus can cause race conditions/crashes when accessing
server state. Solved this by adding a closure evaluator to the
dispatcher.