title: golang request header concurrent read and write
body: <p><strong>Go env:</strong></p>
  <blockquote>
  <p>GOARCH=&quot;amd64&quot;</p>
  <p>GOBIN=&quot;/root/&quot;</p>
  <p>GOEXE=&quot;&quot;</p>
  <p>GOHOSTARCH=&quot;amd64&quot;</p>
  <p>GOHOSTOS=&quot;linux&quot;</p>
  <p>GOOS=&quot;linux&quot;</p>
  <p>GOPATH=&quot;/data/workspace/kubernetes&quot;</p>
  <p>GORACE=&quot;&quot;</p>
  <p>GOROOT=&quot;/usr/local/go&quot;</p>
  <p>GOTOOLDIR=&quot;/usr/local/go/pkg/tool/linux_amd64&quot;</p>
  <p>GO15VENDOREXPERIMENT=&quot;1&quot;</p>
  <p>CC=&quot;gcc&quot;</p>
  <p>GOGCCFLAGS=&quot;-fPIC -m64 -pthread -fmessage-length=0&quot;</p>
  <p>CXX=&quot;g++&quot;</p>
  <p>CGO_ENABLED=&quot;1&quot;</p>
  </blockquote>
  <p><strong>Go version:</strong></p>
  <blockquote>
  <p>go version go1.6.3 linux/amd64</p>
  </blockquote>
  <p>This issues is happend on a “performance test env” kube-apiserver with high load. kube-apiserver panic and exit:</p>
  <pre><code>fatal error: concurrent map read and map write
  
  goroutine 77930636 [running]:
  runtime.throw(0x2f4c4c0, 0x21)
      /root/.gvm/gos/go1.6.3/src/runtime/panic.go:547 +0x90 fp=0xca67b477f0     sp=0xca67b477d8
  runtime.mapaccess1_faststr(0x2a8e520, 0xc9e29000f0, 0x2c11220, 0xa, 0x433e360)
      /root/.gvm/gos/go1.6.3/src/runtime/hashmap_fast.go:202 +0x5b fp=0xca67b47850 sp=0xca67b477f0
  k8s.io/kubernetes/pkg/httplog.(*respLogger).Log(0xcbddf2ae70)
         /data/gerrit/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/pkg/httplog/log.go:180 +0x43d fp=0xca67b47af8 sp=0xca67b47850
  k8s.io/kubernetes/pkg/apiserver.RecoverPanics.func1(0x7f099f157090, 0xcbddf2ae70, 0xcd7569e380)
       /data/gerrit/src/k8s.io/kubernetes/_output/local/go/src/k8s.io/kubernetes/pkg/apiserver/handlers.go:174 +0x15d fp=0xca67b47b50 sp=0xca67b47af8
      net/http.HandlerFunc.ServeHTTP(0xc821a4eac0, 0x7f099f157058, 0xca0f4eb450, 0xcd7569e380)
      /root/.gvm/gos/go1.6.3/src/net/http/server.go:1618 +0x3a fp=0xca67b47b70 sp=0xca67b47b50
  net/http.serverHandler.ServeHTTP(0xc8215a7b80, 0x7f099f157058, 0xca0f4eb450, 0xcd7569e380)
      /root/.gvm/gos/go1.6.3/src/net/http/server.go:2081 +0x19e fp=0xca67b47bd0 sp=0xca67b47b70
  net/http.(*conn).serve(0xc8b5d6b980)
      /root/.gvm/gos/go1.6.3/src/net/http/server.go:1472 +0xf2e fp=0xca67b47f98 sp=0xca67b47bd0
  runtime.goexit()
      /root/.gvm/gos/go1.6.3/src/runtime/asm_amd64.s:1998 +0x1 fp=0xca67b47fa0 sp=0xca67b47f98
  created by net/http.(*Server).Serve
      /root/.gvm/gos/go1.6.3/src/net/http/server.go:2137 +0x44e
  </code></pre>
  <p>corresponding source code:</p>
  <p>pkg/apiserver/handlers.go</p>
  <pre><code>145 func RecoverPanics(handler http.Handler) http.Handler {
  146 return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
  147 defer func() {
  148     if x := recover(); x != nil {
  149         http.Error(w, &quot;apis panic. Look in log for details.&quot;, http.StatusInternalServerError)
  150         glog.Errorf(&quot;APIServer panic'd on %v %v: %v\n%s\n&quot;, req.Method, req.RequestURI, x, debug.Stack())
  151     }
  152 }()
  153 defer httplog.NewLogged(req, &amp;w).StacktraceWhen(
          httplog.StatusIsNot(
              http.StatusOK,
              http.StatusCreated,
              http.StatusAccepted,
              http.StatusBadRequest,
              http.StatusMovedPermanently,
              http.StatusTemporaryRedirect,
              http.StatusConflict,
              http.StatusNotFound,
              http.StatusUnauthorized,
              http.StatusForbidden,
              errors.StatusUnprocessableEntity,
              http.StatusSwitchingProtocols,
              http.StatusRequestTimeout,
              errors.StatusTooManyRequests,
          ),
  170 ).Log()
  
      // Dispatch to the internal handler
      handler.ServeHTTP(w, req)
  174 })
  }
  </code></pre>
  <p>pkg/httplog/log.go:</p>
  <pre><code>159 func (rl *respLogger) Log() {
  160 latency := time.Since(rl.startTime)
  161 if glog.V(2) {
  162     extraInfo := &quot;&quot;
  163     if latency &gt;= time.Millisecond*200 &amp;&amp; latency &lt; time.Second {
          extraInfo = fmt.Sprintf(&quot;%d00.Millisecond&quot;, latency/(time.Millisecond*100))
      } else if latency &gt;= time.Second &amp;&amp; latency &lt; time.Minute { // Warning
          extraInfo = fmt.Sprintf(&quot;%d.Second&quot;, latency/(time.Second))
      } else if latency &gt;= time.Minute { // nce will timeout
          extraInfo = fmt.Sprintf(&quot;%d.Minutes&quot;, latency/(time.Minute))
      }
      method := rl.req.Method
      if len(rl.req.Header[&quot;Detailed-Method&quot;]) &gt; 0 {
          method = rl.req.Header[&quot;Detailed-Method&quot;][0]
      }
  
      remoteIP := rl.getXForwardIPAdress(rl.req)
  
  
      if !rl.hijacked {
          //glog.InfoDepth(1, fmt.Sprintf(&quot;%s %s: (%v) %v%v%v [%s %s]&quot;, rl.req.Method, rl.req.RequestURI, latency, rl.status, rl.statusStack, rl.addedInfo, rl.req.Header[&quot;User-Agent&quot;], rl.req.RemoteAddr))
  180         glog.InfoDepth(1, fmt.Sprintf(&quot;%v %s %s: (%sms) %v%v [%s %s]-%s %v&quot;, rl.req.Header[&quot;X-Requestid&quot;], method, rl.req.RequestURI, GetMilliLatency(latency), rl.status, rl.addedInfo, rl.req.Header[&quot;User-Agent&quot;], remoteIP, extraInfo, rl.statusStack))
      } else {
          //glog.InfoDepth(1, fmt.Sprintf(&quot;%s %s: (%v) hijacked [%s %s]&quot;, rl.req.Method, rl.req.RequestURI, latency, rl.req.Header[&quot;User-Agent&quot;], rl.req.RemoteAddr))
          glog.InfoDepth(1, fmt.Sprintf(&quot;%v %s %s: (%sms) hijacked [%s %s]-%s&quot;, rl.req.Header[&quot;X-Requestid&quot;], method, rl.req.RequestURI, GetMilliLatency(latency), rl.req.Header[&quot;User-Agent&quot;], remoteIP, extraInfo))
      }
  }
  </code></pre>
  <p>}</p>
  <p>In handler.ServeHTTP, I changed request.Header. I cann't find the reason:</p>
  <p>(1) &quot;serveHTTP&quot; and &quot;defer log&quot; works in a serial manner. So  not exist &quot;concurrent read and write&quot;</p>
  <p>(2) Even if exist &quot;concurrent read and write&quot;, there exist &quot;recover&quot; func to deal with panics, so the kube-apiserver shoud  not exit.</p>
  <p>The question has cost me lots of time, who can help me? Thanks</p>