func functionalTest(t *testing.T, ports []string) {
	srvs := []string{}
	for _, port := range ports {
		srvs = append(srvs, fmt.Sprintf("localhost:%s", port))
	}
	client := New(srvs...)

	wg := sync.WaitGroup{}

	server := func(port string) {
		for i := 0; i < 5; i++ {
			ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
			defer cancel()
			cmd := exec.CommandContext(ctx, "memcached", "-p", port)
			err := cmd.Run()
			assertEqualError(t, err, "signal: killed")

			// no randomisation - keep approximately in lockstep to keep the amount of downtime high
			time.Sleep(time.Millisecond*500)
		}
		wg.Done()
	}

	done := make(chan struct{})
	mx := sync.Mutex{}
	resultCounts := map[string]int{}
	networkErrorCategory := "[network error]"
	internalTimeoutError := "[memcachepl timeout]"
	writer := func() {
		for {
			select {
			case <-done:
				break
			default:
				err := client.Set(&Item{
					Key:   "hi",
					Value: []byte("val"),
				})
				errString := fmt.Sprintf("%v", err)
				var netErr net.Error
				if errors.As(err, &netErr) {
					errString = networkErrorCategory
				}
				var cte *ConnectTimeoutError
				if errors.As(err, &cte) {
					errString = internalTimeoutError
				}
				mx.Lock()
				resultCounts[errString]++
				mx.Unlock()
			}
			time.Sleep(time.Duration(rand.Int31n(5)) * time.Millisecond)
		}
	}

	// start servers
	wg.Add(len(srvs))
	for _, port := range ports {
		go server(port)
	}

	// start writers
	go writer()
	go writer()
	go writer()
	go writer()
	go writer()
	go writer()

	// wait for test run to end
	wg.Wait()
	close(done)

	// avoid race-detection errors when we range over results
	mx.Lock()
	defer mx.Unlock()

	unexpectedErrors := map[string]struct{}{}
	total := 0
	for errVal, count := range resultCounts {
		total += count
		switch errVal {

		// filter out success, and expected errors
		case "<nil>":
		case "EOF":
		case internalTimeoutError:
		case networkErrorCategory:
		case ErrNoServers.Error():

		default:
			unexpectedErrors[errVal] = struct{}{}
		}
	}

	t.Log("aggregate result of Set calls", resultCounts)

	for s := range unexpectedErrors {
		t.Errorf("unexpected errors during test - either indicates a bug or something that needs categorisation: %s", s)
	}

	// These ratios are just rough observed values from the configuration above setting the amount
	// of time the memcache nodes are available. They're not tight boundaries as that would create test
	// flakiness. They're designed to ensure that the implementation is having the desired effect.

	// We're up for approximately 2.5 seconds out of every three, so we should have decent number of success too
	// Enforcing success ratio avoids allowing a bug that causes us to return ErrNoServers continually and thus passes the 'more breaker
	// errors than network errors' test below.
	expectedSuccessRatio := 10
	assertGreaterOrEqual(t, resultCounts["<nil>"], total/expectedSuccessRatio, "expected many successes")

	// We want greatly more breaker errors than network errors, reducing the reconnections attempts during unavailability by a large factor
	// Although this is controlled by the test configuration above, what we're demonstrating here is that we do manage to prevent a spike
	// in reconnections during the 5 up->down transitions
	breakerToNetworkErrorRatio := 80
	networkFails := resultCounts[networkErrorCategory] + resultCounts[internalTimeoutError] +
		resultCounts["EOF"]
	assertGreaterOrEqual(t, resultCounts[ErrNoServers.Error()], networkFails*breakerToNetworkErrorRatio, "didn't reduce reconnection load as expected")
}
// This is a functional test that spins up and repeatedly takes down a memcached nodes
// while multiple writer routines attempt to Set keys
This is a functional test that spins up and repeatedly takes down a memcached nodes
while multiple writer routines attempt to Set keys
// This is a functional test that spins up and repeatedly takes down a memcached nodes // while multiple writer routines attempt to Set keys
func functionalTest(t *testing.T, ports []string) {
	srvs := []string{}
	for _, port := range ports {
		srvs = append(srvs, fmt.Sprintf("localhost:%s", port))
	}
	client := New(srvs...)
	wg := sync.WaitGroup{}
	server := func(port string) {
		for i := 0; i < 5; i++ {
			ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
			defer cancel()
			cmd := exec.CommandContext(ctx, "memcached", "-p", port)
			err := cmd.Run()
			assertEqualError(t, err, "signal: killed")
			time.Sleep(time.Millisecond*500)
		}
		wg.Done()
	}
	done := make(chan struct{})
	mx := sync.Mutex{}
	resultCounts := map[string]int{}
	networkErrorCategory := "[network error]"
	internalTimeoutError := "[memcachepl timeout]"
	writer := func() {
		for {
			select {
			case <-done:
				break
			default:
				err := client.Set(&Item{
					Key:   "hi",
					Value: []byte("val"),
				})
				errString := fmt.Sprintf("%v", err)
				var netErr net.Error
				if errors.As(err, &netErr) {
					errString = networkErrorCategory
				}
				var cte *ConnectTimeoutError
				if errors.As(err, &cte) {
					errString = internalTimeoutError
				}
				mx.Lock()
				resultCounts[errString]++
				mx.Unlock()
			}
			time.Sleep(time.Duration(rand.Int31n(5)) * time.Millisecond)
		}
	}
	wg.Add(len(srvs))
	for _, port := range ports {
		go server(port)
	}
	go writer()
	go writer()
	go writer()
	go writer()
	go writer()
	go writer()
	wg.Wait()
	close(done)
	mx.Lock()
	defer mx.Unlock()
	unexpectedErrors := map[string]struct{}{}
	total := 0
	for errVal, count := range resultCounts {
		total += count
		switch errVal {
		case "<nil>":
		case "EOF":
		case internalTimeoutError:
		case networkErrorCategory:
		case ErrNoServers.Error():
		default:
			unexpectedErrors[errVal] = struct{}{}
		}
	}
	t.Log("aggregate result of Set calls", resultCounts)
	for s := range unexpectedErrors {
		t.Errorf("unexpected errors during test - either indicates a bug or something that needs categorisation: %s", s)
	}
	expectedSuccessRatio := 10
	assertGreaterOrEqual(t, resultCounts["<nil>"], total/expectedSuccessRatio, "expected many successes")
	breakerToNetworkErrorRatio := 80
	networkFails := resultCounts[networkErrorCategory] + resultCounts[internalTimeoutError] +
		resultCounts["EOF"]
	assertGreaterOrEqual(t, resultCounts[ErrNoServers.Error()], networkFails*breakerToNetworkErrorRatio, "didn't reduce reconnection load as expected")
}
func functionalTest ( t * testing . T , ports [ ] string ) { srvs := [ ] string { } for _ , port := range ports { srvs = append ( srvs , fmt . Sprintf ( " " , port ) ) } client := New ( srvs ... ) wg := sync . WaitGroup { } server := func ( port string ) { for i := 0 ; i < 5 ; i ++ { ctx , cancel := context . WithTimeout ( context . Background ( ) , time . Second * 3 ) defer cancel ( ) cmd := exec . CommandContext ( ctx , " " , " " , port ) err := cmd . Run ( ) assertEqualError ( t , err , " " ) time . Sleep ( time . Millisecond * 500 ) } wg . Done ( ) } done := make ( chan struct { } ) mx := sync . Mutex { } resultCounts := map [ string ] int { } networkErrorCategory := " " internalTimeoutError := " " writer := func ( ) { for { select { case <- done : break default : err := client . Set ( & Item { Key : " " , Value : [ ] byte ( " " ) , } ) errString := fmt . Sprintf ( " " , err ) var netErr net . Error if errors . As ( err , & netErr ) { errString = networkErrorCategory } var cte * ConnectTimeoutError if errors . As ( err , & cte ) { errString = internalTimeoutError } mx . Lock ( ) resultCounts [ errString ] ++ mx . Unlock ( ) } time . Sleep ( time . Duration ( rand . Int31n ( 5 ) ) * time . Millisecond ) } } wg . Add ( len ( srvs ) ) for _ , port := range ports { go server ( port ) } go writer ( ) go writer ( ) go writer ( ) go writer ( ) go writer ( ) go writer ( ) wg . Wait ( ) close ( done ) mx . Lock ( ) defer mx . Unlock ( ) unexpectedErrors := map [ string ] struct { } { } total := 0 for errVal , count := range resultCounts { total += count switch errVal { case " " : case " " : case internalTimeoutError : case networkErrorCategory : case ErrNoServers . Error ( ) : default : unexpectedErrors [ errVal ] = struct { } { } } } t . Log ( " " , resultCounts ) for s := range unexpectedErrors { t . Errorf ( " " , s ) } expectedSuccessRatio := 10 assertGreaterOrEqual ( t , resultCounts [ " " ] , total / expectedSuccessRatio , " " ) breakerToNetworkErrorRatio := 80 networkFails := resultCounts [ networkErrorCategory ] + resultCounts [ internalTimeoutError ] + resultCounts [ " " ] assertGreaterOrEqual ( t , resultCounts [ ErrNoServers . Error ( ) ] , networkFails * breakerToNetworkErrorRatio , " " ) }
This is a functional test that spins up and repeatedly takes down a memcached nodes
while multiple writer routines attempt to Set keys
This is a functional test that spins up and repeatedly takes down a memcached nodes while multiple writer routines attempt to Set keys
// no randomisation - keep approximately in lockstep to keep the amount of downtime high // start servers // start writers // wait for test run to end // avoid race-detection errors when we range over results // filter out success, and expected errors // These ratios are just rough observed values from the configuration above setting the amount // of time the memcache nodes are available. They're not tight boundaries as that would create test // flakiness. They're designed to ensure that the implementation is having the desired effect. // We're up for approximately 2.5 seconds out of every three, so we should have decent number of success too // Enforcing success ratio avoids allowing a bug that causes us to return ErrNoServers continually and thus passes the 'more breaker // errors than network errors' test below. // We want greatly more breaker errors than network errors, reducing the reconnections attempts during unavailability by a large factor // Although this is controlled by the test configuration above, what we're demonstrating here is that we do manage to prevent a spike // in reconnections during the 5 up->down transitions