Add more memory-monitoring stress tests

.travis.yml

@@ -33,7 +33,8 @@ script:
 - go test -v -covermode=count -coverprofile=server.coverprofile ./server
 - go test -v -covermode=count -coverprofile=psinet.coverprofile ./server/psinet
 - go test -v -covermode=count -coverprofile=psiphon.coverprofile
-- go test -v ./memory_test
+- go test -v ./memory_test -run TestReconnectTunnels
+- go test -v ./memory_test -run TestRestartController
 - $HOME/gopath/bin/gover
 - $HOME/gopath/bin/goveralls -coverprofile=gover.coverprofile -service=travis-ci -repotoken $COVERALLS_TOKEN
 before_install:

psiphon/common/utils.go

@@ -218,3 +218,16 @@ func Decompress(data []byte) ([]byte, error) {
 	}
 	return uncompressedData, nil
 }
+
+// FormatByteCount returns a string representation of the specified
+// byte count in conventional, human-readable format.
+func FormatByteCount(bytes uint64) string {
+	// Based on: https://bitbucket.org/psiphon/psiphon-circumvention-system/src/b2884b0d0a491e55420ed1888aea20d00fefdb45/Android/app/src/main/java/com/psiphon3/psiphonlibrary/Utils.java?at=default#Utils.java-646
+	base := uint64(1024)
+	if bytes < base {
+		return fmt.Sprintf("%dB", bytes)
+	}
+	exp := int(math.Log(float64(bytes)) / math.Log(float64(base)))
+	return fmt.Sprintf(
+		"%.1f%c", float64(bytes)/math.Pow(float64(base), float64(exp)), "KMGTPEZ"[exp-1])
+}

psiphon/common/utils_test.go

@@ -111,3 +111,26 @@ func TestCompress(t *testing.T) {
 		t.Error("decompressed data doesn't match original data")
 	}
 }
+
+func TestFormatByteCount(t *testing.T) {
+
+	testCases := []struct {
+		n              uint64
+		expectedOutput string
+	}{
+		{500, "500B"},
+		{1024, "1.0K"},
+		{10000, "9.8K"},
+		{1024*1024 + 1, "1.0M"},
+		{100*1024*1024 + 99999, "100.1M"},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.expectedOutput, func(t *testing.T) {
+			output := FormatByteCount(testCase.n)
+			if output != testCase.expectedOutput {
+				t.Errorf("unexpected output: %s", output)
+			}
+		})
+	}
+}

psiphon/memory_test/memory_test.go

@@ -33,20 +33,44 @@ import (
 	"time"
 
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
 )
 
-// TestMemoryUsage is a memory stress test that repeatedly
-// establishes a tunnel, immediately terminates it, and
-// start reestablishing.
+// memory_test is a memory stress test suite that repeatedly
+// reestablishes tunnels and restarts the Controller.
 //
 // runtime.MemStats is used to monitor system memory usage
 // during the test.
 //
-// This test is in its own package as its runtime.MemStats
+// These tests are in its own package as its runtime.MemStats
 // checks must not be impacted by other test runs; this
-// test is also long-running.
+// test is also long-running and _may_ require setting the
+// test flag "-timeout" beyond the default of 10 minutes
+// (check the testDuration configured below).
+//
+// For the most accurate memory reporting, run each test
+// individually; e.g.,
+// go test -run [TestReconnectTunnel|TestRestartController|etc.]
+
+const (
+	testModeReconnectTunnel = iota
+	testModeRestartController
+	testModeReconnectAndRestart
+)
+
+func TestReconnectTunnel(t *testing.T) {
+	runMemoryTest(t, testModeReconnectTunnel)
+}
 
-func TestMemoryUsage(t *testing.T) {
+func TestRestartController(t *testing.T) {
+	runMemoryTest(t, testModeRestartController)
+}
+
+func TestReconnectAndRestart(t *testing.T) {
+	runMemoryTest(t, testModeReconnectAndRestart)
+}
+
+func runMemoryTest(t *testing.T, testMode int) {
 
 	testDataDirName, err := ioutil.TempDir("", "psiphon-memory-test")
 	if err != nil {
@@ -80,7 +104,7 @@ func TestMemoryUsage(t *testing.T) {
 	postActiveTunnelTerminateDelay := 250 * time.Millisecond
 	testDuration := 5 * time.Minute
 	memInspectionFrequency := 10 * time.Second
-	maxSysMemory := uint64(10 * 1024 * 1024)
+	maxSysMemory := uint64(11 * 1024 * 1024)
 
 	config.ClientVersion = "999999999"
 	config.TunnelPoolSize = 1
@@ -103,6 +127,10 @@ func TestMemoryUsage(t *testing.T) {
 	}
 
 	var controller *psiphon.Controller
+	var controllerShutdown chan struct{}
+	var controllerWaitGroup *sync.WaitGroup
+	restartController := make(chan bool, 1)
+	reconnectTunnel := make(chan bool, 1)
 	tunnelsEstablished := int32(0)
 
 	psiphon.SetNoticeOutput(psiphon.NewNoticeReceiver(
@@ -116,8 +144,24 @@ func TestMemoryUsage(t *testing.T) {
 				count := int(payload["count"].(float64))
 				if count > 0 {
 					atomic.AddInt32(&tunnelsEstablished, 1)
+
 					time.Sleep(postActiveTunnelTerminateDelay)
-					go controller.TerminateNextActiveTunnel()
+
+					doRestartController := (testMode == testModeRestartController)
+					if testMode == testModeReconnectAndRestart {
+						doRestartController = common.FlipCoin()
+					}
+					if doRestartController {
+						select {
+						case restartController <- true:
+						default:
+						}
+					} else {
+						select {
+						case reconnectTunnel <- true:
+						default:
+						}
+					}
 				}
 			case "Info":
 				message := payload["message"].(string)
@@ -129,28 +173,39 @@ func TestMemoryUsage(t *testing.T) {
 			}
 		}))
 
-	controller, err = psiphon.NewController(config)
-	if err != nil {
-		t.Fatalf("error creating controller: %s", err)
+	startController := func() {
+		controller, err = psiphon.NewController(config)
+		if err != nil {
+			t.Fatalf("error creating controller: %s", err)
+		}
+
+		controllerShutdown = make(chan struct{})
+		controllerWaitGroup = new(sync.WaitGroup)
+		controllerWaitGroup.Add(1)
+		go func() {
+			defer controllerWaitGroup.Done()
+			controller.Run(controllerShutdown)
+		}()
 	}
 
-	shutdownBroadcast := make(chan struct{})
-	controllerWaitGroup := new(sync.WaitGroup)
-	controllerWaitGroup.Add(1)
-	go func() {
-		defer controllerWaitGroup.Done()
-		controller.Run(shutdownBroadcast)
-	}()
+	stopController := func() {
+		close(controllerShutdown)
+		controllerWaitGroup.Wait()
+	}
 
 	testTimer := time.NewTimer(testDuration)
 	memInspectionTicker := time.NewTicker(memInspectionFrequency)
-
 	lastTunnelsEstablished := int32(0)
+
+	startController()
+
 test_loop:
 	for {
 		select {
+
 		case <-testTimer.C:
 			break test_loop
+
 		case <-memInspectionTicker.C:
 			var m runtime.MemStats
 			runtime.ReadMemStats(&m)
@@ -159,15 +214,21 @@ test_loop:
 			} else {
 				n := atomic.LoadInt32(&tunnelsEstablished)
 				fmt.Printf("Tunnels established: %d, MemStats.Sys (peak system memory used): %s, MemStats.TotalAlloc (cumulative allocations): %s\n",
-					n, psiphon.FormatByteCount(m.Sys), psiphon.FormatByteCount(m.TotalAlloc))
+					n, common.FormatByteCount(m.Sys), common.FormatByteCount(m.TotalAlloc))
 				if lastTunnelsEstablished-n >= 0 {
 					t.Fatalf("expected established tunnels")
 				}
 				lastTunnelsEstablished = n
 			}
+
+		case <-reconnectTunnel:
+			controller.TerminateNextActiveTunnel()
+
+		case <-restartController:
+			stopController()
+			startController()
 		}
 	}
 
-	close(shutdownBroadcast)
-	controllerWaitGroup.Wait()
+	stopController()
 }

psiphon/utils.go

@@ -24,7 +24,6 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
-	"math"
 	"net"
 	"net/url"
 	"os"
@@ -189,43 +188,30 @@ func (conn *channelConn) SetWriteDeadline(_ time.Time) error {
 	return common.ContextError(errors.New("unsupported"))
 }
 
-// FormatByteCount returns a string representation of the specified
-// byte count in conventional, human-readable format.
-func FormatByteCount(bytes uint64) string {
-	// Based on: https://bitbucket.org/psiphon/psiphon-circumvention-system/src/b2884b0d0a491e55420ed1888aea20d00fefdb45/Android/app/src/main/java/com/psiphon3/psiphonlibrary/Utils.java?at=default#Utils.java-646
-	base := uint64(1024)
-	if bytes < base {
-		return fmt.Sprintf("%dB", bytes)
-	}
-	exp := int(math.Log(float64(bytes)) / math.Log(float64(base)))
-	return fmt.Sprintf(
-		"%.1f%c", float64(bytes)/math.Pow(float64(base), float64(exp)), "KMGTPEZ"[exp-1])
-}
-
 func emitMemoryMetrics() {
 	var memStats runtime.MemStats
 	runtime.ReadMemStats(&memStats)
 	NoticeInfo("Memory metrics at %s: goroutines %d | total alloc %s | sys %s | heap alloc/sys/idle/inuse/released/objects %s/%s/%s/%s/%s/%d | stack inuse/sys %s/%s | mspan inuse/sys %s/%s | mcached inuse/sys %s/%s | buckhash/gc/other sys %s/%s/%s | nextgc %s",
 		common.GetParentContext(),
 		runtime.NumGoroutine(),
-		FormatByteCount(memStats.TotalAlloc),
-		FormatByteCount(memStats.Sys),
-		FormatByteCount(memStats.HeapAlloc),
-		FormatByteCount(memStats.HeapSys),
-		FormatByteCount(memStats.HeapIdle),
-		FormatByteCount(memStats.HeapInuse),
-		FormatByteCount(memStats.HeapReleased),
+		common.FormatByteCount(memStats.TotalAlloc),
+		common.FormatByteCount(memStats.Sys),
+		common.FormatByteCount(memStats.HeapAlloc),
+		common.FormatByteCount(memStats.HeapSys),
+		common.FormatByteCount(memStats.HeapIdle),
+		common.FormatByteCount(memStats.HeapInuse),
+		common.FormatByteCount(memStats.HeapReleased),
 		memStats.HeapObjects,
-		FormatByteCount(memStats.StackInuse),
-		FormatByteCount(memStats.StackSys),
-		FormatByteCount(memStats.MSpanInuse),
-		FormatByteCount(memStats.MSpanSys),
-		FormatByteCount(memStats.MCacheInuse),
-		FormatByteCount(memStats.MCacheSys),
-		FormatByteCount(memStats.BuckHashSys),
-		FormatByteCount(memStats.GCSys),
-		FormatByteCount(memStats.OtherSys),
-		FormatByteCount(memStats.NextGC))
+		common.FormatByteCount(memStats.StackInuse),
+		common.FormatByteCount(memStats.StackSys),
+		common.FormatByteCount(memStats.MSpanInuse),
+		common.FormatByteCount(memStats.MSpanSys),
+		common.FormatByteCount(memStats.MCacheInuse),
+		common.FormatByteCount(memStats.MCacheSys),
+		common.FormatByteCount(memStats.BuckHashSys),
+		common.FormatByteCount(memStats.GCSys),
+		common.FormatByteCount(memStats.OtherSys),
+		common.FormatByteCount(memStats.NextGC))
 }
 
 func aggressiveGarbageCollection() {