Add customizations for limited memory environments

- Emit runtime.MemStats when starting and ending
  tunnel establishment, which are high memory
  usage phases

- Emit peak concurrent tunnel establishments

- Call debug.FreeOSMemory and debug.SetGCPercent
  to deal with memory pressure around tunnel
  establishment periods

psiphon/config.go

@@ -475,6 +475,11 @@ type Config struct {
 	// file descriptor. The file descriptor is duped in NewController.
 	// When PacketTunnelTunDeviceFileDescriptor is set, TunnelPoolSize must be 1.
 	PacketTunnelTunFileDescriptor int
+
+	// LimitedMemoryEnvironment enables memory usage metrics logging, to track
+	// memory usage, and selective aggressively garbage collection at high memory
+	// pressure phases of operation.
+	LimitedMemoryEnvironment bool
 }
 
 // DownloadURL specifies a URL for downloading resources along with parameters

psiphon/controller.go

@@ -54,6 +54,9 @@ type Controller struct {
 	nextTunnel                        int
 	startedConnectedReporter          bool
 	isEstablishing                    bool
+	concurrentEstablishTunnelsMutex   sync.Mutex
+	concurrentEstablishTunnels        int32
+	peakConcurrentEstablishTunnels    int32
 	establishWaitGroup                *sync.WaitGroup
 	stopEstablishingBroadcast         chan struct{}
 	candidateServerEntries            chan *candidateServerEntry
@@ -1017,6 +1020,16 @@ func (controller *Controller) startEstablishing() {
 	}
 	NoticeInfo("start establishing")
 
+	controller.concurrentEstablishTunnelsMutex.Lock()
+	controller.concurrentEstablishTunnels = 0
+	controller.peakConcurrentEstablishTunnels = 0
+	controller.concurrentEstablishTunnelsMutex.Unlock()
+
+	if controller.config.LimitedMemoryEnvironment {
+		setAggressiveGarbageCollection()
+		emitMemoryMetrics()
+	}
+
 	controller.isEstablishing = true
 	controller.establishWaitGroup = new(sync.WaitGroup)
 	controller.stopEstablishingBroadcast = make(chan struct{})
@@ -1081,6 +1094,18 @@ func (controller *Controller) stopEstablishing() {
 	controller.stopEstablishingBroadcast = nil
 	controller.candidateServerEntries = nil
 	controller.serverAffinityDoneBroadcast = nil
+
+	controller.concurrentEstablishTunnelsMutex.Lock()
+	peakConcurrent := controller.peakConcurrentEstablishTunnels
+	controller.concurrentEstablishTunnels = 0
+	controller.peakConcurrentEstablishTunnels = 0
+	controller.concurrentEstablishTunnelsMutex.Unlock()
+	NoticeInfo("peak concurrent establish tunnels: %d", peakConcurrent)
+
+	if controller.config.LimitedMemoryEnvironment {
+		emitMemoryMetrics()
+		setStandardGarbageCollection()
+	}
 }
 
 // establishCandidateGenerator populates the candidate queue with server entries
@@ -1267,6 +1292,13 @@ loop:
 			continue
 		}
 
+		controller.concurrentEstablishTunnelsMutex.Lock()
+		controller.concurrentEstablishTunnels += 1
+		if controller.concurrentEstablishTunnels > controller.peakConcurrentEstablishTunnels {
+			controller.peakConcurrentEstablishTunnels = controller.concurrentEstablishTunnels
+		}
+		controller.concurrentEstablishTunnelsMutex.Unlock()
+
 		tunnel, err := EstablishTunnel(
 			controller.config,
 			controller.untunneledDialConfig,
@@ -1275,6 +1307,11 @@ loop:
 			candidateServerEntry.serverEntry,
 			candidateServerEntry.adjustedEstablishStartTime,
 			controller) // TunnelOwner
+
+		controller.concurrentEstablishTunnelsMutex.Lock()
+		controller.concurrentEstablishTunnels -= 1
+		controller.concurrentEstablishTunnelsMutex.Unlock()
+
 		if err != nil {
 
 			// Unblock other candidates immediately when

psiphon/utils.go

@@ -24,9 +24,12 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"math"
 	"net"
 	"net/url"
 	"os"
+	"runtime"
+	"runtime/debug"
 	"syscall"
 	"time"
 
@@ -185,3 +188,50 @@ func (conn *channelConn) SetReadDeadline(_ time.Time) error {
 func (conn *channelConn) SetWriteDeadline(_ time.Time) error {
 	return common.ContextError(errors.New("unsupported"))
 }
+
+// Based on: https://bitbucket.org/psiphon/psiphon-circumvention-system/src/b2884b0d0a491e55420ed1888aea20d00fefdb45/Android/app/src/main/java/com/psiphon3/psiphonlibrary/Utils.java?at=default#Utils.java-646
+func byteCountFormatter(bytes uint64) string {
+	base := uint64(1024)
+	if bytes < base {
+		return fmt.Sprintf("%dB", bytes)
+	}
+	exp := int(math.Log(float64(bytes)) / math.Log(float64(base)))
+	return fmt.Sprintf(
+		"%.1f%c", float64(bytes)/math.Pow(float64(base), float64(exp)), "KMGTPEZ"[exp-1])
+}
+
+func emitMemoryMetrics() {
+	var memStats runtime.MemStats
+	runtime.ReadMemStats(&memStats)
+	NoticeInfo("Memory metrics at %s: goroutines %d | total alloc %s | sys %s | heap alloc/sys/idle/inuse/released/objects %s/%s/%s/%s/%s/%d | stack inuse/sys %s/%s | mspan inuse/sys %s/%s | mcached inuse/sys %s/%s | buckhash/gc/other sys %s/%s/%s | nextgc %s",
+		runtime.NumGoroutine(),
+		common.GetParentContext(),
+		byteCountFormatter(memStats.TotalAlloc),
+		byteCountFormatter(memStats.Sys),
+		byteCountFormatter(memStats.HeapAlloc),
+		byteCountFormatter(memStats.HeapSys),
+		byteCountFormatter(memStats.HeapIdle),
+		byteCountFormatter(memStats.HeapInuse),
+		byteCountFormatter(memStats.HeapReleased),
+		memStats.HeapObjects,
+		byteCountFormatter(memStats.StackInuse),
+		byteCountFormatter(memStats.StackSys),
+		byteCountFormatter(memStats.MSpanInuse),
+		byteCountFormatter(memStats.MSpanSys),
+		byteCountFormatter(memStats.MCacheInuse),
+		byteCountFormatter(memStats.MCacheSys),
+		byteCountFormatter(memStats.BuckHashSys),
+		byteCountFormatter(memStats.GCSys),
+		byteCountFormatter(memStats.OtherSys),
+		byteCountFormatter(memStats.NextGC))
+}
+
+func setAggressiveGarbageCollection() {
+	debug.SetGCPercent(20)
+	debug.FreeOSMemory()
+}
+
+func setStandardGarbageCollection() {
+	debug.SetGCPercent(100)
+	debug.FreeOSMemory()
+}