Add option to dump pprof profiles on SIGUSR2

psiphon/server/config.go

@@ -206,6 +206,13 @@ type Config struct {
 	// The default, 0, disables load logging.
 	LoadMonitorPeriodSeconds int
 
+	// ProcessProfileOutputDirectory is the path of a directory to which
+	// process profiles will be written when signaled with SIGUSR2. The
+	// files are overwritten on each invocation. When set to the default
+	// value, blank, no profiles are written on SIGUSR2. Profiles include
+	// the default profiles here: https://golang.org/pkg/runtime/pprof/#Profile.
+	ProcessProfileOutputDirectory string
+
 	// TrafficRulesFilename is the path of a file containing a
 	// JSON-encoded TrafficRulesSet, the traffic rules to apply to
 	// Psiphon client tunnels.

psiphon/server/services.go

@@ -26,7 +26,9 @@ package server
 import (
 	"os"
 	"os/signal"
+	"path/filepath"
 	"runtime"
+	"runtime/pprof"
 	"sync"
 	"syscall"
 	"time"
@@ -121,7 +123,7 @@ func RunServices(configJSON []byte) error {
 	reloadSupportServicesSignal := make(chan os.Signal, 1)
 	signal.Notify(reloadSupportServicesSignal, syscall.SIGUSR1)
 
-	// SIGUSR2 triggers an immediate load log
+	// SIGUSR2 triggers an immediate load log and optional profile dump
 	logServerLoadSignal := make(chan os.Signal, 1)
 	signal.Notify(logServerLoadSignal, syscall.SIGUSR2)
 
@@ -135,11 +137,17 @@ loop:
 			// Reset traffic rules for established clients to reflect reloaded config
 			// TODO: only update when traffic rules config has changed
 			tunnelServer.ResetAllClientTrafficRules()
+
 		case <-logServerLoadSignal:
+			// Profiles are dumped first to ensure some diagnostics are
+			// available in case logServerLoad deadlocks.
+			dumpProcessProfiles(supportServices.Config)
 			logServerLoad(tunnelServer)
+
 		case <-systemStopSignal:
 			log.WithContext().Info("shutdown by system")
 			break loop
+
 		case err = <-errors:
 			log.WithContextFields(LogFields{"error": err}).Error("service failed")
 			break loop
@@ -152,6 +160,35 @@ loop:
 	return err
 }
 
+func dumpProcessProfiles(config *Config) {
+
+	if config.ProcessProfileOutputDirectory != "" {
+
+		for _, profileName := range []string{
+			"goroutine", "heap", "threadcreate", "block"} {
+
+			fileName := filepath.Join(
+				config.ProcessProfileOutputDirectory, profileName+".profile")
+
+			writer, err := os.OpenFile(
+				fileName, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
+
+			if err == nil {
+				err = pprof.Lookup(profileName).WriteTo(writer, 1)
+				writer.Close()
+			}
+
+			if err != nil {
+				log.WithContextFields(
+					LogFields{
+						"error":       err,
+						"profileName": profileName}).Error("write profile failed")
+			}
+		}
+	}
+
+}
+
 func logServerLoad(server *TunnelServer) {
 
 	// golang runtime stats