Merge pull request #347 from geebee/master

Add panicwrapper to Server

Server/main.go

@@ -20,22 +20,31 @@
 package main
 
 import (
+	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"strconv"
 	"strings"
+	"syscall"
+	"time"
 
+	"github.com/Psiphon-Inc/panicwrap"
+	"github.com/Psiphon-Inc/rotate-safe-writer"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/server"
 )
 
+var loadedConfigJSON []byte
+
 func main() {
 
 	var generateTrafficRulesFilename string
 	var generateServerEntryFilename string
 	var generateLogFilename string
+	var generatePanicLogFilename string
 	var generateServerIPaddress string
 	var generateServerNetworkInterface string
 	var generateWebServerPort int
@@ -60,6 +69,12 @@ func main() {
 		"",
 		"set application log file name and path; blank for stderr")
 
+	flag.StringVar(
+		&generatePanicLogFilename,
+		"panicLogFilename",
+		"",
+		"set application log file name and path for recording un-recovered panics; blank for stderr")
+
 	flag.StringVar(
 		&generateServerIPaddress,
 		"ipaddress",
@@ -135,6 +150,7 @@ func main() {
 			server.GenerateConfig(
 				&server.GenerateConfigParams{
 					LogFilename:          generateLogFilename,
+					PanicLogFilename:     generatePanicLogFilename,
 					ServerIPAddress:      serverIPaddress,
 					EnableSSHAPIRequests: true,
 					WebServerPort:        generateWebServerPort,
@@ -172,6 +188,26 @@ func main() {
 			os.Exit(1)
 		}
 
+		loadedConfigJSON = configJSON
+
+		// Comments from: https://github.com/mitchellh/panicwrap#usage
+		// Unhandled panic wrapper. Logs it, then re-executes the current executable
+		exitStatus, err := panicwrap.Wrap(&panicwrap.WrapConfig{
+			Handler:        panicHandler,
+			ForwardSignals: []os.Signal{os.Interrupt, os.Kill, syscall.SIGTERM, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGTSTP, syscall.SIGCONT},
+		})
+		if err != nil {
+			fmt.Printf("failed to set up the panic wrapper: %s\n", err)
+			os.Exit(1)
+		}
+
+		// If exitStatus >= 0, then we're the parent process and the panicwrap
+		// re-executed ourselves and completed. Just exit with the proper status.
+		if exitStatus >= 0 {
+			os.Exit(exitStatus)
+		}
+		// Otherwise, exitStatus < 0 means we're the child. Continue executing as normal
+
 		err = server.RunServices(configJSON)
 		if err != nil {
 			fmt.Printf("run failed: %s\n", err)
@@ -190,3 +226,44 @@ func (list *stringListFlag) Set(flagValue string) error {
 	*list = append(*list, flagValue)
 	return nil
 }
+
+func panicHandler(output string) {
+	if len(loadedConfigJSON) > 0 {
+		config, err := server.LoadConfig([]byte(loadedConfigJSON))
+		if err != nil {
+			fmt.Printf("error parsing configuration file: %s\n%s\n", err, output)
+			os.Exit(1)
+		}
+
+		logEvent := make(map[string]string)
+		logEvent["host_id"] = config.HostID
+		logEvent["build_rev"] = common.GetBuildInfo().BuildRev
+		logEvent["timestamp"] = time.Now().Format(time.RFC3339)
+		logEvent["event_name"] = "panic"
+		logEvent["panic"] = output
+
+		var jsonWriter io.Writer
+		if config.PanicLogFilename != "" {
+			panicLog, err := rotate.NewRotatableFileWriter(config.PanicLogFilename, 0666)
+			if err != nil {
+				fmt.Printf("unable to set panic log output: %s\n%s\n", err, output)
+				os.Exit(1)
+			}
+			jsonWriter = panicLog
+		} else {
+			jsonWriter = os.Stderr
+		}
+
+		enc := json.NewEncoder(jsonWriter)
+		err = enc.Encode(logEvent)
+		if err != nil {
+			fmt.Printf("unable to serialize panic message to JSON: %s\n%s\n", err, output)
+			os.Exit(1)
+		}
+	} else {
+		fmt.Printf("no configuration JSON was loaded, cannot continue\n%s\n", output)
+		os.Exit(1)
+	}
+
+	os.Exit(1)
+}

Server/make.bash

@@ -65,7 +65,7 @@ build_for_linux () {
   chmod 555 psiphond
 
   if [ "$1" == "generate" ]; then
-    ./psiphond --ipaddress 0.0.0.0 --web 3000 --protocol SSH:3001 --protocol OSSH:3002 --logFilename /var/log/psiphond/psiphond.log generate
+    ./psiphond --ipaddress 0.0.0.0 --web 3000 --protocol SSH:3001 --protocol OSSH:3002 --logFilename /var/log/psiphond/psiphond.log --panicLogFilename /var/log/psiphond/psiphond-panics.log generate
 
     chmod 666 psiphond.config
     chmod 666 psiphond-traffic-rules.config

psiphon/server/config.go

@@ -63,6 +63,11 @@ type Config struct {
 	// to. When blank, logs are written to stderr.
 	LogFilename string
 
+	// PanicLogFilename specifies the path of the file to
+	// log unrecovered panics to. When blank, logs are
+	// written to stderr
+	PanicLogFilename string
+
 	// DiscoveryValueHMACKey is the network-wide secret value
 	// used to determine a unique discovery strategy.
 	DiscoveryValueHMACKey string
@@ -339,6 +344,7 @@ func validateNetworkAddress(address string, requireIPaddress bool) error {
 // a generated server config.
 type GenerateConfigParams struct {
 	LogFilename          string
+	PanicLogFilename     string
 	LogLevel             string
 	ServerIPAddress      string
 	WebServerPort        int
@@ -495,6 +501,7 @@ func GenerateConfig(params *GenerateConfigParams) ([]byte, []byte, []byte, error
 	config := &Config{
 		LogLevel:                       logLevel,
 		LogFilename:                    params.LogFilename,
+		PanicLogFilename:               params.PanicLogFilename,
 		GeoIPDatabaseFilenames:         nil,
 		HostID:                         "example-host-id",
 		ServerIPAddress:                params.ServerIPAddress,

psiphon/server/services.go

@@ -134,6 +134,12 @@ func RunServices(configJSON []byte) error {
 		}
 	}()
 
+	// In addition to the actual signal handling here, there is
+	// a list of signals that need to be passed through panicwrap
+	// in 'github.com/Psiphon-Labs/psiphon-tunnel-core/Server/main.go'
+	// where 'panicwrap.Wrap' is called. The handled signals below, and the
+	// list there must be kept in sync to ensure proper signal handling
+
 	// An OS signal triggers an orderly shutdown
 	systemStopSignal := make(chan os.Signal, 1)
 	signal.Notify(systemStopSignal, os.Interrupt, os.Kill, syscall.SIGTERM)