Merge pull request #198 from rod-hynes/master

Logging changes + clientVerification changes

psiphon/server/api.go

@@ -33,7 +33,12 @@ import (
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon"
 )
 
-const MAX_API_PARAMS_SIZE = 256 * 1024 // 256KB
+const (
+	MAX_API_PARAMS_SIZE = 256 * 1024 // 256KB
+
+	CLIENT_PLATFORM_ANDROID = "Android"
+	CLIENT_PLATFORM_WINDOWS = "Windows"
+)
 
 type requestJSONObject map[string]interface{}
 
@@ -123,10 +128,10 @@ func handshakeAPIRequestHandler(
 	// Note: no guarantee that PsinetDatabase won't reload between calls
 
 	handshakeResponse.Homepages = support.PsinetDatabase.GetHomepages(
-		sponsorID, clientRegion, clientPlatform)
+		sponsorID, clientRegion, isMobileClientPlatform(clientPlatform))
 
 	handshakeResponse.UpgradeClientVersion = support.PsinetDatabase.GetUpgradeClientVersion(
-		clientVersion, clientPlatform)
+		clientVersion, normalizeClientPlatform(clientPlatform))
 
 	handshakeResponse.HttpsRequestRegexes = support.PsinetDatabase.GetHttpsRequestRegexes(
 		sponsorID)
@@ -330,7 +335,23 @@ func clientVerificationAPIRequestHandler(
 		return nil, psiphon.ContextError(errors.New("invalid params"))
 	}
 
-	// TODO: implement
+	// Ignoring error as params are validated
+	clientPlatform, _ := getStringRequestParam(params, "client_platform")
+
+	verificationData, err := getJSONObjectRequestParam(params, "verificationData")
+	if err != nil {
+		return nil, psiphon.ContextError(err)
+	}
+
+	var verified bool
+	switch normalizeClientPlatform(clientPlatform) {
+	case CLIENT_PLATFORM_ANDROID:
+		verified = verifySafetyNetPayload(verificationData)
+	}
+
+	if verified {
+		// TODO: change throttling treatment
+	}
 
 	return make([]byte, 0), nil
 }
@@ -548,6 +569,22 @@ func getMapStringInt64RequestParam(params requestJSONObject, name string) (map[s
 	return result, nil
 }
 
+// Normalize reported client platform. Android clients, for example, report
+// OS version, rooted status, and Google Play build status in the clientPlatform
+// string along with "Android".
+func normalizeClientPlatform(clientPlatform string) string {
+
+	if strings.Contains(strings.ToLower(clientPlatform), strings.ToLower(CLIENT_PLATFORM_ANDROID)) {
+		return CLIENT_PLATFORM_ANDROID
+	}
+
+	return CLIENT_PLATFORM_WINDOWS
+}
+
+func isMobileClientPlatform(clientPlatform string) bool {
+	return normalizeClientPlatform(clientPlatform) == CLIENT_PLATFORM_ANDROID
+}
+
 // Input validators follow the legacy validations rules in psi_web.
 
 func isServerSecret(support *SupportServices, value string) bool {

psiphon/server/config.go

@@ -66,16 +66,9 @@ type Config struct {
 	// panic, fatal, error, warn, info, debug
 	LogLevel string
 
-	// SyslogFacility specifies the syslog facility to log to.
-	// When set, the local syslog service is used for message
-	// logging.
-	// Valid values include: "user", "local0", "local1", etc.
-	SyslogFacility string
-
-	// SyslogTag specifies an optional tag for syslog log
-	// messages. The default tag is "psiphon-server". The
-	// fail2ban logs, if enabled, also use this tag.
-	SyslogTag string
+	// LogFilename specifies the path of the file to log
+	// to. When blank, logs are written to stderr.
+	LogFilename string
 
 	// Fail2BanFormat is a string format specifier for the
 	// log message format to use for fail2ban integration for
@@ -88,6 +81,11 @@ type Config struct {
 	// "Authentication failure for psiphon-client from %s".
 	Fail2BanFormat string
 
+	// LogFilename specifies the path of the file to log
+	// fail2ban messages to. When blank, logs are written to
+	// stderr.
+	Fail2BanLogFilename string
+
 	// DiscoveryValueHMACKey is the network-wide secret value
 	// used to determine a unique discovery strategy.
 	DiscoveryValueHMACKey string
@@ -224,7 +222,7 @@ func (config *Config) RunLoadMonitor() bool {
 }
 
 // UseFail2Ban indicates whether to log client IP addresses, in authentication
-// failure cases, to the local syslog service AUTH facility for use by fail2ban.
+// failure cases, for use by fail2ban.
 func (config *Config) UseFail2Ban() bool {
 	return config.Fail2BanFormat != ""
 }
@@ -461,8 +459,6 @@ func GenerateConfig(params *GenerateConfigParams) ([]byte, []byte, []byte, error
 
 	config := &Config{
 		LogLevel:                       "info",
-		SyslogFacility:                 "user",
-		SyslogTag:                      "psiphon-server",
 		Fail2BanFormat:                 "Authentication failure for psiphon-client from %s",
 		GeoIPDatabaseFilename:          "",
 		HostID:                         "example-host-id",

psiphon/server/log.go

@@ -20,12 +20,11 @@
 package server
 
 import (
+	"fmt"
 	"io"
-	"log/syslog"
 	"os"
 
 	"github.com/Psiphon-Inc/logrus"
-	logrus_syslog "github.com/Psiphon-Inc/logrus/hooks/syslog"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon"
 )
 
@@ -68,7 +67,18 @@ func NewLogWriter() *io.PipeWriter {
 	return log.Writer()
 }
 
+// LogFail2Ban logs a message using the format specified by
+// config.Fail2BanFormat and the given client IP address. This
+// is for integration with fail2ban for blocking abusive
+// clients by source IP address.
+func LogFail2Ban(clientIPAddress string) {
+	fail2BanLogger.Info(
+		fmt.Sprintf(fail2BanFormat, clientIPAddress))
+}
+
 var log *ContextLogger
+var fail2BanFormat string
+var fail2BanLogger *logrus.Logger
 
 // InitLogging configures a logger according to the specified
 // config params. If not called, the default logger set by the
@@ -84,76 +94,56 @@ func InitLogging(config *Config) error {
 		return psiphon.ContextError(err)
 	}
 
-	hooks := make(logrus.LevelHooks)
-
-	var syslogHook *logrus_syslog.SyslogHook
-
-	if config.SyslogFacility != "" {
+	logWriter := os.Stderr
 
-		syslogHook, err = logrus_syslog.NewSyslogHook(
-			"", "", getSyslogPriority(config), config.SyslogTag)
+	if config.LogFilename != "" {
+		logWriter, err = os.OpenFile(
+			config.LogFilename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
 		if err != nil {
 			return psiphon.ContextError(err)
 		}
-
-		hooks.Add(syslogHook)
 	}
 
 	log = &ContextLogger{
 		&logrus.Logger{
-			Out:       os.Stderr,
-			Formatter: new(logrus.JSONFormatter),
-			Hooks:     hooks,
+			Out:       logWriter,
+			Formatter: &logrus.JSONFormatter{},
 			Level:     level,
 		},
 	}
 
-	return nil
-}
+	if config.Fail2BanFormat != "" {
 
-// getSyslogPriority determines golang's syslog "priority" value
-// based on the provided config.
-func getSyslogPriority(config *Config) syslog.Priority {
-
-	// TODO: assumes log.Level filter applies?
-	severity := syslog.LOG_DEBUG
-
-	facilityCodes := map[string]syslog.Priority{
-		"kern":     syslog.LOG_KERN,
-		"user":     syslog.LOG_USER,
-		"mail":     syslog.LOG_MAIL,
-		"daemon":   syslog.LOG_DAEMON,
-		"auth":     syslog.LOG_AUTH,
-		"syslog":   syslog.LOG_SYSLOG,
-		"lpr":      syslog.LOG_LPR,
-		"news":     syslog.LOG_NEWS,
-		"uucp":     syslog.LOG_UUCP,
-		"cron":     syslog.LOG_CRON,
-		"authpriv": syslog.LOG_AUTHPRIV,
-		"ftp":      syslog.LOG_FTP,
-		"local0":   syslog.LOG_LOCAL0,
-		"local1":   syslog.LOG_LOCAL1,
-		"local2":   syslog.LOG_LOCAL2,
-		"local3":   syslog.LOG_LOCAL3,
-		"local4":   syslog.LOG_LOCAL4,
-		"local5":   syslog.LOG_LOCAL5,
-		"local6":   syslog.LOG_LOCAL6,
-		"local7":   syslog.LOG_LOCAL7,
-	}
+		fail2BanFormat = config.Fail2BanFormat
 
-	facility, ok := facilityCodes[config.SyslogFacility]
-	if !ok {
-		facility = syslog.LOG_USER
+		fail2BanLogWriter := os.Stderr
+
+		if config.Fail2BanLogFilename != "" {
+			logWriter, err = os.OpenFile(
+				config.Fail2BanLogFilename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
+			if err != nil {
+				return psiphon.ContextError(err)
+			}
+		}
+
+		fail2BanLogger = &logrus.Logger{
+			Out: fail2BanLogWriter,
+			Formatter: &logrus.TextFormatter{
+				DisableColors: true,
+				FullTimestamp: true,
+			},
+			Level: level,
+		}
 	}
 
-	return severity | facility
+	return nil
 }
 
 func init() {
 	log = &ContextLogger{
 		&logrus.Logger{
 			Out:       os.Stderr,
-			Formatter: new(logrus.JSONFormatter),
+			Formatter: &logrus.JSONFormatter{},
 			Hooks:     make(logrus.LevelHooks),
 			Level:     logrus.DebugLevel,
 		},

psiphon/server/psinet/psinet.go

@@ -38,9 +38,6 @@ import (
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon"
 )
 
-var CLIENT_PLATFORM_ANDROID string = "Android"
-var CLIENT_PLATFORM_WINDOWS string = "Windows"
-
 // Database serves Psiphon API data requests. It's safe for
 // concurrent usage. The Reload function supports hot reloading
 // of Psiphon network data while the server is running.
@@ -156,7 +153,7 @@ func (db *Database) Reload(filename string) error {
 
 	configJSON, err := ioutil.ReadFile(filename)
 	if err != nil {
-		return err
+		return psiphon.ContextError(err)
 	}
 
 	// Unmarshal first validates the provided JSON and then
@@ -164,12 +161,12 @@ func (db *Database) Reload(filename string) error {
 	// persists if the new JSON is malformed.
 	err = json.Unmarshal(configJSON, &db)
 
-	return err
+	return psiphon.ContextError(err)
 }
 
 // GetHomepages returns a list of  home pages for the specified sponsor,
 // region, and platform.
-func (db *Database) GetHomepages(sponsorID, clientRegion, clientPlatform string) []string {
+func (db *Database) GetHomepages(sponsorID, clientRegion string, isMobilePlatform bool) []string {
 	db.RLock()
 	defer db.RUnlock()
 
@@ -183,7 +180,7 @@ func (db *Database) GetHomepages(sponsorID, clientRegion, clientPlatform string)
 
 	homePages := sponsor.HomePages
 
-	if getClientPlatform(clientPlatform) == CLIENT_PLATFORM_ANDROID {
+	if isMobilePlatform {
 		if sponsor.MobileHomePages != nil {
 			homePages = sponsor.MobileHomePages
 		}
@@ -213,20 +210,22 @@ func (db *Database) GetHomepages(sponsorID, clientRegion, clientPlatform string)
 
 // GetUpgradeClientVersion returns a new client version when an upgrade is
 // indicated for the specified client current version. The result is "" when
-// no upgrade is available.
+// no upgrade is available. Caller should normalize clientPlatform.
 func (db *Database) GetUpgradeClientVersion(clientVersion, clientPlatform string) string {
 	db.RLock()
 	defer db.RUnlock()
 
 	// Check lastest version number against client version number
-	platform := getClientPlatform(clientPlatform)
 
-	// If no versions exist for this platform
-	clientVersions, ok := db.Versions[platform]
+	clientVersions, ok := db.Versions[clientPlatform]
 	if !ok {
 		return ""
 	}
 
+	if len(clientVersions) == 0 {
+		return ""
+	}
+
 	// NOTE: Assumes versions list is in ascending version order
 	lastVersion := clientVersions[len(clientVersions)-1].Version
 
@@ -525,15 +524,3 @@ func parseSshKeyString(sshKeyString string) (keyType string, key string) {
 
 	return sshKeyArr[0], sshKeyArr[1]
 }
-
-// Parse client platform string for platform identifier
-// and return corresponding platform.
-func getClientPlatform(clientPlatformString string) string {
-	platform := CLIENT_PLATFORM_WINDOWS
-
-	if strings.Contains(strings.ToLower(clientPlatformString), strings.ToLower(CLIENT_PLATFORM_ANDROID)) {
-		platform = CLIENT_PLATFORM_ANDROID
-	}
-
-	return platform
-}

psiphon/server/safetyNet.go

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2016, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package server
+
+func verifySafetyNetPayload(params requestJSONObject) bool {
+
+	// TODO: implement
+
+	return true
+}

psiphon/server/tunnelServer.go

@@ -618,15 +618,14 @@ func (sshClient *sshClient) authLogCallback(conn ssh.ConnMetadata, method string
 			return
 		}
 
-		logFields := LogFields{"error": err, "method": method}
 		if sshClient.sshServer.support.Config.UseFail2Ban() {
 			clientIPAddress := psiphon.IPAddressFromAddr(conn.RemoteAddr())
 			if clientIPAddress != "" {
-				logFields["fail2ban"] = fmt.Sprintf(
-					sshClient.sshServer.support.Config.Fail2BanFormat, clientIPAddress)
+				LogFail2Ban(clientIPAddress)
 			}
 		}
-		log.WithContextFields(logFields).Error("authentication failed")
+
+		log.WithContextFields(LogFields{"error": err, "method": method}).Error("authentication failed")
 
 	} else {