Merge pull request #361 from rod-hynes/master

Misc. fixes/enhancements

psiphon/common/osl/osl.go

@@ -199,9 +199,12 @@ type ClientSeedState struct {
 // ClientSeedProgress tracks client progress towards seeding SLOKs for
 // a particular scheme.
 type ClientSeedProgress struct {
+	// Note: 64-bit ints used with atomic operations are at placed
+	// at the start of struct to ensure 64-bit alignment.
+	// (https://golang.org/pkg/sync/atomic/#pkg-note-BUG)
+	progressSLOKTime int64
 	scheme           *Scheme
 	trafficProgress  []*TrafficValues
-	progressSLOKTime int64
 }
 
 // ClientSeedPortForward map a client port forward, which is relaying

psiphon/common/protocol/protocol.go

@@ -32,10 +32,11 @@ const (
 	TUNNEL_PROTOCOL_FRONTED_MEEK                  = "FRONTED-MEEK-OSSH"
 	TUNNEL_PROTOCOL_FRONTED_MEEK_HTTP             = "FRONTED-MEEK-HTTP-OSSH"
 
-	SERVER_ENTRY_SOURCE_EMBEDDED  = "EMBEDDED"
-	SERVER_ENTRY_SOURCE_REMOTE    = "REMOTE"
-	SERVER_ENTRY_SOURCE_DISCOVERY = "DISCOVERY"
-	SERVER_ENTRY_SOURCE_TARGET    = "TARGET"
+	SERVER_ENTRY_SOURCE_EMBEDDED   = "EMBEDDED"
+	SERVER_ENTRY_SOURCE_REMOTE     = "REMOTE"
+	SERVER_ENTRY_SOURCE_DISCOVERY  = "DISCOVERY"
+	SERVER_ENTRY_SOURCE_TARGET     = "TARGET"
+	SERVER_ENTRY_SOURCE_OBFUSCATED = "OBFUSCATED"
 
 	CAPABILITY_SSH_API_REQUESTS            = "ssh-api-requests"
 	CAPABILITY_UNTUNNELED_WEB_API_REQUESTS = "handshake"
@@ -69,6 +70,7 @@ var SupportedServerEntrySources = []string{
 	SERVER_ENTRY_SOURCE_REMOTE,
 	SERVER_ENTRY_SOURCE_DISCOVERY,
 	SERVER_ENTRY_SOURCE_TARGET,
+	SERVER_ENTRY_SOURCE_OBFUSCATED,
 }
 
 func TunnelProtocolUsesSSH(protocol string) bool {

psiphon/common/protocol/serverEntry.go

@@ -153,7 +153,8 @@ func EncodeServerEntry(serverEntry *ServerEntry) (string, error) {
 //
 // The resulting ServerEntry.LocalSource is populated with serverEntrySource,
 // which should be one of SERVER_ENTRY_SOURCE_EMBEDDED, SERVER_ENTRY_SOURCE_REMOTE,
-// SERVER_ENTRY_SOURCE_DISCOVERY, SERVER_ENTRY_SOURCE_TARGET.
+// SERVER_ENTRY_SOURCE_DISCOVERY, SERVER_ENTRY_SOURCE_TARGET,
+// SERVER_ENTRY_SOURCE_OBFUSCATED.
 // ServerEntry.LocalTimestamp is populated with the provided timestamp, which
 // should be a RFC 3339 formatted string. These local fields are stored with the
 // server entry and reported to the server as stats (a coarse granularity timestamp

psiphon/config.go

@@ -349,14 +349,11 @@ type Config struct {
 	// bytes sent and received.
 	EmitBytesTransferred bool
 
-	// UseIndistinguishableTLS enables use of an alternative TLS stack with a less
+	// UseIndistinguishableTLS enables use of alternative TLS profiles with a less
 	// distinct fingerprint (ClientHello content) than the stock Go TLS.
-	// UseIndistinguishableTLS only applies to untunneled TLS connections. This
-	// parameter is only supported on platforms built with OpenSSL.
-	// Requires TrustedCACertificatesFilename to be set.
 	UseIndistinguishableTLS bool
 
-	// UseTrustedCACertificates toggles use of the trusted CA certs, specified
+	// UseTrustedCACertificatesForStockTLS toggles use of the trusted CA certs, specified
 	// in TrustedCACertificatesFilename, for tunneled TLS connections that expect
 	// server certificates signed with public certificate authorities (currently,
 	// only upgrade downloads). This option is used with stock Go TLS in cases where
@@ -366,8 +363,8 @@ type Config struct {
 
 	// TrustedCACertificatesFilename specifies a file containing trusted CA certs.
 	// The file contents should be compatible with OpenSSL's SSL_CTX_load_verify_locations.
-	// When specified, this enables use of indistinguishable TLS for HTTPS requests
-	// that require typical (system CA) server authentication.
+	// When specified, this enables use of OpenSSL for HTTPS requests that require
+	// typical (system CA) server authentication.
 	TrustedCACertificatesFilename string
 
 	// DisablePeriodicSshKeepAlive indicates whether to send an SSH keepalive every
@@ -475,7 +472,7 @@ type DownloadURL struct {
 
 	// SkipVerify indicates whether to verify HTTPS certificates. It some
 	// circumvention scenarios, verification is not possible. This must
-	// only be set to true when the resource its own verification mechanism.
+	// only be set to true when the resource has its own verification mechanism.
 	SkipVerify bool
 
 	// OnlyAfterAttempts specifies how to schedule this URL when downloading

psiphon/remoteServerList.go

@@ -76,7 +76,7 @@ func FetchCommonRemoteServerList(
 		return fmt.Errorf("failed to unpack common remote server list: %s", common.ContextError(err))
 	}
 
-	err = storeServerEntries(serverListPayload)
+	err = storeServerEntries(serverListPayload, protocol.SERVER_ENTRY_SOURCE_REMOTE)
 	if err != nil {
 		return fmt.Errorf("failed to store common remote server list: %s", common.ContextError(err))
 	}
@@ -263,7 +263,7 @@ func FetchObfuscatedServerLists(
 			continue
 		}
 
-		err = storeServerEntries(serverListPayload)
+		err = storeServerEntries(serverListPayload, protocol.SERVER_ENTRY_SOURCE_OBFUSCATED)
 		if err != nil {
 			failed = true
 			NoticeAlert("failed to store obfuscated server list file (%s): %s", hexID, common.ContextError(err))
@@ -389,12 +389,12 @@ func unpackRemoteServerListFile(
 	return payload, nil
 }
 
-func storeServerEntries(serverList string) error {
+func storeServerEntries(serverList, serverEntrySource string) error {
 
 	serverEntries, err := protocol.DecodeAndValidateServerEntryList(
 		serverList,
 		common.GetCurrentTimestamp(),
-		protocol.SERVER_ENTRY_SOURCE_REMOTE)
+		serverEntrySource)
 	if err != nil {
 		return common.ContextError(err)
 	}

psiphon/server/server_test.go

@@ -286,7 +286,7 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
 	sponsorID, expectedHomepageURL := pavePsinetDatabaseFile(
 		t, runConfig.doDefaultSessionID, psinetFilename)
 
-	// Pave OSL config for SLOk testing
+	// Pave OSL config for SLOK testing
 	oslConfigFilename := filepath.Join(testDataDirName, "osl_config.json")
 	propagationChannelID := paveOSLConfigFile(t, oslConfigFilename)
 

psiphon/server/tunnelServer.go

@@ -42,6 +42,7 @@ import (
 )
 
 const (
+	SSH_AUTH_LOG_PERIOD                   = 30 * time.Minute
 	SSH_HANDSHAKE_TIMEOUT                 = 30 * time.Second
 	SSH_CONNECTION_READ_DEADLINE          = 5 * time.Minute
 	SSH_TCP_PORT_FORWARD_COPY_BUFFER_SIZE = 8192
@@ -236,6 +237,11 @@ func (server *TunnelServer) GetEstablishTunnels() bool {
 }
 
 type sshServer struct {
+	// Note: 64-bit ints used with atomic operations are at placed
+	// at the start of struct to ensure 64-bit alignment.
+	// (https://golang.org/pkg/sync/atomic/#pkg-note-BUG)
+	lastAuthLog          int64
+	authFailedCount      int64
 	support              *SupportServices
 	establishTunnels     int32
 	shutdownBroadcast    <-chan struct{}
@@ -986,12 +992,24 @@ func (sshClient *sshClient) authLogCallback(conn ssh.ConnMetadata, method string
 		//   deliberately blocked; and in any case fail2ban adds iptables rules which can only block
 		//   by direct remote IP, not by original client IP. Fronted meek has the same iptables issue.
 		//
-		// TODO: random scanning and brute forcing of port 22 will result in log noise. To eliminate
-		// this, and to also cover meek protocols, and bad obfuscation keys, and bad inputs to the web
-		// server, consider implementing fail2ban-type logic directly in this server, with the ability
-		// to use X-Forwarded-For (when trustworthy; e.g, from a CDN).
+		// Random scanning and brute forcing of port 22 will result in log noise. To mitigate this,
+		// not every authentication failure is logged. A summary log is emitted periodically to
+		// retain some record of this activity in case this is relevent to, e.g., a performance
+		// investigation.
+
+		atomic.AddInt64(&sshClient.sshServer.authFailedCount, 1)
+
+		lastAuthLog := monotime.Time(atomic.LoadInt64(&sshClient.sshServer.lastAuthLog))
+		if monotime.Since(lastAuthLog) > SSH_AUTH_LOG_PERIOD {
+			now := int64(monotime.Now())
+			if atomic.CompareAndSwapInt64(&sshClient.sshServer.lastAuthLog, int64(lastAuthLog), now) {
+				count := atomic.SwapInt64(&sshClient.sshServer.authFailedCount, 0)
+				log.WithContextFields(
+					LogFields{"lastError": err, "failedCount": count}).Warning("authentication failures")
+			}
+		}
 
-		log.WithContextFields(LogFields{"error": err, "method": method}).Warning("authentication failed")
+		log.WithContextFields(LogFields{"error": err, "method": method}).Debug("authentication failed")
 
 	} else {
 
@@ -1911,6 +1929,7 @@ func (sshClient *sshClient) handleTCPChannel(
 	established = true
 	sshClient.establishedPortForward(portForwardTypeTCP)
 
+	// TODO: 64-bit alignment? https://golang.org/pkg/sync/atomic/#pkg-note-BUG
 	var bytesUp, bytesDown int64
 	defer func() {
 		sshClient.closedPortForward(

psiphon/tlsDialer.go

@@ -118,7 +118,7 @@ type CustomTLSConfig struct {
 	VerifyLegacyCertificate *x509.Certificate
 
 	// UseIndistinguishableTLS specifies whether to try to use an
-	// alternative stack for TLS. From a circumvention perspective,
+	// alternative profile for TLS dials. From a circumvention perspective,
 	// Go's TLS has a distinct fingerprint that may be used for blocking.
 	UseIndistinguishableTLS bool
 

psiphon/tunnel.go

@@ -605,10 +605,8 @@ func dialSsh(
 	var selectedSSHClientVersion bool
 	SSHClientVersion := ""
 	useObfuscatedSsh := false
-	dialCustomHeaders := config.CustomHeaders
 	var directTCPDialAddress string
 	var meekConfig *MeekConfig
-	var selectedUserAgent bool
 	var err error
 
 	switch selectedProtocol {
@@ -629,7 +627,23 @@ func dialSsh(
 		}
 	}
 
-	dialCustomHeaders, selectedUserAgent = UserAgentIfUnset(config.CustomHeaders)
+	// Set User Agent when using meek or an upstream HTTP proxy
+
+	var selectedUserAgent bool
+	dialCustomHeaders := config.CustomHeaders
+	var upstreamProxyType string
+
+	if config.UpstreamProxyUrl != "" {
+		// Note: UpstreamProxyUrl will be validated in the dial
+		proxyURL, err := url.Parse(config.UpstreamProxyUrl)
+		if err == nil {
+			upstreamProxyType = proxyURL.Scheme
+		}
+	}
+
+	if meekConfig != nil || upstreamProxyType == "http" {
+		dialCustomHeaders, selectedUserAgent = UserAgentIfUnset(config.CustomHeaders)
+	}
 
 	// Use an asynchronous callback to record the resolved IP address when
 	// dialing a domain name. Note that DialMeek doesn't immediately
@@ -671,14 +685,8 @@ func dialSsh(
 		dialStats.UserAgent = dialConfig.CustomHeaders.Get("User-Agent")
 	}
 
-	if dialConfig.UpstreamProxyUrl != "" {
-
-		// Note: UpstreamProxyUrl will be validated in the dial
-		proxyURL, err := url.Parse(dialConfig.UpstreamProxyUrl)
-		if err == nil {
-			dialStats.UpstreamProxyType = proxyURL.Scheme
-		}
-
+	if upstreamProxyType != "" {
+		dialStats.UpstreamProxyType = upstreamProxyType
 		dialStats.UpstreamProxyCustomHeaderNames = make([]string, 0)
 		for name, _ := range dialConfig.CustomHeaders {
 			if selectedUserAgent && name == "User-Agent" {