Merge remote-tracking branch 'upstream/master'

psiphon/dataStore.go

@@ -276,6 +276,7 @@ func PromoteServerEntry(ipAddress string) error {
 type ServerEntryIterator struct {
 	region                      string
 	protocol                    string
+	shuffleHeadLength           int
 	transaction                 *sql.Tx
 	cursor                      *sql.Rows
 	isTargetServerEntryIterator bool
@@ -295,6 +296,7 @@ func NewServerEntryIterator(config *Config) (iterator *ServerEntryIterator, err
 	iterator = &ServerEntryIterator{
 		region:                      config.EgressRegion,
 		protocol:                    config.TunnelProtocol,
+		shuffleHeadLength:           config.TunnelPoolSize,
 		isTargetServerEntryIterator: false,
 	}
 	err = iterator.Reset()
@@ -349,13 +351,13 @@ func (iterator *ServerEntryIterator) Reset() error {
 	var cursor *sql.Rows
 
 	// This query implements the Psiphon server candidate selection
-	// algorithm: the first set of server candidates are in rank (priority)
-	// order, to favor previously successful servers; then the remaining
-	// long tail is shuffled to raise up less recent candidates.
+	// algorithm: the first TunnelPoolSize server candidates are in rank
+	// (priority) order, to favor previously successful servers; then the
+	// remaining long tail is shuffled to raise up less recent candidates.
 
 	whereClause, whereParams := makeServerEntryWhereClause(
 		iterator.region, iterator.protocol, nil)
-	headLength := CONNECTION_WORKER_POOL_SIZE
+	headLength := iterator.shuffleHeadLength
 	queryFormat := `
 		select data from serverEntry %s
 		order by case

psiphon/defaults.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, Psiphon Inc.
+ * Copyright (c) 2015, Psiphon Inc.
  * All rights reserved.
  *
  * This program is free software: you can redistribute it and/or modify
@@ -24,26 +24,29 @@ import (
 )
 
 const (
-	VERSION                                    = "0.0.6"
-	DATA_STORE_FILENAME                        = "psiphon.db"
-	CONNECTION_WORKER_POOL_SIZE                = 10
-	TUNNEL_POOL_SIZE                           = 1
-	TUNNEL_CONNECT_TIMEOUT                     = 15 * time.Second
-	TUNNEL_READ_TIMEOUT                        = 0 * time.Second
-	TUNNEL_WRITE_TIMEOUT                       = 5 * time.Second
-	TUNNEL_SSH_KEEP_ALIVE_PERIOD               = 60 * time.Second
-	ESTABLISH_TUNNEL_TIMEOUT                   = 60 * time.Second
-	ESTABLISH_TUNNEL_PAUSE_PERIOD              = 10 * time.Second
-	PORT_FORWARD_FAILURE_THRESHOLD             = 10
-	HTTP_PROXY_ORIGIN_SERVER_TIMEOUT           = 15 * time.Second
-	HTTP_PROXY_MAX_IDLE_CONNECTIONS_PER_HOST   = 50
-	FETCH_REMOTE_SERVER_LIST_TIMEOUT           = 10 * time.Second
-	FETCH_REMOTE_SERVER_LIST_RETRY_PERIOD      = 5 * time.Second
-	FETCH_REMOTE_SERVER_LIST_STALE_PERIOD      = 6 * time.Hour
-	PSIPHON_API_CLIENT_SESSION_ID_LENGTH       = 16
-	PSIPHON_API_SERVER_TIMEOUT                 = 20 * time.Second
-	PSIPHON_API_STATUS_REQUEST_PERIOD_MIN      = 5 * time.Minute
-	PSIPHON_API_STATUS_REQUEST_PERIOD_MAX      = 10 * time.Minute
-	PSIPHON_API_CONNECTED_REQUEST_PERIOD       = 24 * time.Hour
-	PSIPHON_API_CONNECTED_REQUEST_RETRY_PERIOD = 5 * time.Second
+	VERSION                                      = "0.0.6"
+	DATA_STORE_FILENAME                          = "psiphon.db"
+	CONNECTION_WORKER_POOL_SIZE                  = 10
+	TUNNEL_POOL_SIZE                             = 1
+	TUNNEL_CONNECT_TIMEOUT                       = 15 * time.Second
+	TUNNEL_READ_TIMEOUT                          = 0 * time.Second
+	TUNNEL_WRITE_TIMEOUT                         = 5 * time.Second
+	TUNNEL_SSH_KEEP_ALIVE_PAYLOAD_MAX_BYTES      = 256
+	TUNNEL_SSH_KEEP_ALIVE_PERIOD_MIN             = 60 * time.Second
+	TUNNEL_SSH_KEEP_ALIVE_PERIOD_MAX             = 120 * time.Second
+	ESTABLISH_TUNNEL_TIMEOUT                     = 60 * time.Second
+	ESTABLISH_TUNNEL_PAUSE_PERIOD                = 10 * time.Second
+	PORT_FORWARD_FAILURE_THRESHOLD               = 10
+	HTTP_PROXY_ORIGIN_SERVER_TIMEOUT             = 15 * time.Second
+	HTTP_PROXY_MAX_IDLE_CONNECTIONS_PER_HOST     = 50
+	FETCH_REMOTE_SERVER_LIST_TIMEOUT             = 10 * time.Second
+	FETCH_REMOTE_SERVER_LIST_RETRY_PERIOD        = 5 * time.Second
+	FETCH_REMOTE_SERVER_LIST_STALE_PERIOD        = 6 * time.Hour
+	PSIPHON_API_CLIENT_SESSION_ID_LENGTH         = 16
+	PSIPHON_API_SERVER_TIMEOUT                   = 20 * time.Second
+	PSIPHON_API_STATUS_REQUEST_PERIOD_MIN        = 5 * time.Minute
+	PSIPHON_API_STATUS_REQUEST_PERIOD_MAX        = 10 * time.Minute
+	PSIPHON_API_STATUS_REQUEST_PADDING_MAX_BYTES = 256
+	PSIPHON_API_CONNECTED_REQUEST_PERIOD         = 24 * time.Hour
+	PSIPHON_API_CONNECTED_REQUEST_RETRY_PERIOD   = 5 * time.Second
 )

psiphon/notice.go

@@ -64,7 +64,7 @@ func outputNotice(noticeType string, showUser bool, args ...interface{}) {
 	obj["showUser"] = showUser
 	obj["data"] = noticeData
 	obj["timestamp"] = time.Now().UTC().Format(time.RFC3339)
-	for i := 0; i < len(args)/2; i++ {
+	for i := 0; i < len(args)-1; i += 2 {
 		name, ok := args[i].(string)
 		value := args[i+1]
 		if ok {

psiphon/serverApi.go

@@ -21,6 +21,7 @@ package psiphon
 
 import (
 	"bytes"
+	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
@@ -30,7 +31,6 @@ import (
 	"net"
 	"net/http"
 	"strconv"
-	"time"
 )
 
 // Session is a utility struct which holds all of the data associated
@@ -132,23 +132,6 @@ func (session *Session) StatsRegexps() *Regexps {
 	return session.statsRegexps
 }
 
-// NextStatusRequestPeriod returns the amount of time that should be waited before the
-// next time stats are sent. The next wait time is picked at random, from a range,
-// to make the stats send less fingerprintable.
-func NextStatusRequestPeriod() (duration time.Duration) {
-	jitter, err := MakeSecureRandomInt64(
-		PSIPHON_API_STATUS_REQUEST_PERIOD_MAX.Nanoseconds() -
-			PSIPHON_API_STATUS_REQUEST_PERIOD_MIN.Nanoseconds())
-
-	// In case of error we're just going to use zero jitter.
-	if err != nil {
-		NoticeAlert("NextStatusRequestPeriod: make jitter failed")
-	}
-
-	duration = PSIPHON_API_STATUS_REQUEST_PERIOD_MIN + time.Duration(jitter)
-	return
-}
-
 // DoStatusRequest makes a /status request to the server, sending session stats.
 func (session *Session) DoStatusRequest(statsPayload json.Marshaler) error {
 	statsPayloadJSON, err := json.Marshal(statsPayload)
@@ -156,13 +139,20 @@ func (session *Session) DoStatusRequest(statsPayload json.Marshaler) error {
 		return ContextError(err)
 	}
 
+	// Add a random amount of padding to help prevent stats updates from being
+	// a predictable size (which often happens when the connection is quiet).
+	padding := MakeSecureRandomPadding(0, PSIPHON_API_STATUS_REQUEST_PADDING_MAX_BYTES)
+
 	// "connected" is a legacy parameter. This client does not report when
 	// it has disconnected.
 
 	url := session.buildRequestUrl(
 		"status",
 		&ExtraParam{"session_id", session.sessionId},
-		&ExtraParam{"connected", "1"})
+		&ExtraParam{"connected", "1"},
+		// TODO: base64 encoding of padding means the padding
+		// size is not exactly [0, PADDING_MAX_BYTES]
+		&ExtraParam{"padding", base64.StdEncoding.EncodeToString(padding)})
 
 	err = session.doPostRequest(url, "application/json", bytes.NewReader(statsPayloadJSON))
 	if err != nil {

psiphon/serverEntry.go

@@ -96,15 +96,21 @@ func ValidateServerEntry(serverEntry *ServerEntry) error {
 func DecodeAndValidateServerEntryList(encodedServerEntryList string) (serverEntries []*ServerEntry, err error) {
 	serverEntries = make([]*ServerEntry, 0)
 	for _, encodedServerEntry := range strings.Split(encodedServerEntryList, "\n") {
+		if len(encodedServerEntry) == 0 {
+			continue
+		}
+
 		// TODO: skip this entry and continue if can't decode?
 		serverEntry, err := DecodeServerEntry(encodedServerEntry)
 		if err != nil {
 			return nil, ContextError(err)
 		}
+
 		if ValidateServerEntry(serverEntry) != nil {
 			// Skip this entry and continue with the next one
 			continue
 		}
+
 		serverEntries = append(serverEntries, serverEntry)
 	}
 	return serverEntries, nil

psiphon/stats_collector.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, Psiphon Inc.
+ * Copyright (c) 2015, Psiphon Inc.
  * All rights reserved.
  *
  * This program is free software: you can redistribute it and/or modify
@@ -20,7 +20,6 @@
 package psiphon
 
 import (
-	"encoding/base64"
 	"encoding/json"
 	"sync"
 )
@@ -105,23 +104,6 @@ func recordStat(stat *statsUpdate) {
 func (ss serverStats) MarshalJSON() ([]byte, error) {
 	out := make(map[string]interface{})
 
-	// Add a random amount of padding to help prevent stats updates from being
-	// a predictable size (which often happens when the connection is quiet).
-	var padding []byte
-	paddingSize, err := MakeSecureRandomInt(256)
-	// In case of randomness fail, we're going to proceed with zero padding.
-	// TODO: Is this okay?
-	if err != nil {
-		NoticeAlert("stats.serverStats.MarshalJSON: MakeSecureRandomInt failed")
-		padding = make([]byte, 0)
-	} else {
-		padding, err = MakeSecureRandomBytes(paddingSize)
-		if err != nil {
-			NoticeAlert("stats.serverStats.MarshalJSON: MakeSecureRandomBytes failed")
-			padding = make([]byte, 0)
-		}
-	}
-
 	hostBytes := make(map[string]int64)
 	bytesTransferred := int64(0)
 
@@ -134,12 +116,9 @@ func (ss serverStats) MarshalJSON() ([]byte, error) {
 	out["bytes_transferred"] = bytesTransferred
 	out["host_bytes"] = hostBytes
 
-	// Print the notice before adding the padding, since it's not interesting
 	noticeJSON, _ := json.Marshal(out)
 	NoticeInfo("sending stats: %s", noticeJSON)
 
-	out["padding"] = base64.StdEncoding.EncodeToString(padding)
-
 	// We're not using these fields, but the server requires them
 	out["page_views"] = make([]string, 0)
 	out["https_requests"] = make([]string, 0)

psiphon/stats_test.go

@@ -83,15 +83,6 @@ func makeStatsDialer(serverID string, regexps *Regexps) func(network, addr strin
 	}
 }
 
-// TODO: move out of stats test suite
-func (suite *StatsTestSuite) Test_NextStatusRequestPeriod() {
-	res1 := NextStatusRequestPeriod()
-	suite.True(res1 > time.Duration(0), "duration should not be zero")
-
-	res2 := NextStatusRequestPeriod()
-	suite.NotEqual(res1, res2, "duration should have randomness difference between calls")
-}
-
 func (suite *StatsTestSuite) Test_StatsConn() {
 	resp, err := suite.httpClient.Get("http://example.com/index.html")
 	suite.Nil(err, "basic HTTP requests should succeed")

psiphon/tunnel.go

@@ -438,23 +438,42 @@ func dialSsh(
 func (tunnel *Tunnel) operateTunnel(config *Config, tunnelOwner TunnelOwner) {
 	defer tunnel.operateWaitGroup.Done()
 
-	// Note: not using a Ticker since NextStatusRequestPeriod() is not a fixed time period
-	statsTimer := time.NewTimer(NextStatusRequestPeriod())
+	// The next status request and ssh keep alive times are picked at random,
+	// from a range, to make the resulting traffic less fingerprintable,
+	// especially when then tunnel is otherwise idle.
+	// Note: not using Tickers since these are not fixed time periods.
+
+	nextStatusRequestPeriod := func() time.Duration {
+		return MakeRandomPeriod(
+			PSIPHON_API_STATUS_REQUEST_PERIOD_MIN,
+			PSIPHON_API_STATUS_REQUEST_PERIOD_MAX)
+	}
+	nextSshKeepAlivePeriod := func() time.Duration {
+		return MakeRandomPeriod(
+			TUNNEL_SSH_KEEP_ALIVE_PERIOD_MIN,
+			TUNNEL_SSH_KEEP_ALIVE_PERIOD_MAX)
+	}
+
+	statsTimer := time.NewTimer(nextStatusRequestPeriod())
 	defer statsTimer.Stop()
 
-	sshKeepAliveTicker := time.NewTicker(TUNNEL_SSH_KEEP_ALIVE_PERIOD)
-	defer sshKeepAliveTicker.Stop()
+	sshKeepAliveTimer := time.NewTimer(nextSshKeepAlivePeriod())
+	defer sshKeepAliveTimer.Stop()
 
 	var err error
 	for err == nil {
 		select {
 		case <-statsTimer.C:
 			sendStats(tunnel)
-			statsTimer.Reset(NextStatusRequestPeriod())
+			statsTimer.Reset(nextStatusRequestPeriod())
 
-		case <-sshKeepAliveTicker.C:
-			_, _, err := tunnel.sshClient.SendRequest("keepalive@openssh.com", true, nil)
+		case <-sshKeepAliveTimer.C:
+			// Random padding to frustrate fingerprinting
+			_, _, err := tunnel.sshClient.SendRequest(
+				"keepalive@openssh.com", true,
+				MakeSecureRandomPadding(0, TUNNEL_SSH_KEEP_ALIVE_PAYLOAD_MAX_BYTES))
 			err = fmt.Errorf("ssh keep alive failed: %s", err)
+			sshKeepAliveTimer.Reset(nextSshKeepAlivePeriod())
 
 		case failures := <-tunnel.portForwardFailures:
 			// Note: no mutex on portForwardFailureTotal; only referenced here

psiphon/utils.go

@@ -33,6 +33,7 @@ import (
 	"runtime"
 	"strings"
 	"sync"
+	"time"
 )
 
 // Contains is a helper function that returns true
@@ -77,6 +78,38 @@ func MakeSecureRandomBytes(length int) ([]byte, error) {
 	return randomBytes, nil
 }
 
+// MakeSecureRandomPadding selects a random padding length in the indicated
+// range and returns a random byte array of the selected length.
+// In the unlikely case where an  underlying MakeRandom functions fails,
+// the padding is length 0.
+func MakeSecureRandomPadding(minLength, maxLength int) []byte {
+	var padding []byte
+	paddingSize, err := MakeSecureRandomInt(maxLength - minLength)
+	if err != nil {
+		NoticeAlert("MakeSecureRandomPadding: MakeSecureRandomInt failed")
+		return make([]byte, 0)
+	}
+	paddingSize += minLength
+	padding, err = MakeSecureRandomBytes(paddingSize)
+	if err != nil {
+		NoticeAlert("MakeSecureRandomPadding: MakeSecureRandomBytes failed")
+		return make([]byte, 0)
+	}
+	return padding
+}
+
+// MakeRandomPeriod returns a random duration, within a given range.
+// In the unlikely case where an  underlying MakeRandom functions fails,
+// the period is the minimum.
+func MakeRandomPeriod(min, max time.Duration) (duration time.Duration) {
+	period, err := MakeSecureRandomInt64(max.Nanoseconds() - min.Nanoseconds())
+	if err != nil {
+		NoticeAlert("NextRandomRangePeriod: MakeSecureRandomInt64 failed")
+	}
+	duration = min + time.Duration(period)
+	return
+}
+
 func DecodeCertificate(encodedCertificate string) (certificate *x509.Certificate, err error) {
 	derEncodedCertificate, err := base64.StdEncoding.DecodeString(encodedCertificate)
 	if err != nil {

psiphon/utils_test.go

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2014, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package psiphon
+
+import (
+	"testing"
+	"time"
+)
+
+func TestMakeRandomPeriod(t *testing.T) {
+	min := 1 * time.Nanosecond
+	max := 10000 * time.Nanosecond
+
+	res1 := MakeRandomPeriod(min, max)
+
+	if res1 < min {
+		t.Error("duration should not be less than min")
+	}
+
+	if res1 > max {
+		t.Error("duration should not be more than max")
+	}
+
+	res2 := MakeRandomPeriod(min, max)
+	if res1 == res2 {
+		t.Error("duration should have randomness difference between calls")
+	}
+}