Download OSLs complete

- Pave and unpack of OSL directory and files now
  integrates auth package, compression, and, where
  applicable, encryption.
- New end-to-end test in remoteServerList_test.go:
  creates server, paves OSLs, seeds client, runs
  client through disruptor proxy, client must complete
  OSL downloads with multiple resumes to discover
  server entry and successfully connect.
- Added support for secret split threshold = 1.
- Added plain HTTP support in MakeDownloadHttpClient
  for testing.

psiphon/common/osl/osl.go

@@ -30,6 +30,8 @@
 package osl
 
 import (
+	"bytes"
+	"compress/zlib"
 	"crypto/hmac"
 	"crypto/sha256"
 	"encoding/base64"
@@ -38,6 +40,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"net/url"
 	"path"
@@ -237,7 +240,7 @@ func NewConfig(filename string) (*Config, error) {
 	config.ReloadableFile = common.NewReloadableFile(
 		filename,
 		func(fileContent []byte) error {
-			newConfig, err := loadConfig(fileContent)
+			newConfig, err := LoadConfig(fileContent)
 			if err != nil {
 				return common.ContextError(err)
 			}
@@ -254,9 +257,9 @@ func NewConfig(filename string) (*Config, error) {
 	return config, nil
 }
 
-// loadConfig loads, vaildates, and initializes a JSON encoded OSL
+// LoadConfig loads, vaildates, and initializes a JSON encoded OSL
 // configuration.
-func loadConfig(configJSON []byte) (*Config, error) {
+func LoadConfig(configJSON []byte) (*Config, error) {
 
 	var config Config
 	err := json.Unmarshal(configJSON, &config)
@@ -689,14 +692,14 @@ func (config *Config) Pave(
 	propagationChannelID string,
 	signingPublicKey string,
 	signingPrivateKey string,
-	paveServerEntries []map[time.Time][]byte) ([]*PaveFile, error) {
+	paveServerEntries []map[time.Time]string) ([]*PaveFile, error) {
 
 	config.ReloadableFile.RLock()
 	defer config.ReloadableFile.RUnlock()
 
 	var paveFiles []*PaveFile
 
-	Directory := &Directory{}
+	directory := &Directory{}
 
 	if len(paveServerEntries) != len(config.Schemes) {
 		return nil, common.ContextError(errors.New("invalid paveServerEntries"))
@@ -711,7 +714,7 @@ func (config *Config) Pave(
 
 		if common.Contains(scheme.PropagationChannelIDs, propagationChannelID) {
 			oslTime := scheme.epoch
-			for oslTime.Before(endTime) {
+			for !oslTime.After(endTime) {
 
 				firstSLOKTime := oslTime
 				fileKey, fileSpec, err := makeOSLFileSpec(
@@ -720,11 +723,20 @@ func (config *Config) Pave(
 					return nil, common.ContextError(err)
 				}
 
-				Directory.FileSpecs = append(Directory.FileSpecs, fileSpec)
+				directory.FileSpecs = append(directory.FileSpecs, fileSpec)
 
 				serverEntries, ok := paveServerEntries[schemeIndex][oslTime]
 				if ok {
-					boxedServerEntries, err := box(fileKey, serverEntries)
+
+					signedServerEntries, err := common.WriteAuthenticatedDataPackage(
+						serverEntries,
+						signingPublicKey,
+						signingPrivateKey)
+					if err != nil {
+						return nil, common.ContextError(err)
+					}
+
+					boxedServerEntries, err := box(fileKey, compress(signedServerEntries))
 					if err != nil {
 						return nil, common.ContextError(err)
 					}
@@ -745,13 +757,13 @@ func (config *Config) Pave(
 		}
 	}
 
-	jsonDirectory, err := json.Marshal(Directory)
+	directoryJSON, err := json.Marshal(directory)
 	if err != nil {
 		return nil, common.ContextError(err)
 	}
 
 	signedDirectory, err := common.WriteAuthenticatedDataPackage(
-		base64.StdEncoding.EncodeToString(jsonDirectory),
+		base64.StdEncoding.EncodeToString(directoryJSON),
 		signingPublicKey,
 		signingPrivateKey)
 	if err != nil {
@@ -760,7 +772,7 @@ func (config *Config) Pave(
 
 	paveFiles = append(paveFiles, &PaveFile{
 		Name:     DIRECTORY_FILENAME,
-		Contents: signedDirectory,
+		Contents: compress(signedDirectory),
 	})
 
 	return paveFiles, nil
@@ -953,23 +965,36 @@ func GetOSLFilename(baseDirectory string, oslID []byte) string {
 		baseDirectory, fmt.Sprintf(OSL_FILENAME_FORMAT, hex.EncodeToString(oslID)))
 }
 
-// LoadDirectory authenticates the signed directory package -- which is the
-// contents of the paved directory file. It then returns the directory data.
-// Clients call this to process downloaded directory files.
-func LoadDirectory(directoryPackage []byte, signingPublicKey string) (*Directory, error) {
+// UnpackDirectory decompresses, validates, and loads a
+// JSON encoded OSL directory.
+func UnpackDirectory(
+	compressedDirectory []byte, signingPublicKey string) (*Directory, []byte, error) {
+
+	directoryPackage, err := uncompress(compressedDirectory)
+	if err != nil {
+		return nil, nil, common.ContextError(err)
+	}
 
 	encodedDirectory, err := common.ReadAuthenticatedDataPackage(directoryPackage, signingPublicKey)
 	if err != nil {
-		return nil, common.ContextError(err)
+		return nil, nil, common.ContextError(err)
 	}
 
 	directoryJSON, err := base64.StdEncoding.DecodeString(encodedDirectory)
 	if err != nil {
-		return nil, common.ContextError(err)
+		return nil, nil, common.ContextError(err)
 	}
 
+	directory, err := LoadDirectory(directoryJSON)
+	return directory, directoryJSON, err
+}
+
+// LoadDirectory loads a JSON encoded OSL directory.
+// Clients call this to process downloaded directory files.
+func LoadDirectory(directoryJSON []byte) (*Directory, error) {
+
 	var directory Directory
-	err = json.Unmarshal(directoryJSON, &directory)
+	err := json.Unmarshal(directoryJSON, &directory)
 	if err != nil {
 		return nil, common.ContextError(err)
 	}
@@ -1085,29 +1110,48 @@ func (keyShares *KeyShares) reassembleKey(lookup SLOKLookup, unboxKey bool) (boo
 	return true, joinedKey, nil
 }
 
-// DecryptOSL reassembles the key for the OSL specified by oslID and uses
-// that key to decrypt oslFileContents. Clients will call DecryptOSL for
-// OSLs indicated by GetSeededOSLIDs along with their downloaded content.
+// UnpackOSL reassembles the key for the OSL specified by oslID and uses
+// that key to decrypt oslFileContents, uncompress the contents, validate
+// the authenticated package, and extract the payload.
+// Clients will call UnpackOSL for OSLs indicated by GetSeededOSLIDs along
+// with their downloaded content.
 // SLOKLookup is called to determine which SLOKs are seeded with the client.
-func (directory *Directory) DecryptOSL(
-	lookup SLOKLookup, oslID []byte, oslFileContents []byte) ([]byte, error) {
+func (directory *Directory) UnpackOSL(
+	lookup SLOKLookup,
+	oslID []byte,
+	oslFileContents []byte,
+	signingPublicKey string) (string, error) {
 
 	fileSpec, ok := directory.oslIDLookup[string(oslID)]
 	if !ok {
-		return nil, common.ContextError(errors.New("unknown OSL ID"))
+		return "", common.ContextError(errors.New("unknown OSL ID"))
 	}
+
 	ok, fileKey, err := fileSpec.KeyShares.reassembleKey(lookup, true)
 	if err != nil {
-		return nil, common.ContextError(err)
+		return "", common.ContextError(err)
 	}
 	if !ok {
-		return nil, common.ContextError(errors.New("unseeded OSL"))
+		return "", common.ContextError(errors.New("unseeded OSL"))
 	}
-	decryptedOSLFileContents, err := unbox(fileKey, oslFileContents)
+
+	decryptedContents, err := unbox(fileKey, oslFileContents)
 	if err != nil {
-		return nil, common.ContextError(err)
+		return "", common.ContextError(err)
 	}
-	return decryptedOSLFileContents, nil
+
+	dataPackage, err := uncompress(decryptedContents)
+	if err != nil {
+		return "", common.ContextError(err)
+	}
+
+	oslPayload, err := common.ReadAuthenticatedDataPackage(
+		dataPackage, signingPublicKey)
+	if err != nil {
+		return "", common.ContextError(err)
+	}
+
+	return oslPayload, nil
 }
 
 // deriveKeyHKDF implements HKDF-Expand as defined in https://tools.ietf.org/html/rfc5869
@@ -1123,7 +1167,7 @@ func deriveKeyHKDF(masterKey []byte, context ...[]byte) []byte {
 
 // isValidShamirSplit checks sss.Split constraints
 func isValidShamirSplit(total, threshold int) bool {
-	if total < 2 || total > 254 || threshold < 2 || threshold > total {
+	if total < 1 || total > 254 || threshold < 1 || threshold > total {
 		return false
 	}
 	return true
@@ -1135,6 +1179,15 @@ func shamirSplit(secret []byte, total, threshold int) ([][]byte, error) {
 		return nil, common.ContextError(errors.New("invalid parameters"))
 	}
 
+	if threshold == 1 {
+		// Special case: each share is simply the secret
+		shares := make([][]byte, total)
+		for i := 0; i < total; i++ {
+			shares[i] = secret
+		}
+		return shares, nil
+	}
+
 	shareMap, err := sss.Split(byte(total), byte(threshold), secret)
 	if err != nil {
 		return nil, common.ContextError(err)
@@ -1152,6 +1205,11 @@ func shamirSplit(secret []byte, total, threshold int) ([][]byte, error) {
 // shamirCombine is a helper wrapper for sss.Combine
 func shamirCombine(shares [][]byte) []byte {
 
+	if len(shares) == 1 {
+		// Special case: each share is simply the secret
+		return shares[0]
+	}
+
 	// Convert a sparse list into a map
 	shareMap := make(map[byte][]byte)
 	for index, share := range shares {
@@ -1192,3 +1250,24 @@ func unbox(key, box []byte) ([]byte, error) {
 	}
 	return plaintext, nil
 }
+
+func compress(data []byte) []byte {
+	var compressedData bytes.Buffer
+	writer := zlib.NewWriter(&compressedData)
+	writer.Write(data)
+	writer.Close()
+	return compressedData.Bytes()
+}
+
+func uncompress(data []byte) ([]byte, error) {
+	reader, err := zlib.NewReader(bytes.NewReader(data))
+	if err != nil {
+		return nil, common.ContextError(err)
+	}
+	uncompressedData, err := ioutil.ReadAll(reader)
+	reader.Close()
+	if err != nil {
+		return nil, common.ContextError(err)
+	}
+	return uncompressedData, nil
+}

psiphon/common/osl/osl_test.go

@@ -20,7 +20,7 @@
 package osl
 
 import (
-	"bytes"
+	"encoding/base64"
 	"fmt"
 	"net"
 	"testing"
@@ -162,7 +162,7 @@ func TestOSL(t *testing.T) {
 	// periods and 5/10 10 millisecond longer periods. The second scheme requires
 	// sufficient activity within 25/100 1 millisecond periods.
 
-	config, err := loadConfig([]byte(configJSON))
+	config, err := LoadConfig([]byte(configJSON))
 	if err != nil {
 		t.Fatalf("LoadConfig failed: %s", err)
 	}
@@ -317,10 +317,10 @@ func TestOSL(t *testing.T) {
 
 			// Dummy server entry payloads will be the OSL ID, which the following
 			// tests use to verify that the correct OSL file decrypts successfully.
-			paveServerEntries := make([]map[time.Time][]byte, len(config.Schemes))
+			paveServerEntries := make([]map[time.Time]string, len(config.Schemes))
 			for schemeIndex, scheme := range config.Schemes {
 
-				paveServerEntries[schemeIndex] = make(map[time.Time][]byte)
+				paveServerEntries[schemeIndex] = make(map[time.Time]string)
 
 				slokTimePeriodsPerOSL := 1
 				for _, keySplit := range scheme.SeedPeriodKeySplits {
@@ -336,7 +336,8 @@ func TestOSL(t *testing.T) {
 					}
 					firstSLOK := deriveSLOK(scheme, firstSLOKRef)
 					oslID := firstSLOK.ID
-					paveServerEntries[schemeIndex][oslTime] = oslID
+					paveServerEntries[schemeIndex][oslTime] =
+						base64.StdEncoding.EncodeToString(oslID)
 
 					oslTime = oslTime.Add(
 						time.Duration(
@@ -355,7 +356,7 @@ func TestOSL(t *testing.T) {
 			}
 
 			// Check that the paved file name matches the name the client will look for.
-			if len(paveFiles) < 1 || paveFiles[len(paveFiles)-1].Name != GetDirectoryURL("") {
+			if len(paveFiles) < 1 || paveFiles[len(paveFiles)-1].Name != GetOSLDirectoryURL("") {
 				t.Fatalf("invalid directory pave file")
 			}
 
@@ -511,7 +512,7 @@ func TestOSL(t *testing.T) {
 
 			checkDirectoryStartTime := time.Now()
 
-			directory, err := LoadDirectory(
+			directory, _, err := UnpackDirectory(
 				pavedDirectories[testCase.propagationChannelID], signingPublicKey)
 			if err != nil {
 				t.Fatalf("LoadDirectory failed: %s", err)
@@ -539,13 +540,14 @@ func TestOSL(t *testing.T) {
 					t.Fatalf("unknown OSL file name")
 				}
 
-				plaintextOSL, err := directory.DecryptOSL(slokLookup, oslID, oslFileContents)
+				plaintextOSL, err := directory.UnpackOSL(
+					slokLookup, oslID, oslFileContents, signingPublicKey)
 				if err != nil {
 					t.Fatalf("DecryptOSL failed: %s", err)
 				}
 
 				// The decrypted OSL should contain its own ID.
-				if bytes.Compare(plaintextOSL, oslID) != 0 {
+				if plaintextOSL != base64.StdEncoding.EncodeToString(oslID) {
 					t.Fatalf("unexpected OSL file contents")
 				}
 			}

psiphon/net.go

@@ -241,10 +241,14 @@ func ResolveIP(host string, conn net.Conn) (addrs []net.IP, ttls []time.Duration
 // UseIndistinguishableTLS, etc. -- for a specific HTTPS request URL.
 // If verifyLegacyCertificate is not nil, it's used for certificate
 // verification.
+//
 // Because UseIndistinguishableTLS requires a hack to work with
 // net/http, MakeUntunneledHttpClient may return a modified request URL
 // to be used. Callers should always use this return value to make
 // requests, not the input value.
+//
+// MakeUntunneledHttpsClient ignores the input requestUrl scheme,
+// which may be "http" or "https", and always performs HTTPS requests.
 func MakeUntunneledHttpsClient(
 	dialConfig *DialConfig,
 	verifyLegacyCertificate *x509.Certificate,
@@ -352,16 +356,31 @@ func MakeDownloadHttpClient(
 	var err error
 
 	if tunnel != nil {
+		// MakeTunneledHttpClient works with both "http" and "https" schemes
 		httpClient, err = MakeTunneledHttpClient(config, tunnel, requestTimeout)
 		if err != nil {
 			return nil, "", common.ContextError(err)
 		}
 	} else {
-		httpClient, requestUrl, err = MakeUntunneledHttpsClient(
-			untunneledDialConfig, nil, requestUrl, requestTimeout)
+		urlComponents, err := url.Parse(requestUrl)
 		if err != nil {
 			return nil, "", common.ContextError(err)
 		}
+		// MakeUntunneledHttpsClient works only with "https" schemes
+		if urlComponents.Scheme == "https" {
+			httpClient, requestUrl, err = MakeUntunneledHttpsClient(
+				untunneledDialConfig, nil, requestUrl, requestTimeout)
+			if err != nil {
+				return nil, "", common.ContextError(err)
+			}
+		} else {
+			httpClient = &http.Client{
+				Timeout: requestTimeout,
+				Transport: &http.Transport{
+					Dial: NewTCPDialer(untunneledDialConfig),
+				},
+			}
+		}
 	}
 
 	return httpClient, requestUrl, nil

psiphon/remoteServerList.go

@@ -21,6 +21,7 @@ package psiphon
 
 import (
 	"compress/zlib"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -113,7 +114,7 @@ func FetchObfuscatedServerLists(
 	// TODO: should disk-full conditions not trigger retries?
 	var failed bool
 
-	var oslDirectoryPayload string
+	var oslDirectory *osl.Directory
 
 	newETag, err := downloadRemoteServerListFile(
 		config,
@@ -125,31 +126,46 @@ func FetchObfuscatedServerLists(
 		failed = true
 		NoticeAlert("failed to download obfuscated server list directory: %s", common.ContextError(err))
 	} else if newETag != "" {
-		oslDirectoryPayload, err = unpackRemoteServerListFile(config, downloadFilename)
+
+		fileContent, err := ioutil.ReadFile(downloadFilename)
 		if err != nil {
 			failed = true
-			NoticeAlert("failed to unpack obfuscated server list directory: %s", common.ContextError(err))
+			NoticeAlert("failed to read obfuscated server list directory: %s", common.ContextError(err))
 		}
-		err = SetKeyValue(DATA_STORE_OSL_DIRECTORY_KEY, string(oslDirectoryPayload))
-		if err != nil {
-			failed = true
-			NoticeAlert("failed to set cached obfuscated server list directory: %s", common.ContextError(err))
+
+		var oslDirectoryJSON []byte
+		if err == nil {
+			oslDirectory, oslDirectoryJSON, err = osl.UnpackDirectory(
+				fileContent, config.RemoteServerListSignaturePublicKey)
+			if err != nil {
+				failed = true
+				NoticeAlert("failed to unpack obfuscated server list directory: %s", common.ContextError(err))
+			}
+		}
+
+		if err == nil {
+			err = SetKeyValue(DATA_STORE_OSL_DIRECTORY_KEY, string(oslDirectoryJSON))
+			if err != nil {
+				failed = true
+				NoticeAlert("failed to set cached obfuscated server list directory: %s", common.ContextError(err))
+			}
 		}
 	}
 
 	if failed || newETag == "" {
 		// Proceed with the cached OSL directory.
-		oslDirectoryPayload, err = GetKeyValue(DATA_STORE_OSL_DIRECTORY_KEY)
+		oslDirectoryJSON, err := GetKeyValue(DATA_STORE_OSL_DIRECTORY_KEY)
+		if err == nil && oslDirectoryJSON == "" {
+			err = errors.New("not found")
+		}
 		if err != nil {
-			return fmt.Errorf("failed to get cache obfuscated server list directory: %s", common.ContextError(err))
+			return fmt.Errorf("failed to get cached obfuscated server list directory: %s", common.ContextError(err))
 		}
-	}
-
-	// *TODO* fix double authenticated package unwrapping: make LoadDirectory take JSON string
 
-	oslDirectory, err := osl.LoadDirectory([]byte(oslDirectoryPayload), config.RemoteServerListSignaturePublicKey)
-	if err != nil {
-		return fmt.Errorf("failed to load obfuscated server list directory: %s", common.ContextError(err))
+		oslDirectory, err = osl.LoadDirectory([]byte(oslDirectoryJSON))
+		if err != nil {
+			return fmt.Errorf("failed to load obfuscated server list directory: %s", common.ContextError(err))
+		}
 	}
 
 	// When a new directory is downloaded, validated, and parsed, store the
@@ -165,17 +181,17 @@ func FetchObfuscatedServerLists(
 	// Note: we proceed to check individual OSLs even if the direcory is unchanged,
 	// as the set of local SLOKs may have changed.
 
-	oslIDs := oslDirectory.GetSeededOSLIDs(
-
+	lookupSLOKs := func(slokID []byte) []byte {
 		// Lookup SLOKs in local datastore
-		func(slokID []byte) []byte {
-			key, err := GetSLOK(slokID)
-			if err != nil {
-				NoticeAlert("GetSLOK failed: %s", err)
-			}
-			return key
-		},
+		key, err := GetSLOK(slokID)
+		if err != nil {
+			NoticeAlert("GetSLOK failed: %s", err)
+		}
+		return key
+	}
 
+	oslIDs := oslDirectory.GetSeededOSLIDs(
+		lookupSLOKs,
 		func(err error) {
 			NoticeAlert("GetSeededOSLIDs failed: %s", err)
 		})
@@ -183,8 +199,9 @@ func FetchObfuscatedServerLists(
 	for _, oslID := range oslIDs {
 		downloadFilename := osl.GetOSLFilename(config.ObfuscatedServerListDownloadDirectory, oslID)
 		downloadURL := osl.GetOSLFileURL(config.ObfuscatedServerListRootURL, oslID)
+		hexID := hex.EncodeToString(oslID)
 
-		// *TODO* ETags in OSL directory to enable skipping request entirely
+		// TODO: store ETags in OSL directory to enable skipping requests entirely
 
 		newETag, err := downloadRemoteServerListFile(
 			config,
@@ -194,7 +211,7 @@ func FetchObfuscatedServerLists(
 			downloadFilename)
 		if err != nil {
 			failed = true
-			NoticeAlert("failed to download obfuscated server list file (%s): %s", oslID, common.ContextError(err))
+			NoticeAlert("failed to download obfuscated server list file (%s): %s", hexID, common.ContextError(err))
 			continue
 		}
 
@@ -203,19 +220,25 @@ func FetchObfuscatedServerLists(
 			continue
 		}
 
-		// *TODO* DecryptOSL; also, compress before encrypt?
+		fileContent, err := ioutil.ReadFile(downloadFilename)
+		if err != nil {
+			failed = true
+			NoticeAlert("failed to read obfuscated server list file (%s): %s", hexID, common.ContextError(err))
+			continue
+		}
 
-		serverListPayload, err := unpackRemoteServerListFile(config, downloadFilename)
+		serverListPayload, err := oslDirectory.UnpackOSL(
+			lookupSLOKs, oslID, fileContent, config.RemoteServerListSignaturePublicKey)
 		if err != nil {
 			failed = true
-			NoticeAlert("failed to unpack obfuscated server list file (%s): %s", oslID, common.ContextError(err))
+			NoticeAlert("failed to unpack obfuscated server list file (%s): %s", hexID, common.ContextError(err))
 			continue
 		}
 
 		err = storeServerEntries(serverListPayload)
 		if err != nil {
 			failed = true
-			NoticeAlert("failed to store obfuscated server list file (%s): %s", oslID, common.ContextError(err))
+			NoticeAlert("failed to store obfuscated server list file (%s): %s", hexID, common.ContextError(err))
 			continue
 		}
 
@@ -224,7 +247,7 @@ func FetchObfuscatedServerLists(
 		err = SetUrlETag(config.RemoteServerListUrl, newETag)
 		if err != nil {
 			failed = true
-			NoticeAlert("failed to set Etag for obfuscated server list file (%s): %s", oslID, common.ContextError(err))
+			NoticeAlert("failed to set Etag for obfuscated server list file (%s): %s", hexID, common.ContextError(err))
 			continue
 			// This fetch is still reported as a success, even if we can't store the etag
 		}

psiphon/remoteServerList_test.go

@@ -0,0 +1,349 @@
+/*
+ * Copyright (c) 2016, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package psiphon
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	socks "github.com/Psiphon-Inc/goptlib"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/osl"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/server"
+)
+
+// TODO: TestCommonRemoteServerList (this is currently covered by controller_test.go)
+
+func TestObfuscatedRemoteServerLists(t *testing.T) {
+
+	//
+	// create a server
+	//
+
+	var err error
+	serverIPaddress := ""
+	for _, interfaceName := range []string{"eth0", "en0"} {
+		serverIPaddress, err = GetInterfaceIPAddress(interfaceName)
+		if err == nil {
+			break
+		}
+	}
+	if err != nil {
+		t.Fatalf("error getting server IP address: %s", err)
+	}
+
+	serverConfigJSON, _, encodedServerEntry, err := server.GenerateConfig(
+		&server.GenerateConfigParams{
+			ServerIPAddress:      serverIPaddress,
+			EnableSSHAPIRequests: true,
+			WebServerPort:        8000,
+			TunnelProtocolPorts:  map[string]int{"OSSH": 4000},
+		})
+	if err != nil {
+		t.Fatalf("error generating server config: %s", err)
+	}
+
+	//
+	// pave OSLs
+	//
+
+	oslConfigJSONTemplate := `
+    {
+      "Schemes" : [
+        {
+          "Epoch" : "%s",
+          "Regions" : [],
+          "PropagationChannelIDs" : ["%s"],
+          "MasterKey" : "vwab2WY3eNyMBpyFVPtsivMxF4MOpNHM/T7rHJIXctg=",
+          "SeedSpecs" : [
+            {
+              "ID" : "KuP2V6gLcROIFzb/27fUVu4SxtEfm2omUoISlrWv1mA=",
+              "UpstreamSubnets" : ["0.0.0.0/0"],
+              "Targets" :
+              {
+                  "BytesRead" : 1,
+                  "BytesWritten" : 1,
+                  "PortForwardDurationNanoseconds" : 1
+              }
+            }
+          ],
+          "SeedSpecThreshold" : 1,
+          "SeedPeriodNanoseconds" : %d,
+          "SeedPeriodKeySplits": [
+            {
+              "Total": 1,
+              "Threshold": 1
+            }
+          ]
+        }
+      ]
+    }`
+
+	now := time.Now().UTC()
+	seedPeriod := 24 * time.Hour
+	epoch := now.Truncate(seedPeriod)
+	epochStr := epoch.Format(time.RFC3339Nano)
+
+	propagationChannelID, _ := common.MakeRandomStringHex(8)
+
+	oslConfigJSON := fmt.Sprintf(
+		oslConfigJSONTemplate,
+		epochStr,
+		propagationChannelID,
+		seedPeriod)
+
+	oslConfig, err := osl.LoadConfig([]byte(oslConfigJSON))
+	if err != nil {
+		t.Fatalf("error loading OSL config: %s", err)
+	}
+
+	signingPublicKey, signingPrivateKey, err := common.GenerateAuthenticatedDataPackageKeys()
+	if err != nil {
+		t.Fatalf("error generating package keys: %s", err)
+	}
+
+	paveFiles, err := oslConfig.Pave(
+		epoch,
+		propagationChannelID,
+		signingPublicKey,
+		signingPrivateKey,
+		[]map[time.Time]string{
+			map[time.Time]string{
+				epoch: string(encodedServerEntry),
+			},
+		})
+	if err != nil {
+		t.Fatalf("error paving OSL files: %s", err)
+	}
+
+	//
+	// mock seeding SLOKs
+	//
+
+	singleton.db = nil
+	os.Remove(DATA_STORE_FILENAME)
+
+	err = InitDataStore(&Config{})
+	if err != nil {
+		t.Fatalf("error initializing client datastore: %s", err)
+	}
+
+	seedState := oslConfig.NewClientSeedState("", propagationChannelID, nil)
+	seedPortForward := seedState.NewClientSeedPortForward(net.ParseIP("0.0.0.0"))
+	seedPortForward.UpdateProgress(1, 1, 1)
+	payload := seedState.GetSeedPayload()
+	if len(payload.SLOKs) != 1 {
+		t.Fatalf("expected 1 SLOKs, got %d", len(payload.SLOKs))
+	}
+
+	SetSLOK(payload.SLOKs[0].ID, payload.SLOKs[0].Key)
+
+	//
+	// run mock remote server list host
+	//
+
+	downloadRoot := "test-data"
+	os.MkdirAll(downloadRoot, 0700)
+
+	remoteServerListHostAddress := net.JoinHostPort(serverIPaddress, "8080")
+
+	// The common remote server list fetches will 404
+	remoteServerListURL := fmt.Sprintf("http://%s/server_list_compressed", remoteServerListHostAddress)
+	remoteServerListDownloadFilename := filepath.Join(downloadRoot, "server_list_compressed")
+
+	obfuscatedServerListRootURL := fmt.Sprintf("http://%s/", remoteServerListHostAddress)
+	obfuscatedServerListDownloadDirectory := downloadRoot
+
+	go func() {
+		startTime := time.Now()
+		serveMux := http.NewServeMux()
+		for _, paveFile := range paveFiles {
+			file := paveFile
+			serveMux.HandleFunc("/"+file.Name, func(w http.ResponseWriter, req *http.Request) {
+				md5sum := md5.Sum(file.Contents)
+				w.Header().Add("Content-Type", "application/octet-stream")
+				w.Header().Add("ETag", hex.EncodeToString(md5sum[:]))
+				http.ServeContent(w, req, file.Name, startTime, bytes.NewReader(file.Contents))
+			})
+		}
+		httpServer := &http.Server{
+			Addr:    remoteServerListHostAddress,
+			Handler: serveMux,
+		}
+		err := httpServer.ListenAndServe()
+		if err != nil {
+			// TODO: wrong goroutine for t.FatalNow()
+			t.Fatalf("error running remote server list host: %s", err)
+
+		}
+	}()
+
+	//
+	// run Psiphon server
+	//
+
+	go func() {
+		err := server.RunServices(serverConfigJSON)
+		if err != nil {
+			// TODO: wrong goroutine for t.FatalNow()
+			t.Fatalf("error running server: %s", err)
+		}
+	}()
+
+	//
+	// disrupt remote server list downloads
+	//
+
+	disruptorProxyAddress := "127.0.0.1:2162"
+	disruptorProxyURL := "socks4a://" + disruptorProxyAddress
+
+	go func() {
+		listener, err := socks.ListenSocks("tcp", disruptorProxyAddress)
+		if err != nil {
+			fmt.Errorf("disruptor proxy listen error: %s", err)
+			return
+		}
+		for {
+			localConn, err := listener.AcceptSocks()
+			if err != nil {
+				fmt.Errorf("disruptor proxy accept error: %s", err)
+				return
+			}
+			go func() {
+				remoteConn, err := net.Dial("tcp", localConn.Req.Target)
+				if err != nil {
+					fmt.Errorf("disruptor proxy dial error: %s", err)
+					return
+				}
+				err = localConn.Grant(&net.TCPAddr{IP: net.ParseIP("0.0.0.0"), Port: 0})
+				if err != nil {
+					fmt.Errorf("disruptor proxy grant error: %s", err)
+					return
+				}
+
+				waitGroup := new(sync.WaitGroup)
+				waitGroup.Add(1)
+				go func() {
+					defer waitGroup.Done()
+					io.Copy(remoteConn, localConn)
+				}()
+				if localConn.Req.Target == remoteServerListHostAddress {
+					io.CopyN(localConn, remoteConn, 500)
+				} else {
+					io.Copy(localConn, remoteConn)
+				}
+				localConn.Close()
+				remoteConn.Close()
+				waitGroup.Wait()
+			}()
+		}
+	}()
+
+	//
+	// connect to Psiphon server with Psiphon client
+	//
+
+	// Note: calling LoadConfig ensures all *int config fields are initialized
+	clientConfigJSONTemplate := `
+    {
+        "ClientPlatform" : "",
+        "ClientVersion" : "0",
+        "SponsorId" : "0",
+        "PropagationChannelId" : "0",
+        "ConnectionPoolSize" : 1,
+        "EstablishTunnelPausePeriodSeconds" : 1,
+        "FetchRemoteServerListRetryPeriodSeconds" : 1,
+		"RemoteServerListSignaturePublicKey" : "%s",
+		"RemoteServerListUrl" : "%s",
+		"RemoteServerListDownloadFilename" : "%s",
+		"ObfuscatedServerListRootURL" : "%s",
+		"ObfuscatedServerListDownloadDirectory" : "%s",
+		"UpstreamProxyUrl" : "%s"
+    }`
+
+	clientConfigJSON := fmt.Sprintf(
+		clientConfigJSONTemplate,
+		signingPublicKey,
+		remoteServerListURL,
+		remoteServerListDownloadFilename,
+		obfuscatedServerListRootURL,
+		obfuscatedServerListDownloadDirectory,
+		disruptorProxyURL)
+
+	clientConfig, _ := LoadConfig([]byte(clientConfigJSON))
+
+	controller, err := NewController(clientConfig)
+	if err != nil {
+		t.Fatalf("error creating client controller: %s", err)
+	}
+
+	tunnelEstablished := make(chan struct{}, 1)
+
+	SetNoticeOutput(NewNoticeReceiver(
+		func(notice []byte) {
+
+			noticeType, payload, err := GetNotice(notice)
+			if err != nil {
+				return
+			}
+
+			printNotice := false
+
+			switch noticeType {
+			case "Tunnels":
+				printNotice = true
+				count := int(payload["count"].(float64))
+				if count == 1 {
+					tunnelEstablished <- *new(struct{})
+				}
+			case "RemoteServerListResourceDownloadedBytes":
+				// TODO: check for resumed download for each URL
+				//url := payload["url"].(string)
+				printNotice = true
+			case "RemoteServerListResourceDownloaded":
+				printNotice = true
+			}
+
+			if printNotice {
+				fmt.Printf("%s\n", string(notice))
+			}
+		}))
+
+	go func() {
+		controller.Run(make(chan struct{}))
+	}()
+
+	establishTimeout := time.NewTimer(30 * time.Second)
+	select {
+	case <-tunnelEstablished:
+	case <-establishTimeout.C:
+		t.Fatalf("tunnel establish timeout exceeded")
+	}
+}

psiphon/server/tunnelServer.go

@@ -758,7 +758,7 @@ func (sshClient *sshClient) run(clientConn net.Conn) {
 
 	sshClient.runTunnel(result.channels, result.requests)
 
-	// Note: sshServer.unregisterEstablishedClient calls sshClient.Close(),
+	// Note: sshServer.unregisterEstablishedClient calls sshClient.stop(),
 	// which also closes underlying transport Conn.
 }