Merge pull request #430 from rod-hynes/master

OSL download optimizations

psiphon/common/osl/osl.go

@@ -30,6 +30,8 @@
 package osl
 
 import (
+	"crypto/aes"
+	"crypto/cipher"
 	"crypto/hmac"
 	"crypto/md5"
 	"crypto/sha256"
@@ -49,9 +51,9 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/Psiphon-Inc/sss"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/crypto/nacl/secretbox"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/sss"
 )
 
 const (
@@ -123,7 +125,6 @@ type Scheme struct {
 	// sufficiently seeded. And so on. The first level in the list is the
 	// lowest level. The time period for OSLs is determined by the totals in
 	// the KeySplits.
-	// Limitation: thresholds must be at least 2.
 	//
 	// Example:
 	//
@@ -276,7 +277,7 @@ func NewConfig(filename string) (*Config, error) {
 	return config, nil
 }
 
-// LoadConfig loads, vaildates, and initializes a JSON encoded OSL
+// LoadConfig loads, validates, and initializes a JSON encoded OSL
 // configuration.
 func LoadConfig(configJSON []byte) (*Config, error) {
 
@@ -780,7 +781,8 @@ type PaveLogInfo struct {
 // epoch to endTime, and a pave file for each OSL. paveServerEntries is
 // a map from hex-encoded OSL IDs to server entries to pave into that OSL.
 // When entries are found, OSL will contain those entries, newline
-// separated. Otherwise the OSL will still be issued, but be empty.
+// separated. Otherwise the OSL will still be issued, but be empty (unless
+// the scheme is in omitEmptyOSLsSchemes).
 //
 // As OSLs outside the epoch-endTime range will no longer appear in
 // the registry, Pave is intended to be used to create the full set
@@ -794,6 +796,8 @@ func (config *Config) Pave(
 	signingPublicKey string,
 	signingPrivateKey string,
 	paveServerEntries map[string][]string,
+	omitMD5SumsSchemes []int,
+	omitEmptyOSLsSchemes []int,
 	logCallback func(*PaveLogInfo)) ([]*PaveFile, error) {
 
 	config.ReloadableFile.RLock()
@@ -806,6 +810,10 @@ func (config *Config) Pave(
 	for schemeIndex, scheme := range config.Schemes {
 		if common.Contains(scheme.PropagationChannelIDs, propagationChannelID) {
 
+			omitMD5Sums := common.ContainsInt(omitMD5SumsSchemes, schemeIndex)
+
+			omitEmptyOSLs := common.ContainsInt(omitEmptyOSLsSchemes, schemeIndex)
+
 			oslDuration := scheme.GetOSLDuration()
 
 			oslTime := scheme.epoch
@@ -821,47 +829,52 @@ func (config *Config) Pave(
 
 				hexEncodedOSLID := hex.EncodeToString(fileSpec.ID)
 
-				registry.FileSpecs = append(registry.FileSpecs, fileSpec)
-
 				serverEntryCount := len(paveServerEntries[hexEncodedOSLID])
 
-				// serverEntries will be "" when nothing is found in paveServerEntries
-				serverEntries := strings.Join(paveServerEntries[hexEncodedOSLID], "\n")
+				if serverEntryCount > 0 || !omitEmptyOSLs {
 
-				serverEntriesPackage, err := common.WriteAuthenticatedDataPackage(
-					serverEntries,
-					signingPublicKey,
-					signingPrivateKey)
-				if err != nil {
-					return nil, common.ContextError(err)
-				}
+					registry.FileSpecs = append(registry.FileSpecs, fileSpec)
 
-				boxedServerEntries, err := box(fileKey, serverEntriesPackage)
-				if err != nil {
-					return nil, common.ContextError(err)
-				}
+					// serverEntries will be "" when nothing is found in paveServerEntries
+					serverEntries := strings.Join(paveServerEntries[hexEncodedOSLID], "\n")
+
+					serverEntriesPackage, err := common.WriteAuthenticatedDataPackage(
+						serverEntries,
+						signingPublicKey,
+						signingPrivateKey)
+					if err != nil {
+						return nil, common.ContextError(err)
+					}
 
-				md5sum := md5.Sum(boxedServerEntries)
-				fileSpec.MD5Sum = md5sum[:]
-
-				fileName := fmt.Sprintf(
-					OSL_FILENAME_FORMAT, hexEncodedOSLID)
-
-				paveFiles = append(paveFiles, &PaveFile{
-					Name:     fileName,
-					Contents: boxedServerEntries,
-				})
-
-				if logCallback != nil {
-					logCallback(&PaveLogInfo{
-						FileName:             fileName,
-						SchemeIndex:          schemeIndex,
-						PropagationChannelID: propagationChannelID,
-						OSLID:                hexEncodedOSLID,
-						OSLTime:              oslTime,
-						OSLDuration:          oslDuration,
-						ServerEntryCount:     serverEntryCount,
+					boxedServerEntries, err := box(fileKey, serverEntriesPackage)
+					if err != nil {
+						return nil, common.ContextError(err)
+					}
+
+					if !omitMD5Sums {
+						md5sum := md5.Sum(boxedServerEntries)
+						fileSpec.MD5Sum = md5sum[:]
+					}
+
+					fileName := fmt.Sprintf(
+						OSL_FILENAME_FORMAT, hexEncodedOSLID)
+
+					paveFiles = append(paveFiles, &PaveFile{
+						Name:     fileName,
+						Contents: boxedServerEntries,
 					})
+
+					if logCallback != nil {
+						logCallback(&PaveLogInfo{
+							FileName:             fileName,
+							SchemeIndex:          schemeIndex,
+							PropagationChannelID: propagationChannelID,
+							OSLID:                hexEncodedOSLID,
+							OSLTime:              oslTime,
+							OSLDuration:          oslDuration,
+							ServerEntryCount:     serverEntryCount,
+						})
+					}
 				}
 
 				oslTime = oslTime.Add(oslDuration)
@@ -936,19 +949,53 @@ func makeOSLFileSpec(
 	firstSLOK := scheme.deriveSLOK(ref)
 	oslID := firstSLOK.ID
 
-	// Note: previously, this was a random key. Now, the file key
+	// Note: previously, fileKey was a random key. Now, the key
 	// is derived from the master key and OSL ID. This deterministic
 	// derivation ensures that repeated paves of the same OSL
 	// with the same ID and same content yields the same MD5Sum
 	// to avoid wasteful downloads.
+	//
+	// Similarly, the shareKeys generated in divideKey and the Shamir
+	// key splitting random polynomials are now both determinisitcally
+	// generated from a seeded CSPRNG. This ensures that the OSL
+	// registry remains identical for repeated paves of the same config
+	// and parameters.
+	//
+	// The split structure is added to the deterministic key
+	// derivation so that changes to the split configuration will not
+	// expose the same key material to different SLOK combinations.
+
+	splitStructure := make([]byte, 16*(1+len(scheme.SeedPeriodKeySplits)))
+	i := 0
+	binary.LittleEndian.PutUint64(splitStructure[i:], uint64(len(scheme.SeedSpecs)))
+	binary.LittleEndian.PutUint64(splitStructure[i+8:], uint64(scheme.SeedSpecThreshold))
+	i += 16
+	for _, keySplit := range scheme.SeedPeriodKeySplits {
+		binary.LittleEndian.PutUint64(splitStructure[i:], uint64(keySplit.Total))
+		binary.LittleEndian.PutUint64(splitStructure[i+8:], uint64(keySplit.Threshold))
+		i += 16
+	}
 
 	fileKey := deriveKeyHKDF(
 		scheme.MasterKey,
+		splitStructure,
 		[]byte("osl-file-key"),
 		oslID)
 
+	splitKeyMaterialSeed := deriveKeyHKDF(
+		scheme.MasterKey,
+		splitStructure,
+		[]byte("osl-file-split-key-material-seed"),
+		oslID)
+
+	keyMaterialReader, err := newSeededKeyMaterialReader(splitKeyMaterialSeed)
+	if err != nil {
+		return nil, nil, common.ContextError(err)
+	}
+
 	keyShares, err := divideKey(
 		scheme,
+		keyMaterialReader,
 		fileKey,
 		scheme.SeedPeriodKeySplits,
 		propagationChannelID,
@@ -968,6 +1015,7 @@ func makeOSLFileSpec(
 // divideKey recursively constructs a KeyShares tree.
 func divideKey(
 	scheme *Scheme,
+	keyMaterialReader io.Reader,
 	key []byte,
 	keySplits []KeySplit,
 	propagationChannelID string,
@@ -976,7 +1024,11 @@ func divideKey(
 	keySplitIndex := len(keySplits) - 1
 	keySplit := keySplits[keySplitIndex]
 
-	shares, err := shamirSplit(key, keySplit.Total, keySplit.Threshold)
+	shares, err := shamirSplit(
+		key,
+		keySplit.Total,
+		keySplit.Threshold,
+		keyMaterialReader)
 	if err != nil {
 		return nil, common.ContextError(err)
 	}
@@ -986,15 +1038,12 @@ func divideKey(
 
 	for _, share := range shares {
 
-		// Note: for a fully deterministic pave, where the OSL registry
-		// is unchanged when no OSLs change, the share key would need
-		// to be derived (e.g., from the master key, OSL ID, key split
-		// index, and share index). However, since the OSL registry file
-		// content is nondeterministic in any case due to aspects of the
-		// Shamir secret splitting algorithm, there's no reason not to
-		// use a random key here.
+		var shareKey [KEY_LENGTH_BYTES]byte
 
-		shareKey, err := common.MakeSecureRandomBytes(KEY_LENGTH_BYTES)
+		n, err := keyMaterialReader.Read(shareKey[:])
+		if err == nil && n != len(shareKey) {
+			err = errors.New("unexpected length")
+		}
 		if err != nil {
 			return nil, common.ContextError(err)
 		}
@@ -1002,7 +1051,8 @@ func divideKey(
 		if keySplitIndex > 0 {
 			keyShare, err := divideKey(
 				scheme,
-				shareKey,
+				keyMaterialReader,
+				shareKey[:],
 				keySplits[0:keySplitIndex],
 				propagationChannelID,
 				nextSLOKTime)
@@ -1013,7 +1063,8 @@ func divideKey(
 		} else {
 			keyShare, err := divideKeyWithSeedSpecSLOKs(
 				scheme,
-				shareKey,
+				keyMaterialReader,
+				shareKey[:],
 				propagationChannelID,
 				nextSLOKTime)
 			if err != nil {
@@ -1023,7 +1074,7 @@ func divideKey(
 
 			*nextSLOKTime = nextSLOKTime.Add(time.Duration(scheme.SeedPeriodNanoseconds))
 		}
-		boxedShare, err := box(shareKey, share)
+		boxedShare, err := box(shareKey[:], share)
 		if err != nil {
 			return nil, common.ContextError(err)
 		}
@@ -1040,6 +1091,7 @@ func divideKey(
 
 func divideKeyWithSeedSpecSLOKs(
 	scheme *Scheme,
+	keyMaterialReader io.Reader,
 	key []byte,
 	propagationChannelID string,
 	nextSLOKTime *time.Time) (*KeyShares, error) {
@@ -1048,7 +1100,10 @@ func divideKeyWithSeedSpecSLOKs(
 	var slokIDs [][]byte
 
 	shares, err := shamirSplit(
-		key, len(scheme.SeedSpecs), scheme.SeedSpecThreshold)
+		key,
+		len(scheme.SeedSpecs),
+		scheme.SeedSpecThreshold,
+		keyMaterialReader)
 	if err != nil {
 		return nil, common.ContextError(err)
 	}
@@ -1352,6 +1407,38 @@ func NewOSLReader(
 		signingPublicKey)
 }
 
+// zeroReader reads an unlimited stream of zeroes.
+type zeroReader struct {
+}
+
+func (z *zeroReader) Read(p []byte) (int, error) {
+	for i := 0; i < len(p); i++ {
+		p[i] = 0
+	}
+	return len(p), nil
+}
+
+// newSeededKeyMaterialReader constructs a CSPRNG using AES-CTR.
+// The seed is the AES key and the IV is fixed and constant.
+// Using same seed will always produce the same output stream.
+// The data stream is intended to be used to deterministically
+// generate key material and is not intended as a general
+// purpose CSPRNG.
+func newSeededKeyMaterialReader(seed []byte) (io.Reader, error) {
+
+	aesCipher, err := aes.NewCipher(seed)
+	if err != nil {
+		return nil, common.ContextError(err)
+	}
+
+	var iv [aes.BlockSize]byte
+
+	return &cipher.StreamReader{
+		S: cipher.NewCTR(aesCipher, iv[:]),
+		R: new(zeroReader),
+	}, nil
+}
+
 // deriveKeyHKDF implements HKDF-Expand as defined in https://tools.ietf.org/html/rfc5869
 // where masterKey = PRK, context = info, and L = 32; SHA-256 is used so HashLen = 32
 func deriveKeyHKDF(masterKey []byte, context ...[]byte) []byte {
@@ -1372,7 +1459,11 @@ func isValidShamirSplit(total, threshold int) bool {
 }
 
 // shamirSplit is a helper wrapper for sss.Split
-func shamirSplit(secret []byte, total, threshold int) ([][]byte, error) {
+func shamirSplit(
+	secret []byte,
+	total, threshold int,
+	randReader io.Reader) ([][]byte, error) {
+
 	if !isValidShamirSplit(total, threshold) {
 		return nil, common.ContextError(errors.New("invalid parameters"))
 	}
@@ -1386,7 +1477,8 @@ func shamirSplit(secret []byte, total, threshold int) ([][]byte, error) {
 		return shares, nil
 	}
 
-	shareMap, err := sss.Split(byte(total), byte(threshold), secret)
+	shareMap, err := sss.SplitUsingReader(
+		byte(total), byte(threshold), secret, randReader)
 	if err != nil {
 		return nil, common.ContextError(err)
 	}

psiphon/common/osl/osl_test.go

@@ -358,22 +358,56 @@ func TestOSL(t *testing.T) {
 				}
 			}
 
+			// Note: these options are exercised in remoteServerList_test.go
+			omitMD5SumsSchemes := []int{}
+			omitEmptyOSLsSchemes := []int{}
+
+			firstPaveFiles, err := config.Pave(
+				endTime,
+				propagationChannelID,
+				signingPublicKey,
+				signingPrivateKey,
+				paveServerEntries,
+				omitMD5SumsSchemes,
+				omitEmptyOSLsSchemes,
+				nil)
+			if err != nil {
+				t.Fatalf("Pave failed: %s", err)
+			}
+
 			paveFiles, err := config.Pave(
 				endTime,
 				propagationChannelID,
 				signingPublicKey,
 				signingPrivateKey,
 				paveServerEntries,
+				omitMD5SumsSchemes,
+				omitEmptyOSLsSchemes,
 				nil)
 			if err != nil {
 				t.Fatalf("Pave failed: %s", err)
 			}
 
 			// Check that the paved file name matches the name the client will look for.
+
 			if len(paveFiles) < 1 || paveFiles[len(paveFiles)-1].Name != GetOSLRegistryURL("") {
 				t.Fatalf("invalid registry pave file")
 			}
 
+			// Check that the content of two paves is the same: all the crypto should be
+			// deterministic.
+
+			for index, paveFile := range paveFiles {
+				if paveFile.Name != firstPaveFiles[index].Name {
+					t.Fatalf("Pave name mismatch")
+				}
+				if bytes.Compare(paveFile.Contents, firstPaveFiles[index].Contents) != 0 {
+					t.Fatalf("Pave content mismatch")
+				}
+			}
+
+			// Use the paved content in the following tests.
+
 			pavedRegistries[propagationChannelID] = paveFiles[len(paveFiles)-1].Contents
 
 			pavedOSLFileContents[propagationChannelID] = make(map[string][]byte)

psiphon/common/osl/paver/main.go

@@ -29,6 +29,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strconv"
 	"time"
 
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/osl"
@@ -63,6 +64,12 @@ func main() {
 	var listScheme int
 	flag.IntVar(&listScheme, "list-scheme", -1, "list current period OSL IDs for specified scheme; no files are written")
 
+	var omitMD5SumsSchemes ints
+	flag.Var(&omitMD5SumsSchemes, "omit-md5sums", "omit MD5Sum fields for specified scheme(s)")
+
+	var omitEmptyOSLsSchemes ints
+	flag.Var(&omitEmptyOSLsSchemes, "omit-empty", "omit empty OSLs for specified scheme(s)")
+
 	flag.Parse()
 
 	// load config
@@ -211,6 +218,8 @@ func main() {
 			signingPublicKey,
 			signingPrivateKey,
 			paveServerEntries,
+			omitMD5SumsSchemes,
+			omitEmptyOSLsSchemes,
 			func(logInfo *osl.PaveLogInfo) {
 				pavedPayloadOSLID[logInfo.OSLID] = true
 				fmt.Printf(
@@ -266,3 +275,18 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+type ints []int
+
+func (i *ints) String() string {
+	return fmt.Sprint(*i)
+}
+
+func (i *ints) Set(strValue string) error {
+	value, err := strconv.Atoi(strValue)
+	if err != nil {
+		return err
+	}
+	*i = append(*i, value)
+	return nil
+}

psiphon/common/sss/.gitignore

@@ -0,0 +1 @@
+*.test

psiphon/common/sss/.travis.yml

@@ -0,0 +1,9 @@
+language: go
+go:
+  - 1.3.3
+notifications:
+  # See http://about.travis-ci.org/docs/user/build-configuration/ to learn more
+  # about configuring notification recipients and more.
+  email:
+    recipients:
+      - coda.hale@gmail.com

psiphon/common/sss/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Coda Hale
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

psiphon/common/sss/README.md

@@ -0,0 +1,11 @@
+# sss (Shamir's Secret Sharing)
+
+[![Build Status](https://travis-ci.org/codahale/sss.png?branch=master)](https://travis-ci.org/codahale/sss)
+
+A pure Go implementation of
+[Shamir's Secret Sharing algorithm](http://en.wikipedia.org/wiki/Shamir's_Secret_Sharing)
+over GF(2^8).
+
+Inspired by @hbs's [Python implementation](https://github.com/hbs/PySSSS).
+
+For documentation, check [godoc](http://godoc.org/github.com/codahale/sss).

psiphon/common/sss/gf256.go

@@ -0,0 +1,81 @@
+package sss
+
+func mul(e, a byte) byte {
+	if e == 0 || a == 0 {
+		return 0
+	}
+	return exp[(int(log[e])+int(log[a]))%255]
+}
+
+func div(e, a byte) byte {
+	if a == 0 {
+		panic("div by zero")
+	}
+
+	if e == 0 {
+		return 0
+	}
+
+	p := (int(log[e]) - int(log[a])) % 255
+	if p < 0 {
+		p += 255
+	}
+
+	return exp[p]
+}
+
+const (
+	fieldSize = 256 // 2^8
+)
+
+var (
+	// 0x11b prime polynomial and 0x03 as generator
+	exp = [fieldSize]byte{
+		0x01, 0x03, 0x05, 0x0f, 0x11, 0x33, 0x55, 0xff, 0x1a, 0x2e, 0x72, 0x96,
+		0xa1, 0xf8, 0x13, 0x35, 0x5f, 0xe1, 0x38, 0x48, 0xd8, 0x73, 0x95, 0xa4,
+		0xf7, 0x02, 0x06, 0x0a, 0x1e, 0x22, 0x66, 0xaa, 0xe5, 0x34, 0x5c, 0xe4,
+		0x37, 0x59, 0xeb, 0x26, 0x6a, 0xbe, 0xd9, 0x70, 0x90, 0xab, 0xe6, 0x31,
+		0x53, 0xf5, 0x04, 0x0c, 0x14, 0x3c, 0x44, 0xcc, 0x4f, 0xd1, 0x68, 0xb8,
+		0xd3, 0x6e, 0xb2, 0xcd, 0x4c, 0xd4, 0x67, 0xa9, 0xe0, 0x3b, 0x4d, 0xd7,
+		0x62, 0xa6, 0xf1, 0x08, 0x18, 0x28, 0x78, 0x88, 0x83, 0x9e, 0xb9, 0xd0,
+		0x6b, 0xbd, 0xdc, 0x7f, 0x81, 0x98, 0xb3, 0xce, 0x49, 0xdb, 0x76, 0x9a,
+		0xb5, 0xc4, 0x57, 0xf9, 0x10, 0x30, 0x50, 0xf0, 0x0b, 0x1d, 0x27, 0x69,
+		0xbb, 0xd6, 0x61, 0xa3, 0xfe, 0x19, 0x2b, 0x7d, 0x87, 0x92, 0xad, 0xec,
+		0x2f, 0x71, 0x93, 0xae, 0xe9, 0x20, 0x60, 0xa0, 0xfb, 0x16, 0x3a, 0x4e,
+		0xd2, 0x6d, 0xb7, 0xc2, 0x5d, 0xe7, 0x32, 0x56, 0xfa, 0x15, 0x3f, 0x41,
+		0xc3, 0x5e, 0xe2, 0x3d, 0x47, 0xc9, 0x40, 0xc0, 0x5b, 0xed, 0x2c, 0x74,
+		0x9c, 0xbf, 0xda, 0x75, 0x9f, 0xba, 0xd5, 0x64, 0xac, 0xef, 0x2a, 0x7e,
+		0x82, 0x9d, 0xbc, 0xdf, 0x7a, 0x8e, 0x89, 0x80, 0x9b, 0xb6, 0xc1, 0x58,
+		0xe8, 0x23, 0x65, 0xaf, 0xea, 0x25, 0x6f, 0xb1, 0xc8, 0x43, 0xc5, 0x54,
+		0xfc, 0x1f, 0x21, 0x63, 0xa5, 0xf4, 0x07, 0x09, 0x1b, 0x2d, 0x77, 0x99,
+		0xb0, 0xcb, 0x46, 0xca, 0x45, 0xcf, 0x4a, 0xde, 0x79, 0x8b, 0x86, 0x91,
+		0xa8, 0xe3, 0x3e, 0x42, 0xc6, 0x51, 0xf3, 0x0e, 0x12, 0x36, 0x5a, 0xee,
+		0x29, 0x7b, 0x8d, 0x8c, 0x8f, 0x8a, 0x85, 0x94, 0xa7, 0xf2, 0x0d, 0x17,
+		0x39, 0x4b, 0xdd, 0x7c, 0x84, 0x97, 0xa2, 0xfd, 0x1c, 0x24, 0x6c, 0xb4,
+		0xc7, 0x52, 0xf6, 0x01,
+	}
+	log = [fieldSize]byte{
+		0x00, 0x00, 0x19, 0x01, 0x32, 0x02, 0x1a, 0xc6, 0x4b, 0xc7, 0x1b, 0x68,
+		0x33, 0xee, 0xdf, 0x03, 0x64, 0x04, 0xe0, 0x0e, 0x34, 0x8d, 0x81, 0xef,
+		0x4c, 0x71, 0x08, 0xc8, 0xf8, 0x69, 0x1c, 0xc1, 0x7d, 0xc2, 0x1d, 0xb5,
+		0xf9, 0xb9, 0x27, 0x6a, 0x4d, 0xe4, 0xa6, 0x72, 0x9a, 0xc9, 0x09, 0x78,
+		0x65, 0x2f, 0x8a, 0x05, 0x21, 0x0f, 0xe1, 0x24, 0x12, 0xf0, 0x82, 0x45,
+		0x35, 0x93, 0xda, 0x8e, 0x96, 0x8f, 0xdb, 0xbd, 0x36, 0xd0, 0xce, 0x94,
+		0x13, 0x5c, 0xd2, 0xf1, 0x40, 0x46, 0x83, 0x38, 0x66, 0xdd, 0xfd, 0x30,
+		0xbf, 0x06, 0x8b, 0x62, 0xb3, 0x25, 0xe2, 0x98, 0x22, 0x88, 0x91, 0x10,
+		0x7e, 0x6e, 0x48, 0xc3, 0xa3, 0xb6, 0x1e, 0x42, 0x3a, 0x6b, 0x28, 0x54,
+		0xfa, 0x85, 0x3d, 0xba, 0x2b, 0x79, 0x0a, 0x15, 0x9b, 0x9f, 0x5e, 0xca,
+		0x4e, 0xd4, 0xac, 0xe5, 0xf3, 0x73, 0xa7, 0x57, 0xaf, 0x58, 0xa8, 0x50,
+		0xf4, 0xea, 0xd6, 0x74, 0x4f, 0xae, 0xe9, 0xd5, 0xe7, 0xe6, 0xad, 0xe8,
+		0x2c, 0xd7, 0x75, 0x7a, 0xeb, 0x16, 0x0b, 0xf5, 0x59, 0xcb, 0x5f, 0xb0,
+		0x9c, 0xa9, 0x51, 0xa0, 0x7f, 0x0c, 0xf6, 0x6f, 0x17, 0xc4, 0x49, 0xec,
+		0xd8, 0x43, 0x1f, 0x2d, 0xa4, 0x76, 0x7b, 0xb7, 0xcc, 0xbb, 0x3e, 0x5a,
+		0xfb, 0x60, 0xb1, 0x86, 0x3b, 0x52, 0xa1, 0x6c, 0xaa, 0x55, 0x29, 0x9d,
+		0x97, 0xb2, 0x87, 0x90, 0x61, 0xbe, 0xdc, 0xfc, 0xbc, 0x95, 0xcf, 0xcd,
+		0x37, 0x3f, 0x5b, 0xd1, 0x53, 0x39, 0x84, 0x3c, 0x41, 0xa2, 0x6d, 0x47,
+		0x14, 0x2a, 0x9e, 0x5d, 0x56, 0xf2, 0xd3, 0xab, 0x44, 0x11, 0x92, 0xd9,
+		0x23, 0x20, 0x2e, 0x89, 0xb4, 0x7c, 0xb8, 0x26, 0x77, 0x99, 0xe3, 0xa5,
+		0x67, 0x4a, 0xed, 0xde, 0xc5, 0x31, 0xfe, 0x18, 0x0d, 0x63, 0x8c, 0x80,
+		0xc0, 0xf7, 0x70, 0x07,
+	}
+)

psiphon/common/sss/gf256_test.go

@@ -0,0 +1,35 @@
+package sss
+
+import (
+	"testing"
+)
+
+func TestMul(t *testing.T) {
+	if v, want := mul(90, 21), byte(254); v != want {
+		t.Errorf("Was %v, but expected %v", v, want)
+	}
+}
+
+func TestDiv(t *testing.T) {
+	if v, want := div(90, 21), byte(189); v != want {
+		t.Errorf("Was %v, but expected %v", v, want)
+	}
+}
+
+func TestDivZero(t *testing.T) {
+	if v, want := div(0, 2), byte(0); v != want {
+		t.Errorf("Was %v, but expected %v", v, want)
+	}
+}
+
+func TestDivByZero(t *testing.T) {
+	defer func() {
+		m := recover()
+		if m != "div by zero" {
+			t.Error(m)
+		}
+	}()
+
+	div(2, 0)
+	t.Error("Shouldn't have been able to divide those")
+}

psiphon/common/sss/polynomial.go

@@ -0,0 +1,67 @@
+package sss
+
+import "io"
+
+// the degree of the polynomial
+func degree(p []byte) int {
+	return len(p) - 1
+}
+
+// evaluate the polynomial at the given point
+func eval(p []byte, x byte) (result byte) {
+	// Horner's scheme
+	for i := 1; i <= len(p); i++ {
+		result = mul(result, x) ^ p[len(p)-i]
+	}
+	return
+}
+
+// generates a random n-degree polynomial w/ a given x-intercept
+func generate(degree byte, x byte, rand io.Reader) ([]byte, error) {
+	result := make([]byte, degree+1)
+	result[0] = x
+
+	buf := make([]byte, degree-1)
+	if _, err := io.ReadFull(rand, buf); err != nil {
+		return nil, err
+	}
+
+	for i := byte(1); i < degree; i++ {
+		result[i] = buf[i-1]
+	}
+
+	// the Nth term can't be zero, or else it's a (N-1) degree polynomial
+	for {
+		buf = make([]byte, 1)
+		if _, err := io.ReadFull(rand, buf); err != nil {
+			return nil, err
+		}
+
+		if buf[0] != 0 {
+			result[degree] = buf[0]
+			return result, nil
+		}
+	}
+}
+
+// an input/output pair
+type pair struct {
+	x, y byte
+}
+
+// Lagrange interpolation
+func interpolate(points []pair, x byte) (value byte) {
+	for i, a := range points {
+		weight := byte(1)
+		for j, b := range points {
+			if i != j {
+				top := x ^ b.x
+				bottom := a.x ^ b.x
+				factor := div(top, bottom)
+				weight = mul(weight, factor)
+			}
+		}
+		value = value ^ mul(weight, a.y)
+	}
+	return
+}

psiphon/common/sss/polynomial_test.go

@@ -0,0 +1,89 @@
+package sss
+
+import (
+	"bytes"
+	"testing"
+)
+
+var (
+	p  = []byte{1, 0, 2, 3}
+	p2 = []byte{70, 32, 6}
+)
+
+func TestDegree(t *testing.T) {
+	if v, want := degree(p), 3; v != want {
+		t.Errorf("Was %v, but expected %v", v, want)
+	}
+}
+
+func TestEval(t *testing.T) {
+	if v, want := eval(p, 2), byte(17); v != want {
+		t.Errorf("Was %v, but expected %v", v, want)
+	}
+}
+
+func TestGenerate(t *testing.T) {
+	b := []byte{1, 2, 3}
+
+	expected := []byte{10, 1, 2, 3}
+	actual, err := generate(3, 10, bytes.NewReader(b))
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !bytes.Equal(actual, expected) {
+		t.Errorf("Was %v, but expected %v", actual, expected)
+	}
+}
+
+func TestGenerateEOF(t *testing.T) {
+	b := []byte{1}
+
+	p, err := generate(3, 10, bytes.NewReader(b))
+	if p != nil {
+		t.Errorf("Was %v, but expected an error", p)
+	}
+
+	if err == nil {
+		t.Error("No error returned")
+	}
+}
+
+func TestGeneratePolyEOFFullSize(t *testing.T) {
+	b := []byte{1, 2, 0, 0, 0, 0}
+
+	p, err := generate(3, 10, bytes.NewReader(b))
+	if p != nil {
+		t.Errorf("Was %v, but xpected an error", p)
+	}
+
+	if err == nil {
+		t.Error("No error returned")
+	}
+}
+
+func TestGenerateFullSize(t *testing.T) {
+	b := []byte{1, 2, 0, 4}
+
+	expected := []byte{10, 1, 2, 4}
+	actual, err := generate(3, 10, bytes.NewReader(b))
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !bytes.Equal(actual, expected) {
+		t.Errorf("Was %v but expected %v", actual, expected)
+	}
+}
+
+func TestInterpolate(t *testing.T) {
+	in := []pair{
+		pair{x: 1, y: 1},
+		pair{x: 2, y: 2},
+		pair{x: 3, y: 3},
+	}
+
+	if v, want := interpolate(in, 0), byte(0); v != want {
+		t.Errorf("Was %v, but expected %v", v, want)
+	}
+}

psiphon/common/sss/sss.go

@@ -0,0 +1,116 @@
+// Package sss implements Shamir's Secret Sharing algorithm over GF(2^8).
+//
+// Shamir's Secret Sharing algorithm allows you to securely share a secret with
+// N people, allowing the recovery of that secret if K of those people combine
+// their shares.
+//
+// It begins by encoding a secret as a number (e.g., 42), and generating N
+// random polynomial equations of degree K-1 which have an X-intercept equal to
+// the secret. Given K=3, the following equations might be generated:
+//
+//     f1(x) =  78x^2 +  19x + 42
+//     f2(x) = 128x^2 + 171x + 42
+//     f3(x) = 121x^2 +   3x + 42
+//     f4(x) =  91x^2 +  95x + 42
+//     etc.
+//
+// These polynomials are then evaluated for values of X > 0:
+//
+//     f1(1) =  139
+//     f2(2) =  896
+//     f3(3) = 1140
+//     f4(4) = 1783
+//     etc.
+//
+// These (x, y) pairs are the shares given to the parties. In order to combine
+// shares to recover the secret, these (x, y) pairs are used as the input points
+// for Lagrange interpolation, which produces a polynomial which matches the
+// given points. This polynomial can be evaluated for f(0), producing the secret
+// value--the common x-intercept for all the generated polynomials.
+//
+// If fewer than K shares are combined, the interpolated polynomial will be
+// wrong, and the result of f(0) will not be the secret.
+//
+// This package constructs polynomials over the field GF(2^8) for each byte of
+// the secret, allowing for fast splitting and combining of anything which can
+// be encoded as bytes.
+//
+// This package has not been audited by cryptography or security professionals.
+package sss
+
+import (
+	"crypto/rand"
+	"errors"
+	"io"
+)
+
+var (
+	// ErrInvalidCount is returned when the count parameter is invalid.
+	ErrInvalidCount = errors.New("N must be >= K")
+	// ErrInvalidThreshold is returned when the threshold parameter is invalid.
+	ErrInvalidThreshold = errors.New("K must be > 1")
+)
+
+// Split the given secret into N shares of which K are required to recover the
+// secret. Returns a map of share IDs (1-255) to shares.
+func Split(n, k byte, secret []byte) (map[byte][]byte, error) {
+	return split(n, k, secret, rand.Reader)
+}
+
+// SplitUsingReader splits the given secret, as Split does, but using the
+// specified reader to create random polynomials. Use for deterministic
+// splitting; caller must ensure reader is cryptographically secure.
+func SplitUsingReader(
+	n, k byte, secret []byte, reader io.Reader) (map[byte][]byte, error) {
+
+	return split(n, k, secret, reader)
+}
+
+func split(n, k byte, secret []byte, randReader io.Reader) (map[byte][]byte, error) {
+	if k <= 1 {
+		return nil, ErrInvalidThreshold
+	}
+
+	if n < k {
+		return nil, ErrInvalidCount
+	}
+
+	shares := make(map[byte][]byte, n)
+
+	for _, b := range secret {
+		p, err := generate(k-1, b, randReader)
+		if err != nil {
+			return nil, err
+		}
+
+		for x := byte(1); x <= n; x++ {
+			shares[x] = append(shares[x], eval(p, x))
+		}
+	}
+
+	return shares, nil
+}
+
+// Combine the given shares into the original secret.
+//
+// N.B.: There is no way to know whether the returned value is, in fact, the
+// original secret.
+func Combine(shares map[byte][]byte) []byte {
+	var secret []byte
+	for _, v := range shares {
+		secret = make([]byte, len(v))
+		break
+	}
+
+	points := make([]pair, len(shares))
+	for i := range secret {
+		p := 0
+		for k, v := range shares {
+			points[p] = pair{x: k, y: v[i]}
+			p++
+		}
+		secret[i] = interpolate(points, 0)
+	}
+
+	return secret
+}

psiphon/common/sss/sss_test.go

@@ -0,0 +1,32 @@
+package sss
+
+import (
+	"fmt"
+)
+
+func Example() {
+	secret := "well hello there!" // our secret
+	n := byte(30)                 // create 30 shares
+	k := byte(2)                  // require 2 of them to combine
+
+	shares, err := Split(n, k, []byte(secret)) // split into 30 shares
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	// select a random subset of the total shares
+	subset := make(map[byte][]byte, k)
+	for x, y := range shares { // just iterate since maps are randomized
+		subset[x] = y
+		if len(subset) == int(k) {
+			break
+		}
+	}
+
+	// combine two shares and recover the secret
+	recovered := string(Combine(subset))
+	fmt.Println(recovered)
+
+	// Output: well hello there!
+}

psiphon/common/utils.go

@@ -48,6 +48,17 @@ func Contains(list []string, target string) bool {
 	return false
 }
 
+// ContainsInt returns true if the target int is
+// in the list.
+func ContainsInt(list []int, target int) bool {
+	for _, listItem := range list {
+		if listItem == target {
+			return true
+		}
+	}
+	return false
+}
+
 // FlipCoin is a helper function that randomly
 // returns true or false. If the underlying random
 // number generator fails, FlipCoin still returns

psiphon/controller_test.go

@@ -1041,7 +1041,7 @@ func initDisruptor() {
 			localConn, err := listener.AcceptSocks()
 			if err != nil {
 				if e, ok := err.(net.Error); ok && e.Temporary() {
-					fmt.Printf("disruptor proxy temporary accept error: %s", err)
+					fmt.Printf("disruptor proxy temporary accept error: %s\n", err)
 					continue
 				}
 				fmt.Printf("disruptor proxy accept error: %s\n", err)

psiphon/memory_test/memory_test.go

@@ -20,6 +20,7 @@
 package memory_test
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -127,7 +128,8 @@ func runMemoryTest(t *testing.T, testMode int) {
 	}
 
 	var controller *psiphon.Controller
-	var controllerShutdown chan struct{}
+	var controllerCtx context.Context
+	var controllerStopRunning context.CancelFunc
 	var controllerWaitGroup *sync.WaitGroup
 	restartController := make(chan bool, 1)
 	reconnectTunnel := make(chan bool, 1)
@@ -179,21 +181,23 @@ func runMemoryTest(t *testing.T, testMode int) {
 			t.Fatalf("error creating controller: %s", err)
 		}
 
-		controllerShutdown = make(chan struct{})
+		controllerCtx, controllerStopRunning = context.WithCancel(context.Background())
 		controllerWaitGroup = new(sync.WaitGroup)
+
 		controllerWaitGroup.Add(1)
 		go func() {
 			defer controllerWaitGroup.Done()
-			controller.Run(controllerShutdown)
+			controller.Run(controllerCtx)
 		}()
 	}
 
 	stopController := func() {
-		close(controllerShutdown)
+		controllerStopRunning()
 		controllerWaitGroup.Wait()
 	}
 
 	testTimer := time.NewTimer(testDuration)
+	defer testTimer.Stop()
 	memInspectionTicker := time.NewTicker(memInspectionFrequency)
 	lastTunnelsEstablished := int32(0)
 

psiphon/remoteServerList_test.go

@@ -35,6 +35,7 @@ import (
 	"path"
 	"path/filepath"
 	"sync"
+	"syscall"
 	"testing"
 	"time"
 
@@ -47,6 +48,14 @@ import (
 // TODO: TestCommonRemoteServerList (this is currently covered by controller_test.go)
 
 func TestObfuscatedRemoteServerLists(t *testing.T) {
+	testObfuscatedRemoteServerLists(t, false)
+}
+
+func TestObfuscatedRemoteServerListsOmitMD5Sums(t *testing.T) {
+	testObfuscatedRemoteServerLists(t, true)
+}
+
+func testObfuscatedRemoteServerLists(t *testing.T, omitMD5Sums bool) {
 
 	testDataDirName, err := ioutil.TempDir("", "psiphon-remote-server-list-test")
 	if err != nil {
@@ -150,16 +159,24 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 		t.Fatalf("error generating package keys: %s", err)
 	}
 
+	var omitMD5SumsSchemes []int
+	if omitMD5Sums {
+		omitMD5SumsSchemes = []int{0}
+	}
 	// First Pave() call is to get the OSL ID to pave into
 
 	oslID := ""
 
+	omitEmptyOSLsSchemes := []int{}
+
 	paveFiles, err := oslConfig.Pave(
 		epoch,
 		propagationChannelID,
 		signingPublicKey,
 		signingPrivateKey,
 		map[string][]string{},
+		omitMD5SumsSchemes,
+		omitEmptyOSLsSchemes,
 		func(logInfo *osl.PaveLogInfo) {
 			oslID = logInfo.OSLID
 		})
@@ -167,6 +184,8 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 		t.Fatalf("error paving OSL files: %s", err)
 	}
 
+	omitEmptyOSLsSchemes = []int{0}
+
 	paveFiles, err = oslConfig.Pave(
 		epoch,
 		propagationChannelID,
@@ -175,6 +194,8 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 		map[string][]string{
 			oslID: {string(encodedServerEntry)},
 		},
+		omitMD5SumsSchemes,
+		omitEmptyOSLsSchemes,
 		nil)
 	if err != nil {
 		t.Fatalf("error paving OSL files: %s", err)
@@ -211,9 +232,17 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 	//
 
 	// Exercise using multiple download URLs
-	remoteServerListHostAddresses := []string{
-		net.JoinHostPort(serverIPAddress, "8081"),
-		net.JoinHostPort(serverIPAddress, "8082"),
+
+	var remoteServerListListeners [2]net.Listener
+	var remoteServerListHostAddresses [2]string
+
+	for i := 0; i < len(remoteServerListListeners); i++ {
+		remoteServerListListeners[i], err = net.Listen("tcp", net.JoinHostPort(serverIPAddress, "0"))
+		if err != nil {
+			t.Fatalf("net.Listen error: %s", err)
+		}
+		defer remoteServerListListeners[i].Close()
+		remoteServerListHostAddresses[i] = remoteServerListListeners[i].Addr().String()
 	}
 
 	// The common remote server list fetches will 404
@@ -234,7 +263,7 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 			obfuscatedServerListRootURLsJSONConfig += ","
 		}
 
-		go func(remoteServerListHostAddress string) {
+		go func(listener net.Listener, remoteServerListHostAddress string) {
 			startTime := time.Now()
 			serveMux := http.NewServeMux()
 			for _, paveFile := range paveFiles {
@@ -250,12 +279,8 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 				Addr:    remoteServerListHostAddress,
 				Handler: serveMux,
 			}
-			err := httpServer.ListenAndServe()
-			if err != nil {
-				// TODO: wrong goroutine for t.FatalNow()
-				t.Fatalf("error running remote server list host: %s", err)
-			}
-		}(remoteServerListHostAddresses[i])
+			httpServer.Serve(listener)
+		}(remoteServerListListeners[i], remoteServerListHostAddresses[i])
 	}
 
 	obfuscatedServerListDownloadDirectory := testDataDirName
@@ -272,22 +297,34 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 		}
 	}()
 
+	process, err := os.FindProcess(os.Getpid())
+	if err != nil {
+		t.Fatalf("os.FindProcess error: %s", err)
+	}
+	defer process.Signal(syscall.SIGTERM)
+
 	//
 	// disrupt remote server list downloads
 	//
 
-	disruptorProxyAddress := "127.0.0.1:2162"
+	disruptorListener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("net.Listen error: %s", err)
+	}
+	defer disruptorListener.Close()
+
+	disruptorProxyAddress := disruptorListener.Addr().String()
 	disruptorProxyURL := "socks4a://" + disruptorProxyAddress
 
 	go func() {
-		listener, err := socks.ListenSocks("tcp", disruptorProxyAddress)
-		if err != nil {
-			fmt.Printf("disruptor proxy listen error: %s\n", err)
-			return
-		}
+		listener := socks.NewSocksListener(disruptorListener)
 		for {
 			localConn, err := listener.AcceptSocks()
 			if err != nil {
+				if e, ok := err.(net.Error); ok && e.Temporary() {
+					fmt.Printf("disruptor proxy temporary accept error: %s\n", err)
+					continue
+				}
 				fmt.Printf("disruptor proxy accept error: %s\n", err)
 				return
 			}
@@ -309,7 +346,7 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 					defer waitGroup.Done()
 					io.Copy(remoteConn, localConn)
 				}()
-				if common.Contains(remoteServerListHostAddresses, localConn.Req.Target) {
+				if common.Contains(remoteServerListHostAddresses[:], localConn.Req.Target) {
 					io.CopyN(localConn, remoteConn, 500)
 				} else {
 					io.Copy(localConn, remoteConn)