Add server entry exchange support

- Support signing and verifying individual
  server entries

- Export and import an exchange payload
  containing a signed server entry and
  a subset of dial parameters

- psiphond handshake may return own signed
  server entry to clients that don't have
  signed entry

- Simplified psiphond psinet server entry
  representation

MobileLibrary/Android/PsiphonTunnel/PsiphonTunnel.java

@@ -195,6 +195,14 @@ public class PsiphonTunnel implements PsiphonProvider {
         mClientPlatformSuffix.set(suffix);
     }
 
+    public String exportExchangePayload() {
+        return mPsiphonTunnel.exportExchangePayload();
+    }
+
+    public boolean importExchangePayload(String payload) {
+        return mPsiphonTunnel.importExchangePayload(payload);
+    }
+
     // Writes Go runtime profile information to a set of files in the specifiec output directory.
     // cpuSampleDurationSeconds and blockSampleDurationSeconds determines how to long to wait and
     // sample profiles that require active sampling. When set to 0, these profiles are skipped.

MobileLibrary/psi/psi.go

@@ -213,6 +213,45 @@ func SetDynamicConfig(newSponsorID, newAuthorizationsList string) {
 	}
 }
 
+// ExportExchangePayload creates a payload for client-to-client server
+// connection info exchange.
+//
+// ExportExchangePayload will succeed only when Psiphon is running, between
+// Start and Stop.
+//
+// The return value is a payload that may be exchanged with another client;
+// when "", the export failed and a diagnotice has been logged.
+func ExportExchangePayload() string {
+
+	controllerMutex.Lock()
+	defer controllerMutex.Unlock()
+
+	if controller == nil {
+		return ""
+	}
+
+	return controller.ExportExchangePayload()
+}
+
+// ImportExchangePayload imports a payload generated by ExportExchangePayload.
+//
+// If an import occurs when Psiphon is working to establsh a tunnel, the newly
+// imported server entry is prioritized.
+//
+// The return value indicates a successful import. If the import failed, a a
+// diagnostic notice has been logged.
+func ImportExchangePayload(payload string) bool {
+
+	controllerMutex.Lock()
+	defer controllerMutex.Unlock()
+
+	if controller == nil {
+		return false
+	}
+
+	return controller.ImportExchangePayload(payload)
+}
+
 // Encrypt and upload feedback.
 func SendFeedback(configJson, diagnosticsJson, b64EncodedPublicKey, uploadServer, uploadPath, uploadServerHeaders string) error {
 	return psiphon.SendFeedback(configJson, diagnosticsJson, b64EncodedPublicKey, uploadServer, uploadPath, uploadServerHeaders)

psiphon/common/protocol/protocol.go

@@ -47,6 +47,7 @@ const (
 	SERVER_ENTRY_SOURCE_DISCOVERY  = "DISCOVERY"
 	SERVER_ENTRY_SOURCE_TARGET     = "TARGET"
 	SERVER_ENTRY_SOURCE_OBFUSCATED = "OBFUSCATED"
+	SERVER_ENTRY_SOURCE_EXCHANGED  = "EXCHANGED"
 
 	CAPABILITY_SSH_API_REQUESTS            = "ssh-api-requests"
 	CAPABILITY_UNTUNNELED_WEB_API_REQUESTS = "handshake"
@@ -118,6 +119,7 @@ var SupportedServerEntrySources = TunnelProtocols{
 	SERVER_ENTRY_SOURCE_DISCOVERY,
 	SERVER_ENTRY_SOURCE_TARGET,
 	SERVER_ENTRY_SOURCE_OBFUSCATED,
+	SERVER_ENTRY_SOURCE_EXCHANGED,
 }
 
 func TunnelProtocolUsesSSH(protocol string) bool {

psiphon/common/protocol/serverEntry.go

@@ -23,6 +23,7 @@ import (
 	"bufio"
 	"bytes"
 	"crypto/hmac"
+	"crypto/rand"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
@@ -32,8 +33,10 @@ import (
 	"io"
 	"net"
 	"strings"
+	"time"
 
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/crypto/ed25519"
 )
 
 // ServerEntry represents a Psiphon server. It contains information
@@ -69,12 +72,15 @@ type ServerEntry struct {
 	TacticsRequestObfuscatedKey   string   `json:"tacticsRequestObfuscatedKey"`
 	MarionetteFormat              string   `json:"marionetteFormat"`
 	ConfigurationVersion          int      `json:"configurationVersion"`
+	Signature                     string   `json:"signature"`
 
 	// These local fields are not expected to be present in downloaded server
 	// entries. They are added by the client to record and report stats about
 	// how and when server entries are obtained.
-	LocalSource    string `json:"localSource"`
-	LocalTimestamp string `json:"localTimestamp"`
+	// All local fields should be included the list of fields in RemoveUnsignedFields.
+	LocalSource       string `json:"localSource,omitempty"`
+	LocalTimestamp    string `json:"localTimestamp,omitempty"`
+	IsLocalDerivedTag bool   `json:"isLocalDerivedTag,omitempty"`
 }
 
 // ServerEntryFields is an alternate representation of ServerEntry which
@@ -89,6 +95,23 @@ type ServerEntry struct {
 // ServerEntry type.
 type ServerEntryFields map[string]interface{}
 
+// GetServerEntry converts a ServerEntryFields into a ServerEntry.
+func (fields ServerEntryFields) GetServerEntry() (*ServerEntry, error) {
+
+	marshaledServerEntry, err := json.Marshal(fields)
+	if err != nil {
+		return nil, common.ContextError(err)
+	}
+
+	var serverEntry *ServerEntry
+	err = json.Unmarshal(marshaledServerEntry, &serverEntry)
+	if err != nil {
+		return nil, common.ContextError(err)
+	}
+
+	return serverEntry, nil
+}
+
 func (fields ServerEntryFields) GetTag() string {
 	tag, ok := fields["tag"]
 	if !ok {
@@ -101,8 +124,35 @@ func (fields ServerEntryFields) GetTag() string {
 	return tagStr
 }
 
+// SetTag sets a local, derived server entry tag. A tag is an identifier used
+// in server entry pruning and potentially other use cases. An explict tag,
+// set by the Psiphon Network, may be present in a server entry that is
+// imported; otherwise, the client will set a derived tag. The tag should be
+// generated using GenerateServerEntryTag. When SetTag finds a explicit tag,
+// the new, derived tag is ignored. The isLocalTag local field is set to
+// distinguish explict and derived tags and is used in signature verification
+// to determine if the tag field is part of the signature.
 func (fields ServerEntryFields) SetTag(tag string) {
+
+	// Don't replace explicit tag
+	if tag, ok := fields["tag"]; ok {
+		tagStr, ok := tag.(string)
+		if ok && tagStr != "" {
+			isLocalDerivedTag, ok := fields["isLocalDerivedTag"]
+			if !ok {
+				return
+			}
+			isLocalDerivedTagBool, ok := isLocalDerivedTag.(bool)
+			if ok && isLocalDerivedTagBool {
+				return
+			}
+		}
+	}
+
 	fields["tag"] = tag
+
+	// Mark this tag as local
+	fields["isLocalDerivedTag"] = true
 }
 
 func (fields ServerEntryFields) GetIPAddress() string {
@@ -157,10 +207,176 @@ func (fields ServerEntryFields) SetLocalSource(source string) {
 	fields["localSource"] = source
 }
 
+func (fields ServerEntryFields) GetLocalTimestamp() string {
+	localTimestamp, ok := fields["localTimestamp"]
+	if !ok {
+		return ""
+	}
+	localTimestampStr, ok := localTimestamp.(string)
+	if !ok {
+		return ""
+	}
+	return localTimestampStr
+}
+
 func (fields ServerEntryFields) SetLocalTimestamp(timestamp string) {
 	fields["localTimestamp"] = timestamp
 }
 
+func (fields ServerEntryFields) HasSignature() bool {
+	signature, ok := fields["signature"]
+	if !ok {
+		return false
+	}
+	signatureStr, ok := signature.(string)
+	if !ok {
+		return false
+	}
+	return signatureStr != ""
+}
+
+const signaturePublicKeyDigestSize = 8
+
+// AddSignature signs a server entry and attaches a new field containing the
+// signature. Any existing "signature" field will be replaced.
+//
+// The signature incudes a public key ID that is derived from a digest of the
+// public key value. This ID is intended for future use when multiple signing
+// keys may be deployed.
+func (fields ServerEntryFields) AddSignature(publicKey, privateKey string) error {
+
+	// Make a copy so that removing unsigned fields will have no side effects
+	copyFields := make(ServerEntryFields)
+	for k, v := range fields {
+		copyFields[k] = v
+	}
+
+	copyFields.RemoveUnsignedFields()
+
+	delete(copyFields, "signature")
+
+	marshaledFields, err := json.Marshal(copyFields)
+	if err != nil {
+		return common.ContextError(err)
+	}
+
+	decodedPublicKey, err := base64.StdEncoding.DecodeString(publicKey)
+	if err != nil {
+		return common.ContextError(err)
+	}
+
+	publicKeyDigest := sha256.Sum256(decodedPublicKey)
+	publicKeyID := publicKeyDigest[:signaturePublicKeyDigestSize]
+
+	decodedPrivateKey, err := base64.StdEncoding.DecodeString(privateKey)
+	if err != nil {
+		return common.ContextError(err)
+	}
+
+	signature := ed25519.Sign(decodedPrivateKey, marshaledFields)
+
+	fields["signature"] = base64.StdEncoding.EncodeToString(
+		append(publicKeyID, signature...))
+
+	return nil
+}
+
+// VerifySignature verifies the signature set by AddSignature.
+//
+// VerifySignature must be called before using any server entry that is
+// imported from an untrusted source, such as client-to-client exchange.
+func (fields ServerEntryFields) VerifySignature(publicKey string) error {
+
+	// Make a copy so that removing unsigned fields will have no side effects
+	copyFields := make(ServerEntryFields)
+	for k, v := range fields {
+		copyFields[k] = v
+	}
+
+	signatureField, ok := copyFields["signature"]
+	if !ok {
+		return common.ContextError(errors.New("missing signature field"))
+	}
+
+	signatureFieldStr, ok := signatureField.(string)
+	if !ok {
+		return common.ContextError(errors.New("invalid signature field"))
+	}
+
+	decodedSignatureField, err := base64.StdEncoding.DecodeString(signatureFieldStr)
+	if err != nil {
+		return common.ContextError(err)
+	}
+
+	if len(decodedSignatureField) < signaturePublicKeyDigestSize {
+		return common.ContextError(errors.New("invalid signature field length"))
+	}
+
+	publicKeyID := decodedSignatureField[:signaturePublicKeyDigestSize]
+	signature := decodedSignatureField[signaturePublicKeyDigestSize:]
+
+	if len(signature) != ed25519.SignatureSize {
+		return common.ContextError(errors.New("invalid signature length"))
+	}
+
+	decodedPublicKey, err := base64.StdEncoding.DecodeString(publicKey)
+	if err != nil {
+		return common.ContextError(err)
+	}
+
+	publicKeyDigest := sha256.Sum256(decodedPublicKey)
+	expectedPublicKeyID := publicKeyDigest[:signaturePublicKeyDigestSize]
+
+	if bytes.Compare(expectedPublicKeyID, publicKeyID) != 0 {
+		return common.ContextError(errors.New("unexpected public key ID"))
+	}
+
+	copyFields.RemoveUnsignedFields()
+
+	delete(copyFields, "signature")
+
+	marshaledFields, err := json.Marshal(copyFields)
+	if err != nil {
+		return common.ContextError(err)
+	}
+
+	if !ed25519.Verify(decodedPublicKey, marshaledFields, signature) {
+		return common.ContextError(errors.New("invalid signature"))
+	}
+
+	return nil
+}
+
+// RemoveUnsignedFields prepares a server entry for signing or signature
+// verification by removing unsigned fields. The JSON marshalling of the
+// remaining fields is the data that is signed.
+func (fields ServerEntryFields) RemoveUnsignedFields() {
+	delete(fields, "localSource")
+	delete(fields, "localTimestamp")
+
+	// Only non-local, explicit tags are part of the signature
+	isLocalDerivedTag, _ := fields["isLocalDerivedTag"]
+	isLocalDerivedTagBool, ok := isLocalDerivedTag.(bool)
+	if ok && isLocalDerivedTagBool {
+		delete(fields, "tag")
+	}
+	delete(fields, "isLocalDerivedTag")
+}
+
+// NewServerEntrySignatureKeyPair creates an ed25519 key pair for use in
+// server entry signing and verification.
+func NewServerEntrySignatureKeyPair() (string, string, error) {
+
+	publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		return "", "", common.ContextError(err)
+	}
+
+	return base64.StdEncoding.EncodeToString(publicKey),
+		base64.StdEncoding.EncodeToString(privateKey),
+		nil
+}
+
 // GetCapability returns the server capability corresponding
 // to the tunnel protocol.
 func GetCapability(protocol string) string {
@@ -263,6 +479,10 @@ func (serverEntry *ServerEntry) GetUntunneledWebRequestPorts() []string {
 	return ports
 }
 
+func (serverEntry *ServerEntry) HasSignature() bool {
+	return serverEntry.Signature != ""
+}
+
 // GenerateServerEntryTag creates a server entry tag value that is
 // cryptographically derived from the IP address and web server secret in a
 // way that is difficult to reverse the IP address value from the tag or
@@ -364,15 +584,35 @@ func decodeServerEntry(
 }
 
 // ValidateServerEntryFields checks for malformed server entries.
-// Currently, it checks for a valid ipAddress. This is important since
-// the IP address is the key used to store/lookup the server entry.
-// TODO: validate more fields?
 func ValidateServerEntryFields(serverEntryFields ServerEntryFields) error {
+
+	// Checks for a valid ipAddress. This is important since the IP
+	// address is the key used to store/lookup the server entry.
+
 	ipAddress := serverEntryFields.GetIPAddress()
 	if net.ParseIP(ipAddress) == nil {
 		return common.ContextError(
 			fmt.Errorf("server entry has invalid ipAddress: %s", ipAddress))
 	}
+
+	// TODO: validate more fields?
+
+	// Ensure locally initialized fields have been set.
+
+	source := serverEntryFields.GetLocalSource()
+	if !common.Contains(
+		SupportedServerEntrySources, source) {
+		return common.ContextError(
+			fmt.Errorf("server entry has invalid source: %s", source))
+	}
+
+	timestamp := serverEntryFields.GetLocalTimestamp()
+	_, err := time.Parse(time.RFC3339, timestamp)
+	if err != nil {
+		return common.ContextError(
+			fmt.Errorf("server entry has invalid timestamp: %s", err))
+	}
+
 	return nil
 }
 

psiphon/common/protocol/serverEntry_test.go

@@ -22,9 +22,12 @@ package protocol
 import (
 	"bytes"
 	"encoding/hex"
+	"encoding/json"
+	"strconv"
 	"testing"
 
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng"
 )
 
 const (
@@ -152,3 +155,143 @@ func TestDecodeServerEntryStruct(t *testing.T) {
 		t.Errorf("unexpected IP address in decoded server entry: %s", serverEntry.IpAddress)
 	}
 }
+
+func TestServerEntryListSignatures(t *testing.T) {
+	testServerEntryListSignatures(t, true)
+	testServerEntryListSignatures(t, false)
+}
+
+func testServerEntryListSignatures(t *testing.T, setExplicitTag bool) {
+
+	publicKey, privateKey, err := NewServerEntrySignatureKeyPair()
+	if err != nil {
+		t.Fatalf("NewServerEntrySignatureKeyPair failed: %s", err)
+	}
+
+	n := 16
+	serverEntry := &ServerEntry{
+		IpAddress:                     prng.HexString(n),
+		WebServerPort:                 strconv.Itoa(prng.Intn(n)),
+		WebServerSecret:               prng.HexString(n),
+		WebServerCertificate:          prng.HexString(n),
+		SshPort:                       prng.Intn(n),
+		SshUsername:                   prng.HexString(n),
+		SshPassword:                   prng.HexString(n),
+		SshHostKey:                    prng.HexString(n),
+		SshObfuscatedPort:             prng.Intn(n),
+		SshObfuscatedQUICPort:         prng.Intn(n),
+		SshObfuscatedTapdancePort:     prng.Intn(n),
+		SshObfuscatedKey:              prng.HexString(n),
+		Capabilities:                  []string{prng.HexString(n)},
+		Region:                        prng.HexString(n),
+		MeekServerPort:                prng.Intn(n),
+		MeekCookieEncryptionPublicKey: prng.HexString(n),
+		MeekObfuscatedKey:             prng.HexString(n),
+		MeekFrontingHost:              prng.HexString(n),
+		MeekFrontingHosts:             []string{prng.HexString(n)},
+		MeekFrontingDomain:            prng.HexString(n),
+		MeekFrontingAddresses:         []string{prng.HexString(n)},
+		MeekFrontingAddressesRegex:    prng.HexString(n),
+		MeekFrontingDisableSNI:        false,
+		TacticsRequestPublicKey:       prng.HexString(n),
+		TacticsRequestObfuscatedKey:   prng.HexString(n),
+		MarionetteFormat:              prng.HexString(n),
+		ConfigurationVersion:          1,
+	}
+
+	if setExplicitTag {
+		serverEntry.Tag = prng.HexString(n)
+	}
+
+	// Convert ServerEntry to ServerEntryFields
+
+	marshaledServerEntry, err := json.Marshal(serverEntry)
+	if err != nil {
+		t.Fatalf("Marshal failed: %s", err)
+	}
+
+	var serverEntryFields ServerEntryFields
+
+	err = json.Unmarshal(marshaledServerEntry, &serverEntryFields)
+	if err != nil {
+		t.Fatalf("Unmarshal failed: %s", err)
+	}
+
+	// Check that local fields are ignored in the signature
+
+	if !setExplicitTag {
+		serverEntryFields.SetTag(prng.HexString(n))
+	}
+	serverEntryFields.SetLocalSource(prng.HexString(n))
+	serverEntryFields.SetLocalTimestamp(prng.HexString(n))
+
+	// Set dummy signature to check that its overwritten
+
+	serverEntryFields["signature"] = prng.HexString(n)
+
+	err = serverEntryFields.AddSignature(publicKey, privateKey)
+	if err != nil {
+		t.Fatalf("AddSignature failed: %s", err)
+	}
+
+	err = serverEntryFields.VerifySignature(publicKey)
+	if err != nil {
+		t.Fatalf("VerifySignature failed: %s", err)
+	}
+
+	// A 2nd VerifySignature call checks that the first VerifySignature
+	// call leaves the server entry fields intact
+
+	err = serverEntryFields.VerifySignature(publicKey)
+	if err != nil {
+		t.Fatalf("VerifySignature failed: %s", err)
+	}
+
+	// Modify local local fields and check that signature remains valid
+
+	if !setExplicitTag {
+		serverEntryFields.SetTag(prng.HexString(n))
+	}
+	serverEntryFields.SetLocalSource(prng.HexString(n))
+	serverEntryFields.SetLocalTimestamp(prng.HexString(n))
+
+	err = serverEntryFields.VerifySignature(publicKey)
+	if err != nil {
+		t.Fatalf("VerifySignature failed: %s", err)
+	}
+
+	// Check that verification fails when using the wrong public key
+
+	incorrectPublicKey, _, err := NewServerEntrySignatureKeyPair()
+	if err != nil {
+		t.Fatalf("NewServerEntrySignatureKeyPair failed: %s", err)
+	}
+
+	err = serverEntryFields.VerifySignature(incorrectPublicKey)
+	if err == nil {
+		t.Fatalf("VerifySignature unexpectedly succeeded")
+	}
+
+	// Check that an expected, non-local field causes verification to fail
+
+	serverEntryFields[prng.HexString(n)] = prng.HexString(n)
+
+	err = serverEntryFields.VerifySignature(publicKey)
+	if err == nil {
+		t.Fatalf("AddSignature unexpectedly succeeded")
+	}
+
+	// Check that modifying a signed field causes verification to fail
+
+	fieldName := "sshObfuscatedKey"
+	if setExplicitTag {
+		fieldName = "tag"
+	}
+
+	serverEntryFields[fieldName] = prng.HexString(n)
+
+	err = serverEntryFields.VerifySignature(publicKey)
+	if err == nil {
+		t.Fatalf("AddSignature unexpectedly succeeded")
+	}
+}

psiphon/config.go

@@ -482,6 +482,17 @@ type Config struct {
 	// server.
 	Authorizations []string
 
+	// ServerEntrySignaturePublicKey is a base64-encoded, ed25519 public
+	// key value used to verify individual server entry signatures. This value
+	// is supplied by and depends on the Psiphon Network, and is typically
+	// embedded in the client binary.
+	ServerEntrySignaturePublicKey string
+
+	// ExchangeObfuscationKey is a base64-encoded, NaCl secretbox key used to
+	// obfuscate server info exchanges between clients.
+	// Required for the exchange functionality.
+	ExchangeObfuscationKey string
+
 	// TransformHostNameProbability is for testing purposes.
 	TransformHostNameProbability *float64
 

psiphon/controller.go

@@ -74,6 +74,7 @@ type Controller struct {
 	signalFetchObfuscatedServerLists        chan struct{}
 	signalDownloadUpgrade                   chan string
 	signalReportConnected                   chan struct{}
+	signalRestartEstablishing               chan struct{}
 	serverAffinityDoneBroadcast             chan struct{}
 	packetTunnelClient                      *tun.Client
 	packetTunnelTransport                   *PacketTunnelTransport
@@ -122,6 +123,10 @@ func NewController(config *Config) (controller *Controller, err error) {
 		signalFetchObfuscatedServerLists:  make(chan struct{}),
 		signalDownloadUpgrade:             make(chan string),
 		signalReportConnected:             make(chan struct{}),
+
+		// signalRestartEstablishing has a buffer of 1 to ensure sending the
+		// signal doesn't block and receiving won't miss a signal.
+		signalRestartEstablishing: make(chan struct{}, 1),
 	}
 
 	controller.splitTunnelClassifier = NewSplitTunnelClassifier(config, controller)
@@ -291,6 +296,45 @@ func (controller *Controller) TerminateNextActiveTunnel() {
 	}
 }
 
+// ExportExchangePayload creates a payload for client-to-client server
+// connection info exchange. See the comment for psiphon.ExportExchangePayload
+// for more details.
+func (controller *Controller) ExportExchangePayload() string {
+	return ExportExchangePayload(controller.config)
+}
+
+// ImportExchangePayload imports a payload generated by ExportExchangePayload.
+// See the comment for psiphon.ImportExchangePayload for more details about
+// the import.
+//
+// When the import is successful, a signal is set to trigger a restart any
+// establishment in progress. This will cause the newly imported server entry
+// to be prioritized, which it otherwise would not be in later establishment
+// rounds. The establishment process continues after ImportExchangePayload
+// returns.
+//
+// If the client already has a connected tunnel, or a tunnel connection is
+// established concurrently with the import, the signal has no effect as the
+// overall goal is establish _any_ connection.
+func (controller *Controller) ImportExchangePayload(payload string) bool {
+
+	// Race condition: if a new tunnel connection is established concurrently
+	// with the import, either that tunnel's server entry of the imported server
+	// entry may end up as the affinity server.
+
+	ok := ImportExchangePayload(controller.config, payload)
+	if !ok {
+		return false
+	}
+
+	select {
+	case controller.signalRestartEstablishing <- *new(struct{}):
+	default:
+	}
+
+	return true
+}
+
 // remoteServerListFetcher fetches an out-of-band list of server entries
 // for more tunnel candidates. It fetches when signalled, with retries
 // on failure.
@@ -582,6 +626,22 @@ func (controller *Controller) runTunnels() {
 loop:
 	for {
 		select {
+
+		case <-controller.signalRestartEstablishing:
+
+			// signalRestartEstablishing restarts any establishment in progress. One
+			// use case for this is to prioritize a newly imported, exchanged server
+			// entry, which will be in the affinity position.
+			//
+			// It's possible for another connection to establish concurrent to signalling;
+			// since the overall goal remains to establish _any_ connection, we accept that
+			// in some cases the exchanged server entry may not get used.
+
+			if controller.isEstablishing {
+				controller.stopEstablishing()
+				controller.startEstablishing()
+			}
+
 		case failedTunnel := <-controller.failedTunnels:
 			NoticeAlert("tunnel failed: %s", failedTunnel.dialParams.ServerEntry.IpAddress)
 			controller.terminateTunnel(failedTunnel)

psiphon/dataStore.go

@@ -145,6 +145,12 @@ func datastoreUpdate(fn func(tx *datastoreTx) error) error {
 // the entry is skipped; no error is returned.
 func StoreServerEntry(serverEntryFields protocol.ServerEntryFields, replaceIfExists bool) error {
 
+	// TODO: call serverEntryFields.VerifySignature. At this time, we do not do
+	// this as not all server entries have an individual signature field. All
+	// StoreServerEntry callers either call VerifySignature or obtain server
+	// entries from a trusted source (embedded in a signed client, or in a signed
+	// authenticated package).
+
 	// Server entries should already be validated before this point,
 	// so instead of skipping we fail with an error.
 	err := protocol.ValidateServerEntryFields(serverEntryFields)
@@ -368,7 +374,7 @@ func hasServerEntryFilterChanged(config *Config) (bool, error) {
 		bucket := tx.bucket(datastoreKeyValueBucket)
 		previousFilter := bucket.get(datastoreLastServerEntryFilterKey)
 
-		// When not found, previousFilter will be nil; ensure this
+		// When not found, previousFilter will be nil; ensures this
 		// results in "changed", even if currentFilter is len(0).
 		if previousFilter == nil ||
 			bytes.Compare(previousFilter, currentFilter) != 0 {
@@ -1631,6 +1637,61 @@ func GetTacticsStorer() *TacticsStorer {
 	return &TacticsStorer{}
 }
 
+// GetAffinityServerEntryAndDialParameters fetches the current affinity server
+// entry value and any corresponding dial parameters for the specified network
+// ID. An error is returned when no affinity server is available. The
+// DialParameter output may be nil when a server entry is found but has no
+// dial parameters.
+func GetAffinityServerEntryAndDialParameters(
+	networkID string) (protocol.ServerEntryFields, *DialParameters, error) {
+
+	var serverEntryFields protocol.ServerEntryFields
+	var dialParams *DialParameters
+
+	err := datastoreView(func(tx *datastoreTx) error {
+
+		keyValues := tx.bucket(datastoreKeyValueBucket)
+		serverEntries := tx.bucket(datastoreServerEntriesBucket)
+		dialParameters := tx.bucket(datastoreDialParametersBucket)
+
+		affinityServerEntryID := keyValues.get(datastoreAffinityServerEntryIDKey)
+		if affinityServerEntryID == nil {
+			return common.ContextError(errors.New("no affinity server available"))
+		}
+
+		serverEntryRecord := serverEntries.get(affinityServerEntryID)
+		if serverEntryRecord == nil {
+			return common.ContextError(errors.New("affinity server entry not found"))
+		}
+
+		err := json.Unmarshal(
+			serverEntryRecord,
+			&serverEntryFields)
+		if err != nil {
+			return common.ContextError(err)
+		}
+
+		dialParamsKey := makeDialParametersKey(
+			[]byte(serverEntryFields.GetIPAddress()),
+			[]byte(networkID))
+
+		dialParamsRecord := dialParameters.get(dialParamsKey)
+		if dialParamsRecord != nil {
+			err := json.Unmarshal(dialParamsRecord, &dialParams)
+			if err != nil {
+				return common.ContextError(err)
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, nil, common.ContextError(err)
+	}
+
+	return serverEntryFields, dialParams, nil
+}
+
 func setBucketValue(bucket, key, value []byte) error {
 
 	err := datastoreUpdate(func(tx *datastoreTx) error {

psiphon/dialParameters.go

@@ -60,6 +60,8 @@ type DialParameters struct {
 	IsReplay        bool                  `json:"-"`
 	CandidateNumber int                   `json:"-"`
 
+	IsExchanged bool
+
 	LastUsedTimestamp       time.Time
 	LastUsedConfigStateHash []byte
 
@@ -217,12 +219,37 @@ func MakeDialParameters(
 		}
 	}
 
+	// IsExchanged:
+	//
+	// Dial parameters received via client-to-client exchange are partially
+	// initialized. Only the exchange fields are retained, and all other dial
+	// parameters fields must be initialized. This is not considered or logged as
+	// a replay. The exchange case is identified by the IsExchanged flag.
+	//
+	// When previously stored, IsExchanged dial parameters will have set the same
+	// timestamp and state hash used for regular dial parameters and the same
+	// logic above should invalidate expired or invalid exchanged dial
+	// parameters.
+	//
+	// Limitation: metrics will indicate when an exchanged server entry is used
+	// (source "EXCHANGED") but will not indicate when exchanged dial parameters
+	// are used vs. a redial after discarding dial parameters.
+
 	isReplay := (dialParams != nil)
+	isExchanged := isReplay && dialParams.IsExchanged
 
 	if !isReplay {
 		dialParams = &DialParameters{}
 	}
 
+	if isExchanged {
+		// Set isReplay to false to cause all non-exchanged values to be
+		// initialized; this also causes the exchange case to not log as replay.
+		isReplay = false
+	}
+
+	// Set IsExchanged so that full dial parameters are stored and replayed upon success.
+	dialParams.IsExchanged = false
 	dialParams.ServerEntry = serverEntry
 	dialParams.NetworkID = networkID
 	dialParams.IsReplay = isReplay
@@ -241,7 +268,7 @@ func MakeDialParameters(
 	// replaying, existing parameters are retaing, subject to the replay-X
 	// tactics flags.
 
-	if !isReplay {
+	if !isReplay && !isExchanged {
 
 		// TODO: should there be a pre-check of selectProtocol before incurring
 		// overhead of unmarshaling dial parameters? In may be that a server entry
@@ -629,6 +656,87 @@ func (dialParams *DialParameters) Failed(config *Config) {
 	}
 }
 
+// ExchangedDialParameters represents the subset of DialParameters that is
+// shared in a client-to-client exchange of server connection info.
+//
+// The purpose of client-to-client exchange if for one user that can connect
+// to help another user that cannot connect by sharing their connected
+// configuration, including the server entry and dial parameters.
+//
+// There are two concerns regarding which dial parameter fields are safe to
+// exchange:
+//
+// - Unlike signed server entries, there's no independent trust anchor
+//   that can certify that the exchange data is valid.
+//
+// - While users should only perform the exchange with trusted peers,
+//   the user's trust in their peer may be misplaced.
+//
+// This presents the possibility of attack such as the peer sending dial
+// parameters that could be used to trace/monitor/flag the importer; or
+// sending dial parameters, including dial address and SNI, to cause the peer
+// to appear to connect to a banned service.
+//
+// To mitigate these risks, only a subset of dial parameters are exchanged.
+// When exchanged dial parameters and imported and used, all unexchanged
+// parameters are generated locally. At this time, only the tunnel protocol is
+// exchanged. We consider tunnel protocol selection one of the key connection
+// success factors.
+//
+// In addition, the exchange peers may not be on the same network with the
+// same blocking and circumvention characteristics, which is another reason
+// to limit exchanged dial parameter values to broadly applicable fields.
+//
+// Unlike the exchanged (and otherwise acquired) server entry,
+// ExchangedDialParameters does not use the ServerEntry_Fields_ representation
+// which allows older clients to receive and store new, unknown fields. Such a
+// facility is less useful in this case, since exchanged dial parameters and
+// used immediately and have a short lifespan.
+//
+// TODO: exchange more dial parameters, such as TLS profile, QUIC version, etc.
+type ExchangedDialParameters struct {
+	TunnelProtocol string
+}
+
+// NewExchangedDialParameters creates a new ExchangedDialParameters from a
+// DialParameters, including only the exchanged values.
+// NewExchangedDialParameters assumes the input DialParameters has been
+// initialized and populated by MakeDialParameters.
+func NewExchangedDialParameters(dialParams *DialParameters) *ExchangedDialParameters {
+	return &ExchangedDialParameters{
+		TunnelProtocol: dialParams.TunnelProtocol,
+	}
+}
+
+// Validate checks that the ExchangedDialParameters contains only valid values
+// and is compatible with the specified server entry.
+func (dialParams *ExchangedDialParameters) Validate(serverEntry *protocol.ServerEntry) error {
+	if !common.Contains(protocol.SupportedTunnelProtocols, dialParams.TunnelProtocol) {
+		return common.ContextError(fmt.Errorf("unknown tunnel protocol: %s", dialParams.TunnelProtocol))
+	}
+	if !serverEntry.SupportsProtocol(dialParams.TunnelProtocol) {
+		return common.ContextError(fmt.Errorf("unsupported tunnel protocol: %s", dialParams.TunnelProtocol))
+	}
+	return nil
+}
+
+// MakeDialParameters creates a new, partially intitialized DialParameters
+// from the values in ExchangedDialParameters. The returned DialParameters
+// must not be used directly for dialing. It is intended to be stored, and
+// then later fully initialized by MakeDialParameters.
+func (dialParams *ExchangedDialParameters) MakeDialParameters(
+	config *Config,
+	p *parameters.ClientParametersSnapshot,
+	serverEntry *protocol.ServerEntry) *DialParameters {
+
+	return &DialParameters{
+		IsExchanged:             true,
+		LastUsedTimestamp:       time.Now(),
+		LastUsedConfigStateHash: getConfigStateHash(config, p, serverEntry),
+		TunnelProtocol:          dialParams.TunnelProtocol,
+	}
+}
+
 func getConfigStateHash(
 	config *Config,
 	p *parameters.ClientParametersSnapshot,

psiphon/dialParameters_test.go

@@ -518,6 +518,8 @@ func makeMockServerEntries(tunnelProtocol string, count int) []*protocol.ServerE
 			MeekServerPort:             5,
 			MeekFrontingHosts:          []string{"www1.example.org", "www2.example.org", "www3.example.org"},
 			MeekFrontingAddressesRegex: "[a-z0-9]{1,64}.example.org",
+			LocalSource:                protocol.SERVER_ENTRY_SOURCE_EMBEDDED,
+			LocalTimestamp:             common.TruncateTimestampToHour(common.GetCurrentTimestamp()),
 		}
 	}
 

psiphon/exchange.go

@@ -0,0 +1,282 @@
+/*
+ * Copyright (c) 2019, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package psiphon
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/crypto/nacl/secretbox"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/protocol"
+)
+
+// ExportExchangePayload creates a payload for client-to-client server
+// connection info exchange. The payload includes the most recent successful
+// server entry -- the server entry in the affinity position -- and any
+// associated dial parameters, for the current network ID.
+//
+// ExportExchangePayload is intended to be called when the client is
+// connected, as the affinity server will be the currently connected server
+// and there will be dial parameters for the current network ID.
+//
+// Only signed server entries will be exchanged. The signature is created by
+// the Psiphon Network and may be verified using the
+// ServerEntrySignaturePublicKey embedded in clients. This signture defends
+// against attacks by rogue clients and man-in-the-middle operatives which
+// could otherwise cause the importer to receive phony server entry values.
+//
+// Only a subset of dial parameters are exchanged. See the comment for
+// ExchangedDialParameters for more details. When no dial parameters is
+// present the exchange proceeds without dial parameters.
+//
+// The exchange payload is obfuscated with the ExchangeObfuscationKey embedded
+// in clients. The purpose of this obfuscation is to ensure that plaintext
+// server entry info cannot be trivially exported and displayed or published;
+// or at least require an effort equal to what's required without the export
+// feature.
+//
+// There is no success notice for exchange ExportExchangePayload (or
+// ImportExchangePayload) as this would potentially leak a user releationship if
+// two users performed and exchange and subseqently submit diagnostic feedback
+// containg import and export logs at almost the same point in time, along
+// with logs showing connections to the same server, with source "EXCHANGED"
+// in the importer case.
+//
+// Failure notices are logged as, presumably, the event will only appear on
+// one end of the exchange and the error is potentially important diagnostics.
+//
+// There remains some risk of user linkability from Connecting/ConnectedServer
+// diagnostics and metrics alone, because the appearance of "EXCHANGED" may
+// indicate an exchange event. But there are various degrees of ambiguity in
+// this case in terms of determining the server entry was freshly exchanged;
+// and with likely many users often connecting to any given server in a short
+// time period.
+//
+// The return value is a payload that may be exchanged with another client;
+// when "", the export failed and a diagnostic notice has been logged.
+func ExportExchangePayload(config *Config) string {
+	payload, err := exportExchangePayload(config)
+	if err != nil {
+		NoticeAlert("ExportExchangePayload failed: %s", common.ContextError(err))
+		return ""
+	}
+	return payload
+}
+
+// ImportExchangePayload imports a payload generated by ExportExchangePayload.
+// The server entry in the payload is promoted to the affinity position so it
+// will be the first candidate in any establishment that begins after the
+// import.
+//
+// The current network ID. This may not be the same network as the exporter,
+// even if the client-to-client exchange occurs in real time. For example, if
+// the exchange is performed over NFC between two devices, they may be on
+// different mobile or WiFi networks. As mentioned in the comment for
+// ExchangedDialParameters, the exchange dial parameters includes only the
+// most broadly applicable fields.
+//
+// The return value indicates a successful import. If the import failed, a
+// a diagnostic notice has been logged.
+func ImportExchangePayload(config *Config, encodedPayload string) bool {
+	err := importExchangePayload(config, encodedPayload)
+	if err != nil {
+		NoticeAlert("ImportExchangePayload failed: %s", common.ContextError(err))
+		return false
+	}
+	return true
+}
+
+type exchangePayload struct {
+	ServerEntryFields       protocol.ServerEntryFields
+	ExchangedDialParameters *ExchangedDialParameters
+}
+
+func exportExchangePayload(config *Config) (string, error) {
+
+	networkID := config.GetNetworkID()
+
+	key, err := getExchangeObfuscationKey(config)
+	if err != nil {
+		return "", common.ContextError(err)
+	}
+
+	serverEntryFields, dialParams, err :=
+		GetAffinityServerEntryAndDialParameters(networkID)
+	if err != nil {
+		return "", common.ContextError(err)
+	}
+
+	// Fail if the server entry has no signature, as the exchange would be
+	// insecure. Given the mechanism where handshake will return a signed server
+	// entry to clients without one, this case is not expected to occur.
+	if !serverEntryFields.HasSignature() {
+		return "", common.ContextError(errors.New("export server entry not signed"))
+	}
+
+	// RemoveUnsignedFields also removes potentially sensitive local fields, so
+	// explicitly strip these before exchanging.
+	serverEntryFields.RemoveUnsignedFields()
+
+	var exchangedDialParameters *ExchangedDialParameters
+	if dialParams != nil {
+		exchangedDialParameters = NewExchangedDialParameters(dialParams)
+	}
+
+	payload := &exchangePayload{
+		ServerEntryFields:       serverEntryFields,
+		ExchangedDialParameters: exchangedDialParameters,
+	}
+
+	payloadJSON, err := json.Marshal(payload)
+	if err != nil {
+		return "", common.ContextError(err)
+	}
+
+	// A unique nonce is generated and included with the payload as the
+	// obfuscation keys is not single-use.
+	nonce, err := common.MakeSecureRandomBytes(24)
+	if err != nil {
+		return "", common.ContextError(err)
+	}
+
+	var secretboxNonce [24]byte
+	copy(secretboxNonce[:], nonce)
+	var secretboxKey [32]byte
+	copy(secretboxKey[:], key)
+	boxedPayload := secretbox.Seal(
+		nil, payloadJSON, &secretboxNonce, &secretboxKey)
+	boxedPayload = append(secretboxNonce[:], boxedPayload...)
+
+	return base64.StdEncoding.EncodeToString(boxedPayload), nil
+}
+
+func importExchangePayload(config *Config, encodedPayload string) error {
+
+	networkID := config.GetNetworkID()
+
+	key, err := getExchangeObfuscationKey(config)
+	if err != nil {
+		return common.ContextError(err)
+	}
+
+	boxedPayload, err := base64.StdEncoding.DecodeString(encodedPayload)
+	if err != nil {
+		return common.ContextError(err)
+	}
+
+	if len(boxedPayload) <= 24 {
+		return common.ContextError(errors.New("unexpected box length"))
+	}
+
+	var secretboxNonce [24]byte
+	copy(secretboxNonce[:], boxedPayload[:24])
+	var secretboxKey [32]byte
+	copy(secretboxKey[:], key)
+	payloadJSON, ok := secretbox.Open(
+		nil, boxedPayload[24:], &secretboxNonce, &secretboxKey)
+	if !ok {
+		return common.ContextError(errors.New("unbox failed"))
+	}
+
+	var payload *exchangePayload
+	err = json.Unmarshal(payloadJSON, &payload)
+	if err != nil {
+		return common.ContextError(err)
+	}
+
+	// Explicitly strip any unsigned fields that should not be exchanged or
+	// imported.
+	payload.ServerEntryFields.RemoveUnsignedFields()
+
+	err = payload.ServerEntryFields.VerifySignature(
+		config.ServerEntrySignaturePublicKey)
+	if err != nil {
+		return common.ContextError(err)
+	}
+
+	payload.ServerEntryFields.SetLocalSource(
+		protocol.SERVER_ENTRY_SOURCE_EXCHANGED)
+	payload.ServerEntryFields.SetLocalTimestamp(
+		common.TruncateTimestampToHour(common.GetCurrentTimestamp()))
+
+	// The following sequence of datastore calls -- StoreServerEntry,
+	// PromoteServerEntry, SetDialParameters -- is not an atomic transaction but
+	// the  datastore will end up in a consistent state in case of failure to
+	// complete the sequence. The existing calls are reused to avoid redundant
+	// code.
+	//
+	// TODO: refactor existing code to allow reuse in a single transaction?
+
+	err = StoreServerEntry(payload.ServerEntryFields, true)
+	if err != nil {
+		return common.ContextError(err)
+	}
+
+	err = PromoteServerEntry(config, payload.ServerEntryFields.GetIPAddress())
+	if err != nil {
+		return common.ContextError(err)
+	}
+
+	if payload.ExchangedDialParameters != nil {
+
+		serverEntry, err := payload.ServerEntryFields.GetServerEntry()
+		if err != nil {
+			return common.ContextError(err)
+		}
+
+		// Don't abort if Validate fails, as the current client may simply not
+		// support the exchanged dial parameter values (for example, a new tunnel
+		// protocol).
+		//
+		// No notice is issued in the error case for the give linkage reason, as the
+		// notice would be a proxy for an import success log.
+
+		err = payload.ExchangedDialParameters.Validate(serverEntry)
+		if err == nil {
+			dialParams := payload.ExchangedDialParameters.MakeDialParameters(
+				config,
+				config.GetClientParametersSnapshot(),
+				serverEntry)
+
+			err = SetDialParameters(
+				payload.ServerEntryFields.GetIPAddress(),
+				networkID,
+				dialParams)
+			if err != nil {
+				return common.ContextError(err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func getExchangeObfuscationKey(config *Config) ([]byte, error) {
+	key, err := base64.StdEncoding.DecodeString(config.ExchangeObfuscationKey)
+	if err != nil {
+		return nil, common.ContextError(err)
+	}
+	if len(key) != 32 {
+		return nil, common.ContextError(errors.New("invalid key size"))
+	}
+	return key, nil
+}

psiphon/exchange_test.go

@@ -0,0 +1,322 @@
+/*
+ * Copyright (c) 2019, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package psiphon
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/protocol"
+)
+
+func TestServerEntryExchange(t *testing.T) {
+
+	// Prepare an empty database
+
+	testDataDirName, err := ioutil.TempDir("", "psiphon-exchange-test")
+	if err != nil {
+		t.Fatalf("TempDir failed: %s", err)
+	}
+	defer os.RemoveAll(testDataDirName)
+
+	SetNoticeWriter(ioutil.Discard)
+
+	// Generate signing and exchange key material
+
+	obfuscationKeyBytes, err := common.MakeSecureRandomBytes(32)
+	if err != nil {
+		t.Fatalf("MakeRandomBytes failed: %s", err)
+	}
+
+	obfuscationKey := base64.StdEncoding.EncodeToString(obfuscationKeyBytes)
+
+	publicKey, privateKey, err := protocol.NewServerEntrySignatureKeyPair()
+	if err != nil {
+		t.Fatalf("NewServerEntrySignatureKeyPair failed: %s", err)
+	}
+
+	// Initialize config required for datastore operation
+
+	networkID := prng.HexString(8)
+
+	configJSONTemplate := `
+		    {
+                "SponsorId" : "0",
+                "PropagationChannelId" : "0",
+		        "DataStoreDirectory" : "%s",
+		        "ServerEntrySignaturePublicKey" : "%s",
+		        "ExchangeObfuscationKey" : "%s",
+		        "NetworkID" : "%s"
+		    }`
+
+	configJSON := fmt.Sprintf(
+		configJSONTemplate,
+		testDataDirName,
+		publicKey,
+		obfuscationKey,
+		networkID)
+
+	config, err := LoadConfig([]byte(configJSON))
+	if err != nil {
+		t.Fatalf("LoadConfig failed: %s", err)
+	}
+	err = config.Commit()
+	if err != nil {
+		t.Fatalf("Commit failed: %s", err)
+	}
+
+	err = OpenDataStore(config)
+	if err != nil {
+		t.Fatalf("OpenDataStore failed: %s", err)
+	}
+	defer CloseDataStore()
+
+	// Generate server entries to test different cases
+	//
+	// Note: invalid signagture cases are exercised in
+	// protocol.TestServerEntryListSignatures
+
+	makeServerEntryFields := func(IPaddress string) protocol.ServerEntryFields {
+		n := 16
+		fields := make(protocol.ServerEntryFields)
+		fields["ipAddress"] = IPaddress
+		fields["sshPort"] = 22
+		fields["sshUsername"] = prng.HexString(n)
+		fields["sshPassword"] = prng.HexString(n)
+		fields["sshHostKey"] = prng.HexString(n)
+		fields["sshObfuscatedPort"] = 23
+		fields["sshObfuscatedQUICPort"] = 24
+		fields["sshObfuscatedKey"] = prng.HexString(n)
+		fields["capabilities"] = []string{"SSH", "OSSH", "QUIC", "ssh-api-requests"}
+		fields["region"] = "US"
+		fields["configurationVersion"] = 1
+		return fields
+	}
+
+	serverEntry0 := makeServerEntryFields("192.168.1.1")
+	tunnelProtocol0 := "SSH"
+
+	serverEntry1 := makeServerEntryFields("192.168.1.2")
+	err = serverEntry1.AddSignature(publicKey, privateKey)
+	if err != nil {
+		t.Fatalf("AddSignature failed: %s", err)
+	}
+	tunnelProtocol1 := "OSSH"
+
+	serverEntry2 := makeServerEntryFields("192.168.1.3")
+	err = serverEntry2.AddSignature(publicKey, privateKey)
+	if err != nil {
+		t.Fatalf("AddSignature failed: %s", err)
+	}
+	tunnelProtocol2 := "QUIC-OSSH"
+
+	serverEntry3 := makeServerEntryFields("192.168.1.4")
+	err = serverEntry3.AddSignature(publicKey, privateKey)
+	if err != nil {
+		t.Fatalf("AddSignature failed: %s", err)
+	}
+	tunnelProtocol3 := ""
+
+	// paveServerEntry stores a server entry in the datastore with source
+	// EMBEDDED, promotes the server entry to the affinity/export candidate
+	// position, and generates and stores associated dial parameters when
+	// specified. This creates potential candidates for export.
+	//
+	// When tunnelProtocol is "", no dial parameters are created.
+
+	paveServerEntry := func(
+		fields protocol.ServerEntryFields, tunnelProtocol string) {
+
+		fields.SetLocalSource(protocol.SERVER_ENTRY_SOURCE_EMBEDDED)
+		fields.SetLocalTimestamp(
+			common.TruncateTimestampToHour(common.GetCurrentTimestamp()))
+
+		err = StoreServerEntry(fields, true)
+		if err != nil {
+			t.Fatalf("StoreServerEntry failed: %s", err)
+		}
+
+		err = PromoteServerEntry(config, fields["ipAddress"].(string))
+		if err != nil {
+			t.Fatalf("PromoteServerEntry failed: %s", err)
+		}
+
+		if tunnelProtocol != "" {
+
+			serverEntry, err := fields.GetServerEntry()
+			if err != nil {
+				t.Fatalf("ServerEntryFields.GetServerEntry failed: %s", err)
+			}
+
+			canReplay := func(serverEntry *protocol.ServerEntry, replayProtocol string) bool {
+				return true
+			}
+
+			selectProtocol := func(serverEntry *protocol.ServerEntry) (string, bool) {
+				return tunnelProtocol, true
+			}
+
+			dialParams, err := MakeDialParameters(
+				config,
+				canReplay,
+				selectProtocol,
+				serverEntry,
+				false,
+				0)
+			if err != nil {
+				t.Fatalf("MakeDialParameters failed: %s", err)
+			}
+
+			err = SetDialParameters(serverEntry.IpAddress, networkID, dialParams)
+			if err != nil {
+				t.Fatalf("SetDialParameters failed: %s", err)
+			}
+		}
+	}
+
+	// checkFirstServerEntry checks that the current affinity server entry has
+	// the expected ID (IP address), and that any associated, stored dial
+	// parameters are in the expected exchanged state. This is used to verify
+	// that an import has succeed and set the datastore correctly.
+
+	checkFirstServerEntry := func(
+		fields protocol.ServerEntryFields, tunnelProtocol string, isExchanged bool) {
+
+		_, iterator, err := NewServerEntryIterator(config)
+		if err != nil {
+			t.Fatalf("NewServerEntryIterator failed: %s", err)
+		}
+		defer iterator.Close()
+
+		serverEntry, err := iterator.Next()
+		if err != nil {
+			t.Fatalf("ServerEntryIterator.Next failed: %s", err)
+		}
+		if serverEntry == nil {
+			t.Fatalf("unexpected nil server entry")
+		}
+
+		if serverEntry.IpAddress != fields["ipAddress"] {
+			t.Fatalf("unexpected server entry IP address")
+		}
+
+		if isExchanged {
+			if serverEntry.LocalSource != protocol.SERVER_ENTRY_SOURCE_EXCHANGED {
+				t.Fatalf("unexpected non-exchanged server entry source")
+			}
+		} else {
+			if serverEntry.LocalSource == protocol.SERVER_ENTRY_SOURCE_EXCHANGED {
+				t.Fatalf("unexpected exchanged server entry source")
+			}
+		}
+
+		dialParams, err := GetDialParameters(serverEntry.IpAddress, networkID)
+		if err != nil {
+			t.Fatalf("GetDialParameters failed: %s", err)
+		}
+
+		if tunnelProtocol == "" {
+			if dialParams != nil {
+				t.Fatalf("unexpected non-nil dial parameters")
+			}
+		} else if isExchanged {
+			if !dialParams.IsExchanged {
+				t.Fatalf("unexpected non-exchanged dial parameters")
+			}
+			if dialParams.TunnelProtocol != tunnelProtocol {
+				t.Fatalf("unexpected exchanged dial parameters tunnel protocol")
+			}
+		} else {
+			if dialParams.IsExchanged {
+				t.Fatalf("unexpected exchanged dial parameters")
+			}
+			if dialParams.TunnelProtocol != tunnelProtocol {
+				t.Fatalf("unexpected dial parameters tunnel protocol")
+			}
+		}
+	}
+
+	// Test: pave only an unsigned server entry; export should fail
+
+	paveServerEntry(serverEntry0, tunnelProtocol0)
+
+	payload := ExportExchangePayload(config)
+	if payload != "" {
+		t.Fatalf("ExportExchangePayload unexpectedly succeeded")
+	}
+
+	// Test: pave two signed server entries; serverEntry2 is the affinity server
+	// entry and should be the exported server entry
+
+	paveServerEntry(serverEntry1, tunnelProtocol1)
+	paveServerEntry(serverEntry2, tunnelProtocol2)
+
+	payload = ExportExchangePayload(config)
+	if payload == "" {
+		t.Fatalf("ExportExchangePayload failed")
+	}
+
+	// Test: import; serverEntry2 should be imported
+
+	// Before importing the exported payload, move serverEntry1 to the affinity
+	// position. After the import, we expect serverEntry2 to be at the affinity
+	// position and its dial parameters to be IsExchanged and and have the
+	// exchanged tunnel protocol.
+
+	err = PromoteServerEntry(config, serverEntry1["ipAddress"].(string))
+	if err != nil {
+		t.Fatalf("PromoteServerEntry failed: %s", err)
+	}
+
+	checkFirstServerEntry(serverEntry1, tunnelProtocol1, false)
+
+	if !ImportExchangePayload(config, payload) {
+		t.Fatalf("ImportExchangePayload failed")
+	}
+
+	checkFirstServerEntry(serverEntry2, tunnelProtocol2, true)
+
+	// Test: nil exchanged dial parameters case
+
+	paveServerEntry(serverEntry3, tunnelProtocol3)
+
+	payload = ExportExchangePayload(config)
+	if payload == "" {
+		t.Fatalf("ExportExchangePayload failed")
+	}
+
+	err = PromoteServerEntry(config, serverEntry1["ipAddress"].(string))
+	if err != nil {
+		t.Fatalf("PromoteServerEntry failed: %s", err)
+	}
+
+	checkFirstServerEntry(serverEntry1, tunnelProtocol1, false)
+
+	if !ImportExchangePayload(config, payload) {
+		t.Fatalf("ImportExchangePayload failed")
+	}
+
+	checkFirstServerEntry(serverEntry3, tunnelProtocol3, true)
+}

psiphon/server/api.go

@@ -175,7 +175,9 @@ func dispatchAPIRequestHandler(
 var handshakeRequestParams = append(
 	append(
 		// Note: legacy clients may not send "session_id" in handshake
-		[]requestParamSpec{{"session_id", isHexDigits, requestParamOptional}},
+		[]requestParamSpec{
+			{"session_id", isHexDigits, requestParamOptional},
+			{"missing_server_entry_signature", isBooleanFlag, requestParamOptional | requestParamLogFlagAsBool}},
 		tacticsParams...),
 	baseRequestParams...)
 
@@ -288,13 +290,25 @@ func handshakeAPIRequestHandler(
 
 	pad_response, _ := getPaddingSizeRequestParam(params, "pad_response")
 
+	encodedServerList := db.DiscoverServers(geoIPData.DiscoveryValue)
+
+	// When the client indicates that it used an unsigned server entry for this
+	// connection, return a signed copy of the server entry for the client to
+	// upgrade to. See also: comment in psiphon.doHandshakeRequest.
+	if getOptionalBooleanFlagRequestParam(params, "missing_server_entry_signature") {
+		ownServerEntry, ok := db.OwnServerEntry()
+		if ok {
+			encodedServerList = append(encodedServerList, ownServerEntry)
+		}
+	}
+
 	handshakeResponse := protocol.HandshakeResponse{
 		SSHSessionID:           sessionID,
 		Homepages:              db.GetRandomizedHomepages(sponsorID, geoIPData.Country, isMobile),
 		UpgradeClientVersion:   db.GetUpgradeClientVersion(clientVersion, normalizedPlatform),
 		PageViewRegexes:        make([]map[string]string, 0),
 		HttpsRequestRegexes:    httpsRequestRegexes,
-		EncodedServerList:      db.DiscoverServers(geoIPData.DiscoveryValue),
+		EncodedServerList:      encodedServerList,
 		ClientRegion:           geoIPData.Country,
 		ServerTimestamp:        common.GetCurrentTimestamp(),
 		ActiveAuthorizationIDs: activeAuthorizationIDs,
@@ -1006,6 +1020,19 @@ func makeSpeedTestSamplesLogField(samples []interface{}) []interface{} {
 	return logSamples
 }
 
+// getOptionalBooleanFlagRequestParam returns true only if the field exists,
+// and is a true flag value, "1". Otherwise, it returns false.
+func getOptionalBooleanFlagRequestParam(params common.APIParameters, name string) bool {
+	if params[name] == nil {
+		return false
+	}
+	value, ok := params[name].(string)
+	if !ok {
+		return false
+	}
+	return value == "1"
+}
+
 func getStringRequestParam(params common.APIParameters, name string) (string, error) {
 	if params[name] == nil {
 		return "", common.ContextError(fmt.Errorf("missing param: %s", name))

psiphon/server/psinet/psinet.go

@@ -24,9 +24,7 @@
 package psinet
 
 import (
-	"encoding/hex"
 	"encoding/json"
-	"fmt"
 	"math"
 	"math/rand"
 	"strconv"
@@ -46,53 +44,19 @@ const (
 type Database struct {
 	common.ReloadableFile
 
-	Hosts                map[string]Host            `json:"hosts"`
-	Servers              []Server                   `json:"servers"`
-	Sponsors             map[string]Sponsor         `json:"sponsors"`
-	Versions             map[string][]ClientVersion `json:"client_versions"`
-	DefaultSponsorID     string                     `json:"default_sponsor_id"`
-	ValidServerEntryTags map[string]bool            `json:"valid_server_entry_tags"`
+	Sponsors              map[string]*Sponsor        `json:"sponsors"`
+	Versions              map[string][]ClientVersion `json:"client_versions"`
+	DefaultSponsorID      string                     `json:"default_sponsor_id"`
+	ValidServerEntryTags  map[string]bool            `json:"valid_server_entry_tags"`
+	OwnEncodedServerEntry string                     `json:"own_encoded_server_entry"`
+	DiscoveryServers      []*DiscoveryServer         `json:"discovery_servers`
 
 	fileModTime time.Time
 }
 
-type Host struct {
-	DatacenterName                string `json:"datacenter_name"`
-	Id                            string `json:"id"`
-	IpAddress                     string `json:"ip_address"`
-	IsTCS                         bool   `json:"is_TCS"`
-	MeekCookieEncryptionPublicKey string `json:"meek_cookie_encryption_public_key"`
-	MeekServerObfuscatedKey       string `json:"meek_server_obfuscated_key"`
-	MeekServerPort                int    `json:"meek_server_port"`
-	TacticsRequestPublicKey       string `json:"tactics_request_public_key"`
-	TacticsRequestObfuscatedKey   string `json:"tactics_request_obfuscated_key"`
-	Region                        string `json:"region"`
-}
-
-type Server struct {
-	AlternateSshObfuscatedPorts []string        `json:"alternate_ssh_obfuscated_ports"`
-	Capabilities                map[string]bool `json:"capabilities"`
-	DiscoveryDateRange          []string        `json:"discovery_date_range"`
-	EgressIpAddress             string          `json:"egress_ip_address"`
-	HostId                      string          `json:"host_id"`
-	Id                          string          `json:"id"`
-	InternalIpAddress           string          `json:"internal_ip_address"`
-	IpAddress                   string          `json:"ip_address"`
-	IsEmbedded                  bool            `json:"is_embedded"`
-	IsPermanent                 bool            `json:"is_permanent"`
-	PropogationChannelId        string          `json:"propagation_channel_id"`
-	SshHostKey                  string          `json:"ssh_host_key"`
-	SshObfuscatedKey            string          `json:"ssh_obfuscated_key"`
-	SshObfuscatedPort           int             `json:"ssh_obfuscated_port"`
-	SshObfuscatedQUICPort       int             `json:"ssh_obfuscated_quic_port"`
-	SshObfuscatedTapdancePort   int             `json:"ssh_obfuscated_tapdance_port"`
-	SshPassword                 string          `json:"ssh_password"`
-	SshPort                     string          `json:"ssh_port"`
-	SshUsername                 string          `json:"ssh_username"`
-	WebServerCertificate        string          `json:"web_server_certificate"`
-	WebServerPort               string          `json:"web_server_port"`
-	WebServerSecret             string          `json:"web_server_secret"`
-	ConfigurationVersion        int             `json:"configuration_version"`
+type DiscoveryServer struct {
+	DiscoveryDateRange []time.Time `json:"discovery_date_range"`
+	EncodedServerEntry string      `json:"encoded_server_entry"`
 }
 
 type Sponsor struct {
@@ -148,12 +112,12 @@ func NewDatabase(filename string) (*Database, error) {
 			}
 			// Note: an unmarshal directly into &database would fail
 			// to reset to zero value fields not present in the JSON.
-			database.Hosts = newDatabase.Hosts
-			database.Servers = newDatabase.Servers
 			database.Sponsors = newDatabase.Sponsors
 			database.Versions = newDatabase.Versions
 			database.DefaultSponsorID = newDatabase.DefaultSponsorID
 			database.ValidServerEntryTags = newDatabase.ValidServerEntryTags
+			database.OwnEncodedServerEntry = newDatabase.OwnEncodedServerEntry
+			database.DiscoveryServers = newDatabase.DiscoveryServers
 			database.fileModTime = fileModTime
 
 			return nil
@@ -292,6 +256,24 @@ func (db *Database) GetHttpsRequestRegexes(sponsorID string) []map[string]string
 	return regexes
 }
 
+// OwnServerEntry returns the server's own server entry. This is returned, in
+// the handshake, to clients that don't yet have a signed copy of this server
+// entry.
+//
+// For purposed of compartmentalization, each server stores only its own
+// server entry, along with the discovery server entries necessary for the
+// discovery feature.
+func (db *Database) OwnServerEntry() (string, bool) {
+	db.ReloadableFile.RLock()
+	defer db.ReloadableFile.RUnlock()
+
+	if db.OwnEncodedServerEntry != "" {
+		return db.OwnEncodedServerEntry, true
+	}
+
+	return "", false
+}
+
 // DiscoverServers selects new encoded server entries to be "discovered" by
 // the client, using the discoveryValue -- a function of the client's IP
 // address -- as the input into the discovery algorithm.
@@ -302,29 +284,18 @@ func (db *Database) DiscoverServers(discoveryValue int) []string {
 	db.ReloadableFile.RLock()
 	defer db.ReloadableFile.RUnlock()
 
-	var servers []Server
+	var servers []*DiscoveryServer
 
 	discoveryDate := time.Now().UTC()
-	candidateServers := make([]Server, 0)
-
-	for _, server := range db.Servers {
-		var start time.Time
-		var end time.Time
-		var err error
+	candidateServers := make([]*DiscoveryServer, 0)
 
+	for _, server := range db.DiscoveryServers {
 		// All servers that are discoverable on this day are eligible for discovery
-		if len(server.DiscoveryDateRange) != 0 {
-			start, err = time.Parse("2006-01-02T15:04:05", server.DiscoveryDateRange[0])
-			if err != nil {
-				continue
-			}
-			end, err = time.Parse("2006-01-02T15:04:05", server.DiscoveryDateRange[1])
-			if err != nil {
-				continue
-			}
-			if discoveryDate.After(start) && discoveryDate.Before(end) {
-				candidateServers = append(candidateServers, server)
-			}
+		if len(server.DiscoveryDateRange) == 2 &&
+			discoveryDate.After(server.DiscoveryDateRange[0]) &&
+			discoveryDate.Before(server.DiscoveryDateRange[1]) {
+
+			candidateServers = append(candidateServers, server)
 		}
 	}
 
@@ -334,7 +305,7 @@ func (db *Database) DiscoverServers(discoveryValue int) []string {
 	encodedServerEntries := make([]string, 0)
 
 	for _, server := range servers {
-		encodedServerEntries = append(encodedServerEntries, db.getEncodedServerEntry(server))
+		encodedServerEntries = append(encodedServerEntries, server.EncodedServerEntry)
 	}
 
 	return encodedServerEntries
@@ -355,7 +326,9 @@ func (db *Database) DiscoverServers(discoveryValue int) []string {
 // both aspects determine which server is selected. IP address is given the
 // priority: if there are only a couple of servers, for example, IP address alone
 // determines the outcome.
-func selectServers(servers []Server, timeInSeconds, discoveryValue int) []Server {
+func selectServers(
+	servers []*DiscoveryServer, timeInSeconds, discoveryValue int) []*DiscoveryServer {
+
 	TIME_GRANULARITY := 3600
 
 	if len(servers) == 0 {
@@ -388,7 +361,7 @@ func selectServers(servers []Server, timeInSeconds, discoveryValue int) []Server
 
 	server := bucket[timeStrategyValue%len(bucket)]
 
-	serverList := make([]Server, 1)
+	serverList := make([]*DiscoveryServer, 1)
 	serverList[0] = server
 
 	return serverList
@@ -401,7 +374,7 @@ func calculateBucketCount(length int) int {
 }
 
 // bucketizeServerList creates nearly equal sized slices of the input list.
-func bucketizeServerList(servers []Server, bucketCount int) [][]Server {
+func bucketizeServerList(servers []*DiscoveryServer, bucketCount int) [][]*DiscoveryServer {
 
 	// This code creates the same partitions as legacy servers:
 	// https://bitbucket.org/psiphon/psiphon-circumvention-system/src/03bc1a7e51e7c85a816e370bb3a6c755fd9c6fee/Automation/psi_ops_discovery.py
@@ -412,7 +385,7 @@ func bucketizeServerList(servers []Server, bucketCount int) [][]Server {
 	// TODO: this partition is constant for fixed Database content, so it could
 	// be done once and cached in the Database ReloadableFile reloadAction.
 
-	buckets := make([][]Server, bucketCount)
+	buckets := make([][]*DiscoveryServer, bucketCount)
 
 	division := float64(len(servers)) / float64(bucketCount)
 
@@ -425,139 +398,6 @@ func bucketizeServerList(servers []Server, bucketCount int) [][]Server {
 	return buckets
 }
 
-// Return hex encoded server entry string for comsumption by client.
-// Newer clients ignore the legacy fields and only utilize the extended (new) config.
-func (db *Database) getEncodedServerEntry(server Server) string {
-
-	host, hostExists := db.Hosts[server.HostId]
-	if !hostExists {
-		return ""
-	}
-
-	// TCS web server certificate has PEM headers and newlines, so strip those now
-	// for legacy format compatibility
-	webServerCertificate := server.WebServerCertificate
-	if host.IsTCS {
-		splitCert := strings.Split(server.WebServerCertificate, "\n")
-		if len(splitCert) <= 2 {
-			webServerCertificate = ""
-		} else {
-			webServerCertificate = strings.Join(splitCert[1:len(splitCert)-2], "")
-		}
-	}
-
-	// Double-check that we're not giving our blank server credentials
-	if len(server.IpAddress) <= 1 || len(server.WebServerPort) <= 1 || len(server.WebServerSecret) <= 1 || len(webServerCertificate) <= 1 {
-		return ""
-	}
-
-	// Extended (new) entry fields are in a JSON string
-	var extendedConfig struct {
-		IpAddress                     string   `json:"ipAddress"`
-		WebServerPort                 string   `json:"webServerPort"` // not an int
-		WebServerSecret               string   `json:"webServerSecret"`
-		WebServerCertificate          string   `json:"webServerCertificate"`
-		SshPort                       int      `json:"sshPort"`
-		SshUsername                   string   `json:"sshUsername"`
-		SshPassword                   string   `json:"sshPassword"`
-		SshHostKey                    string   `json:"sshHostKey"`
-		SshObfuscatedPort             int      `json:"sshObfuscatedPort"`
-		SshObfuscatedQUICPort         int      `json:"sshObfuscatedQUICPort"`
-		SshObfuscatedTapdancePort     int      `json:"sshObfuscatedTapdancePort"`
-		SshObfuscatedKey              string   `json:"sshObfuscatedKey"`
-		Capabilities                  []string `json:"capabilities"`
-		Region                        string   `json:"region"`
-		MeekServerPort                int      `json:"meekServerPort"`
-		MeekCookieEncryptionPublicKey string   `json:"meekCookieEncryptionPublicKey"`
-		MeekObfuscatedKey             string   `json:"meekObfuscatedKey"`
-		TacticsRequestPublicKey       string   `json:"tacticsRequestPublicKey"`
-		TacticsRequestObfuscatedKey   string   `json:"tacticsRequestObfuscatedKey"`
-		ConfigurationVersion          int      `json:"configurationVersion"`
-	}
-
-	// NOTE: also putting original values in extended config for easier parsing by new clients
-	extendedConfig.IpAddress = server.IpAddress
-	extendedConfig.WebServerPort = server.WebServerPort
-	extendedConfig.WebServerSecret = server.WebServerSecret
-	extendedConfig.WebServerCertificate = webServerCertificate
-
-	sshPort, err := strconv.Atoi(server.SshPort)
-	if err != nil {
-		extendedConfig.SshPort = 0
-	} else {
-		extendedConfig.SshPort = sshPort
-	}
-
-	extendedConfig.SshUsername = server.SshUsername
-	extendedConfig.SshPassword = server.SshPassword
-
-	sshHostKeyType, sshHostKey := parseSshKeyString(server.SshHostKey)
-
-	if strings.Compare(sshHostKeyType, "ssh-rsa") == 0 {
-		extendedConfig.SshHostKey = sshHostKey
-	} else {
-		extendedConfig.SshHostKey = ""
-	}
-
-	extendedConfig.SshObfuscatedPort = server.SshObfuscatedPort
-	// Use the latest alternate port unless tunneling through meek
-	if len(server.AlternateSshObfuscatedPorts) > 0 && !server.Capabilities["UNFRONTED-MEEK"] {
-		port, err := strconv.Atoi(server.AlternateSshObfuscatedPorts[len(server.AlternateSshObfuscatedPorts)-1])
-		if err == nil {
-			extendedConfig.SshObfuscatedPort = port
-		}
-	}
-
-	extendedConfig.SshObfuscatedQUICPort = server.SshObfuscatedQUICPort
-	extendedConfig.SshObfuscatedTapdancePort = server.SshObfuscatedTapdancePort
-
-	extendedConfig.SshObfuscatedKey = server.SshObfuscatedKey
-	extendedConfig.Region = host.Region
-	extendedConfig.MeekCookieEncryptionPublicKey = host.MeekCookieEncryptionPublicKey
-	extendedConfig.MeekServerPort = host.MeekServerPort
-	extendedConfig.MeekObfuscatedKey = host.MeekServerObfuscatedKey
-	extendedConfig.TacticsRequestPublicKey = host.TacticsRequestPublicKey
-	extendedConfig.TacticsRequestObfuscatedKey = host.TacticsRequestObfuscatedKey
-
-	serverCapabilities := make(map[string]bool, 0)
-	for capability, enabled := range server.Capabilities {
-		serverCapabilities[capability] = enabled
-	}
-
-	if serverCapabilities["UNFRONTED-MEEK"] && host.MeekServerPort == 443 {
-		serverCapabilities["UNFRONTED-MEEK"] = false
-		serverCapabilities["UNFRONTED-MEEK-HTTPS"] = true
-	}
-
-	for capability, enabled := range serverCapabilities {
-		if enabled == true {
-			extendedConfig.Capabilities = append(extendedConfig.Capabilities, capability)
-		}
-	}
-
-	extendedConfig.ConfigurationVersion = server.ConfigurationVersion
-
-	jsonDump, err := json.Marshal(extendedConfig)
-	if err != nil {
-		return ""
-	}
-
-	// Legacy format + extended (new) config
-	prefixString := fmt.Sprintf("%s %s %s %s ", server.IpAddress, server.WebServerPort, server.WebServerSecret, webServerCertificate)
-
-	return hex.EncodeToString(append([]byte(prefixString)[:], []byte(jsonDump)[:]...))
-}
-
-// Parse string of format "ssh-key-type ssh-key".
-func parseSshKeyString(sshKeyString string) (keyType string, key string) {
-	sshKeyArr := strings.Split(sshKeyString, " ")
-	if len(sshKeyArr) != 2 {
-		return "", ""
-	}
-
-	return sshKeyArr[0], sshKeyArr[1]
-}
-
 // IsValidServerEntryTag checks if the specified server entry tag is valid.
 func (db *Database) IsValidServerEntryTag(serverEntryTag string) bool {
 	db.ReloadableFile.RLock()

psiphon/server/psinet/psinet_test.go

@@ -20,33 +20,33 @@
 package psinet
 
 import (
-	"fmt"
+	"strconv"
 	"testing"
 	"time"
 )
 
 func TestDiscoveryBuckets(t *testing.T) {
 
-	checkBuckets := func(buckets [][]Server, expectedIDs [][]int) {
-		if len(buckets) != len(expectedIDs) {
+	checkBuckets := func(buckets [][]*DiscoveryServer, expectedServerEntries [][]int) {
+		if len(buckets) != len(expectedServerEntries) {
 			t.Errorf(
 				"unexpected bucket count: got %d expected %d",
-				len(buckets), len(expectedIDs))
+				len(buckets), len(expectedServerEntries))
 			return
 		}
 		for i := 0; i < len(buckets); i++ {
-			if len(buckets[i]) != len(expectedIDs[i]) {
+			if len(buckets[i]) != len(expectedServerEntries[i]) {
 				t.Errorf(
 					"unexpected bucket %d size: got %d expected %d",
-					i, len(buckets[i]), len(expectedIDs[i]))
+					i, len(buckets[i]), len(expectedServerEntries[i]))
 				return
 			}
 			for j := 0; j < len(buckets[i]); j++ {
-				expectedID := fmt.Sprintf("%d", expectedIDs[i][j])
-				if buckets[i][j].Id != expectedID {
+				expectedServerEntry := strconv.Itoa(expectedServerEntries[i][j])
+				if buckets[i][j].EncodedServerEntry != expectedServerEntry {
 					t.Errorf(
 						"unexpected bucket %d item %d: got %s expected %s",
-						i, j, buckets[i][j].Id, expectedID)
+						i, j, buckets[i][j].EncodedServerEntry, expectedServerEntry)
 					return
 				}
 			}
@@ -56,9 +56,9 @@ func TestDiscoveryBuckets(t *testing.T) {
 	// Partition test cases from:
 	// http://stackoverflow.com/questions/2659900/python-slicing-a-list-into-n-nearly-equal-length-partitions
 
-	servers := make([]Server, 0)
+	servers := make([]*DiscoveryServer, 0)
 	for i := 0; i < 105; i++ {
-		servers = append(servers, Server{Id: fmt.Sprintf("%d", i)})
+		servers = append(servers, &DiscoveryServer{EncodedServerEntry: strconv.Itoa(i)})
 	}
 
 	t.Run("5 servers, 5 buckets", func(t *testing.T) {
@@ -109,7 +109,7 @@ func TestDiscoveryBuckets(t *testing.T) {
 
 		for i := 0; i < 1000; i++ {
 			for _, server := range selectServers(servers, i*int(time.Hour/time.Second), discoveryValue) {
-				discoveredServers[server.Id] = true
+				discoveredServers[server.EncodedServerEntry] = true
 			}
 		}
 
@@ -125,8 +125,8 @@ func TestDiscoveryBuckets(t *testing.T) {
 		}
 
 		for _, bucketServer := range buckets[0] {
-			if _, ok := discoveredServers[bucketServer.Id]; !ok {
-				t.Errorf("unexpected missing discovery server: %s", bucketServer.Id)
+			if _, ok := discoveredServers[bucketServer.EncodedServerEntry]; !ok {
+				t.Errorf("unexpected missing discovery server: %s", bucketServer.EncodedServerEntry)
 				return
 			}
 		}

psiphon/server/server_test.go

@@ -1862,7 +1862,7 @@ func initializePruneServerEntriesTest(
 	oldTimeStamp := time.Now().Add(-30 * 24 * time.Hour).UTC().Format(time.RFC3339)
 
 	// Test Cases:
-	// - ExplicitTag: server entry includes a tag; vs. generate a derive tag
+	// - ExplicitTag: server entry includes a tag; vs. generate a derived tag
 	// - LocalTimestamp: server entry is sufficiently old to be pruned; vs. not
 	// - PsinetValid: server entry is reported valid by psinet; vs. deleted
 	// - ExpectPrune: prune outcome based on flags above

psiphon/serverApi.go

@@ -116,6 +116,20 @@ func (serverContext *ServerContext) doHandshakeRequest(
 
 	params := serverContext.getBaseAPIParameters()
 
+	// The server will return a signed copy of its own server entry when the
+	// client specifies this 'missing_server_entry_signature' flag.
+	//
+	// The purpose of this mechanism is to rapidly upgrade client local storage
+	// from unsigned to signed server entries, and to ensure that the client has
+	// a signed server entry for its currently connected server as required for
+	// the client-to-client exchange feature.
+	//
+	// The server entry will be included in handshakeResponse.EncodedServerList,
+	// along side discovery servers.
+	if !serverContext.tunnel.dialParams.ServerEntry.HasSignature() {
+		params["missing_server_entry_signature"] = "1"
+	}
+
 	doTactics := !serverContext.tunnel.config.DisableTactics
 
 	networkID := ""
@@ -218,9 +232,6 @@ func (serverContext *ServerContext) doHandshakeRequest(
 		serverEntries = append(serverEntries, serverEntryFields)
 	}
 
-	// The reason we are storing the entire array of server entries at once rather
-	// than one at a time is that some desirable side-effects get triggered by
-	// StoreServerEntries that don't get triggered by StoreServerEntry.
 	err = StoreServerEntries(
 		serverContext.tunnel.config,
 		serverEntries,