Merge branch 'master' into staging-client

MobileLibrary/Android/make.bash

@@ -26,6 +26,9 @@ GOVERSION=$(go version | perl -ne '/go version (.*?) / && print $1')
 #
 # TODO: conditional on PSIPHON_ENABLE_INPROXY build tag?
 
+# 16KB page size alignment for Android compatibility
+export CGO_LDFLAGS="${CGO_LDFLAGS:-} -Wl,-z,max-page-size=16384,-z,common-page-size=16384"
+
 LDFLAGS="\
 -checklinkname=0 \
 -s \
@@ -34,6 +37,7 @@ LDFLAGS="\
 -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.buildRepo=$BUILDREPO \
 -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.buildRev=$BUILDREV \
 -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.goVersion=$GOVERSION \
+-extldflags=-Wl,-z,max-page-size=16384,-z,common-page-size=16384 \
 "
 
 echo -e "${BUILDDATE}\n${BUILDREPO}\n${BUILDREV}\n" > $BUILDINFOFILE

psiphon/common/inproxy/broker.go

@@ -151,6 +151,12 @@ type BrokerConfig struct {
 	// adds server-side enforcement.
 	AllowDomainFrontedDestinations func(common.GeoIPData) bool
 
+	// AllowMatch is a callback which can indicate whether a proxy and client
+	// pair, with the given, respective GeoIP data, is allowed to match
+	// together. Pairs are always allowed to match based on personal
+	// compartment ID.
+	AllowMatch func(common.GeoIPData, common.GeoIPData) bool
+
 	// LookupGeoIP provides GeoIP lookup service.
 	LookupGeoIP LookupGeoIP
 
@@ -209,6 +215,22 @@ type BrokerConfig struct {
 	MaxCompartmentIDs int
 }
 
+// BrokerLoggedEvent is an error type which indicates that the broker has
+// already logged an event recording the underlying error. This may be used
+// by the outer server layer to avoid redundant logs for HandleSessionPacket
+// errors.
+type BrokerLoggedEvent struct {
+	err error
+}
+
+func NewBrokerLoggedEvent(err error) *BrokerLoggedEvent {
+	return &BrokerLoggedEvent{err: err}
+}
+
+func (e *BrokerLoggedEvent) Error() string {
+	return e.err.Error()
+}
+
 // NewBroker initializes a new Broker.
 func NewBroker(config *BrokerConfig) (*Broker, error) {
 
@@ -261,6 +283,8 @@ func NewBroker(config *BrokerConfig) (*Broker, error) {
 			ProxyQualityState: proxyQuality,
 
 			IsLoadLimiting: config.IsLoadLimiting,
+
+			AllowMatch: config.AllowMatch,
 		}),
 
 		proxyQualityState: proxyQuality,
@@ -601,6 +625,9 @@ func (b *Broker) handleProxyAnnounce(
 		logFields.Add(transportLogFields)
 		logFields.Add(matchMetrics.GetMetrics())
 		b.config.Logger.LogMetric(brokerMetricName, logFields)
+		if retErr != nil {
+			retErr = NewBrokerLoggedEvent(retErr)
+		}
 	}()
 
 	announceRequest, err := UnmarshalProxyAnnounceRequest(requestPayload)
@@ -960,6 +987,9 @@ func (b *Broker) handleClientOffer(
 		logFields.Add(transportLogFields)
 		logFields.Add(matchMetrics.GetMetrics())
 		b.config.Logger.LogMetric(brokerMetricName, logFields)
+		if retErr != nil {
+			retErr = NewBrokerLoggedEvent(retErr)
+		}
 	}()
 
 	offerRequest, err := UnmarshalClientOfferRequest(requestPayload)
@@ -1280,6 +1310,9 @@ func (b *Broker) handleProxyAnswer(
 		}
 		logFields.Add(transportLogFields)
 		b.config.Logger.LogMetric(brokerMetricName, logFields)
+		if retErr != nil {
+			retErr = NewBrokerLoggedEvent(retErr)
+		}
 	}()
 
 	answerRequest, err := UnmarshalProxyAnswerRequest(requestPayload)
@@ -1414,6 +1447,9 @@ func (b *Broker) handleServerProxyQuality(
 		}
 		logFields.Add(transportLogFields)
 		b.config.Logger.LogMetric(brokerMetricName, logFields)
+		if retErr != nil {
+			retErr = NewBrokerLoggedEvent(retErr)
+		}
 	}()
 
 	qualityRequest, err := UnmarshalServerProxyQualityRequest(requestPayload)
@@ -1491,6 +1527,9 @@ func (b *Broker) handleClientRelayedPacket(
 		}
 		logFields.Add(transportLogFields)
 		b.config.Logger.LogMetric(brokerMetricName, logFields)
+		if retErr != nil {
+			retErr = NewBrokerLoggedEvent(retErr)
+		}
 	}()
 
 	relayedPacketRequest, err := UnmarshalClientRelayedPacketRequest(requestPayload)

psiphon/common/inproxy/coordinator.go

@@ -60,7 +60,7 @@ func NewRoundTripperFailedError(err error) *RoundTripperFailedError {
 	return &RoundTripperFailedError{err: err}
 }
 
-func (e RoundTripperFailedError) Error() string {
+func (e *RoundTripperFailedError) Error() string {
 	return e.err.Error()
 }
 

psiphon/common/inproxy/inproxy_test.go

@@ -285,6 +285,7 @@ func runTestInproxy(doMustUpgrade bool) error {
 		AllowProxy:                     func(common.GeoIPData) bool { return true },
 		AllowClient:                    func(common.GeoIPData) bool { return true },
 		AllowDomainFrontedDestinations: func(common.GeoIPData) bool { return true },
+		AllowMatch:                     func(common.GeoIPData, common.GeoIPData) bool { return true },
 	}
 
 	broker, err := NewBroker(brokerConfig)

psiphon/common/inproxy/matcher.go

@@ -132,8 +132,11 @@ type MatcherConfig struct {
 	// Proxy quality state.
 	ProxyQualityState *ProxyQualityState
 
-	// Broker process load limit state callback. See Broker.Config.
+	// Broker process load limit state callback. See BrokerConfig.
 	IsLoadLimiting func() bool
+
+	// Proxy/client allow match callback. See BrokerConfig.
+	AllowMatch func(common.GeoIPData, common.GeoIPData) bool
 }
 
 // MatchProperties specifies the compartment, GeoIP, and network topology
@@ -852,20 +855,27 @@ func (m *Matcher) matchOffer(offerEntry *offerEntry) (*announcementEntry, int) {
 			continue
 		}
 
-		// Disallow matching the same country and ASN, except for personal
+		// Disallow matching the same country and ASN, or GeoIP combinations
+		// prohibited by the AllowMatch callback, except for personal
 		// compartment ID matches.
 		//
 		// For common matching, hopping through the same ISP is assumed to
 		// have no circumvention benefit. For personal matching, the user may
 		// wish to hop their their own or their friend's proxy regardless.
 
-		if isCommonCompartments &&
-			!GetAllowCommonASNMatching() &&
-			(offerProperties.GeoIPData.Country ==
-				announcementProperties.GeoIPData.Country &&
-				offerProperties.GeoIPData.ASN ==
-					announcementProperties.GeoIPData.ASN) {
-			continue
+		if isCommonCompartments {
+			if !GetAllowCommonASNMatching() &&
+				(offerProperties.GeoIPData.Country ==
+					announcementProperties.GeoIPData.Country &&
+					offerProperties.GeoIPData.ASN ==
+						announcementProperties.GeoIPData.ASN) {
+				continue
+			}
+			if !m.config.AllowMatch(
+				announcementProperties.GeoIPData,
+				offerProperties.GeoIPData) {
+				continue
+			}
 		}
 
 		// Check if this is a preferred NAT match. Ultimately, a match may be

psiphon/common/inproxy/matcher_test.go

@@ -62,6 +62,8 @@ func runTestMatcher() error {
 			OfferRateLimitInterval: rateLimitInterval,
 
 			ProxyQualityState: NewProxyQuality(),
+
+			AllowMatch: func(common.GeoIPData, common.GeoIPData) bool { return true },
 		})
 	err := m.Start()
 	if err != nil {
@@ -572,6 +574,48 @@ func runTestMatcher() error {
 		return errors.Tracef("unexpected result: %v", err)
 	}
 
+	// Test: AllowMatch disallow
+
+	m.config.AllowMatch = func(proxy common.GeoIPData, client common.GeoIPData) bool {
+		return proxy != geoIPData1.GeoIPData && client != geoIPData2.GeoIPData
+	}
+
+	go proxyFunc(proxyResultChan, proxyIP, compartment1, 10*time.Millisecond, nil, true)
+	go clientFunc(clientResultChan, clientIP, compartment1And3, 10*time.Millisecond)
+
+	err = <-proxyResultChan
+	if err == nil || !strings.HasSuffix(err.Error(), "context deadline exceeded") {
+		return errors.Tracef("unexpected result: %v", err)
+	}
+
+	err = <-clientResultChan
+	if err == nil || !strings.HasSuffix(err.Error(), "context deadline exceeded") {
+		return errors.Tracef("unexpected result: %v", err)
+	}
+
+	// Test: AllowMatch allow
+
+	m.config.AllowMatch = func(proxy common.GeoIPData, client common.GeoIPData) bool {
+		return proxy == geoIPData1.GeoIPData && client == geoIPData2.GeoIPData
+	}
+
+	go proxyFunc(proxyResultChan, proxyIP, compartment1, 10*time.Millisecond, nil, true)
+	go clientFunc(clientResultChan, clientIP, compartment1And3, 10*time.Millisecond)
+
+	err = <-proxyResultChan
+	if err != nil {
+		return errors.Trace(err)
+	}
+
+	err = <-clientResultChan
+	if err != nil {
+		return errors.Trace(err)
+	}
+
+	m.config.AllowMatch = func(proxy common.GeoIPData, client common.GeoIPData) bool {
+		return true
+	}
+
 	// Test: downgrade-compatible protocol version match
 
 	protocolVersion1 := &MatchProperties{
@@ -1120,7 +1164,8 @@ func BenchmarkMatcherQueue(b *testing.B) {
 
 				m = NewMatcher(
 					&MatcherConfig{
-						Logger: newTestLogger(),
+						Logger:     newTestLogger(),
+						AllowMatch: func(common.GeoIPData, common.GeoIPData) bool { return true },
 					})
 
 				for j := 0; j < size; j++ {

psiphon/common/inproxy/obfuscation.go

@@ -223,7 +223,7 @@ func NewDeobfuscationAnomoly(err error) *DeobfuscationAnomoly {
 	return &DeobfuscationAnomoly{err: err}
 }
 
-func (e DeobfuscationAnomoly) Error() string {
+func (e *DeobfuscationAnomoly) Error() string {
 	return e.err.Error()
 }
 

psiphon/common/osl/osl.go

@@ -86,6 +86,10 @@ type Scheme struct {
 	// specified in UTC and must be a multiple of SeedPeriodNanoseconds.
 	Epoch string
 
+	// PaveDataOSLCount indicates how many active OSLs GetPaveData should
+	// return. Must be must be > 0 when using GetPaveData.
+	PaveDataOSLCount int
+
 	// Regions is a list of client country codes this scheme applies to.
 	// If empty, the scheme applies to all regions.
 	Regions []string
@@ -790,10 +794,13 @@ type Registry struct {
 // MD5 is not cryptographically secure and this checksum is not
 // relied upon for OSL verification. MD5 is used for compatibility
 // with out-of-band distribution hosts.
+//
+// OSLFileSpec supports compact CBOR encoding for use in alternative,
+// fileless mechanisms.
 type OSLFileSpec struct {
-	ID        []byte
-	KeyShares *KeyShares
-	MD5Sum    []byte
+	ID        []byte     `cbor:"1,keyasint,omitempty"`
+	KeyShares *KeyShares `cbor:"2,keyasint,omitempty"`
+	MD5Sum    []byte     `cbor:"3,keyasint,omitempty"`
 }
 
 // KeyShares is a tree data structure which describes the
@@ -802,11 +809,14 @@ type OSLFileSpec struct {
 // are required to reconstruct the secret key. The keys for BoxedShares
 // are either SLOKs (referenced by SLOK ID) or random keys that are
 // themselves split as described in child KeyShares.
+//
+// KeyShares supports compact CBOR encoding for use in alternative,
+// fileless mechanisms.
 type KeyShares struct {
-	Threshold   int
-	BoxedShares [][]byte
-	SLOKIDs     [][]byte
-	KeyShares   []*KeyShares
+	Threshold   int          `cbor:"1,keyasint,omitempty"`
+	BoxedShares [][]byte     `cbor:"2,keyasint,omitempty"`
+	SLOKIDs     [][]byte     `cbor:"3,keyasint,omitempty"`
+	KeyShares   []*KeyShares `cbor:"4,keyasint,omitempty"`
 }
 
 type PaveLogInfo struct {
@@ -996,6 +1006,84 @@ func (config *Config) CurrentOSLIDs(schemeIndex int) (map[string]string, error)
 	return OSLIDs, nil
 }
 
+// PaveData is the per-OSL data used by Pave, for use in alternative, fileless
+// mechanisms, such as proof-of-knowledge of keys. PaveData.FileSpec is the
+// OSL FileSpec that would be paved into the registry file, and
+// PaveData.FileKey is the key that would be used to encrypt OSL files.
+type PaveData struct {
+	FileSpec *OSLFileSpec
+	FileKey  []byte
+}
+
+// GetPaveData returns, for each propagation channel ID in the specified
+// scheme, the list of OSL PaveData for the Config.PaveDataOSLCount most
+// recent OSLs from now. GetPaveData is the equivilent of Pave that is for
+// use in alternative, fileless mechanisms, such as proof-of-knowledge of
+// keys
+func (config *Config) GetPaveData(schemeIndex int) (map[string][]*PaveData, error) {
+
+	config.ReloadableFile.RLock()
+	defer config.ReloadableFile.RUnlock()
+
+	if schemeIndex < 0 || schemeIndex >= len(config.Schemes) {
+		return nil, errors.TraceNew("invalid scheme index")
+	}
+
+	scheme := config.Schemes[schemeIndex]
+
+	oslDuration := scheme.GetOSLDuration()
+
+	// Using PaveDataOSLCount, initialize startTime and EndTime values that
+	// are similar to the Pave inputs. As in Pave, logic in the following
+	// loop will align these time values to actual OSL periods.
+
+	if scheme.PaveDataOSLCount < 1 {
+		return nil, errors.TraceNew("invalid OSL count")
+	}
+	endTime := time.Now()
+	startTime := endTime.Add(-time.Duration(scheme.PaveDataOSLCount) * oslDuration)
+	if startTime.Before(scheme.epoch) {
+		startTime = scheme.epoch
+	}
+
+	allPaveData := make(map[string][]*PaveData)
+
+	for _, propagationChannelID := range scheme.PropagationChannelIDs {
+
+		if !common.Contains(scheme.PropagationChannelIDs, propagationChannelID) {
+			return nil, errors.TraceNew("invalid propagationChannelID")
+		}
+
+		var paveData []*PaveData
+
+		oslTime := scheme.epoch
+
+		if !startTime.IsZero() && !startTime.Before(scheme.epoch) {
+			for oslTime.Before(startTime) {
+				oslTime = oslTime.Add(oslDuration)
+			}
+		}
+
+		for !oslTime.After(endTime) {
+
+			firstSLOKTime := oslTime
+			fileKey, fileSpec, err := makeOSLFileSpec(
+				scheme, propagationChannelID, firstSLOKTime)
+			if err != nil {
+				return nil, errors.Trace(err)
+			}
+
+			paveData = append(paveData, &PaveData{FileSpec: fileSpec, FileKey: fileKey})
+
+			oslTime = oslTime.Add(oslDuration)
+		}
+
+		allPaveData[propagationChannelID] = paveData
+	}
+
+	return allPaveData, nil
+}
+
 // makeOSLFileSpec creates an OSL file key, splits it according to the
 // scheme's key splits, and sets the OSL ID as its first SLOK ID. The
 // returned key is used to encrypt the OSL payload and then discarded;
@@ -1487,6 +1575,26 @@ func NewOSLReader(
 		signingPublicKey)
 }
 
+// ReassembleOSLKey returns a reassembled OSL key, for use in alternative,
+// fileless mechanisms, such as proof-of-knowledge of keys.
+func ReassembleOSLKey(
+	fileSpec *OSLFileSpec,
+	lookup SLOKLookup) (bool, []byte, error) {
+
+	ok, fileKey, err := fileSpec.KeyShares.reassembleKey(lookup, true)
+	if err != nil {
+		return false, nil, errors.Trace(err)
+	}
+	if !ok {
+		return false, nil, nil
+	}
+	if len(fileKey) != KEY_LENGTH_BYTES {
+		return false, nil, errors.TraceNew("invalid key length")
+	}
+
+	return true, fileKey, nil
+}
+
 // zeroReader reads an unlimited stream of zeroes.
 type zeroReader struct {
 }

psiphon/common/osl/osl_test.go

@@ -40,6 +40,8 @@ func TestOSL(t *testing.T) {
     {
       "Epoch" : "%s",
 
+      "PaveDataOSLCount" : %d,
+
       "Regions" : ["US", "CA"],
 
       "PropagationChannelIDs" : ["2995DB0C968C59C4F23E87988D9C0D41", "E742C25A6D8BA8C17F37E725FA628569", "B4A780E67695595FA486E9B900EA7335"],
@@ -101,6 +103,8 @@ func TestOSL(t *testing.T) {
     {
       "Epoch" : "%s",
 
+      "PaveDataOSLCount" : %d,
+
       "Regions" : ["US", "CA"],
 
       "PropagationChannelIDs" : ["36F1CF2DF1250BF0C7BA0629CE3DC657", "B4A780E67695595FA486E9B900EA7335"],
@@ -157,11 +161,14 @@ func TestOSL(t *testing.T) {
   ]
 }
 `
+	// Pave sufficient OSLs to cover simulated elapsed time of all test cases.
+	paveOSLCount := 1000
+
 	seedPeriod := 5 * time.Millisecond // "SeedPeriodNanoseconds" : 5000000
 	now := time.Now().UTC()
 	epoch := now.Add(-seedPeriod).Truncate(seedPeriod)
 	epochStr := epoch.Format(time.RFC3339Nano)
-	configJSON := fmt.Sprintf(configJSONTemplate, epochStr, epochStr)
+	configJSON := fmt.Sprintf(configJSONTemplate, epochStr, paveOSLCount, epochStr, paveOSLCount)
 
 	// The first scheme requires sufficient activity within 5/10 5 millisecond
 	// periods and 5/10 50 millisecond longer periods. The second scheme requires
@@ -387,8 +394,7 @@ func TestOSL(t *testing.T) {
 
 	t.Run("pave OSLs", func(t *testing.T) {
 
-		// Pave sufficient OSLs to cover simulated elapsed time of all test cases.
-		endTime := epoch.Add(1000 * seedPeriod)
+		endTime := epoch.Add(time.Duration(paveOSLCount) * seedPeriod)
 
 		// In actual deployment, paved files for each propagation channel ID
 		// are dropped in distinct distribution sites.
@@ -569,6 +575,7 @@ func TestOSL(t *testing.T) {
 			[]int{0},
 			0,
 		},
+
 		{
 			"single split scheme: sufficient SLOKs",
 			singleSplitPropagationChannelID,
@@ -718,6 +725,36 @@ func TestOSL(t *testing.T) {
 			if seededOSLCount != testCase.expectedOSLCount {
 				t.Fatalf("expected %d OSLs got %d", testCase.expectedOSLCount, seededOSLCount)
 			}
+
+			// Test the alternative, file-less API.
+
+			seededOSLCount = 0
+			for schemeIndex := range len(config.Schemes) {
+				schemePaveData, err := config.GetPaveData(schemeIndex)
+				if err != nil {
+					t.Fatalf("GetPaveData failed: %s", err)
+				}
+				propagationChannelPaveData, ok := schemePaveData[testCase.propagationChannelID]
+				if !ok {
+					continue
+				}
+				for _, paveData := range propagationChannelPaveData {
+					ok, reassembledKey, err := ReassembleOSLKey(paveData.FileSpec, lookupSLOKs)
+					if err != nil {
+						t.Fatalf("ReassembleOSLKey failed: %s", err)
+					}
+					if ok {
+						if !bytes.Equal(reassembledKey, paveData.FileKey) {
+							t.Fatalf("unexpected reassembled key")
+						}
+						seededOSLCount++
+					}
+				}
+			}
+
+			if seededOSLCount != testCase.expectedOSLCount {
+				t.Fatalf("expected %d OSLs got %d", testCase.expectedOSLCount, seededOSLCount)
+			}
 		})
 	}
 }

psiphon/common/parameters/parameters.go

@@ -403,6 +403,10 @@ const (
 	InproxyAllowProxy                                  = "InproxyAllowProxy"
 	InproxyAllowClient                                 = "InproxyAllowClient"
 	InproxyAllowDomainFrontedDestinations              = "InproxyAllowDomainFrontedDestinations"
+	InproxyAllowMatchByRegion                          = "InproxyAllowMatchByRegion"
+	InproxyAllowMatchByASN                             = "InproxyAllowMatchByASN"
+	InproxyDisallowMatchByRegion                       = "InproxyDisallowMatchByRegion"
+	InproxyDisallowMatchByASN                          = "InproxyDisallowMatchByASN"
 	InproxyAllBrokerSpecs                              = "InproxyAllBrokerSpecs"
 	InproxyBrokerSpecs                                 = "InproxyBrokerSpecs"
 	InproxyPersonalPairingBrokerSpecs                  = "InproxyPersonalPairingBrokerSpecs"
@@ -986,6 +990,10 @@ var defaultParameters = map[string]struct {
 	InproxyAllowProxy:                                  {value: false},
 	InproxyAllowClient:                                 {value: false, flags: serverSideOnly},
 	InproxyAllowDomainFrontedDestinations:              {value: false, flags: serverSideOnly},
+	InproxyAllowMatchByRegion:                          {value: KeyStrings{}, flags: serverSideOnly},
+	InproxyAllowMatchByASN:                             {value: KeyStrings{}, flags: serverSideOnly},
+	InproxyDisallowMatchByRegion:                       {value: KeyStrings{}, flags: serverSideOnly},
+	InproxyDisallowMatchByASN:                          {value: KeyStrings{}, flags: serverSideOnly},
 	InproxyTunnelProtocolSelectionProbability:          {value: 1.0, minimum: 0.0},
 	InproxyAllBrokerPublicKeys:                         {value: []string{}, flags: serverSideOnly},
 	InproxyAllBrokerSpecs:                              {value: InproxyBrokerSpecsValue{}, flags: serverSideOnly},

psiphon/common/parameters/transferURLs.go

@@ -32,20 +32,20 @@ type TransferURL struct {
 
 	// URL is the location of the resource. This string is slightly obfuscated
 	// with base64 encoding to mitigate trivial binary executable string scanning.
-	URL string
+	URL string `json:",omitempty"`
 
 	// SkipVerify indicates whether to verify HTTPS certificates. In some
 	// circumvention scenarios, verification is not possible. This must
 	// only be set to true when the resource has its own verification mechanism.
 	// Overridden when a FrontingSpec in FrontingSpecs has verification fields
 	// set.
-	SkipVerify bool
+	SkipVerify bool `json:",omitempty"`
 
 	// OnlyAfterAttempts specifies how to schedule this URL when transferring
 	// the same resource (same entity, same ETag) from multiple different
 	// candidate locations. For a value of N, this URL is only a candidate
 	// after N rounds of attempting the transfer to or from other URLs.
-	OnlyAfterAttempts int
+	OnlyAfterAttempts int `json:",omitempty"`
 
 	// B64EncodedPublicKey is a base64-encoded RSA public key to be used for
 	// encrypting the resource, when uploading, or for verifying a signature of
@@ -59,7 +59,7 @@ type TransferURL struct {
 
 	// FrontingSpecs is an optional set of domain fronting configurations to
 	// apply to any requests made to the destination.
-	FrontingSpecs FrontingSpecs
+	FrontingSpecs FrontingSpecs `json:",omitempty"`
 }
 
 // TransferURLs is a list of transfer URLs.

psiphon/common/protocol/protocol.go

@@ -819,6 +819,7 @@ type SSHPasswordPayload struct {
 	SessionId          string   `json:"SessionId"`
 	SshPassword        string   `json:"SshPassword"`
 	ClientCapabilities []string `json:"ClientCapabilities"`
+	SponsorID          string   `json:"SponsorId"`
 }
 
 type MeekCookieData struct {

psiphon/common/tactics/tactics.go

@@ -439,11 +439,18 @@ func GenerateKeys() (encodedRequestPublicKey, encodedRequestPrivateKey, encodedO
 // The logger and logFieldFormatter callbacks are used to log errors and
 // metrics. The apiParameterValidator callback is used to validate client
 // API parameters submitted to the tactics request.
+//
+// The optional requestPublicKey, requestPrivateKey, and requestObfuscatedKey
+// base64 encoded string parameters may be used to specify and override the
+// corresponding Server config values.
 func NewServer(
 	logger common.Logger,
 	logFieldFormatter common.APIParameterLogFieldFormatter,
 	apiParameterValidator common.APIParameterValidator,
-	configFilename string) (*Server, error) {
+	configFilename string,
+	requestPublicKey string,
+	requestPrivateKey string,
+	requestObfuscatedKey string) (*Server, error) {
 
 	server := &Server{
 		logger:                logger,
@@ -464,6 +471,30 @@ func NewServer(
 				return errors.Trace(err)
 			}
 
+			if requestPublicKey != "" {
+				newServer.RequestPublicKey, err =
+					base64.StdEncoding.DecodeString(requestPublicKey)
+				if err != nil {
+					return errors.Trace(err)
+				}
+			}
+
+			if requestPrivateKey != "" {
+				newServer.RequestPrivateKey, err =
+					base64.StdEncoding.DecodeString(requestPrivateKey)
+				if err != nil {
+					return errors.Trace(err)
+				}
+			}
+
+			if requestObfuscatedKey != "" {
+				newServer.RequestObfuscatedKey, err =
+					base64.StdEncoding.DecodeString(requestObfuscatedKey)
+				if err != nil {
+					return errors.Trace(err)
+				}
+			}
+
 			err = newServer.Validate()
 			if err != nil {
 				return errors.Trace(err)

psiphon/common/tactics/tactics_test.go

@@ -202,7 +202,10 @@ func TestTactics(t *testing.T) {
 		logger,
 		formatter,
 		validator,
-		configFileName)
+		configFileName,
+		"",
+		"",
+		"")
 	if err != nil {
 		t.Fatalf("NewServer failed: %s", err)
 	}
@@ -800,17 +803,16 @@ func TestTacticsFilterGeoIPScope(t *testing.T) {
 		t.Fatalf("GenerateKeys failed: %s", err)
 	}
 
-	tacticsConfigTemplate := fmt.Sprintf(`
+	// Exercise specifying keys in NewServer instead of config file.
+
+	tacticsConfigTemplate := `
     {
-      "RequestPublicKey" : "%s",
-      "RequestPrivateKey" : "%s",
-      "RequestObfuscatedKey" : "%s",
       "DefaultTactics" : {
         "TTL" : "60s"
       },
-      %%s
+      %s
     }
-    `, encodedRequestPublicKey, encodedRequestPrivateKey, encodedObfuscatedKey)
+    `
 
 	// Test: region-only scope
 
@@ -848,7 +850,10 @@ func TestTacticsFilterGeoIPScope(t *testing.T) {
 		nil,
 		nil,
 		nil,
-		configFileName)
+		configFileName,
+		encodedRequestPublicKey,
+		encodedRequestPrivateKey,
+		encodedObfuscatedKey)
 	if err != nil {
 		t.Fatalf("NewServer failed: %s", err)
 	}

psiphon/common/tun/tun_linux.go

@@ -217,15 +217,14 @@ func configureServerInterface(
 		return errors.Tracef("failed to get interface %s: %w", tunDeviceName, err)
 	}
 
-	_, ipv4Net, err := net.ParseCIDR(serverIPv4AddressCIDR)
+	ipv4Addr, err := netlink.ParseAddr(serverIPv4AddressCIDR)
 	if err != nil {
 		return errors.Tracef("failed to parse server IPv4 address: %s: %w", serverIPv4AddressCIDR, err)
 	}
 
-	ipv4Addr := &netlink.Addr{IPNet: ipv4Net}
 	err = netlink.AddrAdd(link, ipv4Addr)
 	if err != nil {
-		return errors.Tracef("failed to add IPv4 address to interface: %s: %w", ipv4Net.String(), err)
+		return errors.Tracef("failed to add IPv4 address to interface: %s: %w", ipv4Addr.String(), err)
 	}
 
 	err = netlink.LinkSetMTU(link, getMTU(config.MTU))
@@ -238,14 +237,13 @@ func configureServerInterface(
 		return errors.Tracef("failed to set interface up: %w", err)
 	}
 
-	_, ipv6Net, err := net.ParseCIDR(serverIPv6AddressCIDR)
+	ipv6Addr, err := netlink.ParseAddr(serverIPv6AddressCIDR)
 	if err != nil {
-		err = errors.Tracef("failed to parse server IPv6 address: %s: %w", serverIPv4AddressCIDR, err)
+		err = errors.Tracef("failed to parse server IPv6 address: %s: %w", serverIPv6AddressCIDR, err)
 	} else {
-		ipv6Addr := &netlink.Addr{IPNet: ipv6Net}
 		err = netlink.AddrAdd(link, ipv6Addr)
 		if err != nil {
-			err = errors.Tracef("failed to add IPv6 address to interface: %s: %w", ipv6Net.String(), err)
+			err = errors.Tracef("failed to add IPv6 address to interface: %s: %w", ipv6Addr.String(), err)
 		}
 	}
 
@@ -334,32 +332,33 @@ func configureClientInterface(
 	// Set tun device network addresses and MTU
 	link, err := netlink.LinkByName(tunDeviceName)
 	if err != nil {
-		return errors.Trace(fmt.Errorf("failed to get interface %s: %w", tunDeviceName, err))
+		return errors.Tracef("failed to get interface %s: %w", tunDeviceName, err)
 	}
 
-	_, ipv4Net, err := net.ParseCIDR(config.IPv4AddressCIDR)
+	ipv4Addr, err := netlink.ParseAddr(config.IPv4AddressCIDR)
 	if err != nil {
 		return errors.Trace(err)
 	}
 
-	ipv4Addr := &netlink.Addr{IPNet: ipv4Net}
-	if err := netlink.AddrAdd(link, ipv4Addr); err != nil {
+	err = netlink.AddrAdd(link, ipv4Addr)
+	if err != nil {
 		return errors.Trace(err)
 	}
 
-	if err := netlink.LinkSetMTU(link, getMTU(config.MTU)); err != nil {
+	err = netlink.LinkSetMTU(link, getMTU(config.MTU))
+	if err != nil {
 		return errors.Trace(err)
 	}
 
-	if err := netlink.LinkSetUp(link); err != nil {
+	err = netlink.LinkSetUp(link)
+	if err != nil {
 		return errors.Trace(err)
 	}
 
-	_, ipv6Net, err := net.ParseCIDR(config.IPv6AddressCIDR)
+	ipv6Addr, err := netlink.ParseAddr(config.IPv6AddressCIDR)
 	if err != nil {
 		err = errors.Trace(err)
 	} else {
-		ipv6Addr := &netlink.Addr{IPNet: ipv6Net}
 		err = netlink.AddrAdd(link, ipv6Addr)
 		if err != nil {
 			err = errors.Trace(err)

psiphon/server/api.go

@@ -45,6 +45,8 @@ const (
 	CLIENT_PLATFORM_ANDROID = "Android"
 	CLIENT_PLATFORM_WINDOWS = "Windows"
 	CLIENT_PLATFORM_IOS     = "iOS"
+
+	SPONSOR_ID_LENGTH = 16
 )
 
 // sshAPIRequestHandler routes Psiphon API requests transported as
@@ -1101,7 +1103,7 @@ const (
 // requests and log events.
 var baseParams = []requestParamSpec{
 	{"propagation_channel_id", isHexDigits, 0},
-	{"sponsor_id", isHexDigits, 0},
+	{"sponsor_id", isSponsorID, 0},
 	{"client_version", isIntString, requestParamLogStringAsInt},
 	{"client_platform", isClientPlatform, 0},
 	{"client_features", isAnyString, requestParamOptional | requestParamArray},
@@ -1750,6 +1752,10 @@ func isMobileClientPlatform(clientPlatform string) bool {
 
 // Input validators follow the legacy validations rules in psi_web.
 
+func isSponsorID(config *Config, value string) bool {
+	return len(value) == SPONSOR_ID_LENGTH && isHexDigits(config, value)
+}
+
 func isHexDigits(_ *Config, value string) bool {
 	// Allows both uppercase in addition to lowercase, for legacy support.
 	return -1 == strings.IndexFunc(value, func(c rune) bool {

psiphon/server/config.go

@@ -75,11 +75,11 @@ type Config struct {
 	// panic, fatal, error, warn, info, debug
 	//
 	// Some debug logs can contain user traffic destination address information.
-	LogLevel string
+	LogLevel string `json:",omitempty"`
 
 	// LogFilename specifies the path of the file to log
 	// to. When blank, logs are written to stderr.
-	LogFilename string
+	LogFilename string `json:",omitempty"`
 
 	// LogFileReopenRetries specifies how many retries, each with a 1ms delay,
 	// will be attempted after reopening a rotated log file fails. Retries
@@ -87,7 +87,7 @@ type Config struct {
 	// performed by external log managers, such as logrotate.
 	//
 	// When omitted, DEFAULT_LOG_FILE_REOPEN_RETRIES is used.
-	LogFileReopenRetries *int
+	LogFileReopenRetries *int `json:",omitempty"`
 
 	// LogFileCreateMode specifies that the Psiphon server should create a new
 	// log file when one is not found, such as after rotation with logrotate
@@ -95,20 +95,20 @@ type Config struct {
 	// creating the file.
 	//
 	// When omitted, the Psiphon server does not create log files.
-	LogFileCreateMode *int
+	LogFileCreateMode *int `json:",omitempty"`
 
 	// When LogDNSServerLoadMetrics is true, server_load logs will include a
 	// break down of DNS request counts, failure rates, etc. per DNS server.
 	// Otherwise, only the overall DNS metrics are logged.
-	LogDNSServerLoadMetrics bool
+	LogDNSServerLoadMetrics bool `json:",omitempty"`
 
 	// SkipPanickingLogWriter disables panicking when
 	// unable to write any logs.
-	SkipPanickingLogWriter bool
+	SkipPanickingLogWriter bool `json:",omitempty"`
 
 	// DiscoveryValueHMACKey is the network-wide secret value
 	// used to determine a unique discovery strategy.
-	DiscoveryValueHMACKey string
+	DiscoveryValueHMACKey string `json:",omitempty"`
 
 	// GeoIPDatabaseFilenames are paths of GeoIP2/GeoLite2
 	// MaxMind database files. When empty, no GeoIP lookups are
@@ -116,21 +116,21 @@ type Config struct {
 	// logged fields: country code, city, and ISP. Multiple
 	// file support accommodates the MaxMind distribution where
 	// ISP data in a separate file.
-	GeoIPDatabaseFilenames []string
+	GeoIPDatabaseFilenames []string `json:",omitempty"`
 
 	// PsinetDatabaseFilename is the path of the file containing
 	// psinet.Database data.
-	PsinetDatabaseFilename string
+	PsinetDatabaseFilename string `json:",omitempty"`
 
 	// HostID identifies the server host; this value is included with all logs.
-	HostID string
+	HostID string `json:",omitempty"`
 
 	// HostProvider identifies the server host provider; this value is
 	// included with all logs and logged only when not blank.
-	HostProvider string
+	HostProvider string `json:",omitempty"`
 
 	// ServerIPAddress is the public IP address of the server.
-	ServerIPAddress string
+	ServerIPAddress string `json:",omitempty"`
 
 	// TunnelProtocolPorts specifies which tunnel protocols to run
 	// and which ports to listen on for each protocol. Valid tunnel
@@ -139,7 +139,7 @@ type Config struct {
 	// "UNFRONTED-MEEK-SESSION-TICKET-OSSH", "FRONTED-MEEK-OSSH",
 	// "FRONTED-MEEK-QUIC-OSSH", "FRONTED-MEEK-HTTP-OSSH", "QUIC-OSSH",
 	// "TAPDANCE-OSSH", "CONJURE-OSSH", and "SHADOWSOCKS-OSSH".
-	TunnelProtocolPorts map[string]int
+	TunnelProtocolPorts map[string]int `json:",omitempty"`
 
 	// TunnelProtocolPassthroughAddresses specifies passthrough addresses to be
 	// used for tunnel protocols configured in TunnelProtocolPorts. Passthrough
@@ -149,61 +149,61 @@ type Config struct {
 	// TunnelProtocolPassthroughAddresses is supported for:
 	// "TLS-OSSH", "UNFRONTED-MEEK-HTTPS-OSSH",
 	// "UNFRONTED-MEEK-SESSION-TICKET-OSSH", "UNFRONTED-MEEK-OSSH".
-	TunnelProtocolPassthroughAddresses map[string]string
+	TunnelProtocolPassthroughAddresses map[string]string `json:",omitempty"`
 
 	// LegacyPassthrough indicates whether to expect legacy passthrough messages
 	// from clients attempting to connect. This should be set for existing/legacy
 	// passthrough servers only.
-	LegacyPassthrough bool
+	LegacyPassthrough bool `json:",omitempty"`
 
 	// EnableGQUIC indicates whether to enable legacy gQUIC QUIC-OSSH
 	// versions, for backwards compatibility with all versions used by older
 	// clients. Enabling gQUIC degrades the anti-probing stance of QUIC-OSSH,
 	// as the legacy gQUIC stack will respond to probing packets.
-	EnableGQUIC bool
+	EnableGQUIC bool `json:",omitempty"`
 
 	// SSHPrivateKey is the SSH host key. The same key is used for
 	// all protocols, run by this server instance, which use SSH.
-	SSHPrivateKey string
+	SSHPrivateKey string `json:",omitempty"`
 
 	// SSHServerVersion is the server version presented in the
 	// identification string. The same value is used for all
 	// protocols, run by this server instance, which use SSH.
-	SSHServerVersion string
+	SSHServerVersion string `json:",omitempty"`
 
 	// SSHUserName is the SSH user name to be presented by the
 	// the tunnel-core client. The same value is used for all
 	// protocols, run by this server instance, which use SSH.
-	SSHUserName string
+	SSHUserName string `json:",omitempty"`
 
 	// SSHPassword is the SSH password to be presented by the
 	// the tunnel-core client. The same value is used for all
 	// protocols, run by this server instance, which use SSH.
-	SSHPassword string
+	SSHPassword string `json:",omitempty"`
 
 	// SSHBeginHandshakeTimeoutMilliseconds specifies the timeout
 	// for clients queueing to begin an SSH handshake. The default
 	// is SSH_BEGIN_HANDSHAKE_TIMEOUT.
-	SSHBeginHandshakeTimeoutMilliseconds *int
+	SSHBeginHandshakeTimeoutMilliseconds *int `json:",omitempty"`
 
 	// SSHHandshakeTimeoutMilliseconds specifies the timeout
 	// before which a client must complete its handshake. The default
 	// is SSH_HANDSHAKE_TIMEOUT.
-	SSHHandshakeTimeoutMilliseconds *int
+	SSHHandshakeTimeoutMilliseconds *int `json:",omitempty"`
 
 	// ObfuscatedSSHKey is the secret key for use in the Obfuscated
 	// SSH protocol. The same secret key is used for all protocols,
 	// run by this server instance, which use Obfuscated SSH.
-	ObfuscatedSSHKey string
+	ObfuscatedSSHKey string `json:",omitempty"`
 
 	// ShadowsocksKey is the secret key for use in the Shadowsocks
 	// protocol.
-	ShadowsocksKey string
+	ShadowsocksKey string `json:",omitempty"`
 
 	// MeekCookieEncryptionPrivateKey is the NaCl private key used
 	// to decrypt meek cookie payload sent from clients. The same
 	// key is used for all meek protocols run by this server instance.
-	MeekCookieEncryptionPrivateKey string
+	MeekCookieEncryptionPrivateKey string `json:",omitempty"`
 
 	// MeekObfuscatedKey is the secret key used for obfuscating
 	// meek cookies sent from clients. The same key is used for all
@@ -214,27 +214,27 @@ type Config struct {
 	// passthrough capability, to connect with TLS-OSSH to the servers
 	// corresponding to those server entries, which now support TLS-OSSH by
 	// demultiplexing meek-https and TLS-OSSH over the meek-https port.
-	MeekObfuscatedKey string
+	MeekObfuscatedKey string `json:",omitempty"`
 
 	// MeekProhibitedHeaders is a list of HTTP headers to check for
 	// in client requests. If one of these headers is found, the
 	// request fails. This is used to defend against abuse.
-	MeekProhibitedHeaders []string
+	MeekProhibitedHeaders []string `json:",omitempty"`
 
 	// MeekRequiredHeaders is a list of HTTP header names and values that must
 	// appear in requests. This is used to defend against abuse.
-	MeekRequiredHeaders map[string]string
+	MeekRequiredHeaders map[string]string `json:",omitempty"`
 
 	// MeekServerCertificate specifies an optional certificate to use for meek
 	// servers, in place of the default, randomly generate certificate. When
 	// specified, the corresponding private key must be supplied in
 	// MeekServerPrivateKey. Any specified certificate is used for all meek
 	// listeners.
-	MeekServerCertificate string
+	MeekServerCertificate string `json:",omitempty"`
 
 	// MeekServerPrivateKey is the private key corresponding to the optional
 	// MeekServerCertificate parameter.
-	MeekServerPrivateKey string
+	MeekServerPrivateKey string `json:",omitempty"`
 
 	// MeekProxyForwardedForHeaders is a list of HTTP headers which
 	// may be added by downstream HTTP proxies or CDNs in front
@@ -247,37 +247,37 @@ type Config struct {
 	// the header if any one is present and the value is a valid
 	// IP address; otherwise the direct connection remote address is
 	// used as the client IP.
-	MeekProxyForwardedForHeaders []string
+	MeekProxyForwardedForHeaders []string `json:",omitempty"`
 
 	// MeekTurnAroundTimeoutMilliseconds specifies the amount of time meek will
 	// wait for downstream bytes before responding to a request. The default is
 	// MEEK_DEFAULT_TURN_AROUND_TIMEOUT.
-	MeekTurnAroundTimeoutMilliseconds *int
+	MeekTurnAroundTimeoutMilliseconds *int `json:",omitempty"`
 
 	// MeekExtendedTurnAroundTimeoutMilliseconds specifies the extended amount of
 	// time meek will wait for downstream bytes, as long as bytes arrive every
 	// MeekTurnAroundTimeoutMilliseconds, before responding to a request. The
 	// default is MEEK_DEFAULT_EXTENDED_TURN_AROUND_TIMEOUT.
-	MeekExtendedTurnAroundTimeoutMilliseconds *int
+	MeekExtendedTurnAroundTimeoutMilliseconds *int `json:",omitempty"`
 
 	// MeekSkipExtendedTurnAroundThresholdBytes specifies when to skip the
 	// extended turn around. When the number of bytes received in the client
 	// request meets the threshold, optimize for upstream flows with quicker
 	// round trip turn arounds.
-	MeekSkipExtendedTurnAroundThresholdBytes *int
+	MeekSkipExtendedTurnAroundThresholdBytes *int `json:",omitempty"`
 
 	// MeekMaxSessionStalenessMilliseconds specifies the TTL for meek sessions.
 	// The default is MEEK_DEFAULT_MAX_SESSION_STALENESS.
-	MeekMaxSessionStalenessMilliseconds *int
+	MeekMaxSessionStalenessMilliseconds *int `json:",omitempty"`
 
 	// MeekHTTPClientIOTimeoutMilliseconds specifies meek HTTP server I/O
 	// timeouts. The default is MEEK_DEFAULT_HTTP_CLIENT_IO_TIMEOUT.
-	MeekHTTPClientIOTimeoutMilliseconds *int
+	MeekHTTPClientIOTimeoutMilliseconds *int `json:",omitempty"`
 
 	// MeekFrontedHTTPClientIOTimeoutMilliseconds specifies meek HTTP server
 	// I/O timeouts for fronted protocols. The default is
 	// MEEK_DEFAULT_FRONTED_HTTP_CLIENT_IO_TIMEOUT.
-	MeekFrontedHTTPClientIOTimeoutMilliseconds *int
+	MeekFrontedHTTPClientIOTimeoutMilliseconds *int `json:",omitempty"`
 
 	// MeekCachedResponseBufferSize is the size of a private,
 	// fixed-size buffer allocated for every meek client. The buffer
@@ -290,7 +290,7 @@ type Config struct {
 	// response payload is a function of client activity, network
 	// throughput and throttling.
 	// A default of 64K is used when MeekCachedResponseBufferSize is 0.
-	MeekCachedResponseBufferSize int
+	MeekCachedResponseBufferSize int `json:",omitempty"`
 
 	// MeekCachedResponsePoolBufferSize is the size of a fixed-size,
 	// shared buffer used to temporarily extend a private buffer when
@@ -299,7 +299,7 @@ type Config struct {
 	// without allocating large buffers for all clients.
 	// A default of 64K is used when MeekCachedResponsePoolBufferSize
 	// is 0.
-	MeekCachedResponsePoolBufferSize int
+	MeekCachedResponsePoolBufferSize int `json:",omitempty"`
 
 	// MeekCachedResponsePoolBufferCount is the number of shared
 	// buffers. Shared buffers are allocated on first use and remain
@@ -307,12 +307,12 @@ type Config struct {
 	// overhead of this facility.
 	// A default of 2048 is used when MeekCachedResponsePoolBufferCount
 	// is 0.
-	MeekCachedResponsePoolBufferCount int
+	MeekCachedResponsePoolBufferCount int `json:",omitempty"`
 
 	// MeekCachedResponsePoolBufferClientLimit is the maximum number of of
 	// shared buffers a single client may consume at once. A default of 32 is
 	// used when MeekCachedResponsePoolBufferClientLimit is 0.
-	MeekCachedResponsePoolBufferClientLimit int
+	MeekCachedResponsePoolBufferClientLimit int `json:",omitempty"`
 
 	// UDPInterceptUdpgwServerAddress specifies the network address of
 	// a udpgw server which clients may be port forwarding to. When
@@ -325,76 +325,76 @@ type Config struct {
 	// validated against SSH_DISALLOWED_PORT_FORWARD_HOSTS and
 	// AllowTCPPorts. So the intercept address may be any otherwise
 	// prohibited destination.
-	UDPInterceptUdpgwServerAddress string
+	UDPInterceptUdpgwServerAddress string `json:",omitempty"`
 
 	// DNSResolverIPAddress specifies the IP address of a DNS server
 	// to be used when "/etc/resolv.conf" doesn't exist or fails to
 	// parse. When blank, "/etc/resolv.conf" must contain a usable
 	// "nameserver" entry.
-	DNSResolverIPAddress string
+	DNSResolverIPAddress string `json:",omitempty"`
 
 	// LoadMonitorPeriodSeconds indicates how frequently to log server
 	// load information (number of connected clients per tunnel protocol,
 	// number of running goroutines, amount of memory allocated, etc.)
 	// The default, 0, disables load logging.
-	LoadMonitorPeriodSeconds int
+	LoadMonitorPeriodSeconds int `json:",omitempty"`
 
 	// PeakUpstreamFailureRateMinimumSampleSize specifies the minimum number
 	// of samples (e.g., upstream port forward attempts) that are required
 	// before taking a failure rate snapshot which may be recorded as
 	// peak_dns_failure_rate/peak_tcp_port_forward_failure_rate.  The default
 	// is PEAK_UPSTREAM_FAILURE_RATE_SAMPLE_SIZE.
-	PeakUpstreamFailureRateMinimumSampleSize *int
+	PeakUpstreamFailureRateMinimumSampleSize *int `json:",omitempty"`
 
 	// ProcessProfileOutputDirectory is the path of a directory to which
 	// process profiles will be written when signaled with SIGUSR2. The
 	// files are overwritten on each invocation. When set to the default
 	// value, blank, no profiles are written on SIGUSR2. Profiles include
 	// the default profiles here: https://golang.org/pkg/runtime/pprof/#Profile.
-	ProcessProfileOutputDirectory string
+	ProcessProfileOutputDirectory string `json:",omitempty"`
 
 	// ProcessBlockProfileDurationSeconds specifies the sample duration for
 	// "block" profiling. For the default, 0, no "block" profile is taken.
-	ProcessBlockProfileDurationSeconds int
+	ProcessBlockProfileDurationSeconds int `json:",omitempty"`
 
 	// ProcessCPUProfileDurationSeconds specifies the sample duration for
 	// CPU profiling. For the default, 0, no CPU profile is taken.
-	ProcessCPUProfileDurationSeconds int
+	ProcessCPUProfileDurationSeconds int `json:",omitempty"`
 
 	// TrafficRulesFilename is the path of a file containing a JSON-encoded
 	// TrafficRulesSet, the traffic rules to apply to Psiphon client tunnels.
-	TrafficRulesFilename string
+	TrafficRulesFilename string `json:",omitempty"`
 
 	// OSLConfigFilename is the path of a file containing a JSON-encoded
 	// OSL Config, the OSL schemes to apply to Psiphon client tunnels.
-	OSLConfigFilename string
+	OSLConfigFilename string `json:",omitempty"`
 
 	// RunPacketTunnel specifies whether to run a packet tunnel.
-	RunPacketTunnel bool
+	RunPacketTunnel bool `json:",omitempty"`
 
 	// PacketTunnelEgressInterface specifies tun.ServerConfig.EgressInterface.
-	PacketTunnelEgressInterface string
+	PacketTunnelEgressInterface string `json:",omitempty"`
 
 	// PacketTunnelEnableDNSFlowTracking sets
 	// tun.ServerConfig.EnableDNSFlowTracking.
-	PacketTunnelEnableDNSFlowTracking bool
+	PacketTunnelEnableDNSFlowTracking bool `json:",omitempty"`
 
 	// PacketTunnelDownstreamPacketQueueSize specifies
 	// tun.ServerConfig.DownStreamPacketQueueSize.
-	PacketTunnelDownstreamPacketQueueSize int
+	PacketTunnelDownstreamPacketQueueSize int `json:",omitempty"`
 
 	// PacketTunnelSessionIdleExpirySeconds specifies
 	// tun.ServerConfig.SessionIdleExpirySeconds.
-	PacketTunnelSessionIdleExpirySeconds int
+	PacketTunnelSessionIdleExpirySeconds int `json:",omitempty"`
 
 	// PacketTunnelSudoNetworkConfigCommands sets
 	// tun.ServerConfig.SudoNetworkConfigCommands,
 	// packetman.Config.SudoNetworkConfigCommands, and
 	// SudoNetworkConfigCommands for configureIptablesAcceptRateLimitChain.
-	PacketTunnelSudoNetworkConfigCommands bool
+	PacketTunnelSudoNetworkConfigCommands bool `json:",omitempty"`
 
 	// RunPacketManipulator specifies whether to run a packet manipulator.
-	RunPacketManipulator bool
+	RunPacketManipulator bool `json:",omitempty"`
 
 	// MaxConcurrentSSHHandshakes specifies a limit on the number of concurrent
 	// SSH handshake negotiations. This is set to mitigate spikes in memory
@@ -404,13 +404,13 @@ type Config struct {
 	// be disconnected after a short wait for the number of concurrent handshakes
 	// to drop below the limit.
 	// The default, 0 is no limit.
-	MaxConcurrentSSHHandshakes int
+	MaxConcurrentSSHHandshakes int `json:",omitempty"`
 
 	// PeriodicGarbageCollectionSeconds turns on periodic calls to
 	// debug.FreeOSMemory, every specified number of seconds, to force garbage
 	// collection and memory scavenging. Specify 0 to disable. The default is
 	// PERIODIC_GARBAGE_COLLECTION.
-	PeriodicGarbageCollectionSeconds *int
+	PeriodicGarbageCollectionSeconds *int `json:",omitempty"`
 
 	// StopEstablishTunnelsEstablishedClientThreshold sets the established client
 	// threshold for dumping profiles when SIGTSTP is signaled. When there are
@@ -419,34 +419,49 @@ type Config struct {
 	// occur when few clients are connected and load should be relatively low. A
 	// profile dump is attempted at most once per process lifetime, the first
 	// time the threshold is met. Disabled when < 0.
-	StopEstablishTunnelsEstablishedClientThreshold *int
+	StopEstablishTunnelsEstablishedClientThreshold *int `json:",omitempty"`
 
 	// AccessControlVerificationKeyRing is the access control authorization
 	// verification key ring used to verify signed authorizations presented
 	// by clients. Verified, active (unexpired) access control types will be
 	// available for matching in the TrafficRulesFilter for the client via
 	// AuthorizedAccessTypes. All other authorizations are ignored.
-	AccessControlVerificationKeyRing accesscontrol.VerificationKeyRing
+	AccessControlVerificationKeyRing *accesscontrol.VerificationKeyRing `json:",omitempty"`
 
 	// TacticsConfigFilename is the path of a file containing a JSON-encoded
 	// tactics server configuration.
-	TacticsConfigFilename string
+	TacticsConfigFilename string `json:",omitempty"`
+
+	// TacticsRequestPublicKey is an optional, base64 encoded
+	// tactics.Server.RequestPublicKey which overrides the value in the
+	// tactics configuration file.
+	TacticsRequestPublicKey string `json:",omitempty"`
+
+	// TacticsRequestPrivateKey is an optional, base64 encoded
+	// tactics.Server.RequestPrivateKey which overrides the value in the
+	// tactics configuration file.
+	TacticsRequestPrivateKey string `json:",omitempty"`
+
+	// TacticsRequestObfuscatedKey is an optional, base64 encoded
+	// tactics.Server.RequestObfuscatedKey which overrides the value in the
+	// tactics configuration file.
+	TacticsRequestObfuscatedKey string `json:",omitempty"`
 
 	// BlocklistFilename is the path of a file containing a CSV-encoded
 	// blocklist configuration. See NewBlocklist for more file format
 	// documentation.
-	BlocklistFilename string
+	BlocklistFilename string `json:",omitempty"`
 
 	// BlocklistActive indicates whether to actively prevent blocklist hits in
 	// addition to logging events.
-	BlocklistActive bool
+	BlocklistActive bool `json:",omitempty"`
 
 	// AllowBogons disables port forward bogon checks. This should be used only
 	// for testing.
-	AllowBogons bool
+	AllowBogons bool `json:",omitempty"`
 
 	// EnableSteeringIPs enables meek server steering IP support.
-	EnableSteeringIPs bool
+	EnableSteeringIPs bool `json:",omitempty"`
 
 	// OwnEncodedServerEntries is a list of the server's own encoded server
 	// entries, idenfified by server entry tag. These values are used in the
@@ -457,53 +472,53 @@ type Config struct {
 	// server entries here; and, besides the discovery server entries, in
 	// psinet.Database, necessary for the discovery feature, no other server
 	// entries are stored on a Psiphon server.
-	OwnEncodedServerEntries map[string]string
+	OwnEncodedServerEntries map[string]string `json:",omitempty"`
 
 	// MeekServerRunInproxyBroker indicates whether to run an in-proxy broker
 	// endpoint and service under the meek server.
-	MeekServerRunInproxyBroker bool
+	MeekServerRunInproxyBroker bool `json:",omitempty"`
 
 	// MeekServerInproxyBrokerOnly indicates whether to run only an in-proxy
 	// broker under the meek server, and not run any meek tunnel protocol. To
 	// run the meek listener, a meek server protocol and port must still be
 	// specified in TunnelProtocolPorts, but no other tunnel protocol
 	// parameters are required.
-	MeekServerInproxyBrokerOnly bool
+	MeekServerInproxyBrokerOnly bool `json:",omitempty"`
 
 	// InproxyBrokerSessionPrivateKey specifies the broker's in-proxy session
 	// private key and derived public key used by in-proxy clients and
 	// proxies. This value is required when running an in-proxy broker.
-	InproxyBrokerSessionPrivateKey string
+	InproxyBrokerSessionPrivateKey string `json:",omitempty"`
 
 	// InproxyBrokerObfuscationRootSecret specifies the broker's in-proxy
 	// session root obfuscation secret used by in-proxy clients and proxies.
 	// This value is required when running an in-proxy broker.
-	InproxyBrokerObfuscationRootSecret string
+	InproxyBrokerObfuscationRootSecret string `json:",omitempty"`
 
 	// InproxyBrokerServerEntrySignaturePublicKey specifies the public key
 	// used to verify Psiphon server entry signature. This value is required
 	// when running an in-proxy broker.
-	InproxyBrokerServerEntrySignaturePublicKey string
+	InproxyBrokerServerEntrySignaturePublicKey string `json:",omitempty"`
 
 	// InproxyBrokerAllowCommonASNMatching overrides the default broker
 	// matching behavior which doesn't match non-personal in-proxy clients
 	// and proxies from the same ASN. This parameter is for testing only.
-	InproxyBrokerAllowCommonASNMatching bool
+	InproxyBrokerAllowCommonASNMatching bool `json:",omitempty"`
 
 	// InproxyBrokerAllowBogonWebRTCConnections overrides the default broker
 	// SDP validation behavior, which doesn't allow private network WebRTC
 	// candidates. This parameter is for testing only.
-	InproxyBrokerAllowBogonWebRTCConnections bool
+	InproxyBrokerAllowBogonWebRTCConnections bool `json:",omitempty"`
 
 	// InproxyServerSessionPrivateKey specifies the server's in-proxy session
 	// private key and derived public key used by brokers. This value is
 	// required when running in-proxy tunnel protocols.
-	InproxyServerSessionPrivateKey string
+	InproxyServerSessionPrivateKey string `json:",omitempty"`
 
 	// InproxyServerObfuscationRootSecret specifies the server's in-proxy
 	// session root obfuscation secret used by brokers. This value is
 	// required when running in-proxy tunnel protocols.
-	InproxyServerObfuscationRootSecret string
+	InproxyServerObfuscationRootSecret string `json:",omitempty"`
 
 	// IptablesAcceptRateLimitChainName, when set, enables programmatic
 	// configuration of iptables rules to allow and apply rate limits to
@@ -511,13 +526,13 @@ type Config struct {
 	// specified chain.
 	//
 	// For details, see configureIptablesAcceptRateLimitChain.
-	IptablesAcceptRateLimitChainName string
+	IptablesAcceptRateLimitChainName string `json:",omitempty"`
 
 	// IptablesAcceptRateLimitTunnelProtocolRateLimits specifies custom
 	// iptables rate limits by tunnel protocol name. See
 	// configureIptablesAcceptRateLimitChain details about the rate limit
 	// values.
-	IptablesAcceptRateLimitTunnelProtocolRateLimits map[string][2]int
+	IptablesAcceptRateLimitTunnelProtocolRateLimits map[string][2]int `json:",omitempty"`
 
 	sshBeginHandshakeTimeout                       time.Duration
 	sshHandshakeTimeout                            time.Duration
@@ -812,10 +827,12 @@ func LoadConfig(configJSON []byte) (*Config, error) {
 		config.stopEstablishTunnelsEstablishedClientThreshold = *config.StopEstablishTunnelsEstablishedClientThreshold
 	}
 
-	err = accesscontrol.ValidateVerificationKeyRing(&config.AccessControlVerificationKeyRing)
-	if err != nil {
-		return nil, errors.Tracef(
-			"AccessControlVerificationKeyRing is invalid: %s", err)
+	if config.AccessControlVerificationKeyRing != nil {
+		err = accesscontrol.ValidateVerificationKeyRing(config.AccessControlVerificationKeyRing)
+		if err != nil {
+			return nil, errors.Tracef(
+				"AccessControlVerificationKeyRing is invalid: %s", err)
+		}
 	}
 
 	// Limitation: the following is a shortcut which extracts the server's

psiphon/server/listener_test.go

@@ -86,10 +86,7 @@ func TestListener(t *testing.T) {
 	}
 
 	tacticsServer, err := tactics.NewServer(
-		nil,
-		nil,
-		nil,
-		tacticsConfigFilename)
+		nil, nil, nil, tacticsConfigFilename, "", "", "")
 	if err != nil {
 		t.Fatalf("NewServer failed: %s", err)
 	}

psiphon/server/meek.go

@@ -138,6 +138,7 @@ type MeekServer struct {
 	rateLimitSignalGC               chan struct{}
 	normalizer                      *transforms.HTTPNormalizerListener
 	inproxyBroker                   *inproxy.Broker
+	inproxyCheckAllowMatch          atomic.Value
 }
 
 // NewMeekServer initializes a new meek server.
@@ -354,6 +355,7 @@ func NewMeekServer(
 				PrioritizeProxy:                meekServer.inproxyBrokerPrioritizeProxy,
 				AllowClient:                    meekServer.inproxyBrokerAllowClient,
 				AllowDomainFrontedDestinations: meekServer.inproxyBrokerAllowDomainFrontedDestinations,
+				AllowMatch:                     meekServer.inproxyBrokerAllowMatch,
 				LookupGeoIP:                    lookupGeoIPData,
 				APIParameterValidator:          getInproxyBrokerAPIParameterValidator(support.Config),
 				APIParameterLogFieldFormatter:  getInproxyBrokerAPIParameterLogFieldFormatter(),
@@ -650,7 +652,17 @@ func (server *MeekServer) ServeHTTP(responseWriter http.ResponseWriter, request
 				responseWriter,
 				request)
 			if err != nil {
-				log.WithTraceFields(LogFields{"error": err}).Warning("inproxyBrokerHandler failed")
+
+				var brokerLoggedEvent *inproxy.BrokerLoggedEvent
+				var deobfuscationAnomoly *inproxy.DeobfuscationAnomoly
+				alreadyLogged := std_errors.As(err, &brokerLoggedEvent) ||
+					std_errors.As(err, &deobfuscationAnomoly)
+
+				if !alreadyLogged {
+					log.WithTraceFields(
+						LogFields{"error": err}).Warning("inproxyBrokerHandler failed")
+				}
+
 				server.handleError(responseWriter, request)
 			}
 		}
@@ -1871,6 +1883,62 @@ func (server *MeekServer) inproxyReloadTactics() error {
 		p.Duration(parameters.InproxyProxyQualityPendingFailedMatchDeadline),
 		p.Int(parameters.InproxyProxyQualityFailedMatchThreshold))
 
+	// Configure proxy/client match checklists.
+	//
+	// When an allow list is set, the client GeoIP data must appear in the
+	// proxy's list or the match isn't allowed. When a disallow list is set,
+	// the match isn't allowed if the client GeoIP data appears in the
+	// proxy's list.
+
+	makeCheckListLookup := func(
+		lists map[string][]string,
+		isAllowList bool) func(string, string) bool {
+
+		if len(lists) == 0 {
+			return func(string, string) bool {
+				// Allow when no list
+				return true
+			}
+		}
+		lookup := make(map[string]map[string]struct{})
+		for key, items := range lists {
+			// TODO: use linear search for lists below stringLookupThreshold?
+			itemLookup := make(map[string]struct{})
+			for _, item := range items {
+				itemLookup[item] = struct{}{}
+			}
+			lookup[key] = itemLookup
+		}
+		return func(key, item string) bool {
+			itemLookup := lookup[key]
+			if itemLookup == nil {
+				// Allow when no list
+				return true
+			}
+			_, found := itemLookup[item]
+			// Allow or disallow based on list type
+			return found == isAllowList
+		}
+	}
+
+	inproxyCheckAllowMatchByRegion := makeCheckListLookup(p.KeyStringsValue(
+		parameters.InproxyAllowMatchByRegion), true)
+	inproxyCheckAllowMatchByASN := makeCheckListLookup(p.KeyStringsValue(
+		parameters.InproxyAllowMatchByASN), true)
+	inproxyCheckDisallowMatchByRegion := makeCheckListLookup(p.KeyStringsValue(
+		parameters.InproxyDisallowMatchByRegion), false)
+	inproxyCheckDisallowMatchByASN := makeCheckListLookup(p.KeyStringsValue(
+		parameters.InproxyDisallowMatchByASN), false)
+
+	checkAllowMatch := func(proxyGeoIPData, clientGeoIPData common.GeoIPData) bool {
+		return inproxyCheckAllowMatchByRegion(proxyGeoIPData.Country, clientGeoIPData.Country) &&
+			inproxyCheckAllowMatchByASN(proxyGeoIPData.ASN, clientGeoIPData.ASN) &&
+			inproxyCheckDisallowMatchByRegion(proxyGeoIPData.Country, clientGeoIPData.Country) &&
+			inproxyCheckDisallowMatchByASN(proxyGeoIPData.ASN, clientGeoIPData.ASN)
+	}
+
+	server.inproxyCheckAllowMatch.Store(checkAllowMatch)
+
 	return nil
 }
 
@@ -1901,6 +1969,13 @@ func (server *MeekServer) inproxyBrokerAllowDomainFrontedDestinations(clientGeoI
 	return server.lookupAllowTactic(clientGeoIPData, parameters.InproxyAllowDomainFrontedDestinations)
 }
 
+func (server *MeekServer) inproxyBrokerAllowMatch(
+	proxyGeoIPData common.GeoIPData, clientGeoIPData common.GeoIPData) bool {
+
+	return server.inproxyCheckAllowMatch.Load().(func(proxy, client common.GeoIPData) bool)(
+		proxyGeoIPData, clientGeoIPData)
+}
+
 func (server *MeekServer) inproxyBrokerPrioritizeProxy(
 	proxyInproxyProtocolVersion int,
 	proxyGeoIPData common.GeoIPData,

psiphon/server/meek_test.go

@@ -560,7 +560,7 @@ func runTestMeekAccessControl(t *testing.T, rateLimit, restrictProvider, missing
 	}
 	mockSupport.GeoIPService, _ = NewGeoIPService([]string{})
 
-	tacticsServer, err := tactics.NewServer(nil, nil, nil, tacticsConfigFilename)
+	tacticsServer, err := tactics.NewServer(nil, nil, nil, tacticsConfigFilename, "", "", "")
 	if err != nil {
 		t.Fatalf("tactics.NewServer failed: %s", err)
 	}

psiphon/server/server_test.go

@@ -846,7 +846,7 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
 		t.Fatalf("error creating access control key pair: %s", err)
 	}
 
-	accessControlVerificationKeyRing := accesscontrol.VerificationKeyRing{
+	accessControlVerificationKeyRing := &accesscontrol.VerificationKeyRing{
 		Keys: []*accesscontrol.VerificationKey{accessControlVerificationKey},
 	}
 
@@ -879,7 +879,6 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
 		runConfig.doLegacyDestinationBytes ||
 		runConfig.doTunneledDomainRequest
 
-	// All servers require a tactics config with valid keys.
 	tacticsRequestPublicKey, tacticsRequestPrivateKey, tacticsRequestObfuscatedKey, err :=
 		tactics.GenerateKeys()
 	if err != nil {
@@ -944,6 +943,21 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
 		generateConfigParams.FrontingProviderID = prng.HexString(8)
 	}
 
+	var configTacticsRequestPublicKey, configTacticsRequestPrivateKey, configTacticsRequestObfuscatedKey string
+	if prng.FlipCoin() {
+
+		// Exercise specifying the tactics key parameters in the main server
+		// config file and not in the tactics config file.
+
+		configTacticsRequestPublicKey = tacticsRequestPublicKey
+		configTacticsRequestPrivateKey = tacticsRequestPrivateKey
+		configTacticsRequestObfuscatedKey = tacticsRequestObfuscatedKey
+
+		tacticsRequestPublicKey = ""
+		tacticsRequestPrivateKey = ""
+		tacticsRequestObfuscatedKey = ""
+	}
+
 	serverConfigJSON, _, _, _, encodedServerEntry, err := GenerateConfig(generateConfigParams)
 	if err != nil {
 		t.Fatalf("error generating server config: %s", err)
@@ -1046,6 +1060,19 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
 	serverConfig["OSLConfigFilename"] = oslConfigFilename
 	if doServerTactics {
 		serverConfig["TacticsConfigFilename"] = tacticsConfigFilename
+
+		if configTacticsRequestPublicKey != "" {
+			serverConfig["TacticsRequestPublicKey"] = configTacticsRequestPublicKey
+
+		}
+		if configTacticsRequestPrivateKey != "" {
+			serverConfig["TacticsRequestPrivateKey"] = configTacticsRequestPrivateKey
+
+		}
+		if configTacticsRequestObfuscatedKey != "" {
+			serverConfig["TacticsRequestObfuscatedKey"] = configTacticsRequestObfuscatedKey
+
+		}
 	}
 	serverConfig["BlocklistFilename"] = blocklistFilename
 
@@ -2011,11 +2038,6 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
 
 	// Test: all expected server logs were emitted
 
-	// TODO: stops should be fully synchronous, but, intermittently,
-	// server_tunnel fails to appear ("missing server tunnel log")
-	// without this delay.
-	time.Sleep(100 * time.Millisecond)
-
 	// For in-proxy tunnel protocols, client BPF tactics are currently ignored and not applied by the 2nd hop.
 	expectClientBPFField := psiphon.ClientBPFEnabled() && doClientTactics && !protocol.TunnelProtocolUsesInproxy(runConfig.tunnelProtocol)
 	expectServerBPFField := ServerBPFEnabled() && protocol.TunnelProtocolIsDirect(runConfig.tunnelProtocol) && doServerTactics
@@ -2425,6 +2447,9 @@ func checkExpectedServerTunnelLogFields(
 		"established_tunnels_count",
 		"network_latency_multiplier",
 		"network_type",
+		"bytes",
+		"ssh_protocol_bytes",
+		"ssh_protocol_bytes_overhead",
 
 		// The test run ensures that logServerLoad is invoked while the client
 		// is connected, so the following must be logged.
@@ -2436,6 +2461,14 @@ func checkExpectedServerTunnelLogFields(
 		}
 	}
 
+	if !(fields["ssh_protocol_bytes"].(float64) > 0) {
+		return fmt.Errorf("unexpected zero ssh_protocol_bytes")
+	}
+
+	if !(fields["ssh_protocol_bytes"].(float64) > fields["bytes"].(float64)) {
+		return fmt.Errorf("unexpected ssh_protocol_bytes < bytes")
+	}
+
 	appliedTacticsTag := len(fields[tactics.APPLIED_TACTICS_TAG_PARAMETER_NAME].(string)) > 0
 	if expectAppliedTacticsTag != appliedTacticsTag {
 		return fmt.Errorf("unexpected applied_tactics_tag")
@@ -4162,6 +4195,10 @@ func generateInproxyTestConfig(
 	tacticsParametersJSONFormat := `
             "InproxyAllowProxy": true,
             "InproxyAllowClient": true,
+            "InproxyAllowMatchByRegion": {"%s":["%s"]},
+            "InproxyAllowMatchByASN": {"%s":["%s"]},
+            "InproxyDisallowMatchByRegion": {"%s":["%s"]},
+            "InproxyDisallowMatchByASN": {"%s":["%s"]},
             "InproxyTunnelProtocolSelectionProbability": 1.0,
             "InproxyAllBrokerSpecs": %s,
             "InproxyBrokerSpecs": %s,
@@ -4191,6 +4228,10 @@ func generateInproxyTestConfig(
 
 	tacticsParametersJSON := fmt.Sprintf(
 		tacticsParametersJSONFormat,
+		testGeoIPCountry, testGeoIPCountry,
+		testGeoIPASN, testGeoIPASN,
+		testGeoIPCountry, "_"+testGeoIPCountry,
+		testGeoIPASN, "_"+testGeoIPASN,
 		allBrokerSpecsJSON,
 		brokerSpecsJSON,
 		proxyBrokerSpecsJSON,

psiphon/server/services.go

@@ -621,7 +621,10 @@ func NewSupportServices(config *Config) (*SupportServices, error) {
 		CommonLogger(log),
 		getTacticsAPIParameterLogFieldFormatter(),
 		getTacticsAPIParameterValidator(config),
-		config.TacticsConfigFilename)
+		config.TacticsConfigFilename,
+		config.TacticsRequestPublicKey,
+		config.TacticsRequestPrivateKey,
+		config.TacticsRequestObfuscatedKey)
 	if err != nil {
 		return nil, errors.Trace(err)
 	}

psiphon/server/tactics_test.go

@@ -142,10 +142,7 @@ func TestServerTacticsParametersCache(t *testing.T) {
 	}
 
 	tacticsServer, err := tactics.NewServer(
-		nil,
-		nil,
-		nil,
-		tacticsConfigFilename)
+		nil, nil, nil, tacticsConfigFilename, "", "", "")
 	if err != nil {
 		t.Fatalf("NewServer failed: %s", err)
 	}

psiphon/server/tunnelServer.go

@@ -1651,6 +1651,7 @@ func (sshServer *sshServer) stopClients() {
 		go func(c *sshClient) {
 			defer waitGroup.Done()
 			c.stop()
+			c.awaitStopped()
 		}(client)
 	}
 	waitGroup.Wait()
@@ -1926,6 +1927,7 @@ type sshClient struct {
 	sessionID                            string
 	isFirstTunnelInSession               bool
 	supportsServerRequests               bool
+	sponsorID                            string
 	handshakeState                       handshakeState
 	udpgwChannelHandler                  *udpgwPortForwardMultiplexer
 	totalUdpgwChannelCount               int
@@ -1956,6 +1958,7 @@ type sshClient struct {
 	requestCheckServerEntryTags          int
 	checkedServerEntryTags               int
 	invalidServerEntryTags               int
+	sshProtocolBytesTracker              *sshProtocolBytesTracker
 }
 
 type trafficState struct {
@@ -2173,15 +2176,18 @@ func (lookup *splitTunnelLookup) lookup(region string) bool {
 }
 
 type inproxyProxyQualityTracker struct {
+	// Note: 64-bit ints used with atomic operations are placed
+	// at the start of struct to ensure 64-bit alignment.
+	// (https://golang.org/pkg/sync/atomic/#pkg-note-BUG)
+	bytesUp         int64
+	bytesDown       int64
+	reportTriggered int32
+
 	sshClient       *sshClient
 	targetBytesUp   int64
 	targetBytesDown int64
 	targetDuration  time.Duration
 	startTime       time.Time
-
-	bytesUp         int64
-	bytesDown       int64
-	reportTriggered int32
 }
 
 func newInproxyProxyQualityTracker(
@@ -2251,6 +2257,31 @@ func (t *inproxyProxyQualityTracker) UpdateProgress(
 	}
 }
 
+type sshProtocolBytesTracker struct {
+	// Note: 64-bit ints used with atomic operations are placed
+	// at the start of struct to ensure 64-bit alignment.
+	// (https://golang.org/pkg/sync/atomic/#pkg-note-BUG)
+	totalBytesRead    int64
+	totalBytesWritten int64
+}
+
+func newSSHProtocolBytesTracker(sshClient *sshClient) *sshProtocolBytesTracker {
+	return &sshProtocolBytesTracker{
+		totalBytesRead:    0,
+		totalBytesWritten: 0,
+	}
+}
+
+func (t *sshProtocolBytesTracker) UpdateProgress(
+	bytesRead, bytesWritten, _ int64) {
+
+	// Concurrency: UpdateProgress may be called concurrently; all accesses to
+	// mutated fields use atomic operations.
+
+	atomic.AddInt64(&t.totalBytesRead, bytesRead)
+	atomic.AddInt64(&t.totalBytesWritten, bytesWritten)
+}
+
 func newSshClient(
 	sshServer *sshServer,
 	sshListener *sshListener,
@@ -2700,10 +2731,10 @@ func (sshClient *sshClient) run(
 
 	sshClient.runTunnel(result.channels, result.requests)
 
-	// Note: sshServer.unregisterEstablishedClient calls sshClient.stop(),
-	// which also closes underlying transport Conn.
+	// sshClient.stop closes the underlying transport conn, ensuring all
+	// network trafic is complete before calling logTunnel.
 
-	sshClient.sshServer.unregisterEstablishedClient(sshClient)
+	sshClient.stop()
 
 	// Log tunnel metrics.
 
@@ -2726,7 +2757,7 @@ func (sshClient *sshClient) run(
 
 	if burstConn != nil {
 		// Any outstanding burst should be recorded by burstConn.Close which should
-		// be called by unregisterEstablishedClient.
+		// be called via sshClient.stop.
 		additionalMetrics = append(
 			additionalMetrics, LogFields(burstConn.GetMetrics(activityConn.GetStartTime())))
 	}
@@ -2803,6 +2834,13 @@ func (sshClient *sshClient) run(
 	// disconnects supports first-tunnel-in-session and duplicate
 	// authorization logic.
 	sshClient.sshServer.markGeoIPSessionCacheToExpire(sshClient.sessionID)
+
+	// unregisterEstablishedClient removes the client from sshServer.clients.
+	// This call must come after logTunnel to ensure all logTunnel calls
+	// complete before a sshServer.stopClients returns, in the case of a
+	// server shutdown.
+
+	sshClient.sshServer.unregisterEstablishedClient(sshClient)
 }
 
 func (sshClient *sshClient) passwordCallback(conn ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) {
@@ -2850,6 +2888,13 @@ func (sshClient *sshClient) passwordCallback(conn ssh.ConnMetadata, password []b
 	supportsServerRequests := common.Contains(
 		sshPasswordPayload.ClientCapabilities, protocol.CLIENT_CAPABILITY_SERVER_REQUESTS)
 
+	// This optional, early sponsor ID will be logged with server_tunnel if
+	// the tunnel doesn't reach handshakeState.completed.
+	sponsorID := sshPasswordPayload.SponsorID
+	if sponsorID != "" && !isSponsorID(sshClient.sshServer.support.Config, sponsorID) {
+		return nil, errors.Tracef("invalid sponsor ID")
+	}
+
 	sshClient.Lock()
 
 	// After this point, these values are read-only as they are read
@@ -2857,6 +2902,7 @@ func (sshClient *sshClient) passwordCallback(conn ssh.ConnMetadata, password []b
 	sshClient.sessionID = sessionID
 	sshClient.isFirstTunnelInSession = isFirstTunnelInSession
 	sshClient.supportsServerRequests = supportsServerRequests
+	sshClient.sponsorID = sponsorID
 
 	sshClient.Unlock()
 
@@ -3605,6 +3651,13 @@ func (sshClient *sshClient) logTunnel(additionalMetrics []LogFields) {
 
 	logFields["handshake_completed"] = sshClient.handshakeState.completed
 
+	// Use the handshake sponsor ID unless the handshake did not complete.
+	//
+	// TODO: check that the handshake sponsor ID matches the early sponsor ID?
+	if !sshClient.handshakeState.completed {
+		logFields["sponsor_id"] = sshClient.sponsorID
+	}
+
 	logFields["is_first_tunnel_in_session"] = sshClient.isFirstTunnelInSession
 
 	if sshClient.preHandshakeRandomStreamMetrics.count > 0 {
@@ -3742,10 +3795,17 @@ func (sshClient *sshClient) logTunnel(additionalMetrics []LogFields) {
 
 	// Pre-calculate a total-tunneled-bytes field. This total is used
 	// extensively in analytics and is more performant when pre-calculated.
-	logFields["bytes"] = sshClient.tcpTrafficState.bytesUp +
+	bytes := sshClient.tcpTrafficState.bytesUp +
 		sshClient.tcpTrafficState.bytesDown +
 		sshClient.udpTrafficState.bytesUp +
 		sshClient.udpTrafficState.bytesDown
+	logFields["bytes"] = bytes
+
+	// Pre-calculate ssh protocol bytes and overhead.
+	sshProtocolBytes := sshClient.sshProtocolBytesTracker.totalBytesWritten +
+		sshClient.sshProtocolBytesTracker.totalBytesRead
+	logFields["ssh_protocol_bytes"] = sshProtocolBytes
+	logFields["ssh_protocol_bytes_overhead"] = sshProtocolBytes - bytes
 
 	if sshClient.additionalTransportData != nil &&
 		sshClient.additionalTransportData.steeringIP != "" {
@@ -4132,10 +4192,16 @@ func (sshClient *sshClient) setHandshakeState(
 			break
 		}
 
+		if sshClient.sshServer.support.Config.AccessControlVerificationKeyRing == nil {
+			if i == 0 {
+				log.WithTrace().Warning("authorization not configured")
+			}
+			continue
+		}
+
 		verifiedAuthorization, err := accesscontrol.VerifyAuthorization(
-			&sshClient.sshServer.support.Config.AccessControlVerificationKeyRing,
+			sshClient.sshServer.support.Config.AccessControlVerificationKeyRing,
 			authorization)
-
 		if err != nil {
 			log.WithTraceFields(
 				LogFields{"error": err}).Warning("verify authorization failed")
@@ -4655,6 +4721,17 @@ func (sshClient *sshClient) reportProxyQuality() {
 		sshClient.clientGeoIPData.ASN)
 }
 
+func (sshClient *sshClient) newSSHProtocolBytesTracker() *sshProtocolBytesTracker {
+	sshClient.Lock()
+	defer sshClient.Unlock()
+
+	tracker := newSSHProtocolBytesTracker(sshClient)
+
+	sshClient.sshProtocolBytesTracker = tracker
+
+	return tracker
+}
+
 func (sshClient *sshClient) getTunnelActivityUpdaters() []common.ActivityUpdater {
 
 	var updaters []common.ActivityUpdater
@@ -4664,6 +4741,9 @@ func (sshClient *sshClient) getTunnelActivityUpdaters() []common.ActivityUpdater
 		updaters = append(updaters, inproxyProxyQualityTracker)
 	}
 
+	sshProtocolBytesTracker := sshClient.newSSHProtocolBytesTracker()
+	updaters = append(updaters, sshProtocolBytesTracker)
+
 	return updaters
 }
 

psiphon/tunnel.go

@@ -1105,6 +1105,7 @@ func dialTunnel(
 		SessionId:          config.SessionID,
 		SshPassword:        dialParams.ServerEntry.SshPassword,
 		ClientCapabilities: []string{protocol.CLIENT_CAPABILITY_SERVER_REQUESTS},
+		SponsorID:          config.GetSponsorID(),
 	}
 
 	payload, err := json.Marshal(sshPasswordPayload)