Inproxy: add InproxyProxyOnBrokerClientFailedRetryPeriod

- Also log selected broker ID in NewInproxyBrokerClientInstance

psiphon/common/parameters/parameters.go

@@ -451,6 +451,7 @@ const (
 	InproxyClientNoMatchFailoverPersonalProbability    = "InproxyClientNoMatchFailoverPersonalProbability"
 	InproxyFrontingProviderClientMaxRequestTimeouts    = "InproxyFrontingProviderClientMaxRequestTimeouts"
 	InproxyFrontingProviderServerMaxRequestTimeouts    = "InproxyFrontingProviderServerMaxRequestTimeouts"
+	InproxyProxyOnBrokerClientFailedRetryPeriod        = "InproxyProxyOnBrokerClientFailedRetryPeriod"
 
 	// Retired parameters
 
@@ -960,6 +961,7 @@ var defaultParameters = map[string]struct {
 	InproxyClientNoMatchFailoverPersonalProbability:    {value: 1.0, minimum: 0.0},
 	InproxyFrontingProviderClientMaxRequestTimeouts:    {value: KeyDurations{}},
 	InproxyFrontingProviderServerMaxRequestTimeouts:    {value: KeyDurations{}, flags: serverSideOnly},
+	InproxyProxyOnBrokerClientFailedRetryPeriod:        {value: 30 * time.Second, minimum: time.Duration(0)},
 }
 
 // IsServerSideOnly indicates if the parameter specified by name is used

psiphon/config.go

@@ -972,65 +972,66 @@ type Config struct {
 	SteeringIPProbability     *float64
 
 	// The following in-proxy fields are for testing purposes only.
-	InproxyAllowProxy                                      *bool
-	InproxyAllowClient                                     *bool
-	InproxyTunnelProtocolSelectionProbability              *float64
-	InproxyBrokerSpecs                                     parameters.InproxyBrokerSpecsValue
-	InproxyPersonalPairingBrokerSpecs                      parameters.InproxyBrokerSpecsValue
-	InproxyProxyBrokerSpecs                                parameters.InproxyBrokerSpecsValue
-	InproxyProxyPersonalPairingBrokerSpecs                 parameters.InproxyBrokerSpecsValue
-	InproxyClientBrokerSpecs                               parameters.InproxyBrokerSpecsValue
-	InproxyClientPersonalPairingBrokerSpecs                parameters.InproxyBrokerSpecsValue
-	InproxyReplayBrokerDialParametersTTLSeconds            *int
-	InproxyReplayBrokerUpdateFrequencySeconds              *int
-	InproxyReplayBrokerDialParametersProbability           *float64
-	InproxyReplayBrokerRetainFailedProbability             *float64
-	InproxyCommonCompartmentIDs                            parameters.InproxyCompartmentIDsValue
-	InproxyMaxCompartmentIDListLength                      *int
-	InproxyProxyAnnounceRequestTimeoutMilliseconds         *int
-	InproxyProxyAnnounceDelayMilliseconds                  *int
-	InproxyProxyAnnounceDelayJitter                        *float64
-	InproxyProxyAnswerRequestTimeoutMilliseconds           *int
-	InproxyClientOfferRequestTimeoutMilliseconds           *int
-	InproxyClientOfferRequestPersonalTimeoutMilliseconds   *int
-	InproxyClientOfferRetryDelayMilliseconds               *int
-	InproxyClientOfferRetryJitter                          *float64
-	InproxyClientRelayedPacketRequestTimeoutMilliseconds   *int
-	InproxyDTLSRandomizationProbability                    *float64
-	InproxyDataChannelTrafficShapingProbability            *float64
-	InproxyDataChannelTrafficShapingParameters             *parameters.InproxyDataChannelTrafficShapingParametersValue
-	InproxySTUNServerAddresses                             []string
-	InproxySTUNServerAddressesRFC5780                      []string
-	InproxyProxySTUNServerAddresses                        []string
-	InproxyProxySTUNServerAddressesRFC5780                 []string
-	InproxyClientSTUNServerAddresses                       []string
-	InproxyClientSTUNServerAddressesRFC5780                []string
-	InproxyClientDiscoverNATProbability                    *float64
-	InproxyDisableSTUN                                     *bool
-	InproxyDisablePortMapping                              *bool
-	InproxyDisableInboundForMobileNetworks                 *bool
-	InproxyDisableIPv6ICECandidates                        *bool
-	InproxyProxyDisableSTUN                                *bool
-	InproxyProxyDisablePortMapping                         *bool
-	InproxyProxyDisableInboundForMobileNetworks            *bool
-	InproxyProxyDisableIPv6ICECandidates                   *bool
-	InproxyClientDisableSTUN                               *bool
-	InproxyClientDisablePortMapping                        *bool
-	InproxyClientDisableInboundForMobileNetworks           *bool
-	InproxyClientDisableIPv6ICECandidates                  *bool
-	InproxyProxyDiscoverNATTimeoutMilliseconds             *int
-	InproxyClientDiscoverNATTimeoutMilliseconds            *int
-	InproxyWebRTCAnswerTimeoutMilliseconds                 *int
-	InproxyProxyWebRTCAwaitDataChannelTimeoutMilliseconds  *int
-	InproxyClientWebRTCAwaitDataChannelTimeoutMilliseconds *int
-	InproxyProxyDestinationDialTimeoutMilliseconds         *int
-	InproxyPsiphonAPIRequestTimeoutMilliseconds            *int
-	InproxyProxyTotalActivityNoticePeriodMilliseconds      *int
-	InproxyClientDialRateLimitQuantity                     *int
-	InproxyClientDialRateLimitIntervalMilliseconds         *int
-	InproxyClientNoMatchFailoverProbability                *float64
-	InproxyClientNoMatchFailoverPersonalProbability        *float64
-	InproxyFrontingProviderClientMaxRequestTimeouts        map[string]string
+	InproxyAllowProxy                                       *bool
+	InproxyAllowClient                                      *bool
+	InproxyTunnelProtocolSelectionProbability               *float64
+	InproxyBrokerSpecs                                      parameters.InproxyBrokerSpecsValue
+	InproxyPersonalPairingBrokerSpecs                       parameters.InproxyBrokerSpecsValue
+	InproxyProxyBrokerSpecs                                 parameters.InproxyBrokerSpecsValue
+	InproxyProxyPersonalPairingBrokerSpecs                  parameters.InproxyBrokerSpecsValue
+	InproxyClientBrokerSpecs                                parameters.InproxyBrokerSpecsValue
+	InproxyClientPersonalPairingBrokerSpecs                 parameters.InproxyBrokerSpecsValue
+	InproxyReplayBrokerDialParametersTTLSeconds             *int
+	InproxyReplayBrokerUpdateFrequencySeconds               *int
+	InproxyReplayBrokerDialParametersProbability            *float64
+	InproxyReplayBrokerRetainFailedProbability              *float64
+	InproxyCommonCompartmentIDs                             parameters.InproxyCompartmentIDsValue
+	InproxyMaxCompartmentIDListLength                       *int
+	InproxyProxyAnnounceRequestTimeoutMilliseconds          *int
+	InproxyProxyAnnounceDelayMilliseconds                   *int
+	InproxyProxyAnnounceDelayJitter                         *float64
+	InproxyProxyAnswerRequestTimeoutMilliseconds            *int
+	InproxyClientOfferRequestTimeoutMilliseconds            *int
+	InproxyClientOfferRequestPersonalTimeoutMilliseconds    *int
+	InproxyClientOfferRetryDelayMilliseconds                *int
+	InproxyClientOfferRetryJitter                           *float64
+	InproxyClientRelayedPacketRequestTimeoutMilliseconds    *int
+	InproxyDTLSRandomizationProbability                     *float64
+	InproxyDataChannelTrafficShapingProbability             *float64
+	InproxyDataChannelTrafficShapingParameters              *parameters.InproxyDataChannelTrafficShapingParametersValue
+	InproxySTUNServerAddresses                              []string
+	InproxySTUNServerAddressesRFC5780                       []string
+	InproxyProxySTUNServerAddresses                         []string
+	InproxyProxySTUNServerAddressesRFC5780                  []string
+	InproxyClientSTUNServerAddresses                        []string
+	InproxyClientSTUNServerAddressesRFC5780                 []string
+	InproxyClientDiscoverNATProbability                     *float64
+	InproxyDisableSTUN                                      *bool
+	InproxyDisablePortMapping                               *bool
+	InproxyDisableInboundForMobileNetworks                  *bool
+	InproxyDisableIPv6ICECandidates                         *bool
+	InproxyProxyDisableSTUN                                 *bool
+	InproxyProxyDisablePortMapping                          *bool
+	InproxyProxyDisableInboundForMobileNetworks             *bool
+	InproxyProxyDisableIPv6ICECandidates                    *bool
+	InproxyClientDisableSTUN                                *bool
+	InproxyClientDisablePortMapping                         *bool
+	InproxyClientDisableInboundForMobileNetworks            *bool
+	InproxyClientDisableIPv6ICECandidates                   *bool
+	InproxyProxyDiscoverNATTimeoutMilliseconds              *int
+	InproxyClientDiscoverNATTimeoutMilliseconds             *int
+	InproxyWebRTCAnswerTimeoutMilliseconds                  *int
+	InproxyProxyWebRTCAwaitDataChannelTimeoutMilliseconds   *int
+	InproxyClientWebRTCAwaitDataChannelTimeoutMilliseconds  *int
+	InproxyProxyDestinationDialTimeoutMilliseconds          *int
+	InproxyPsiphonAPIRequestTimeoutMilliseconds             *int
+	InproxyProxyTotalActivityNoticePeriodMilliseconds       *int
+	InproxyClientDialRateLimitQuantity                      *int
+	InproxyClientDialRateLimitIntervalMilliseconds          *int
+	InproxyClientNoMatchFailoverProbability                 *float64
+	InproxyClientNoMatchFailoverPersonalProbability         *float64
+	InproxyFrontingProviderClientMaxRequestTimeouts         map[string]string
+	InproxyProxyOnBrokerClientFailedRetryPeriodMilliseconds *int
 
 	InproxySkipAwaitFullyConnected  bool
 	InproxyEnableWebRTCDebugLogging bool
@@ -2664,6 +2665,10 @@ func (config *Config) makeConfigParameters() map[string]interface{} {
 		applyParameters[parameters.InproxyFrontingProviderClientMaxRequestTimeouts] = config.InproxyFrontingProviderClientMaxRequestTimeouts
 	}
 
+	if config.InproxyProxyOnBrokerClientFailedRetryPeriodMilliseconds != nil {
+		applyParameters[parameters.InproxyProxyOnBrokerClientFailedRetryPeriod] = fmt.Sprintf("%dms", *config.InproxyProxyOnBrokerClientFailedRetryPeriodMilliseconds)
+	}
+
 	// When adding new config dial parameters that may override tactics, also
 	// update setDialParametersHash.
 

psiphon/inproxy.go

@@ -288,9 +288,11 @@ type InproxyBrokerClientInstance struct {
 	relayedPacketRequestTimeout   time.Duration
 	replayRetainFailedProbability float64
 	replayUpdateFrequency         time.Duration
+	retryOnFailedPeriod           time.Duration
 
 	mutex           sync.Mutex
 	lastStoreReplay time.Time
+	lastSuccess     time.Time
 }
 
 // NewInproxyBrokerClientInstance creates a new InproxyBrokerClientInstance.
@@ -441,6 +443,10 @@ func NewInproxyBrokerClientInstance(
 		brokerSpec = brokerSpecs[0]
 	}
 
+	// The broker ID is the broker's session public key.
+	brokerID := brokerSpec.BrokerPublicKey
+	NoticeInfo("inproxy: selected broker %s", brokerID)
+
 	// Generate new broker dial parameters if not replaying. Later, isReplay
 	// is used to report the replay metric.
 
@@ -525,6 +531,12 @@ func NewInproxyBrokerClientInstance(
 		replayUpdateFrequency:         p.Duration(parameters.InproxyReplayBrokerUpdateFrequency),
 	}
 
+	if isProxy && !config.IsInproxyPersonalPairingMode() {
+		// This retry is applied only for proxies and only in common pairing
+		// mode. See comment in BrokerClientRoundTripperFailed.
+		b.retryOnFailedPeriod = p.Duration(parameters.InproxyProxyOnBrokerClientFailedRetryPeriod)
+	}
+
 	// Adjust long-polling request timeouts to respect any maximum request
 	// timeout supported by the provider fronting the request.
 	maxRequestTimeout, ok := p.KeyDurations(
@@ -811,6 +823,9 @@ func (b *InproxyBrokerClientInstance) BrokerClientRoundTripperSucceeded(roundTri
 		return
 	}
 
+	now := time.Now()
+	b.lastSuccess = now
+
 	// Set replay or extend the broker dial parameters replay TTL after a
 	// success. With tunnel dial parameters, the replay TTL is extended after
 	// every successful tunnel connection. Since there are potentially more
@@ -818,7 +833,6 @@ func (b *InproxyBrokerClientInstance) BrokerClientRoundTripperSucceeded(roundTri
 	// extended after some target duration has elapsed, to avoid excessive
 	// datastore writes.
 
-	now := time.Now()
 	if b.replayEnabled && now.Sub(b.lastStoreReplay) > b.replayUpdateFrequency {
 		b.brokerDialParams.LastUsedTimestamp = time.Now()
 
@@ -857,6 +871,39 @@ func (b *InproxyBrokerClientInstance) BrokerClientRoundTripperFailed(roundTrippe
 		return
 	}
 
+	// For common pairing proxies, skip both the replay deletion and the
+	// InproxyBrokerClientInstance reset for a short duration after a recent
+	// round trip success. In this case, subsequent broker requests will use
+	// the existing round tripper, wired up with the same dial parameters and
+	// fronting provider selection. If the failure was due to a transient
+	// TLS/TCP network failure, the net/http round tripper should establish a
+	// new connection on the next request.
+	//
+	// This retry is intended to retain proxy affinity with its currently
+	// selected broker in cases such as broker service upgrades/restarts or
+	// brief network interruptions, mitigating load balancing issues that
+	// otherwise occur (e.g., all proxies fail over to other brokers, leaving
+	// no supply on a restarted broker).
+	//
+	// In common pairing mode, clients do not perform this retry and
+	// immediately reset, as is appropriate for the tunnel establishment
+	// race. In personal pairing mode, neither proxies nor clients retry and
+	// instead follow the personal pairing broker selection scheme in an
+	// effort to rendezvous at the same broker with minimal delay.
+	//
+	// A delay before retrying announce requests is appropriate, but there is
+	// no delay added here since Proxy.proxyOneClient already schedule delays
+	// between announcements.
+	if b.brokerClientManager.isProxy &&
+		!b.config.IsInproxyPersonalPairingMode() &&
+		b.retryOnFailedPeriod > 0 &&
+		!b.lastSuccess.IsZero() &&
+		time.Since(b.lastSuccess) <= b.retryOnFailedPeriod {
+
+		NoticeWarning("BrokerClientRoundTripperFailed: retry roundTripper")
+		return
+	}
+
 	// Delete any persistent replay dial parameters. Unlike with the success
 	// case, consecutive, repeated deletes shouldn't write to storage, so
 	// they are not avoided.