Add client verification mechanism (and remove generic server API request)

AndroidLibrary/psi/psi.go

@@ -123,15 +123,15 @@ func Stop() {
 	}
 }
 
-// See description in Controller.MakeServerAPIRequest.
-func MakeServerAPIRequest(
-	requestID, requestName, requestPayloadJSON string, retryDelaySeconds int) {
+// This is a passthrough to Controller.SetClientVerificationPayload.
+// Note: should only be called after Start() and before Stop(); otherwise,
+// will silently take no action.
+func SetClientVerificationPayload(clientVerificationPayload string) {
 
 	controllerMutex.Lock()
 	defer controllerMutex.Unlock()
 
 	if controller != nil {
-		controller.MakeServerAPIRequest(
-			requestID, requestName, requestPayloadJSON, retryDelaySeconds)
+		controller.SetClientVerificationPayload(clientVerificationPayload)
 	}
 }

psiphon/config.go

@@ -31,47 +31,48 @@ import (
 // TODO: allow all params to be configured
 
 const (
-	LEGACY_DATA_STORE_FILENAME                     = "psiphon.db"
-	DATA_STORE_FILENAME                            = "psiphon.boltdb"
-	CONNECTION_WORKER_POOL_SIZE                    = 10
-	TUNNEL_POOL_SIZE                               = 1
-	TUNNEL_CONNECT_TIMEOUT_SECONDS                 = 20
-	TUNNEL_OPERATE_SHUTDOWN_TIMEOUT                = 1 * time.Second
-	TUNNEL_PORT_FORWARD_DIAL_TIMEOUT_SECONDS       = 10
-	TUNNEL_SSH_KEEP_ALIVE_PAYLOAD_MAX_BYTES        = 256
-	TUNNEL_SSH_KEEP_ALIVE_PERIOD_MIN               = 60 * time.Second
-	TUNNEL_SSH_KEEP_ALIVE_PERIOD_MAX               = 120 * time.Second
-	TUNNEL_SSH_KEEP_ALIVE_PERIODIC_TIMEOUT_SECONDS = 30
-	TUNNEL_SSH_KEEP_ALIVE_PERIODIC_INACTIVE_PERIOD = 10 * time.Second
-	TUNNEL_SSH_KEEP_ALIVE_PROBE_TIMEOUT_SECONDS    = 5
-	TUNNEL_SSH_KEEP_ALIVE_PROBE_INACTIVE_PERIOD    = 10 * time.Second
-	ESTABLISH_TUNNEL_TIMEOUT_SECONDS               = 300
-	ESTABLISH_TUNNEL_WORK_TIME                     = 60 * time.Second
-	ESTABLISH_TUNNEL_PAUSE_PERIOD_SECONDS          = 5
-	ESTABLISH_TUNNEL_SERVER_AFFINITY_GRACE_PERIOD  = 1 * time.Second
-	HTTP_PROXY_ORIGIN_SERVER_TIMEOUT_SECONDS       = 15
-	HTTP_PROXY_MAX_IDLE_CONNECTIONS_PER_HOST       = 50
-	FETCH_REMOTE_SERVER_LIST_TIMEOUT_SECONDS       = 30
-	FETCH_REMOTE_SERVER_LIST_RETRY_PERIOD_SECONDS  = 30
-	FETCH_REMOTE_SERVER_LIST_STALE_PERIOD          = 6 * time.Hour
-	PSIPHON_API_CLIENT_SESSION_ID_LENGTH           = 16
-	PSIPHON_API_SERVER_TIMEOUT_SECONDS             = 20
-	PSIPHON_API_SHUTDOWN_SERVER_TIMEOUT            = 1 * time.Second
-	PSIPHON_API_STATUS_REQUEST_PERIOD_MIN          = 5 * time.Minute
-	PSIPHON_API_STATUS_REQUEST_PERIOD_MAX          = 10 * time.Minute
-	PSIPHON_API_STATUS_REQUEST_SHORT_PERIOD_MIN    = 5 * time.Second
-	PSIPHON_API_STATUS_REQUEST_SHORT_PERIOD_MAX    = 10 * time.Second
-	PSIPHON_API_STATUS_REQUEST_PADDING_MAX_BYTES   = 256
-	PSIPHON_API_CONNECTED_REQUEST_PERIOD           = 24 * time.Hour
-	PSIPHON_API_CONNECTED_REQUEST_RETRY_PERIOD     = 5 * time.Second
-	PSIPHON_API_TUNNEL_STATS_MAX_COUNT             = 1000
-	FETCH_ROUTES_TIMEOUT_SECONDS                   = 60
-	DOWNLOAD_UPGRADE_TIMEOUT                       = 15 * time.Minute
-	DOWNLOAD_UPGRADE_RETRY_PERIOD_SECONDS          = 30
-	DOWNLOAD_UPGRADE_STALE_PERIOD                  = 6 * time.Hour
-	IMPAIRED_PROTOCOL_CLASSIFICATION_DURATION      = 2 * time.Minute
-	IMPAIRED_PROTOCOL_CLASSIFICATION_THRESHOLD     = 3
-	TOTAL_BYTES_TRANSFERRED_NOTICE_PERIOD          = 5 * time.Minute
+	LEGACY_DATA_STORE_FILENAME                           = "psiphon.db"
+	DATA_STORE_FILENAME                                  = "psiphon.boltdb"
+	CONNECTION_WORKER_POOL_SIZE                          = 10
+	TUNNEL_POOL_SIZE                                     = 1
+	TUNNEL_CONNECT_TIMEOUT_SECONDS                       = 20
+	TUNNEL_OPERATE_SHUTDOWN_TIMEOUT                      = 1 * time.Second
+	TUNNEL_PORT_FORWARD_DIAL_TIMEOUT_SECONDS             = 10
+	TUNNEL_SSH_KEEP_ALIVE_PAYLOAD_MAX_BYTES              = 256
+	TUNNEL_SSH_KEEP_ALIVE_PERIOD_MIN                     = 60 * time.Second
+	TUNNEL_SSH_KEEP_ALIVE_PERIOD_MAX                     = 120 * time.Second
+	TUNNEL_SSH_KEEP_ALIVE_PERIODIC_TIMEOUT_SECONDS       = 30
+	TUNNEL_SSH_KEEP_ALIVE_PERIODIC_INACTIVE_PERIOD       = 10 * time.Second
+	TUNNEL_SSH_KEEP_ALIVE_PROBE_TIMEOUT_SECONDS          = 5
+	TUNNEL_SSH_KEEP_ALIVE_PROBE_INACTIVE_PERIOD          = 10 * time.Second
+	ESTABLISH_TUNNEL_TIMEOUT_SECONDS                     = 300
+	ESTABLISH_TUNNEL_WORK_TIME                           = 60 * time.Second
+	ESTABLISH_TUNNEL_PAUSE_PERIOD_SECONDS                = 5
+	ESTABLISH_TUNNEL_SERVER_AFFINITY_GRACE_PERIOD        = 1 * time.Second
+	HTTP_PROXY_ORIGIN_SERVER_TIMEOUT_SECONDS             = 15
+	HTTP_PROXY_MAX_IDLE_CONNECTIONS_PER_HOST             = 50
+	FETCH_REMOTE_SERVER_LIST_TIMEOUT_SECONDS             = 30
+	FETCH_REMOTE_SERVER_LIST_RETRY_PERIOD_SECONDS        = 30
+	FETCH_REMOTE_SERVER_LIST_STALE_PERIOD                = 6 * time.Hour
+	PSIPHON_API_CLIENT_SESSION_ID_LENGTH                 = 16
+	PSIPHON_API_SERVER_TIMEOUT_SECONDS                   = 20
+	PSIPHON_API_SHUTDOWN_SERVER_TIMEOUT                  = 1 * time.Second
+	PSIPHON_API_STATUS_REQUEST_PERIOD_MIN                = 5 * time.Minute
+	PSIPHON_API_STATUS_REQUEST_PERIOD_MAX                = 10 * time.Minute
+	PSIPHON_API_STATUS_REQUEST_SHORT_PERIOD_MIN          = 5 * time.Second
+	PSIPHON_API_STATUS_REQUEST_SHORT_PERIOD_MAX          = 10 * time.Second
+	PSIPHON_API_STATUS_REQUEST_PADDING_MAX_BYTES         = 256
+	PSIPHON_API_CONNECTED_REQUEST_PERIOD                 = 24 * time.Hour
+	PSIPHON_API_CONNECTED_REQUEST_RETRY_PERIOD           = 5 * time.Second
+	PSIPHON_API_TUNNEL_STATS_MAX_COUNT                   = 1000
+	PSIPHON_API_CLIENT_VERIFICATION_REQUEST_RETRY_PERIOD = 5 * time.Second
+	FETCH_ROUTES_TIMEOUT_SECONDS                         = 60
+	DOWNLOAD_UPGRADE_TIMEOUT                             = 15 * time.Minute
+	DOWNLOAD_UPGRADE_RETRY_PERIOD_SECONDS                = 30
+	DOWNLOAD_UPGRADE_STALE_PERIOD                        = 6 * time.Hour
+	IMPAIRED_PROTOCOL_CLASSIFICATION_DURATION            = 2 * time.Minute
+	IMPAIRED_PROTOCOL_CLASSIFICATION_THRESHOLD           = 3
+	TOTAL_BYTES_TRANSFERRED_NOTICE_PERIOD                = 5 * time.Minute
 )
 
 // To distinguish omitted timeout params from explicit 0 value timeout

psiphon/controller.go

@@ -60,6 +60,7 @@ type Controller struct {
 	impairedProtocolClassification map[string]int
 	signalReportConnected          chan struct{}
 	serverAffinityDoneBroadcast    chan struct{}
+	newClientVerificationPayload   chan string
 }
 
 type candidateServerEntry struct {
@@ -107,7 +108,7 @@ func NewController(config *Config) (controller *Controller, err error) {
 		sessionId: sessionId,
 		// componentFailureSignal receives a signal from a component (including socks and
 		// http local proxies) if they unexpectedly fail. Senders should not block.
-		// A buffer allows at least one stop signal to be sent before there is a receiver.
+		// Buffer allows at least one stop signal to be sent before there is a receiver.
 		componentFailureSignal: make(chan struct{}, 1),
 		shutdownBroadcast:      make(chan struct{}),
 		runWaitGroup:           new(sync.WaitGroup),
@@ -129,6 +130,9 @@ func NewController(config *Config) (controller *Controller, err error) {
 		signalFetchRemoteServerList: make(chan struct{}),
 		signalDownloadUpgrade:       make(chan string),
 		signalReportConnected:       make(chan struct{}),
+		// Buffer allows SetClientVerificationPayload to submit one new payload
+		// without blocking or dropping it.
+		newClientVerificationPayload: make(chan string, 1),
 	}
 
 	controller.splitTunnelClassifier = NewSplitTunnelClassifier(config, controller)
@@ -242,6 +246,31 @@ func (controller *Controller) SignalComponentFailure() {
 	}
 }
 
+// SetClientVerificationPayload sets the client verification payload
+// that is to be sent in client verification requests to all established
+// tunnels. Calling this function both sets the payload to be used for
+// all future tunnels as wells as triggering requests with this payload
+// for all currently established tunneled.
+//
+// Client verification is used to verify that the client is a
+// valid Psiphon client, which will determine how the server treats
+// the client traffic. The proof-of-validity is platform-specific
+// and the payload is opaque to this function but assumed to be JSON.
+//
+// Since, in some cases, verification payload cannot be determined until
+// after tunnel-core starts, the payload cannot be simply specified in
+// the Config.
+//
+// SetClientVerificationPayload will not block enqueuing a new verification
+// payload. One new payload can be enqueued, after which additional payloads
+// will be dropped if a payload is still enqueued.
+func (controller *Controller) SetClientVerificationPayload(clientVerificationPayload string) {
+	select {
+	case controller.newClientVerificationPayload <- clientVerificationPayload:
+	default:
+	}
+}
+
 // remoteServerListFetcher fetches an out-of-band list of server entries
 // for more tunnel candidates. It fetches when signalled, with retries
 // on failure.
@@ -481,61 +510,6 @@ downloadLoop:
 	NoticeInfo("exiting upgrade downloader")
 }
 
-// MakeServerAPIRequest submits a caller-defined request to the
-// Psiphon API via a currently established tunnel. This is used
-// by higher-level client code to perform non-tunnel-core API requests.
-//
-// requestName and requestPayloadJSON define the API request. The "common"
-// API inputs are added to the request as query parameters. The request
-// is a POST with "application/json" encoding.
-//
-// requestID is a caller-selected unique ID used to identify response
-// Notices for this request. The request is sent in a background goroutine.
-// This function does not block. After the request completes, the requestID
-// is reported in a Notice.
-//
-// If the request does not complete successfully or there is no active tunnel,
-// the request will be retried. retryDelaySeconds specifies a pause period
-// between retries.
-//
-// Current limitations:
-// - Assumes HTTP status code 200 is expected; will retry on all other HTTP
-//   status codes
-// - GET requests unsupported
-// - response payloads unsupported
-//
-func (controller *Controller) MakeServerAPIRequest(
-	requestID, requestName, requestPayloadJSON string, retryDelaySeconds int) {
-
-	controller.runWaitGroup.Add(1)
-	go func() {
-		defer controller.runWaitGroup.Done()
-	loop:
-		for {
-
-			tunnel := controller.getNextActiveTunnel()
-			if tunnel != nil {
-				err := tunnel.serverContext.DoServerAPIRequest(
-					requestName, requestPayloadJSON)
-				if err == nil {
-					NoticeServerAPIRequestCompleted(requestID)
-					break loop
-				}
-				NoticeServerAPIRequestFailed(requestID, err)
-			}
-
-			timeout := time.After(time.Duration(retryDelaySeconds) * time.Second)
-			select {
-			case <-timeout:
-				// Attempt the request again
-
-			case <-controller.shutdownBroadcast:
-				break loop
-			}
-		}
-	}()
-}
-
 // runTunnels is the controller tunnel management main loop. It starts and stops
 // establishing tunnels based on the target tunnel pool size and the current size
 // of the pool. Tunnels are established asynchronously using worker goroutines.
@@ -553,6 +527,8 @@ func (controller *Controller) MakeServerAPIRequest(
 func (controller *Controller) runTunnels() {
 	defer controller.runWaitGroup.Done()
 
+	var clientVerificationPayload string
+
 	// Start running
 
 	controller.startEstablishing()
@@ -610,6 +586,10 @@ loop:
 				break
 			}
 
+			if clientVerificationPayload != "" {
+				establishedTunnel.SetClientVerificationPayload(clientVerificationPayload)
+			}
+
 			NoticeActiveTunnel(establishedTunnel.serverEntry.IpAddress, establishedTunnel.protocol)
 
 			if tunnelCount == 1 {
@@ -652,6 +632,9 @@ loop:
 				controller.stopEstablishing()
 			}
 
+		case clientVerificationPayload = <-controller.newClientVerificationPayload:
+			controller.setClientVerificationPayloadForActiveTunnels(clientVerificationPayload)
+
 		case <-controller.shutdownBroadcast:
 			break loop
 		}
@@ -872,6 +855,19 @@ func (controller *Controller) isActiveTunnelServerEntry(serverEntry *ServerEntry
 	return false
 }
 
+// setClientVerificationPayloadForActiveTunnels triggers the client verification
+// request for all active tunnels.
+func (controller *Controller) setClientVerificationPayloadForActiveTunnels(
+	clientVerificationPayload string) {
+
+	controller.tunnelMutex.Lock()
+	defer controller.tunnelMutex.Unlock()
+
+	for _, activeTunnel := range controller.tunnels {
+		activeTunnel.SetClientVerificationPayload(clientVerificationPayload)
+	}
+}
+
 // Dial selects an active tunnel and establishes a port forward
 // connection through the selected tunnel. Failure to connect is considered
 // a port foward failure, for the purpose of monitoring tunnel health.

psiphon/notice.go

@@ -337,12 +337,8 @@ func NoticeRemoteServerListDownloaded(filename string) {
 	outputNotice("RemoteServerListDownloaded", false, false, "filename", filename)
 }
 
-func NoticeServerAPIRequestCompleted(requestID string) {
-	outputNotice("NoticeServerAPIRequestCompleted", false, false, "requestID", requestID)
-}
-
-func NoticeServerAPIRequestFailed(requestID string, err error) {
-	outputNotice("NoticeServerAPIRequestFailed", false, false, "requestID", requestID, "error", err.Error())
+func NoticeClientVerificationRequestCompleted(ipAddress string) {
+	outputNotice("NoticeClientVerificationRequestCompleted", true, false, "ipAddress", ipAddress)
 }
 
 type repetitiveNoticeState struct {

psiphon/serverApi.go

@@ -553,15 +553,18 @@ func RecordTunnelStats(
 	return StoreTunnelStats(tunnelStatsJson)
 }
 
-// DoServerAPIRequest performs a caller-defined Server API request.
-// See description in Controller.MakeServerAPIRequest.
-func (serverContext *ServerContext) DoServerAPIRequest(
-	requestName, requestPayload string) error {
+// DoClientVerificationRequest performs the client_verification API
+// request. This request is used to verify that the client is a
+// valid Psiphon client, which will determine how the server treats
+// the client traffic. The proof-of-validity is platform-specific
+// and the payload is opaque to this function but assumed to be JSON.
+func (serverContext *ServerContext) DoClientVerificationRequest(
+	verificationPayload string) error {
 
 	return serverContext.doPostRequest(
-		buildRequestUrl(serverContext.baseRequestUrl, requestName),
+		buildRequestUrl(serverContext.baseRequestUrl, "client_verification"),
 		"application/json",
-		bytes.NewReader([]byte(requestPayload)))
+		bytes.NewReader([]byte(verificationPayload)))
 }
 
 // doGetRequest makes a tunneled HTTPS request and returns the response body.

psiphon/tunnel.go

@@ -63,22 +63,23 @@ type TunnelOwner interface {
 // tunnel includes a network connection to the specified server
 // and an SSH session built on top of that transport.
 type Tunnel struct {
-	mutex                    *sync.Mutex
-	config                   *Config
-	untunneledDialConfig     *DialConfig
-	isDiscarded              bool
-	isClosed                 bool
-	serverEntry              *ServerEntry
-	serverContext            *ServerContext
-	protocol                 string
-	conn                     net.Conn
-	sshClient                *ssh.Client
-	operateWaitGroup         *sync.WaitGroup
-	shutdownOperateBroadcast chan struct{}
-	signalPortForwardFailure chan struct{}
-	totalPortForwardFailures int
-	startTime                time.Time
-	meekStats                *MeekStats
+	mutex                        *sync.Mutex
+	config                       *Config
+	untunneledDialConfig         *DialConfig
+	isDiscarded                  bool
+	isClosed                     bool
+	serverEntry                  *ServerEntry
+	serverContext                *ServerContext
+	protocol                     string
+	conn                         net.Conn
+	sshClient                    *ssh.Client
+	operateWaitGroup             *sync.WaitGroup
+	shutdownOperateBroadcast     chan struct{}
+	signalPortForwardFailure     chan struct{}
+	totalPortForwardFailures     int
+	startTime                    time.Time
+	meekStats                    *MeekStats
+	newClientVerificationPayload chan string
 }
 
 // EstablishTunnel first makes a network transport connection to the
@@ -135,6 +136,9 @@ func EstablishTunnel(
 		// not listening. Senders should not block.
 		signalPortForwardFailure: make(chan struct{}, 1),
 		meekStats:                meekStats,
+		// Buffer allows SetClientVerificationPayload to submit one new payload
+		// without blocking or dropping it.
+		newClientVerificationPayload: make(chan string, 1),
 	}
 
 	// Create a new Psiphon API server context for this tunnel. This includes
@@ -262,6 +266,17 @@ func (tunnel *Tunnel) SignalComponentFailure() {
 	tunnel.Close(false)
 }
 
+// SetClientVerificationPayload triggers a client verification request, for this
+// tunnel, with the specified verifiction payload. If the tunnel is not yet established,
+// the payload/request is enqueued. If a payload/request is already eneueued, the
+// new payload is dropped.
+func (tunnel *Tunnel) SetClientVerificationPayload(clientVerificationPayload string) {
+	select {
+	case tunnel.newClientVerificationPayload <- clientVerificationPayload:
+	default:
+	}
+}
+
 // TunneledConn implements net.Conn and wraps a port foward connection.
 // It is used to hook into Read and Write to observe I/O errors and
 // report these errors back to the tunnel monitor as port forward failures.
@@ -767,6 +782,39 @@ func (tunnel *Tunnel) operateTunnel(tunnelOwner TunnelOwner) {
 		}
 	}()
 
+	requestsWaitGroup.Add(1)
+	signalStopClientVerificationRequests := make(chan struct{})
+	go func() {
+		defer requestsWaitGroup.Done()
+
+		clientVerificationPayload := ""
+		for {
+			// TODO: use reflect.SelectCase?
+			if clientVerificationPayload == "" {
+				select {
+				case clientVerificationPayload = <-tunnel.newClientVerificationPayload:
+				case <-signalStopClientVerificationRequests:
+					return
+				}
+			} else {
+				// When clientVerificationPayload is not "", the request for that
+				// payload so retry after a delay. Will use a new payload instead
+				// if that arrives in the meantime.
+				timeout := time.After(PSIPHON_API_CLIENT_VERIFICATION_REQUEST_RETRY_PERIOD)
+				select {
+				case <-timeout:
+				case clientVerificationPayload = <-tunnel.newClientVerificationPayload:
+				case <-signalStopClientVerificationRequests:
+					return
+				}
+			}
+			if sendClientVerification(tunnel, clientVerificationPayload) {
+				clientVerificationPayload = ""
+			}
+
+		}
+	}()
+
 	shutdown := false
 	var err error
 	for !shutdown && err == nil {
@@ -833,6 +881,7 @@ func (tunnel *Tunnel) operateTunnel(tunnelOwner TunnelOwner) {
 
 	close(signalSshKeepAlive)
 	close(signalStatusRequest)
+	close(signalStopClientVerificationRequests)
 	requestsWaitGroup.Wait()
 
 	// Capture bytes transferred since the last noticeBytesTransferredTicker tick
@@ -972,3 +1021,27 @@ func sendUntunneledStats(tunnel *Tunnel, isShutdown bool) {
 		NoticeAlert("TryUntunneledStatusRequest failed for %s: %s", tunnel.serverEntry.IpAddress, err)
 	}
 }
+
+// sendClientVerification is a helper for sending a client verification request
+// to the server.
+func sendClientVerification(tunnel *Tunnel, clientVerificationPayload string) bool {
+
+	// Tunnel does not have a serverContext when DisableApi is set
+	if tunnel.serverContext == nil {
+		return true
+	}
+
+	// Skip when tunnel is discarded
+	if tunnel.IsDiscarded() {
+		return true
+	}
+
+	err := tunnel.serverContext.DoClientVerificationRequest(clientVerificationPayload)
+	if err != nil {
+		NoticeAlert("DoClientVerificationRequest failed for %s: %s", tunnel.serverEntry.IpAddress, err)
+	} else {
+		NoticeClientVerificationRequestCompleted(tunnel.serverEntry.IpAddress)
+	}
+
+	return err == nil
+}