Merge pull request #184 from rod-hynes/master

Psiphon API implementation

psiphon/config.go

@@ -64,7 +64,7 @@ const (
 	PSIPHON_API_STATUS_REQUEST_PADDING_MAX_BYTES         = 256
 	PSIPHON_API_CONNECTED_REQUEST_PERIOD                 = 24 * time.Hour
 	PSIPHON_API_CONNECTED_REQUEST_RETRY_PERIOD           = 5 * time.Second
-	PSIPHON_API_TUNNEL_STATS_MAX_COUNT                   = 1000
+	PSIPHON_API_TUNNEL_STATS_MAX_COUNT                   = 100
 	PSIPHON_API_CLIENT_VERIFICATION_REQUEST_RETRY_PERIOD = 5 * time.Second
 	FETCH_ROUTES_TIMEOUT_SECONDS                         = 60
 	DOWNLOAD_UPGRADE_TIMEOUT                             = 15 * time.Minute

psiphon/server/api.go

@@ -0,0 +1,602 @@
+/*
+ * Copyright (c) 2016, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package server
+
+import (
+	"crypto/subtle"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"regexp"
+	"strconv"
+	"strings"
+	"unicode"
+
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon"
+)
+
+const MAX_API_PARAMS_SIZE = 256 * 1024 // 256KB
+
+type requestJSONObject map[string]interface{}
+
+// sshAPIRequestHandler routes Psiphon API requests transported as
+// JSON objects via the SSH request mechanism.
+//
+// The API request handlers, handshakeAPIRequestHandler, etc., are
+// reused by webServer which offers the Psiphon API via web transport.
+//
+// The API request parameters and event log values follow the legacy
+// psi_web protocol and naming conventions. The API is compatible all
+// tunnel-core clients but are not backwards compatible with older
+// clients.
+//
+func sshAPIRequestHandler(
+	config *Config,
+	psinetDatabase *PsinetDatabase,
+	geoIPData GeoIPData,
+	name string,
+	requestPayload []byte) ([]byte, error) {
+
+	// Note: for SSH requests, MAX_API_PARAMS_SIZE is implicitly enforced
+	// by max SSH reqest packet size.
+
+	var params requestJSONObject
+	err := json.Unmarshal(requestPayload, &params)
+	if err != nil {
+		return nil, psiphon.ContextError(err)
+	}
+
+	switch name {
+	case psiphon.SERVER_API_HANDSHAKE_REQUEST_NAME:
+		return handshakeAPIRequestHandler(config, psinetDatabase, geoIPData, params)
+	case psiphon.SERVER_API_CONNECTED_REQUEST_NAME:
+		return connectedAPIRequestHandler(config, geoIPData, params)
+	case psiphon.SERVER_API_STATUS_REQUEST_NAME:
+		return statusAPIRequestHandler(config, geoIPData, params)
+	case psiphon.SERVER_API_CLIENT_VERIFICATION_REQUEST_NAME:
+		return clientVerificationAPIRequestHandler(config, geoIPData, params)
+	}
+
+	return nil, psiphon.ContextError(fmt.Errorf("invalid request name: %s", name))
+}
+
+// handshakeAPIRequestHandler implements the "handshake" API request.
+// Clients make the handshake immediately after establishing a tunnel
+// connection; the response tells the client what homepage to open, what
+// stats to record, etc.
+func handshakeAPIRequestHandler(
+	config *Config,
+	psinetDatabase *PsinetDatabase,
+	geoIPData GeoIPData,
+	params requestJSONObject) ([]byte, error) {
+
+	// Note: ignoring "known_servers" params
+
+	err := validateRequestParams(config, params, baseRequestParams)
+	if err != nil {
+		// TODO: fail2ban?
+		return nil, psiphon.ContextError(errors.New("invalid params"))
+	}
+
+	log.WithContextFields(
+		getRequestLogFields(
+			config,
+			"handshake",
+			geoIPData,
+			params,
+			baseRequestParams)).Info("API event")
+
+	// TODO: share struct definition with psiphon/serverApi.go?
+	// TODO: populate response data using psinet database
+
+	var handshakeResponse struct {
+		Homepages            []string            `json:"homepages"`
+		UpgradeClientVersion string              `json:"upgrade_client_version"`
+		PageViewRegexes      []map[string]string `json:"page_view_regexes"`
+		HttpsRequestRegexes  []map[string]string `json:"https_request_regexes"`
+		EncodedServerList    []string            `json:"encoded_server_list"`
+		ClientRegion         string              `json:"client_region"`
+		ServerTimestamp      string              `json:"server_timestamp"`
+	}
+
+	handshakeResponse.Homepages = psinetDatabase.GetHomepages(
+		"", "", "") // TODO: sponsorID, clientRegion, clientPlatform)
+
+	handshakeResponse.UpgradeClientVersion = psinetDatabase.GetUpgradeClientVersion(
+		"") // TODO: clientVersion)
+
+	handshakeResponse.HttpsRequestRegexes = psinetDatabase.GetHttpsRequestRegexes(
+		"", "", "") // TODO: sponsorID, clientRegion, clientPlatform)
+
+	handshakeResponse.EncodedServerList = psinetDatabase.DiscoverServers(
+		"", 0) // TODO: propagationChannelID, discoveryValue)
+
+	handshakeResponse.ClientRegion = geoIPData.Country
+
+	handshakeResponse.ServerTimestamp = psiphon.GetCurrentTimestamp()
+
+	responsePayload, err := json.Marshal(handshakeResponse)
+	if err != nil {
+		return nil, psiphon.ContextError(err)
+	}
+
+	return responsePayload, nil
+}
+
+var connectedRequestParams = append(
+	[]requestParamSpec{requestParamSpec{"last_connected", isLastConnected, 0}},
+	baseRequestParams...)
+
+// connectedAPIRequestHandler implements the "connected" API request.
+// Clients make the connected request once a tunnel connection has been
+// established and at least once per day. The last_connected input value,
+// which should be a connected_timestamp output from a previous connected
+// response, is used to calculate unique user stats.
+func connectedAPIRequestHandler(
+	config *Config, geoIPData GeoIPData, params requestJSONObject) ([]byte, error) {
+
+	err := validateRequestParams(config, params, connectedRequestParams)
+	if err != nil {
+		// TODO: fail2ban?
+		return nil, psiphon.ContextError(errors.New("invalid params"))
+	}
+
+	log.WithContextFields(
+		getRequestLogFields(
+			config,
+			"connected",
+			geoIPData,
+			params,
+			connectedRequestParams)).Info("API event")
+
+	var connectedResponse struct {
+		ConnectedTimestamp string `json:"connected_timestamp"`
+	}
+
+	connectedResponse.ConnectedTimestamp =
+		psiphon.TruncateTimestampToHour(psiphon.GetCurrentTimestamp())
+
+	responsePayload, err := json.Marshal(connectedResponse)
+	if err != nil {
+		return nil, psiphon.ContextError(err)
+	}
+
+	return responsePayload, nil
+}
+
+var statusRequestParams = append(
+	[]requestParamSpec{requestParamSpec{"connected", isBooleanFlag, 0}},
+	baseRequestParams...)
+
+// statusAPIRequestHandler implements the "status" API request.
+// Clients make periodic status requests which deliver client-side
+// recorded data transfer and tunnel duration stats.
+func statusAPIRequestHandler(
+	config *Config, geoIPData GeoIPData, params requestJSONObject) ([]byte, error) {
+
+	err := validateRequestParams(config, params, statusRequestParams)
+	if err != nil {
+		// TODO: fail2ban?
+		return nil, psiphon.ContextError(errors.New("invalid params"))
+	}
+
+	statusData, err := getJSONObjectRequestParam(params, "statusData")
+	if err != nil {
+		return nil, psiphon.ContextError(err)
+	}
+
+	// Overall bytes transferred stats
+
+	bytesTransferred, err := getInt64RequestParam(statusData, "bytes_transferred")
+	if err != nil {
+		return nil, psiphon.ContextError(err)
+	}
+	bytesTransferredFields := getRequestLogFields(
+		config, "bytes_transferred", geoIPData, params, statusRequestParams)
+	bytesTransferredFields["bytes"] = bytesTransferred
+	log.WithContextFields(bytesTransferredFields).Info("API event")
+
+	// Domain bytes transferred stats
+
+	hostBytes, err := getMapStringInt64RequestParam(statusData, "host_bytes")
+	if err != nil {
+		return nil, psiphon.ContextError(err)
+	}
+	domainBytesFields := getRequestLogFields(
+		config, "domain_bytes", geoIPData, params, statusRequestParams)
+	for domain, bytes := range hostBytes {
+		domainBytesFields["domain"] = domain
+		domainBytesFields["bytes"] = bytes
+		log.WithContextFields(domainBytesFields).Info("API event")
+	}
+
+	// Tunnel duration and bytes transferred stats
+
+	tunnelStats, err := getJSONObjectArrayRequestParam(statusData, "tunnel_stats")
+	if err != nil {
+		return nil, psiphon.ContextError(err)
+	}
+	sessionFields := getRequestLogFields(
+		config, "session", geoIPData, params, statusRequestParams)
+	for _, tunnelStat := range tunnelStats {
+
+		sessionID, err := getStringRequestParam(tunnelStat, "session_id")
+		if err != nil {
+			return nil, psiphon.ContextError(err)
+		}
+		sessionFields["session_id"] = sessionID
+
+		tunnelNumber, err := getInt64RequestParam(tunnelStat, "tunnel_number")
+		if err != nil {
+			return nil, psiphon.ContextError(err)
+		}
+		sessionFields["tunnel_number"] = tunnelNumber
+
+		tunnelServerIPAddress, err := getStringRequestParam(tunnelStat, "tunnel_server_ip_address")
+		if err != nil {
+			return nil, psiphon.ContextError(err)
+		}
+		sessionFields["tunnel_server_ip_address"] = tunnelServerIPAddress
+
+		serverHandshakeTimestamp, err := getStringRequestParam(tunnelStat, "server_handshake_timestamp")
+		if err != nil {
+			return nil, psiphon.ContextError(err)
+		}
+		sessionFields["server_handshake_timestamp"] = serverHandshakeTimestamp
+
+		duration, err := getInt64RequestParam(tunnelStat, "duration")
+		if err != nil {
+			return nil, psiphon.ContextError(err)
+		}
+		// Client reports durations in nanoseconds; divide to get to milliseconds
+		sessionFields["duration"] = duration / 1000000
+
+		totalBytesSent, err := getInt64RequestParam(tunnelStat, "total_bytes_sent")
+		if err != nil {
+			return nil, psiphon.ContextError(err)
+		}
+		sessionFields["total_bytes_sent"] = totalBytesSent
+
+		totalBytesReceived, err := getInt64RequestParam(tunnelStat, "total_bytes_received")
+		if err != nil {
+			return nil, psiphon.ContextError(err)
+		}
+		sessionFields["total_bytes_received"] = totalBytesReceived
+
+		log.WithContextFields(sessionFields).Info("API event")
+	}
+
+	return make([]byte, 0), nil
+}
+
+// clientVerificationAPIRequestHandler implements the
+// "client verification" API request. Clients make the client
+// verification request once per tunnel connection. The payload
+// attests that client is a legitimate Psiphon client.
+func clientVerificationAPIRequestHandler(
+	config *Config, geoIPData GeoIPData, params requestJSONObject) ([]byte, error) {
+
+	err := validateRequestParams(config, params, baseRequestParams)
+	if err != nil {
+		// TODO: fail2ban?
+		return nil, psiphon.ContextError(errors.New("invalid params"))
+	}
+
+	// TODO: implement
+
+	return make([]byte, 0), nil
+}
+
+type requestParamSpec struct {
+	name      string
+	validator func(*Config, string) bool
+	flags     int32
+}
+
+const (
+	requestParamOptional  = 1
+	requestParamNotLogged = 2
+)
+
+// baseRequestParams is the list of required and optional
+// request parameters; derived from COMMON_INPUTS and
+// OPTIONAL_COMMON_INPUTS in psi_web.
+var baseRequestParams = []requestParamSpec{
+	requestParamSpec{"server_secret", isServerSecret, requestParamNotLogged},
+	requestParamSpec{"client_session_id", isHexDigits, 0},
+	requestParamSpec{"propagation_channel_id", isHexDigits, 0},
+	requestParamSpec{"sponsor_id", isHexDigits, 0},
+	requestParamSpec{"client_version", isDigits, 0},
+	requestParamSpec{"client_platform", isClientPlatform, 0},
+	requestParamSpec{"relay_protocol", isRelayProtocol, 0},
+	requestParamSpec{"tunnel_whole_device", isBooleanFlag, 0},
+	requestParamSpec{"device_region", isRegionCode, requestParamOptional},
+	requestParamSpec{"meek_dial_address", isDialAddress, requestParamOptional},
+	requestParamSpec{"meek_resolved_ip_address", isIPAddress, requestParamOptional},
+	requestParamSpec{"meek_sni_server_name", isDomain, requestParamOptional},
+	requestParamSpec{"meek_host_header", isHostHeader, requestParamOptional},
+	requestParamSpec{"meek_transformed_host_name", isBooleanFlag, requestParamOptional},
+	requestParamSpec{"server_entry_region", isRegionCode, requestParamOptional},
+	requestParamSpec{"server_entry_source", isServerEntrySource, requestParamOptional},
+	requestParamSpec{"server_entry_timestamp", isISO8601Date, requestParamOptional},
+}
+
+func validateRequestParams(
+	config *Config,
+	params requestJSONObject,
+	expectedParams []requestParamSpec) error {
+
+	for _, expectedParam := range expectedParams {
+		value := params[expectedParam.name]
+		if value == nil {
+			if expectedParam.flags&requestParamOptional != 0 {
+				continue
+			}
+			return psiphon.ContextError(
+				fmt.Errorf("missing required param: %s", expectedParam.name))
+		}
+		strValue, ok := value.(string)
+		if !ok {
+			return psiphon.ContextError(
+				fmt.Errorf("unexpected param type: %s", expectedParam.name))
+		}
+		if !expectedParam.validator(config, strValue) {
+			return psiphon.ContextError(
+				fmt.Errorf("invalid param: %s", expectedParam.name))
+		}
+	}
+
+	return nil
+}
+
+// getRequestLogFields makes LogFields to log the API event following
+// the legacy psi_web and current ELK naming conventions.
+func getRequestLogFields(
+	config *Config,
+	eventName string,
+	geoIPData GeoIPData,
+	params requestJSONObject,
+	expectedParams []requestParamSpec) LogFields {
+
+	logFields := make(LogFields)
+
+	logFields["event_name"] = eventName
+	logFields["host_id"] = config.HostID
+
+	// In psi_web, the space replacement was done to accommodate space
+	// delimited logging, which is no longer required; we retain the
+	// transformation so that stats aggregation isn't impacted.
+	logFields["client_region"] = strings.Replace(geoIPData.Country, " ", "_", -1)
+	logFields["client_city"] = strings.Replace(geoIPData.City, " ", "_", -1)
+	logFields["client_isp"] = strings.Replace(geoIPData.ISP, " ", "_", -1)
+
+	for _, expectedParam := range expectedParams {
+		value := params[expectedParam.name]
+		if value == nil {
+			// Skip optional params
+			continue
+		}
+		strValue, ok := value.(string)
+		if !ok {
+			// This type assertion should be checked already in
+			// validateRequestParams, so failure is unexpected.
+			continue
+		}
+		// Special cases:
+		// - Number fields are encoded as integer types.
+		// - For ELK performance we record these domain-or-IP
+		//   fields as one of two different values based on type;
+		//   we also omit port from host:port fields for now.
+		switch expectedParam.name {
+		case "client_version":
+			intValue, _ := strconv.Atoi(strValue)
+			logFields[expectedParam.name] = intValue
+		case "meek_dial_address":
+			host, _, _ := net.SplitHostPort(strValue)
+			if isIPAddress(config, host) {
+				logFields["meek_dial_ip_address"] = host
+			} else {
+				logFields["meek_dial_domain"] = host
+			}
+		case "meek_host_header":
+			host, _, _ := net.SplitHostPort(strValue)
+			logFields[expectedParam.name] = host
+		default:
+			logFields[expectedParam.name] = strValue
+		}
+	}
+
+	return logFields
+}
+
+func getStringRequestParam(params requestJSONObject, name string) (string, error) {
+	if params[name] == nil {
+		return "", psiphon.ContextError(errors.New("missing param"))
+	}
+	value, ok := params[name].(string)
+	if !ok {
+		return "", psiphon.ContextError(errors.New("invalid param"))
+	}
+	return value, nil
+}
+
+func getInt64RequestParam(params requestJSONObject, name string) (int64, error) {
+	if params[name] == nil {
+		return 0, psiphon.ContextError(errors.New("missing param"))
+	}
+	value, ok := params[name].(int64)
+	if !ok {
+		return 0, psiphon.ContextError(errors.New("invalid param"))
+	}
+	return value, nil
+}
+
+func getJSONObjectRequestParam(params requestJSONObject, name string) (requestJSONObject, error) {
+	if params[name] == nil {
+		return nil, psiphon.ContextError(errors.New("missing param"))
+	}
+	value, ok := params[name].(requestJSONObject)
+	if !ok {
+		return nil, psiphon.ContextError(errors.New("invalid param"))
+	}
+	return value, nil
+}
+
+func getJSONObjectArrayRequestParam(params requestJSONObject, name string) ([]requestJSONObject, error) {
+	if params[name] == nil {
+		return nil, psiphon.ContextError(errors.New("missing param"))
+	}
+	value, ok := params[name].([]requestJSONObject)
+	if !ok {
+		return nil, psiphon.ContextError(errors.New("invalid param"))
+	}
+	return value, nil
+}
+
+func getMapStringInt64RequestParam(params requestJSONObject, name string) (map[string]int64, error) {
+	if params[name] == nil {
+		return nil, psiphon.ContextError(errors.New("missing param"))
+	}
+	value, ok := params[name].(map[string]int64)
+	if !ok {
+		return nil, psiphon.ContextError(errors.New("invalid param"))
+	}
+	return value, nil
+}
+
+// Input validators follow the legacy validations rules in psi_web.
+
+func isServerSecret(config *Config, value string) bool {
+	return subtle.ConstantTimeCompare(
+		[]byte(value),
+		[]byte(config.WebServerSecret)) == 1
+}
+
+func isHexDigits(_ *Config, value string) bool {
+	return -1 == strings.IndexFunc(value, func(c rune) bool {
+		return !unicode.Is(unicode.ASCII_Hex_Digit, c)
+	})
+}
+
+func isDigits(_ *Config, value string) bool {
+	return -1 == strings.IndexFunc(value, func(c rune) bool {
+		return c < '0' || c > '9'
+	})
+}
+
+func isClientPlatform(_ *Config, value string) bool {
+	return -1 == strings.IndexFunc(value, func(c rune) bool {
+		// Note: stricter than psi_web's Python string.whitespace
+		return unicode.Is(unicode.White_Space, c)
+	})
+}
+
+func isRelayProtocol(_ *Config, value string) bool {
+	return psiphon.Contains(psiphon.SupportedTunnelProtocols, value)
+}
+
+func isBooleanFlag(_ *Config, value string) bool {
+	return value == "0" || value == "1"
+}
+
+func isRegionCode(_ *Config, value string) bool {
+	if len(value) != 2 {
+		return false
+	}
+	return -1 == strings.IndexFunc(value, func(c rune) bool {
+		return c < 'A' || c > 'Z'
+	})
+}
+
+func isDialAddress(config *Config, value string) bool {
+	// "<host>:<port>", where <host> is a domain or IP address
+	parts := strings.Split(value, ":")
+	if len(parts) != 2 {
+		return false
+	}
+	if !isIPAddress(config, parts[0]) && !isDomain(config, parts[0]) {
+		return false
+	}
+	if !isDigits(config, parts[1]) {
+		return false
+	}
+	port, err := strconv.Atoi(parts[1])
+	if err != nil {
+		return false
+	}
+	return port > 0 && port < 65536
+}
+
+func isIPAddress(_ *Config, value string) bool {
+	return net.ParseIP(value) != nil
+}
+
+var isDomainRegex = regexp.MustCompile("[a-zA-Z\\d-]{1,63}$")
+
+func isDomain(_ *Config, value string) bool {
+
+	// From: http://stackoverflow.com/questions/2532053/validate-a-hostname-string
+	//
+	// "ensures that each segment
+	//    * contains at least one character and a maximum of 63 characters
+	//    * consists only of allowed characters
+	//    * doesn't begin or end with a hyphen"
+	//
+
+	if len(value) > 255 {
+		return false
+	}
+	value = strings.TrimSuffix(value, ".")
+	for _, part := range strings.Split(value, ".") {
+		// Note: regexp doesn't support the following Perl expression which
+		// would check for '-' prefix/suffix: "(?!-)[a-zA-Z\\d-]{1,63}(?<!-)$"
+		if strings.HasPrefix(part, "-") || strings.HasSuffix(part, "-") {
+			return false
+		}
+		if !isDomainRegex.Match([]byte(part)) {
+			return false
+		}
+	}
+	return true
+}
+
+func isHostHeader(config *Config, value string) bool {
+	// "<host>:<port>", where <host> is a domain or IP address and ":<port>" is optional
+	if strings.Contains(value, ":") {
+		return isDialAddress(config, value)
+	}
+	return isIPAddress(config, value) || isDomain(config, value)
+}
+
+func isServerEntrySource(_ *Config, value string) bool {
+	return psiphon.Contains(psiphon.SupportedServerEntrySources, value)
+}
+
+var isISO8601DateRegex = regexp.MustCompile(
+	"(?P<year>[0-9]{4})-(?P<month>[0-9]{1,2})-(?P<day>[0-9]{1,2})T(?P<hour>[0-9]{2}):(?P<minute>[0-9]{2}):(?P<second>[0-9]{2})(\\.(?P<fraction>[0-9]+))?(?P<timezone>Z|(([-+])([0-9]{2}):([0-9]{2})))")
+
+func isISO8601Date(_ *Config, value string) bool {
+	return isISO8601DateRegex.Match([]byte(value))
+}
+
+func isLastConnected(config *Config, value string) bool {
+	return value == "None" || value == "Unknown" || isISO8601Date(config, value)
+}

psiphon/server/config.go

@@ -105,6 +105,14 @@ type Config struct {
 	// set, redis is used to store per-session GeoIP information.
 	RedisServerAddress string
 
+	// PsinetDatabaseFilename is the path of the Psiphon automation
+	// jsonpickle format Psiphon API data file.
+	PsinetDatabaseFilename string
+
+	// HostID is the ID of the server host; this is used for API
+	// event logging.
+	HostID string
+
 	// ServerIPAddress is the public IP address of the server.
 	ServerIPAddress string
 
@@ -561,6 +569,7 @@ func GenerateConfig(
 		SyslogTag:                      "psiphon-server",
 		Fail2BanFormat:                 "Authentication failure for psiphon-client from %s",
 		GeoIPDatabaseFilename:          "",
+		HostID:                         "example-host-id",
 		ServerIPAddress:                serverIPaddress,
 		DiscoveryValueHMACKey:          discoveryValueHMACKey,
 		WebServerPort:                  webServerPort,
@@ -609,7 +618,11 @@ func GenerateConfig(
 	lines := strings.Split(webServerCertificate, "\n")
 	strippedWebServerCertificate := strings.Join(lines[1:len(lines)-2], "")
 
-	capabilities := make([]string, 0)
+	capabilities := []string{psiphon.CAPABILITY_SSH_API_REQUESTS}
+
+	if webServerPort != 0 {
+		capabilities = append(capabilities, psiphon.CAPABILITY_UNTUNNELED_WEB_API_REQUESTS)
+	}
 
 	for protocol, _ := range tunnelProtocolPorts {
 		capabilities = append(capabilities, psiphon.GetCapability(protocol))

psiphon/server/psinet.go

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2016, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package server
+
+// PsinetDatabase serves Psiphon API data requests. It's safe for
+// concurrent usage.
+type PsinetDatabase struct {
+}
+
+// NewPsinetDatabase initializes a PsinetDatabase. It loads the specified
+// file, which should be in the Psiphon automation jsonpickle format, and
+// prepares to serve data requests.
+// The input "" is valid and returns a functional PsinetDatabase with no
+// data.
+func NewPsinetDatabase(filename string) (*PsinetDatabase, error) {
+
+	// TODO: implement
+
+	return &PsinetDatabase{}, nil
+}
+
+// GetHomepages returns a list of  home pages for the specified sponsor,
+// region, and platform.
+func (psinet *PsinetDatabase) GetHomepages(sponsorID, clientRegion, clientPlatform string) []string {
+
+	// TODO: implement
+
+	return make([]string, 0)
+}
+
+// GetUpgradeClientVersion returns a new client version when an upgrade is
+// indicated for the specified client current version. The result is "" when
+// no upgrade is available.
+func (psinet *PsinetDatabase) GetUpgradeClientVersion(clientVersion string) string {
+
+	// TODO: implement
+
+	return ""
+}
+
+// GetHttpsRequestRegexes returns bytes transferred stats regexes for the
+// specified client inputs.
+func (psinet *PsinetDatabase) GetHttpsRequestRegexes(sponsorID, clientRegion, clientPlatform string) []map[string]string {
+
+	return make([]map[string]string, 0)
+}
+
+// DiscoverServers selects new encoded server entries to be "discovered" by
+// the client, using the discoveryValue as the input into the discovery algorithm.
+func (psinet *PsinetDatabase) DiscoverServers(propagationChannelID string, discoveryValue int) []string {
+
+	// TODO: implement
+
+	return make([]string, 0)
+}

psiphon/server/services.go

@@ -58,6 +58,12 @@ func RunServices(encodedConfigs [][]byte) error {
 		return psiphon.ContextError(err)
 	}
 
+	psinetDatabase, err := NewPsinetDatabase(config.PsinetDatabaseFilename)
+	if err != nil {
+		log.WithContextFields(LogFields{"error": err}).Error("init PsinetDatabase failed")
+		return psiphon.ContextError(err)
+	}
+
 	if config.UseRedis() {
 		err = InitRedis(config)
 		if err != nil {
@@ -70,7 +76,7 @@ func RunServices(encodedConfigs [][]byte) error {
 	shutdownBroadcast := make(chan struct{})
 	errors := make(chan error)
 
-	tunnelServer, err := NewTunnelServer(config, shutdownBroadcast)
+	tunnelServer, err := NewTunnelServer(config, psinetDatabase, shutdownBroadcast)
 	if err != nil {
 		log.WithContextFields(LogFields{"error": err}).Error("init tunnel server failed")
 		return psiphon.ContextError(err)
@@ -97,7 +103,7 @@ func RunServices(encodedConfigs [][]byte) error {
 		waitGroup.Add(1)
 		go func() {
 			defer waitGroup.Done()
-			err := RunWebServer(config, shutdownBroadcast)
+			err := RunWebServer(config, psinetDatabase, shutdownBroadcast)
 			select {
 			case errors <- err:
 			default:

psiphon/server/tunnelServer.go

@@ -52,9 +52,12 @@ type TunnelServer struct {
 
 // NewTunnelServer initializes a new tunnel server.
 func NewTunnelServer(
-	config *Config, shutdownBroadcast <-chan struct{}) (*TunnelServer, error) {
+	config *Config,
+	psinetDatabase *PsinetDatabase,
+	shutdownBroadcast <-chan struct{}) (*TunnelServer, error) {
 
-	sshServer, err := newSSHServer(config, shutdownBroadcast)
+	sshServer, err := newSSHServer(
+		config, psinetDatabase, shutdownBroadcast)
 	if err != nil {
 		return nil, psiphon.ContextError(err)
 	}
@@ -180,6 +183,7 @@ type sshClientID uint64
 
 type sshServer struct {
 	config            *Config
+	psinetDatabase    *PsinetDatabase
 	shutdownBroadcast <-chan struct{}
 	sshHostKey        ssh.Signer
 	nextClientID      sshClientID
@@ -190,6 +194,7 @@ type sshServer struct {
 
 func newSSHServer(
 	config *Config,
+	psinetDatabase *PsinetDatabase,
 	shutdownBroadcast <-chan struct{}) (*sshServer, error) {
 
 	privateKey, err := ssh.ParseRawPrivateKey([]byte(config.SSHPrivateKey))
@@ -205,6 +210,7 @@ func newSSHServer(
 
 	return &sshServer{
 		config:            config,
+		psinetDatabase:    psinetDatabase,
 		shutdownBroadcast: shutdownBroadcast,
 		sshHostKey:        signer,
 		nextClientID:      1,
@@ -462,9 +468,7 @@ func (sshServer *sshServer) handleClient(tunnelProtocol string, clientConn net.C
 	}
 	defer sshServer.unregisterClient(clientID)
 
-	go ssh.DiscardRequests(result.requests)
-
-	sshClient.handleChannels(result.channels)
+	sshClient.runClient(result.channels, result.requests)
 
 	// TODO: clientConn.Close()?
 }
@@ -593,7 +597,40 @@ func (sshClient *sshClient) stop() {
 	sshClient.Unlock()
 }
 
-func (sshClient *sshClient) handleChannels(channels <-chan ssh.NewChannel) {
+// runClient handles/dispatches new channel and new requests from the client.
+// When the SSH client connection closes, both the channels and requests channels
+// will close and runClient will exit.
+func (sshClient *sshClient) runClient(
+	channels <-chan ssh.NewChannel, requests <-chan *ssh.Request) {
+
+	requestsWaitGroup := new(sync.WaitGroup)
+	requestsWaitGroup.Add(1)
+	go func() {
+		defer requestsWaitGroup.Done()
+
+		for request := range requests {
+
+			// requests are processed serially; responses must be sent in request order.
+			responsePayload, err := sshAPIRequestHandler(
+				sshClient.sshServer.config,
+				sshClient.sshServer.psinetDatabase,
+				sshClient.geoIPData,
+				request.Type,
+				request.Payload)
+
+			if err == nil {
+				err = request.Reply(true, responsePayload)
+			} else {
+				log.WithContextFields(LogFields{"error": err}).Warning("request failed")
+				err = request.Reply(false, nil)
+			}
+			if err != nil {
+				log.WithContextFields(LogFields{"error": err}).Warning("response failed")
+			}
+
+		}
+	}()
+
 	for newChannel := range channels {
 
 		if newChannel.ChannelType() != "direct-tcpip" {
@@ -605,6 +642,8 @@ func (sshClient *sshClient) handleChannels(channels <-chan ssh.NewChannel) {
 		sshClient.channelHandlerWaitGroup.Add(1)
 		go sshClient.handleNewPortForwardChannel(newChannel)
 	}
+
+	requestsWaitGroup.Wait()
 }
 
 func (sshClient *sshClient) rejectNewChannel(newChannel ssh.NewChannel, reason ssh.RejectionReason, message string) {

psiphon/server/webServer.go

@@ -20,7 +20,6 @@
 package server
 
 import (
-	"crypto/subtle"
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
@@ -34,20 +33,33 @@ import (
 )
 
 type webServer struct {
-	serveMux *http.ServeMux
-	config   *Config
+	serveMux       *http.ServeMux
+	config         *Config
+	psinetDatabase *PsinetDatabase
 }
 
-// RunWebServer runs a web server which responds to the following Psiphon API
-// web requests: handshake, connected, and status. At this time, this web
-// server is a stub for stand-alone testing. It provides the minimal response
-// required by the tunnel-core client. It doesn't return read landing pages,
-// serer discovery results, or other handshake response values. It also does
-// not log stats in the standard way.
-func RunWebServer(config *Config, shutdownBroadcast <-chan struct{}) error {
+// RunWebServer runs a web server which supports tunneled and untunneled
+// Psiphon API requests.
+//
+// The HTTP request handlers are light wrappers around the base Psiphon
+// API request handlers from the SSH API transport. The SSH API transport
+// is preferred by new clients; however the web API transport is still
+// required for untunneled final status requests. The web API transport
+// may be retired once untunneled final status requests are made obsolete
+// (e.g., by server-side bytes transferred stats, by client-side local
+// storage of stats for retry, or some other future development).
+//
+// The API is compatible with all tunnel-core clients but not backwards
+// compatible with older clients.
+//
+func RunWebServer(
+	config *Config,
+	psinetDatabase *PsinetDatabase,
+	shutdownBroadcast <-chan struct{}) error {
 
 	webServer := &webServer{
-		config: config,
+		config:         config,
+		psinetDatabase: psinetDatabase,
 	}
 
 	serveMux := http.NewServeMux()
@@ -73,11 +85,12 @@ func RunWebServer(config *Config, shutdownBroadcast <-chan struct{}) error {
 
 	server := &psiphon.HTTPSServer{
 		http.Server{
-			Handler:      serveMux,
-			TLSConfig:    tlsConfig,
-			ReadTimeout:  WEB_SERVER_READ_TIMEOUT,
-			WriteTimeout: WEB_SERVER_WRITE_TIMEOUT,
-			ErrorLog:     golanglog.New(logWriter, "", 0),
+			MaxHeaderBytes: MAX_API_PARAMS_SIZE,
+			Handler:        serveMux,
+			TLSConfig:      tlsConfig,
+			ReadTimeout:    WEB_SERVER_READ_TIMEOUT,
+			WriteTimeout:   WEB_SERVER_WRITE_TIMEOUT,
+			ErrorLog:       golanglog.New(logWriter, "", 0),
 		},
 	}
 
@@ -131,54 +144,72 @@ func RunWebServer(config *Config, shutdownBroadcast <-chan struct{}) error {
 	return err
 }
 
-func (webServer *webServer) checkWebServerSecret(r *http.Request) bool {
-	return subtle.ConstantTimeCompare(
-		[]byte(r.URL.Query().Get("server_secret")),
-		[]byte(webServer.config.WebServerSecret)) == 1
+// convertHTTPRequestToAPIRequest converts the HTTP request query
+// parameters and request body to the JSON object import format
+// expected by the API request handlers.
+func convertHTTPRequestToAPIRequest(
+	w http.ResponseWriter,
+	r *http.Request,
+	requestBodyName string) (requestJSONObject, error) {
+
+	params := make(requestJSONObject)
+
+	for name, values := range r.URL.Query() {
+		for _, value := range values {
+			params[name] = value
+			// Note: multiple values per name are ignored
+			break
+		}
+	}
+
+	if requestBodyName != "" {
+		r.Body = http.MaxBytesReader(w, r.Body, MAX_API_PARAMS_SIZE)
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			return nil, psiphon.ContextError(err)
+		}
+		var bodyParams requestJSONObject
+		err = json.Unmarshal(body, &bodyParams)
+		if err != nil {
+			return nil, psiphon.ContextError(err)
+		}
+		params[requestBodyName] = bodyParams
+	}
+
+	return params, nil
 }
 
-func (webServer *webServer) handshakeHandler(w http.ResponseWriter, r *http.Request) {
+func (webServer *webServer) lookupGeoIPData(params requestJSONObject) GeoIPData {
 
-	if !webServer.checkWebServerSecret(r) {
-		// TODO: log more details?
-		log.WithContext().Warning("checkWebServerSecret failed")
-		// TODO: psi_web returns NotFound in this case
-		w.WriteHeader(http.StatusForbidden)
-		return
-	}
+	// TODO: implement
 
-	// TODO: validate; proper log
-	log.WithContextFields(LogFields{"queryParams": r.URL.Query()}).Info("handshake")
+	return NewGeoIPData()
+}
 
-	// TODO: necessary, in case client sends bogus request body?
-	_, err := ioutil.ReadAll(r.Body)
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		return
-	}
+func (webServer *webServer) handshakeHandler(w http.ResponseWriter, r *http.Request) {
 
-	// TODO: backwards compatibility cases (only sending the new JSON format response line)
-	// TODO: share struct definition with psiphon/serverApi.go?
-	// TODO: populate more response data
-
-	var handshakeConfig struct {
-		Homepages            []string            `json:"homepages"`
-		UpgradeClientVersion string              `json:"upgrade_client_version"`
-		PageViewRegexes      []map[string]string `json:"page_view_regexes"`
-		HttpsRequestRegexes  []map[string]string `json:"https_request_regexes"`
-		EncodedServerList    []string            `json:"encoded_server_list"`
-		ClientRegion         string              `json:"client_region"`
-		ServerTimestamp      string              `json:"server_timestamp"`
-	}
+	params, err := convertHTTPRequestToAPIRequest(w, r, "")
 
-	handshakeConfig.ServerTimestamp = psiphon.GetCurrentTimestamp()
+	var responsePayload []byte
+	if err == nil {
+		responsePayload, err = handshakeAPIRequestHandler(
+			webServer.config,
+			webServer.psinetDatabase,
+			webServer.lookupGeoIPData(params),
+			params)
+	}
 
-	jsonPayload, err := json.Marshal(handshakeConfig)
 	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
+		log.WithContextFields(LogFields{"error": err}).Warning("failed")
+		w.WriteHeader(http.StatusNotFound)
 		return
 	}
-	responseBody := append([]byte("Config: "), jsonPayload...)
+
+	// The legacy response format is newline seperated, name prefixed values.
+	// Within that legacy format, the modern JSON response (containing all the
+	// legacy response values and more) is single value with a "Config:" prefix.
+	// This response uses the legacy format but omits all but the JSON value.
+	responseBody := append([]byte("Config: "), responsePayload...)
 
 	w.WriteHeader(http.StatusOK)
 	w.Write(responseBody)
@@ -186,89 +217,56 @@ func (webServer *webServer) handshakeHandler(w http.ResponseWriter, r *http.Requ
 
 func (webServer *webServer) connectedHandler(w http.ResponseWriter, r *http.Request) {
 
-	if !webServer.checkWebServerSecret(r) {
-		// TODO: log more details?
-		log.WithContext().Warning("checkWebServerSecret failed")
-		// TODO: psi_web does NotFound in this case
-		w.WriteHeader(http.StatusForbidden)
-		return
-	}
-
-	// TODO: validate; proper log
-	log.WithContextFields(LogFields{"queryParams": r.URL.Query()}).Info("connected")
+	params, err := convertHTTPRequestToAPIRequest(w, r, "")
 
-	// TODO: necessary, in case client sends bogus request body?
-	_, err := ioutil.ReadAll(r.Body)
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		return
+	var responsePayload []byte
+	if err == nil {
+		responsePayload, err = connectedAPIRequestHandler(
+			webServer.config, webServer.lookupGeoIPData(params), params)
 	}
 
-	var connectedResponse struct {
-		ConnectedTimestamp string `json:"connected_timestamp"`
-	}
-
-	connectedResponse.ConnectedTimestamp =
-		psiphon.TruncateTimestampToHour(psiphon.GetCurrentTimestamp())
-
-	responseBody, err := json.Marshal(connectedResponse)
 	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
+		log.WithContextFields(LogFields{"error": err}).Warning("failed")
+		w.WriteHeader(http.StatusNotFound)
 		return
 	}
 
 	w.WriteHeader(http.StatusOK)
-	w.Write(responseBody)
+	w.Write(responsePayload)
 }
 
 func (webServer *webServer) statusHandler(w http.ResponseWriter, r *http.Request) {
 
-	if !webServer.checkWebServerSecret(r) {
-		// TODO: log more details?
-		log.WithContext().Warning("checkWebServerSecret failed")
-		// TODO: psi_web does NotFound in this case
-		w.WriteHeader(http.StatusForbidden)
-		return
-	}
+	params, err := convertHTTPRequestToAPIRequest(w, r, "statusData")
 
-	// TODO: validate; proper log
-	log.WithContextFields(LogFields{"queryParams": r.URL.Query()}).Info("status")
+	if err == nil {
+		_, err = statusAPIRequestHandler(
+			webServer.config, webServer.lookupGeoIPData(params), params)
+	}
 
-	// TODO: use json.NewDecoder(r.Body)? But will that handle bogus extra data in request body?
-	requestBody, err := ioutil.ReadAll(r.Body)
 	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
+		log.WithContextFields(LogFields{"error": err}).Warning("failed")
+		w.WriteHeader(http.StatusNotFound)
 		return
 	}
 
-	// TODO: parse payload; validate; proper logs
-	log.WithContextFields(LogFields{"payload": string(requestBody)}).Info("status payload")
-
 	w.WriteHeader(http.StatusOK)
 }
 
 func (webServer *webServer) clientVerificationHandler(w http.ResponseWriter, r *http.Request) {
 
-	if !webServer.checkWebServerSecret(r) {
-		// TODO: log more details?
-		log.WithContext().Warning("checkWebServerSecret failed")
-		// TODO: psi_web does NotFound in this case
-		w.WriteHeader(http.StatusForbidden)
-		return
-	}
+	params, err := convertHTTPRequestToAPIRequest(w, r, "verificationData")
 
-	// TODO: validate; proper log
-	log.WithContextFields(LogFields{"queryParams": r.URL.Query()}).Info("client_verification")
+	if err == nil {
+		_, err = clientVerificationAPIRequestHandler(
+			webServer.config, webServer.lookupGeoIPData(params), params)
+	}
 
-	// TODO: use json.NewDecoder(r.Body)? But will that handle bogus extra data in request body?
-	requestBody, err := ioutil.ReadAll(r.Body)
 	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
+		log.WithContextFields(LogFields{"error": err}).Warning("failed")
+		w.WriteHeader(http.StatusNotFound)
 		return
 	}
 
-	// TODO: parse payload; validate; proper logs
-	log.WithContextFields(LogFields{"payload": string(requestBody)}).Info("client_verification payload")
-
 	w.WriteHeader(http.StatusOK)
 }

psiphon/serverApi.go

@@ -37,14 +37,22 @@ import (
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/transferstats"
 )
 
+const (
+	SERVER_API_HANDSHAKE_REQUEST_NAME           = "psiphon-handshake"
+	SERVER_API_CONNECTED_REQUEST_NAME           = "psiphon-connected"
+	SERVER_API_STATUS_REQUEST_NAME              = "psiphon-status"
+	SERVER_API_CLIENT_VERIFICATION_REQUEST_NAME = "psiphon-client-verification"
+)
+
 // ServerContext is a utility struct which holds all of the data associated
 // with a Psiphon server connection. In addition to the established tunnel, this
-// includes data associated with Psiphon API requests and a persistent http
-// client configured to make tunneled Psiphon API requests.
+// includes data and transport mechanisms for Psiphon API requests. Legacy servers
+// offer the Psiphon API through a web service; newer servers offer the Psiphon
+// API through SSH requests made directly through the tunnel's SSH client.
 type ServerContext struct {
 	sessionId                string
 	tunnelNumber             int64
-	baseRequestUrl           string
+	tunnel                   *Tunnel
 	psiphonHttpsClient       *http.Client
 	statsRegexps             *transferstats.Regexps
 	clientRegion             string
@@ -87,19 +95,25 @@ func MakeSessionId() (sessionId string, err error) {
 // requests (e.g., periodic connected and status requests).
 func NewServerContext(tunnel *Tunnel, sessionId string) (*ServerContext, error) {
 
-	psiphonHttpsClient, err := makePsiphonHttpsClient(tunnel)
-	if err != nil {
-		return nil, ContextError(err)
+	// For legacy servers, set up psiphonHttpsClient for
+	// accessing the Psiphon API via the web service.
+	var psiphonHttpsClient *http.Client
+	if !tunnel.serverEntry.SupportsSSHAPIRequests() {
+		var err error
+		psiphonHttpsClient, err = makePsiphonHttpsClient(tunnel)
+		if err != nil {
+			return nil, ContextError(err)
+		}
 	}
 
 	serverContext := &ServerContext{
 		sessionId:          sessionId,
 		tunnelNumber:       atomic.AddInt64(&nextTunnelNumber, 1),
-		baseRequestUrl:     makeBaseRequestUrl(tunnel, "", sessionId),
+		tunnel:             tunnel,
 		psiphonHttpsClient: psiphonHttpsClient,
 	}
 
-	err = serverContext.doHandshakeRequest()
+	err := serverContext.doHandshakeRequest()
 	if err != nil {
 		return nil, ContextError(err)
 	}
@@ -107,42 +121,67 @@ func NewServerContext(tunnel *Tunnel, sessionId string) (*ServerContext, error)
 	return serverContext, nil
 }
 
-// doHandshakeRequest performs the handshake API request. The handshake
+// doHandshakeRequest performs the "handshake" API request. The handshake
 // returns upgrade info, newly discovered server entries -- which are
 // stored -- and sponsor info (home pages, stat regexes).
 func (serverContext *ServerContext) doHandshakeRequest() error {
-	extraParams := make([]*ExtraParam, 0)
-	serverEntryIpAddresses, err := GetServerEntryIpAddresses()
-	if err != nil {
-		return ContextError(err)
-	}
-	// Submit a list of known servers -- this will be used for
-	// discovery statistics.
-	for _, ipAddress := range serverEntryIpAddresses {
-		extraParams = append(extraParams, &ExtraParam{"known_server", ipAddress})
-	}
-	url := buildRequestUrl(serverContext.baseRequestUrl, "handshake", extraParams...)
-	responseBody, err := serverContext.doGetRequest(url)
-	if err != nil {
-		return ContextError(err)
-	}
-	// Skip legacy format lines and just parse the JSON config line
-	configLinePrefix := []byte("Config: ")
-	var configLine []byte
-	for _, line := range bytes.Split(responseBody, []byte("\n")) {
-		if bytes.HasPrefix(line, configLinePrefix) {
-			configLine = line[len(configLinePrefix):]
-			break
+
+	params := serverContext.getBaseParams()
+
+	// *TODO*: this is obsolete?
+	/*
+		serverEntryIpAddresses, err := GetServerEntryIpAddresses()
+		if err != nil {
+			return ContextError(err)
+		}
+
+		// Submit a list of known servers -- this will be used for
+		// discovery statistics.
+		for _, ipAddress := range serverEntryIpAddresses {
+			params = append(params, requestParam{"known_server", ipAddress})
+		}
+	*/
+
+	var response []byte
+	if serverContext.psiphonHttpsClient == nil {
+
+		request, err := makeSSHAPIRequestPayload(params)
+		if err != nil {
+			return ContextError(err)
+		}
+
+		response, err = serverContext.tunnel.SendAPIRequest(
+			SERVER_API_HANDSHAKE_REQUEST_NAME, request)
+		if err != nil {
+			return ContextError(err)
+		}
+
+	} else {
+
+		// Legacy web service API request
+
+		responseBody, err := serverContext.doGetRequest(
+			makeRequestUrl(serverContext.tunnel, "", "handshake", params))
+		if err != nil {
+			return ContextError(err)
+		}
+		// Skip legacy format lines and just parse the JSON config line
+		configLinePrefix := []byte("Config: ")
+		for _, line := range bytes.Split(responseBody, []byte("\n")) {
+			if bytes.HasPrefix(line, configLinePrefix) {
+				response = line[len(configLinePrefix):]
+				break
+			}
+		}
+		if len(response) == 0 {
+			return ContextError(errors.New("no config line found"))
 		}
-	}
-	if len(configLine) == 0 {
-		return ContextError(errors.New("no config line found"))
 	}
 
 	// Note:
 	// - 'preemptive_reconnect_lifetime_milliseconds' is currently unused
 	// - 'ssh_session_id' is ignored; client session ID is used instead
-	var handshakeConfig struct {
+	var handshakeResponse struct {
 		Homepages                  []string            `json:"homepages"`
 		UpgradeClientVersion       string              `json:"upgrade_client_version"`
 		PageViewRegexes            []map[string]string `json:"page_view_regexes"`
@@ -152,12 +191,12 @@ func (serverContext *ServerContext) doHandshakeRequest() error {
 		ServerTimestamp            string              `json:"server_timestamp"`
 		ClientVerificationRequired bool                `json:"client_verification_required"`
 	}
-	err = json.Unmarshal(configLine, &handshakeConfig)
+	err := json.Unmarshal(response, &handshakeResponse)
 	if err != nil {
 		return ContextError(err)
 	}
 
-	serverContext.clientRegion = handshakeConfig.ClientRegion
+	serverContext.clientRegion = handshakeResponse.ClientRegion
 	NoticeClientRegion(serverContext.clientRegion)
 
 	var decodedServerEntries []*ServerEntry
@@ -165,11 +204,11 @@ func (serverContext *ServerContext) doHandshakeRequest() error {
 	// Store discovered server entries
 	// We use the server's time, as it's available here, for the server entry
 	// timestamp since this is more reliable than the client time.
-	for _, encodedServerEntry := range handshakeConfig.EncodedServerList {
+	for _, encodedServerEntry := range handshakeResponse.EncodedServerList {
 
 		serverEntry, err := DecodeServerEntry(
 			encodedServerEntry,
-			TruncateTimestampToHour(handshakeConfig.ServerTimestamp),
+			TruncateTimestampToHour(handshakeResponse.ServerTimestamp),
 			SERVER_ENTRY_SOURCE_DISCOVERY)
 		if err != nil {
 			return ContextError(err)
@@ -194,42 +233,45 @@ func (serverContext *ServerContext) doHandshakeRequest() error {
 
 	// TODO: formally communicate the sponsor and upgrade info to an
 	// outer client via some control interface.
-	for _, homepage := range handshakeConfig.Homepages {
+	for _, homepage := range handshakeResponse.Homepages {
 		NoticeHomepage(homepage)
 	}
 
-	serverContext.clientUpgradeVersion = handshakeConfig.UpgradeClientVersion
-	if handshakeConfig.UpgradeClientVersion != "" {
-		NoticeClientUpgradeAvailable(handshakeConfig.UpgradeClientVersion)
+	serverContext.clientUpgradeVersion = handshakeResponse.UpgradeClientVersion
+	if handshakeResponse.UpgradeClientVersion != "" {
+		NoticeClientUpgradeAvailable(handshakeResponse.UpgradeClientVersion)
 	} else {
 		NoticeClientIsLatestVersion("")
 	}
 
 	var regexpsNotices []string
 	serverContext.statsRegexps, regexpsNotices = transferstats.MakeRegexps(
-		handshakeConfig.PageViewRegexes,
-		handshakeConfig.HttpsRequestRegexes)
+		handshakeResponse.PageViewRegexes,
+		handshakeResponse.HttpsRequestRegexes)
 
 	for _, notice := range regexpsNotices {
 		NoticeAlert(notice)
 	}
 
-	serverContext.serverHandshakeTimestamp = handshakeConfig.ServerTimestamp
+	serverContext.serverHandshakeTimestamp = handshakeResponse.ServerTimestamp
 
-	if handshakeConfig.ClientVerificationRequired {
+	if handshakeResponse.ClientVerificationRequired {
 		NoticeClientVerificationRequired()
 	}
 
 	return nil
 }
 
-// DoConnectedRequest performs the connected API request. This request is
+// DoConnectedRequest performs the "connected" API request. This request is
 // used for statistics. The server returns a last_connected token for
 // the client to store and send next time it connects. This token is
 // a timestamp (using the server clock, and should be rounded to the
 // nearest hour) which is used to determine when a connection represents
 // a unique user for a time period.
 func (serverContext *ServerContext) DoConnectedRequest() error {
+
+	params := serverContext.getBaseParams()
+
 	const DATA_STORE_LAST_CONNECTED_KEY = "lastConnected"
 	lastConnected, err := GetKeyValue(DATA_STORE_LAST_CONNECTED_KEY)
 	if err != nil {
@@ -238,25 +280,44 @@ func (serverContext *ServerContext) DoConnectedRequest() error {
 	if lastConnected == "" {
 		lastConnected = "None"
 	}
-	url := buildRequestUrl(
-		serverContext.baseRequestUrl,
-		"connected",
-		&ExtraParam{"session_id", serverContext.sessionId},
-		&ExtraParam{"last_connected", lastConnected})
-	responseBody, err := serverContext.doGetRequest(url)
-	if err != nil {
-		return ContextError(err)
+
+	params["last_connected"] = lastConnected
+
+	var response []byte
+	if serverContext.psiphonHttpsClient == nil {
+
+		request, err := makeSSHAPIRequestPayload(params)
+		if err != nil {
+			return ContextError(err)
+		}
+
+		response, err = serverContext.tunnel.SendAPIRequest(
+			SERVER_API_CONNECTED_REQUEST_NAME, request)
+		if err != nil {
+			return ContextError(err)
+		}
+
+	} else {
+
+		// Legacy web service API request
+
+		response, err = serverContext.doGetRequest(
+			makeRequestUrl(serverContext.tunnel, "", "connected", params))
+		if err != nil {
+			return ContextError(err)
+		}
 	}
 
-	var response struct {
+	var connectedResponse struct {
 		ConnectedTimestamp string `json:"connected_timestamp"`
 	}
-	err = json.Unmarshal(responseBody, &response)
+	err = json.Unmarshal(response, &connectedResponse)
 	if err != nil {
 		return ContextError(err)
 	}
 
-	err = SetKeyValue(DATA_STORE_LAST_CONNECTED_KEY, response.ConnectedTimestamp)
+	err = SetKeyValue(
+		DATA_STORE_LAST_CONNECTED_KEY, connectedResponse.ConnectedTimestamp)
 	if err != nil {
 		return ContextError(err)
 	}
@@ -268,36 +329,67 @@ func (serverContext *ServerContext) StatsRegexps() *transferstats.Regexps {
 	return serverContext.statsRegexps
 }
 
-// DoStatusRequest makes a /status request to the server, sending session stats.
+// DoStatusRequest makes a "status" API request to the server, sending session stats.
 func (serverContext *ServerContext) DoStatusRequest(tunnel *Tunnel) error {
 
-	url := makeStatusRequestUrl(serverContext.sessionId, serverContext.baseRequestUrl, true)
+	params := serverContext.getStatusParams(true)
+
+	// Note: ensure putBackStatusRequestPayload is called, to replace
+	// payload for future attempt, in all failure cases.
 
-	payload, payloadInfo, err := makeStatusRequestPayload(tunnel.serverEntry.IpAddress)
+	statusPayload, statusPayloadInfo, err := makeStatusRequestPayload(
+		tunnel.serverEntry.IpAddress)
 	if err != nil {
 		return ContextError(err)
 	}
 
-	err = serverContext.doPostRequest(url, "application/json", bytes.NewReader(payload))
+	if serverContext.psiphonHttpsClient == nil {
+
+		params["statusData"] = json.RawMessage(statusPayload)
+
+		var request []byte
+		request, err = makeSSHAPIRequestPayload(params)
+
+		if err == nil {
+			_, err = serverContext.tunnel.SendAPIRequest(
+				SERVER_API_STATUS_REQUEST_NAME, request)
+		}
+
+	} else {
+
+		// Legacy web service API request
+		err = serverContext.doPostRequest(
+			makeRequestUrl(serverContext.tunnel, "", "status", params),
+			"application/json",
+			bytes.NewReader(statusPayload))
+	}
+
 	if err != nil {
 
 		// Resend the transfer stats and tunnel stats later
 		// Note: potential duplicate reports if the server received and processed
 		// the request but the client failed to receive the response.
-		putBackStatusRequestPayload(payloadInfo)
+		putBackStatusRequestPayload(statusPayloadInfo)
 
 		return ContextError(err)
 	}
-	confirmStatusRequestPayload(payloadInfo)
+
+	confirmStatusRequestPayload(statusPayloadInfo)
 
 	return nil
 }
 
-func makeStatusRequestUrl(sessionId, baseRequestUrl string, isTunneled bool) string {
+func (serverContext *ServerContext) getStatusParams(isTunneled bool) requestJSONObject {
+
+	params := serverContext.getBaseParams()
 
 	// Add a random amount of padding to help prevent stats updates from being
 	// a predictable size (which often happens when the connection is quiet).
-	padding := MakeSecureRandomPadding(0, PSIPHON_API_STATUS_REQUEST_PADDING_MAX_BYTES)
+	// TODO: base64 encoding of padding means the padding size is not exactly
+	// [0, PADDING_MAX_BYTES].
+
+	randomPadding := MakeSecureRandomPadding(0, PSIPHON_API_STATUS_REQUEST_PADDING_MAX_BYTES)
+	params["padding"] = base64.StdEncoding.EncodeToString(randomPadding)
 
 	// Legacy clients set "connected" to "0" when disconnecting, and this value
 	// is used to calculate session duration estimates. This is now superseded
@@ -311,15 +403,9 @@ func makeStatusRequestUrl(sessionId, baseRequestUrl string, isTunneled bool) str
 	if !isTunneled {
 		connected = "0"
 	}
+	params["connected"] = connected
 
-	return buildRequestUrl(
-		baseRequestUrl,
-		"status",
-		&ExtraParam{"session_id", sessionId},
-		&ExtraParam{"connected", connected},
-		// TODO: base64 encoding of padding means the padding
-		// size is not exactly [0, PADDING_MAX_BYTES]
-		&ExtraParam{"padding", base64.StdEncoding.EncodeToString(padding)})
+	return params
 }
 
 // statusRequestPayloadInfo is a temporary structure for data used to
@@ -402,28 +488,25 @@ func confirmStatusRequestPayload(payloadInfo *statusRequestPayloadInfo) {
 // The tunnel is assumed to be closed, but its config, protocol, and
 // context values must still be valid.
 // TryUntunneledStatusRequest emits notices detailing failed attempts.
-func TryUntunneledStatusRequest(tunnel *Tunnel, isShutdown bool) error {
+func (serverContext *ServerContext) TryUntunneledStatusRequest(isShutdown bool) error {
 
-	for _, port := range tunnel.serverEntry.GetDirectWebRequestPorts() {
-		err := doUntunneledStatusRequest(tunnel, port, isShutdown)
+	for _, port := range serverContext.tunnel.serverEntry.GetUntunneledWebRequestPorts() {
+		err := serverContext.doUntunneledStatusRequest(port, isShutdown)
 		if err == nil {
 			return nil
 		}
 		NoticeAlert("doUntunneledStatusRequest failed for %s:%s: %s",
-			tunnel.serverEntry.IpAddress, port, err)
+			serverContext.tunnel.serverEntry.IpAddress, port, err)
 	}
 
 	return errors.New("all attempts failed")
 }
 
-// doUntunneledStatusRequest attempts an untunneled stratus request.
-func doUntunneledStatusRequest(
-	tunnel *Tunnel, port string, isShutdown bool) error {
+// doUntunneledStatusRequest attempts an untunneled status request.
+func (serverContext *ServerContext) doUntunneledStatusRequest(
+	port string, isShutdown bool) error {
 
-	url := makeStatusRequestUrl(
-		tunnel.serverContext.sessionId,
-		makeBaseRequestUrl(tunnel, port, tunnel.serverContext.sessionId),
-		false)
+	tunnel := serverContext.tunnel
 
 	certificate, err := DecodeCertificate(tunnel.serverEntry.WebServerCertificate)
 	if err != nil {
@@ -444,7 +527,9 @@ func doUntunneledStatusRequest(
 		*dialConfig = *tunnel.untunneledDialConfig
 	}
 
-	httpClient, requestUrl, err := MakeUntunneledHttpsClient(
+	url := makeRequestUrl(tunnel, port, "status", serverContext.getStatusParams(false))
+
+	httpClient, url, err := MakeUntunneledHttpsClient(
 		dialConfig,
 		certificate,
 		url,
@@ -453,15 +538,15 @@ func doUntunneledStatusRequest(
 		return ContextError(err)
 	}
 
-	payload, payloadInfo, err := makeStatusRequestPayload(tunnel.serverEntry.IpAddress)
+	statusPayload, statusPayloadInfo, err := makeStatusRequestPayload(tunnel.serverEntry.IpAddress)
 	if err != nil {
 		return ContextError(err)
 	}
 
 	bodyType := "application/json"
-	body := bytes.NewReader(payload)
+	body := bytes.NewReader(statusPayload)
 
-	response, err := httpClient.Post(requestUrl, bodyType, body)
+	response, err := httpClient.Post(url, bodyType, body)
 	if err == nil && response.StatusCode != http.StatusOK {
 		response.Body.Close()
 		err = fmt.Errorf("HTTP POST request failed with response code: %d", response.StatusCode)
@@ -471,12 +556,12 @@ func doUntunneledStatusRequest(
 		// Resend the transfer stats and tunnel stats later
 		// Note: potential duplicate reports if the server received and processed
 		// the request but the client failed to receive the response.
-		putBackStatusRequestPayload(payloadInfo)
+		putBackStatusRequestPayload(statusPayloadInfo)
 
 		// Trim this error since it may include long URLs
 		return ContextError(TrimError(err))
 	}
-	confirmStatusRequestPayload(payloadInfo)
+	confirmStatusRequestPayload(statusPayloadInfo)
 	response.Body.Close()
 
 	return nil
@@ -558,18 +643,44 @@ func RecordTunnelStats(
 	return StoreTunnelStats(tunnelStatsJson)
 }
 
-// DoClientVerificationRequest performs the client_verification API
-// request. This request is used to verify that the client is a
-// valid Psiphon client, which will determine how the server treats
-// the client traffic. The proof-of-validity is platform-specific
-// and the payload is opaque to this function but assumed to be JSON.
+// DoClientVerificationRequest performs the "client_verification" API
+// request. This request is used to verify that the client is a valid
+// Psiphon client, which will determine how the server treats the client
+// traffic. The proof-of-validity is platform-specific and the payload
+// is opaque to this function but assumed to be JSON.
 func (serverContext *ServerContext) DoClientVerificationRequest(
 	verificationPayload string) error {
 
-	return serverContext.doPostRequest(
-		buildRequestUrl(serverContext.baseRequestUrl, "client_verification"),
-		"application/json",
-		bytes.NewReader([]byte(verificationPayload)))
+	params := serverContext.getBaseParams()
+
+	if serverContext.psiphonHttpsClient == nil {
+
+		params["verificationData"] = json.RawMessage(verificationPayload)
+
+		request, err := makeSSHAPIRequestPayload(params)
+		if err != nil {
+			return ContextError(err)
+		}
+
+		_, err = serverContext.tunnel.SendAPIRequest(
+			SERVER_API_CLIENT_VERIFICATION_REQUEST_NAME, request)
+		if err != nil {
+			return ContextError(err)
+		}
+
+	} else {
+
+		// Legacy web service API request
+		err := serverContext.doPostRequest(
+			makeRequestUrl(serverContext.tunnel, "", "client_verification", params),
+			"application/json",
+			bytes.NewReader([]byte(verificationPayload)))
+		if err != nil {
+			return ContextError(err)
+		}
+	}
+
+	return nil
 }
 
 // doGetRequest makes a tunneled HTTPS request and returns the response body.
@@ -610,83 +721,60 @@ func (serverContext *ServerContext) doPostRequest(
 	return nil
 }
 
-// makeBaseRequestUrl makes a URL containing all the common parameters
-// that are included with Psiphon API requests. These common parameters
-// are used for statistics.
-func makeBaseRequestUrl(tunnel *Tunnel, port, sessionId string) string {
-	var requestUrl bytes.Buffer
+type requestJSONObject map[string]interface{}
 
-	if port == "" {
-		port = tunnel.serverEntry.WebServerPort
-	}
+// getBaseParams returns all the common API parameters that are included
+// with each Psiphon API request. These common parameters are used for
+// statistics.
+func (serverContext *ServerContext) getBaseParams() requestJSONObject {
 
-	// Note: don't prefix with HTTPS scheme, see comment in doGetRequest.
-	// e.g., don't do this: requestUrl.WriteString("https://")
-	requestUrl.WriteString("http://")
-	requestUrl.WriteString(tunnel.serverEntry.IpAddress)
-	requestUrl.WriteString(":")
-	requestUrl.WriteString(port)
-	requestUrl.WriteString("/")
-	// Placeholder for the path component of a request
-	requestUrl.WriteString("%s")
-	requestUrl.WriteString("?client_session_id=")
-	requestUrl.WriteString(sessionId)
-	requestUrl.WriteString("&server_secret=")
-	requestUrl.WriteString(tunnel.serverEntry.WebServerSecret)
-	requestUrl.WriteString("&propagation_channel_id=")
-	requestUrl.WriteString(tunnel.config.PropagationChannelId)
-	requestUrl.WriteString("&sponsor_id=")
-	requestUrl.WriteString(tunnel.config.SponsorId)
-	requestUrl.WriteString("&client_version=")
-	requestUrl.WriteString(tunnel.config.ClientVersion)
-	// TODO: client_tunnel_core_version
-	requestUrl.WriteString("&relay_protocol=")
-	requestUrl.WriteString(tunnel.protocol)
-	requestUrl.WriteString("&client_platform=")
-	requestUrl.WriteString(tunnel.config.ClientPlatform)
-	requestUrl.WriteString("&tunnel_whole_device=")
-	requestUrl.WriteString(strconv.Itoa(tunnel.config.TunnelWholeDevice))
+	params := make(requestJSONObject)
+
+	tunnel := serverContext.tunnel
+
+	params["session_id"] = serverContext.sessionId
+	params["client_session_id"] = serverContext.sessionId
+	params["server_secret"] = tunnel.serverEntry.WebServerSecret
+	params["propagation_channel_id"] = tunnel.config.PropagationChannelId
+	params["sponsor_id"] = tunnel.config.SponsorId
+	params["client_version"] = tunnel.config.ClientVersion
+	// TODO: client_tunnel_core_version?
+	params["relay_protocol"] = tunnel.protocol
+	params["client_platform"] = tunnel.config.ClientPlatform
+	params["tunnel_whole_device"] = strconv.Itoa(tunnel.config.TunnelWholeDevice)
 
 	// The following parameters may be blank and must
 	// not be sent to the server if blank.
 
 	if tunnel.config.DeviceRegion != "" {
-		requestUrl.WriteString("&device_region=")
-		requestUrl.WriteString(tunnel.config.DeviceRegion)
+		params["device_region"] = tunnel.config.DeviceRegion
 	}
 	if tunnel.meekStats != nil {
 		if tunnel.meekStats.DialAddress != "" {
-			requestUrl.WriteString("&meek_dial_address=")
-			requestUrl.WriteString(tunnel.meekStats.DialAddress)
+			params["meek_dial_address"] = tunnel.meekStats.DialAddress
 		}
 		if tunnel.meekStats.ResolvedIPAddress != "" {
-			requestUrl.WriteString("&meek_resolved_ip_address=")
-			requestUrl.WriteString(tunnel.meekStats.ResolvedIPAddress)
+			params["meek_resolved_ip_address"] = tunnel.meekStats.ResolvedIPAddress
 		}
 		if tunnel.meekStats.SNIServerName != "" {
-			requestUrl.WriteString("&meek_sni_server_name=")
-			requestUrl.WriteString(tunnel.meekStats.SNIServerName)
+			params["meek_sni_server_name"] = tunnel.meekStats.SNIServerName
 		}
 		if tunnel.meekStats.HostHeader != "" {
-			requestUrl.WriteString("&meek_host_header=")
-			requestUrl.WriteString(tunnel.meekStats.HostHeader)
+			params["meek_host_header"] = tunnel.meekStats.HostHeader
 		}
-		requestUrl.WriteString("&meek_transformed_host_name=")
+		transformedHostName := "0"
 		if tunnel.meekStats.TransformedHostName {
-			requestUrl.WriteString("1")
-		} else {
-			requestUrl.WriteString("0")
+			transformedHostName = "1"
 		}
+		params["meek_transformed_host_name"] = transformedHostName
 	}
 
 	if tunnel.serverEntry.Region != "" {
-		requestUrl.WriteString("&server_entry_region=")
-		requestUrl.WriteString(tunnel.serverEntry.Region)
+		params["server_entry_region"] = tunnel.serverEntry.Region
 	}
 
 	if tunnel.serverEntry.LocalSource != "" {
-		requestUrl.WriteString("&server_entry_source=")
-		requestUrl.WriteString(tunnel.serverEntry.LocalSource)
+		params["server_entry_source"] = tunnel.serverEntry.LocalSource
 	}
 
 	// As with last_connected, this timestamp stat, which may be
@@ -695,32 +783,57 @@ func makeBaseRequestUrl(tunnel *Tunnel, port, sessionId string) string {
 	// cross-session user trace into server logs.
 	localServerEntryTimestamp := TruncateTimestampToHour(tunnel.serverEntry.LocalTimestamp)
 	if localServerEntryTimestamp != "" {
-		requestUrl.WriteString("&server_entry_timestamp=")
-		requestUrl.WriteString(localServerEntryTimestamp)
+		params["server_entry_timestamp"] = localServerEntryTimestamp
 	}
 
-	return requestUrl.String()
+	return params
 }
 
-type ExtraParam struct{ name, value string }
+// makeSSHAPIRequestPayload makes a JSON payload for an SSH API request.
+func makeSSHAPIRequestPayload(params requestJSONObject) ([]byte, error) {
+	jsonPayload, err := json.Marshal(params)
+	if err != nil {
+		return nil, ContextError(err)
+	}
+	return jsonPayload, nil
+}
 
-// buildRequestUrl makes a URL for an API request. The URL includes the
-// base request URL and any extra parameters for the specific request.
-func buildRequestUrl(baseRequestUrl, path string, extraParams ...*ExtraParam) string {
+// makeRequestUrl makes a URL for a web service API request.
+func makeRequestUrl(tunnel *Tunnel, port, path string, params requestJSONObject) string {
 	var requestUrl bytes.Buffer
-	requestUrl.WriteString(fmt.Sprintf(baseRequestUrl, path))
-	for _, extraParam := range extraParams {
-		requestUrl.WriteString("&")
-		requestUrl.WriteString(extraParam.name)
-		requestUrl.WriteString("=")
-		requestUrl.WriteString(extraParam.value)
+
+	if port == "" {
+		port = tunnel.serverEntry.WebServerPort
+	}
+
+	// Note: don't prefix with HTTPS scheme, see comment in doGetRequest.
+	// e.g., don't do this: requestUrl.WriteString("https://")
+	requestUrl.WriteString("http://")
+	requestUrl.WriteString(tunnel.serverEntry.IpAddress)
+	requestUrl.WriteString(":")
+	requestUrl.WriteString(port)
+	requestUrl.WriteString("/")
+	requestUrl.WriteString(path)
+	firstParam := true
+	for name, value := range params {
+		if strValue, ok := value.(string); ok {
+			if firstParam {
+				requestUrl.WriteString("?")
+				firstParam = false
+			} else {
+				requestUrl.WriteString("&")
+			}
+			requestUrl.WriteString(name)
+			requestUrl.WriteString("=")
+			requestUrl.WriteString(strValue)
+		}
 	}
 	return requestUrl.String()
 }
 
-// makePsiphonHttpsClient creates a Psiphon HTTPS client that tunnels requests and which validates
-// the web server using the Psiphon server entry web server certificate.
-// This is not a general purpose HTTPS client.
+// makePsiphonHttpsClient creates a Psiphon HTTPS client that tunnels web service API
+// requests and which validates the web server using the Psiphon server entry web server
+// certificate. This is not a general purpose HTTPS client.
 // As the custom dialer makes an explicit TLS connection, URLs submitted to the returned
 // http.Client should use the "http://" scheme. Otherwise http.Transport will try to do another TLS
 // handshake inside the explicit TLS session.

psiphon/serverEntry.go

@@ -36,6 +36,14 @@ const (
 	TUNNEL_PROTOCOL_UNFRONTED_MEEK_HTTPS = "UNFRONTED-MEEK-HTTPS-OSSH"
 	TUNNEL_PROTOCOL_FRONTED_MEEK         = "FRONTED-MEEK-OSSH"
 	TUNNEL_PROTOCOL_FRONTED_MEEK_HTTP    = "FRONTED-MEEK-HTTP-OSSH"
+
+	SERVER_ENTRY_SOURCE_EMBEDDED  = "EMBEDDED"
+	SERVER_ENTRY_SOURCE_REMOTE    = "REMOTE"
+	SERVER_ENTRY_SOURCE_DISCOVERY = "DISCOVERY"
+	SERVER_ENTRY_SOURCE_TARGET    = "TARGET"
+
+	CAPABILITY_SSH_API_REQUESTS            = "ssh-api-requests"
+	CAPABILITY_UNTUNNELED_WEB_API_REQUESTS = "handshake"
 )
 
 var SupportedTunnelProtocols = []string{
@@ -47,6 +55,13 @@ var SupportedTunnelProtocols = []string{
 	TUNNEL_PROTOCOL_SSH,
 }
 
+var SupportedServerEntrySources = []string{
+	SERVER_ENTRY_SOURCE_EMBEDDED,
+	SERVER_ENTRY_SOURCE_REMOTE,
+	SERVER_ENTRY_SOURCE_DISCOVERY,
+	SERVER_ENTRY_SOURCE_TARGET,
+}
+
 // ServerEntry represents a Psiphon server. It contains information
 // about how to establish a tunnel connection to the server through
 // several protocols. Server entries are JSON records downloaded from
@@ -81,15 +96,6 @@ type ServerEntry struct {
 	LocalTimestamp string `json:"localTimestamp"`
 }
 
-type ServerEntrySource string
-
-const (
-	SERVER_ENTRY_SOURCE_EMBEDDED  ServerEntrySource = "EMBEDDED"
-	SERVER_ENTRY_SOURCE_REMOTE    ServerEntrySource = "REMOTE"
-	SERVER_ENTRY_SOURCE_DISCOVERY ServerEntrySource = "DISCOVERY"
-	SERVER_ENTRY_SOURCE_TARGET    ServerEntrySource = "TARGET"
-)
-
 func TunnelProtocolUsesSSH(protocol string) bool {
 	return true
 }
@@ -154,9 +160,15 @@ func (serverEntry *ServerEntry) DisableImpairedProtocols(impairedProtocols []str
 	serverEntry.Capabilities = capabilities
 }
 
-func (serverEntry *ServerEntry) GetDirectWebRequestPorts() []string {
+// SupportsSSHAPIRequests returns true when the server supports
+// SSH API requests.
+func (serverEntry *ServerEntry) SupportsSSHAPIRequests() bool {
+	return Contains(serverEntry.Capabilities, CAPABILITY_SSH_API_REQUESTS)
+}
+
+func (serverEntry *ServerEntry) GetUntunneledWebRequestPorts() []string {
 	ports := make([]string, 0)
-	if Contains(serverEntry.Capabilities, "handshake") {
+	if Contains(serverEntry.Capabilities, CAPABILITY_UNTUNNELED_WEB_API_REQUESTS) {
 		// Server-side configuration quirk: there's a port forward from
 		// port 443 to the web server, which we can try, except on servers
 		// running FRONTED_MEEK, which listens on port 443.
@@ -196,8 +208,8 @@ func EncodeServerEntry(serverEntry *ServerEntry) (string, error) {
 // server entry and reported to the server as stats (a coarse granularity timestamp
 // is reported).
 func DecodeServerEntry(
-	encodedServerEntry, timestamp string,
-	serverEntrySource ServerEntrySource) (serverEntry *ServerEntry, err error) {
+	encodedServerEntry, timestamp,
+	serverEntrySource string) (serverEntry *ServerEntry, err error) {
 
 	hexDecodedServerEntry, err := hex.DecodeString(encodedServerEntry)
 	if err != nil {
@@ -217,7 +229,7 @@ func DecodeServerEntry(
 	}
 
 	// NOTE: if the source JSON happens to have values in these fields, they get clobbered.
-	serverEntry.LocalSource = string(serverEntrySource)
+	serverEntry.LocalSource = serverEntrySource
 	serverEntry.LocalTimestamp = timestamp
 
 	return serverEntry, nil
@@ -245,8 +257,8 @@ func ValidateServerEntry(serverEntry *ServerEntry) error {
 // Each server entry is validated and invalid entries are skipped.
 // See DecodeServerEntry for note on serverEntrySource/timestamp.
 func DecodeAndValidateServerEntryList(
-	encodedServerEntryList, timestamp string,
-	serverEntrySource ServerEntrySource) (serverEntries []*ServerEntry, err error) {
+	encodedServerEntryList, timestamp,
+	serverEntrySource string) (serverEntries []*ServerEntry, err error) {
 
 	serverEntries = make([]*ServerEntry, 0)
 	for _, encodedServerEntry := range strings.Split(encodedServerEntryList, "\n") {

psiphon/tunnel.go

@@ -197,6 +197,13 @@ func (tunnel *Tunnel) Close(isDiscarded bool) {
 	}
 }
 
+// IsClosed returns the tunnel's closed status.
+func (tunnel *Tunnel) IsClosed() bool {
+	tunnel.mutex.Lock()
+	defer tunnel.mutex.Unlock()
+	return tunnel.isClosed
+}
+
 // IsDiscarded returns the tunnel's discarded flag.
 func (tunnel *Tunnel) IsDiscarded() bool {
 	tunnel.mutex.Lock()
@@ -204,17 +211,38 @@ func (tunnel *Tunnel) IsDiscarded() bool {
 	return tunnel.isDiscarded
 }
 
+// SendAPIRequest sends an API request as an SSH request through the tunnel.
+// This function blocks awaiting a response. Only one request may be in-flight
+// at once; a concurrent SendAPIRequest will block until an active request
+// receives its response (or the SSH connection is terminated).
+func (tunnel *Tunnel) SendAPIRequest(
+	name string, requestPayload []byte) ([]byte, error) {
+
+	if tunnel.IsClosed() {
+		return nil, ContextError(errors.New("tunnel is closed"))
+	}
+
+	ok, responsePayload, err := tunnel.sshClient.Conn.SendRequest(
+		name, true, requestPayload)
+
+	if err != nil {
+		return nil, ContextError(err)
+	}
+
+	if !ok {
+		return nil, ContextError(errors.New("API request rejected"))
+	}
+
+	return responsePayload, nil
+}
+
 // Dial establishes a port forward connection through the tunnel
 // This Dial doesn't support split tunnel, so alwaysTunnel is not referenced
 func (tunnel *Tunnel) Dial(
 	remoteAddr string, alwaysTunnel bool, downstreamConn net.Conn) (conn net.Conn, err error) {
 
-	tunnel.mutex.Lock()
-	isClosed := tunnel.isClosed
-	tunnel.mutex.Unlock()
-
-	if isClosed {
-		return nil, errors.New("tunnel is closed")
+	if tunnel.IsClosed() {
+		return nil, ContextError(errors.New("tunnel is closed"))
 	}
 
 	type tunnelDialResult struct {
@@ -482,8 +510,7 @@ func dialSsh(
 	pendingConns *Conns,
 	serverEntry *ServerEntry,
 	selectedProtocol,
-	sessionId string) (
-	conn net.Conn, sshClient *ssh.Client, meekStats *MeekStats, err error) {
+	sessionId string) (net.Conn, *ssh.Client, *MeekStats, error) {
 
 	// The meek protocols tunnel obfuscated SSH. Obfuscated SSH is layered on top of SSH.
 	// So depending on which protocol is used, multiple layers are initialized.
@@ -491,6 +518,7 @@ func dialSsh(
 	useObfuscatedSsh := false
 	var directTCPDialAddress string
 	var meekConfig *MeekConfig
+	var err error
 
 	switch selectedProtocol {
 	case TUNNEL_PROTOCOL_OBFUSCATED_SSH:
@@ -539,6 +567,7 @@ func dialSsh(
 		DeviceRegion:                  config.DeviceRegion,
 		ResolvedIPCallback:            setResolvedIPAddress,
 	}
+	var conn net.Conn
 	if meekConfig != nil {
 		conn, err = DialMeek(meekConfig, dialConfig)
 		if err != nil {
@@ -554,14 +583,13 @@ func dialSsh(
 	cleanupConn := conn
 	defer func() {
 		// Cleanup on error
-		if err != nil {
+		if cleanupConn != nil {
 			cleanupConn.Close()
 		}
 	}()
 
 	// Add obfuscated SSH layer
-	var sshConn net.Conn
-	sshConn = conn
+	sshConn := conn
 	if useObfuscatedSsh {
 		sshConn, err = NewObfuscatedSshConn(
 			OBFUSCATION_CONN_MODE_CLIENT, conn, serverEntry.SshObfuscatedKey)
@@ -570,7 +598,7 @@ func dialSsh(
 		}
 	}
 
-	// Now establish the SSH session over the sshConn transport
+	// Now establish the SSH session over the conn transport
 	expectedPublicKey, err := base64.StdEncoding.DecodeString(serverEntry.SshHostKey)
 	if err != nil {
 		return nil, nil, nil, ContextError(err)
@@ -638,6 +666,7 @@ func dialSsh(
 		return nil, nil, nil, ContextError(result.err)
 	}
 
+	var meekStats *MeekStats
 	if meekConfig != nil {
 		meekStats = &MeekStats{
 			DialAddress:         meekConfig.DialAddress,
@@ -650,6 +679,8 @@ func dialSsh(
 		NoticeConnectedMeekStats(serverEntry.IpAddress, meekStats)
 	}
 
+	cleanupConn = nil
+
 	return conn, result.sshClient, meekStats, nil
 }
 
@@ -1017,7 +1048,7 @@ func sendUntunneledStats(tunnel *Tunnel, isShutdown bool) {
 		return
 	}
 
-	err := TryUntunneledStatusRequest(tunnel, isShutdown)
+	err := tunnel.serverContext.TryUntunneledStatusRequest(isShutdown)
 	if err != nil {
 		NoticeAlert("TryUntunneledStatusRequest failed for %s: %s", tunnel.serverEntry.IpAddress, err)
 	}