/*
* Copyright (c) 2016, Psiphon Inc.
* All rights reserved.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see .
*
*/
package server
import (
"crypto/hmac"
"crypto/sha256"
"crypto/subtle"
"encoding/base64"
"encoding/json"
std_errors "errors"
"net"
"regexp"
"strconv"
"strings"
"time"
"unicode"
"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/errors"
"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/fragmentor"
"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/protocol"
"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/tactics"
)
const (
MAX_API_PARAMS_SIZE = 256 * 1024 // 256KB
PADDING_MAX_BYTES = 16 * 1024
CLIENT_PLATFORM_ANDROID = "Android"
CLIENT_PLATFORM_WINDOWS = "Windows"
CLIENT_PLATFORM_IOS = "iOS"
)
// sshAPIRequestHandler routes Psiphon API requests transported as
// JSON objects via the SSH request mechanism.
//
// The API request handlers, handshakeAPIRequestHandler, etc., are
// reused by webServer which offers the Psiphon API via web transport.
//
// The API request parameters and event log values follow the legacy
// psi_web protocol and naming conventions. The API is compatible with
// all tunnel-core clients but are not backwards compatible with all
// legacy clients.
//
func sshAPIRequestHandler(
support *SupportServices,
clientAddr string,
geoIPData GeoIPData,
authorizedAccessTypes []string,
name string,
requestPayload []byte) ([]byte, error) {
// Notes:
//
// - For SSH requests, MAX_API_PARAMS_SIZE is implicitly enforced
// by max SSH request packet size.
//
// - The param protocol.PSIPHON_API_HANDSHAKE_AUTHORIZATIONS is an
// array of base64-encoded strings; the base64 representation should
// not be decoded to []byte values. The default behavior of
// https://golang.org/pkg/encoding/json/#Unmarshal for a target of
// type map[string]interface{} will unmarshal a base64-encoded string
// to a string, not a decoded []byte, as required.
var params common.APIParameters
err := json.Unmarshal(requestPayload, ¶ms)
if err != nil {
return nil, errors.Tracef(
"invalid payload for request name: %s: %s", name, err)
}
return dispatchAPIRequestHandler(
support,
protocol.PSIPHON_SSH_API_PROTOCOL,
clientAddr,
geoIPData,
authorizedAccessTypes,
name,
params)
}
// dispatchAPIRequestHandler is the common dispatch point for both
// web and SSH API requests.
func dispatchAPIRequestHandler(
support *SupportServices,
apiProtocol string,
clientAddr string,
geoIPData GeoIPData,
authorizedAccessTypes []string,
name string,
params common.APIParameters) (response []byte, reterr error) {
// Before invoking the handlers, enforce some preconditions:
//
// - A handshake request must precede any other requests.
// - When the handshake results in a traffic rules state where
// the client is immediately exhausted, no requests
// may succeed. This case ensures that blocked clients do
// not log "connected", etc.
//
// Only one handshake request may be made. There is no check here
// to enforce that handshakeAPIRequestHandler will be called at
// most once. The SetHandshakeState call in handshakeAPIRequestHandler
// enforces that only a single handshake is made; enforcing that there
// ensures no race condition even if concurrent requests are
// in flight.
if name != protocol.PSIPHON_API_HANDSHAKE_REQUEST_NAME {
// TODO: same session-ID-lookup TODO in handshakeAPIRequestHandler
// applies here.
sessionID, err := getStringRequestParam(params, "client_session_id")
if err == nil {
// Note: follows/duplicates baseParams validation
if !isHexDigits(support.Config, sessionID) {
err = std_errors.New("invalid param: client_session_id")
}
}
if err != nil {
return nil, errors.Trace(err)
}
completed, exhausted, err := support.TunnelServer.GetClientHandshaked(sessionID)
if err != nil {
return nil, errors.Trace(err)
}
if !completed {
return nil, errors.TraceNew("handshake not completed")
}
if exhausted {
return nil, errors.TraceNew("exhausted after handshake")
}
}
switch name {
case protocol.PSIPHON_API_HANDSHAKE_REQUEST_NAME:
return handshakeAPIRequestHandler(
support, apiProtocol, clientAddr, geoIPData, params)
case protocol.PSIPHON_API_CONNECTED_REQUEST_NAME:
return connectedAPIRequestHandler(
support, clientAddr, geoIPData, authorizedAccessTypes, params)
case protocol.PSIPHON_API_STATUS_REQUEST_NAME:
return statusAPIRequestHandler(
support, clientAddr, geoIPData, authorizedAccessTypes, params)
case protocol.PSIPHON_API_CLIENT_VERIFICATION_REQUEST_NAME:
return clientVerificationAPIRequestHandler(
support, clientAddr, geoIPData, authorizedAccessTypes, params)
}
return nil, errors.Tracef("invalid request name: %s", name)
}
var handshakeRequestParams = append(
append(
append(
[]requestParamSpec{
// Legacy clients may not send "session_id" in handshake
{"session_id", isHexDigits, requestParamOptional},
{"missing_server_entry_signature", isBase64String, requestParamOptional}},
baseParams...),
baseDialParams...),
tacticsParams...)
// handshakeAPIRequestHandler implements the "handshake" API request.
// Clients make the handshake immediately after establishing a tunnel
// connection; the response tells the client what homepage to open, what
// stats to record, etc.
func handshakeAPIRequestHandler(
support *SupportServices,
apiProtocol string,
clientAddr string,
geoIPData GeoIPData,
params common.APIParameters) ([]byte, error) {
// Note: ignoring legacy "known_servers" params
err := validateRequestParams(support.Config, params, handshakeRequestParams)
if err != nil {
return nil, errors.Trace(err)
}
sessionID, _ := getStringRequestParam(params, "client_session_id")
sponsorID, _ := getStringRequestParam(params, "sponsor_id")
clientVersion, _ := getStringRequestParam(params, "client_version")
clientPlatform, _ := getStringRequestParam(params, "client_platform")
isMobile := isMobileClientPlatform(clientPlatform)
normalizedPlatform := normalizeClientPlatform(clientPlatform)
// establishedTunnelsCount is used in traffic rule selection. When omitted by
// the client, a value of 0 will be used.
establishedTunnelsCount, _ := getIntStringRequestParam(params, "established_tunnels_count")
// splitTunnel indicates if the client is using split tunnel mode. When
// omitted by the client, the value will be false.
splitTunnel, _ := getBoolStringRequestParam(params, "split_tunnel")
var authorizations []string
if params[protocol.PSIPHON_API_HANDSHAKE_AUTHORIZATIONS] != nil {
authorizations, err = getStringArrayRequestParam(params, protocol.PSIPHON_API_HANDSHAKE_AUTHORIZATIONS)
if err != nil {
return nil, errors.Trace(err)
}
}
// Note: no guarantee that PsinetDatabase won't reload between database calls
db := support.PsinetDatabase
httpsRequestRegexes := db.GetHttpsRequestRegexes(sponsorID)
// Flag the SSH client as having completed its handshake. This
// may reselect traffic rules and starts allowing port forwards.
// TODO: in the case of SSH API requests, the actual sshClient could
// be passed in and used here. The session ID lookup is only strictly
// necessary to support web API requests.
handshakeStateInfo, err := support.TunnelServer.SetClientHandshakeState(
sessionID,
handshakeState{
completed: true,
apiProtocol: apiProtocol,
apiParams: copyBaseSessionAndDialParams(params),
expectDomainBytes: len(httpsRequestRegexes) > 0,
establishedTunnelsCount: establishedTunnelsCount,
splitTunnel: splitTunnel,
},
authorizations)
if err != nil {
return nil, errors.Trace(err)
}
tacticsPayload, err := support.TacticsServer.GetTacticsPayload(
common.GeoIPData(geoIPData), params)
if err != nil {
return nil, errors.Trace(err)
}
var marshaledTacticsPayload []byte
if tacticsPayload != nil {
marshaledTacticsPayload, err = json.Marshal(tacticsPayload)
if err != nil {
return nil, errors.Trace(err)
}
// Log a metric when new tactics are issued. Logging here indicates that
// the handshake tactics mechanism is active; but logging for every
// handshake creates unneccesary log data.
if len(tacticsPayload.Tactics) > 0 {
logFields := getRequestLogFields(
tactics.TACTICS_METRIC_EVENT_NAME,
geoIPData,
handshakeStateInfo.authorizedAccessTypes,
params,
handshakeRequestParams)
logFields[tactics.NEW_TACTICS_TAG_LOG_FIELD_NAME] = tacticsPayload.Tag
logFields[tactics.IS_TACTICS_REQUEST_LOG_FIELD_NAME] = false
log.LogRawFieldsWithTimestamp(logFields)
}
}
// The log comes _after_ SetClientHandshakeState, in case that call rejects
// the state change (for example, if a second handshake is performed)
//
// The handshake event is no longer shipped to log consumers, so this is
// simply a diagnostic log. Since the "server_tunnel" event includes all
// common API parameters and "handshake_completed" flag, this handshake
// log is mostly redundant and set to debug level.
log.WithTraceFields(
getRequestLogFields(
"",
geoIPData,
handshakeStateInfo.authorizedAccessTypes,
params,
handshakeRequestParams)).Debug("handshake")
pad_response, _ := getPaddingSizeRequestParam(params, "pad_response")
// Discover new servers
host, _, err := net.SplitHostPort(clientAddr)
if err != nil {
return nil, errors.Trace(err)
}
clientIP := net.ParseIP(host)
if clientIP == nil {
return nil, errors.TraceNew("missing client IP")
}
encodedServerList := db.DiscoverServers(
calculateDiscoveryValue(support.Config.DiscoveryValueHMACKey, clientIP))
// When the client indicates that it used an unsigned server entry for this
// connection, return a signed copy of the server entry for the client to
// upgrade to. See also: comment in psiphon.doHandshakeRequest.
//
// The missing_server_entry_signature parameter value is a server entry tag,
// which is used to select the correct server entry for servers with multiple
// entries. Identifying the server entries tags instead of server IPs prevents
// an enumeration attack, where a malicious client can abuse this facilty to
// check if an arbitrary IP address is a Psiphon server.
serverEntryTag, ok := getOptionalStringRequestParam(
params, "missing_server_entry_signature")
if ok {
ownServerEntry, ok := support.Config.GetOwnEncodedServerEntry(serverEntryTag)
if ok {
encodedServerList = append(encodedServerList, ownServerEntry)
}
}
// PageViewRegexes is obsolete and not used by any tunnel-core clients. In
// the JSON response, return an empty array instead of null for legacy
// clients.
handshakeResponse := protocol.HandshakeResponse{
SSHSessionID: sessionID,
Homepages: db.GetRandomizedHomepages(sponsorID, geoIPData.Country, geoIPData.ASN, isMobile),
UpgradeClientVersion: db.GetUpgradeClientVersion(clientVersion, normalizedPlatform),
PageViewRegexes: make([]map[string]string, 0),
HttpsRequestRegexes: httpsRequestRegexes,
EncodedServerList: encodedServerList,
ClientRegion: geoIPData.Country,
ClientAddress: clientAddr,
ServerTimestamp: common.GetCurrentTimestamp(),
ActiveAuthorizationIDs: handshakeStateInfo.activeAuthorizationIDs,
TacticsPayload: marshaledTacticsPayload,
UpstreamBytesPerSecond: handshakeStateInfo.upstreamBytesPerSecond,
DownstreamBytesPerSecond: handshakeStateInfo.downstreamBytesPerSecond,
Padding: strings.Repeat(" ", pad_response),
}
responsePayload, err := json.Marshal(handshakeResponse)
if err != nil {
return nil, errors.Trace(err)
}
return responsePayload, nil
}
// calculateDiscoveryValue derives a value from the client IP address to be
// used as input in the server discovery algorithm.
// See https://github.com/Psiphon-Inc/psiphon-automation/tree/master/Automation/psi_ops_discovery.py
// for full details.
func calculateDiscoveryValue(discoveryValueHMACKey string, ipAddress net.IP) int {
// From: psi_ops_discovery.calculate_ip_address_strategy_value:
// # Mix bits from all octets of the client IP address to determine the
// # bucket. An HMAC is used to prevent pre-calculation of buckets for IPs.
// return ord(hmac.new(HMAC_KEY, ip_address, hashlib.sha256).digest()[0])
// TODO: use 3-octet algorithm?
hash := hmac.New(sha256.New, []byte(discoveryValueHMACKey))
hash.Write([]byte(ipAddress.String()))
return int(hash.Sum(nil)[0])
}
// uniqueUserParams are the connected request parameters which are logged for
// unique_user events.
var uniqueUserParams = append(
[]requestParamSpec{
{"last_connected", isLastConnected, 0}},
baseSessionParams...)
var connectedRequestParams = append(
[]requestParamSpec{
{"establishment_duration", isIntString, requestParamOptional | requestParamLogStringAsInt}},
uniqueUserParams...)
// updateOnConnectedParamNames are connected request parameters which are
// copied to update data logged with server_tunnel: these fields either only
// ship with or ship newer data with connected requests.
var updateOnConnectedParamNames = append(
[]string{
"last_connected",
"establishment_duration",
},
fragmentor.GetUpstreamMetricsNames()...)
// connectedAPIRequestHandler implements the "connected" API request. Clients
// make the connected request once a tunnel connection has been established
// and at least once per 24h for long-running tunnels. The last_connected
// input value, which should be a connected_timestamp output from a previous
// connected response, is used to calculate unique user stats.
// connected_timestamp is truncated as a privacy measure.
func connectedAPIRequestHandler(
support *SupportServices,
clientAddr string,
geoIPData GeoIPData,
authorizedAccessTypes []string,
params common.APIParameters) ([]byte, error) {
err := validateRequestParams(support.Config, params, connectedRequestParams)
if err != nil {
return nil, errors.Trace(err)
}
sessionID, _ := getStringRequestParam(params, "client_session_id")
lastConnected, _ := getStringRequestParam(params, "last_connected")
// Update, for server_tunnel logging, upstream fragmentor metrics, as the
// client may have performed more upstream fragmentation since the previous
// metrics reported by the handshake request. Also, additional fields that
// are reported only in the connected request are added to server_tunnel
// here.
// TODO: same session-ID-lookup TODO in handshakeAPIRequestHandler
// applies here.
err = support.TunnelServer.UpdateClientAPIParameters(
sessionID, copyUpdateOnConnectedParams(params))
if err != nil {
return nil, errors.Trace(err)
}
connectedTimestamp := common.TruncateTimestampToHour(common.GetCurrentTimestamp())
// The finest required granularity for unique users is daily. To save space,
// only record a "unique_user" log event when the client's last_connected is
// in the previous day relative to the new connected_timestamp.
logUniqueUser := false
if lastConnected == "None" {
logUniqueUser = true
} else {
t1, _ := time.Parse(time.RFC3339, lastConnected)
year, month, day := t1.Date()
d1 := time.Date(year, month, day, 0, 0, 0, 0, time.UTC)
t2, _ := time.Parse(time.RFC3339, connectedTimestamp)
year, month, day = t2.Date()
d2 := time.Date(year, month, day, 0, 0, 0, 0, time.UTC)
if t1.Before(t2) && d1 != d2 {
logUniqueUser = true
}
}
if logUniqueUser {
log.LogRawFieldsWithTimestamp(
getRequestLogFields(
"unique_user",
geoIPData,
authorizedAccessTypes,
params,
uniqueUserParams))
}
pad_response, _ := getPaddingSizeRequestParam(params, "pad_response")
connectedResponse := protocol.ConnectedResponse{
ConnectedTimestamp: connectedTimestamp,
Padding: strings.Repeat(" ", pad_response),
}
responsePayload, err := json.Marshal(connectedResponse)
if err != nil {
return nil, errors.Trace(err)
}
return responsePayload, nil
}
var statusRequestParams = baseSessionParams
var remoteServerListStatParams = append(
[]requestParamSpec{
{"client_download_timestamp", isISO8601Date, 0},
{"tunneled", isBooleanFlag, requestParamOptional | requestParamLogFlagAsBool},
{"url", isAnyString, 0},
{"etag", isAnyString, 0},
{"bytes", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"duration", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"authenticated", isBooleanFlag, requestParamOptional | requestParamLogFlagAsBool}},
baseSessionParams...)
// Backwards compatibility case: legacy clients do not include these fields in
// the remote_server_list_stats entries. Use the values from the outer status
// request as an approximation (these values reflect the client at persistent
// stat shipping time, which may differ from the client at persistent stat
// recording time). Note that all but client_build_rev and device_region are
// required fields.
var remoteServerListStatBackwardsCompatibilityParamNames = []string{
"session_id",
"propagation_channel_id",
"sponsor_id",
"client_version",
"client_platform",
"client_build_rev",
"device_region",
}
var failedTunnelStatParams = append(
[]requestParamSpec{
{"server_entry_tag", isAnyString, requestParamOptional},
{"session_id", isHexDigits, 0},
{"last_connected", isLastConnected, 0},
{"client_failed_timestamp", isISO8601Date, 0},
{"liveness_test_upstream_bytes", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"liveness_test_sent_upstream_bytes", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"liveness_test_downstream_bytes", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"liveness_test_received_downstream_bytes", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"bytes_up", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"bytes_down", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"tunnel_error", isAnyString, 0}},
baseSessionAndDialParams...)
// statusAPIRequestHandler implements the "status" API request.
// Clients make periodic status requests which deliver client-side
// recorded data transfer and tunnel duration stats.
// Note from psi_web implementation: no input validation on domains;
// any string is accepted (regex transform may result in arbitrary
// string). Stats processor must handle this input with care.
func statusAPIRequestHandler(
support *SupportServices,
clientAddr string,
geoIPData GeoIPData,
authorizedAccessTypes []string,
params common.APIParameters) ([]byte, error) {
err := validateRequestParams(support.Config, params, statusRequestParams)
if err != nil {
return nil, errors.Trace(err)
}
sessionID, _ := getStringRequestParam(params, "client_session_id")
statusData, err := getJSONObjectRequestParam(params, "statusData")
if err != nil {
return nil, errors.Trace(err)
}
// Logs are queued until the input is fully validated. Otherwise, stats
// could be double counted if the client has a bug in its request
// formatting: partial stats would be logged (counted), the request would
// fail, and clients would then resend all the same stats again.
logQueue := make([]LogFields, 0)
// Domain bytes transferred stats
// Older clients may not submit this data
// Clients are expected to send host_bytes/domain_bytes stats only when
// configured to do so in the handshake reponse. Legacy clients may still
// report "(OTHER)" host_bytes when no regexes are set. Drop those stats.
domainBytesExpected, err := support.TunnelServer.ExpectClientDomainBytes(sessionID)
if err != nil {
return nil, errors.Trace(err)
}
if domainBytesExpected && statusData["host_bytes"] != nil {
hostBytes, err := getMapStringInt64RequestParam(statusData, "host_bytes")
if err != nil {
return nil, errors.Trace(err)
}
for domain, bytes := range hostBytes {
domainBytesFields := getRequestLogFields(
"domain_bytes",
geoIPData,
authorizedAccessTypes,
params,
statusRequestParams)
domainBytesFields["domain"] = domain
domainBytesFields["bytes"] = bytes
logQueue = append(logQueue, domainBytesFields)
}
}
// Limitation: for "persistent" stats, host_id and geolocation is time-of-sending
// not time-of-recording.
// Remote server list download persistent stats.
// Older clients may not submit this data.
if statusData["remote_server_list_stats"] != nil {
remoteServerListStats, err := getJSONObjectArrayRequestParam(statusData, "remote_server_list_stats")
if err != nil {
return nil, errors.Trace(err)
}
for _, remoteServerListStat := range remoteServerListStats {
for _, name := range remoteServerListStatBackwardsCompatibilityParamNames {
if _, ok := remoteServerListStat[name]; !ok {
if field, ok := params[name]; ok {
remoteServerListStat[name] = field
}
}
}
// For validation, copy expected fields from the outer
// statusRequestParams.
remoteServerListStat["server_secret"] = params["server_secret"]
remoteServerListStat["client_session_id"] = params["client_session_id"]
err := validateRequestParams(support.Config, remoteServerListStat, remoteServerListStatParams)
if err != nil {
// Occasionally, clients may send corrupt persistent stat data. Do not
// fail the status request, as this will lead to endless retries.
log.WithTraceFields(LogFields{"error": err}).Warning("remote_server_list_stats entry dropped")
continue
}
remoteServerListFields := getRequestLogFields(
"remote_server_list",
geoIPData,
authorizedAccessTypes,
remoteServerListStat,
remoteServerListStatParams)
logQueue = append(logQueue, remoteServerListFields)
}
}
// Failed tunnel persistent stats.
// Older clients may not submit this data.
var invalidServerEntryTags map[string]bool
if statusData["failed_tunnel_stats"] != nil {
// Note: no guarantee that PsinetDatabase won't reload between database calls
db := support.PsinetDatabase
invalidServerEntryTags = make(map[string]bool)
failedTunnelStats, err := getJSONObjectArrayRequestParam(statusData, "failed_tunnel_stats")
if err != nil {
return nil, errors.Trace(err)
}
for _, failedTunnelStat := range failedTunnelStats {
// failed_tunnel supplies a full set of base params, but the server secret
// must use the correct value from the outer statusRequestParams.
failedTunnelStat["server_secret"] = params["server_secret"]
err := validateRequestParams(support.Config, failedTunnelStat, failedTunnelStatParams)
if err != nil {
// Occasionally, clients may send corrupt persistent stat data. Do not
// fail the status request, as this will lead to endless retries.
//
// TODO: trigger pruning if the data corruption indicates corrupt server
// entry storage?
log.WithTraceFields(LogFields{"error": err}).Warning("failed_tunnel_stats entry dropped")
continue
}
failedTunnelFields := getRequestLogFields(
"failed_tunnel",
geoIPData,
authorizedAccessTypes,
failedTunnelStat,
failedTunnelStatParams)
// Return a list of servers, identified by server entry tag, that are
// invalid and presumed to be deleted. This information is used by clients
// to prune deleted servers from their local datastores and stop attempting
// connections to servers that no longer exist.
//
// This mechanism uses tags instead of server IPs: (a) to prevent an
// enumeration attack, where a malicious client can query the entire IPv4
// range and build a map of the Psiphon network; (b) to deal with recyling
// cases where a server deleted and its IP is reused for a new server with
// a distinct server entry.
//
// IsValidServerEntryTag ensures that the local copy of psinet is not stale
// before returning a negative result, to mitigate accidental pruning.
//
// In addition, when the reported dial port number is 0, flag the server
// entry as invalid to trigger client pruning. This covers a class of
// invalid/semi-functional server entries, found in practice to be stored
// by clients, where some protocol port number has been omitted -- due to
// historical bugs in various server entry handling implementations. When
// missing from a server entry loaded by a client, the port number
// evaluates to 0, the zero value, which is not a valid port number even if
// were not missing.
serverEntryTag, ok := getOptionalStringRequestParam(failedTunnelStat, "server_entry_tag")
if ok {
serverEntryValid := db.IsValidServerEntryTag(serverEntryTag)
if serverEntryValid {
dialPortNumber, err := getIntStringRequestParam(failedTunnelStat, "dial_port_number")
if err == nil && dialPortNumber == 0 {
serverEntryValid = false
}
}
if !serverEntryValid {
invalidServerEntryTags[serverEntryTag] = true
}
// Add a field to the failed_tunnel log indicating if the server entry is
// valid.
failedTunnelFields["server_entry_valid"] = serverEntryValid
}
// Log failed_tunnel.
logQueue = append(logQueue, failedTunnelFields)
}
}
for _, logItem := range logQueue {
log.LogRawFieldsWithTimestamp(logItem)
}
pad_response, _ := getPaddingSizeRequestParam(params, "pad_response")
statusResponse := protocol.StatusResponse{
Padding: strings.Repeat(" ", pad_response),
}
if len(invalidServerEntryTags) > 0 {
statusResponse.InvalidServerEntryTags = make([]string, len(invalidServerEntryTags))
i := 0
for tag := range invalidServerEntryTags {
statusResponse.InvalidServerEntryTags[i] = tag
i++
}
}
responsePayload, err := json.Marshal(statusResponse)
if err != nil {
return nil, errors.Trace(err)
}
return responsePayload, nil
}
// clientVerificationAPIRequestHandler is just a compliance stub
// for older Android clients that still send verification requests
func clientVerificationAPIRequestHandler(
support *SupportServices,
clientAddr string,
geoIPData GeoIPData,
authorizedAccessTypes []string,
params common.APIParameters) ([]byte, error) {
return make([]byte, 0), nil
}
var tacticsParams = []requestParamSpec{
{tactics.STORED_TACTICS_TAG_PARAMETER_NAME, isAnyString, requestParamOptional},
{tactics.SPEED_TEST_SAMPLES_PARAMETER_NAME, nil, requestParamOptional | requestParamJSON},
}
var tacticsRequestParams = append(
append([]requestParamSpec(nil), tacticsParams...),
baseSessionAndDialParams...)
func getTacticsAPIParameterValidator(config *Config) common.APIParameterValidator {
return func(params common.APIParameters) error {
return validateRequestParams(config, params, tacticsRequestParams)
}
}
func getTacticsAPIParameterLogFieldFormatter() common.APIParameterLogFieldFormatter {
return func(geoIPData common.GeoIPData, params common.APIParameters) common.LogFields {
logFields := getRequestLogFields(
tactics.TACTICS_METRIC_EVENT_NAME,
GeoIPData(geoIPData),
nil, // authorizedAccessTypes are not known yet
params,
tacticsRequestParams)
return common.LogFields(logFields)
}
}
// requestParamSpec defines a request parameter. Each param is expected to be
// a string, unless requestParamArray is specified, in which case an array of
// strings is expected.
type requestParamSpec struct {
name string
validator func(*Config, string) bool
flags uint32
}
const (
requestParamOptional = 1
requestParamNotLogged = 1 << 1
requestParamArray = 1 << 2
requestParamJSON = 1 << 3
requestParamLogStringAsInt = 1 << 4
requestParamLogStringAsFloat = 1 << 5
requestParamLogStringLengthAsInt = 1 << 6
requestParamLogFlagAsBool = 1 << 7
requestParamLogOnlyForFrontedMeek = 1 << 8
requestParamNotLoggedForUnfrontedMeekNonTransformedHeader = 1 << 9
)
// baseParams are the basic request parameters that are expected for all API
// requests and log events.
var baseParams = []requestParamSpec{
{"server_secret", isServerSecret, requestParamNotLogged},
{"client_session_id", isHexDigits, requestParamNotLogged},
{"propagation_channel_id", isHexDigits, 0},
{"sponsor_id", isHexDigits, 0},
{"client_version", isIntString, requestParamLogStringAsInt},
{"client_platform", isClientPlatform, 0},
{"client_features", isAnyString, requestParamOptional | requestParamArray},
{"client_build_rev", isHexDigits, requestParamOptional},
{"device_region", isAnyString, requestParamOptional},
}
// baseSessionParams adds to baseParams the required session_id parameter. For
// all requests except handshake, all existing clients are expected to send
// session_id. Legacy clients may not send "session_id" in handshake.
var baseSessionParams = append(
[]requestParamSpec{
{"session_id", isHexDigits, 0}},
baseParams...)
// baseDialParams are the dial parameters, per-tunnel network protocol and
// obfuscation metrics which are logged with server_tunnel, failed_tunnel, and
// tactics.
var baseDialParams = []requestParamSpec{
{"relay_protocol", isRelayProtocol, 0},
{"ssh_client_version", isAnyString, requestParamOptional},
{"upstream_proxy_type", isUpstreamProxyType, requestParamOptional},
{"upstream_proxy_custom_header_names", isAnyString, requestParamOptional | requestParamArray},
{"fronting_provider_id", isAnyString, requestParamOptional},
{"meek_dial_address", isDialAddress, requestParamOptional | requestParamLogOnlyForFrontedMeek},
{"meek_resolved_ip_address", isIPAddress, requestParamOptional | requestParamLogOnlyForFrontedMeek},
{"meek_sni_server_name", isDomain, requestParamOptional},
{"meek_host_header", isHostHeader, requestParamOptional | requestParamNotLoggedForUnfrontedMeekNonTransformedHeader},
{"meek_transformed_host_name", isBooleanFlag, requestParamOptional | requestParamLogFlagAsBool},
{"user_agent", isAnyString, requestParamOptional},
{"tls_profile", isAnyString, requestParamOptional},
{"tls_version", isAnyString, requestParamOptional},
{"server_entry_region", isRegionCode, requestParamOptional},
{"server_entry_source", isServerEntrySource, requestParamOptional},
{"server_entry_timestamp", isISO8601Date, requestParamOptional},
{tactics.APPLIED_TACTICS_TAG_PARAMETER_NAME, isAnyString, requestParamOptional},
{"dial_port_number", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"quic_version", isAnyString, requestParamOptional},
{"quic_dial_sni_address", isAnyString, requestParamOptional},
{"upstream_bytes_fragmented", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"upstream_min_bytes_written", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"upstream_max_bytes_written", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"upstream_min_delayed", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"upstream_max_delayed", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"padding", isAnyString, requestParamOptional | requestParamLogStringLengthAsInt},
{"pad_response", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"is_replay", isBooleanFlag, requestParamOptional | requestParamLogFlagAsBool},
{"egress_region", isRegionCode, requestParamOptional},
{"dial_duration", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"candidate_number", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"established_tunnels_count", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"upstream_ossh_padding", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"meek_cookie_size", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"meek_limit_request", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"meek_tls_padding", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"network_latency_multiplier", isFloatString, requestParamOptional | requestParamLogStringAsFloat},
{"client_bpf", isAnyString, requestParamOptional},
{"network_type", isAnyString, requestParamOptional},
{"conjure_cached", isBooleanFlag, requestParamOptional | requestParamLogFlagAsBool},
{"conjure_delay", isIntString, requestParamOptional | requestParamLogStringAsInt},
{"conjure_transport", isAnyString, requestParamOptional},
{"split_tunnel", isBooleanFlag, requestParamOptional | requestParamLogFlagAsBool},
}
// baseSessionAndDialParams adds baseDialParams to baseSessionParams.
var baseSessionAndDialParams = append(
append(
[]requestParamSpec{},
baseSessionParams...),
baseDialParams...)
func validateRequestParams(
config *Config,
params common.APIParameters,
expectedParams []requestParamSpec) error {
for _, expectedParam := range expectedParams {
value := params[expectedParam.name]
if value == nil {
if expectedParam.flags&requestParamOptional != 0 {
continue
}
return errors.Tracef("missing param: %s", expectedParam.name)
}
var err error
switch {
case expectedParam.flags&requestParamArray != 0:
err = validateStringArrayRequestParam(config, expectedParam, value)
case expectedParam.flags&requestParamJSON != 0:
// No validation: the JSON already unmarshalled; the parameter
// user will validate that the JSON contains the expected
// objects/data.
// TODO: without validation, any valid JSON will be logged
// by getRequestLogFields, even if the parameter user validates
// and rejects the parameter.
default:
err = validateStringRequestParam(config, expectedParam, value)
}
if err != nil {
return errors.Trace(err)
}
}
return nil
}
// copyBaseSessionAndDialParams makes a copy of the params which includes only
// the baseSessionAndDialParams.
func copyBaseSessionAndDialParams(params common.APIParameters) common.APIParameters {
// Note: not a deep copy; assumes baseSessionAndDialParams values are all
// scalar types (int, string, etc.)
paramsCopy := make(common.APIParameters)
for _, baseParam := range baseSessionAndDialParams {
value := params[baseParam.name]
if value == nil {
continue
}
paramsCopy[baseParam.name] = value
}
return paramsCopy
}
func copyUpdateOnConnectedParams(params common.APIParameters) common.APIParameters {
// Note: not a deep copy
paramsCopy := make(common.APIParameters)
for _, name := range updateOnConnectedParamNames {
value := params[name]
if value == nil {
continue
}
paramsCopy[name] = value
}
return paramsCopy
}
func validateStringRequestParam(
config *Config,
expectedParam requestParamSpec,
value interface{}) error {
strValue, ok := value.(string)
if !ok {
return errors.Tracef("unexpected string param type: %s", expectedParam.name)
}
if !expectedParam.validator(config, strValue) {
return errors.Tracef("invalid param: %s: %s", expectedParam.name, strValue)
}
return nil
}
func validateStringArrayRequestParam(
config *Config,
expectedParam requestParamSpec,
value interface{}) error {
arrayValue, ok := value.([]interface{})
if !ok {
return errors.Tracef("unexpected string param type: %s", expectedParam.name)
}
for _, value := range arrayValue {
err := validateStringRequestParam(config, expectedParam, value)
if err != nil {
return errors.Trace(err)
}
}
return nil
}
// getRequestLogFields makes LogFields to log the API event following
// the legacy psi_web and current ELK naming conventions.
func getRequestLogFields(
eventName string,
geoIPData GeoIPData,
authorizedAccessTypes []string,
params common.APIParameters,
expectedParams []requestParamSpec) LogFields {
logFields := make(LogFields)
if eventName != "" {
logFields["event_name"] = eventName
}
geoIPData.SetLogFields(logFields)
if len(authorizedAccessTypes) > 0 {
logFields["authorized_access_types"] = authorizedAccessTypes
}
if params == nil {
return logFields
}
for _, expectedParam := range expectedParams {
if expectedParam.flags&requestParamNotLogged != 0 {
continue
}
var tunnelProtocol string
if value, ok := params["relay_protocol"]; ok {
tunnelProtocol, _ = value.(string)
}
if expectedParam.flags&requestParamLogOnlyForFrontedMeek != 0 &&
!protocol.TunnelProtocolUsesFrontedMeek(tunnelProtocol) {
continue
}
if expectedParam.flags&requestParamNotLoggedForUnfrontedMeekNonTransformedHeader != 0 &&
protocol.TunnelProtocolUsesMeek(tunnelProtocol) &&
!protocol.TunnelProtocolUsesFrontedMeek(tunnelProtocol) {
// Non-HTTP unfronted meek protocols never tranform the host header.
if protocol.TunnelProtocolUsesMeekHTTPS(tunnelProtocol) {
continue
}
var transformedHostName string
if value, ok := params["meek_transformed_host_name"]; ok {
transformedHostName, _ = value.(string)
}
if transformedHostName != "1" {
continue
}
}
value := params[expectedParam.name]
if value == nil {
// Special case: older clients don't send this value,
// so log a default.
if expectedParam.name == "tunnel_whole_device" {
value = "0"
} else {
// Skip omitted, optional params
continue
}
}
switch v := value.(type) {
case string:
strValue := v
// Special cases:
// - Number fields are encoded as integer types.
// - For ELK performance we record certain domain-or-IP
// fields as one of two different values based on type;
// we also omit port from these host:port fields for now.
// - Boolean fields that come into the api as "1"/"0"
// must be logged as actual boolean values
switch expectedParam.name {
case "meek_dial_address":
host, _, _ := net.SplitHostPort(strValue)
if isIPAddress(nil, host) {
logFields["meek_dial_ip_address"] = host
} else {
logFields["meek_dial_domain"] = host
}
case "upstream_proxy_type":
// Submitted value could be e.g., "SOCKS5" or "socks5"; log lowercase
logFields[expectedParam.name] = strings.ToLower(strValue)
case tactics.SPEED_TEST_SAMPLES_PARAMETER_NAME:
// Due to a client bug, clients may deliever an incorrect ""
// value for speed_test_samples via the web API protocol. Omit
// the field in this case.
case "tunnel_error":
// net/url.Error, returned from net/url.Parse, contains the original input
// URL, which may contain PII. New clients strip this out by using
// common.SafeParseURL. Legacy clients will still send the full error
// message, so strip it out here. The target substring should be unique to
// legacy clients.
target := "upstreamproxy error: proxyURI url.Parse: parse "
index := strings.Index(strValue, target)
if index != -1 {
strValue = strValue[:index+len(target)] + ""
}
logFields[expectedParam.name] = strValue
default:
if expectedParam.flags&requestParamLogStringAsInt != 0 {
intValue, _ := strconv.Atoi(strValue)
logFields[expectedParam.name] = intValue
} else if expectedParam.flags&requestParamLogStringAsFloat != 0 {
floatValue, _ := strconv.ParseFloat(strValue, 64)
logFields[expectedParam.name] = floatValue
} else if expectedParam.flags&requestParamLogStringLengthAsInt != 0 {
logFields[expectedParam.name] = len(strValue)
} else if expectedParam.flags&requestParamLogFlagAsBool != 0 {
// Submitted value could be "0" or "1"
// "0" and non "0"/"1" values should be transformed to false
// "1" should be transformed to true
if strValue == "1" {
logFields[expectedParam.name] = true
} else {
logFields[expectedParam.name] = false
}
} else {
logFields[expectedParam.name] = strValue
}
}
case []interface{}:
if expectedParam.name == tactics.SPEED_TEST_SAMPLES_PARAMETER_NAME {
logFields[expectedParam.name] = makeSpeedTestSamplesLogField(v)
} else {
logFields[expectedParam.name] = v
}
default:
logFields[expectedParam.name] = v
}
}
return logFields
}
// makeSpeedTestSamplesLogField renames the tactics.SpeedTestSample json tag
// fields to more verbose names for metrics.
func makeSpeedTestSamplesLogField(samples []interface{}) []interface{} {
// TODO: use reflection and add additional tags, e.g.,
// `json:"s" log:"timestamp"` to remove hard-coded
// tag value dependency?
logSamples := make([]interface{}, len(samples))
for i, sample := range samples {
logSample := make(map[string]interface{})
if m, ok := sample.(map[string]interface{}); ok {
for k, v := range m {
logK := k
switch k {
case "s":
logK = "timestamp"
case "r":
logK = "server_region"
case "p":
logK = "relay_protocol"
case "t":
logK = "round_trip_time_ms"
case "u":
logK = "bytes_up"
case "d":
logK = "bytes_down"
}
logSample[logK] = v
}
}
logSamples[i] = logSample
}
return logSamples
}
func getOptionalStringRequestParam(params common.APIParameters, name string) (string, bool) {
if params[name] == nil {
return "", false
}
value, ok := params[name].(string)
if !ok {
return "", false
}
return value, true
}
func getStringRequestParam(params common.APIParameters, name string) (string, error) {
if params[name] == nil {
return "", errors.Tracef("missing param: %s", name)
}
value, ok := params[name].(string)
if !ok {
return "", errors.Tracef("invalid param: %s", name)
}
return value, nil
}
func getIntStringRequestParam(params common.APIParameters, name string) (int, error) {
if params[name] == nil {
return 0, errors.Tracef("missing param: %s", name)
}
valueStr, ok := params[name].(string)
if !ok {
return 0, errors.Tracef("invalid param: %s", name)
}
value, err := strconv.Atoi(valueStr)
if !ok {
return 0, errors.Trace(err)
}
return value, nil
}
func getBoolStringRequestParam(params common.APIParameters, name string) (bool, error) {
if params[name] == nil {
return false, errors.Tracef("missing param: %s", name)
}
valueStr, ok := params[name].(string)
if !ok {
return false, errors.Tracef("invalid param: %s", name)
}
if valueStr == "1" {
return true, nil
}
return false, nil
}
func getPaddingSizeRequestParam(params common.APIParameters, name string) (int, error) {
value, err := getIntStringRequestParam(params, name)
if err != nil {
return 0, errors.Trace(err)
}
if value < 0 {
value = 0
}
if value > PADDING_MAX_BYTES {
value = PADDING_MAX_BYTES
}
return int(value), nil
}
func getJSONObjectRequestParam(params common.APIParameters, name string) (common.APIParameters, error) {
if params[name] == nil {
return nil, errors.Tracef("missing param: %s", name)
}
// Note: generic unmarshal of JSON produces map[string]interface{}, not common.APIParameters
value, ok := params[name].(map[string]interface{})
if !ok {
return nil, errors.Tracef("invalid param: %s", name)
}
return common.APIParameters(value), nil
}
func getJSONObjectArrayRequestParam(params common.APIParameters, name string) ([]common.APIParameters, error) {
if params[name] == nil {
return nil, errors.Tracef("missing param: %s", name)
}
value, ok := params[name].([]interface{})
if !ok {
return nil, errors.Tracef("invalid param: %s", name)
}
result := make([]common.APIParameters, len(value))
for i, item := range value {
// Note: generic unmarshal of JSON produces map[string]interface{}, not common.APIParameters
resultItem, ok := item.(map[string]interface{})
if !ok {
return nil, errors.Tracef("invalid param: %s", name)
}
result[i] = common.APIParameters(resultItem)
}
return result, nil
}
func getMapStringInt64RequestParam(params common.APIParameters, name string) (map[string]int64, error) {
if params[name] == nil {
return nil, errors.Tracef("missing param: %s", name)
}
// TODO: can't use common.APIParameters type?
value, ok := params[name].(map[string]interface{})
if !ok {
return nil, errors.Tracef("invalid param: %s", name)
}
result := make(map[string]int64)
for k, v := range value {
numValue, ok := v.(float64)
if !ok {
return nil, errors.Tracef("invalid param: %s", name)
}
result[k] = int64(numValue)
}
return result, nil
}
func getStringArrayRequestParam(params common.APIParameters, name string) ([]string, error) {
if params[name] == nil {
return nil, errors.Tracef("missing param: %s", name)
}
value, ok := params[name].([]interface{})
if !ok {
return nil, errors.Tracef("invalid param: %s", name)
}
result := make([]string, len(value))
for i, v := range value {
strValue, ok := v.(string)
if !ok {
return nil, errors.Tracef("invalid param: %s", name)
}
result[i] = strValue
}
return result, nil
}
// Normalize reported client platform. Android clients, for example, report
// OS version, rooted status, and Google Play build status in the clientPlatform
// string along with "Android".
func normalizeClientPlatform(clientPlatform string) string {
if strings.Contains(strings.ToLower(clientPlatform), strings.ToLower(CLIENT_PLATFORM_ANDROID)) {
return CLIENT_PLATFORM_ANDROID
} else if strings.HasPrefix(clientPlatform, CLIENT_PLATFORM_IOS) {
return CLIENT_PLATFORM_IOS
}
return CLIENT_PLATFORM_WINDOWS
}
func isAnyString(config *Config, value string) bool {
return true
}
func isMobileClientPlatform(clientPlatform string) bool {
normalizedClientPlatform := normalizeClientPlatform(clientPlatform)
return normalizedClientPlatform == CLIENT_PLATFORM_ANDROID ||
normalizedClientPlatform == CLIENT_PLATFORM_IOS
}
// Input validators follow the legacy validations rules in psi_web.
func isServerSecret(config *Config, value string) bool {
return subtle.ConstantTimeCompare(
[]byte(value),
[]byte(config.WebServerSecret)) == 1
}
func isHexDigits(_ *Config, value string) bool {
// Allows both uppercase in addition to lowercase, for legacy support.
return -1 == strings.IndexFunc(value, func(c rune) bool {
return !unicode.Is(unicode.ASCII_Hex_Digit, c)
})
}
func isBase64String(_ *Config, value string) bool {
_, err := base64.StdEncoding.DecodeString(value)
return err == nil
}
func isDigits(_ *Config, value string) bool {
return -1 == strings.IndexFunc(value, func(c rune) bool {
return c < '0' || c > '9'
})
}
func isIntString(_ *Config, value string) bool {
_, err := strconv.Atoi(value)
return err == nil
}
func isFloatString(_ *Config, value string) bool {
_, err := strconv.ParseFloat(value, 64)
return err == nil
}
func isClientPlatform(_ *Config, value string) bool {
return -1 == strings.IndexFunc(value, func(c rune) bool {
// Note: stricter than psi_web's Python string.whitespace
return unicode.Is(unicode.White_Space, c)
})
}
func isRelayProtocol(_ *Config, value string) bool {
return common.Contains(protocol.SupportedTunnelProtocols, value)
}
func isBooleanFlag(_ *Config, value string) bool {
return value == "0" || value == "1"
}
func isUpstreamProxyType(_ *Config, value string) bool {
value = strings.ToLower(value)
return value == "http" || value == "socks5" || value == "socks4a"
}
func isRegionCode(_ *Config, value string) bool {
if len(value) != 2 {
return false
}
return -1 == strings.IndexFunc(value, func(c rune) bool {
return c < 'A' || c > 'Z'
})
}
func isDialAddress(_ *Config, value string) bool {
// ":", where is a domain or IP address
parts := strings.Split(value, ":")
if len(parts) != 2 {
return false
}
if !isIPAddress(nil, parts[0]) && !isDomain(nil, parts[0]) {
return false
}
if !isDigits(nil, parts[1]) {
return false
}
_, err := strconv.Atoi(parts[1])
if err != nil {
return false
}
// Allow port numbers outside [0,65535] to accommodate failed_tunnel cases.
return true
}
func isIPAddress(_ *Config, value string) bool {
return net.ParseIP(value) != nil
}
var isDomainRegex = regexp.MustCompile(`[a-zA-Z\d-]{1,63}$`)
func isDomain(_ *Config, value string) bool {
// From: http://stackoverflow.com/questions/2532053/validate-a-hostname-string
//
// "ensures that each segment
// * contains at least one character and a maximum of 63 characters
// * consists only of allowed characters
// * doesn't begin or end with a hyphen"
//
if len(value) > 255 {
return false
}
value = strings.TrimSuffix(value, ".")
for _, part := range strings.Split(value, ".") {
// Note: regexp doesn't support the following Perl expression which
// would check for '-' prefix/suffix: "(?!-)[a-zA-Z\\d-]{1,63}(?:", where is a domain or IP address and ":" is optional
if strings.Contains(value, ":") {
return isDialAddress(nil, value)
}
return isIPAddress(nil, value) || isDomain(nil, value)
}
func isServerEntrySource(_ *Config, value string) bool {
return common.Contains(protocol.SupportedServerEntrySources, value)
}
var isISO8601DateRegex = regexp.MustCompile(
`(?P[0-9]{4})-(?P[0-9]{1,2})-(?P[0-9]{1,2})T(?P[0-9]{2}):(?P[0-9]{2}):(?P[0-9]{2})(\.(?P[0-9]+))?(?PZ|(([-+])([0-9]{2}):([0-9]{2})))`)
func isISO8601Date(_ *Config, value string) bool {
return isISO8601DateRegex.Match([]byte(value))
}
func isLastConnected(_ *Config, value string) bool {
return value == "None" || isISO8601Date(nil, value)
}