Merge pull request #211 from rod-hynes/master

Upstream proxy automated tests; misc. fixes

psiphon/controller_test.go

@@ -35,12 +35,14 @@ import (
 	"time"
 
 	socks "github.com/Psiphon-Inc/goptlib"
+	"github.com/elazarl/goproxy"
 )
 
 func TestMain(m *testing.M) {
 	flag.Parse()
 	os.Remove(DATA_STORE_FILENAME)
 	initDisruptor()
+	initUpstreamProxy()
 	SetEmitDiagnosticNotices(true)
 	os.Exit(m.Run())
 }
@@ -79,6 +81,7 @@ func TestUntunneledUpgradeDownload(t *testing.T) {
 			disableEstablishing:      true,
 			disableApi:               false,
 			tunnelPoolSize:           1,
+			useUpstreamProxy:         false,
 			disruptNetwork:           false,
 			useHostNameTransformer:   false,
 			runDuration:              0,
@@ -95,6 +98,7 @@ func TestUntunneledResumableUpgradeDownload(t *testing.T) {
 			disableEstablishing:      true,
 			disableApi:               false,
 			tunnelPoolSize:           1,
+			useUpstreamProxy:         false,
 			disruptNetwork:           true,
 			useHostNameTransformer:   false,
 			runDuration:              0,
@@ -111,6 +115,7 @@ func TestUntunneledUpgradeClientIsLatestVersion(t *testing.T) {
 			disableEstablishing:      true,
 			disableApi:               false,
 			tunnelPoolSize:           1,
+			useUpstreamProxy:         false,
 			disruptNetwork:           false,
 			useHostNameTransformer:   false,
 			runDuration:              0,
@@ -127,6 +132,7 @@ func TestUntunneledResumableFetchRemoveServerList(t *testing.T) {
 			disableEstablishing:      false,
 			disableApi:               false,
 			tunnelPoolSize:           1,
+			useUpstreamProxy:         false,
 			disruptNetwork:           true,
 			useHostNameTransformer:   false,
 			runDuration:              0,
@@ -143,6 +149,7 @@ func TestTunneledUpgradeClientIsLatestVersion(t *testing.T) {
 			disableEstablishing:      false,
 			disableApi:               false,
 			tunnelPoolSize:           1,
+			useUpstreamProxy:         false,
 			disruptNetwork:           false,
 			useHostNameTransformer:   false,
 			runDuration:              0,
@@ -167,6 +174,7 @@ func TestImpairedProtocols(t *testing.T) {
 			disableEstablishing:      false,
 			disableApi:               false,
 			tunnelPoolSize:           40,
+			useUpstreamProxy:         false,
 			disruptNetwork:           true,
 			useHostNameTransformer:   false,
 			runDuration:              1 * time.Minute,
@@ -183,6 +191,7 @@ func TestSSH(t *testing.T) {
 			disableEstablishing:      false,
 			disableApi:               false,
 			tunnelPoolSize:           1,
+			useUpstreamProxy:         false,
 			disruptNetwork:           false,
 			useHostNameTransformer:   false,
 			runDuration:              0,
@@ -199,6 +208,7 @@ func TestObfuscatedSSH(t *testing.T) {
 			disableEstablishing:      false,
 			disableApi:               false,
 			tunnelPoolSize:           1,
+			useUpstreamProxy:         false,
 			disruptNetwork:           false,
 			useHostNameTransformer:   false,
 			runDuration:              0,
@@ -215,6 +225,7 @@ func TestUnfrontedMeek(t *testing.T) {
 			disableEstablishing:      false,
 			disableApi:               false,
 			tunnelPoolSize:           1,
+			useUpstreamProxy:         false,
 			disruptNetwork:           false,
 			useHostNameTransformer:   false,
 			runDuration:              0,
@@ -231,6 +242,7 @@ func TestUnfrontedMeekWithTransformer(t *testing.T) {
 			disableEstablishing:      false,
 			disableApi:               false,
 			tunnelPoolSize:           1,
+			useUpstreamProxy:         false,
 			disruptNetwork:           false,
 			useHostNameTransformer:   true,
 			runDuration:              0,
@@ -247,6 +259,7 @@ func TestFrontedMeek(t *testing.T) {
 			disableEstablishing:      false,
 			disableApi:               false,
 			tunnelPoolSize:           1,
+			useUpstreamProxy:         false,
 			disruptNetwork:           false,
 			useHostNameTransformer:   false,
 			runDuration:              0,
@@ -263,6 +276,7 @@ func TestFrontedMeekWithTransformer(t *testing.T) {
 			disableEstablishing:      false,
 			disableApi:               false,
 			tunnelPoolSize:           1,
+			useUpstreamProxy:         false,
 			disruptNetwork:           false,
 			useHostNameTransformer:   true,
 			runDuration:              0,
@@ -279,6 +293,7 @@ func TestFrontedMeekHTTP(t *testing.T) {
 			disableEstablishing:      false,
 			disableApi:               false,
 			tunnelPoolSize:           1,
+			useUpstreamProxy:         false,
 			disruptNetwork:           false,
 			useHostNameTransformer:   false,
 			runDuration:              0,
@@ -295,6 +310,7 @@ func TestUnfrontedMeekHTTPS(t *testing.T) {
 			disableEstablishing:      false,
 			disableApi:               false,
 			tunnelPoolSize:           1,
+			useUpstreamProxy:         false,
 			disruptNetwork:           false,
 			useHostNameTransformer:   false,
 			runDuration:              0,
@@ -311,6 +327,7 @@ func TestUnfrontedMeekHTTPSWithTransformer(t *testing.T) {
 			disableEstablishing:      false,
 			disableApi:               false,
 			tunnelPoolSize:           1,
+			useUpstreamProxy:         false,
 			disruptNetwork:           false,
 			useHostNameTransformer:   true,
 			runDuration:              0,
@@ -327,6 +344,58 @@ func TestDisabledApi(t *testing.T) {
 			disableEstablishing:      false,
 			disableApi:               true,
 			tunnelPoolSize:           1,
+			useUpstreamProxy:         false,
+			disruptNetwork:           false,
+			useHostNameTransformer:   false,
+			runDuration:              0,
+		})
+}
+
+func TestObfuscatedSSHWithUpstreamProxy(t *testing.T) {
+	controllerRun(t,
+		&controllerRunConfig{
+			expectNoServerEntries:    false,
+			protocol:                 TUNNEL_PROTOCOL_OBFUSCATED_SSH,
+			clientIsLatestVersion:    false,
+			disableUntunneledUpgrade: true,
+			disableEstablishing:      false,
+			disableApi:               false,
+			tunnelPoolSize:           1,
+			useUpstreamProxy:         true,
+			disruptNetwork:           false,
+			useHostNameTransformer:   false,
+			runDuration:              0,
+		})
+}
+
+func TestUnfrontedMeekWithUpstreamProxy(t *testing.T) {
+	controllerRun(t,
+		&controllerRunConfig{
+			expectNoServerEntries:    false,
+			protocol:                 TUNNEL_PROTOCOL_UNFRONTED_MEEK,
+			clientIsLatestVersion:    false,
+			disableUntunneledUpgrade: true,
+			disableEstablishing:      false,
+			disableApi:               false,
+			tunnelPoolSize:           1,
+			useUpstreamProxy:         true,
+			disruptNetwork:           false,
+			useHostNameTransformer:   false,
+			runDuration:              0,
+		})
+}
+
+func TestUnfrontedMeekHTTPSWithUpstreamProxy(t *testing.T) {
+	controllerRun(t,
+		&controllerRunConfig{
+			expectNoServerEntries:    false,
+			protocol:                 TUNNEL_PROTOCOL_UNFRONTED_MEEK_HTTPS,
+			clientIsLatestVersion:    false,
+			disableUntunneledUpgrade: true,
+			disableEstablishing:      false,
+			disableApi:               false,
+			tunnelPoolSize:           1,
+			useUpstreamProxy:         true,
 			disruptNetwork:           false,
 			useHostNameTransformer:   false,
 			runDuration:              0,
@@ -339,11 +408,12 @@ type controllerRunConfig struct {
 	clientIsLatestVersion    bool
 	disableUntunneledUpgrade bool
 	disableEstablishing      bool
+	disableApi               bool
 	tunnelPoolSize           int
+	useUpstreamProxy         bool
 	disruptNetwork           bool
 	useHostNameTransformer   bool
 	runDuration              time.Duration
-	disableApi               bool
 }
 
 func controllerRun(t *testing.T, runConfig *controllerRunConfig) {
@@ -379,8 +449,14 @@ func controllerRun(t *testing.T, runConfig *controllerRunConfig) {
 		config.UpgradeDownloadClientVersionHeader = ""
 	}
 
+	if runConfig.useUpstreamProxy && runConfig.disruptNetwork {
+		t.Fatalf("cannot use multiple upstream proxies")
+	}
 	if runConfig.disruptNetwork {
 		config.UpstreamProxyUrl = disruptorProxyURL
+	} else if runConfig.useUpstreamProxy {
+		config.UpstreamProxyUrl = upstreamProxyURL
+		config.UpstreamProxyCustomHeaders = upstreamProxyCustomHeaders
 	}
 
 	if runConfig.useHostNameTransformer {
@@ -841,3 +917,51 @@ func initDisruptor() {
 		}
 	}()
 }
+
+const upstreamProxyURL = "http://127.0.0.1:2161"
+
+var upstreamProxyCustomHeaders = map[string][]string{"X-Test-Header-Name": []string{"test-header-value1", "test-header-value2"}}
+
+func hasExpectedCustomHeaders(h http.Header) bool {
+	for name, values := range upstreamProxyCustomHeaders {
+		if h[name] == nil {
+			return false
+		}
+		// Order may not be the same
+		for _, value := range values {
+			if !Contains(h[name], value) {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+func initUpstreamProxy() {
+	go func() {
+		proxy := goproxy.NewProxyHttpServer()
+
+		proxy.OnRequest().DoFunc(
+			func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
+				if !hasExpectedCustomHeaders(r.Header) {
+					return nil, goproxy.NewResponse(r, goproxy.ContentTypeText, http.StatusUnauthorized, "")
+				}
+				return r, nil
+			})
+
+		proxy.OnRequest().HandleConnectFunc(
+			func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) {
+				if !hasExpectedCustomHeaders(ctx.Req.Header) {
+					return goproxy.RejectConnect, host
+				}
+				return goproxy.OkConnect, host
+			})
+
+		err := http.ListenAndServe("127.0.0.1:2161", proxy)
+		if err != nil {
+			fmt.Printf("upstream proxy failed: %s", err)
+		}
+	}()
+
+	// TODO: wait until listener is active?
+}

psiphon/meekConn.go

@@ -196,6 +196,7 @@ func DialMeek(
 			TrustedCACertificatesFilename: meekDialConfig.TrustedCACertificatesFilename,
 		})
 
+		// TODO: wrap in an http.Client and use http.Client.Timeout which actually covers round trip
 		transport = &http.Transport{
 			Dial: dialer,
 			ResponseHeaderTimeout: MEEK_ROUND_TRIP_TIMEOUT,
@@ -229,6 +230,7 @@ func DialMeek(
 			dialer = NewTCPDialer(meekDialConfig)
 		}
 
+		// TODO: wrap in an http.Client and use http.Client.Timeout which actually covers round trip
 		httpTransport := &http.Transport{
 			Proxy: proxyUrl,
 			Dial:  dialer,

psiphon/net.go

@@ -481,7 +481,6 @@ func MakeTunneledHttpClient(
 
 	transport := &http.Transport{
 		Dial: tunneledDialer,
-		ResponseHeaderTimeout: requestTimeout,
 	}
 
 	if config.UseTrustedCACertificatesForStockTLS {

psiphon/server/api.go

@@ -35,6 +35,9 @@ import (
 const (
 	MAX_API_PARAMS_SIZE = 256 * 1024 // 256KB
 
+	CLIENT_VERIFICATION_REQUIRED    = true
+	CLIENT_VERIFICATION_TTL_SECONDS = 60 * 60 * 24 * 7 // 7 days
+
 	CLIENT_PLATFORM_ANDROID = "Android"
 	CLIENT_PLATFORM_WINDOWS = "Windows"
 )
@@ -108,13 +111,17 @@ func handshakeAPIRequestHandler(
 
 	// TODO: share struct definition with psiphon/serverApi.go?
 	var handshakeResponse struct {
-		Homepages            []string            `json:"homepages"`
-		UpgradeClientVersion string              `json:"upgrade_client_version"`
-		PageViewRegexes      []map[string]string `json:"page_view_regexes"`
-		HttpsRequestRegexes  []map[string]string `json:"https_request_regexes"`
-		EncodedServerList    []string            `json:"encoded_server_list"`
-		ClientRegion         string              `json:"client_region"`
-		ServerTimestamp      string              `json:"server_timestamp"`
+		Homepages                     []string            `json:"homepages"`
+		UpgradeClientVersion          string              `json:"upgrade_client_version"`
+		PageViewRegexes               []map[string]string `json:"page_view_regexes"`
+		HttpsRequestRegexes           []map[string]string `json:"https_request_regexes"`
+		EncodedServerList             []string            `json:"encoded_server_list"`
+		ClientRegion                  string              `json:"client_region"`
+		ServerTimestamp               string              `json:"server_timestamp"`
+		ClientVerificationRequired    bool                `json:"client_verification_required"`
+		ClientVerificationServerNonce string              `json:"client_verification_server_nonce"`
+		ClientVerificationTTLSeconds  int                 `json:"client_verification_ttl_seconds"`
+		ClientVerificationResetCache  bool                `json:"client_verification_reset_cache"`
 	}
 
 	// Ignoring errors as params are validated
@@ -141,6 +148,11 @@ func handshakeAPIRequestHandler(
 
 	handshakeResponse.ServerTimestamp = psiphon.GetCurrentTimestamp()
 
+	handshakeResponse.ClientVerificationRequired = CLIENT_VERIFICATION_REQUIRED
+	handshakeResponse.ClientVerificationServerNonce = ""
+	handshakeResponse.ClientVerificationTTLSeconds = CLIENT_VERIFICATION_TTL_SECONDS
+	handshakeResponse.ClientVerificationResetCache = false
+
 	responsePayload, err := json.Marshal(handshakeResponse)
 	if err != nil {
 		return nil, psiphon.ContextError(err)
@@ -389,7 +401,7 @@ var baseRequestParams = []requestParamSpec{
 	requestParamSpec{"relay_protocol", isRelayProtocol, 0},
 	requestParamSpec{"tunnel_whole_device", isBooleanFlag, requestParamOptional},
 	requestParamSpec{"device_region", isRegionCode, requestParamOptional},
-	requestParamSpec{"upstream_proxy_type", isAnyString, requestParamOptional},
+	requestParamSpec{"upstream_proxy_type", isUpstreamProxyType, requestParamOptional},
 	requestParamSpec{"upstream_proxy_custom_header_names", isAnyString, requestParamOptional | requestParamArray},
 	requestParamSpec{"meek_dial_address", isDialAddress, requestParamOptional},
 	requestParamSpec{"meek_resolved_ip_address", isIPAddress, requestParamOptional},
@@ -528,6 +540,9 @@ func getRequestLogFields(
 			case "meek_host_header":
 				host, _, _ := net.SplitHostPort(strValue)
 				logFields[expectedParam.name] = host
+			case "upstream_proxy_type":
+				// Submitted value could be e.g., "SOCKS5" or "socks5"; log lowercase
+				logFields[expectedParam.name] = strings.ToLower(strValue)
 			default:
 				logFields[expectedParam.name] = strValue
 			}
@@ -636,16 +651,16 @@ func normalizeClientPlatform(clientPlatform string) string {
 	return CLIENT_PLATFORM_WINDOWS
 }
 
+func isAnyString(support *SupportServices, value string) bool {
+	return true
+}
+
 func isMobileClientPlatform(clientPlatform string) bool {
 	return normalizeClientPlatform(clientPlatform) == CLIENT_PLATFORM_ANDROID
 }
 
 // Input validators follow the legacy validations rules in psi_web.
 
-func isAnyString(support *SupportServices, value string) bool {
-	return true
-}
-
 func isServerSecret(support *SupportServices, value string) bool {
 	return subtle.ConstantTimeCompare(
 		[]byte(value),
@@ -679,6 +694,11 @@ func isBooleanFlag(_ *SupportServices, value string) bool {
 	return value == "0" || value == "1"
 }
 
+func isUpstreamProxyType(_ *SupportServices, value string) bool {
+	value = strings.ToLower(value)
+	return value == "http" || value == "socks5" || value == "socks4a"
+}
+
 func isRegionCode(_ *SupportServices, value string) bool {
 	if len(value) != 2 {
 		return false

psiphon/serverApi.go

@@ -181,7 +181,7 @@ func (serverContext *ServerContext) doHandshakeRequest() error {
 		ClientRegion                  string              `json:"client_region"`
 		ServerTimestamp               string              `json:"server_timestamp"`
 		ClientVerificationRequired    bool                `json:"client_verification_required"`
-		ClientVerificationServerNonce string              `json:"client_verification__server_nonce"`
+		ClientVerificationServerNonce string              `json:"client_verification_server_nonce"`
 		ClientVerificationTTLSeconds  int                 `json:"client_verification_ttl_seconds"`
 		ClientVerificationResetCache  bool                `json:"client_verification_reset_cache"`
 	}
@@ -905,7 +905,6 @@ func makePsiphonHttpsClient(tunnel *Tunnel) (httpsClient *http.Client, err error
 		})
 	transport := &http.Transport{
 		Dial: dialer,
-		ResponseHeaderTimeout: timeout,
 	}
 	return &http.Client{
 		Transport: transport,