Merge pull request #147 from rod-hynes/master

Meek refactor; untunneled upgrade; bug fixes

.travis.yml

@@ -16,5 +16,5 @@ before_install:
 - go get github.com/axw/gocov/gocov
 - go get github.com/mattn/goveralls
 - if ! go get github.com/golang/tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi
-- openssl aes-256-cbc -K $encrypted_9e40808ea1e2_key -iv $encrypted_9e40808ea1e2_iv
+- openssl aes-256-cbc -K $encrypted_ae0fe824cc69_key -iv $encrypted_ae0fe824cc69_iv
   -in psiphon/controller_test.config.enc -out psiphon/controller_test.config -d

SampleApps/Psibot/app/src/main/java/ca/psiphon/PsiphonTunnel.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, Psiphon Inc.
+ * Copyright (c) 2015, Psiphon Inc.
  * All rights reserved.
  *
  * This program is free software: you can redistribute it and/or modify
@@ -91,6 +91,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     }
 
     private final HostService mHostService;
+    private AtomicBoolean mVpnMode;
     private PrivateAddress mPrivateAddress;
     private AtomicReference<ParcelFileDescriptor> mTunFd;
     private AtomicInteger mLocalSocksProxyPort;
@@ -114,6 +115,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
 
     private PsiphonTunnel(HostService hostService) {
         mHostService = hostService;
+        mVpnMode = new AtomicBoolean(false);
         mTunFd = new AtomicReference<ParcelFileDescriptor>();
         mLocalSocksProxyPort = new AtomicInteger(0);
         mRoutingThroughTunnel = new AtomicBoolean(false);
@@ -149,10 +151,11 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         stopPsiphon();
         startPsiphon("");
     }
-
+    
     public synchronized void stop() {
         stopVpn();
         stopPsiphon();
+        mVpnMode.set(false);
         mLocalSocksProxyPort.set(0);
     }
 
@@ -165,7 +168,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     private final static int UDPGW_SERVER_PORT = 7300;
     private final static String DEFAULT_PRIMARY_DNS_SERVER = "8.8.4.4";
     private final static String DEFAULT_SECONDARY_DNS_SERVER = "8.8.8.8";
-
+    
     // Note: Atomic variables used for getting/setting local proxy port, routing flag, and
     // tun fd, as these functions may be called via PsiphonProvider callbacks. Do not use
     // synchronized functions as stop() is synchronized and a deadlock is possible as callbacks
@@ -174,6 +177,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     @TargetApi(Build.VERSION_CODES.ICE_CREAM_SANDWICH)
     private boolean startVpn() throws Exception {
 
+        mVpnMode.set(true);
         mPrivateAddress = selectPrivateAddress();
 
         Locale previousLocale = Locale.getDefault();
@@ -184,20 +188,21 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             Locale.setDefault(new Locale("en"));
 
             ParcelFileDescriptor tunFd =
-                    ((VpnService.Builder) mHostService.newVpnServiceBuilder())
-                            .setSession(mHostService.getAppName())
-                            .setMtu(VPN_INTERFACE_MTU)
-                            .addAddress(mPrivateAddress.mIpAddress, mPrivateAddress.mPrefixLength)
-                            .addRoute("0.0.0.0", 0)
-                            .addRoute(mPrivateAddress.mSubnet, mPrivateAddress.mPrefixLength)
-                            .addDnsServer(mPrivateAddress.mRouter)
-                            .establish();
+                ((VpnService.Builder) mHostService.newVpnServiceBuilder())
+                    .setSession(mHostService.getAppName())
+                    .setMtu(VPN_INTERFACE_MTU)
+                    .addAddress(mPrivateAddress.mIpAddress, mPrivateAddress.mPrefixLength)
+                    .addRoute("0.0.0.0", 0)
+                    .addRoute(mPrivateAddress.mSubnet, mPrivateAddress.mPrefixLength)
+                    .addDnsServer(mPrivateAddress.mRouter)
+                    .establish();
             if (tunFd == null) {
                 // As per http://developer.android.com/reference/android/net/VpnService.Builder.html#establish%28%29,
                 // this application is no longer prepared or was revoked.
                 return false;
             }
             mTunFd.set(tunFd);
+            mRoutingThroughTunnel.set(false);
 
             mHostService.onDiagnosticMessage("VPN established");
 
@@ -216,7 +221,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     }
 
     private boolean isVpnMode() {
-        return mTunFd.get() != null;
+        return mVpnMode.get();
     }
 
     private void setLocalSocksProxyPort(int port) {
@@ -227,10 +232,14 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         if (!mRoutingThroughTunnel.compareAndSet(false, true)) {
             return;
         }
+        ParcelFileDescriptor tunFd = mTunFd.getAndSet(null);
+        if (tunFd == null) {
+            return;
+        }
         String socksServerAddress = "127.0.0.1:" + Integer.toString(mLocalSocksProxyPort.get());
         String udpgwServerAddress = "127.0.0.1:" + Integer.toString(UDPGW_SERVER_PORT);
         startTun2Socks(
-                mTunFd.get(),
+                tunFd,
                 VPN_INTERFACE_MTU,
                 mPrivateAddress.mRouter,
                 VPN_INTERFACE_NETMASK,
@@ -244,6 +253,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     }
 
     private void stopVpn() {
+        stopTun2Socks();
         ParcelFileDescriptor tunFd = mTunFd.getAndSet(null);
         if (tunFd != null) {
             try {
@@ -251,10 +261,9 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             } catch (IOException e) {
             }
         }
-        waitStopTun2Socks();
         mRoutingThroughTunnel.set(false);
     }
-
+    
     //----------------------------------------------------------------------------------------------
     // PsiphonProvider (Core support) interface implementation
     //----------------------------------------------------------------------------------------------
@@ -312,10 +321,10 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         mHostService.onDiagnosticMessage("starting Psiphon library");
         try {
             Psi.Start(
-                    loadPsiphonConfig(mHostService.getContext()),
-                    embeddedServerEntries,
-                    this,
-                    isVpnMode());
+                loadPsiphonConfig(mHostService.getContext()),
+                embeddedServerEntries,
+                this,
+                isVpnMode());
         } catch (java.lang.Exception e) {
             throw new Exception("failed to start Psiphon library", e);
         }
@@ -334,7 +343,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         // Load settings from the raw resource JSON config file and
         // update as necessary. Then write JSON to disk for the Go client.
         JSONObject json = new JSONObject(mHostService.getPsiphonConfig());
-
+        
         // On Android, these directories must be set to the app private storage area.
         // The Psiphon library won't be able to use its current working directory
         // and the standard temporary directories do not exist.
@@ -361,7 +370,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         }
 
         json.put("UseIndistinguishableTLS", true);
-
+        
         if (Build.VERSION.SDK_INT < Build.VERSION_CODES.ICE_CREAM_SANDWICH) {
             json.put("UseTrustedCACertificatesForStockTLS", true);
         }
@@ -370,12 +379,12 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             // Also enable indistinguishable TLS for HTTPS requests that
             // require system CAs.
             json.put(
-                    "TrustedCACertificatesFilename",
-                    setupTrustedCertificates(mHostService.getContext()));
+                "TrustedCACertificatesFilename",
+                setupTrustedCertificates(mHostService.getContext()));
         } catch (Exception e) {
             mHostService.onDiagnosticMessage(e.getMessage());
         }
-
+        
         json.put("DeviceRegion", getDeviceRegion(mHostService.getContext()));
 
         return json.toString();
@@ -386,10 +395,10 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             // All notices are sent on as diagnostic messages
             // except those that may contain private user data.
             boolean diagnostic = true;
-
+            
             JSONObject notice = new JSONObject(noticeJSON);
             String noticeType = notice.getString("noticeType");
-
+            
             if (noticeType.equals("Tunnels")) {
                 int count = notice.getJSONObject("data").getInt("count");
                 if (count > 0) {
@@ -408,7 +417,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
                     regions.add(egressRegions.getString(i));
                 }
                 mHostService.onAvailableEgressRegions(regions);
-
+                
             } else if (noticeType.equals("SocksProxyPortInUse")) {
                 mHostService.onSocksProxyPortInUse(notice.getJSONObject("data").getInt("port"));
 
@@ -557,7 +566,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         }
         return region.toUpperCase(Locale.US);
     }
-
+    
     //----------------------------------------------------------------------------------------------
     // Tun2Socks
     //----------------------------------------------------------------------------------------------
@@ -571,11 +580,14 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             final String socksServerAddress,
             final String udpgwServerAddress,
             final boolean udpgwTransparentDNS) {
+        if (mTun2SocksThread != null) {
+            return;
+        }
         mTun2SocksThread = new Thread(new Runnable() {
             @Override
             public void run() {
                 runTun2Socks(
-                        vpnInterfaceFileDescriptor.getFd(),
+                        vpnInterfaceFileDescriptor.detachFd(),
                         vpnInterfaceMTU,
                         vpnIpAddress,
                         vpnNetMask,
@@ -588,10 +600,10 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         mHostService.onDiagnosticMessage("tun2socks started");
     }
 
-    private void waitStopTun2Socks() {
+    private void stopTun2Socks() {
         if (mTun2SocksThread != null) {
             try {
-                // Assumes mTunFd has been closed, which signals tun2socks to exit
+                terminateTun2Socks();
                 mTun2SocksThread.join();
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
@@ -614,6 +626,8 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             String socksServerAddress,
             String udpgwServerAddress,
             int udpgwTransparentDNS);
+    
+    private native static int terminateTun2Socks();
 
     static {
         System.loadLibrary("tun2socks");

psiphon/config.go

@@ -22,7 +22,9 @@ package psiphon
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"os"
+	"strconv"
 	"time"
 )
 
@@ -65,7 +67,8 @@ const (
 	PSIPHON_API_TUNNEL_STATS_MAX_COUNT             = 1000
 	FETCH_ROUTES_TIMEOUT                           = 1 * time.Minute
 	DOWNLOAD_UPGRADE_TIMEOUT                       = 15 * time.Minute
-	DOWNLOAD_UPGRADE_RETRY_PAUSE_PERIOD            = 5 * time.Second
+	DOWNLOAD_UPGRADE_RETRY_PERIOD                  = 5 * time.Second
+	DOWNLOAD_UPGRADE_STALE_PERIOD                  = 6 * time.Hour
 	IMPAIRED_PROTOCOL_CLASSIFICATION_DURATION      = 2 * time.Minute
 	IMPAIRED_PROTOCOL_CLASSIFICATION_THRESHOLD     = 3
 	TOTAL_BYTES_TRANSFERRED_NOTICE_PERIOD          = 5 * time.Minute
@@ -208,6 +211,10 @@ type Config struct {
 	// This parameter is only applicable to library deployments.
 	DnsServerGetter DnsServerGetter
 
+	// HostNameTransformer is an interface that enables pluggable hostname
+	// transformation circumvention strategies.
+	HostNameTransformer HostNameTransformer
+
 	// TargetServerEntry is an encoded server entry. When specified, this server entry
 	// is used exclusively and all other known servers are ignored.
 	TargetServerEntry string
@@ -248,6 +255,13 @@ type Config struct {
 	// typically embedded in the client binary.
 	UpgradeDownloadUrl string
 
+	// UpgradeDownloadClientVersionHeader specifies the HTTP header name for the
+	// entity at UpgradeDownloadUrl which specifies the client version (an integer
+	// value). A HEAD request may be made to check the version number available at
+	// UpgradeDownloadUrl. UpgradeDownloadClientVersionHeader is required when
+	// UpgradeDownloadUrl is specified.
+	UpgradeDownloadClientVersionHeader string
+
 	// UpgradeDownloadFilename is the local target filename for an upgrade download.
 	// This parameter is required when UpgradeDownloadUrl is specified.
 	UpgradeDownloadFilename string
@@ -293,7 +307,7 @@ type Config struct {
 	// EmitDiagnosticNotices indicates whether to output notices containing detailed
 	// information about the Psiphon session. As these notices may contain sensitive
 	// network information, they should not be insecurely distributed or displayed
-	// to users.
+	// to users. Default is off.
 	EmitDiagnosticNotices bool
 }
 
@@ -306,6 +320,11 @@ func LoadConfig(configJson []byte) (*Config, error) {
 		return nil, ContextError(err)
 	}
 
+	// Do setEmitDiagnosticNotices first, to ensure config file errors are emitted.
+	if config.EmitDiagnosticNotices {
+		setEmitDiagnosticNotices(true)
+	}
+
 	// These fields are required; the rest are optional
 	if config.PropagationChannelId == "" {
 		return nil, ContextError(
@@ -327,6 +346,12 @@ func LoadConfig(configJson []byte) (*Config, error) {
 		config.ClientVersion = "0"
 	}
 
+	_, err = strconv.Atoi(config.ClientVersion)
+	if err != nil {
+		return nil, ContextError(
+			fmt.Errorf("invalid client version: %s", err))
+	}
+
 	if config.TunnelProtocol != "" {
 		if !Contains(SupportedTunnelProtocols, config.TunnelProtocol) {
 			return nil, ContextError(
@@ -359,8 +384,14 @@ func LoadConfig(configJson []byte) (*Config, error) {
 		return nil, ContextError(errors.New("DnsServerGetter interface must be set at runtime"))
 	}
 
-	if config.EmitDiagnosticNotices {
-		setEmitDiagnosticNotices(true)
+	if config.HostNameTransformer != nil {
+		return nil, ContextError(errors.New("HostNameTransformer interface must be set at runtime"))
+	}
+
+	if config.UpgradeDownloadUrl != "" &&
+		(config.UpgradeDownloadClientVersionHeader == "" || config.UpgradeDownloadFilename == "") {
+		return nil, ContextError(errors.New(
+			"UpgradeDownloadUrl requires UpgradeDownloadClientVersionHeader and UpgradeDownloadFilename"))
 	}
 
 	return &config, nil

psiphon/controller.go

@@ -47,7 +47,6 @@ type Controller struct {
 	tunnels                        []*Tunnel
 	nextTunnel                     int
 	startedConnectedReporter       bool
-	startedUpgradeDownloader       bool
 	isEstablishing                 bool
 	establishWaitGroup             *sync.WaitGroup
 	stopEstablishingBroadcast      chan struct{}
@@ -57,6 +56,7 @@ type Controller struct {
 	untunneledDialConfig           *DialConfig
 	splitTunnelClassifier          *SplitTunnelClassifier
 	signalFetchRemoteServerList    chan struct{}
+	signalDownloadUpgrade          chan string
 	impairedProtocolClassification map[string]int
 	signalReportConnected          chan struct{}
 	serverAffinityDoneBroadcast    chan struct{}
@@ -73,6 +73,11 @@ func NewController(config *Config) (controller *Controller, err error) {
 	// Needed by regen, at least
 	rand.Seed(int64(time.Now().Nanosecond()))
 
+	// Supply a default HostNameTransformer
+	if config.HostNameTransformer == nil {
+		config.HostNameTransformer = &IdentityHostNameTransformer{}
+	}
+
 	// Generate a session ID for the Psiphon server API. This session ID is
 	// used across all tunnels established by the controller.
 	sessionId, err := MakeSessionId()
@@ -110,7 +115,6 @@ func NewController(config *Config) (controller *Controller, err error) {
 		tunnels:                        make([]*Tunnel, 0),
 		establishedOnce:                false,
 		startedConnectedReporter:       false,
-		startedUpgradeDownloader:       false,
 		isEstablishing:                 false,
 		establishPendingConns:          new(Conns),
 		untunneledPendingConns:         untunneledPendingConns,
@@ -120,6 +124,7 @@ func NewController(config *Config) (controller *Controller, err error) {
 		// starting? Trade-off is potential back-to-back fetch remotes. As-is,
 		// establish will eventually signal another fetch remote.
 		signalFetchRemoteServerList: make(chan struct{}),
+		signalDownloadUpgrade:       make(chan string),
 		signalReportConnected:       make(chan struct{}),
 	}
 
@@ -168,6 +173,13 @@ func (controller *Controller) Run(shutdownBroadcast <-chan struct{}) {
 		go controller.remoteServerListFetcher()
 	}
 
+	if controller.config.UpgradeDownloadUrl != "" &&
+		controller.config.UpgradeDownloadFilename != "" {
+
+		controller.runWaitGroup.Add(1)
+		go controller.upgradeDownloader()
+	}
+
 	/// Note: the connected reporter isn't started until a tunnel is
 	// established
 
@@ -365,63 +377,76 @@ func (controller *Controller) startOrSignalConnectedReporter() {
 // download. DownloadUpgrade() is resumable, so each attempt has potential for
 // getting closer to completion, even in conditions where the download or
 // tunnel is repeatedly interrupted.
-// Once the download is complete, the downloader exits and is not run again:
+// An upgrade download is triggered by either a handshake response indicating
+// that a new version is available; or after failing to connect, in which case
+// it's useful to check, out-of-band, for an upgrade with new circumvention
+// capabilities.
+// Once the download operation completes successfully, the downloader exits
+// and is not run again: either there is not a newer version, or the upgrade
+// has been downloaded and is ready to be applied.
 // We're assuming that the upgrade will be applied and the entire system
 // restarted before another upgrade is to be downloaded.
-func (controller *Controller) upgradeDownloader(clientUpgradeVersion string) {
+//
+// TODO: refactor upgrade downloader and remote server list fetcher to use
+// common code (including the resumable download routines).
+//
+func (controller *Controller) upgradeDownloader() {
 	defer controller.runWaitGroup.Done()
 
-loop:
-	for {
-		// Pick any active tunnel and make the next download attempt. No error
-		// is logged if there's no active tunnel, as that's not an unexpected condition.
-		tunnel := controller.getNextActiveTunnel()
-		if tunnel != nil {
-			err := DownloadUpgrade(controller.config, clientUpgradeVersion, tunnel)
-			if err == nil {
-				break loop
-			}
-			NoticeAlert("upgrade download failed: %s", err)
-		}
+	var lastDownloadTime time.Time
 
-		timeout := time.After(DOWNLOAD_UPGRADE_RETRY_PAUSE_PERIOD)
+downloadLoop:
+	for {
+		// Wait for a signal before downloading
+		var handshakeVersion string
 		select {
-		case <-timeout:
-			// Make another download attempt
+		case handshakeVersion = <-controller.signalDownloadUpgrade:
 		case <-controller.shutdownBroadcast:
-			break loop
+			break downloadLoop
 		}
-	}
 
-	NoticeInfo("exiting upgrade downloader")
-}
+		// Skip download entirely when a recent download was successful
+		if time.Now().Before(lastDownloadTime.Add(DOWNLOAD_UPGRADE_STALE_PERIOD)) {
+			continue
+		}
 
-func (controller *Controller) startClientUpgradeDownloader(
-	serverContext *ServerContext) {
+	retryLoop:
+		for {
+			// Don't attempt to download while there is no network connectivity,
+			// to avoid alert notice noise.
+			if !WaitForNetworkConnectivity(
+				controller.config.NetworkConnectivityChecker,
+				controller.shutdownBroadcast) {
+				break downloadLoop
+			}
 
-	// serverContext is nil when DisableApi is set
-	if controller.config.DisableApi {
-		return
-	}
+			// Pick any active tunnel and make the next download attempt. If there's
+			// no active tunnel, the untunneledDialConfig will be used.
+			tunnel := controller.getNextActiveTunnel()
 
-	if controller.config.UpgradeDownloadUrl == "" ||
-		controller.config.UpgradeDownloadFilename == "" {
-		// No upgrade is desired
-		return
-	}
+			err := DownloadUpgrade(
+				controller.config,
+				handshakeVersion,
+				tunnel,
+				controller.untunneledDialConfig)
 
-	if serverContext.clientUpgradeVersion == "" {
-		// No upgrade is offered
-		return
-	}
+			if err == nil {
+				lastDownloadTime = time.Now()
+				break retryLoop
+			}
 
-	// Start the client upgrade downloaded after the first tunnel is established.
-	// Concurrency note: only the runTunnels goroutine may access startClientUpgradeDownloader.
-	if !controller.startedUpgradeDownloader {
-		controller.startedUpgradeDownloader = true
-		controller.runWaitGroup.Add(1)
-		go controller.upgradeDownloader(serverContext.clientUpgradeVersion)
+			NoticeAlert("failed to download upgrade: %s", err)
+
+			timeout := time.After(DOWNLOAD_UPGRADE_RETRY_PERIOD)
+			select {
+			case <-timeout:
+			case <-controller.shutdownBroadcast:
+				break downloadLoop
+			}
+		}
 	}
+
+	NoticeInfo("exiting upgrade downloader")
 }
 
 // runTunnels is the controller tunnel management main loop. It starts and stops
@@ -499,8 +524,18 @@ loop:
 					// tunnel is established.
 					controller.startOrSignalConnectedReporter()
 
-					controller.startClientUpgradeDownloader(
-						establishedTunnel.serverContext)
+					// If the handshake indicated that a new client version is available,
+					// trigger an upgrade download.
+					// Note: serverContext is nil when DisableApi is set
+					if establishedTunnel.serverContext != nil &&
+						establishedTunnel.serverContext.clientUpgradeVersion != "" {
+
+						handshakeVersion := establishedTunnel.serverContext.clientUpgradeVersion
+						select {
+						case controller.signalDownloadUpgrade <- handshakeVersion:
+						default:
+						}
+					}
 				}
 
 			} else {
@@ -943,6 +978,14 @@ loop:
 		default:
 		}
 
+		// Trigger an out-of-band upgrade availability check and download.
+		// Since we may have failed to connect, we may benefit from upgrading
+		// to a new client version with new circumvention capabilities.
+		select {
+		case controller.signalDownloadUpgrade <- "":
+		default:
+		}
+
 		// After a complete iteration of candidate servers, pause before iterating again.
 		// This helps avoid some busy wait loop conditions, and also allows some time for
 		// network conditions to change. Also allows for fetch remote to complete,

psiphon/controller_test.go

@@ -20,41 +20,76 @@
 package psiphon
 
 import (
+	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
+
+	socks "github.com/Psiphon-Inc/goptlib"
 )
 
+func TestMain(m *testing.M) {
+	flag.Parse()
+	os.Remove(DATA_STORE_FILENAME)
+	initDisruptor()
+	setEmitDiagnosticNotices(true)
+	os.Exit(m.Run())
+}
+
+// Note: untunneled upgrade tests must execute before
+// the "Run" tests to ensure no tunnel is established.
+// We need a way to reset the datastore after it's been
+// initialized in order to to clear out its data entries
+// and be able to arbitrarily order the tests.
+
+/*
+
+TODO: temporarily disabled pending automation
+      support for untunneled upgrade downloads.
+
+func TestUntunneledUpgradeDownload(t *testing.T) {
+	doUntunnledUpgradeDownload(t, false)
+}
+
+func TestUntunneledResumableUpgradeDownload(t *testing.T) {
+	doUntunnledUpgradeDownload(t, true)
+}
+*/
+
 func TestControllerRunSSH(t *testing.T) {
-	controllerRun(t, TUNNEL_PROTOCOL_SSH)
+	controllerRun(t, TUNNEL_PROTOCOL_SSH, false)
 }
 
 func TestControllerRunObfuscatedSSH(t *testing.T) {
-	controllerRun(t, TUNNEL_PROTOCOL_OBFUSCATED_SSH)
+	controllerRun(t, TUNNEL_PROTOCOL_OBFUSCATED_SSH, false)
 }
 
 func TestControllerRunUnfrontedMeek(t *testing.T) {
-	controllerRun(t, TUNNEL_PROTOCOL_UNFRONTED_MEEK)
+	controllerRun(t, TUNNEL_PROTOCOL_UNFRONTED_MEEK, true)
 }
 
 func TestControllerRunFrontedMeek(t *testing.T) {
-	controllerRun(t, TUNNEL_PROTOCOL_FRONTED_MEEK)
+	controllerRun(t, TUNNEL_PROTOCOL_FRONTED_MEEK, true)
 }
 
 func TestControllerRunFrontedMeekHTTP(t *testing.T) {
-	controllerRun(t, TUNNEL_PROTOCOL_FRONTED_MEEK_HTTP)
+	controllerRun(t, TUNNEL_PROTOCOL_FRONTED_MEEK_HTTP, false)
 }
 
 func TestControllerRunUnfrontedMeekHTTPS(t *testing.T) {
-	controllerRun(t, TUNNEL_PROTOCOL_UNFRONTED_MEEK_HTTPS)
+	controllerRun(t, TUNNEL_PROTOCOL_UNFRONTED_MEEK_HTTPS, true)
 }
 
-func controllerRun(t *testing.T, protocol string) {
+func doUntunnledUpgradeDownload(t *testing.T, disrupt bool) {
 
 	configFileContents, err := ioutil.ReadFile("controller_test.config")
 	if err != nil {
@@ -63,21 +98,155 @@ func controllerRun(t *testing.T, protocol string) {
 	}
 	config, err := LoadConfig(configFileContents)
 	if err != nil {
-		t.Errorf("error processing configuration file: %s", err)
-		t.FailNow()
+		t.Fatalf("error processing configuration file: %s", err)
 	}
+
+	if disrupt {
+		config.UpstreamProxyUrl = disruptorProxyURL
+	}
+
+	// Clear remote server list so tunnel cannot be established and
+	// untunneled upgrade download case is tested.
+	config.RemoteServerListUrl = ""
+
+	os.Remove(config.UpgradeDownloadFilename)
+
+	err = InitDataStore(config)
+	if err != nil {
+		t.Fatalf("error initializing datastore: %s", err)
+	}
+
+	controller, err := NewController(config)
+	if err != nil {
+		t.Fatalf("error creating controller: %s", err)
+	}
+
+	upgradeDownloaded := make(chan struct{}, 1)
+
+	var clientUpgradeDownloadedBytesCount int32
+
+	SetNoticeOutput(NewNoticeReceiver(
+		func(notice []byte) {
+			// TODO: log notices without logging server IPs:
+			// fmt.Fprintf(os.Stderr, "%s\n", string(notice))
+			noticeType, payload, err := GetNotice(notice)
+			if err != nil {
+				return
+			}
+			switch noticeType {
+			case "Tunnels":
+				count := int(payload["count"].(float64))
+				if count > 0 {
+					// TODO: wrong goroutine for t.FatalNow()
+					t.Fatalf("tunnel established unexpectedly")
+				}
+			case "ClientUpgradeDownloadedBytes":
+				atomic.AddInt32(&clientUpgradeDownloadedBytesCount, 1)
+				t.Logf("ClientUpgradeDownloadedBytes: %d", int(payload["bytes"].(float64)))
+			case "ClientUpgradeDownloaded":
+				select {
+				case upgradeDownloaded <- *new(struct{}):
+				default:
+				}
+			}
+		}))
+
+	// Run controller
+
+	shutdownBroadcast := make(chan struct{})
+	controllerWaitGroup := new(sync.WaitGroup)
+	controllerWaitGroup.Add(1)
+	go func() {
+		defer controllerWaitGroup.Done()
+		controller.Run(shutdownBroadcast)
+	}()
+
+	defer func() {
+		// Test: shutdown must complete within 10 seconds
+
+		close(shutdownBroadcast)
+
+		shutdownTimeout := time.NewTimer(10 * time.Second)
+
+		shutdownOk := make(chan struct{}, 1)
+		go func() {
+			controllerWaitGroup.Wait()
+			shutdownOk <- *new(struct{})
+		}()
+
+		select {
+		case <-shutdownOk:
+		case <-shutdownTimeout.C:
+			t.Fatalf("controller shutdown timeout exceeded")
+		}
+	}()
+
+	// Test: upgrade must be downloaded within 120 seconds
+
+	downloadTimeout := time.NewTimer(120 * time.Second)
+
+	select {
+	case <-upgradeDownloaded:
+		// TODO: verify downloaded file
+
+	case <-downloadTimeout.C:
+		t.Fatalf("upgrade download timeout exceeded")
+	}
+
+	// Test: with disrupt, must be multiple download progress notices
+
+	if disrupt {
+		count := atomic.LoadInt32(&clientUpgradeDownloadedBytesCount)
+		if count <= 1 {
+			t.Fatalf("unexpected upgrade download progress: %d", count)
+		}
+	}
+}
+
+type TestHostNameTransformer struct {
+}
+
+func (TestHostNameTransformer) TransformHostName(string) (string, bool) {
+	return "example.com", true
+}
+
+func controllerRun(t *testing.T, protocol string, protocolUsesHostNameTransformer bool) {
+	doControllerRun(t, protocol, nil)
+	if protocolUsesHostNameTransformer {
+		t.Log("running with testHostNameTransformer")
+		doControllerRun(t, protocol, &TestHostNameTransformer{})
+	}
+}
+
+func doControllerRun(t *testing.T, protocol string, hostNameTransformer HostNameTransformer) {
+
+	configFileContents, err := ioutil.ReadFile("controller_test.config")
+	if err != nil {
+		// Skip, don't fail, if config file is not present
+		t.Skipf("error loading configuration file: %s", err)
+	}
+	config, err := LoadConfig(configFileContents)
+	if err != nil {
+		t.Fatalf("error processing configuration file: %s", err)
+	}
+
+	// Disable untunneled upgrade downloader to ensure tunneled case is tested
+	config.UpgradeDownloadClientVersionHeader = ""
+
+	os.Remove(config.UpgradeDownloadFilename)
+
 	config.TunnelProtocol = protocol
 
+	config.HostNameTransformer = hostNameTransformer
+
 	err = InitDataStore(config)
 	if err != nil {
-		t.Errorf("error initializing datastore: %s", err)
-		t.FailNow()
+		t.Fatalf("error initializing datastore: %s", err)
 	}
 
 	controller, err := NewController(config)
 	if err != nil {
-		t.Errorf("error creating controller: %s", err)
-		t.FailNow()
+		t.Fatalf("error creating controller: %s", err)
 	}
 
 	// Monitor notices for "Tunnels" with count > 1, the
@@ -88,6 +257,8 @@ func controllerRun(t *testing.T, protocol string) {
 	httpProxyPort := 0
 
 	tunnelEstablished := make(chan struct{}, 1)
+	upgradeDownloaded := make(chan struct{}, 1)
+
 	SetNoticeOutput(NewNoticeReceiver(
 		func(notice []byte) {
 			// TODO: log notices without logging server IPs:
@@ -105,13 +276,20 @@ func controllerRun(t *testing.T, protocol string) {
 					default:
 					}
 				}
+			case "ClientUpgradeDownloadedBytes":
+				t.Logf("ClientUpgradeDownloadedBytes: %d", int(payload["bytes"].(float64)))
+			case "ClientUpgradeDownloaded":
+				select {
+				case upgradeDownloaded <- *new(struct{}):
+				default:
+				}
 			case "ListeningHttpProxyPort":
 				httpProxyPort = int(payload["port"].(float64))
 			case "ConnectingServer":
 				serverProtocol := payload["protocol"]
 				if serverProtocol != protocol {
-					t.Errorf("wrong protocol selected: %s", serverProtocol)
-					t.FailNow()
+					// TODO: wrong goroutine for t.FatalNow()
+					t.Fatalf("wrong protocol selected: %s", serverProtocol)
 				}
 			}
 		}))
@@ -126,6 +304,26 @@ func controllerRun(t *testing.T, protocol string) {
 		controller.Run(shutdownBroadcast)
 	}()
 
+	defer func() {
+		// Test: shutdown must complete within 10 seconds
+
+		close(shutdownBroadcast)
+
+		shutdownTimeout := time.NewTimer(10 * time.Second)
+
+		shutdownOk := make(chan struct{}, 1)
+		go func() {
+			controllerWaitGroup.Wait()
+			shutdownOk <- *new(struct{})
+		}()
+
+		select {
+		case <-shutdownOk:
+		case <-shutdownTimeout.C:
+			t.Fatalf("controller shutdown timeout exceeded")
+		}
+	}()
+
 	// Test: tunnel must be established within 60 seconds
 
 	establishTimeout := time.NewTimer(60 * time.Second)
@@ -133,33 +331,26 @@ func controllerRun(t *testing.T, protocol string) {
 	select {
 	case <-tunnelEstablished:
 
-		// Allow for known race condition described in NewHttpProxy():
-		time.Sleep(1 * time.Second)
-
-		// Test: fetch website through tunnel
-		fetchWebsite(t, httpProxyPort)
-
 	case <-establishTimeout.C:
-		t.Errorf("tunnel establish timeout exceeded")
-		// ...continue with cleanup
+		t.Fatalf("tunnel establish timeout exceeded")
 	}
 
-	close(shutdownBroadcast)
+	// Allow for known race condition described in NewHttpProxy():
+	time.Sleep(1 * time.Second)
 
-	// Test: shutdown must complete within 10 seconds
+	// Test: fetch website through tunnel
+	fetchWebsite(t, httpProxyPort)
 
-	shutdownTimeout := time.NewTimer(10 * time.Second)
+	// Test: upgrade must be downloaded within 60 seconds
 
-	shutdownOk := make(chan struct{}, 1)
-	go func() {
-		controllerWaitGroup.Wait()
-		shutdownOk <- *new(struct{})
-	}()
+	downloadTimeout := time.NewTimer(60 * time.Second)
 
 	select {
-	case <-shutdownOk:
-	case <-shutdownTimeout.C:
-		t.Errorf("controller shutdown timeout exceeded")
+	case <-upgradeDownloaded:
+		// TODO: verify downloaded file
+
+	case <-downloadTimeout.C:
+		t.Fatalf("upgrade download timeout exceeded")
 	}
 }
 
@@ -177,8 +368,7 @@ func fetchWebsite(t *testing.T, httpProxyPort int) {
 
 	proxyUrl, err := url.Parse(fmt.Sprintf("http://127.0.0.1:%d", httpProxyPort))
 	if err != nil {
-		t.Errorf("error initializing proxied HTTP request: %s", err)
-		t.FailNow()
+		t.Fatalf("error initializing proxied HTTP request: %s", err)
 	}
 
 	httpClient := &http.Client{
@@ -190,20 +380,17 @@ func fetchWebsite(t *testing.T, httpProxyPort int) {
 
 	response, err := httpClient.Get(testUrl)
 	if err != nil {
-		t.Errorf("error sending proxied HTTP request: %s", err)
-		t.FailNow()
+		t.Fatalf("error sending proxied HTTP request: %s", err)
 	}
 
 	body, err := ioutil.ReadAll(response.Body)
 	if err != nil {
-		t.Errorf("error reading proxied HTTP response: %s", err)
-		t.FailNow()
+		t.Fatalf("error reading proxied HTTP response: %s", err)
 	}
 	response.Body.Close()
 
 	if !checkResponse(string(body)) {
-		t.Errorf("unexpected proxied HTTP response")
-		t.FailNow()
+		t.Fatalf("unexpected proxied HTTP response")
 	}
 
 	// Test: use direct URL proxy
@@ -217,20 +404,17 @@ func fetchWebsite(t *testing.T, httpProxyPort int) {
 		fmt.Sprintf("http://127.0.0.1:%d/direct/%s",
 			httpProxyPort, url.QueryEscape(testUrl)))
 	if err != nil {
-		t.Errorf("error sending direct URL request: %s", err)
-		t.FailNow()
+		t.Fatalf("error sending direct URL request: %s", err)
 	}
 
 	body, err = ioutil.ReadAll(response.Body)
 	if err != nil {
-		t.Errorf("error reading direct URL response: %s", err)
-		t.FailNow()
+		t.Fatalf("error reading direct URL response: %s", err)
 	}
 	response.Body.Close()
 
 	if !checkResponse(string(body)) {
-		t.Errorf("unexpected direct URL response")
-		t.FailNow()
+		t.Fatalf("unexpected direct URL response")
 	}
 
 	// Test: use tunneled URL proxy
@@ -239,19 +423,69 @@ func fetchWebsite(t *testing.T, httpProxyPort int) {
 		fmt.Sprintf("http://127.0.0.1:%d/tunneled/%s",
 			httpProxyPort, url.QueryEscape(testUrl)))
 	if err != nil {
-		t.Errorf("error sending tunneled URL request: %s", err)
-		t.FailNow()
+		t.Fatalf("error sending tunneled URL request: %s", err)
 	}
 
 	body, err = ioutil.ReadAll(response.Body)
 	if err != nil {
-		t.Errorf("error reading tunneled URL response: %s", err)
-		t.FailNow()
+		t.Fatalf("error reading tunneled URL response: %s", err)
 	}
 	response.Body.Close()
 
 	if !checkResponse(string(body)) {
-		t.Errorf("unexpected tunneled URL response")
-		t.FailNow()
+		t.Fatalf("unexpected tunneled URL response")
 	}
 }
+
+const disruptorProxyAddress = "127.0.0.1:2160"
+const disruptorProxyURL = "socks4a://" + disruptorProxyAddress
+const disruptorMaxConnectionBytes = 2000000
+const disruptorMaxConnectionTime = 15 * time.Second
+
+func initDisruptor() {
+
+	go func() {
+		listener, err := socks.ListenSocks("tcp", disruptorProxyAddress)
+		if err != nil {
+			fmt.Errorf("disruptor proxy listen error: %s", err)
+			return
+		}
+		for {
+			localConn, err := listener.AcceptSocks()
+			if err != nil {
+				fmt.Errorf("disruptor proxy accept error: %s", err)
+				return
+			}
+			go func() {
+				defer localConn.Close()
+				remoteConn, err := net.Dial("tcp", localConn.Req.Target)
+				if err != nil {
+					fmt.Errorf("disruptor proxy dial error: %s", err)
+					return
+				}
+				defer remoteConn.Close()
+				err = localConn.Grant(&net.TCPAddr{IP: net.ParseIP("0.0.0.0"), Port: 0})
+				if err != nil {
+					fmt.Errorf("disruptor proxy grant error: %s", err)
+					return
+				}
+
+				// Cut connection after disruptorMaxConnectionTime
+				time.AfterFunc(disruptorMaxConnectionTime, func() {
+					localConn.Close()
+					remoteConn.Close()
+				})
+
+				// Relay connection, but only up to disruptorMaxConnectionBytes
+				waitGroup := new(sync.WaitGroup)
+				waitGroup.Add(1)
+				go func() {
+					defer waitGroup.Done()
+					io.CopyN(localConn, remoteConn, disruptorMaxConnectionBytes)
+				}()
+				io.CopyN(remoteConn, localConn, disruptorMaxConnectionBytes)
+				waitGroup.Wait()
+			}()
+		}
+	}()
+}

psiphon/meekConn.go

@@ -59,6 +59,37 @@ const (
 	MEEK_ROUND_TRIP_TIMEOUT        = 20 * time.Second
 )
 
+// MeekConfig specifies the behavior of a MeekConn
+type MeekConfig struct {
+
+	// DialAddress is the actual network address to dial to establish a
+	// connection to the meek server. This may be either a fronted or
+	// direct address. The address must be in the form "host:port",
+	// where host may be a domain name or IP address.
+	DialAddress string
+
+	// UseHTTPS indicates whether to use HTTPS (true) or HTTP (false).
+	UseHTTPS bool
+
+	// SNIServerName is the value to place in the TLS SNI server_name
+	// field when HTTPS is used.
+	SNIServerName string
+
+	// HostHeader is the value to place in the HTTP request Host header.
+	HostHeader string
+
+	// TransformedHostName records whether a HostNameTransformer
+	// transformation is in effect. This value is used for stats reporting.
+	TransformedHostName bool
+
+	// The following values are used to create the obfuscated meek cookie.
+
+	PsiphonServerAddress          string
+	SessionID                     string
+	MeekCookieEncryptionPublicKey string
+	MeekObfuscatedKey             string
+}
+
 // MeekConn is a network connection that tunnels TCP over HTTP and supports "fronting". Meek sends
 // client->server flow in HTTP request bodies and receives server->client flow in HTTP response bodies.
 // Polling is used to achieve full duplex TCP.
@@ -71,9 +102,8 @@ const (
 // MeekConn also operates in unfronted mode, in which plain HTTP connections are made without routing
 // through a CDN.
 type MeekConn struct {
-	frontingAddress      string
-	useHTTPS             bool
 	url                  *url.URL
+	additionalHeaders    map[string]string
 	cookie               *http.Cookie
 	pendingConns         *Conns
 	transport            transporter
@@ -105,10 +135,8 @@ type transporter interface {
 // When frontingAddress is not "", fronting is used. This option assumes caller has
 // already checked server entry capabilities.
 func DialMeek(
-	serverEntry *ServerEntry, sessionId string,
-	useHTTPS, useSNI bool,
-	frontingAddress, frontingHost string,
-	config *DialConfig) (meek *MeekConn, err error) {
+	meekConfig *MeekConfig,
+	dialConfig *DialConfig) (meek *MeekConn, err error) {
 
 	// Configure transport
 	// Note: MeekConn has its own PendingConns to manage the underlying HTTP transport connections,
@@ -118,136 +146,121 @@ func DialMeek(
 	pendingConns := new(Conns)
 
 	// Use a copy of DialConfig with the meek pendingConns
-	meekConfig := new(DialConfig)
-	*meekConfig = *config
-	meekConfig.PendingConns = pendingConns
-
-	var host string
-	var dialer Dialer
-	var proxyUrl func(*http.Request) (*url.URL, error)
-
-	if frontingAddress != "" {
-
-		// In this case, host is not what is dialed but is what ends up in the HTTP Host header
-		host = frontingHost
-
-		if useHTTPS {
-
-			// Custom TLS dialer:
-			//
-			//  1. ignores the HTTP request address and uses the fronting domain
-			//  2. optionally disables SNI -- SNI breaks fronting when used with certain CDNs.
-			//  3. skips verifying the server cert.
-			//
-			// Reasoning for #3:
-			//
-			// With a TLS MiM attack in place, and server certs verified, we'll fail to connect because the client
-			// will refuse to connect. That's not a successful outcome.
-			//
-			// With a MiM attack in place, and server certs not verified, we'll fail to connect if the MiM is actively
-			// targeting Psiphon and classifying the HTTP traffic by Host header or payload signature.
-			//
-			// However, in the case of a passive MiM that's just recording traffic or an active MiM that's targeting
-			// something other than Psiphon, the client will connect. This is a successful outcome.
-			//
-			// What is exposed to the MiM? The Host header does not contain a Psiphon server IP address, just an
-			// unrelated, randomly generated domain name which cannot be used to block direct connections. The
-			// Psiphon server IP is sent over meek, but it's in the encrypted cookie.
-			//
-			// The payload (user traffic) gets its confidentiality and integrity from the underlying SSH protocol.
-			// So, nothing is leaked to the MiM apart from signatures which could be used to classify the traffic
-			// as Psiphon to possibly block it; but note that not revealing that the client is Psiphon is outside
-			// our threat model; we merely seek to evade mass blocking by taking steps that require progressively
-			// more effort to block.
-			//
-			// There is a subtle attack remaining: an adversary that can MiM some CDNs but not others (and so can
-			// classify Psiphon traffic on some CDNs but not others) may throttle non-MiM CDNs so that our server
-			// selection always chooses tunnels to the MiM CDN (without any server cert verification, we won't
-			// exclusively connect to non-MiM CDNs); then the adversary kills the underlying TCP connection after
-			// some short period. This is mitigated by the "impaired" protocol classification mechanism.
-
-			customTLSConfig := &CustomTLSConfig{
-				FrontingAddr:                  fmt.Sprintf("%s:%d", frontingAddress, 443),
-				Dial:                          NewTCPDialer(meekConfig),
-				Timeout:                       meekConfig.ConnectTimeout,
-				SendServerName:                useSNI,
-				SkipVerify:                    true,
-				UseIndistinguishableTLS:       config.UseIndistinguishableTLS,
-				TrustedCACertificatesFilename: config.TrustedCACertificatesFilename,
-			}
+	meekDialConfig := new(DialConfig)
+	*meekDialConfig = *dialConfig
+	meekDialConfig.PendingConns = pendingConns
 
-			dialer = NewCustomTLSDialer(customTLSConfig)
+	var transport transporter
 
-		} else { // !useHTTPS
+	if meekConfig.UseHTTPS {
+		// Custom TLS dialer:
+		//
+		//  1. ignores the HTTP request address and uses the fronting domain
+		//  2. optionally disables SNI -- SNI breaks fronting when used with certain CDNs.
+		//  3. skips verifying the server cert.
+		//
+		// Reasoning for #3:
+		//
+		// With a TLS MiM attack in place, and server certs verified, we'll fail to connect because the client
+		// will refuse to connect. That's not a successful outcome.
+		//
+		// With a MiM attack in place, and server certs not verified, we'll fail to connect if the MiM is actively
+		// targeting Psiphon and classifying the HTTP traffic by Host header or payload signature.
+		//
+		// However, in the case of a passive MiM that's just recording traffic or an active MiM that's targeting
+		// something other than Psiphon, the client will connect. This is a successful outcome.
+		//
+		// What is exposed to the MiM? The Host header does not contain a Psiphon server IP address, just an
+		// unrelated, randomly generated domain name which cannot be used to block direct connections. The
+		// Psiphon server IP is sent over meek, but it's in the encrypted cookie.
+		//
+		// The payload (user traffic) gets its confidentiality and integrity from the underlying SSH protocol.
+		// So, nothing is leaked to the MiM apart from signatures which could be used to classify the traffic
+		// as Psiphon to possibly block it; but note that not revealing that the client is Psiphon is outside
+		// our threat model; we merely seek to evade mass blocking by taking steps that require progressively
+		// more effort to block.
+		//
+		// There is a subtle attack remaining: an adversary that can MiM some CDNs but not others (and so can
+		// classify Psiphon traffic on some CDNs but not others) may throttle non-MiM CDNs so that our server
+		// selection always chooses tunnels to the MiM CDN (without any server cert verification, we won't
+		// exclusively connect to non-MiM CDNs); then the adversary kills the underlying TCP connection after
+		// some short period. This is mitigated by the "impaired" protocol classification mechanism.
+
+		dialer := NewCustomTLSDialer(&CustomTLSConfig{
+			DialAddr:                      meekConfig.DialAddress,
+			Dial:                          NewTCPDialer(meekDialConfig),
+			Timeout:                       meekDialConfig.ConnectTimeout,
+			SNIServerName:                 meekConfig.SNIServerName,
+			SkipVerify:                    true,
+			UseIndistinguishableTLS:       meekDialConfig.UseIndistinguishableTLS,
+			TrustedCACertificatesFilename: meekDialConfig.TrustedCACertificatesFilename,
+		})
+
+		transport = &http.Transport{
+			Dial: dialer,
+			ResponseHeaderTimeout: MEEK_ROUND_TRIP_TIMEOUT,
+		}
+	} else {
 
-			dialer = func(string, string) (net.Conn, error) {
-				return NewTCPDialer(meekConfig)("tcp", frontingAddress+":80")
+		// For HTTP meek, we let the http.Transport handle proxying. http.Transport will
+		// put the the HTTP server address in the HTTP request line. In this case, we can
+		// use an HTTP proxy that does not support CONNECT.
+		var proxyUrl func(*http.Request) (*url.URL, error)
+		if strings.HasPrefix(meekDialConfig.UpstreamProxyUrl, "http://") &&
+			(meekConfig.DialAddress == meekConfig.HostHeader ||
+				meekConfig.DialAddress == meekConfig.HostHeader+":80") {
+			url, err := url.Parse(meekDialConfig.UpstreamProxyUrl)
+			if err != nil {
+				return nil, ContextError(err)
 			}
+			proxyUrl = http.ProxyURL(url)
+			meekDialConfig.UpstreamProxyUrl = ""
 		}
 
-	} else { // frontingAddress == ""
-
-		// host is both what is dialed and what ends up in the HTTP Host header
-		host = fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.MeekServerPort)
-
-		if useHTTPS {
-
-			customTLSConfig := &CustomTLSConfig{
-				Dial:                          NewTCPDialer(meekConfig),
-				Timeout:                       meekConfig.ConnectTimeout,
-				SendServerName:                useSNI,
-				SkipVerify:                    true,
-				UseIndistinguishableTLS:       config.UseIndistinguishableTLS,
-				TrustedCACertificatesFilename: config.TrustedCACertificatesFilename,
-			}
+		// dialer ignores address that http.Transport will pass in (derived from
+		// the HTTP request URL) and always dials meekConfig.DialAddress.
+		dialer := func(string, string) (net.Conn, error) {
+			return NewTCPDialer(meekDialConfig)("tcp", meekConfig.DialAddress)
+		}
 
-			dialer = NewCustomTLSDialer(customTLSConfig)
-
-		} else { // !useHTTPS
-
-			if strings.HasPrefix(meekConfig.UpstreamProxyUrl, "http://") {
-				// For unfronted meek, we let the http.Transport handle proxying, as the
-				// target server hostname has to be in the HTTP request line. Also, in this
-				// case, we don't require the proxy to support CONNECT and so we can work
-				// through HTTP proxies that don't support it.
-				url, err := url.Parse(meekConfig.UpstreamProxyUrl)
-				if err != nil {
-					return nil, ContextError(err)
-				}
-				proxyUrl = http.ProxyURL(url)
-				meekConfig.UpstreamProxyUrl = ""
+		httpTransport := &http.Transport{
+			Proxy: proxyUrl,
+			Dial:  dialer,
+			ResponseHeaderTimeout: MEEK_ROUND_TRIP_TIMEOUT,
+		}
+		if proxyUrl != nil {
+			// Wrap transport with a transport that can perform HTTP proxy auth negotiation
+			transport, err = upstreamproxy.NewProxyAuthTransport(httpTransport)
+			if err != nil {
+				return nil, ContextError(err)
 			}
-
-			dialer = NewTCPDialer(meekConfig)
+		} else {
+			transport = httpTransport
 		}
-
 	}
 
 	// Scheme is always "http". Otherwise http.Transport will try to do another TLS
 	// handshake inside the explicit TLS session (in fronting mode).
 	url := &url.URL{
 		Scheme: "http",
-		Host:   host,
+		Host:   meekConfig.HostHeader,
 		Path:   "/",
 	}
-	cookie, err := makeCookie(serverEntry, sessionId)
-	if err != nil {
-		return nil, ContextError(err)
-	}
-	httpTransport := &http.Transport{
-		Proxy: proxyUrl,
-		Dial:  dialer,
-		ResponseHeaderTimeout: MEEK_ROUND_TRIP_TIMEOUT,
-	}
-	var transport transporter
-	if proxyUrl != nil {
-		// Wrap transport with a transport that can perform HTTP proxy auth negotiation
-		transport, err = upstreamproxy.NewProxyAuthTransport(httpTransport)
+
+	var additionalHeaders map[string]string
+	if meekConfig.UseHTTPS {
+		host, _, err := net.SplitHostPort(meekConfig.DialAddress)
 		if err != nil {
 			return nil, ContextError(err)
 		}
-	} else {
-		transport = httpTransport
+		additionalHeaders = map[string]string{
+			"X-Psiphon-Fronting-Address": host,
+		}
+	}
+
+	cookie, err := makeCookie(meekConfig)
+	if err != nil {
+		return nil, ContextError(err)
 	}
 
 	// The main loop of a MeekConn is run in the relay() goroutine.
@@ -267,9 +280,8 @@ func DialMeek(
 	// Write() calls and relay() are synchronized in a similar way, using a single
 	// sendBuffer.
 	meek = &MeekConn{
-		frontingAddress:      frontingAddress,
-		useHTTPS:             useHTTPS,
 		url:                  url,
+		additionalHeaders:    additionalHeaders,
 		cookie:               cookie,
 		pendingConns:         pendingConns,
 		transport:            transport,
@@ -290,7 +302,7 @@ func DialMeek(
 	go meek.relay()
 
 	// Enable interruption
-	if !config.PendingConns.Add(meek) {
+	if !dialConfig.PendingConns.Add(meek) {
 		meek.Close()
 		return nil, ContextError(errors.New("pending connections already closed"))
 	}
@@ -524,16 +536,16 @@ func (meek *MeekConn) roundTrip(sendPayload []byte) (receivedPayload io.ReadClos
 		return nil, ContextError(err)
 	}
 
-	if meek.useHTTPS {
-		request.Header.Set("X-Psiphon-Fronting-Address", meek.frontingAddress)
-	}
-
 	// Don't use the default user agent ("Go 1.1 package http").
 	// For now, just omit the header (net/http/request.go: "may be blank to not send the header").
 	request.Header.Set("User-Agent", "")
 
 	request.Header.Set("Content-Type", "application/octet-stream")
 
+	for name, value := range meek.additionalHeaders {
+		request.Header.Set(name, value)
+	}
+
 	request.AddCookie(meek.cookie)
 
 	// The retry mitigates intermittent failures between the client and front/server.
@@ -624,13 +636,13 @@ type meekCookieData struct {
 // all consequent HTTP requests
 // In unfronted meek mode, the cookie is visible over the adversary network, so the
 // cookie is encrypted and obfuscated.
-func makeCookie(serverEntry *ServerEntry, sessionId string) (cookie *http.Cookie, err error) {
+func makeCookie(meekConfig *MeekConfig) (cookie *http.Cookie, err error) {
 
 	// Make the JSON data
-	serverAddress := fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.SshObfuscatedPort)
+	serverAddress := meekConfig.PsiphonServerAddress
 	cookieData := &meekCookieData{
 		ServerAddress:       serverAddress,
-		SessionID:           sessionId,
+		SessionID:           meekConfig.SessionID,
 		MeekProtocolVersion: MEEK_PROTOCOL_VERSION,
 	}
 	serializedCookie, err := json.Marshal(cookieData)
@@ -647,7 +659,7 @@ func makeCookie(serverEntry *ServerEntry, sessionId string) (cookie *http.Cookie
 	// different messages if the messages are sent to two different public keys."
 	var nonce [24]byte
 	var publicKey [32]byte
-	decodedPublicKey, err := base64.StdEncoding.DecodeString(serverEntry.MeekCookieEncryptionPublicKey)
+	decodedPublicKey, err := base64.StdEncoding.DecodeString(meekConfig.MeekCookieEncryptionPublicKey)
 	if err != nil {
 		return nil, ContextError(err)
 	}
@@ -663,7 +675,7 @@ func makeCookie(serverEntry *ServerEntry, sessionId string) (cookie *http.Cookie
 
 	// Obfuscate the encrypted data
 	obfuscator, err := NewObfuscator(
-		&ObfuscatorConfig{Keyword: serverEntry.MeekObfuscatedKey, MaxPadding: MEEK_COOKIE_MAX_PADDING})
+		&ObfuscatorConfig{Keyword: meekConfig.MeekObfuscatedKey, MaxPadding: MEEK_COOKIE_MAX_PADDING})
 	if err != nil {
 		return nil, ContextError(err)
 	}

psiphon/net.go

@@ -97,11 +97,6 @@ type DialConfig struct {
 	ResolvedIPCallback func(string)
 }
 
-// DeviceBinder defines the interface to the external BindToDevice provider
-type DeviceBinder interface {
-	BindToDevice(fileDescriptor int) error
-}
-
 // NetworkConnectivityChecker defines the interface to the external
 // HasNetworkConnectivity provider
 type NetworkConnectivityChecker interface {
@@ -109,12 +104,31 @@ type NetworkConnectivityChecker interface {
 	HasNetworkConnectivity() int
 }
 
+// DeviceBinder defines the interface to the external BindToDevice provider
+type DeviceBinder interface {
+	BindToDevice(fileDescriptor int) error
+}
+
 // DnsServerGetter defines the interface to the external GetDnsServer provider
 type DnsServerGetter interface {
 	GetPrimaryDnsServer() string
 	GetSecondaryDnsServer() string
 }
 
+// HostNameTransformer defines the interface for pluggable hostname
+// transformation circumvention strategies.
+type HostNameTransformer interface {
+	TransformHostName(hostname string) (string, bool)
+}
+
+// IdentityHostNameTransformer is the default HostNameTransformer, which
+// returns the hostname unchanged.
+type IdentityHostNameTransformer struct{}
+
+func (IdentityHostNameTransformer) TransformHostName(hostname string) (string, bool) {
+	return hostname, false
+}
+
 // TimeoutError implements the error interface
 type TimeoutError struct{}
 
@@ -276,26 +290,15 @@ func MakeUntunneledHttpsClient(
 	requestUrl string,
 	requestTimeout time.Duration) (*http.Client, string, error) {
 
-	dialer := NewCustomTLSDialer(
-		// Note: when verifyLegacyCertificate is not nil, some
-		// of the other CustomTLSConfig is overridden.
-		&CustomTLSConfig{
-			Dial: NewTCPDialer(dialConfig),
-			VerifyLegacyCertificate:       verifyLegacyCertificate,
-			SendServerName:                true,
-			SkipVerify:                    false,
-			UseIndistinguishableTLS:       dialConfig.UseIndistinguishableTLS,
-			TrustedCACertificatesFilename: dialConfig.TrustedCACertificatesFilename,
-		})
+	// Change the scheme to "http"; otherwise http.Transport will try to do
+	// another TLS handshake inside the explicit TLS session. Also need to
+	// force an explicit port, as the default for "http", 80, won't talk TLS.
 
 	urlComponents, err := url.Parse(requestUrl)
 	if err != nil {
 		return nil, "", ContextError(err)
 	}
 
-	// Change the scheme to "http"; otherwise http.Transport will try to do
-	// another TLS handshake inside the explicit TLS session. Also need to
-	// force an explicit port, as the default for "http", 80, won't talk TLS.
 	urlComponents.Scheme = "http"
 	host, port, err := net.SplitHostPort(urlComponents.Host)
 	if err != nil {
@@ -308,6 +311,21 @@ func MakeUntunneledHttpsClient(
 	}
 	urlComponents.Host = net.JoinHostPort(host, port)
 
+	// Note: IndistinguishableTLS mode doesn't support VerifyLegacyCertificate
+	useIndistinguishableTLS := dialConfig.UseIndistinguishableTLS && verifyLegacyCertificate == nil
+
+	dialer := NewCustomTLSDialer(
+		// Note: when verifyLegacyCertificate is not nil, some
+		// of the other CustomTLSConfig is overridden.
+		&CustomTLSConfig{
+			Dial: NewTCPDialer(dialConfig),
+			VerifyLegacyCertificate:       verifyLegacyCertificate,
+			SNIServerName:                 host,
+			SkipVerify:                    false,
+			UseIndistinguishableTLS:       useIndistinguishableTLS,
+			TrustedCACertificatesFilename: dialConfig.TrustedCACertificatesFilename,
+		})
+
 	transport := &http.Transport{
 		Dial: dialer,
 	}

psiphon/notice.go

@@ -137,9 +137,24 @@ func NoticeAvailableEgressRegions(regions []string) {
 }
 
 // NoticeConnectingServer is details on a connection attempt
-func NoticeConnectingServer(ipAddress, region, protocol, frontingAddress string) {
-	outputNotice("ConnectingServer", true, false, "ipAddress", ipAddress, "region",
-		region, "protocol", protocol, "frontingAddress", frontingAddress)
+func NoticeConnectingServer(ipAddress, region, protocol, directTCPDialAddress string, meekConfig *MeekConfig) {
+	if meekConfig == nil {
+		outputNotice("ConnectingServer", true, false,
+			"ipAddress", ipAddress,
+			"region", region,
+			"protocol", protocol,
+			"directTCPDialAddress", directTCPDialAddress)
+	} else {
+		outputNotice("ConnectingServer", true, false,
+			"ipAddress", ipAddress,
+			"region", region,
+			"protocol", protocol,
+			"meekDialAddress", meekConfig.DialAddress,
+			"meekUseHTTPS", meekConfig.UseHTTPS,
+			"meekSNIServerName", meekConfig.SNIServerName,
+			"meekHostHeader", meekConfig.HostHeader,
+			"meekTransformedHostName", meekConfig.TransformedHostName)
+	}
 }
 
 // NoticeActiveTunnel is a successful connection that is used as an active tunnel for port forwarding
@@ -213,6 +228,11 @@ func NoticeUpstreamProxyError(err error) {
 	outputNotice("UpstreamProxyError", false, true, "message", err.Error())
 }
 
+// NoticeClientUpgradeDownloadedBytes reports client upgrade download progress.
+func NoticeClientUpgradeDownloadedBytes(bytes int64) {
+	outputNotice("ClientUpgradeDownloadedBytes", true, false, "bytes", bytes)
+}
+
 // NoticeClientUpgradeDownloaded indicates that a client upgrade download
 // is complete and available at the destination specified.
 func NoticeClientUpgradeDownloaded(filename string) {
@@ -263,14 +283,15 @@ func NoticeLocalProxyError(proxyType string, err error) {
 		"LocalProxyError", true, false, "message", err.Error())
 }
 
-// NoticeFrontedMeekStats reports extra network details for a
-// FRONTED-MEEK-OSSH or FRONTED-MEEK-HTTP-OSSH tunnel connection.
-func NoticeFrontedMeekStats(ipAddress string, frontedMeekStats *FrontedMeekStats) {
-	outputNotice("NoticeFrontedMeekStats", true, false, "ipAddress", ipAddress,
-		"frontingAddress", frontedMeekStats.frontingAddress,
-		"resolvedIPAddress", frontedMeekStats.resolvedIPAddress,
-		"enabledSNI", frontedMeekStats.enabledSNI,
-		"frontingHost", frontedMeekStats.frontingHost)
+// NoticeConnectedMeekStats reports extra network details for a meek tunnel connection.
+func NoticeConnectedMeekStats(ipAddress string, meekStats *MeekStats) {
+	outputNotice("NoticeConnectedMeekStats", true, false,
+		"ipAddress", ipAddress,
+		"dialAddress", meekStats.DialAddress,
+		"resolvedIPAddress", meekStats.ResolvedIPAddress,
+		"sniServerName", meekStats.SNIServerName,
+		"hostHeader", meekStats.HostHeader,
+		"transformedHostName", meekStats.TransformedHostName)
 }
 
 // NoticeBuildInfo reports build version info.

psiphon/opensslConn.go

@@ -96,12 +96,12 @@ func newOpenSSLConn(rawConn net.Conn, hostname string, config *CustomTLSConfig)
 		return nil, ContextError(err)
 	}
 
-	if config.SendServerName {
+	if config.SNIServerName {
 		// Explicitly exclude IPs:
 		// - "Literal IPv4 and IPv6 addresses are not permitted": https://tools.ietf.org/html/rfc6066#page-6.
 		// - OpenSSL does not appear to enforce this rule itself.
-		if net.ParseIP(hostname) == nil {
-			err = conn.SetTlsExtHostName(hostname)
+		if net.ParseIP(config.SNIServerName) == nil {
+			err = conn.SetTlsExtHostName(config.SNIServerName)
 			if err != nil {
 				return nil, ContextError(err)
 			}

psiphon/serverApi.go

@@ -51,13 +51,13 @@ type ServerContext struct {
 	serverHandshakeTimestamp string
 }
 
-// FrontedMeekStats holds extra stats that are only gathered for
-// FRONTED-MEEK-OSSH, FRONTED-MEEK-HTTP-OSSH.
-type FrontedMeekStats struct {
-	frontingAddress   string
-	resolvedIPAddress string
-	enabledSNI        bool
-	frontingHost      string
+// MeekStats holds extra stats that are only gathered for meek tunnels.
+type MeekStats struct {
+	DialAddress         string
+	ResolvedIPAddress   string
+	SNIServerName       string
+	HostHeader          string
+	TransformedHostName bool
 }
 
 // nextTunnelNumber is a monotonically increasing number assigned to each
@@ -623,20 +623,29 @@ func makeBaseRequestUrl(tunnel *Tunnel, port, sessionId string) string {
 		requestUrl.WriteString("&device_region=")
 		requestUrl.WriteString(tunnel.config.DeviceRegion)
 	}
-
-	if tunnel.frontedMeekStats != nil {
-		requestUrl.WriteString("&fronting_address=")
-		requestUrl.WriteString(tunnel.frontedMeekStats.frontingAddress)
-		requestUrl.WriteString("&fronting_resolved_ip_address=")
-		requestUrl.WriteString(tunnel.frontedMeekStats.resolvedIPAddress)
-		requestUrl.WriteString("&fronting_enabled_sni=")
-		if tunnel.frontedMeekStats.enabledSNI {
+	if tunnel.meekStats != nil {
+		if tunnel.meekStats.DialAddress != "" {
+			requestUrl.WriteString("&meek_dial_address=")
+			requestUrl.WriteString(tunnel.meekStats.DialAddress)
+		}
+		if tunnel.meekStats.ResolvedIPAddress != "" {
+			requestUrl.WriteString("&meek_resolved_ip_address=")
+			requestUrl.WriteString(tunnel.meekStats.ResolvedIPAddress)
+		}
+		if tunnel.meekStats.SNIServerName != "" {
+			requestUrl.WriteString("&meek_sni_server_name=")
+			requestUrl.WriteString(tunnel.meekStats.SNIServerName)
+		}
+		if tunnel.meekStats.HostHeader != "" {
+			requestUrl.WriteString("&meek_host_header=")
+			requestUrl.WriteString(tunnel.meekStats.HostHeader)
+		}
+		requestUrl.WriteString("&meek_transformed_host_name=")
+		if tunnel.meekStats.TransformedHostName {
 			requestUrl.WriteString("1")
 		} else {
 			requestUrl.WriteString("0")
 		}
-		requestUrl.WriteString("&fronting_host=")
-		requestUrl.WriteString(tunnel.frontedMeekStats.frontingHost)
 	}
 
 	if tunnel.serverEntry.Region != "" {

psiphon/tlsDialer.go

@@ -91,12 +91,14 @@ type CustomTLSConfig struct {
 	// connection dial and TLS handshake.
 	Timeout time.Duration
 
-	// FrontingAddr overrides the "addr" input to Dial when specified
-	FrontingAddr string
+	// DialAddr overrides the "addr" input to Dial when specified
+	DialAddr string
 
-	// SendServerName specifies whether to use SNI
-	// (tlsdialer functionality)
-	SendServerName bool
+	// SNIServerName specifies the value to set in the SNI
+	// server_name field. When blank, SNI is omitted. Note that
+	// underlying TLS code also automatically omits SNI when
+	// the server_name is an IP address.
+	SNIServerName string
 
 	// SkipVerify completely disables server certificate verification.
 	SkipVerify bool
@@ -151,8 +153,8 @@ func CustomTLSDial(network, addr string, config *CustomTLSConfig) (net.Conn, err
 	}
 
 	dialAddr := addr
-	if config.FrontingAddr != "" {
-		dialAddr = config.FrontingAddr
+	if config.DialAddr != "" {
+		dialAddr = config.DialAddr
 	}
 
 	rawConn, err := config.Dial(network, dialAddr)
@@ -172,12 +174,12 @@ func CustomTLSDial(network, addr string, config *CustomTLSConfig) (net.Conn, err
 		tlsConfig.InsecureSkipVerify = true
 	}
 
-	if config.SendServerName && config.VerifyLegacyCertificate == nil {
+	if config.SNIServerName != "" && config.VerifyLegacyCertificate == nil {
 		// Set the ServerName and rely on the usual logic in
 		// tls.Conn.Handshake() to do its verification.
 		// Note: Go TLS will automatically omit this ServerName when it's an IP address
 		if net.ParseIP(hostname) == nil {
-			tlsConfig.ServerName = hostname
+			tlsConfig.ServerName = config.SNIServerName
 		}
 	} else {
 		// No SNI.

psiphon/tunnel.go

@@ -78,7 +78,7 @@ type Tunnel struct {
 	signalPortForwardFailure chan struct{}
 	totalPortForwardFailures int
 	startTime                time.Time
-	frontedMeekStats         *FrontedMeekStats
+	meekStats                *MeekStats
 }
 
 // EstablishTunnel first makes a network transport connection to the
@@ -105,7 +105,7 @@ func EstablishTunnel(
 	}
 
 	// Build transport layers and establish SSH connection
-	conn, sshClient, frontedMeekStats, err := dialSsh(
+	conn, sshClient, meekStats, err := dialSsh(
 		config, pendingConns, serverEntry, selectedProtocol, sessionId)
 	if err != nil {
 		return nil, ContextError(err)
@@ -134,7 +134,7 @@ func EstablishTunnel(
 		// A buffer allows at least one signal to be sent even when the receiver is
 		// not listening. Senders should not block.
 		signalPortForwardFailure: make(chan struct{}, 1),
-		frontedMeekStats:         frontedMeekStats,
+		meekStats:                meekStats,
 	}
 
 	// Create a new Psiphon API server context for this tunnel. This includes
@@ -334,91 +334,160 @@ func selectProtocol(config *Config, serverEntry *ServerEntry) (selectedProtocol
 	return selectedProtocol, nil
 }
 
-// dialSsh is a helper that builds the transport layers and establishes the SSH connection.
-// When  FRONTED-MEEK-OSSH is selected, additional FrontedMeekStats are recorded and returned.
-func dialSsh(
+// selectFrontingParameters is a helper which selects/generates meek fronting
+// parameters where the server entry provides multiple options or patterns.
+func selectFrontingParameters(
+	serverEntry *ServerEntry) (frontingAddress, frontingHost string, err error) {
+
+	if len(serverEntry.MeekFrontingAddressesRegex) > 0 {
+
+		// Generate a front address based on the regex.
+
+		frontingAddress, err = regen.Generate(serverEntry.MeekFrontingAddressesRegex)
+		if err != nil {
+			return "", "", ContextError(err)
+		}
+	} else {
+
+		// Randomly select, for this connection attempt, one front address for
+		// fronting-capable servers.
+
+		if len(serverEntry.MeekFrontingAddresses) == 0 {
+			return "", "", ContextError(errors.New("MeekFrontingAddresses is empty"))
+		}
+		index, err := MakeSecureRandomInt(len(serverEntry.MeekFrontingAddresses))
+		if err != nil {
+			return "", "", ContextError(err)
+		}
+		frontingAddress = serverEntry.MeekFrontingAddresses[index]
+	}
+
+	if len(serverEntry.MeekFrontingHosts) > 0 {
+		index, err := MakeSecureRandomInt(len(serverEntry.MeekFrontingHosts))
+		if err != nil {
+			return "", "", ContextError(err)
+		}
+		frontingHost = serverEntry.MeekFrontingHosts[index]
+	} else {
+		// Backwards compatibility case
+		frontingHost = serverEntry.MeekFrontingHost
+	}
+
+	return
+}
+
+// initMeekConfig is a helper that creates a MeekConfig suitable for the
+// selected meek tunnel protocol.
+func initMeekConfig(
 	config *Config,
-	pendingConns *Conns,
 	serverEntry *ServerEntry,
 	selectedProtocol,
-	sessionId string) (
-	conn net.Conn, sshClient *ssh.Client, frontedMeekStats *FrontedMeekStats, err error) {
+	sessionId string) (*MeekConfig, error) {
+
+	// The meek protocol always uses OSSH
+	psiphonServerAddress := fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.SshObfuscatedPort)
+
+	var dialAddress string
+	useHTTPS := false
+	var SNIServerName, hostHeader string
+	transformedHostName := false
 
-	// The meek protocols tunnel obfuscated SSH. Obfuscated SSH is layered on top of SSH.
-	// So depending on which protocol is used, multiple layers are initialized.
-	port := 0
-	useMeek := false
-	useMeekHTTPS := false
-	useMeekSNI := false
-	useFronting := false
-	useObfuscatedSsh := false
 	switch selectedProtocol {
 	case TUNNEL_PROTOCOL_FRONTED_MEEK:
-		useMeek = true
-		useMeekHTTPS = true
-		useMeekSNI = !serverEntry.MeekFrontingDisableSNI
-		useFronting = true
-		useObfuscatedSsh = true
+		frontingAddress, frontingHost, err := selectFrontingParameters(serverEntry)
+		if err != nil {
+			return nil, ContextError(err)
+		}
+		dialAddress = fmt.Sprintf("%s:443", frontingAddress)
+		useHTTPS = true
+		if !serverEntry.MeekFrontingDisableSNI {
+			SNIServerName, transformedHostName =
+				config.HostNameTransformer.TransformHostName(frontingAddress)
+		}
+		hostHeader = frontingHost
+
 	case TUNNEL_PROTOCOL_FRONTED_MEEK_HTTP:
-		useMeek = true
-		useMeekHTTPS = false
-		useMeekSNI = false
-		useFronting = true
-		useObfuscatedSsh = true
+		frontingAddress, frontingHost, err := selectFrontingParameters(serverEntry)
+		if err != nil {
+			return nil, ContextError(err)
+		}
+		dialAddress = fmt.Sprintf("%s:80", frontingAddress)
+		hostHeader = frontingHost
+
 	case TUNNEL_PROTOCOL_UNFRONTED_MEEK:
-		useMeek = true
-		useMeekHTTPS = false
-		useMeekSNI = false
-		useObfuscatedSsh = true
-		port = serverEntry.SshObfuscatedPort
+		dialAddress = fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.MeekServerPort)
+		hostname := serverEntry.IpAddress
+		hostname, transformedHostName = config.HostNameTransformer.TransformHostName(hostname)
+		if serverEntry.MeekServerPort == 80 {
+			hostHeader = hostname
+		} else {
+			hostHeader = fmt.Sprintf("%s:%d", hostname, serverEntry.MeekServerPort)
+		}
+
 	case TUNNEL_PROTOCOL_UNFRONTED_MEEK_HTTPS:
-		useMeek = true
-		useMeekHTTPS = true
-		useMeekSNI = false
-		useObfuscatedSsh = true
-		port = serverEntry.SshObfuscatedPort
-	case TUNNEL_PROTOCOL_OBFUSCATED_SSH:
-		useObfuscatedSsh = true
-		port = serverEntry.SshObfuscatedPort
-	case TUNNEL_PROTOCOL_SSH:
-		port = serverEntry.SshPort
+		dialAddress = fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.MeekServerPort)
+		useHTTPS = true
+		SNIServerName, transformedHostName =
+			config.HostNameTransformer.TransformHostName(serverEntry.IpAddress)
+		if serverEntry.MeekServerPort == 443 {
+			hostHeader = serverEntry.IpAddress
+		} else {
+			hostHeader = fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.MeekServerPort)
+		}
+
+	default:
+		return nil, ContextError(errors.New("unexpected selectedProtocol"))
 	}
 
-	frontingAddress := ""
-	frontingHost := ""
-	if useFronting {
-		if len(serverEntry.MeekFrontingAddressesRegex) > 0 {
+	// The unnderlying TLS will automatically disable SNI for IP address server name
+	// values; we have this explicit check here so we record the correct value for stats.
+	if net.ParseIP(SNIServerName) != nil {
+		SNIServerName = ""
+	}
 
-			// Generate a front address based on the regex.
+	return &MeekConfig{
+		DialAddress:                   dialAddress,
+		UseHTTPS:                      useHTTPS,
+		SNIServerName:                 SNIServerName,
+		HostHeader:                    hostHeader,
+		TransformedHostName:           transformedHostName,
+		PsiphonServerAddress:          psiphonServerAddress,
+		SessionID:                     sessionId,
+		MeekCookieEncryptionPublicKey: serverEntry.MeekCookieEncryptionPublicKey,
+		MeekObfuscatedKey:             serverEntry.MeekObfuscatedKey,
+	}, nil
+}
 
-			frontingAddress, err = regen.Generate(serverEntry.MeekFrontingAddressesRegex)
-			if err != nil {
-				return nil, nil, nil, ContextError(err)
-			}
-		} else {
+// dialSsh is a helper that builds the transport layers and establishes the SSH connection.
+// When a meek protocols is selected, additional MeekStats are recorded and returned.
+func dialSsh(
+	config *Config,
+	pendingConns *Conns,
+	serverEntry *ServerEntry,
+	selectedProtocol,
+	sessionId string) (
+	conn net.Conn, sshClient *ssh.Client, meekStats *MeekStats, err error) {
 
-			// Randomly select, for this connection attempt, one front address for
-			// fronting-capable servers.
+	// The meek protocols tunnel obfuscated SSH. Obfuscated SSH is layered on top of SSH.
+	// So depending on which protocol is used, multiple layers are initialized.
 
-			if len(serverEntry.MeekFrontingAddresses) == 0 {
-				return nil, nil, nil, ContextError(errors.New("MeekFrontingAddresses is empty"))
-			}
-			index, err := MakeSecureRandomInt(len(serverEntry.MeekFrontingAddresses))
-			if err != nil {
-				return nil, nil, nil, ContextError(err)
-			}
-			frontingAddress = serverEntry.MeekFrontingAddresses[index]
-		}
+	useObfuscatedSsh := false
+	var directTCPDialAddress string
+	var meekConfig *MeekConfig
 
-		if len(serverEntry.MeekFrontingHosts) > 0 {
-			index, err := MakeSecureRandomInt(len(serverEntry.MeekFrontingHosts))
-			if err != nil {
-				return nil, nil, nil, ContextError(err)
-			}
-			frontingHost = serverEntry.MeekFrontingHosts[index]
-		} else {
-			// Backwards compatibility case
-			frontingHost = serverEntry.MeekFrontingHost
+	switch selectedProtocol {
+	case TUNNEL_PROTOCOL_OBFUSCATED_SSH:
+		useObfuscatedSsh = true
+		directTCPDialAddress = fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.SshObfuscatedPort)
+
+	case TUNNEL_PROTOCOL_SSH:
+		directTCPDialAddress = fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.SshPort)
+
+	default:
+		useObfuscatedSsh = true
+		meekConfig, err = initMeekConfig(config, serverEntry, selectedProtocol, sessionId)
+		if err != nil {
+			return nil, nil, nil, ContextError(err)
 		}
 	}
 
@@ -426,7 +495,8 @@ func dialSsh(
 		serverEntry.IpAddress,
 		serverEntry.Region,
 		selectedProtocol,
-		frontingAddress)
+		directTCPDialAddress,
+		meekConfig)
 
 	// Use an asynchronous callback to record the resolved IP address when
 	// dialing a domain name. Note that DialMeek doesn't immediately
@@ -452,14 +522,13 @@ func dialSsh(
 		DeviceRegion:                  config.DeviceRegion,
 		ResolvedIPCallback:            setResolvedIPAddress,
 	}
-	if useMeek {
-		conn, err = DialMeek(
-			serverEntry, sessionId, useMeekHTTPS, useMeekSNI, frontingAddress, frontingHost, dialConfig)
+	if meekConfig != nil {
+		conn, err = DialMeek(meekConfig, dialConfig)
 		if err != nil {
 			return nil, nil, nil, ContextError(err)
 		}
 	} else {
-		conn, err = DialTCP(fmt.Sprintf("%s:%d", serverEntry.IpAddress, port), dialConfig)
+		conn, err = DialTCP(directTCPDialAddress, dialConfig)
 		if err != nil {
 			return nil, nil, nil, ContextError(err)
 		}
@@ -549,20 +618,19 @@ func dialSsh(
 		return nil, nil, nil, ContextError(result.err)
 	}
 
-	if selectedProtocol == TUNNEL_PROTOCOL_FRONTED_MEEK ||
-		selectedProtocol == TUNNEL_PROTOCOL_FRONTED_MEEK_HTTP {
-
-		frontedMeekStats = &FrontedMeekStats{
-			frontingAddress:   frontingAddress,
-			resolvedIPAddress: resolvedIPAddress.Load().(string),
-			enabledSNI:        useMeekSNI,
-			frontingHost:      frontingHost,
+	if meekConfig != nil {
+		meekStats = &MeekStats{
+			DialAddress:         meekConfig.DialAddress,
+			ResolvedIPAddress:   resolvedIPAddress.Load().(string),
+			SNIServerName:       meekConfig.SNIServerName,
+			HostHeader:          meekConfig.HostHeader,
+			TransformedHostName: meekConfig.TransformedHostName,
 		}
 
-		NoticeFrontedMeekStats(serverEntry.IpAddress, frontedMeekStats)
+		NoticeConnectedMeekStats(serverEntry.IpAddress, meekStats)
 	}
 
-	return conn, result.sshClient, frontedMeekStats, nil
+	return conn, result.sshClient, meekStats, nil
 }
 
 // operateTunnel monitors the health of the tunnel and performs

psiphon/upgradeDownload.go

@@ -26,32 +26,121 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"strconv"
 )
 
-// DownloadUpgrade performs a tunneled, resumable download of client upgrade files.
+// DownloadUpgrade performs a resumable download of client upgrade files.
+//
 // While downloading/resuming, a temporary file is used. Once the download is complete,
 // a notice is issued and the upgrade is available at the destination specified in
 // config.UpgradeDownloadFilename.
-// NOTE: this code does not check that any existing file at config.UpgradeDownloadFilename
-// is actually the version specified in clientUpgradeVersion.
-func DownloadUpgrade(config *Config, clientUpgradeVersion string, tunnel *Tunnel) error {
+//
+// The upgrade download may be either tunneled or untunneled. As the untunneled case may
+// happen with no handshake request response, the downloader cannot rely on having the
+// upgrade_client_version output from handshake and instead this logic performs a
+// comparison between the config.ClientVersion and the client version recorded in the
+// remote entity's UpgradeDownloadClientVersionHeader. A HEAD request is made to check the
+// version before proceeding with a full download.
+//
+// NOTE: This code does not check that any existing file at config.UpgradeDownloadFilename
+// is actually the version specified in handshakeVersion.
+//
+// TODO: This logic requires the outer client to *omit* config.UpgradeDownloadFilename
+// when there's already a downloaded upgrade pending. Because the outer client currently
+// handles the authenticated package phase, and because the outer client deletes the
+// intermediate files (including config.UpgradeDownloadFilename), if the outer client
+// does not omit config.UpgradeDownloadFilename then the new version will be downloaded
+// repeatedly. Implement a new scheme where tunnel core does the authenticated package phase
+// and tracks the the output by version number so that (a) tunnel core knows when it's not
+// necessary to re-download; (b) newer upgrades will be downloaded even when an older
+// upgrade is still pending install by the outer client.
+func DownloadUpgrade(
+	config *Config,
+	handshakeVersion string,
+	tunnel *Tunnel,
+	untunneledDialConfig *DialConfig) error {
 
 	// Check if complete file already downloaded
+
 	if _, err := os.Stat(config.UpgradeDownloadFilename); err == nil {
 		NoticeClientUpgradeDownloaded(config.UpgradeDownloadFilename)
 		return nil
 	}
 
-	httpClient, err := MakeTunneledHttpClient(config, tunnel, DOWNLOAD_UPGRADE_TIMEOUT)
-	if err != nil {
-		return ContextError(err)
+	requestUrl := config.UpgradeDownloadUrl
+	var httpClient *http.Client
+	var err error
+
+	// Select tunneled or untunneled configuration
+
+	if tunnel != nil {
+		httpClient, err = MakeTunneledHttpClient(config, tunnel, DOWNLOAD_UPGRADE_TIMEOUT)
+		if err != nil {
+			return ContextError(err)
+		}
+	} else {
+		httpClient, requestUrl, err = MakeUntunneledHttpsClient(
+			untunneledDialConfig, nil, requestUrl, DOWNLOAD_UPGRADE_TIMEOUT)
+		if err != nil {
+			return ContextError(err)
+		}
 	}
 
+	// If no handshake version is supplied, make an initial HEAD request
+	// to get the current version from the version header.
+
+	availableClientVersion := handshakeVersion
+	if availableClientVersion == "" {
+		request, err := http.NewRequest("HEAD", requestUrl, nil)
+		if err != nil {
+			return ContextError(err)
+		}
+		response, err := httpClient.Do(request)
+		if err == nil && response.StatusCode != http.StatusOK {
+			response.Body.Close()
+			err = fmt.Errorf("unexpected response status code: %d", response.StatusCode)
+		}
+		if err != nil {
+			return ContextError(err)
+		}
+		defer response.Body.Close()
+
+		currentClientVersion, err := strconv.Atoi(config.ClientVersion)
+		if err != nil {
+			return ContextError(err)
+		}
+
+		// Note: if the header is missing, Header.Get returns "" and then
+		// strconv.Atoi returns a parse error.
+		availableClientVersion := response.Header.Get(config.UpgradeDownloadClientVersionHeader)
+		checkAvailableClientVersion, err := strconv.Atoi(availableClientVersion)
+		if err != nil {
+			// If the header is missing or malformed, we can't determine the available
+			// version number. This is unexpected; but if it happens, it's likely due
+			// to a server-side configuration issue. In this one case, we don't
+			// return an error so that we don't go into a rapid retry loop making
+			// ineffective HEAD requests (the client may still signal an upgrade
+			// download later in the session).
+			NoticeAlert(
+				"failed to download upgrade: invalid %s header value %s: %s",
+				config.UpgradeDownloadClientVersionHeader, availableClientVersion, err)
+			return nil
+		}
+
+		if currentClientVersion >= checkAvailableClientVersion {
+			NoticeInfo(
+				"skipping download of available client version %d",
+				checkAvailableClientVersion)
+		}
+	}
+
+	// Proceed with full download
+
 	partialFilename := fmt.Sprintf(
-		"%s.%s.part", config.UpgradeDownloadFilename, clientUpgradeVersion)
+		"%s.%s.part", config.UpgradeDownloadFilename, availableClientVersion)
 
 	partialETagFilename := fmt.Sprintf(
-		"%s.%s.part.etag", config.UpgradeDownloadFilename, clientUpgradeVersion)
+		"%s.%s.part.etag", config.UpgradeDownloadFilename, availableClientVersion)
 
 	file, err := os.OpenFile(partialFilename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
 	if err != nil {
@@ -85,7 +174,7 @@ func DownloadUpgrade(config *Config, clientUpgradeVersion string, tunnel *Tunnel
 
 	}
 
-	request, err := http.NewRequest("GET", config.UpgradeDownloadUrl, nil)
+	request, err := http.NewRequest("GET", requestUrl, nil)
 	if err != nil {
 		return ContextError(err)
 	}
@@ -131,12 +220,13 @@ func DownloadUpgrade(config *Config, clientUpgradeVersion string, tunnel *Tunnel
 	// A partial download occurs when this copy is interrupted. The io.Copy
 	// will fail, leaving a partial download in place (.part and .part.etag).
 	n, err := io.Copy(NewSyncFileWriter(file), response.Body)
+
+	NoticeClientUpgradeDownloadedBytes(n)
+
 	if err != nil {
 		return ContextError(err)
 	}
 
-	NoticeInfo("client upgrade downloaded bytes: %d", n)
-
 	// Ensure the file is flushed to disk. The deferred close
 	// will be a noop when this succeeds.
 	err = file.Close()