Merge remote-tracking branch 'upstream/master'

AndroidLibrary/psi/psi.go

@@ -26,7 +26,6 @@ package psi
 
 import (
 	"fmt"
-	"log"
 	"sync"
 
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon"
@@ -43,7 +42,10 @@ var controller *psiphon.Controller
 var shutdownBroadcast chan struct{}
 var controllerWaitGroup *sync.WaitGroup
 
-func Start(configJson, embeddedServerEntryList string, provider PsiphonProvider) error {
+func Start(
+	configJson, embeddedServerEntryList string,
+	provider PsiphonProvider,
+	useDeviceBinder bool) error {
 
 	if controller != nil {
 		return fmt.Errorf("already started")
@@ -54,8 +56,11 @@ func Start(configJson, embeddedServerEntryList string, provider PsiphonProvider)
 		return fmt.Errorf("error loading configuration file: %s", err)
 	}
 	config.NetworkConnectivityChecker = provider
-	config.DeviceBinder = provider
-	config.DnsServerGetter = provider
+
+	if useDeviceBinder {
+		config.DeviceBinder = provider
+		config.DnsServerGetter = provider
+	}
 
 	psiphon.SetNoticeOutput(psiphon.NewNoticeReceiver(
 		func(notice []byte) {
@@ -69,13 +74,40 @@ func Start(configJson, embeddedServerEntryList string, provider PsiphonProvider)
 		return fmt.Errorf("error initializing datastore: %s", err)
 	}
 
-	serverEntries, err := psiphon.DecodeAndValidateServerEntryList(embeddedServerEntryList)
-	if err != nil {
-		log.Fatalf("error decoding embedded server entry list: %s", err)
-	}
-	err = psiphon.StoreServerEntries(serverEntries, false)
-	if err != nil {
-		log.Fatalf("error storing embedded server entry list: %s", err)
+	// If specified, the embedded server list is loaded and stored. When there
+	// are no server candidates at all, we wait for this import to complete
+	// before starting the Psiphon controller. Otherwise, we import while
+	// concurrently starting the controller to minimize delay before attempting
+	// to connect to existing candidate servers.
+	// If the import fails, an error notice is emitted, but the controller is
+	// still started: either existing candidate servers may suffice, or the
+	// remote server list fetch may obtain candidate servers.
+	// TODO: duplicates logic in psiphonClient.go -- refactor?
+	if embeddedServerEntryList != "" {
+		embeddedServerListWaitGroup := new(sync.WaitGroup)
+		embeddedServerListWaitGroup.Add(1)
+		go func() {
+			defer embeddedServerListWaitGroup.Done()
+			// TODO: stream embedded server list data?
+			serverEntries, err := psiphon.DecodeAndValidateServerEntryList(embeddedServerEntryList)
+			if err != nil {
+				psiphon.NoticeError("error decoding embedded server entry list file: %s", err)
+				return
+			}
+			// Since embedded server list entries may become stale, they will not
+			// overwrite existing stored entries for the same server.
+			err = psiphon.StoreServerEntries(serverEntries, false)
+			if err != nil {
+				psiphon.NoticeError("error storing embedded server entry list data: %s", err)
+				return
+			}
+		}()
+
+		if psiphon.CountServerEntries(config.EgressRegion, config.TunnelProtocol) == 0 {
+			embeddedServerListWaitGroup.Wait()
+		} else {
+			defer embeddedServerListWaitGroup.Wait()
+		}
 	}
 
 	controller, err = psiphon.NewController(config)

README.md

@@ -41,7 +41,7 @@ Setup
         "ConnectionWorkerPoolSize" : 10,
         "TunnelPoolSize" : 1,
         "PortForwardFailureThreshold" : 10,
-        "UpstreamHttpProxyAddress" : ""
+        "UpstreamProxyUrl" : ""
     }
     ```
 <!--END-SAMPLE-CONFIG-->

SampleApps/Psibot/app/src/main/java/ca/psiphon/psibot/Service.java

@@ -110,10 +110,10 @@ public class Service extends VpnService
                     preferences.getString(
                             getString(R.string.preferenceTunnelProtocol),
                             getString(R.string.preferenceTunnelProtocolDefaultValue)));
-            config.put("UpstreamHttpProxyAddress",
+            config.put("UpstreamProxyUrl",
                     preferences.getString(
-                            getString(R.string.preferenceUpstreamHttpProxyAddress),
-                            getString(R.string.preferenceUpstreamHttpProxyAddressDefaultValue)));
+                            getString(R.string.preferenceUpstreamProxyUrl),
+                            getString(R.string.preferenceUpstreamProxyUrlDefaultValue)));
             config.put("LocalHttpProxyPort",
                     Integer.parseInt(
                             preferences.getString(

SampleApps/Psibot/app/src/main/res/raw/psiphon_config_stub

@@ -13,5 +13,5 @@
     "ConnectionWorkerPoolSize" : 10,
     "TunnelPoolSize" : 1,
     "PortForwardFailureThreshold" : 10,
-    "UpstreamHttpProxyAddress" : ""
+    "UpstreamProxyUrl" : ""
 }

SampleApps/Psibot/app/src/main/res/values/strings.xml

@@ -27,7 +27,7 @@
         <item>Fronted Meek</item>
     </string-array>
 
-    <string name="preference_upstream_http_proxy_address">Upstream HTTP Proxy Address</string>
+    <string name="preference_upstream_proxy_url">Upstream Proxy URL</string>
     <string name="preference_local_http_proxy_port">Local HTTP Proxy Port</string>
     <string name="preference_local_socks_proxy_port">Local SOCKS Proxy Port</string>
     <string name="preference_connection_worker_pool_size">Connection Workers</string>

SampleApps/Psibot/app/src/main/res/values/symbols.xml

@@ -22,8 +22,8 @@
     </string-array>
     <string name="preferenceTunnelProtocolDefaultValue"></string>
 
-    <string name="preferenceUpstreamHttpProxyAddress">preferenceUpstreamHttpProxyAddress</string>
-    <string name="preferenceUpstreamHttpProxyAddressDefaultValue"></string>
+    <string name="preferenceUpstreamProxyUrl">preferenceUpstreamProxyUrl</string>
+    <string name="preferenceUpstreamProxyUrlDefaultValue"></string>
 
     <string name="preferenceLocalHttpProxyPort">preferenceLocalHttpProxyPort</string>
     <string name="preferenceLocalHttpProxyPortDefaultValue">0</string>

SampleApps/Psibot/app/src/main/res/xml/preferences.xml

@@ -18,10 +18,10 @@
         android:defaultValue="@string/preferenceTunnelProtocolDefaultValue" />
 
     <EditTextPreference
-        android:key="@string/preferenceUpstreamHttpProxyAddress"
-        android:title="@string/preference_upstream_http_proxy_address"
-        android:dialogTitle="@string/preference_upstream_http_proxy_address"
-        android:defaultValue="@string/preferenceUpstreamHttpProxyAddressDefaultValue" />
+        android:key="@string/preferenceUpstreamProxyUrl"
+        android:title="@string/preference_upstream_proxy_url"
+        android:dialogTitle="@string/preference_upstream_proxy_url"
+        android:defaultValue="@string/preferenceUpstreamProxyUrlDefaultValue" />
 
     <EditTextPreference
         android:numeric="integer"

psiphon/TCPConn.go

@@ -24,13 +24,18 @@ import (
 	"net"
 	"sync"
 	"time"
+
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/upstreamproxy"
 )
 
 // TCPConn is a customized TCP connection that:
 // - can be interrupted while connecting;
+// - implements a connect timeout;
 // - implements idle read/write timeouts;
+// - uses an upstream proxy when specified, and includes
+//   upstream proxy dialing in the connect timeout;
 // - can be bound to a specific system device (for Android VpnService
-//   routing compatibility, for example).
+//   routing compatibility, for example);
 // - implements the psiphon.Conn interface
 type TCPConn struct {
 	net.Conn
@@ -44,15 +49,58 @@ type TCPConn struct {
 
 // NewTCPDialer creates a TCPDialer.
 func NewTCPDialer(config *DialConfig) Dialer {
-	return func(network, addr string) (net.Conn, error) {
+
+	dialer := func(network, addr string) (net.Conn, error) {
 		if network != "tcp" {
 			return nil, errors.New("unsupported network type in NewTCPDialer")
 		}
 		return DialTCP(addr, config)
 	}
+
+	if config.UpstreamProxyUrl != "" {
+
+		upstreamDialer := upstreamproxy.NewProxyDialFunc(
+			&upstreamproxy.UpstreamProxyConfig{
+				ForwardDialFunc: dialer,
+				ProxyURIString:  config.UpstreamProxyUrl,
+			})
+
+		dialer = func(network, addr string) (net.Conn, error) {
+
+			// The entire upstream dial is wrapped in an explicit timeout. This
+			// may include network connection read and writes when proxy auth negotation
+			// is performed.
+
+			type upstreamDialResult struct {
+				conn net.Conn
+				err  error
+			}
+			resultChannel := make(chan *upstreamDialResult, 2)
+			time.AfterFunc(config.ConnectTimeout, func() {
+				// TODO: we could "interrupt" the underlying TCPConn at this point, as
+				// it's being abandoned. But we don't have a reference to it. It's left
+				// to the outer DialConfig.PendingConns to track and clean up that TCPConn.
+				resultChannel <- &upstreamDialResult{nil, errors.New("upstreamproxy dial timeout")}
+			})
+			go func() {
+				conn, err := upstreamDialer(network, addr)
+				resultChannel <- &upstreamDialResult{conn, err}
+			}()
+			result := <-resultChannel
+
+			if _, ok := result.err.(upstreamproxy.Error); ok {
+				NoticeUpstreamProxyError(result.err)
+			}
+
+			return result.conn, result.err
+		}
+	}
+
+	return dialer
 }
 
-// TCPConn creates a new, connected TCPConn.
+// TCPConn creates a new, connected TCPConn. It uses an upstream proxy
+// when specified.
 func DialTCP(addr string, config *DialConfig) (conn *TCPConn, err error) {
 	conn, err = interruptibleTCPDial(addr, config)
 	if err != nil {

psiphon/TCPConn_unix.go

@@ -68,24 +68,9 @@ func interruptibleTCPDial(addr string, config *DialConfig) (conn *TCPConn, err e
 		}
 	}
 
-	// When using an upstream HTTP proxy, first connect to the proxy,
-	// then use HTTP CONNECT to connect to the original destination.
-	dialAddr := addr
-	if config.UpstreamHttpProxyAddress != "" {
-		dialAddr = config.UpstreamHttpProxyAddress
-
-		// Report connection errors in a notice, as user may have input
-		// invalid proxy address or credential
-		defer func() {
-			if err != nil {
-				NoticeUpstreamProxyError(err)
-			}
-		}()
-	}
-
 	// Get the remote IP and port, resolving a domain name if necessary
 	// TODO: domain name resolution isn't interruptible
-	host, strPort, err := net.SplitHostPort(dialAddr)
+	host, strPort, err := net.SplitHostPort(addr)
 	if err != nil {
 		return nil, ContextError(err)
 	}
@@ -174,15 +159,6 @@ func interruptibleTCPDial(addr string, config *DialConfig) (conn *TCPConn, err e
 
 	conn.mutex.Unlock()
 
-	// Going through upstream HTTP proxy
-	if config.UpstreamHttpProxyAddress != "" {
-		// This call can be interrupted by closing the pending conn
-		err = HttpProxyConnect(conn, addr)
-		if err != nil {
-			return nil, ContextError(err)
-		}
-	}
-
 	return conn, nil
 }
 

psiphon/TCPConn_windows.go

@@ -63,28 +63,7 @@ func interruptibleTCPDial(addr string, config *DialConfig) (conn *TCPConn, err e
 	// Call the blocking Dial in a goroutine
 	results := conn.interruptible.results
 	go func() {
-
-		// When using an upstream HTTP proxy, first connect to the proxy,
-		// then use HTTP CONNECT to connect to the original destination.
-		dialAddr := addr
-		if config.UpstreamHttpProxyAddress != "" {
-			dialAddr = config.UpstreamHttpProxyAddress
-		}
-
-		netConn, err := net.DialTimeout("tcp", dialAddr, config.ConnectTimeout)
-
-		if config.UpstreamHttpProxyAddress != "" {
-			if err == nil {
-				err = HttpProxyConnect(netConn, addr)
-			}
-			if err != nil {
-				NoticeUpstreamProxyError(err)
-			}
-		}
-		if err != nil {
-			netConn = nil
-		}
-
+		netConn, err := net.DialTimeout("tcp", addr, config.ConnectTimeout)
 		results <- &interruptibleDialResult{netConn, err}
 	}()
 

psiphon/config.go

@@ -40,6 +40,7 @@ const (
 	TUNNEL_SSH_KEEP_ALIVE_PAYLOAD_MAX_BYTES      = 256
 	TUNNEL_SSH_KEEP_ALIVE_PERIOD_MIN             = 60 * time.Second
 	TUNNEL_SSH_KEEP_ALIVE_PERIOD_MAX             = 120 * time.Second
+	TUNNEL_SSH_KEEP_ALIVE_TIMEOUT                = 10 * time.Second
 	ESTABLISH_TUNNEL_TIMEOUT_SECONDS             = 300
 	ESTABLISH_TUNNEL_WORK_TIME_SECONDS           = 60 * time.Second
 	ESTABLISH_TUNNEL_PAUSE_PERIOD                = 5 * time.Second
@@ -56,7 +57,9 @@ const (
 	PSIPHON_API_STATUS_REQUEST_PADDING_MAX_BYTES = 256
 	PSIPHON_API_CONNECTED_REQUEST_PERIOD         = 24 * time.Hour
 	PSIPHON_API_CONNECTED_REQUEST_RETRY_PERIOD   = 5 * time.Second
-	FETCH_ROUTES_TIMEOUT                         = 10 * time.Second
+	FETCH_ROUTES_TIMEOUT                         = 1 * time.Minute
+	DOWNLOAD_UPGRADE_TIMEOUT                     = 15 * time.Minute
+	DOWNLOAD_UPGRADE_RETRY_PAUSE_PERIOD          = 5 * time.Second
 )
 
 // To distinguish omitted timeout params from explicit 0 value timeout
@@ -82,7 +85,7 @@ type Config struct {
 	ConnectionWorkerPoolSize            int
 	TunnelPoolSize                      int
 	PortForwardFailureThreshold         int
-	UpstreamHttpProxyAddress            string
+	UpstreamProxyUrl                    string
 	NetworkConnectivityChecker          NetworkConnectivityChecker
 	DeviceBinder                        DeviceBinder
 	DnsServerGetter                     DnsServerGetter
@@ -92,6 +95,8 @@ type Config struct {
 	SplitTunnelRoutesUrlFormat          string
 	SplitTunnelRoutesSignaturePublicKey string
 	SplitTunnelDnsServer                string
+	UpgradeDownloadUrl                  string
+	UpgradeDownloadFilename             string
 }
 
 // LoadConfig parses and validates a JSON format Psiphon config JSON

psiphon/controller.go

@@ -46,6 +46,7 @@ type Controller struct {
 	tunnels                     []*Tunnel
 	nextTunnel                  int
 	startedConnectedReporter    bool
+	startedUpgradeDownloader    bool
 	isEstablishing              bool
 	establishWaitGroup          *sync.WaitGroup
 	stopEstablishingBroadcast   chan struct{}
@@ -72,10 +73,10 @@ func NewController(config *Config) (controller *Controller, err error) {
 	// used to exclude these requests and connection from VPN routing.
 	untunneledPendingConns := new(Conns)
 	untunneledDialConfig := &DialConfig{
-		UpstreamHttpProxyAddress: config.UpstreamHttpProxyAddress,
-		PendingConns:             untunneledPendingConns,
-		DeviceBinder:             config.DeviceBinder,
-		DnsServerGetter:          config.DnsServerGetter,
+		UpstreamProxyUrl: config.UpstreamProxyUrl,
+		PendingConns:     untunneledPendingConns,
+		DeviceBinder:     config.DeviceBinder,
+		DnsServerGetter:  config.DnsServerGetter,
 	}
 
 	controller = &Controller{
@@ -94,6 +95,7 @@ func NewController(config *Config) (controller *Controller, err error) {
 		tunnels:                  make([]*Tunnel, 0),
 		establishedOnce:          false,
 		startedConnectedReporter: false,
+		startedUpgradeDownloader: false,
 		isEstablishing:           false,
 		establishPendingConns:    new(Conns),
 		untunneledPendingConns:   untunneledPendingConns,
@@ -317,6 +319,66 @@ func (controller *Controller) startConnectedReporter() {
 	}
 }
 
+// upgradeDownloader makes periodic attemps to complete a client upgrade
+// download. DownloadUpgrade() is resumable, so each attempt has potential for
+// getting closer to completion, even in conditions where the download or
+// tunnel is repeatedly interrupted.
+// Once the download is complete, the downloader exits and is not run again:
+// We're assuming that the upgrade will be applied and the entire system
+// restarted before another upgrade is to be downloaded.
+func (controller *Controller) upgradeDownloader(clientUpgradeVersion string) {
+	defer controller.runWaitGroup.Done()
+
+loop:
+	for {
+		// Pick any active tunnel and make the next download attempt. No error
+		// is logged if there's no active tunnel, as that's not an unexpected condition.
+		tunnel := controller.getNextActiveTunnel()
+		if tunnel != nil {
+			err := DownloadUpgrade(controller.config, clientUpgradeVersion, tunnel)
+			if err == nil {
+				break loop
+			}
+			NoticeAlert("upgrade download failed: ", err)
+		}
+
+		timeout := time.After(DOWNLOAD_UPGRADE_RETRY_PAUSE_PERIOD)
+		select {
+		case <-timeout:
+			// Make another download attempt
+		case <-controller.shutdownBroadcast:
+			break loop
+		}
+	}
+
+	NoticeInfo("exiting upgrade downloader")
+}
+
+func (controller *Controller) startClientUpgradeDownloader(clientUpgradeVersion string) {
+	if controller.config.DisableApi {
+		return
+	}
+
+	if controller.config.UpgradeDownloadUrl == "" ||
+		controller.config.UpgradeDownloadFilename == "" {
+		// No upgrade is desired
+		return
+	}
+
+	if clientUpgradeVersion == "" {
+		// No upgrade is offered
+		return
+	}
+
+	// Start the client upgrade downloaded after the first tunnel is established.
+	// Concurrency note: only the runTunnels goroutine may access startClientUpgradeDownloader.
+	if !controller.startedUpgradeDownloader {
+		controller.startedUpgradeDownloader = true
+		controller.runWaitGroup.Add(1)
+		go controller.upgradeDownloader(clientUpgradeVersion)
+	}
+}
+
 // runTunnels is the controller tunnel management main loop. It starts and stops
 // establishing tunnels based on the target tunnel pool size and the current size
 // of the pool. Tunnels are established asynchronously using worker goroutines.
@@ -361,6 +423,7 @@ loop:
 				controller.stopEstablishing()
 			}
 			controller.startConnectedReporter()
+			controller.startClientUpgradeDownloader(establishedTunnel.session.clientUpgradeVersion)
 
 		case <-controller.shutdownBroadcast:
 			break loop

psiphon/meekConn.go

@@ -33,6 +33,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/upstreamproxy"
 	"golang.org/x/crypto/nacl/box"
 )
 
@@ -54,6 +55,7 @@ const (
 	POLL_INTERNAL_MULTIPLIER       = 1.5
 	MEEK_ROUND_TRIP_RETRY_DEADLINE = 1 * time.Second
 	MEEK_ROUND_TRIP_RETRY_DELAY    = 50 * time.Millisecond
+	MEEK_ROUND_TRIP_TIMEOUT        = 20 * time.Second
 )
 
 // MeekConn is a network connection that tunnels TCP over HTTP and supports "fronting". Meek sends
@@ -72,7 +74,7 @@ type MeekConn struct {
 	url                  *url.URL
 	cookie               *http.Cookie
 	pendingConns         *Conns
-	transport            *http.Transport
+	transport            transporter
 	mutex                sync.Mutex
 	isClosed             bool
 	closedSignal         chan struct{}
@@ -86,6 +88,14 @@ type MeekConn struct {
 	fullSendBuffer       chan *bytes.Buffer
 }
 
+// transporter is implemented by both http.Transport and upstreamproxy.ProxyAuthTransport.
+type transporter interface {
+	CancelRequest(req *http.Request)
+	CloseIdleConnections()
+	RegisterProtocol(scheme string, rt http.RoundTripper)
+	RoundTrip(req *http.Request) (resp *http.Response, err error)
+}
+
 // DialMeek returns an initialized meek connection. A meek connection is
 // an HTTP session which does not depend on an underlying socket connection (although
 // persistent HTTP connections are used for performance). This function does not
@@ -160,20 +170,20 @@ func DialMeek(
 				SkipVerify:     true,
 			})
 	} else {
-		// In this case, host is both what is dialed and what ends up in the HTTP Host header
+		// In the unfronted case, host is both what is dialed and what ends up in the HTTP Host header
 		host = fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.MeekServerPort)
 
-		if meekConfig.UpstreamHttpProxyAddress != "" {
+		if meekConfig.UpstreamProxyUrl != "" {
 			// For unfronted meek, we let the http.Transport handle proxying, as the
 			// target server hostname has to be in the HTTP request line. Also, in this
 			// case, we don't require the proxy to support CONNECT and so we can work
 			// through HTTP proxies that don't support it.
-			url, err := url.Parse(fmt.Sprintf("http://%s", meekConfig.UpstreamHttpProxyAddress))
+			url, err := url.Parse(meekConfig.UpstreamProxyUrl)
 			if err != nil {
 				return nil, ContextError(err)
 			}
 			proxyUrl = http.ProxyURL(url)
-			meekConfig.UpstreamHttpProxyAddress = ""
+			meekConfig.UpstreamProxyUrl = ""
 		}
 
 		dialer = NewTCPDialer(meekConfig)
@@ -190,10 +200,20 @@ func DialMeek(
 	if err != nil {
 		return nil, ContextError(err)
 	}
-	transport := &http.Transport{
+	httpTransport := &http.Transport{
 		Proxy: proxyUrl,
 		Dial:  dialer,
-		ResponseHeaderTimeout: TUNNEL_WRITE_TIMEOUT,
+		ResponseHeaderTimeout: MEEK_ROUND_TRIP_TIMEOUT,
+	}
+	var transport transporter
+	if proxyUrl != nil {
+		// Wrap transport with a transport that can perform HTTP proxy auth negotiation
+		transport, err = upstreamproxy.NewProxyAuthTransport(httpTransport)
+		if err != nil {
+			return nil, ContextError(err)
+		}
+	} else {
+		transport = httpTransport
 	}
 
 	// The main loop of a MeekConn is run in the relay() goroutine.
@@ -472,7 +492,7 @@ func (meek *MeekConn) readPayload(receivedPayload io.ReadCloser) (totalSize int6
 func (meek *MeekConn) roundTrip(sendPayload []byte) (receivedPayload io.ReadCloser, err error) {
 	request, err := http.NewRequest("POST", meek.url.String(), bytes.NewReader(sendPayload))
 	if err != nil {
-		return nil, err
+		return nil, ContextError(err)
 	}
 
 	if meek.frontingAddress != "" && nil == net.ParseIP(meek.frontingAddress) {

psiphon/net.go

@@ -20,12 +20,8 @@
 package psiphon
 
 import (
-	"bufio"
-	"errors"
-	"fmt"
 	"io"
 	"net"
-	"net/http"
 	"sync"
 	"time"
 
@@ -38,10 +34,16 @@ const DNS_PORT = 53
 // of a Psiphon dialer (TCPDial, MeekDial, etc.)
 type DialConfig struct {
 
-	// UpstreamHttpProxyAddress specifies an HTTP proxy to connect through
-	// (the proxy must support HTTP CONNECT). The address may be a hostname
-	// or IP address and must include a port number.
-	UpstreamHttpProxyAddress string
+	// UpstreamProxyUrl specifies a proxy to connect through.
+	// E.g., "http://proxyhost:8080"
+	//       "socks5://user:password@proxyhost:1080"
+	//       "socks4a://proxyhost:1080"
+	//       "http://NTDOMAIN\NTUser:password@proxyhost:3375"
+	//
+	// Certain tunnel protocols require HTTP CONNECT support
+	// when a HTTP proxy is specified. If CONNECT is not
+	// supported, those protocols will not connect.
+	UpstreamProxyUrl string
 
 	ConnectTimeout time.Duration
 	ReadTimeout    time.Duration
@@ -82,6 +84,13 @@ type DnsServerGetter interface {
 	GetDnsServer() string
 }
 
+// TimeoutError implements the error interface
+type TimeoutError struct{}
+
+func (TimeoutError) Error() string   { return "timed out" }
+func (TimeoutError) Timeout() bool   { return true }
+func (TimeoutError) Temporary() bool { return true }
+
 // Dialer is a custom dialer compatible with http.Transport.Dial.
 type Dialer func(string, string) (net.Conn, error)
 
@@ -166,39 +175,6 @@ func Relay(localConn, remoteConn net.Conn) {
 	copyWaitGroup.Wait()
 }
 
-// HttpProxyConnect establishes a HTTP CONNECT tunnel to addr through
-// an established network connection to an HTTP proxy. It is assumed that
-// no payload bytes have been sent through the connection to the proxy.
-func HttpProxyConnect(rawConn net.Conn, addr string) (err error) {
-	hostname, _, err := net.SplitHostPort(addr)
-	if err != nil {
-		return ContextError(err)
-	}
-
-	// TODO: use the proxy request/response code from net/http/transport.go?
-	connectRequest := fmt.Sprintf(
-		"CONNECT %s HTTP/1.1\r\nHost: %s\r\nConnection: Keep-Alive\r\n\r\n",
-		addr, hostname)
-	_, err = rawConn.Write([]byte(connectRequest))
-	if err != nil {
-		return ContextError(err)
-	}
-
-	// Adapted from dialConn in net/http/transport.go:
-	// Read response.
-	// Okay to use and discard buffered reader here, because
-	// TLS server will not speak until spoken to.
-	response, err := http.ReadResponse(bufio.NewReader(rawConn), nil)
-	if err != nil {
-		return ContextError(err)
-	}
-	if response.StatusCode != 200 {
-		return ContextError(errors.New(response.Status))
-	}
-
-	return nil
-}
-
 // WaitForNetworkConnectivity uses a NetworkConnectivityChecker to
 // periodically check for network connectivity. It returns true if
 // no NetworkConnectivityChecker is provided (waiting is disabled)

psiphon/notice.go

@@ -185,6 +185,12 @@ func NoticeUpstreamProxyError(err error) {
 	outputNotice("UpstreamProxyError", true, "message", fmt.Sprintf("%s", err))
 }
 
+// NoticeClientUpgradeDownloaded indicates that a client upgrade download
+// is complete and available at the destination specified.
+func NoticeClientUpgradeDownloaded(filename string) {
+	outputNotice("ClientUpgradeDownloaded", false, "filename", filename)
+}
+
 type noticeObject struct {
 	NoticeType string          `json:"noticeType"`
 	Data       json.RawMessage `json:"data"`

psiphon/serverApi.go

@@ -39,12 +39,13 @@ import (
 // includes the session ID (used for Psiphon API requests) and a http
 // client configured to make tunneled Psiphon API requests.
 type Session struct {
-	sessionId          string
-	baseRequestUrl     string
-	psiphonHttpsClient *http.Client
-	statsRegexps       *transferstats.Regexps
-	statsServerId      string
-	clientRegion       string
+	sessionId            string
+	baseRequestUrl       string
+	psiphonHttpsClient   *http.Client
+	statsRegexps         *transferstats.Regexps
+	statsServerId        string
+	clientRegion         string
+	clientUpgradeVersion string
 }
 
 // MakeSessionId creates a new session ID. Making the session ID is not done
@@ -246,6 +247,8 @@ func (session *Session) doHandshakeRequest() error {
 	for _, homepage := range handshakeConfig.Homepages {
 		NoticeHomepage(homepage)
 	}
+
+	session.clientUpgradeVersion = handshakeConfig.UpgradeClientVersion
 	if handshakeConfig.UpgradeClientVersion != "" {
 		NoticeClientUpgradeAvailable(handshakeConfig.UpgradeClientVersion)
 	}

psiphon/tlsDialer.go

@@ -79,12 +79,6 @@ import (
 	"time"
 )
 
-type timeoutError struct{}
-
-func (timeoutError) Error() string   { return "tls: DialWithDialer timed out" }
-func (timeoutError) Timeout() bool   { return true }
-func (timeoutError) Temporary() bool { return true }
-
 // CustomTLSConfig contains parameters to determine the behavior
 // of CustomTLSDial.
 type CustomTLSConfig struct {
@@ -139,7 +133,7 @@ func CustomTLSDial(network, addr string, config *CustomTLSConfig) (*tls.Conn, er
 	if config.Timeout != 0 {
 		errChannel = make(chan error, 2)
 		time.AfterFunc(config.Timeout, func() {
-			errChannel <- timeoutError{}
+			errChannel <- TimeoutError{}
 		})
 	}
 

psiphon/tunnel.go

@@ -384,13 +384,13 @@ func dialSsh(
 
 	// Create the base transport: meek or direct connection
 	dialConfig := &DialConfig{
-		UpstreamHttpProxyAddress: config.UpstreamHttpProxyAddress,
-		ConnectTimeout:           TUNNEL_CONNECT_TIMEOUT,
-		ReadTimeout:              TUNNEL_READ_TIMEOUT,
-		WriteTimeout:             TUNNEL_WRITE_TIMEOUT,
-		PendingConns:             pendingConns,
-		DeviceBinder:             config.DeviceBinder,
-		DnsServerGetter:          config.DnsServerGetter,
+		UpstreamProxyUrl: config.UpstreamProxyUrl,
+		ConnectTimeout:   TUNNEL_CONNECT_TIMEOUT,
+		ReadTimeout:      TUNNEL_READ_TIMEOUT,
+		WriteTimeout:     TUNNEL_WRITE_TIMEOUT,
+		PendingConns:     pendingConns,
+		DeviceBinder:     config.DeviceBinder,
+		DnsServerGetter:  config.DnsServerGetter,
 	}
 	if useMeek {
 		conn, err = DialMeek(serverEntry, sessionId, frontingAddress, dialConfig)
@@ -565,11 +565,7 @@ func (tunnel *Tunnel) operateTunnel(config *Config, tunnelOwner TunnelOwner) {
 			statsTimer.Reset(nextStatusRequestPeriod())
 
 		case <-sshKeepAliveTimer.C:
-			// Random padding to frustrate fingerprinting
-			_, _, err := tunnel.sshClient.SendRequest(
-				"keepalive@openssh.com", true,
-				MakeSecureRandomPadding(0, TUNNEL_SSH_KEEP_ALIVE_PAYLOAD_MAX_BYTES))
-			err = fmt.Errorf("ssh keep alive failed: %s", err)
+			err = sendSshKeepAlive(tunnel.sshClient)
 			sshKeepAliveTimer.Reset(nextSshKeepAlivePeriod())
 
 		case failures := <-tunnel.portForwardFailures:
@@ -579,6 +575,14 @@ func (tunnel *Tunnel) operateTunnel(config *Config, tunnelOwner TunnelOwner) {
 				tunnel.serverEntry.IpAddress, tunnel.portForwardFailureTotal)
 			if tunnel.portForwardFailureTotal > config.PortForwardFailureThreshold {
 				err = errors.New("tunnel exceeded port forward failure threshold")
+			} else {
+				// Try an SSH keep alive to check the state of the SSH connection
+				// Some port forward failures are due to intermittent conditions
+				// on the server, so we don't abort the connection until the threshold
+				// is hit. But if we can't make a simple round trip request to the
+				// server, we'll immediately abort.
+				err = sendSshKeepAlive(tunnel.sshClient)
+				sshKeepAliveTimer.Reset(nextSshKeepAlivePeriod())
 			}
 
 		case <-tunnel.closedSignal:
@@ -598,6 +602,27 @@ func (tunnel *Tunnel) operateTunnel(config *Config, tunnelOwner TunnelOwner) {
 	}
 }
 
+// sendSshKeepAlive is a helper which sends a keepalive@openssh.com request
+// on the specified SSH connections and returns true of the request succeeds
+// within a specified timeout.
+func sendSshKeepAlive(sshClient *ssh.Client) error {
+
+	errChannel := make(chan error, 2)
+	time.AfterFunc(TUNNEL_SSH_KEEP_ALIVE_TIMEOUT, func() {
+		errChannel <- TimeoutError{}
+	})
+
+	go func() {
+		// Random padding to frustrate fingerprinting
+		_, _, err := sshClient.SendRequest(
+			"keepalive@openssh.com", true,
+			MakeSecureRandomPadding(0, TUNNEL_SSH_KEEP_ALIVE_PAYLOAD_MAX_BYTES))
+		errChannel <- err
+	}()
+
+	return ContextError(<-errChannel)
+}
+
 // sendStats is a helper for sending session stats to the server.
 func sendStats(tunnel *Tunnel) {
 

psiphon/upgradeDownload.go

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2015, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package psiphon
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+)
+
+// DownloadUpgrade performs a tunneled, resumable download of client upgrade files.
+// While downloading/resuming, a temporary file is used. Once the download is complete,
+// a notice is issued and the upgrade is available at the destination specified in
+// config.UpgradeDownloadFilename.
+// NOTE: this code does not check that any existing file at config.UpgradeDownloadFilename
+// is actually the version specified in clientUpgradeVersion.
+func DownloadUpgrade(config *Config, clientUpgradeVersion string, tunnel *Tunnel) error {
+
+	// Check if complete file already downloaded
+	if _, err := os.Stat(config.UpgradeDownloadFilename); err == nil {
+		NoticeClientUpgradeDownloaded(config.UpgradeDownloadFilename)
+		return nil
+	}
+
+	partialFilename := fmt.Sprintf(
+		"%s.%s.part", config.UpgradeDownloadFilename, clientUpgradeVersion)
+
+	file, err := os.OpenFile(partialFilename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+	if err != nil {
+		return ContextError(err)
+	}
+	defer file.Close()
+
+	fileInfo, err := file.Stat()
+	if err != nil {
+		return ContextError(err)
+	}
+
+	request, err := http.NewRequest("GET", config.UpgradeDownloadUrl, nil)
+	if err != nil {
+		return ContextError(err)
+	}
+	request.Header.Add("Range", fmt.Sprintf("bytes=%d-", fileInfo.Size()))
+
+	tunneledDialer := func(_, addr string) (conn net.Conn, err error) {
+		return tunnel.sshClient.Dial("tcp", addr)
+	}
+	transport := &http.Transport{
+		Dial: tunneledDialer,
+		ResponseHeaderTimeout: DOWNLOAD_UPGRADE_TIMEOUT,
+	}
+	httpClient := &http.Client{
+		Transport: transport,
+		Timeout:   DOWNLOAD_UPGRADE_TIMEOUT,
+	}
+
+	response, err := httpClient.Do(request)
+	if err != nil {
+		return ContextError(err)
+	}
+	defer response.Body.Close()
+
+	n, err := io.Copy(NewSyncFileWriter(file), response.Body)
+	if err != nil {
+		return ContextError(err)
+	}
+
+	NoticeInfo("client upgrade downloaded bytes: %d", n)
+
+	// Ensure the file is flushed to disk. The deferred close
+	// will be a noop when this succeeds.
+	err = file.Close()
+	if err != nil {
+		return ContextError(err)
+	}
+
+	err = os.Rename(partialFilename, config.UpgradeDownloadFilename)
+	if err != nil {
+		return ContextError(err)
+	}
+
+	NoticeClientUpgradeDownloaded(config.UpgradeDownloadFilename)
+
+	return nil
+}

psiphon/utils.go

@@ -182,3 +182,33 @@ func IsAddressInUseError(err error) bool {
 	}
 	return false
 }
+
+// SyncFileWriter wraps a file and exposes an io.Writer. At predefined
+// steps, the file is synced (flushed to disk) while writing.
+type SyncFileWriter struct {
+	file  *os.File
+	step  int
+	count int
+}
+
+// NewSyncFileWriter creates a SyncFileWriter.
+func NewSyncFileWriter(file *os.File) *SyncFileWriter {
+	return &SyncFileWriter{
+		file:  file,
+		step:  2 << 16,
+		count: 0}
+}
+
+// Write implements io.Writer with periodic file syncing.
+func (writer *SyncFileWriter) Write(p []byte) (n int, err error) {
+	n, err = writer.file.Write(p)
+	if err != nil {
+		return
+	}
+	writer.count += n
+	if writer.count >= writer.step {
+		err = writer.file.Sync()
+		writer.count = 0
+	}
+	return
+}