Support multiple download URLs

- For upgrade download, and remote and obfuscated
  server lists fetches
- Configurable download strategy
- Deprecate legacy URL parameters
- Backwards compatible with existing configs

psiphon/config.go

@@ -20,6 +20,7 @@
 package psiphon
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -120,8 +121,20 @@ type Config struct {
 	// be established to known servers.
 	// This value is supplied by and depends on the Psiphon Network, and is
 	// typically embedded in the client binary.
+	//
+	// Deprecated: Use RemoteServerListURLs. When RemoteServerListURLs is
+	// not nil, this parameter is ignored.
 	RemoteServerListUrl string
 
+	// RemoteServerListURLs is list of URLs which specify locations to fetch
+	// out-of-band server entries. This facility is used when a tunnel cannot
+	// be established to known servers.
+	// This value is supplied by and depends on the Psiphon Network, and is
+	// typically embedded in the client binary.
+	// All URLs must point to the same entity with the same ETag. At least
+	// one DownloadURL must have OnlyAfterAttempts = 0.
+	RemoteServerListURLs []*DownloadURL
+
 	// RemoteServerListDownloadFilename specifies a target filename for
 	// storing the remote server list download. Data is stored in co-located
 	// files (RemoteServerListDownloadFilename.part*) to allow for resumable
@@ -138,8 +151,19 @@ type Config struct {
 	// from which to fetch obfuscated server list files.
 	// This value is supplied by and depends on the Psiphon Network, and is
 	// typically embedded in the client binary.
+	//
+	// Deprecated: Use ObfuscatedServerListRootURLs. When
+	// ObfuscatedServerListRootURLs is not nil, this parameter is ignored.
 	ObfuscatedServerListRootURL string
 
+	// ObfuscatedServerListRootURLs is a list of URLs which specify root
+	// locations from which to fetch obfuscated server list files.
+	// This value is supplied by and depends on the Psiphon Network, and is
+	// typically embedded in the client binary.
+	// All URLs must point to the same entity with the same ETag. At least
+	// one DownloadURL must have OnlyAfterAttempts = 0.
+	ObfuscatedServerListRootURLs []*DownloadURL
+
 	// ObfuscatedServerListDownloadDirectory specifies a target directory for
 	// storing the obfuscated remote server list downloads. Data is stored in
 	// co-located files (<OSL filename>.part*) to allow for resumable
@@ -284,17 +308,30 @@ type Config struct {
 	// download facility which downloads this resource and emits a notice when complete.
 	// This value is supplied by and depends on the Psiphon Network, and is
 	// typically embedded in the client binary.
+	//
+	// Deprecated: Use UpgradeDownloadURLs. When UpgradeDownloadURLs
+	// is not nil, this parameter is ignored.
 	UpgradeDownloadUrl string
 
+	// UpgradeDownloadURLs is list of URLs which specify locations from which to
+	// download a host client upgrade file, when one is available. The core tunnel
+	// controller provides a resumable download facility which downloads this resource
+	// and emits a notice when complete.
+	// This value is supplied by and depends on the Psiphon Network, and is
+	// typically embedded in the client binary.
+	// All URLs must point to the same entity with the same ETag. At least
+	// one DownloadURL must have OnlyAfterAttempts = 0.
+	UpgradeDownloadURLs []*DownloadURL
+
 	// UpgradeDownloadClientVersionHeader specifies the HTTP header name for the
-	// entity at UpgradeDownloadUrl which specifies the client version (an integer
+	// entity at UpgradeDownloadURLs which specifies the client version (an integer
 	// value). A HEAD request may be made to check the version number available at
-	// UpgradeDownloadUrl. UpgradeDownloadClientVersionHeader is required when
-	// UpgradeDownloadUrl is specified.
+	// UpgradeDownloadURLs. UpgradeDownloadClientVersionHeader is required when
+	// UpgradeDownloadURLs is specified.
 	UpgradeDownloadClientVersionHeader string
 
 	// UpgradeDownloadFilename is the local target filename for an upgrade download.
-	// This parameter is required when UpgradeDownloadUrl is specified.
+	// This parameter is required when UpgradeDownloadURLs is specified.
 	// Data is stored in co-located files (UpgradeDownloadFilename.part*) to allow
 	// for resumable downloading.
 	UpgradeDownloadFilename string
@@ -414,6 +451,26 @@ type Config struct {
 	EmitSLOKs bool
 }
 
+// DownloadURL specifies a URL for downloading resources along with parameters
+// for the download strategy.
+type DownloadURL struct {
+
+	// URL is the location of the resource. This string is slightly obfuscated
+	// with base64 encoding to mitigate trivial binary executable string scanning.
+	URL string
+
+	// SkipVerify indicates whether to verify HTTPS certificates. It some
+	// circumvention scenarios, verification is not possible. This must
+	// only be set to true when the resource its own verification mechanism.
+	SkipVerify bool
+
+	// OnlyAfterAttempts specifies how to schedule this URL when downloading
+	// the same resource (same entity, same ETag) from multiple different
+	// candidate locations. For a value of N, this URL is only a candidate
+	// after N rounds of attempting the download from other URLs.
+	OnlyAfterAttempts int
+}
+
 // LoadConfig parses and validates a JSON format Psiphon config JSON
 // string and returns a Config struct populated with config values.
 func LoadConfig(configJson []byte) (*Config, error) {
@@ -506,15 +563,37 @@ func LoadConfig(configJson []byte) (*Config, error) {
 			errors.New("invalid TargetApiProtocol"))
 	}
 
-	if config.UpgradeDownloadUrl != "" &&
-		(config.UpgradeDownloadClientVersionHeader == "" || config.UpgradeDownloadFilename == "") {
-		return nil, common.ContextError(errors.New(
-			"UpgradeDownloadUrl requires UpgradeDownloadClientVersionHeader and UpgradeDownloadFilename"))
+	if config.UpgradeDownloadUrl != "" && config.UpgradeDownloadURLs == nil {
+		config.UpgradeDownloadURLs = promoteLegacyDownloadURL(config.UpgradeDownloadUrl)
+	}
+
+	if config.UpgradeDownloadURLs != nil {
+
+		err := decodeAndValidateDownloadURLs("UpgradeDownloadURLs", config.UpgradeDownloadURLs)
+		if err != nil {
+			return nil, common.ContextError(err)
+		}
+
+		if config.UpgradeDownloadClientVersionHeader == "" {
+			return nil, common.ContextError(errors.New("missing UpgradeDownloadClientVersionHeader"))
+		}
+		if config.UpgradeDownloadFilename == "" {
+			return nil, common.ContextError(errors.New("missing UpgradeDownloadFilename"))
+		}
 	}
 
 	if !config.DisableRemoteServerListFetcher {
 
-		if config.RemoteServerListUrl != "" {
+		if config.RemoteServerListUrl != "" && config.RemoteServerListURLs == nil {
+			config.RemoteServerListURLs = promoteLegacyDownloadURL(config.RemoteServerListUrl)
+		}
+
+		if config.RemoteServerListURLs != nil {
+
+			err := decodeAndValidateDownloadURLs("RemoteServerListURLs", config.RemoteServerListURLs)
+			if err != nil {
+				return nil, common.ContextError(err)
+			}
 
 			if config.RemoteServerListSignaturePublicKey == "" {
 				return nil, common.ContextError(errors.New("missing RemoteServerListSignaturePublicKey"))
@@ -525,7 +604,16 @@ func LoadConfig(configJson []byte) (*Config, error) {
 			}
 		}
 
-		if config.ObfuscatedServerListRootURL != "" {
+		if config.ObfuscatedServerListRootURL != "" && config.ObfuscatedServerListRootURLs == nil {
+			config.ObfuscatedServerListRootURLs = promoteLegacyDownloadURL(config.ObfuscatedServerListRootURL)
+		}
+
+		if config.ObfuscatedServerListRootURLs != nil {
+
+			err := decodeAndValidateDownloadURLs("ObfuscatedServerListRootURLs", config.ObfuscatedServerListRootURLs)
+			if err != nil {
+				return nil, common.ContextError(err)
+			}
 
 			if config.RemoteServerListSignaturePublicKey == "" {
 				return nil, common.ContextError(errors.New("missing RemoteServerListSignaturePublicKey"))
@@ -594,3 +682,60 @@ func LoadConfig(configJson []byte) (*Config, error) {
 
 	return &config, nil
 }
+
+func promoteLegacyDownloadURL(URL string) []*DownloadURL {
+	downloadURLs := make([]*DownloadURL, 1)
+	downloadURLs[0] = &DownloadURL{
+		URL:               base64.StdEncoding.EncodeToString([]byte(URL)),
+		SkipVerify:        false,
+		OnlyAfterAttempts: 0,
+	}
+	return downloadURLs
+}
+
+func decodeAndValidateDownloadURLs(name string, downloadURLs []*DownloadURL) error {
+
+	hasOnlyAfterZero := false
+	for _, downloadURL := range downloadURLs {
+		if downloadURL.OnlyAfterAttempts == 0 {
+			hasOnlyAfterZero = true
+		}
+		decodedURL, err := base64.StdEncoding.DecodeString(downloadURL.URL)
+		if err != nil {
+			return fmt.Errorf("failed to decode URL in %s: %s", name, err)
+		}
+
+		downloadURL.URL = string(decodedURL)
+	}
+
+	var err error
+	if !hasOnlyAfterZero {
+		err = fmt.Errorf("must be at least one DownloadURL with OnlyAfterAttempts = 0 in %s", name)
+	}
+
+	return err
+}
+
+func selectDownloadURL(attempt int, downloadURLs []*DownloadURL) (string, bool) {
+
+	candidates := make([]int, 0)
+	for index, URL := range downloadURLs {
+		if attempt >= URL.OnlyAfterAttempts {
+			candidates = append(candidates, index)
+		}
+	}
+
+	if len(candidates) < 1 {
+		// This case is not expected, as decodeAndValidateDownloadURLs
+		// should reject configs that would have no candidates for
+		// 0 attempts.
+		return "", true
+	}
+
+	selection, err := common.MakeSecureRandomInt(len(candidates))
+	if err != nil {
+		selection = 0
+	}
+
+	return downloadURLs[candidates[selection]].URL, downloadURLs[candidates[selection]].SkipVerify
+}

psiphon/controller.go

@@ -184,7 +184,7 @@ func (controller *Controller) Run(shutdownBroadcast <-chan struct{}) {
 		retryPeriod := time.Duration(
 			*controller.config.FetchRemoteServerListRetryPeriodSeconds) * time.Second
 
-		if controller.config.RemoteServerListUrl != "" {
+		if controller.config.RemoteServerListURLs != nil {
 			controller.runWaitGroup.Add(1)
 			go controller.remoteServerListFetcher(
 				"common",
@@ -194,7 +194,7 @@ func (controller *Controller) Run(shutdownBroadcast <-chan struct{}) {
 				FETCH_REMOTE_SERVER_LIST_STALE_PERIOD)
 		}
 
-		if controller.config.ObfuscatedServerListRootURL != "" {
+		if controller.config.ObfuscatedServerListRootURLs != nil {
 			controller.runWaitGroup.Add(1)
 			go controller.remoteServerListFetcher(
 				"obfuscated",
@@ -205,9 +205,7 @@ func (controller *Controller) Run(shutdownBroadcast <-chan struct{}) {
 		}
 	}
 
-	if controller.config.UpgradeDownloadUrl != "" &&
-		controller.config.UpgradeDownloadFilename != "" {
-
+	if controller.config.UpgradeDownloadURLs != nil {
 		controller.runWaitGroup.Add(1)
 		go controller.upgradeDownloader()
 	}
@@ -324,7 +322,7 @@ fetcherLoop:
 		}
 
 	retryLoop:
-		for {
+		for attempt := 0; ; attempt++ {
 			// Don't attempt to fetch while there is no network connectivity,
 			// to avoid alert notice noise.
 			if !WaitForNetworkConnectivity(
@@ -339,6 +337,7 @@ fetcherLoop:
 
 			err := fetcher(
 				controller.config,
+				attempt,
 				tunnel,
 				controller.untunneledDialConfig)
 
@@ -491,7 +490,7 @@ downloadLoop:
 		}
 
 	retryLoop:
-		for {
+		for attempt := 0; ; attempt++ {
 			// Don't attempt to download while there is no network connectivity,
 			// to avoid alert notice noise.
 			if !WaitForNetworkConnectivity(
@@ -506,6 +505,7 @@ downloadLoop:
 
 			err := DownloadUpgrade(
 				controller.config,
+				attempt,
 				handshakeVersion,
 				tunnel,
 				controller.untunneledDialConfig)

psiphon/feedback.go

@@ -151,7 +151,8 @@ func SendFeedback(configJson, diagnosticsJson, b64EncodedPublicKey, uploadServer
 
 // Attempt to upload feedback data to server.
 func uploadFeedback(config *DialConfig, feedbackData []byte, url string, headerPieces []string) error {
-	client, parsedUrl, err := MakeUntunneledHttpsClient(config, nil, url, time.Duration(FEEDBACK_UPLOAD_TIMEOUT_SECONDS*time.Second))
+	client, parsedUrl, err := MakeUntunneledHttpsClient(
+		config, nil, url, false, time.Duration(FEEDBACK_UPLOAD_TIMEOUT_SECONDS*time.Second))
 	if err != nil {
 		return err
 	}

psiphon/net.go

@@ -239,11 +239,16 @@ func MakeUntunneledHttpsClient(
 	dialConfig *DialConfig,
 	verifyLegacyCertificate *x509.Certificate,
 	requestUrl string,
+	skipVerify bool,
 	requestTimeout time.Duration) (*http.Client, string, error) {
 
 	// Change the scheme to "http"; otherwise http.Transport will try to do
 	// another TLS handshake inside the explicit TLS session. Also need to
 	// force an explicit port, as the default for "http", 80, won't talk TLS.
+	//
+	// TODO: set http.Transport.DialTLS instead of Dial to avoid this hack?
+	// See: https://golang.org/pkg/net/http/#Transport. DialTLS was added in
+	// Go 1.4 but this code may pre-date that.
 
 	urlComponents, err := url.Parse(requestUrl)
 	if err != nil {
@@ -272,7 +277,7 @@ func MakeUntunneledHttpsClient(
 			Dial: NewTCPDialer(dialConfig),
 			VerifyLegacyCertificate:       verifyLegacyCertificate,
 			SNIServerName:                 host,
-			SkipVerify:                    false,
+			SkipVerify:                    skipVerify,
 			UseIndistinguishableTLS:       useIndistinguishableTLS,
 			TrustedCACertificatesFilename: dialConfig.TrustedCACertificatesFilename,
 		})
@@ -297,6 +302,7 @@ func MakeUntunneledHttpsClient(
 func MakeTunneledHttpClient(
 	config *Config,
 	tunnel *Tunnel,
+	skipVerify bool,
 	requestTimeout time.Duration) (*http.Client, error) {
 
 	tunneledDialer := func(_, addr string) (conn net.Conn, err error) {
@@ -307,7 +313,12 @@ func MakeTunneledHttpClient(
 		Dial: tunneledDialer,
 	}
 
-	if config.UseTrustedCACertificatesForStockTLS {
+	if skipVerify {
+
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+
+	} else if config.UseTrustedCACertificatesForStockTLS {
+
 		if config.TrustedCACertificatesFilename == "" {
 			return nil, common.ContextError(errors.New(
 				"UseTrustedCACertificatesForStockTLS requires TrustedCACertificatesFilename"))
@@ -336,6 +347,7 @@ func MakeDownloadHttpClient(
 	tunnel *Tunnel,
 	untunneledDialConfig *DialConfig,
 	requestUrl string,
+	skipVerify bool,
 	requestTimeout time.Duration) (*http.Client, string, error) {
 
 	var httpClient *http.Client
@@ -343,7 +355,8 @@ func MakeDownloadHttpClient(
 
 	if tunnel != nil {
 		// MakeTunneledHttpClient works with both "http" and "https" schemes
-		httpClient, err = MakeTunneledHttpClient(config, tunnel, requestTimeout)
+		httpClient, err = MakeTunneledHttpClient(
+			config, tunnel, skipVerify, requestTimeout)
 		if err != nil {
 			return nil, "", common.ContextError(err)
 		}
@@ -355,7 +368,7 @@ func MakeDownloadHttpClient(
 		// MakeUntunneledHttpsClient works only with "https" schemes
 		if urlComponents.Scheme == "https" {
 			httpClient, requestUrl, err = MakeUntunneledHttpsClient(
-				untunneledDialConfig, nil, requestUrl, requestTimeout)
+				untunneledDialConfig, nil, requestUrl, skipVerify, requestTimeout)
 			if err != nil {
 				return nil, "", common.ContextError(err)
 			}
@@ -467,6 +480,10 @@ func ResumeDownload(
 	// receive 412 on ETag mismatch.
 	if err == nil &&
 		(response.StatusCode != http.StatusPartialContent &&
+
+			// Certain http servers return 200 OK where we expect 206, so accept that.
+			response.StatusCode != http.StatusOK &&
+
 			response.StatusCode != http.StatusRequestedRangeNotSatisfiable &&
 			response.StatusCode != http.StatusPreconditionFailed &&
 			response.StatusCode != http.StatusNotModified) {

psiphon/remoteServerList.go

@@ -34,7 +34,7 @@ import (
 )
 
 type RemoteServerListFetcher func(
-	config *Config, tunnel *Tunnel, untunneledDialConfig *DialConfig) error
+	config *Config, attempt int, tunnel *Tunnel, untunneledDialConfig *DialConfig) error
 
 // FetchCommonRemoteServerList downloads the common remote server list from
 // config.RemoteServerListUrl. It validates its digital signature using the
@@ -45,16 +45,20 @@ type RemoteServerListFetcher func(
 // be unique and persistent.
 func FetchCommonRemoteServerList(
 	config *Config,
+	attempt int,
 	tunnel *Tunnel,
 	untunneledDialConfig *DialConfig) error {
 
 	NoticeInfo("fetching common remote server list")
 
+	downloadURL, skipVerify := selectDownloadURL(attempt, config.RemoteServerListURLs)
+
 	newETag, err := downloadRemoteServerListFile(
 		config,
 		tunnel,
 		untunneledDialConfig,
-		config.RemoteServerListUrl,
+		downloadURL,
+		skipVerify,
 		"",
 		config.RemoteServerListDownloadFilename)
 	if err != nil {
@@ -100,13 +104,16 @@ func FetchCommonRemoteServerList(
 // must be unique and persistent.
 func FetchObfuscatedServerLists(
 	config *Config,
+	attempt int,
 	tunnel *Tunnel,
 	untunneledDialConfig *DialConfig) error {
 
 	NoticeInfo("fetching obfuscated remote server lists")
 
 	downloadFilename := osl.GetOSLRegistryFilename(config.ObfuscatedServerListDownloadDirectory)
-	downloadURL := osl.GetOSLRegistryURL(config.ObfuscatedServerListRootURL)
+
+	rootURL, skipVerify := selectDownloadURL(attempt, config.ObfuscatedServerListRootURLs)
+	downloadURL := osl.GetOSLRegistryURL(rootURL)
 
 	// failed is set if any operation fails and should trigger a retry. When the OSL registry
 	// fails to download, any cached registry is used instead; when any single OSL fails
@@ -122,6 +129,7 @@ func FetchObfuscatedServerLists(
 		tunnel,
 		untunneledDialConfig,
 		downloadURL,
+		skipVerify,
 		"",
 		downloadFilename)
 	if err != nil {
@@ -199,8 +207,11 @@ func FetchObfuscatedServerLists(
 		})
 
 	for _, oslID := range oslIDs {
+
 		downloadFilename := osl.GetOSLFilename(config.ObfuscatedServerListDownloadDirectory, oslID)
-		downloadURL := osl.GetOSLFileURL(config.ObfuscatedServerListRootURL, oslID)
+
+		downloadURL := osl.GetOSLFileURL(rootURL, oslID)
+
 		hexID := hex.EncodeToString(oslID)
 
 		// Note: the MD5 checksum step assumes the remote server list host's ETag uses MD5
@@ -218,6 +229,7 @@ func FetchObfuscatedServerLists(
 			tunnel,
 			untunneledDialConfig,
 			downloadURL,
+			skipVerify,
 			remoteETag,
 			downloadFilename)
 		if err != nil {
@@ -280,7 +292,10 @@ func downloadRemoteServerListFile(
 	config *Config,
 	tunnel *Tunnel,
 	untunneledDialConfig *DialConfig,
-	sourceURL, sourceETag, destinationFilename string) (string, error) {
+	sourceURL string,
+	skipVerify bool,
+	sourceETag string,
+	destinationFilename string) (string, error) {
 
 	lastETag, err := GetUrlETag(sourceURL)
 	if err != nil {
@@ -304,6 +319,7 @@ func downloadRemoteServerListFile(
 		tunnel,
 		untunneledDialConfig,
 		sourceURL,
+		skipVerify,
 		time.Duration(*config.FetchRemoteServerListTimeoutSeconds)*time.Second)
 	if err != nil {
 		return "", common.ContextError(err)

psiphon/serverApi.go

@@ -525,6 +525,7 @@ func (serverContext *ServerContext) doUntunneledStatusRequest(
 		dialConfig,
 		certificate,
 		url,
+		false,
 		timeout)
 	if err != nil {
 		return common.ContextError(err)

psiphon/upgradeDownload.go

@@ -55,6 +55,7 @@ import (
 // upgrade is still pending install by the outer client.
 func DownloadUpgrade(
 	config *Config,
+	attempt int,
 	handshakeVersion string,
 	tunnel *Tunnel,
 	untunneledDialConfig *DialConfig) error {
@@ -68,11 +69,14 @@ func DownloadUpgrade(
 
 	// Select tunneled or untunneled configuration
 
+	downloadURL, skipVerify := selectDownloadURL(attempt, config.UpgradeDownloadURLs)
+
 	httpClient, requestUrl, err := MakeDownloadHttpClient(
 		config,
 		tunnel,
 		untunneledDialConfig,
-		config.UpgradeDownloadUrl,
+		downloadURL,
+		skipVerify,
 		DOWNLOAD_UPGRADE_TIMEOUT)
 
 	// If no handshake version is supplied, make an initial HEAD request