Store md5sums of OSL files in the registry

For remote server list distribution hosts that
supports ETags based on md5sum, use this info in
the registry is to entirely skip redownloading
unchanged OSLs without making any HTTP request.

psiphon/common/osl/osl.go

@@ -21,7 +21,7 @@
 // mechanism is a method of distributing server lists only to clients that
 // demonstrate certain behavioral traits. Clients are seeded with Server
 // List Obfuscation Keys (SLOKs) as they meet the configured criteria. These
-// keys are stored and later comboined to assemble keys to decrypt out-of-band
+// keys are stored and later combined to assemble keys to decrypt out-of-band
 // distributed OSL files that contain server lists.
 //
 // This package contains the core routines used in psiphond (to track client
@@ -33,6 +33,7 @@ import (
 	"bytes"
 	"compress/zlib"
 	"crypto/hmac"
+	"crypto/md5"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/binary"
@@ -653,9 +654,16 @@ type Registry struct {
 // An OSLFileSpec includes an ID which is used to reference the
 // OSL file and describes the key splits used to divide the OSL
 // file key along with the SLOKs required to reassemble those keys.
+//
+// The MD5Sum field is a checksum of the contents of the OSL file
+// to be used to skip redownloading previously downloaded files.
+// MD5 is not cryptogrpahically secure and this checksum is not
+// relied upon for OSL verification. MD5 is used for compatibility
+// with out-of-band distribution hosts.
 type OSLFileSpec struct {
 	ID        []byte
 	KeyShares *KeyShares
+	MD5Sum    []byte
 }
 
 // KeyShares is a tree data structure which describes the
@@ -684,6 +692,10 @@ type KeyShares struct {
 // paveServerEntries. paveServerEntries is a list of maps, one for each
 // scheme, from the first SLOK time period identifying an OSL to a
 // payload to encrypt and pave.
+// The registry file spec MD5 checksum values are populated only for
+// OSLs referenced in paveServerEntries. To ensure a registry is fully
+// populated with hashes for skipping redownloading, all OSLs should
+// be paved.
 //
 // Automation is responsible for consistently distributing server entries
 // to OSLs in the case where OSLs are repaved in subsequent calls.
@@ -741,6 +753,9 @@ func (config *Config) Pave(
 						return nil, common.ContextError(err)
 					}
 
+					md5sum := md5.Sum(boxedServerEntries)
+					fileSpec.MD5Sum = md5sum[:]
+
 					fileName := fmt.Sprintf(
 						OSL_FILENAME_FORMAT, hex.EncodeToString(fileSpec.ID))
 
@@ -1045,6 +1060,17 @@ func (registry *Registry) GetSeededOSLIDs(lookup SLOKLookup, errorLogger func(er
 	return OSLIDs
 }
 
+// GetOSLMD5Sum returns the MD5 checksum for the specified OSL.
+func (registry *Registry) GetOSLMD5Sum(oslID []byte) ([]byte, error) {
+
+	fileSpec, ok := registry.oslIDLookup[string(oslID)]
+	if !ok {
+		return nil, common.ContextError(errors.New("unknown OSL ID"))
+	}
+
+	return fileSpec.MD5Sum, nil
+}
+
 // reassembleKey recursively traverses a KeyShares tree, determining
 // whether there exists suffient SLOKs to reassemble the root key and
 // performing the key assembly as required.

psiphon/remoteServerList.go

@@ -55,6 +55,7 @@ func FetchCommonRemoteServerList(
 		tunnel,
 		untunneledDialConfig,
 		config.RemoteServerListUrl,
+		"",
 		config.RemoteServerListDownloadFilename)
 	if err != nil {
 		return fmt.Errorf("failed to download common remote server list: %s", common.ContextError(err))
@@ -121,6 +122,7 @@ func FetchObfuscatedServerLists(
 		tunnel,
 		untunneledDialConfig,
 		downloadURL,
+		"",
 		downloadFilename)
 	if err != nil {
 		failed = true
@@ -201,6 +203,14 @@ func FetchObfuscatedServerLists(
 		downloadURL := osl.GetOSLFileURL(config.ObfuscatedServerListRootURL, oslID)
 		hexID := hex.EncodeToString(oslID)
 
+		// Note: the MD5 checksum step assumes the remote server list host's ETag uses MD5
+		// with a hex encoding. If this is not the case, the remoteETag should be left blank.
+		remoteETag := ""
+		md5sum, err := oslRegistry.GetOSLMD5Sum(oslID)
+		if err == nil {
+			remoteETag = hex.EncodeToString(md5sum)
+		}
+
 		// TODO: store ETags in OSL registry to enable skipping requests entirely
 
 		newETag, err := downloadRemoteServerListFile(
@@ -208,6 +218,7 @@ func FetchObfuscatedServerLists(
 			tunnel,
 			untunneledDialConfig,
 			downloadURL,
+			remoteETag,
 			downloadFilename)
 		if err != nil {
 			failed = true
@@ -254,7 +265,7 @@ func FetchObfuscatedServerLists(
 	}
 
 	if failed {
-		return errors.New("failed to fetch obfuscated remote server lists")
+		return errors.New("one or more operations failed")
 	}
 	return nil
 }
@@ -269,7 +280,21 @@ func downloadRemoteServerListFile(
 	config *Config,
 	tunnel *Tunnel,
 	untunneledDialConfig *DialConfig,
-	sourceURL, destinationFilename string) (string, error) {
+	sourceURL, sourceETag, destinationFilename string) (string, error) {
+
+	lastETag, err := GetUrlETag(sourceURL)
+	if err != nil {
+		return "", common.ContextError(err)
+	}
+
+	// sourceETag, when specified, is prior knowlegde of the
+	// remote ETag that can be used to skip the request entirely.
+	// This will be set in the case of OSL files, from the MD5Sum
+	// values stored in the registry.
+	if lastETag != "" && sourceETag == lastETag {
+		// TODO: notice?
+		return "", nil
+	}
 
 	// MakeDownloadHttpClient will select either a tunneled
 	// or untunneled configuration.
@@ -284,11 +309,6 @@ func downloadRemoteServerListFile(
 		return "", common.ContextError(err)
 	}
 
-	lastETag, err := GetUrlETag(sourceURL)
-	if err != nil {
-		return "", common.ContextError(err)
-	}
-
 	n, responseETag, err := ResumeDownload(
 		httpClient, requestURL, destinationFilename, lastETag)
 

psiphon/remoteServerList_test.go

@@ -279,6 +279,8 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 	// connect to Psiphon server with Psiphon client
 	//
 
+	SetEmitDiagnosticNotices(true)
+
 	// Note: calling LoadConfig ensures all *int config fields are initialized
 	clientConfigJSONTemplate := `
     {