Update paver to support payload

- osl.Config.Pave() now takes payload keyed
  by OSL ID instead of scheme/time; and
  now paves all OSLs in the given time range,
  regardless of payload
- paver "-payload" parameter takes a JSON
  file and converts that to the payload
  parameter expected by osl.Config.Pave()
- paver "-offset" and "-period" now have
  useful defaults
- paver does a dry run, for the purpose
  of illustrating OSL IDs, when "-output"
  is omitted
- paver will fail when the payload
  includes unknown/unpaved OSL IDs

psiphon/common/osl/osl.go

@@ -43,6 +43,7 @@ import (
 	"net/url"
 	"path"
 	"path/filepath"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -535,7 +536,7 @@ func (state *ClientSeedState) issueSLOKs() {
 			slok, ok := state.scheme.derivedSLOKCache[*ref]
 			state.scheme.derivedSLOKCacheMutex.RUnlock()
 			if !ok {
-				slok = deriveSLOK(state.scheme, ref)
+				slok = state.scheme.deriveSLOK(ref)
 				state.scheme.derivedSLOKCacheMutex.Lock()
 				state.scheme.derivedSLOKCache[*ref] = slok
 				state.scheme.derivedSLOKCacheMutex.Unlock()
@@ -569,31 +570,6 @@ func getSLOKTime(seedPeriodNanoseconds int64) int64 {
 	return time.Now().UTC().Truncate(time.Duration(seedPeriodNanoseconds)).UnixNano()
 }
 
-// deriveSLOK produces SLOK secret keys and IDs using HKDF-Expand
-// defined in https://tools.ietf.org/html/rfc5869.
-func deriveSLOK(
-	scheme *Scheme, ref *slokReference) *SLOK {
-
-	timeBytes := make([]byte, 8)
-	binary.LittleEndian.PutUint64(timeBytes, uint64(ref.Time.UnixNano()))
-
-	key := deriveKeyHKDF(
-		scheme.MasterKey,
-		[]byte(ref.PropagationChannelID),
-		[]byte(ref.SeedSpecID),
-		timeBytes)
-
-	// TODO: is ID derivation cryptographically sound?
-	id := deriveKeyHKDF(
-		scheme.MasterKey,
-		key)
-
-	return &SLOK{
-		ID:  id,
-		Key: key,
-	}
-}
-
 // GetSeedPayload issues any pending SLOKs and returns the accumulated
 // SLOKs for a given client. psiphond will calls this when it receives
 // signalIssueSLOKs which is the trigger to check for new SLOKs.
@@ -631,6 +607,44 @@ func (state *ClientSeedState) ClearSeedPayload() {
 	state.payloadSLOKs = nil
 }
 
+// deriveSLOK produces SLOK secret keys and IDs using HKDF-Expand
+// defined in https://tools.ietf.org/html/rfc5869.
+func (scheme *Scheme) deriveSLOK(ref *slokReference) *SLOK {
+
+	timeBytes := make([]byte, 8)
+	binary.LittleEndian.PutUint64(timeBytes, uint64(ref.Time.UnixNano()))
+
+	key := deriveKeyHKDF(
+		scheme.MasterKey,
+		[]byte(ref.PropagationChannelID),
+		[]byte(ref.SeedSpecID),
+		timeBytes)
+
+	// TODO: is ID derivation cryptographically sound?
+	id := deriveKeyHKDF(
+		scheme.MasterKey,
+		key)
+
+	return &SLOK{
+		ID:  id,
+		Key: key,
+	}
+}
+
+// GetOSLDuration returns the total time duration of an OSL,
+// which is a function of the scheme's SeedPeriodNanoSeconds,
+// the duration of a single SLOK, and the scheme's SeedPeriodKeySplits,
+// the number of SLOKs associated with an OSL.
+func (scheme *Scheme) GetOSLDuration() time.Duration {
+	slokTimePeriodsPerOSL := 1
+	for _, keySplit := range scheme.SeedPeriodKeySplits {
+		slokTimePeriodsPerOSL *= keySplit.Total
+	}
+
+	return time.Duration(
+		int64(slokTimePeriodsPerOSL) * scheme.SeedPeriodNanoseconds)
+}
+
 // PaveFile describes an OSL data file to be paved to an out-of-band
 // distribution drop site. There are two types of files: a registry,
 // which describes how to assemble keys for OSLs, and the encrypted
@@ -677,6 +691,16 @@ type KeyShares struct {
 	KeyShares   []*KeyShares
 }
 
+type PaveLogInfo struct {
+	FileName             string
+	SchemeIndex          int
+	PropagationChannelID string
+	OSLID                string
+	OSLTime              time.Time
+	OSLDuration          time.Duration
+	ServerEntryCount     int
+}
+
 // Pave creates the full set of OSL files, for all schemes in the
 // configuration, to be dropped in an out-of-band distribution site.
 // Only OSLs for the propagation channel ID associated with the
@@ -686,14 +710,14 @@ type KeyShares struct {
 // the client functions GetRegistryURL and GetOSLFileURL.
 //
 // Pave returns a pave file for the entire registry of all OSLs from
-// epoch. It only returns pave files for OSLs referenced in
-// paveServerEntries. paveServerEntries is a list of maps, one for each
-// scheme, from the first SLOK time period identifying an OSL to a
-// payload to encrypt and pave.
-// The registry file spec MD5 checksum values are populated only for
-// OSLs referenced in paveServerEntries. To ensure a registry is fully
-// populated with hashes for skipping redownloading, all OSLs should
-// be paved.
+// epoch to endTime, and a pave file for each OSL. paveServerEntries is
+// a map from hex-encoded OSL IDs to server entries to pave into that OSL.
+// When entries are found, OSL will contain those entries, newline
+// seperated. Otherwise the OSL will still be issued, but be empty.
+//
+// As OSLs outside the epoch-endTime range will no longer appear in
+// the registry, Pave is intended to be used to create the full set
+// of OSLs for a distribution site; i.e., not incrementally.
 //
 // Automation is responsible for consistently distributing server entries
 // to OSLs in the case where OSLs are repaved in subsequent calls.
@@ -702,8 +726,8 @@ func (config *Config) Pave(
 	propagationChannelID string,
 	signingPublicKey string,
 	signingPrivateKey string,
-	paveServerEntries []map[time.Time]string,
-	logCallback func(int, time.Time, string)) ([]*PaveFile, error) {
+	paveServerEntries map[string][]string,
+	logCallback func(*PaveLogInfo)) ([]*PaveFile, error) {
 
 	config.ReloadableFile.RLock()
 	defer config.ReloadableFile.RUnlock()
@@ -712,19 +736,13 @@ func (config *Config) Pave(
 
 	registry := &Registry{}
 
-	if len(paveServerEntries) != len(config.Schemes) {
-		return nil, common.ContextError(errors.New("invalid paveServerEntries"))
-	}
-
 	for schemeIndex, scheme := range config.Schemes {
+		if common.Contains(scheme.PropagationChannelIDs, propagationChannelID) {
 
-		slokTimePeriodsPerOSL := 1
-		for _, keySplit := range scheme.SeedPeriodKeySplits {
-			slokTimePeriodsPerOSL *= keySplit.Total
-		}
+			oslDuration := scheme.GetOSLDuration()
 
-		if common.Contains(scheme.PropagationChannelIDs, propagationChannelID) {
 			oslTime := scheme.epoch
+
 			for !oslTime.After(endTime) {
 
 				firstSLOKTime := oslTime
@@ -734,43 +752,52 @@ func (config *Config) Pave(
 					return nil, common.ContextError(err)
 				}
 
-				registry.FileSpecs = append(registry.FileSpecs, fileSpec)
+				hexEncodedOSLID := hex.EncodeToString(fileSpec.ID)
 
-				serverEntries, ok := paveServerEntries[schemeIndex][oslTime]
-				if ok {
+				registry.FileSpecs = append(registry.FileSpecs, fileSpec)
 
-					signedServerEntries, err := common.WriteAuthenticatedDataPackage(
-						serverEntries,
-						signingPublicKey,
-						signingPrivateKey)
-					if err != nil {
-						return nil, common.ContextError(err)
-					}
+				serverEntryCount := len(paveServerEntries[hexEncodedOSLID])
 
-					boxedServerEntries, err := box(fileKey, common.Compress(signedServerEntries))
-					if err != nil {
-						return nil, common.ContextError(err)
-					}
+				// serverEntries will be "" when nothing is found in paveServerEntries
+				serverEntries := strings.Join(paveServerEntries[hexEncodedOSLID], "\n")
 
-					md5sum := md5.Sum(boxedServerEntries)
-					fileSpec.MD5Sum = md5sum[:]
+				signedServerEntries, err := common.WriteAuthenticatedDataPackage(
+					serverEntries,
+					signingPublicKey,
+					signingPrivateKey)
+				if err != nil {
+					return nil, common.ContextError(err)
+				}
 
-					fileName := fmt.Sprintf(
-						OSL_FILENAME_FORMAT, hex.EncodeToString(fileSpec.ID))
+				boxedServerEntries, err := box(fileKey, common.Compress(signedServerEntries))
+				if err != nil {
+					return nil, common.ContextError(err)
+				}
 
-					paveFiles = append(paveFiles, &PaveFile{
-						Name:     fileName,
-						Contents: boxedServerEntries,
+				md5sum := md5.Sum(boxedServerEntries)
+				fileSpec.MD5Sum = md5sum[:]
+
+				fileName := fmt.Sprintf(
+					OSL_FILENAME_FORMAT, hexEncodedOSLID)
+
+				paveFiles = append(paveFiles, &PaveFile{
+					Name:     fileName,
+					Contents: boxedServerEntries,
+				})
+
+				if logCallback != nil {
+					logCallback(&PaveLogInfo{
+						FileName:             fileName,
+						SchemeIndex:          schemeIndex,
+						PropagationChannelID: propagationChannelID,
+						OSLID:                hexEncodedOSLID,
+						OSLTime:              oslTime,
+						OSLDuration:          oslDuration,
+						ServerEntryCount:     serverEntryCount,
 					})
-
-					if logCallback != nil {
-						logCallback(schemeIndex, oslTime, fileName)
-					}
 				}
 
-				oslTime = oslTime.Add(
-					time.Duration(
-						int64(slokTimePeriodsPerOSL) * scheme.SeedPeriodNanoseconds))
+				oslTime = oslTime.Add(oslDuration)
 			}
 		}
 	}
@@ -811,7 +838,7 @@ func makeOSLFileSpec(
 		SeedSpecID:           string(scheme.SeedSpecs[0].ID),
 		Time:                 firstSLOKTime,
 	}
-	firstSLOK := deriveSLOK(scheme, ref)
+	firstSLOK := scheme.deriveSLOK(ref)
 	oslID := firstSLOK.ID
 
 	fileKey, err := common.MakeSecureRandomBytes(KEY_LENGTH_BYTES)
@@ -922,7 +949,7 @@ func divideKeyWithSeedSpecSLOKs(
 			SeedSpecID:           string(seedSpec.ID),
 			Time:                 *nextSLOKTime,
 		}
-		slok := deriveSLOK(scheme, ref)
+		slok := scheme.deriveSLOK(ref)
 
 		boxedShare, err := box(slok.Key, shares[index])
 		if err != nil {

psiphon/common/osl/osl_test.go

@@ -21,6 +21,7 @@ package osl
 
 import (
 	"encoding/base64"
+	"encoding/hex"
 	"fmt"
 	"net"
 	"testing"
@@ -317,31 +318,25 @@ func TestOSL(t *testing.T) {
 
 			// Dummy server entry payloads will be the OSL ID, which the following
 			// tests use to verify that the correct OSL file decrypts successfully.
-			paveServerEntries := make([]map[time.Time]string, len(config.Schemes))
-			for schemeIndex, scheme := range config.Schemes {
+			paveServerEntries := make(map[string][]string)
+			for _, scheme := range config.Schemes {
 
-				paveServerEntries[schemeIndex] = make(map[time.Time]string)
-
-				slokTimePeriodsPerOSL := 1
-				for _, keySplit := range scheme.SeedPeriodKeySplits {
-					slokTimePeriodsPerOSL *= keySplit.Total
-				}
+				oslDuration := scheme.GetOSLDuration()
 
 				oslTime := scheme.epoch
 				for oslTime.Before(endTime) {
+
 					firstSLOKRef := &slokReference{
 						PropagationChannelID: propagationChannelID,
 						SeedSpecID:           string(scheme.SeedSpecs[0].ID),
 						Time:                 oslTime,
 					}
-					firstSLOK := deriveSLOK(scheme, firstSLOKRef)
+					firstSLOK := scheme.deriveSLOK(firstSLOKRef)
 					oslID := firstSLOK.ID
-					paveServerEntries[schemeIndex][oslTime] =
-						base64.StdEncoding.EncodeToString(oslID)
+					paveServerEntries[hex.EncodeToString(oslID)] =
+						[]string{base64.StdEncoding.EncodeToString(oslID)}
 
-					oslTime = oslTime.Add(
-						time.Duration(
-							int64(slokTimePeriodsPerOSL) * scheme.SeedPeriodNanoseconds))
+					oslTime = oslTime.Add(oslDuration)
 				}
 			}
 
@@ -492,8 +487,7 @@ func TestOSL(t *testing.T) {
 			for _, timePeriod := range testCase.issueSLOKTimePeriods {
 				for _, seedSpecIndex := range testCase.issueSLOKSeedSpecIndexes {
 
-					slok := deriveSLOK(
-						testCase.scheme,
+					slok := testCase.scheme.deriveSLOK(
 						&slokReference{
 							PropagationChannelID: testCase.propagationChannelID,
 							SeedSpecID:           string(testCase.scheme.SeedSpecs[seedSpecIndex].ID),

psiphon/common/osl/paver/main.go

@@ -22,6 +22,7 @@ package main
 import (
 	"crypto/x509"
 	"encoding/base64"
+	"encoding/json"
 	"encoding/pem"
 	"flag"
 	"fmt"
@@ -36,22 +37,33 @@ import (
 func main() {
 
 	var configFilename string
-	flag.StringVar(&configFilename, "config", "", "OSL configuration file")
+	flag.StringVar(&configFilename, "config", "", "OSL configuration filename")
 
 	var offset time.Duration
-	flag.DurationVar(&offset, "offset", 0, "pave OSL start time (offset from now)")
+	flag.DurationVar(
+		&offset, "offset", 0,
+		"pave OSL start time (offset from now); default, 0, selects earliest epoch")
 
 	var period time.Duration
-	flag.DurationVar(&period, "period", 0, "pave OSL total period (starting from offset)")
+	flag.DurationVar(
+		&period, "period", 0,
+		"pave OSL total period (starting from offset); default, 0, selects at least one OSL period from now for all schemes")
 
 	var signingKeyPairFilename string
-	flag.StringVar(&signingKeyPairFilename, "key", "", "signing public key pair")
+	flag.StringVar(&signingKeyPairFilename, "key", "", "signing public key pair filename")
+
+	var payloadFilename string
+	flag.StringVar(&payloadFilename, "payload", "", "server entries to pave into OSLs")
 
 	var destinationDirectory string
-	flag.StringVar(&destinationDirectory, "output", "", "destination directory for output files")
+	flag.StringVar(
+		&destinationDirectory, "output", "",
+		"destination directory for output files; when omitted, no files are written (dry run mode)")
 
 	flag.Parse()
 
+	// load config
+
 	configJSON, err := ioutil.ReadFile(configFilename)
 	if err != nil {
 		fmt.Printf("failed loading configuration file: %s\n", err)
@@ -64,6 +76,8 @@ func main() {
 		os.Exit(1)
 	}
 
+	// load key pair
+
 	keyPairPEM, err := ioutil.ReadFile(signingKeyPairFilename)
 	if err != nil {
 		fmt.Printf("failed loading signing public key pair file: %s\n", err)
@@ -97,47 +111,84 @@ func main() {
 	signingPublicKey := base64.StdEncoding.EncodeToString(publicKeyBytes)
 	signingPrivateKey := base64.StdEncoding.EncodeToString(privateKeyBytes)
 
-	paveTime := time.Now().UTC()
-	startTime := paveTime.Add(offset)
-	endTime := startTime.Add(period)
-
-	schemeOSLTimePeriods := make(map[int]time.Duration)
-	for index, scheme := range config.Schemes {
-		slokTimePeriodsPerOSL := 1
-		for _, keySplit := range scheme.SeedPeriodKeySplits {
-			slokTimePeriodsPerOSL *= keySplit.Total
+	// load payload
+
+	paveServerEntries := make(map[string][]string)
+
+	pavedPayloadOSLID := make(map[string]bool)
+
+	if payloadFilename != "" {
+		payloadJSON, err := ioutil.ReadFile(payloadFilename)
+		if err != nil {
+			fmt.Printf("failed loading payload file: %s\n", err)
+			os.Exit(1)
 		}
-		schemeOSLTimePeriods[index] =
-			time.Duration(scheme.SeedPeriodNanoseconds * int64(slokTimePeriodsPerOSL))
-	}
 
-	allPropagationChannelIDs := make(map[string][]int)
-	for index, scheme := range config.Schemes {
-		for _, propagationChannelID := range scheme.PropagationChannelIDs {
-			allPropagationChannelIDs[propagationChannelID] =
-				append(allPropagationChannelIDs[propagationChannelID], index)
+		var payload []*struct {
+			OSLIDs      []string
+			ServerEntry string
 		}
-	}
 
-	for propagationChannelID, schemeIndexes := range allPropagationChannelIDs {
+		err = json.Unmarshal(payloadJSON, &payload)
+		if err != nil {
+			fmt.Printf("failed unmarshaling payload file: %s\n", err)
+			os.Exit(1)
+		}
 
-		paveServerEntries := make([]map[time.Time]string, len(config.Schemes))
+		for _, item := range payload {
+			for _, oslID := range item.OSLIDs {
+				paveServerEntries[oslID] = append(
+					paveServerEntries[oslID], item.ServerEntry)
+				pavedPayloadOSLID[oslID] = false
+			}
+		}
+	}
 
-		for _, index := range schemeIndexes {
+	// determine pave time range
 
-			paveServerEntries[index] = make(map[time.Time]string)
+	paveTime := time.Now().UTC()
 
-			oslTime, _ := time.Parse(time.RFC3339, config.Schemes[index].Epoch)
-			for !oslTime.After(endTime) {
-				if !oslTime.Before(startTime) {
-					paveServerEntries[index][oslTime] = ""
-				}
-				oslTime = oslTime.Add(schemeOSLTimePeriods[index])
+	var startTime, endTime time.Time
+
+	if offset != 0 {
+		startTime = paveTime.Add(offset)
+	} else {
+		// Default to the earliest scheme epoch.
+		startTime = paveTime
+		for _, scheme := range config.Schemes {
+			epoch, _ := time.Parse(time.RFC3339, scheme.Epoch)
+			if epoch.Before(startTime) {
+				startTime = epoch
 			}
+		}
+	}
 
-			fmt.Printf("Paving propagation channel %s, scheme #%d, [%s - %s], %s\n",
-				propagationChannelID, index, startTime, endTime, schemeOSLTimePeriods[index])
+	if period != 0 {
+		endTime = startTime.Add(period)
+	} else {
+		// Default to at least one OSL period after "now",
+		// considering all schemes.
+		endTime = paveTime
+		for _, scheme := range config.Schemes {
+			oslDuration := scheme.GetOSLDuration()
+			if endTime.Add(oslDuration).After(endTime) {
+				endTime = endTime.Add(oslDuration)
+			}
 		}
+	}
+
+	// build list of all participating propagation channel IDs
+
+	allPropagationChannelIDs := make(map[string]bool)
+	for _, scheme := range config.Schemes {
+		for _, propagationChannelID := range scheme.PropagationChannelIDs {
+			allPropagationChannelIDs[propagationChannelID] = true
+		}
+	}
+
+	// pave a directory for each propagation channel
+
+	for propagationChannelID, _ := range allPropagationChannelIDs {
 
 		paveFiles, err := config.Pave(
 			endTime,
@@ -145,29 +196,58 @@ func main() {
 			signingPublicKey,
 			signingPrivateKey,
 			paveServerEntries,
-			func(schemeIndex int, oslTime time.Time, fileName string) {
-				fmt.Printf("\tPaved scheme %d %s: %s\n", schemeIndex, oslTime, fileName)
+			func(logInfo *osl.PaveLogInfo) {
+				pavedPayloadOSLID[logInfo.OSLID] = true
+				fmt.Printf(
+					"paved %s: scheme %d, propagation channel ID %s, "+
+						"OSL time %s, OSL duration %s, server entries: %d\n",
+					logInfo.FileName,
+					logInfo.SchemeIndex,
+					logInfo.PropagationChannelID,
+					logInfo.OSLTime,
+					logInfo.OSLDuration,
+					logInfo.ServerEntryCount)
 			})
 		if err != nil {
 			fmt.Printf("failed paving: %s\n", err)
 			os.Exit(1)
 		}
 
-		directory := filepath.Join(destinationDirectory, propagationChannelID)
+		if destinationDirectory != "" {
 
-		err = os.MkdirAll(directory, 0755)
-		if err != nil {
-			fmt.Printf("failed creating output directory: %s\n", err)
-			os.Exit(1)
-		}
+			directory := filepath.Join(destinationDirectory, propagationChannelID)
 
-		for _, paveFile := range paveFiles {
-			filename := filepath.Join(directory, paveFile.Name)
-			err = ioutil.WriteFile(filename, paveFile.Contents, 0755)
+			err = os.MkdirAll(directory, 0755)
 			if err != nil {
-				fmt.Printf("error writing output file: %s\n", err)
+				fmt.Printf("failed creating output directory: %s\n", err)
 				os.Exit(1)
 			}
+
+			for _, paveFile := range paveFiles {
+				filename := filepath.Join(directory, paveFile.Name)
+				err = ioutil.WriteFile(filename, paveFile.Contents, 0755)
+				if err != nil {
+					fmt.Printf("error writing output file: %s\n", err)
+					os.Exit(1)
+				}
+			}
+		}
+	}
+
+	// fail if payload contains OSL IDs not in the config and time range
+
+	unknown := false
+	for oslID, paved := range pavedPayloadOSLID {
+		if !paved {
+			fmt.Printf(
+				"ignored %d server entries for unknown OSL ID: %s\n",
+				len(paveServerEntries[oslID]),
+				oslID)
+			unknown = true
 		}
 	}
+	if unknown {
+		fmt.Printf("payload contains unknown OSL IDs\n")
+		os.Exit(1)
+	}
 }