OSL paver enhancements

- Support propagation channel IDs in multiple schemes
- Switch to time range for pave target
- Emit progress messages

psiphon/common/osl/osl.go

@@ -705,7 +705,8 @@ func (config *Config) Pave(
 	propagationChannelID string,
 	signingPublicKey string,
 	signingPrivateKey string,
-	paveServerEntries []map[time.Time]string) ([]*PaveFile, error) {
+	paveServerEntries []map[time.Time]string,
+	logCallback func(int, time.Time, string)) ([]*PaveFile, error) {
 
 	config.ReloadableFile.RLock()
 	defer config.ReloadableFile.RUnlock()
@@ -764,6 +765,10 @@ func (config *Config) Pave(
 						Name:     fileName,
 						Contents: boxedServerEntries,
 					})
+
+					if logCallback != nil {
+						logCallback(schemeIndex, oslTime, fileName)
+					}
 				}
 
 				oslTime = oslTime.Add(

psiphon/common/osl/osl_test.go

@@ -350,7 +350,8 @@ func TestOSL(t *testing.T) {
 				propagationChannelID,
 				signingPublicKey,
 				signingPrivateKey,
-				paveServerEntries)
+				paveServerEntries,
+				nil)
 			if err != nil {
 				t.Fatalf("Pave failed: %s", err)
 			}

psiphon/common/osl/paver/README.md

@@ -3,13 +3,12 @@
 Example usage:
 
 ```
-./paver -config osl_config.json -key signing_key.pem -count 3
+./paver -config osl_config.json -key signing_key.pem -offset -1h -period 2h
 ```
 
 * Paver is a tool that generates OSL files for paving.
 * Output is one directory for each propagation channel ID containing the files to upload to the appropriate campaign buckets.
 * Each output OSL is empty. Support for specifying and paving server entries is pending.
-* The example will pave 3 OSLs (e.g., OSLs for 3 time periods from epoch, where the time period is determined by the config) for each propagation channel ID.
+* The example will pave all OSLs, for each propagation channel ID, within a 2 hour period starting 1 hour ago.
   * `osl_config.json` is the OSL config in `psinet`.
   * `signing_key.pem` is `psinet._PsiphonNetwork__get_remote_server_list_signing_key_pair().pem_key_pair`.
-

psiphon/common/osl/paver/main.go

@@ -38,14 +38,11 @@ func main() {
 	var configFilename string
 	flag.StringVar(&configFilename, "config", "", "OSL configuration file")
 
-	var scheme int
-	flag.IntVar(&scheme, "scheme", 0, "scheme to pave")
+	var offset time.Duration
+	flag.DurationVar(&offset, "offset", 0, "pave OSL start time (offset from now)")
 
-	var oslOffset int
-	flag.IntVar(&oslOffset, "offset", 0, "OSL offset")
-
-	var oslCount int
-	flag.IntVar(&oslCount, "count", 1, "OSL count")
+	var period time.Duration
+	flag.DurationVar(&period, "period", 0, "pave OSL total period (starting from offset)")
 
 	var signingKeyPairFilename string
 	flag.StringVar(&signingKeyPairFilename, "key", "", "signing public key pair")
@@ -67,11 +64,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	if scheme < 0 || scheme >= len(config.Schemes) {
-		fmt.Printf("failed: invalid scheme\n")
-		os.Exit(1)
-	}
-
 	keyPairPEM, err := ioutil.ReadFile(signingKeyPairFilename)
 	if err != nil {
 		fmt.Printf("failed loading signing public key pair file: %s\n", err)
@@ -105,28 +97,57 @@ func main() {
 	signingPublicKey := base64.StdEncoding.EncodeToString(publicKeyBytes)
 	signingPrivateKey := base64.StdEncoding.EncodeToString(privateKeyBytes)
 
-	slokTimePeriodsPerOSL := 1
-	for _, keySplit := range config.Schemes[scheme].SeedPeriodKeySplits {
-		slokTimePeriodsPerOSL *= keySplit.Total
+	paveTime := time.Now().UTC()
+	startTime := paveTime.Add(offset)
+	endTime := startTime.Add(period)
+
+	schemeOSLTimePeriods := make(map[int]time.Duration)
+	for index, scheme := range config.Schemes {
+		slokTimePeriodsPerOSL := 1
+		for _, keySplit := range scheme.SeedPeriodKeySplits {
+			slokTimePeriodsPerOSL *= keySplit.Total
+		}
+		schemeOSLTimePeriods[index] =
+			time.Duration(scheme.SeedPeriodNanoseconds * int64(slokTimePeriodsPerOSL))
 	}
-	oslTimePeriod := time.Duration(config.Schemes[scheme].SeedPeriodNanoseconds * int64(slokTimePeriodsPerOSL))
 
-	for _, propagationChannelID := range config.Schemes[scheme].PropagationChannelIDs {
+	allPropagationChannelIDs := make(map[string][]int)
+	for index, scheme := range config.Schemes {
+		for _, propagationChannelID := range scheme.PropagationChannelIDs {
+			allPropagationChannelIDs[propagationChannelID] =
+				append(allPropagationChannelIDs[propagationChannelID], index)
+		}
+	}
+
+	for propagationChannelID, schemeIndexes := range allPropagationChannelIDs {
 
 		paveServerEntries := make([]map[time.Time]string, len(config.Schemes))
-		paveServerEntries[0] = make(map[time.Time]string)
 
-		epoch, _ := time.Parse(time.RFC3339, config.Schemes[scheme].Epoch)
-		for i := oslOffset; i < oslOffset+oslCount; i++ {
-			paveServerEntries[0][epoch.Add(time.Duration(i)*oslTimePeriod)] = ""
+		for _, index := range schemeIndexes {
+
+			paveServerEntries[index] = make(map[time.Time]string)
+
+			oslTime, _ := time.Parse(time.RFC3339, config.Schemes[index].Epoch)
+			for !oslTime.After(endTime) {
+				if !oslTime.Before(startTime) {
+					paveServerEntries[index][oslTime] = ""
+				}
+				oslTime = oslTime.Add(schemeOSLTimePeriods[index])
+			}
+
+			fmt.Printf("Paving propagation channel %s, scheme #%d, [%s - %s], %s\n",
+				propagationChannelID, index, startTime, endTime, schemeOSLTimePeriods[index])
 		}
 
 		paveFiles, err := config.Pave(
-			epoch.Add(time.Duration(oslOffset+oslCount)*oslTimePeriod),
+			endTime,
 			propagationChannelID,
 			signingPublicKey,
 			signingPrivateKey,
-			paveServerEntries)
+			paveServerEntries,
+			func(schemeIndex int, oslTime time.Time, fileName string) {
+				fmt.Printf("\tPaved scheme %d %s: %s\n", schemeIndex, oslTime, fileName)
+			})
 		if err != nil {
 			fmt.Printf("failed paving: %s\n", err)
 			os.Exit(1)