Stream OSL registry and files

- Rewrite client-side OSL registry and file processing
  to avoid loading all data in memory.

- Cached OSL registry is now a file, not a datastore
  record, as boltdb does not support record value
  streaming.

psiphon/common/authPackage.go

@@ -34,7 +34,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"os"
 	"sync"
 )
 
@@ -170,30 +169,15 @@ func ReadAuthenticatedDataPackage(
 	return authenticatedDataPackage.Data, nil
 }
 
-// StreamingReadAuthenticatedDataPackage extracts and verifies authenticated
+// NewAuthenticatedDataPackageReader extracts and verifies authenticated
 // data from an AuthenticatedDataPackage stored in the specified file. The
 // package must have been signed with the given key.
-// StreamingReadAuthenticatedDataPackage does not load the entire package nor
+// NewAuthenticatedDataPackageReader does not load the entire package nor
 // the entire data into memory. It streams the package while verifying, and
-// returns an io.ReadCloser that the caller may use to stream the authenticated
-// data payload. The caller _must_ close the io.Closer to free resources and
-// close the underlying file.
-func StreamingReadAuthenticatedDataPackage(
-	packageFileName string, signingPublicKey string) (io.ReadCloser, error) {
-
-	file, err := os.Open(packageFileName)
-	if err != nil {
-		return nil, ContextError(err)
-	}
-
-	closeOnError := file
-	defer func() {
-		if closeOnError != nil {
-			closeOnError.Close()
-		}
-	}()
-
-	var payload io.ReadCloser
+// returns an io.Reader that the caller may use to stream the authenticated
+// data payload.
+func NewAuthenticatedDataPackageReader(
+	dataPackage io.ReadSeeker, signingPublicKey string) (io.Reader, error) {
 
 	// The file is streamed in 2 passes. The first pass verifies the package
 	// signature. No payload data should be accepted/processed until the signature
@@ -203,18 +187,19 @@ func StreamingReadAuthenticatedDataPackage(
 	// Note: No exclusive file lock is held between passes, so it's possible to
 	// verify the data in one pass, and read different data in the second pass.
 	// For Psiphon's use cases, this will not happen in practise -- the packageFileName
-	// will not change while StreamingReadAuthenticatedDataPackage is running -- unless
-	// the client host is compromised; a compromised client host is outside of our threat
-	// model.
+	// will not change while the returned io.Reader is used -- unless the client host
+	// is compromised; a compromised client host is outside of our threat model.
+
+	var payload io.Reader
 
 	for pass := 0; pass < 2; pass++ {
 
-		_, err = file.Seek(0, 0)
+		_, err := dataPackage.Seek(0, io.SeekStart)
 		if err != nil {
 			return nil, ContextError(err)
 		}
 
-		decompressor, err := zlib.NewReader(file)
+		decompressor, err := zlib.NewReader(dataPackage)
 		if err != nil {
 			return nil, ContextError(err)
 		}
@@ -328,18 +313,10 @@ func StreamingReadAuthenticatedDataPackage(
 				return nil, ContextError(errors.New("missing expected field"))
 			}
 
-			payload = struct {
-				io.Reader
-				io.Closer
-			}{
-				jsonData,
-				file,
-			}
+			payload = jsonData
 		}
 	}
 
-	closeOnError = nil
-
 	return payload, nil
 }
 

psiphon/common/crypto/nacl/secretbox/secretbox_reader.go

@@ -0,0 +1,507 @@
+/*
+ * Copyright (c) 2017, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package secretbox // import "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/crypto/nacl/secretbox"
+
+import (
+	"crypto/subtle"
+	"encoding/binary"
+	"fmt"
+	"io"
+
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/crypto/poly1305"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/crypto/salsa20/salsa"
+)
+
+// NewOpenReadSeeker is a streaming variant of Open.
+//
+// NewOpenReadSeeker is intended only for use in Psiphon with a payload that is
+// independently authenticated; and consideration has been given only for client-side
+// operation. Non-optimized reference implementation poly1305 and salsa20 code is used.
+//
+// The box is accessed through an io.ReadSeeker, which allows for an initial
+// poly1305 verification pass followed by a payload decryption pass, both
+// without loading the entire box into memory. As such, this implementation
+// should not be subject to the use-before-authentication or truncation attacks
+// discussed here:
+// https://github.com/golang/crypto/commit/9ba3862cf6a5452ae579de98f9364dd2e544844c#diff-9a969aca62172940631ad143523794ee
+// https://github.com/golang/go/issues/17673#issuecomment-275732868
+func NewOpenReadSeeker(box io.ReadSeeker, nonce *[24]byte, key *[32]byte) (io.ReadSeeker, error) {
+
+	r := &salsa20ReadSeeker{
+		box:   box,
+		nonce: *nonce,
+		key:   *key,
+	}
+
+	err := r.reset()
+	if err != nil {
+		return nil, err
+	}
+
+	return r, nil
+}
+
+type salsa20ReadSeeker struct {
+	box         io.ReadSeeker
+	nonce       [24]byte
+	key         [32]byte
+	subKey      [32]byte
+	counter     [16]byte
+	block       [64]byte
+	blockOffset int
+}
+
+// Open x/crypto/nacl/secretbox/secretbox.go, adapted to streaming and rewinding.
+func (r *salsa20ReadSeeker) reset() error {
+
+	// See comments in Open
+
+	_, err := r.box.Seek(0, io.SeekStart)
+	if err != nil {
+		return fmt.Errorf("initial seek failed: %s", err)
+	}
+
+	var tag [poly1305.TagSize]byte
+
+	_, err = io.ReadFull(r.box, tag[:])
+	if err != nil {
+		return fmt.Errorf("read tag failed: %s", err)
+	}
+
+	var subKey [32]byte
+	var counter [16]byte
+	setup(&subKey, &counter, &r.nonce, &r.key)
+
+	// The Poly1305 key is generated by encrypting 32 bytes of zeros. Since
+	// Salsa20 works with 64-byte blocks, we also generate 32 bytes of
+	// keystream as a side effect.
+	var firstBlock [64]byte
+	salsa.XORKeyStream(firstBlock[:], firstBlock[:], &counter, &subKey)
+
+	var poly1305Key [32]byte
+	copy(poly1305Key[:], firstBlock[:])
+
+	err = poly1305VerifyReader(&tag, r.box, &poly1305Key)
+	if err != nil {
+		return err
+	}
+
+	_, err = r.box.Seek(int64(len(tag)), io.SeekStart)
+	if err != nil {
+		return fmt.Errorf("rewind seek failed: %s", err)
+	}
+
+	counter[8] = 1
+
+	r.subKey = subKey
+	r.counter = counter
+
+	// We XOR up to 32 bytes of box with the keystream generated from
+	// the first block.
+
+	r.block = firstBlock
+	r.blockOffset = 32
+
+	return nil
+}
+
+func (r *salsa20ReadSeeker) Read(p []byte) (int, error) {
+
+	n, err := r.box.Read(p)
+
+	for i := 0; i < n; i++ {
+		if r.blockOffset == 64 {
+			salsa20Core(&r.block, &r.counter, &r.subKey, &salsa.Sigma)
+
+			u := uint32(1)
+			for i := 8; i < 16; i++ {
+				u += uint32(r.counter[i])
+				r.counter[i] = byte(u)
+				u >>= 8
+			}
+			r.blockOffset = 0
+		}
+		p[i] = p[i] ^ r.block[r.blockOffset]
+		r.blockOffset++
+	}
+
+	return n, err
+}
+
+func (r *salsa20ReadSeeker) Seek(offset int64, whence int) (int64, error) {
+
+	// Currently only supports Seek(0, io.SeekStart) as required for Psiphon.
+
+	if offset != 0 || whence != io.SeekStart {
+		return -1, fmt.Errorf("unsupported")
+	}
+
+	// TODO: could skip poly1305 verify after 1st reset.
+
+	err := r.reset()
+	if err != nil {
+		return -1, err
+	}
+
+	return 0, nil
+}
+
+// Verify from crypto/poly1305/poly1305.go, modifed to use an io.Reader.
+func poly1305VerifyReader(mac *[16]byte, m io.Reader, key *[32]byte) error {
+	var tmp [16]byte
+	err := poly1305SumReader(&tmp, m, key)
+	if err != nil {
+		return err
+	}
+	if subtle.ConstantTimeCompare(tmp[:], mac[:]) != 1 {
+		return fmt.Errorf("verify failed")
+	}
+	return nil
+}
+
+// Sum from crypto/poly1305/sum_ref.go, modifed to use an io.Reader.
+func poly1305SumReader(out *[poly1305.TagSize]byte, msg io.Reader, key *[32]byte) error {
+	var (
+		h0, h1, h2, h3, h4 uint32 // the hash accumulators
+		r0, r1, r2, r3, r4 uint64 // the r part of the key
+	)
+
+	r0 = uint64(binary.LittleEndian.Uint32(key[0:]) & 0x3ffffff)
+	r1 = uint64((binary.LittleEndian.Uint32(key[3:]) >> 2) & 0x3ffff03)
+	r2 = uint64((binary.LittleEndian.Uint32(key[6:]) >> 4) & 0x3ffc0ff)
+	r3 = uint64((binary.LittleEndian.Uint32(key[9:]) >> 6) & 0x3f03fff)
+	r4 = uint64((binary.LittleEndian.Uint32(key[12:]) >> 8) & 0x00fffff)
+
+	R1, R2, R3, R4 := r1*5, r2*5, r3*5, r4*5
+
+	var in [poly1305.TagSize]byte
+
+	for {
+		n, err := msg.Read(in[:])
+
+		if n == poly1305.TagSize {
+
+			// h += msg
+			h0 += binary.LittleEndian.Uint32(in[0:]) & 0x3ffffff
+			h1 += (binary.LittleEndian.Uint32(in[3:]) >> 2) & 0x3ffffff
+			h2 += (binary.LittleEndian.Uint32(in[6:]) >> 4) & 0x3ffffff
+			h3 += (binary.LittleEndian.Uint32(in[9:]) >> 6) & 0x3ffffff
+			h4 += (binary.LittleEndian.Uint32(in[12:]) >> 8) | (1 << 24)
+
+		} else if n > 0 {
+
+			in[n] = 0x01
+			for i := n + 1; i < poly1305.TagSize; i++ {
+				in[i] = 0
+			}
+
+			// h += msg
+			h0 += binary.LittleEndian.Uint32(in[0:]) & 0x3ffffff
+			h1 += (binary.LittleEndian.Uint32(in[3:]) >> 2) & 0x3ffffff
+			h2 += (binary.LittleEndian.Uint32(in[6:]) >> 4) & 0x3ffffff
+			h3 += (binary.LittleEndian.Uint32(in[9:]) >> 6) & 0x3ffffff
+			h4 += (binary.LittleEndian.Uint32(in[12:]) >> 8)
+		}
+
+		if n > 0 {
+
+			// h *= r
+			d0 := (uint64(h0) * r0) + (uint64(h1) * R4) + (uint64(h2) * R3) + (uint64(h3) * R2) + (uint64(h4) * R1)
+			d1 := (d0 >> 26) + (uint64(h0) * r1) + (uint64(h1) * r0) + (uint64(h2) * R4) + (uint64(h3) * R3) + (uint64(h4) * R2)
+			d2 := (d1 >> 26) + (uint64(h0) * r2) + (uint64(h1) * r1) + (uint64(h2) * r0) + (uint64(h3) * R4) + (uint64(h4) * R3)
+			d3 := (d2 >> 26) + (uint64(h0) * r3) + (uint64(h1) * r2) + (uint64(h2) * r1) + (uint64(h3) * r0) + (uint64(h4) * R4)
+			d4 := (d3 >> 26) + (uint64(h0) * r4) + (uint64(h1) * r3) + (uint64(h2) * r2) + (uint64(h3) * r1) + (uint64(h4) * r0)
+
+			// h %= p
+			h0 = uint32(d0) & 0x3ffffff
+			h1 = uint32(d1) & 0x3ffffff
+			h2 = uint32(d2) & 0x3ffffff
+			h3 = uint32(d3) & 0x3ffffff
+			h4 = uint32(d4) & 0x3ffffff
+
+			h0 += uint32(d4>>26) * 5
+			h1 += h0 >> 26
+			h0 = h0 & 0x3ffffff
+		}
+
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			return err
+		}
+	}
+
+	// h %= p reduction
+	h2 += h1 >> 26
+	h1 &= 0x3ffffff
+	h3 += h2 >> 26
+	h2 &= 0x3ffffff
+	h4 += h3 >> 26
+	h3 &= 0x3ffffff
+	h0 += 5 * (h4 >> 26)
+	h4 &= 0x3ffffff
+	h1 += h0 >> 26
+	h0 &= 0x3ffffff
+
+	// h - p
+	t0 := h0 + 5
+	t1 := h1 + (t0 >> 26)
+	t2 := h2 + (t1 >> 26)
+	t3 := h3 + (t2 >> 26)
+	t4 := h4 + (t3 >> 26) - (1 << 26)
+	t0 &= 0x3ffffff
+	t1 &= 0x3ffffff
+	t2 &= 0x3ffffff
+	t3 &= 0x3ffffff
+
+	// select h if h < p else h - p
+	t_mask := (t4 >> 31) - 1
+	h_mask := ^t_mask
+	h0 = (h0 & h_mask) | (t0 & t_mask)
+	h1 = (h1 & h_mask) | (t1 & t_mask)
+	h2 = (h2 & h_mask) | (t2 & t_mask)
+	h3 = (h3 & h_mask) | (t3 & t_mask)
+	h4 = (h4 & h_mask) | (t4 & t_mask)
+
+	// h %= 2^128
+	h0 |= h1 << 26
+	h1 = ((h1 >> 6) | (h2 << 20))
+	h2 = ((h2 >> 12) | (h3 << 14))
+	h3 = ((h3 >> 18) | (h4 << 8))
+
+	// s: the s part of the key
+	// tag = (h + s) % (2^128)
+	t := uint64(h0) + uint64(binary.LittleEndian.Uint32(key[16:]))
+	h0 = uint32(t)
+	t = uint64(h1) + uint64(binary.LittleEndian.Uint32(key[20:])) + (t >> 32)
+	h1 = uint32(t)
+	t = uint64(h2) + uint64(binary.LittleEndian.Uint32(key[24:])) + (t >> 32)
+	h2 = uint32(t)
+	t = uint64(h3) + uint64(binary.LittleEndian.Uint32(key[28:])) + (t >> 32)
+	h3 = uint32(t)
+
+	binary.LittleEndian.PutUint32(out[0:], h0)
+	binary.LittleEndian.PutUint32(out[4:], h1)
+	binary.LittleEndian.PutUint32(out[8:], h2)
+	binary.LittleEndian.PutUint32(out[12:], h3)
+
+	return nil
+}
+
+// core from x/crypto/salsa20/salsa/salsa20_ref.go.
+func salsa20Core(out *[64]byte, in *[16]byte, k *[32]byte, c *[16]byte) {
+	j0 := uint32(c[0]) | uint32(c[1])<<8 | uint32(c[2])<<16 | uint32(c[3])<<24
+	j1 := uint32(k[0]) | uint32(k[1])<<8 | uint32(k[2])<<16 | uint32(k[3])<<24
+	j2 := uint32(k[4]) | uint32(k[5])<<8 | uint32(k[6])<<16 | uint32(k[7])<<24
+	j3 := uint32(k[8]) | uint32(k[9])<<8 | uint32(k[10])<<16 | uint32(k[11])<<24
+	j4 := uint32(k[12]) | uint32(k[13])<<8 | uint32(k[14])<<16 | uint32(k[15])<<24
+	j5 := uint32(c[4]) | uint32(c[5])<<8 | uint32(c[6])<<16 | uint32(c[7])<<24
+	j6 := uint32(in[0]) | uint32(in[1])<<8 | uint32(in[2])<<16 | uint32(in[3])<<24
+	j7 := uint32(in[4]) | uint32(in[5])<<8 | uint32(in[6])<<16 | uint32(in[7])<<24
+	j8 := uint32(in[8]) | uint32(in[9])<<8 | uint32(in[10])<<16 | uint32(in[11])<<24
+	j9 := uint32(in[12]) | uint32(in[13])<<8 | uint32(in[14])<<16 | uint32(in[15])<<24
+	j10 := uint32(c[8]) | uint32(c[9])<<8 | uint32(c[10])<<16 | uint32(c[11])<<24
+	j11 := uint32(k[16]) | uint32(k[17])<<8 | uint32(k[18])<<16 | uint32(k[19])<<24
+	j12 := uint32(k[20]) | uint32(k[21])<<8 | uint32(k[22])<<16 | uint32(k[23])<<24
+	j13 := uint32(k[24]) | uint32(k[25])<<8 | uint32(k[26])<<16 | uint32(k[27])<<24
+	j14 := uint32(k[28]) | uint32(k[29])<<8 | uint32(k[30])<<16 | uint32(k[31])<<24
+	j15 := uint32(c[12]) | uint32(c[13])<<8 | uint32(c[14])<<16 | uint32(c[15])<<24
+
+	x0, x1, x2, x3, x4, x5, x6, x7, x8 := j0, j1, j2, j3, j4, j5, j6, j7, j8
+	x9, x10, x11, x12, x13, x14, x15 := j9, j10, j11, j12, j13, j14, j15
+
+	const rounds = 20
+
+	for i := 0; i < rounds; i += 2 {
+		u := x0 + x12
+		x4 ^= u<<7 | u>>(32-7)
+		u = x4 + x0
+		x8 ^= u<<9 | u>>(32-9)
+		u = x8 + x4
+		x12 ^= u<<13 | u>>(32-13)
+		u = x12 + x8
+		x0 ^= u<<18 | u>>(32-18)
+
+		u = x5 + x1
+		x9 ^= u<<7 | u>>(32-7)
+		u = x9 + x5
+		x13 ^= u<<9 | u>>(32-9)
+		u = x13 + x9
+		x1 ^= u<<13 | u>>(32-13)
+		u = x1 + x13
+		x5 ^= u<<18 | u>>(32-18)
+
+		u = x10 + x6
+		x14 ^= u<<7 | u>>(32-7)
+		u = x14 + x10
+		x2 ^= u<<9 | u>>(32-9)
+		u = x2 + x14
+		x6 ^= u<<13 | u>>(32-13)
+		u = x6 + x2
+		x10 ^= u<<18 | u>>(32-18)
+
+		u = x15 + x11
+		x3 ^= u<<7 | u>>(32-7)
+		u = x3 + x15
+		x7 ^= u<<9 | u>>(32-9)
+		u = x7 + x3
+		x11 ^= u<<13 | u>>(32-13)
+		u = x11 + x7
+		x15 ^= u<<18 | u>>(32-18)
+
+		u = x0 + x3
+		x1 ^= u<<7 | u>>(32-7)
+		u = x1 + x0
+		x2 ^= u<<9 | u>>(32-9)
+		u = x2 + x1
+		x3 ^= u<<13 | u>>(32-13)
+		u = x3 + x2
+		x0 ^= u<<18 | u>>(32-18)
+
+		u = x5 + x4
+		x6 ^= u<<7 | u>>(32-7)
+		u = x6 + x5
+		x7 ^= u<<9 | u>>(32-9)
+		u = x7 + x6
+		x4 ^= u<<13 | u>>(32-13)
+		u = x4 + x7
+		x5 ^= u<<18 | u>>(32-18)
+
+		u = x10 + x9
+		x11 ^= u<<7 | u>>(32-7)
+		u = x11 + x10
+		x8 ^= u<<9 | u>>(32-9)
+		u = x8 + x11
+		x9 ^= u<<13 | u>>(32-13)
+		u = x9 + x8
+		x10 ^= u<<18 | u>>(32-18)
+
+		u = x15 + x14
+		x12 ^= u<<7 | u>>(32-7)
+		u = x12 + x15
+		x13 ^= u<<9 | u>>(32-9)
+		u = x13 + x12
+		x14 ^= u<<13 | u>>(32-13)
+		u = x14 + x13
+		x15 ^= u<<18 | u>>(32-18)
+	}
+	x0 += j0
+	x1 += j1
+	x2 += j2
+	x3 += j3
+	x4 += j4
+	x5 += j5
+	x6 += j6
+	x7 += j7
+	x8 += j8
+	x9 += j9
+	x10 += j10
+	x11 += j11
+	x12 += j12
+	x13 += j13
+	x14 += j14
+	x15 += j15
+
+	out[0] = byte(x0)
+	out[1] = byte(x0 >> 8)
+	out[2] = byte(x0 >> 16)
+	out[3] = byte(x0 >> 24)
+
+	out[4] = byte(x1)
+	out[5] = byte(x1 >> 8)
+	out[6] = byte(x1 >> 16)
+	out[7] = byte(x1 >> 24)
+
+	out[8] = byte(x2)
+	out[9] = byte(x2 >> 8)
+	out[10] = byte(x2 >> 16)
+	out[11] = byte(x2 >> 24)
+
+	out[12] = byte(x3)
+	out[13] = byte(x3 >> 8)
+	out[14] = byte(x3 >> 16)
+	out[15] = byte(x3 >> 24)
+
+	out[16] = byte(x4)
+	out[17] = byte(x4 >> 8)
+	out[18] = byte(x4 >> 16)
+	out[19] = byte(x4 >> 24)
+
+	out[20] = byte(x5)
+	out[21] = byte(x5 >> 8)
+	out[22] = byte(x5 >> 16)
+	out[23] = byte(x5 >> 24)
+
+	out[24] = byte(x6)
+	out[25] = byte(x6 >> 8)
+	out[26] = byte(x6 >> 16)
+	out[27] = byte(x6 >> 24)
+
+	out[28] = byte(x7)
+	out[29] = byte(x7 >> 8)
+	out[30] = byte(x7 >> 16)
+	out[31] = byte(x7 >> 24)
+
+	out[32] = byte(x8)
+	out[33] = byte(x8 >> 8)
+	out[34] = byte(x8 >> 16)
+	out[35] = byte(x8 >> 24)
+
+	out[36] = byte(x9)
+	out[37] = byte(x9 >> 8)
+	out[38] = byte(x9 >> 16)
+	out[39] = byte(x9 >> 24)
+
+	out[40] = byte(x10)
+	out[41] = byte(x10 >> 8)
+	out[42] = byte(x10 >> 16)
+	out[43] = byte(x10 >> 24)
+
+	out[44] = byte(x11)
+	out[45] = byte(x11 >> 8)
+	out[46] = byte(x11 >> 16)
+	out[47] = byte(x11 >> 24)
+
+	out[48] = byte(x12)
+	out[49] = byte(x12 >> 8)
+	out[50] = byte(x12 >> 16)
+	out[51] = byte(x12 >> 24)
+
+	out[52] = byte(x13)
+	out[53] = byte(x13 >> 8)
+	out[54] = byte(x13 >> 16)
+	out[55] = byte(x13 >> 24)
+
+	out[56] = byte(x14)
+	out[57] = byte(x14 >> 8)
+	out[58] = byte(x14 >> 16)
+	out[59] = byte(x14 >> 24)
+
+	out[60] = byte(x15)
+	out[61] = byte(x15 >> 8)
+	out[62] = byte(x15 >> 16)
+	out[63] = byte(x15 >> 24)
+}

psiphon/common/osl/osl.go

@@ -39,6 +39,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"net/url"
 	"path"
@@ -727,10 +728,6 @@ type PaveFile struct {
 // Registry describes a set of OSL files.
 type Registry struct {
 	FileSpecs []*OSLFileSpec
-
-	// The following fields are ephemeral state.
-
-	oslIDLookup map[string]*OSLFileSpec
 }
 
 // An OSLFileSpec includes an ID which is used to reference the
@@ -739,7 +736,7 @@ type Registry struct {
 //
 // The MD5Sum field is a checksum of the contents of the OSL file
 // to be used to skip redownloading previously downloaded files.
-// MD5 is not cryptogrpahically secure and this checksum is not
+// MD5 is not cryptographically secure and this checksum is not
 // relied upon for OSL verification. MD5 is used for compatibility
 // with out-of-band distribution hosts.
 type OSLFileSpec struct {
@@ -915,7 +912,7 @@ func makeOSLFileSpec(
 	// is derived from the master key and OSL ID. This deterministic
 	// derivation ensures that repeated paves of the same OSL
 	// with the same ID and same content yields the same MD5Sum
-	// to avoid wastful downloads.
+	// to avoid wasteful downloads.
 
 	fileKey := deriveKeyHKDF(
 		scheme.MasterKey,
@@ -1054,6 +1051,72 @@ func divideKeyWithSeedSpecSLOKs(
 	}, nil
 }
 
+// reassembleKey recursively traverses a KeyShares tree, determining
+// whether there exists suffient SLOKs to reassemble the root key and
+// performing the key assembly as required.
+func (keyShares *KeyShares) reassembleKey(lookup SLOKLookup, unboxKey bool) (bool, []byte, error) {
+
+	if (len(keyShares.SLOKIDs) > 0 && len(keyShares.KeyShares) > 0) ||
+		(len(keyShares.SLOKIDs) > 0 && len(keyShares.SLOKIDs) != len(keyShares.BoxedShares)) ||
+		(len(keyShares.KeyShares) > 0 && len(keyShares.KeyShares) != len(keyShares.BoxedShares)) {
+		return false, nil, common.ContextError(errors.New("unexpected KeyShares format"))
+	}
+
+	shareCount := 0
+	var shares [][]byte
+	if unboxKey {
+		// Note: shamirCombine infers share indices from slice offset, so the full
+		// keyShares.Total slots are allocated and missing shares are left nil.
+		shares = make([][]byte, len(keyShares.BoxedShares))
+	}
+	if len(keyShares.SLOKIDs) > 0 {
+		for i := 0; i < len(keyShares.SLOKIDs) && shareCount < keyShares.Threshold; i++ {
+			slokKey := lookup(keyShares.SLOKIDs[i])
+			if slokKey == nil {
+				continue
+			}
+			shareCount += 1
+			if unboxKey {
+				share, err := unbox(slokKey, keyShares.BoxedShares[i])
+				if err != nil {
+					return false, nil, common.ContextError(err)
+				}
+				shares[i] = share
+			}
+		}
+	} else {
+		for i := 0; i < len(keyShares.KeyShares) && shareCount < keyShares.Threshold; i++ {
+			ok, key, err := keyShares.KeyShares[i].reassembleKey(lookup, unboxKey)
+			if err != nil {
+				return false, nil, common.ContextError(err)
+			}
+			if !ok {
+				continue
+			}
+			shareCount += 1
+			if unboxKey {
+				share, err := unbox(key, keyShares.BoxedShares[i])
+				if err != nil {
+					return false, nil, common.ContextError(err)
+				}
+				shares[i] = share
+			}
+		}
+	}
+
+	if shareCount < keyShares.Threshold {
+		return false, nil, nil
+	}
+
+	if !unboxKey {
+		return true, nil, nil
+	}
+
+	joinedKey := shamirCombine(shares)
+
+	return true, joinedKey, nil
+}
+
 // GetOSLRegistryURL returns the URL for an OSL registry. Clients
 // call this when fetching the registry from out-of-band
 // distribution sites.
@@ -1096,194 +1159,169 @@ func GetOSLFilename(baseDirectory string, oslID []byte) string {
 		baseDirectory, fmt.Sprintf(OSL_FILENAME_FORMAT, hex.EncodeToString(oslID)))
 }
 
-// UnpackRegistry validates and loads a JSON encoded OSL registry.
-func UnpackRegistry(
-	registryPackage []byte, signingPublicKey string) (*Registry, []byte, error) {
-
-	encodedRegistry, err := common.ReadAuthenticatedDataPackage(
-		registryPackage, true, signingPublicKey)
-	if err != nil {
-		return nil, nil, common.ContextError(err)
-	}
-
-	registryJSON, err := base64.StdEncoding.DecodeString(encodedRegistry)
-	if err != nil {
-		return nil, nil, common.ContextError(err)
-	}
+// SLOKLookup is a callback to lookup SLOK keys by ID.
+type SLOKLookup func([]byte) []byte
 
-	registry, err := LoadRegistry(registryJSON)
-	return registry, registryJSON, err
+// RegistryStreamer authenticates and processes a JSON encoded OSL registry.
+// The streamer processes the registry without loading the entire file
+// into memory, parsing each OSL file spec in turn and returning those
+// OSL file specs for which the client has sufficient SLOKs to reassemble
+// the OSL key and decrypt.
+//
+// At this stage, SLOK reassembly simply does SLOK ID lookups and threshold
+// counting and does not derive keys for every OSL. This allows the client
+// to defer key derivation until NewOSLReader for cases where it has not
+// already imported the OSL.
+//
+// The client's propagation channel ID is used implicitly: it determines the
+// base URL used to download the registry and OSL files. If the client has
+// seeded SLOKs from a propagation channel ID different than the one associated
+// with its present base URL, they will not appear in the registry and not
+// be used.
+type RegistryStreamer struct {
+	jsonDecoder *json.Decoder
+	lookup      SLOKLookup
 }
 
-// LoadRegistry loads a JSON encoded OSL registry.
-// Clients call this to process downloaded registry files.
-func LoadRegistry(registryJSON []byte) (*Registry, error) {
+// NewRegistryStreamer creates a new RegistryStreamer.
+func NewRegistryStreamer(
+	registryFileContent io.ReadSeeker,
+	signingPublicKey string,
+	lookup SLOKLookup) (*RegistryStreamer, error) {
 
-	var registry Registry
-	err := json.Unmarshal(registryJSON, &registry)
+	payloadReader, err := common.NewAuthenticatedDataPackageReader(
+		registryFileContent, signingPublicKey)
 	if err != nil {
 		return nil, common.ContextError(err)
 	}
 
-	registry.oslIDLookup = make(map[string]*OSLFileSpec)
-	for _, fileSpec := range registry.FileSpecs {
-		registry.oslIDLookup[string(fileSpec.ID)] = fileSpec
-	}
+	base64Decoder := base64.NewDecoder(base64.StdEncoding, payloadReader)
 
-	return &registry, nil
-}
+	// A json.Decoder is used to stream the JSON payload, which
+	// is expected to be of the following form, corresponding
+	// to the Registry struct type:
+	//
+	// {"FileSpecs" : [{...}, {...}, ..., {...}]}
 
-// SLOKLookup is a callback to lookup SLOK keys by ID.
-type SLOKLookup func([]byte) []byte
+	jsonDecoder := json.NewDecoder(base64Decoder)
 
-// GetSeededOSLIDs examines each OSL in the registry and returns a list for
-// which the client has sufficient SLOKs to reassemble the OSL key and
-// decrypt. This function simply does SLOK ID lookups and threshold counting
-// and does not derive keys for every OSL.
-// The client is responsible for using the resulting list of OSL IDs to fetch
-// the OSL files and process.
-//
-// The client's propagation channel ID is used implicitly: it determines the
-// base URL used to download the registry and OSL files. If the client has
-// seeded SLOKs from a propagation channel ID different than the one associated
-// with its present base URL, they will not appear in the registry and not
-// be used.
-//
-// SLOKLookup is called to determine which SLOKs are seeded with the client.
-// errorLogger is a callback to log errors; GetSeededOSLIDs will continue to
-// process each candidate OSL even in the case of an error processing a
-// particular one.
-func (registry *Registry) GetSeededOSLIDs(lookup SLOKLookup, errorLogger func(error)) [][]byte {
-
-	var OSLIDs [][]byte
-	for _, fileSpec := range registry.FileSpecs {
-		ok, _, err := fileSpec.KeyShares.reassembleKey(lookup, false)
-		if err != nil {
-			errorLogger(err)
-			continue
-		}
-		if ok {
-			OSLIDs = append(OSLIDs, fileSpec.ID)
-		}
+	err = expectJSONDelimiter(jsonDecoder, "{")
+	if err != nil {
+		return nil, common.ContextError(err)
 	}
 
-	return OSLIDs
-}
-
-// GetOSLMD5Sum returns the MD5 checksum for the specified OSL.
-func (registry *Registry) GetOSLMD5Sum(oslID []byte) ([]byte, error) {
+	token, err := jsonDecoder.Token()
+	if err != nil {
+		return nil, common.ContextError(err)
+	}
+	if name, ok := token.(string); !ok || name != "FileSpecs" {
+		return nil, common.ContextError(
+			fmt.Errorf("unexpected name: %s", name))
+	}
 
-	fileSpec, ok := registry.oslIDLookup[string(oslID)]
-	if !ok {
-		return nil, common.ContextError(errors.New("unknown OSL ID"))
+	err = expectJSONDelimiter(jsonDecoder, "[")
+	if err != nil {
+		return nil, common.ContextError(err)
 	}
 
-	return fileSpec.MD5Sum, nil
+	return &RegistryStreamer{
+		jsonDecoder: jsonDecoder,
+		lookup:      lookup,
+	}, nil
 }
 
-// reassembleKey recursively traverses a KeyShares tree, determining
-// whether there exists suffient SLOKs to reassemble the root key and
-// performing the key assembly as required.
-func (keyShares *KeyShares) reassembleKey(lookup SLOKLookup, unboxKey bool) (bool, []byte, error) {
+// Next returns the next OSL file spec that the client
+// has sufficient SLOKs to decrypt. The client calls
+// NewOSLReader with the file spec to process that OSL.
+// Next returns nil at EOF.
+func (s *RegistryStreamer) Next() (*OSLFileSpec, error) {
 
-	if (len(keyShares.SLOKIDs) > 0 && len(keyShares.KeyShares) > 0) ||
-		(len(keyShares.SLOKIDs) > 0 && len(keyShares.SLOKIDs) != len(keyShares.BoxedShares)) ||
-		(len(keyShares.KeyShares) > 0 && len(keyShares.KeyShares) != len(keyShares.BoxedShares)) {
-		return false, nil, common.ContextError(errors.New("unexpected KeyShares format"))
-	}
+	for {
+		if s.jsonDecoder.More() {
 
-	shareCount := 0
-	var shares [][]byte
-	if unboxKey {
-		// Note: shamirCombine infers share indices from slice offset, so the full
-		// keyShares.Total slots are allocated and missing shares are left nil.
-		shares = make([][]byte, len(keyShares.BoxedShares))
-	}
-	if len(keyShares.SLOKIDs) > 0 {
-		for i := 0; i < len(keyShares.SLOKIDs) && shareCount < keyShares.Threshold; i++ {
-			slokKey := lookup(keyShares.SLOKIDs[i])
-			if slokKey == nil {
-				continue
+			var fileSpec OSLFileSpec
+			err := s.jsonDecoder.Decode(&fileSpec)
+			if err != nil {
+				return nil, common.ContextError(err)
 			}
-			shareCount += 1
-			if unboxKey {
-				share, err := unbox(slokKey, keyShares.BoxedShares[i])
-				if err != nil {
-					return false, nil, common.ContextError(err)
-				}
-				shares[i] = share
+
+			ok, _, err := fileSpec.KeyShares.reassembleKey(s.lookup, false)
+			if err != nil {
+				return nil, common.ContextError(err)
 			}
-		}
-	} else {
-		for i := 0; i < len(keyShares.KeyShares) && shareCount < keyShares.Threshold; i++ {
-			ok, key, err := keyShares.KeyShares[i].reassembleKey(lookup, unboxKey)
+
+			if ok {
+				return &fileSpec, nil
+			}
+
+		} else {
+
+			// Expect the end of the FileSpecs array.
+			err := expectJSONDelimiter(s.jsonDecoder, "]")
 			if err != nil {
-				return false, nil, common.ContextError(err)
+				return nil, common.ContextError(err)
 			}
-			if !ok {
-				continue
+
+			// Expect the end of the Registry object.
+			err = expectJSONDelimiter(s.jsonDecoder, "}")
+			if err != nil {
+				return nil, common.ContextError(err)
 			}
-			shareCount += 1
-			if unboxKey {
-				share, err := unbox(key, keyShares.BoxedShares[i])
-				if err != nil {
-					return false, nil, common.ContextError(err)
-				}
-				shares[i] = share
+
+			// Expect the end of the registry content.
+			_, err = s.jsonDecoder.Token()
+			if err != io.EOF {
+				return nil, common.ContextError(err)
 			}
+
+			return nil, nil
 		}
 	}
+}
 
-	if shareCount < keyShares.Threshold {
-		return false, nil, nil
+func expectJSONDelimiter(jsonDecoder *json.Decoder, delimiter string) error {
+	token, err := jsonDecoder.Token()
+	if err != nil {
+		return common.ContextError(err)
 	}
-
-	if !unboxKey {
-		return true, nil, nil
+	if delim, ok := token.(json.Delim); !ok || delim.String() != delimiter {
+		return common.ContextError(
+			fmt.Errorf("unexpected delimiter: %s", delim.String()))
 	}
-
-	joinedKey := shamirCombine(shares)
-
-	return true, joinedKey, nil
+	return nil
 }
 
-// UnpackOSL reassembles the key for the OSL specified by oslID and uses
-// that key to decrypt oslFileContents, validate the authenticated package,
-// and extract the payload.
-// Clients will call UnpackOSL for OSLs indicated by GetSeededOSLIDs along
-// with their downloaded content.
-// SLOKLookup is called to determine which SLOKs are seeded with the client.
-func (registry *Registry) UnpackOSL(
+// NewOSLReader decrypts, authenticates and streams an OSL payload.
+func NewOSLReader(
+	oslFileContent io.ReadSeeker,
+	fileSpec *OSLFileSpec,
 	lookup SLOKLookup,
-	oslID []byte,
-	oslFileContents []byte,
-	signingPublicKey string) (string, error) {
-
-	fileSpec, ok := registry.oslIDLookup[string(oslID)]
-	if !ok {
-		return "", common.ContextError(errors.New("unknown OSL ID"))
-	}
+	signingPublicKey string) (io.Reader, error) {
 
 	ok, fileKey, err := fileSpec.KeyShares.reassembleKey(lookup, true)
 	if err != nil {
-		return "", common.ContextError(err)
+		return nil, common.ContextError(err)
 	}
 	if !ok {
-		return "", common.ContextError(errors.New("unseeded OSL"))
+		return nil, common.ContextError(errors.New("unseeded OSL"))
 	}
 
-	dataPackage, err := unbox(fileKey, oslFileContents)
-	if err != nil {
-		return "", common.ContextError(err)
+	if len(fileKey) != 32 {
+		return nil, common.ContextError(errors.New("invalid key length"))
 	}
 
-	oslPayload, err := common.ReadAuthenticatedDataPackage(
-		dataPackage, true, signingPublicKey)
+	var nonce [24]byte
+	var key [32]byte
+	copy(key[:], fileKey)
+
+	unboxer, err := secretbox.NewOpenReadSeeker(oslFileContent, &nonce, &key)
 	if err != nil {
-		return "", common.ContextError(err)
+		return nil, common.ContextError(err)
 	}
 
-	return oslPayload, nil
+	return common.NewAuthenticatedDataPackageReader(
+		unboxer,
+		signingPublicKey)
 }
 
 // deriveKeyHKDF implements HKDF-Expand as defined in https://tools.ietf.org/html/rfc5869
@@ -1355,7 +1393,7 @@ func shamirCombine(shares [][]byte) []byte {
 }
 
 // box is a helper wrapper for secretbox.Seal.
-// A constant  nonce is used, which is secure so long as
+// A constant nonce is used, which is secure so long as
 // each key is used to encrypt only one message.
 func box(key, plaintext []byte) ([]byte, error) {
 	if len(key) != 32 {

psiphon/common/osl/osl_test.go

@@ -20,9 +20,11 @@
 package osl
 
 import (
+	"bytes"
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"testing"
 	"time"
@@ -515,54 +517,69 @@ func TestOSL(t *testing.T) {
 				}
 			}
 
-			t.Logf("SLOK count: %d", len(slokMap))
+			startTime := time.Now()
 
-			slokLookup := func(slokID []byte) []byte {
+			lookupSLOKs := func(slokID []byte) []byte {
 				return slokMap[string(slokID)]
 			}
 
-			checkRegistryStartTime := time.Now()
-
-			registry, _, err := UnpackRegistry(
-				pavedRegistries[testCase.propagationChannelID], signingPublicKey)
+			registryStreamer, err := NewRegistryStreamer(
+				bytes.NewReader(pavedRegistries[testCase.propagationChannelID]),
+				signingPublicKey,
+				lookupSLOKs)
 			if err != nil {
-				t.Fatalf("UnpackRegistry failed: %s", err)
+				t.Fatalf("NewRegistryStreamer failed: %s", err)
 			}
 
-			t.Logf("registry size: %d", len(pavedRegistries[testCase.propagationChannelID]))
-			t.Logf("registry OSL count: %d", len(registry.FileSpecs))
+			seededOSLCount := 0
 
-			oslIDs := registry.GetSeededOSLIDs(
-				slokLookup,
-				func(err error) {
-					// Actual client will treat errors as warnings.
-					t.Fatalf("GetSeededOSLIDs failed: %s", err)
-				})
+			for {
 
-			t.Logf("check registry elapsed time: %s", time.Since(checkRegistryStartTime))
+				fileSpec, err := registryStreamer.Next()
+				if err != nil {
+					t.Fatalf("Next failed: %s", err)
+				}
 
-			if len(oslIDs) != testCase.expectedOSLCount {
-				t.Fatalf("expected %d OSLs got %d", testCase.expectedOSLCount, len(oslIDs))
-			}
+				if fileSpec == nil {
+					break
+				}
+
+				seededOSLCount += 1
 
-			for _, oslID := range oslIDs {
 				oslFileContents, ok :=
-					pavedOSLFileContents[testCase.propagationChannelID][GetOSLFileURL("", oslID)]
+					pavedOSLFileContents[testCase.propagationChannelID][GetOSLFileURL("", fileSpec.ID)]
 				if !ok {
 					t.Fatalf("unknown OSL file name")
 				}
 
-				plaintextOSL, err := registry.UnpackOSL(
-					slokLookup, oslID, oslFileContents, signingPublicKey)
+				payloadReader, err := NewOSLReader(
+					bytes.NewReader(oslFileContents),
+					fileSpec,
+					lookupSLOKs,
+					signingPublicKey)
 				if err != nil {
-					t.Fatalf("DecryptOSL failed: %s", err)
+					t.Fatalf("NewOSLReader failed: %s", err)
+				}
+
+				payload, err := ioutil.ReadAll(payloadReader)
+				if err != nil {
+					t.Fatalf("ReadAll failed: %s", err)
 				}
 
 				// The decrypted OSL should contain its own ID.
-				if plaintextOSL != base64.StdEncoding.EncodeToString(oslID) {
+				if string(payload) != base64.StdEncoding.EncodeToString(fileSpec.ID) {
 					t.Fatalf("unexpected OSL file contents")
 				}
 			}
+
+			t.Logf("registry size: %d", len(pavedRegistries[testCase.propagationChannelID]))
+			t.Logf("SLOK count: %d", len(slokMap))
+			t.Logf("seeded OSL count: %d", seededOSLCount)
+			t.Logf("elapsed time: %s", time.Since(startTime))
+
+			if seededOSLCount != testCase.expectedOSLCount {
+				t.Fatalf("expected %d OSLs got %d", testCase.expectedOSLCount, seededOSLCount)
+			}
 		})
 	}
 }

psiphon/dataStore.go

@@ -65,7 +65,6 @@ const (
 
 const (
 	DATA_STORE_LAST_CONNECTED_KEY           = "lastConnected"
-	DATA_STORE_OSL_REGISTRY_KEY             = "OSLRegistry"
 	PERSISTENT_STAT_TYPE_TUNNEL             = tunnelStatsBucket
 	PERSISTENT_STAT_TYPE_REMOTE_SERVER_LIST = remoteServerListStatsBucket
 )

psiphon/remoteServerList.go

@@ -23,7 +23,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"time"
 
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
@@ -69,17 +69,23 @@ func FetchCommonRemoteServerList(
 		return nil
 	}
 
-	serverListPayload, err := common.StreamingReadAuthenticatedDataPackage(
-		config.RemoteServerListDownloadFilename,
+	file, err := os.Open(config.RemoteServerListDownloadFilename)
+	if err != nil {
+		return fmt.Errorf("failed to open common remote server list: %s", common.ContextError(err))
+
+	}
+	defer file.Close()
+
+	serverListPayloadReader, err := common.NewAuthenticatedDataPackageReader(
+		file,
 		config.RemoteServerListSignaturePublicKey)
 	if err != nil {
 		return fmt.Errorf("failed to read remote server list: %s", common.ContextError(err))
 	}
-	defer serverListPayload.Close()
 
 	err = StreamingStoreServerEntries(
 		protocol.NewStreamingServerEntryDecoder(
-			serverListPayload,
+			serverListPayloadReader,
 			common.GetCurrentTimestamp(),
 			protocol.SERVER_ENTRY_SOURCE_REMOTE),
 		true)
@@ -118,6 +124,7 @@ func FetchObfuscatedServerLists(
 	NoticeInfo("fetching obfuscated remote server lists")
 
 	downloadFilename := osl.GetOSLRegistryFilename(config.ObfuscatedServerListDownloadDirectory)
+	cachedFilename := downloadFilename + ".cached"
 
 	rootURL, canonicalRootURL, skipVerify := selectDownloadURL(attempt, config.ObfuscatedServerListRootURLs)
 	downloadURL := osl.GetOSLRegistryURL(rootURL)
@@ -130,7 +137,11 @@ func FetchObfuscatedServerLists(
 	// TODO: should disk-full conditions not trigger retries?
 	var failed bool
 
-	var oslRegistry *osl.Registry
+	// updateCache is set when modifed registry content is downloaded. Both the cached
+	// file and the persisted ETag will be updated in this case. The update is deferred
+	// until after the registry has been authenticated.
+	updateCache := false
+	registryFilename := cachedFilename
 
 	newETag, err := downloadRemoteServerListFile(
 		config,
@@ -144,95 +155,68 @@ func FetchObfuscatedServerLists(
 	if err != nil {
 		failed = true
 		NoticeAlert("failed to download obfuscated server list registry: %s", common.ContextError(err))
+		// Proceed with any existing cached OSL registry.
 	} else if newETag != "" {
-
-		fileContent, err := ioutil.ReadFile(downloadFilename)
-		if err != nil {
-			failed = true
-			NoticeAlert("failed to read obfuscated server list registry: %s", common.ContextError(err))
-		}
-
-		var oslRegistryJSON []byte
-		if err == nil {
-			oslRegistry, oslRegistryJSON, err = osl.UnpackRegistry(
-				fileContent, config.RemoteServerListSignaturePublicKey)
-			if err != nil {
-				failed = true
-				NoticeAlert("failed to unpack obfuscated server list registry: %s", common.ContextError(err))
-			}
-		}
-
-		if err == nil {
-			err = SetKeyValue(DATA_STORE_OSL_REGISTRY_KEY, string(oslRegistryJSON))
-			if err != nil {
-				failed = true
-				NoticeAlert("failed to set cached obfuscated server list registry: %s", common.ContextError(err))
-			}
-		}
+		updateCache = true
+		registryFilename = downloadFilename
 	}
 
-	if failed || newETag == "" {
-		// Proceed with the cached OSL registry.
-		oslRegistryJSON, err := GetKeyValue(DATA_STORE_OSL_REGISTRY_KEY)
-		if err == nil && oslRegistryJSON == "" {
-			err = errors.New("not found")
-		}
+	lookupSLOKs := func(slokID []byte) []byte {
+		// Lookup SLOKs in local datastore
+		key, err := GetSLOK(slokID)
 		if err != nil {
-			return fmt.Errorf("failed to get cached obfuscated server list registry: %s", common.ContextError(err))
+			NoticeAlert("GetSLOK failed: %s", err)
 		}
+		return key
+	}
 
-		oslRegistry, err = osl.LoadRegistry([]byte(oslRegistryJSON))
-		if err != nil {
-			return fmt.Errorf("failed to load obfuscated server list registry: %s", common.ContextError(err))
-		}
+	registryFile, err := os.Open(registryFilename)
+	if err != nil {
+		return fmt.Errorf("failed to read obfuscated server list registry: %s", common.ContextError(err))
 	}
+	defer registryFile.Close()
 
-	// When a new registry is downloaded, validated, and parsed, store the
-	// response ETag so we won't re-download this same data again.
-	if !failed && newETag != "" {
-		err = SetUrlETag(canonicalURL, newETag)
-		if err != nil {
-			NoticeAlert("failed to set ETag for obfuscated server list registry: %s", common.ContextError(err))
-			// This fetch is still reported as a success, even if we can't store the etag
-		}
+	registryStreamer, err := osl.NewRegistryStreamer(
+		registryFile,
+		config.RemoteServerListSignaturePublicKey,
+		lookupSLOKs)
+	if err != nil {
+		// TODO: delete file? redownload if corrupt?
+		return fmt.Errorf("failed to read obfuscated server list registry: %s", common.ContextError(err))
 	}
 
-	// Note: we proceed to check individual OSLs even if the direcory is unchanged,
+	// NewRegistryStreamer authenticates the downloaded registry, so now it would be
+	// ok to update the cache. However, we defer that until after processing so we
+	// can close the file first before copying it, avoiding related complications on
+	// platforms such as Windows.
+
+	// Note: we proceed to check individual OSLs even if the directory is unchanged,
 	// as the set of local SLOKs may have changed.
 
-	lookupSLOKs := func(slokID []byte) []byte {
-		// Lookup SLOKs in local datastore
-		key, err := GetSLOK(slokID)
+	for {
+
+		oslFileSpec, err := registryStreamer.Next()
 		if err != nil {
-			NoticeAlert("GetSLOK failed: %s", err)
+			failed = true
+			NoticeAlert("failed to stream obfuscated server list registry: %s", common.ContextError(err))
+			break
 		}
-		return key
-	}
 
-	oslIDs := oslRegistry.GetSeededOSLIDs(
-		lookupSLOKs,
-		func(err error) {
-			NoticeAlert("GetSeededOSLIDs failed: %s", err)
-		})
-
-	for _, oslID := range oslIDs {
+		if oslFileSpec == nil {
+			break
+		}
 
-		downloadFilename := osl.GetOSLFilename(config.ObfuscatedServerListDownloadDirectory, oslID)
+		downloadFilename := osl.GetOSLFilename(
+			config.ObfuscatedServerListDownloadDirectory, oslFileSpec.ID)
 
-		downloadURL := osl.GetOSLFileURL(rootURL, oslID)
-		canonicalURL := osl.GetOSLFileURL(canonicalRootURL, oslID)
+		downloadURL := osl.GetOSLFileURL(rootURL, oslFileSpec.ID)
+		canonicalURL := osl.GetOSLFileURL(canonicalRootURL, oslFileSpec.ID)
 
-		hexID := hex.EncodeToString(oslID)
+		hexID := hex.EncodeToString(oslFileSpec.ID)
 
 		// Note: the MD5 checksum step assumes the remote server list host's ETag uses MD5
 		// with a hex encoding. If this is not the case, the sourceETag should be left blank.
-		sourceETag := ""
-		md5sum, err := oslRegistry.GetOSLMD5Sum(oslID)
-		if err == nil {
-			sourceETag = fmt.Sprintf("\"%s\"", hex.EncodeToString(md5sum))
-		}
-
-		// TODO: store ETags in OSL registry to enable skipping requests entirely
+		sourceETag := fmt.Sprintf("\"%s\"", hex.EncodeToString(oslFileSpec.MD5Sum))
 
 		newETag, err := downloadRemoteServerListFile(
 			config,
@@ -254,23 +238,34 @@ func FetchObfuscatedServerLists(
 			continue
 		}
 
-		fileContent, err := ioutil.ReadFile(downloadFilename)
+		file, err := os.Open(downloadFilename)
 		if err != nil {
 			failed = true
-			NoticeAlert("failed to read obfuscated server list file (%s): %s", hexID, common.ContextError(err))
+			NoticeAlert("failed to open obfuscated server list file (%s): %s", hexID, common.ContextError(err))
 			continue
 		}
+		// Note: don't defer file.Close() since we're in a loop
 
-		serverListPayload, err := oslRegistry.UnpackOSL(
-			lookupSLOKs, oslID, fileContent, config.RemoteServerListSignaturePublicKey)
+		serverListPayloadReader, err := osl.NewOSLReader(
+			file,
+			oslFileSpec,
+			lookupSLOKs,
+			config.RemoteServerListSignaturePublicKey)
 		if err != nil {
+			file.Close()
 			failed = true
-			NoticeAlert("failed to unpack obfuscated server list file (%s): %s", hexID, common.ContextError(err))
+			NoticeAlert("failed to read obfuscated server list file (%s): %s", hexID, common.ContextError(err))
 			continue
 		}
 
-		err = storeServerEntries(serverListPayload, protocol.SERVER_ENTRY_SOURCE_OBFUSCATED)
+		err = StreamingStoreServerEntries(
+			protocol.NewStreamingServerEntryDecoder(
+				serverListPayloadReader,
+				common.GetCurrentTimestamp(),
+				protocol.SERVER_ENTRY_SOURCE_OBFUSCATED),
+			true)
 		if err != nil {
+			file.Close()
 			failed = true
 			NoticeAlert("failed to store obfuscated server list file (%s): %s", hexID, common.ContextError(err))
 			continue
@@ -280,16 +275,39 @@ func FetchObfuscatedServerLists(
 		// ETag so we won't re-download this same data again.
 		err = SetUrlETag(canonicalURL, newETag)
 		if err != nil {
-			failed = true
-			NoticeAlert("failed to set Etag for obfuscated server list file (%s): %s", hexID, common.ContextError(err))
+			file.Close()
+			NoticeAlert("failed to set ETag for obfuscated server list file (%s): %s", hexID, common.ContextError(err))
 			continue
-			// This fetch is still reported as a success, even if we can't store the etag
+			// This fetch is still reported as a success, even if we can't store the ETag
+		}
+
+		file.Close()
+	}
+
+	// Now that a new registry is downloaded, validated, and parsed, store
+	// the response ETag so we won't re-download this same data again. First
+	// close the file to avoid complications on platforms such as Windows.
+	if updateCache {
+
+		registryFile.Close()
+
+		err := os.Rename(downloadFilename, cachedFilename)
+		if err != nil {
+			NoticeAlert("failed to set cached obfuscated server list registry: %s", common.ContextError(err))
+			// This fetch is still reported as a success, even if we can't update the cache
+		}
+
+		err = SetUrlETag(canonicalURL, newETag)
+		if err != nil {
+			NoticeAlert("failed to set ETag for obfuscated server list registry: %s", common.ContextError(err))
+			// This fetch is still reported as a success, even if we can't store the ETag
 		}
 	}
 
 	if failed {
 		return errors.New("one or more operations failed")
 	}
+
 	return nil
 }
 
@@ -362,21 +380,3 @@ func downloadRemoteServerListFile(
 
 	return responseETag, nil
 }
-
-func storeServerEntries(serverList, serverEntrySource string) error {
-
-	serverEntries, err := protocol.DecodeServerEntryList(
-		serverList,
-		common.GetCurrentTimestamp(),
-		serverEntrySource)
-	if err != nil {
-		return common.ContextError(err)
-	}
-
-	err = StoreServerEntries(serverEntries, true)
-	if err != nil {
-		return common.ContextError(err)
-	}
-
-	return nil
-}