Merge branch 'master' into production-client

MobileLibrary/Android/Dockerfile

@@ -92,6 +92,10 @@ RUN mkdir -p $GOPATH/pkg/gomobile/dl \
   && git clone https://github.com/golang/mobile \
   && cd mobile \
   && git checkout -b pinned $GOMOBILE_PINNED_REV \
+  && mv ./cmd/gomobile/build.go ./cmd/gomobile/build.go.orig \
+  && sed -e 's/"-tags="+strconv.Quote(strings.Join(ctx.BuildTags, ",")),/"-tags",strings.Join(ctx.BuildTags, " "),/g' ./cmd/gomobile/build.go.orig > ./cmd/gomobile/build.go \
+  && mv ./cmd/gomobile/build.go ./cmd/gomobile/build.go.orig \
+  && sed -e 's/"strconv"//g' ./cmd/gomobile/build.go.orig > ./cmd/gomobile/build.go \
   && echo "master: $(git rev-parse master)\npinned: $(git rev-parse pinned)" | tee $GOROOT/MOBILE \
   && go install golang.org/x/mobile/cmd/gomobile \
   && gomobile init -v

MobileLibrary/Android/make.bash

@@ -16,11 +16,6 @@ BUILD_TAGS="OPENSSL ${PRIVATE_PLUGINS_TAG}"
 # the latest versions. Outside of Docker, be aware that these dependencies
 # will not be overridden w/ new versions if they already exist in $GOPATH
 
-GOOS=arm go get -d -v -tags "${BUILD_TAGS}" github.com/Psiphon-Inc/openssl
-if [ $? != 0 ]; then
-  echo "..'go get -d -v github.com/psiphon-inc/openssl' failed, exiting"
-  exit $?
-fi
 GOOS=arm go get -d -v -tags "${BUILD_TAGS}" github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon
 if [ $? != 0 ]; then
   echo "..'go get -d -v github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon' failed, exiting"

MobileLibrary/iOS/build-psiphon-framework.sh

@@ -6,7 +6,7 @@ set -x -u -e
 
 # Reset the PATH to macOS default. This is mainly so we don't execute the wrong
 # gomobile executable.
-PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
+PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/go/bin
 
 BASE_DIR=$(cd "$(dirname "$0")" ; pwd -P)
 cd ${BASE_DIR}
@@ -162,6 +162,10 @@ if [[ $rc != 0 ]]; then
     cd ${GOPATH}/src/golang.org/x/mobile/cmd/gomobile
     git checkout master
     git checkout -b pinned ${GOMOBILE_PINNED_REV}
+    mv ./build.go ./build.go.orig
+    sed -e 's/"-tags="+strconv.Quote(strings.Join(ctx.BuildTags, ",")),/"-tags",strings.Join(ctx.BuildTags, " "),/g' ./build.go.orig > ./build.go
+    mv ./build.go ./build.go.orig
+    sed -e 's/"strconv"//g' ./build.go.orig > ./build.go
     go install
     ${GOPATH}/bin/gomobile init -v
     if [[ $? != 0 ]]; then

Server/make.bash

@@ -57,7 +57,7 @@ build_for_linux () {
     exit $?
   fi
 
-  GOOS=linux GOARCH=amd64 go build -tags "${BUILD_TAGS}" -ldflags "$LDFLAGS" -o psiphond main.go
+  GOOS=linux GOARCH=amd64 go build -tags "${BUILD_TAGS}" -ldflags "$LDFLAGS" -o psiphond
   if [ $? != 0 ]; then
     echo "...'go build' failed, exiting"
     exit $?

psiphon/common/osl/paver/main.go

@@ -109,14 +109,14 @@ func main() {
 	for _, keySplit := range config.Schemes[scheme].SeedPeriodKeySplits {
 		slokTimePeriodsPerOSL *= keySplit.Total
 	}
-	oslTimePeriod := time.Duration(config.Schemes[0].SeedPeriodNanoseconds * int64(slokTimePeriodsPerOSL))
+	oslTimePeriod := time.Duration(config.Schemes[scheme].SeedPeriodNanoseconds * int64(slokTimePeriodsPerOSL))
 
-	for _, propagationChannelID := range config.Schemes[0].PropagationChannelIDs {
+	for _, propagationChannelID := range config.Schemes[scheme].PropagationChannelIDs {
 
 		paveServerEntries := make([]map[time.Time]string, len(config.Schemes))
 		paveServerEntries[0] = make(map[time.Time]string)
 
-		epoch, _ := time.Parse(time.RFC3339, config.Schemes[0].Epoch)
+		epoch, _ := time.Parse(time.RFC3339, config.Schemes[scheme].Epoch)
 		for i := oslOffset; i < oslOffset+oslCount; i++ {
 			paveServerEntries[0][epoch.Add(time.Duration(i)*oslTimePeriod)] = ""
 		}

psiphon/server/psinet/psinet.go

@@ -128,13 +128,19 @@ func NewDatabase(filename string) (*Database, error) {
 	database.ReloadableFile = common.NewReloadableFile(
 		filename,
 		func(fileContent []byte) error {
-			err := json.Unmarshal(fileContent, &database)
+			var newDatabase Database
+			err := json.Unmarshal(fileContent, &newDatabase)
 			if err != nil {
-				// On error, state remains the same
-				// (Unmarshal first validates the provided
-				//  JOSN and then populates the interface)
 				return common.ContextError(err)
 			}
+			// Note: an unmarshal directly into &database would fail
+			// to reset to zero value fields not present in the JSON.
+			database.Hosts = newDatabase.Hosts
+			database.Servers = newDatabase.Servers
+			database.Sponsors = newDatabase.Sponsors
+			database.Versions = newDatabase.Versions
+			database.DefaultSponsorID = newDatabase.DefaultSponsorID
+
 			return nil
 		})
 

psiphon/server/server_test.go

@@ -72,6 +72,7 @@ func TestSSH(t *testing.T) {
 			tunnelProtocol:       "SSH",
 			enableSSHAPIRequests: true,
 			doHotReload:          false,
+			doDefaultSessionID:   false,
 			denyTrafficRules:     false,
 			doClientVerification: true,
 			doTunneledWebRequest: true,
@@ -85,6 +86,7 @@ func TestOSSH(t *testing.T) {
 			tunnelProtocol:       "OSSH",
 			enableSSHAPIRequests: true,
 			doHotReload:          false,
+			doDefaultSessionID:   false,
 			denyTrafficRules:     false,
 			doClientVerification: false,
 			doTunneledWebRequest: true,
@@ -98,6 +100,7 @@ func TestUnfrontedMeek(t *testing.T) {
 			tunnelProtocol:       "UNFRONTED-MEEK-OSSH",
 			enableSSHAPIRequests: true,
 			doHotReload:          false,
+			doDefaultSessionID:   false,
 			denyTrafficRules:     false,
 			doClientVerification: false,
 			doTunneledWebRequest: true,
@@ -111,6 +114,7 @@ func TestUnfrontedMeekHTTPS(t *testing.T) {
 			tunnelProtocol:       "UNFRONTED-MEEK-HTTPS-OSSH",
 			enableSSHAPIRequests: true,
 			doHotReload:          false,
+			doDefaultSessionID:   false,
 			denyTrafficRules:     false,
 			doClientVerification: false,
 			doTunneledWebRequest: true,
@@ -124,6 +128,7 @@ func TestUnfrontedMeekSessionTicket(t *testing.T) {
 			tunnelProtocol:       "UNFRONTED-MEEK-SESSION-TICKET-OSSH",
 			enableSSHAPIRequests: true,
 			doHotReload:          false,
+			doDefaultSessionID:   false,
 			denyTrafficRules:     false,
 			doClientVerification: false,
 			doTunneledWebRequest: true,
@@ -137,6 +142,7 @@ func TestWebTransportAPIRequests(t *testing.T) {
 			tunnelProtocol:       "OSSH",
 			enableSSHAPIRequests: false,
 			doHotReload:          false,
+			doDefaultSessionID:   false,
 			denyTrafficRules:     false,
 			doClientVerification: true,
 			doTunneledWebRequest: true,
@@ -150,6 +156,21 @@ func TestHotReload(t *testing.T) {
 			tunnelProtocol:       "OSSH",
 			enableSSHAPIRequests: true,
 			doHotReload:          true,
+			doDefaultSessionID:   false,
+			denyTrafficRules:     false,
+			doClientVerification: false,
+			doTunneledWebRequest: true,
+			doTunneledNTPRequest: true,
+		})
+}
+
+func TestDefaultSessionID(t *testing.T) {
+	runServer(t,
+		&runServerConfig{
+			tunnelProtocol:       "OSSH",
+			enableSSHAPIRequests: true,
+			doHotReload:          true,
+			doDefaultSessionID:   true,
 			denyTrafficRules:     false,
 			doClientVerification: false,
 			doTunneledWebRequest: true,
@@ -163,6 +184,7 @@ func TestDenyTrafficRules(t *testing.T) {
 			tunnelProtocol:       "OSSH",
 			enableSSHAPIRequests: true,
 			doHotReload:          true,
+			doDefaultSessionID:   false,
 			denyTrafficRules:     true,
 			doClientVerification: false,
 			doTunneledWebRequest: true,
@@ -176,6 +198,7 @@ func TestTCPOnlySLOK(t *testing.T) {
 			tunnelProtocol:       "OSSH",
 			enableSSHAPIRequests: true,
 			doHotReload:          false,
+			doDefaultSessionID:   false,
 			denyTrafficRules:     false,
 			doClientVerification: false,
 			doTunneledWebRequest: true,
@@ -189,6 +212,7 @@ func TestUDPOnlySLOK(t *testing.T) {
 			tunnelProtocol:       "OSSH",
 			enableSSHAPIRequests: true,
 			doHotReload:          false,
+			doDefaultSessionID:   false,
 			denyTrafficRules:     false,
 			doClientVerification: false,
 			doTunneledWebRequest: false,
@@ -200,6 +224,7 @@ type runServerConfig struct {
 	tunnelProtocol       string
 	enableSSHAPIRequests bool
 	doHotReload          bool
+	doDefaultSessionID   bool
 	denyTrafficRules     bool
 	doClientVerification bool
 	doTunneledWebRequest bool
@@ -258,16 +283,18 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
 
 	// Pave psinet with random values to test handshake homepages.
 	psinetFilename := filepath.Join(testDataDirName, "psinet.json")
-	sponsorID, expectedHomepageURL := pavePsinetDatabaseFile(t, psinetFilename)
+	sponsorID, expectedHomepageURL := pavePsinetDatabaseFile(
+		t, runConfig.doDefaultSessionID, psinetFilename)
+
+	// Pave OSL config for SLOk testing
+	oslConfigFilename := filepath.Join(testDataDirName, "osl_config.json")
+	propagationChannelID := paveOSLConfigFile(t, oslConfigFilename)
 
 	// Pave traffic rules file which exercises handshake parameter filtering. Client
 	// must handshake with specified sponsor ID in order to allow ports for tunneled
 	// requests.
 	trafficRulesFilename := filepath.Join(testDataDirName, "traffic_rules.json")
-	paveTrafficRulesFile(t, trafficRulesFilename, sponsorID, runConfig.denyTrafficRules)
-
-	oslConfigFilename := filepath.Join(testDataDirName, "osl_config.json")
-	propagationChannelID := paveOSLConfigFile(t, oslConfigFilename)
+	paveTrafficRulesFile(t, trafficRulesFilename, propagationChannelID, runConfig.denyTrafficRules)
 
 	var serverConfig map[string]interface{}
 	json.Unmarshal(serverConfigJSON, &serverConfig)
@@ -320,9 +347,14 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
 
 	if runConfig.doHotReload {
 
-		// Pave a new psinet and traffic rules with different random values.
-		sponsorID, expectedHomepageURL = pavePsinetDatabaseFile(t, psinetFilename)
-		paveTrafficRulesFile(t, trafficRulesFilename, sponsorID, runConfig.denyTrafficRules)
+		// Pave new config files with different random values.
+		sponsorID, expectedHomepageURL = pavePsinetDatabaseFile(
+			t, runConfig.doDefaultSessionID, psinetFilename)
+
+		propagationChannelID = paveOSLConfigFile(t, oslConfigFilename)
+
+		paveTrafficRulesFile(
+			t, trafficRulesFilename, propagationChannelID, runConfig.denyTrafficRules)
 
 		p, _ := os.FindProcess(os.Getpid())
 		p.Signal(syscall.SIGUSR1)
@@ -358,7 +390,9 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
     }`
 	clientConfig, _ := psiphon.LoadConfig([]byte(clientConfigJSON))
 
-	clientConfig.SponsorId = sponsorID
+	if !runConfig.doDefaultSessionID {
+		clientConfig.SponsorId = sponsorID
+	}
 	clientConfig.PropagationChannelId = propagationChannelID
 	clientConfig.ConnectionWorkerPoolSize = numTunnels
 	clientConfig.TunnelPoolSize = numTunnels
@@ -721,7 +755,8 @@ func makeTunneledNTPRequest(t *testing.T, localSOCKSProxyPort int, udpgwServerAd
 	return nil
 }
 
-func pavePsinetDatabaseFile(t *testing.T, psinetFilename string) (string, string) {
+func pavePsinetDatabaseFile(
+	t *testing.T, useDefaultSponsorID bool, psinetFilename string) (string, string) {
 
 	sponsorID, _ := common.MakeRandomStringHex(8)
 
@@ -731,6 +766,7 @@ func pavePsinetDatabaseFile(t *testing.T, psinetFilename string) (string, string
 
 	psinetJSONFormat := `
     {
+        "default_sponsor_id" : "%s",
         "sponsors": {
             "%s": {
                 "home_pages": {
@@ -745,7 +781,14 @@ func pavePsinetDatabaseFile(t *testing.T, psinetFilename string) (string, string
         }
     }
 	`
-	psinetJSON := fmt.Sprintf(psinetJSONFormat, sponsorID, expectedHomepageURL)
+
+	defaultSponsorID := ""
+	if useDefaultSponsorID {
+		defaultSponsorID = sponsorID
+	}
+
+	psinetJSON := fmt.Sprintf(
+		psinetJSONFormat, defaultSponsorID, sponsorID, expectedHomepageURL)
 
 	err := ioutil.WriteFile(psinetFilename, []byte(psinetJSON), 0600)
 	if err != nil {
@@ -755,7 +798,8 @@ func pavePsinetDatabaseFile(t *testing.T, psinetFilename string) (string, string
 	return sponsorID, expectedHomepageURL
 }
 
-func paveTrafficRulesFile(t *testing.T, trafficRulesFilename, sponsorID string, deny bool) {
+func paveTrafficRulesFile(
+	t *testing.T, trafficRulesFilename, propagationChannelID string, deny bool) {
 
 	allowTCPPorts := "443"
 	allowUDPPorts := "53, 123"
@@ -779,7 +823,7 @@ func paveTrafficRulesFile(t *testing.T, trafficRulesFilename, sponsorID string,
             {
                 "Filter" : {
                     "HandshakeParameters" : {
-                        "sponsor_id" : ["%s"]
+                        "propagation_channel_id" : ["%s"]
                     }
                 },
                 "Rules" : {
@@ -796,7 +840,7 @@ func paveTrafficRulesFile(t *testing.T, trafficRulesFilename, sponsorID string,
     `
 
 	trafficRulesJSON := fmt.Sprintf(
-		trafficRulesJSONFormat, sponsorID, allowTCPPorts, allowUDPPorts)
+		trafficRulesJSONFormat, propagationChannelID, allowTCPPorts, allowUDPPorts)
 
 	err := ioutil.WriteFile(trafficRulesFilename, []byte(trafficRulesJSON), 0600)
 	if err != nil {