Fix automated tests

- All automated tests working with new
  obfuscated server list changes.
- Automated tests now create test files
  in unique temporary directories.

psiphon/common/reloader_test.go

@@ -22,12 +22,21 @@ package common
 import (
 	"bytes"
 	"io/ioutil"
+	"os"
+	"path/filepath"
 	"testing"
 )
 
 func TestReloader(t *testing.T) {
 
-	fileName := "reloader_test.dat"
+	dirname, err := ioutil.TempDir("", "psiphon-reloader-test")
+	if err != nil {
+		t.Fatalf("TempDir failed: %s", err)
+	}
+	defer os.RemoveAll(dirname)
+
+	fileName := filepath.Join(dirname, "reloader_test.dat")
+
 	initialContents := []byte("contents1\n")
 	modifiedContents := []byte("contents2\n")
 
@@ -45,7 +54,7 @@ func TestReloader(t *testing.T) {
 
 	// Test: initial load
 
-	err := ioutil.WriteFile(fileName, initialContents, 0600)
+	err = ioutil.WriteFile(fileName, initialContents, 0600)
 	if err != nil {
 		t.Fatalf("WriteFile failed: %s", err)
 	}

psiphon/config_test.go

@@ -59,6 +59,10 @@ func (suite *ConfigTestSuite) SetupSuite() {
 			suite.nonRequiredFields = append(suite.nonRequiredFields, k)
 		}
 	}
+
+	// Force skipping required RSL fields
+	obj["DisableRemoteServerListFetcher"] = true
+	suite.confStubBlob, _ = json.Marshal(obj)
 }
 
 func TestConfigTestSuite(t *testing.T) {

psiphon/controller_test.go

@@ -20,6 +20,7 @@
 package psiphon
 
 import (
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
@@ -28,6 +29,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -41,12 +43,26 @@ import (
 	"github.com/elazarl/goproxy"
 )
 
+var testDataDirName string
+
 func TestMain(m *testing.M) {
 	flag.Parse()
-	os.Remove(DATA_STORE_FILENAME)
+
+	var err error
+	testDataDirName, err = ioutil.TempDir("", "psiphon-controller-test")
+	if err != nil {
+		fmt.Printf("TempDir failed: %s", err)
+		os.Exit(1)
+	}
+	defer os.RemoveAll(testDataDirName)
+
+	os.Remove(filepath.Join(testDataDirName, DATA_STORE_FILENAME))
+
+	SetEmitDiagnosticNotices(true)
+
 	initDisruptor()
 	initUpstreamProxy()
-	SetEmitDiagnosticNotices(true)
+
 	os.Exit(m.Run())
 }
 
@@ -125,7 +141,7 @@ func TestUntunneledUpgradeClientIsLatestVersion(t *testing.T) {
 		})
 }
 
-func TestUntunneledResumableFetchRemoveServerList(t *testing.T) {
+func TestUntunneledResumableFetchRemoteServerList(t *testing.T) {
 	controllerRun(t,
 		&controllerRunConfig{
 			expectNoServerEntries:    true,
@@ -421,12 +437,23 @@ type controllerRunConfig struct {
 
 func controllerRun(t *testing.T, runConfig *controllerRunConfig) {
 
-	configFileContents, err := ioutil.ReadFile("controller_test.config")
+	configJSON, err := ioutil.ReadFile("controller_test.config")
 	if err != nil {
 		// Skip, don't fail, if config file is not present
 		t.Skipf("error loading configuration file: %s", err)
 	}
-	config, err := LoadConfig(configFileContents)
+
+	// These fields must be filled in before calling LoadConfig
+	var modifyConfig map[string]interface{}
+	json.Unmarshal(configJSON, &modifyConfig)
+	modifyConfig["DataStoreDirectory"] = testDataDirName
+	modifyConfig["RemoteServerListDownloadFilename"] = filepath.Join(testDataDirName, "server_list_compressed")
+	modifyConfig["ObfuscatedServerListDownloadDirectory"] = testDataDirName
+	modifyConfig["ObfuscatedServerListRootURL"] = "http://127.0.0.1/osl" // will fail
+	modifyConfig["UpgradeDownloadFilename"] = filepath.Join(testDataDirName, "upgrade")
+	configJSON, _ = json.Marshal(modifyConfig)
+
+	config, err := LoadConfig(configJSON)
 	if err != nil {
 		t.Fatalf("error processing configuration file: %s", err)
 	}
@@ -438,7 +465,7 @@ func controllerRun(t *testing.T, runConfig *controllerRunConfig) {
 	if runConfig.disableEstablishing {
 		// Clear remote server list so tunnel cannot be established.
 		// TODO: also delete all server entries in the datastore.
-		config.RemoteServerListUrl = ""
+		config.DisableRemoteServerListFetcher = true
 	}
 
 	if runConfig.disableApi {
@@ -475,10 +502,11 @@ func controllerRun(t *testing.T, runConfig *controllerRunConfig) {
 	establishTunnelPausePeriodSeconds := 1
 	config.EstablishTunnelPausePeriodSeconds = &establishTunnelPausePeriodSeconds
 
-	os.Remove(config.UpgradeDownloadFilename)
-
 	config.TunnelProtocol = runConfig.protocol
 
+	os.Remove(config.UpgradeDownloadFilename)
+	os.Remove(config.RemoteServerListDownloadFilename)
+
 	err = InitDataStore(config)
 	if err != nil {
 		t.Fatalf("error initializing datastore: %s", err)
@@ -573,16 +601,23 @@ func controllerRun(t *testing.T, runConfig *controllerRunConfig) {
 				default:
 				}
 
-			case "RemoteServerListDownloadedBytes":
+			case "RemoteServerListResourceDownloadedBytes":
 
-				atomic.AddInt32(&remoteServerListDownloadedBytesCount, 1)
-				t.Logf("RemoteServerListDownloadedBytes: %d", int(payload["bytes"].(float64)))
+				url := payload["url"].(string)
+				if url == config.RemoteServerListUrl {
+					t.Logf("RemoteServerListResourceDownloadedBytes: %d", int(payload["bytes"].(float64)))
+					atomic.AddInt32(&remoteServerListDownloadedBytesCount, 1)
+				}
 
-			case "RemoteServerListDownloaded":
+			case "RemoteServerListResourceDownloaded":
 
-				select {
-				case remoteServerListDownloaded <- *new(struct{}):
-				default:
+				url := payload["url"].(string)
+				if url == config.RemoteServerListUrl {
+					t.Logf("RemoteServerListResourceDownloaded")
+					select {
+					case remoteServerListDownloaded <- *new(struct{}):
+					default:
+					}
 				}
 
 			case "ImpairedProtocolClassification":

psiphon/remoteServerList_test.go

@@ -25,6 +25,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
@@ -43,11 +44,16 @@ import (
 
 func TestObfuscatedRemoteServerLists(t *testing.T) {
 
+	testDataDirName, err := ioutil.TempDir("", "psiphon-remote-server-list-test")
+	if err != nil {
+		t.Fatalf("TempDir failed: %s", err)
+	}
+	defer os.RemoveAll(testDataDirName)
+
 	//
 	// create a server
 	//
 
-	var err error
 	serverIPaddress := ""
 	for _, interfaceName := range []string{"eth0", "en0"} {
 		serverIPaddress, err = GetInterfaceIPAddress(interfaceName)
@@ -147,14 +153,18 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 	// mock seeding SLOKs
 	//
 
-	singleton.db = nil
-	os.Remove(DATA_STORE_FILENAME)
+	singleton = dataStore{}
+	os.Remove(filepath.Join(testDataDirName, DATA_STORE_FILENAME))
 
-	err = InitDataStore(&Config{})
+	err = InitDataStore(&Config{DataStoreDirectory: testDataDirName})
 	if err != nil {
 		t.Fatalf("error initializing client datastore: %s", err)
 	}
 
+	if CountServerEntries("", "") > 0 {
+		t.Fatalf("unexpected server entries")
+	}
+
 	seedState := oslConfig.NewClientSeedState("", propagationChannelID, nil)
 	seedPortForward := seedState.NewClientSeedPortForward(net.ParseIP("0.0.0.0"))
 	seedPortForward.UpdateProgress(1, 1, 1)
@@ -169,17 +179,14 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 	// run mock remote server list host
 	//
 
-	downloadRoot := "test-data"
-	os.MkdirAll(downloadRoot, 0700)
-
 	remoteServerListHostAddress := net.JoinHostPort(serverIPaddress, "8080")
 
 	// The common remote server list fetches will 404
 	remoteServerListURL := fmt.Sprintf("http://%s/server_list_compressed", remoteServerListHostAddress)
-	remoteServerListDownloadFilename := filepath.Join(downloadRoot, "server_list_compressed")
+	remoteServerListDownloadFilename := filepath.Join(testDataDirName, "server_list_compressed")
 
 	obfuscatedServerListRootURL := fmt.Sprintf("http://%s/", remoteServerListHostAddress)
-	obfuscatedServerListDownloadDirectory := downloadRoot
+	obfuscatedServerListDownloadDirectory := testDataDirName
 
 	go func() {
 		startTime := time.Now()

psiphon/server/server_test.go

@@ -29,6 +29,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strconv"
 	"sync"
 	"syscall"
@@ -40,11 +41,25 @@ import (
 	"golang.org/x/net/proxy"
 )
 
+var testDataDirName string
+
 func TestMain(m *testing.M) {
 	flag.Parse()
-	os.Remove(psiphon.DATA_STORE_FILENAME)
+
+	var err error
+	testDataDirName, err = ioutil.TempDir("", "psiphon-server-test")
+	if err != nil {
+		fmt.Printf("TempDir failed: %s", err)
+		os.Exit(1)
+	}
+	defer os.RemoveAll(testDataDirName)
+
+	os.Remove(filepath.Join(testDataDirName, psiphon.DATA_STORE_FILENAME))
+
 	psiphon.SetEmitDiagnosticNotices(true)
+
 	CLIENT_VERIFICATION_REQUIRED = true
+
 	os.Exit(m.Run())
 }
 
@@ -219,29 +234,29 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
 	// customize server config
 
 	// Pave psinet with random values to test handshake homepages.
-	psinetFilename := "psinet.json"
+	psinetFilename := filepath.Join(testDataDirName, "psinet.json")
 	sponsorID, expectedHomepageURL := pavePsinetDatabaseFile(t, psinetFilename)
 
 	// Pave traffic rules file which exercises handshake parameter filtering. Client
 	// must handshake with specified sponsor ID in order to allow ports for tunneled
 	// requests.
-	trafficRulesFilename := "traffic_rules.json"
+	trafficRulesFilename := filepath.Join(testDataDirName, "traffic_rules.json")
 	paveTrafficRulesFile(t, trafficRulesFilename, sponsorID, runConfig.denyTrafficRules)
 
-	oslConfigFilename := "osl_config.json"
+	oslConfigFilename := filepath.Join(testDataDirName, "osl_config.json")
 	propagationChannelID := paveOSLConfigFile(t, oslConfigFilename)
 
-	var serverConfig interface{}
+	var serverConfig map[string]interface{}
 	json.Unmarshal(serverConfigJSON, &serverConfig)
-	serverConfig.(map[string]interface{})["GeoIPDatabaseFilename"] = ""
-	serverConfig.(map[string]interface{})["PsinetDatabaseFilename"] = psinetFilename
-	serverConfig.(map[string]interface{})["TrafficRulesFilename"] = trafficRulesFilename
-	serverConfig.(map[string]interface{})["OSLConfigFilename"] = oslConfigFilename
-	serverConfig.(map[string]interface{})["LogLevel"] = "debug"
+	serverConfig["GeoIPDatabaseFilename"] = ""
+	serverConfig["PsinetDatabaseFilename"] = psinetFilename
+	serverConfig["TrafficRulesFilename"] = trafficRulesFilename
+	serverConfig["OSLConfigFilename"] = oslConfigFilename
+	serverConfig["LogLevel"] = "error"
 
 	// 1 second is the minimum period; should be small enough to emit a log during the
 	// test run, but not guaranteed
-	serverConfig.(map[string]interface{})["LoadMonitorPeriodSeconds"] = 1
+	serverConfig["LoadMonitorPeriodSeconds"] = 1
 
 	serverConfigJSON, _ = json.Marshal(serverConfig)
 
@@ -314,7 +329,8 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
         "ClientPlatform" : "Android",
         "ClientVersion" : "0",
         "SponsorId" : "0",
-        "PropagationChannelId" : "0"
+        "PropagationChannelId" : "0",
+        "DisableRemoteServerListFetcher" : true
     }`
 	clientConfig, _ := psiphon.LoadConfig([]byte(clientConfigJSON))
 
@@ -322,7 +338,6 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
 	clientConfig.PropagationChannelId = propagationChannelID
 	clientConfig.ConnectionWorkerPoolSize = numTunnels
 	clientConfig.TunnelPoolSize = numTunnels
-	clientConfig.DisableRemoteServerListFetcher = true
 	clientConfig.EstablishTunnelPausePeriodSeconds = &establishTunnelPausePeriodSeconds
 	clientConfig.TargetServerEntry = string(encodedServerEntry)
 	clientConfig.TunnelProtocol = runConfig.tunnelProtocol
@@ -330,6 +345,7 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
 	clientConfig.LocalHttpProxyPort = localHTTPProxyPort
 	clientConfig.ReportSLOKs = true
 
+	clientConfig.DataStoreDirectory = testDataDirName
 	err = psiphon.InitDataStore(clientConfig)
 	if err != nil {
 		t.Fatalf("error initializing client datastore: %s", err)
@@ -443,7 +459,7 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
 
 		// Test: tunneled UDP packets
 
-		udpgwServerAddress := serverConfig.(map[string]interface{})["UDPInterceptUdpgwServerAddress"].(string)
+		udpgwServerAddress := serverConfig["UDPInterceptUdpgwServerAddress"].(string)
 
 		err = makeTunneledNTPRequest(t, localSOCKSProxyPort, udpgwServerAddress)