Add integration test

- Fix integration bugs

.travis.yml

@@ -15,6 +15,8 @@ script:
 - go test -race -v ./common/fragmentor
 - go test -race -v ./common/obfuscator
 - go test -race -v ./common/osl
+# TODO: enable all packetman tests with tag PACKET_MANIPULATOR_TEST; requires CAP_NETADMIN and CAP_RAW
+- go test -race -v ./common/packetman
 - go test -race -v ./common/parameters
 - go test -race -v ./common/protocol
 - go test -race -v ./common/quic
@@ -27,6 +29,7 @@ script:
 - go test -race -v ./common/values
 - go test -race -v ./common/wildcard
 - go test -race -v ./transferstats
+# TODO: seet "packetman" comment above
 - go test -race -v ./server
 - go test -race -v ./server/psinet
 - go test -race -v ../Server/logging/analysis
@@ -39,10 +42,11 @@ script:
 - go test -v -covermode=count -coverprofile=obfuscator.coverprofile ./common/obfuscator
 - go test -v -covermode=count -coverprofile=osl.coverprofile ./common/osl
 - go test -v -covermode=count -coverprofile=parameters.coverprofile ./common/parameters
+- go test -v -covermode=count -coverprofile=packetman.coverprofile ./common/packetman
 - go test -v -covermode=count -coverprofile=protocol.coverprofile ./common/protocol
 - go test -v -covermode=count -coverprofile=quic.coverprofile ./common/quic
 - go test -v -covermode=count -coverprofile=tactics.coverprofile ./common/tactics
-# TODO: see comment above
+# TODO: see "tun" test comment above
 #- sudo -E env "PATH=$PATH" go test -v -covermode=count -coverprofile=tun.coverprofile ./common/tun
 - go test -v -covermode=count -coverprofile=values.coverprofile ./common/values
 - go test -v -covermode=count -coverprofile=wildcard.coverprofile ./common/wildcard

psiphon/common/packetman/packetman_linux.go

@@ -279,7 +279,7 @@ func (m *Manipulator) Stop() {
 func (m *Manipulator) SetSpecs(specs []*Spec) error {
 
 	compiledSpecs := make(map[string]*compiledSpec)
-	for _, spec := range config.Specs {
+	for _, spec := range m.config.Specs {
 		if spec.Name == "" {
 			return errors.TraceNew("invalid spec name")
 		}

psiphon/common/packetman/packetman_linux_test.go

@@ -1,3 +1,5 @@
+// +build PACKET_MANIPULATOR_TEST
+
 /*
  * Copyright (c) 2020, Psiphon Inc.
  * All rights reserved.
@@ -20,7 +22,6 @@
 package packetman
 
 import (
-	"context"
 	"fmt"
 	"io"
 	"io/ioutil"

psiphon/server/meek.go

@@ -1240,17 +1240,19 @@ func newMeekConn(
 	protocolVersion int) *meekConn {
 
 	conn := &meekConn{
-		meekServer:        meekServer,
-		meekSession:       meekSession,
-		remoteAddr:        remoteAddr,
-		protocolVersion:   protocolVersion,
-		closeBroadcast:    make(chan struct{}),
-		closed:            0,
-		emptyReadBuffer:   make(chan *bytes.Buffer, 1),
-		partialReadBuffer: make(chan *bytes.Buffer, 1),
-		fullReadBuffer:    make(chan *bytes.Buffer, 1),
-		nextWriteBuffer:   make(chan []byte, 1),
-		writeResult:       make(chan error, 1),
+		meekServer:           meekServer,
+		meekSession:          meekSession,
+		underlyingLocalAddr:  underlyingLocalAddr,
+		underlyingRemoteAddr: underlyingRemoteAddr,
+		remoteAddr:           remoteAddr,
+		protocolVersion:      protocolVersion,
+		closeBroadcast:       make(chan struct{}),
+		closed:               0,
+		emptyReadBuffer:      make(chan *bytes.Buffer, 1),
+		partialReadBuffer:    make(chan *bytes.Buffer, 1),
+		fullReadBuffer:       make(chan *bytes.Buffer, 1),
+		nextWriteBuffer:      make(chan []byte, 1),
+		writeResult:          make(chan error, 1),
 	}
 	// Read() calls and pumpReads() are synchronized by exchanging control
 	// of a single readBuffer. This is the same scheme used in and described

psiphon/server/server_packetman_test.go

@@ -0,0 +1,46 @@
+// +build PACKET_MANIPULATOR_TEST
+
+/*
+ * Copyright (c) 2020, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package server
+
+import (
+	"testing"
+)
+
+func TestServerPacketManipulation(t *testing.T) {
+	runServer(t,
+		&runServerConfig{
+			tunnelProtocol:       "UNFRONTED-MEEK-SESSION-TICKET-OSSH",
+			enableSSHAPIRequests: true,
+			doHotReload:          false,
+			doDefaultSponsorID:   false,
+			denyTrafficRules:     false,
+			requireAuthorization: true,
+			omitAuthorization:    false,
+			doTunneledWebRequest: true,
+			doTunneledNTPRequest: true,
+			forceFragmenting:     false,
+			forceLivenessTest:    false,
+			doPruneServerEntries: false,
+			doDanglingTCPConn:    true,
+			doPacketManipulation: true,
+		})
+}

psiphon/server/server_test.go

@@ -131,6 +131,7 @@ func TestSSH(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    true,
+			doPacketManipulation: false,
 		})
 }
 
@@ -150,6 +151,7 @@ func TestOSSH(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    true,
+			doPacketManipulation: false,
 		})
 }
 
@@ -169,6 +171,7 @@ func TestFragmentedOSSH(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    true,
+			doPacketManipulation: false,
 		})
 }
 
@@ -188,6 +191,7 @@ func TestUnfrontedMeek(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    true,
+			doPacketManipulation: false,
 		})
 }
 
@@ -208,6 +212,7 @@ func TestUnfrontedMeekHTTPS(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    true,
+			doPacketManipulation: false,
 		})
 }
 
@@ -228,6 +233,7 @@ func TestUnfrontedMeekHTTPSTLS13(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    true,
+			doPacketManipulation: false,
 		})
 }
 
@@ -248,6 +254,7 @@ func TestUnfrontedMeekSessionTicket(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    true,
+			doPacketManipulation: false,
 		})
 }
 
@@ -268,6 +275,7 @@ func TestUnfrontedMeekSessionTicketTLS13(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    true,
+			doPacketManipulation: false,
 		})
 }
 
@@ -290,6 +298,7 @@ func TestQUICOSSH(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    false,
+			doPacketManipulation: false,
 		})
 }
 
@@ -312,6 +321,7 @@ func TestMarionetteOSSH(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    false,
+			doPacketManipulation: false,
 		})
 }
 
@@ -331,6 +341,7 @@ func TestWebTransportAPIRequests(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    false,
+			doPacketManipulation: false,
 		})
 }
 
@@ -350,6 +361,7 @@ func TestHotReload(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    false,
+			doPacketManipulation: false,
 		})
 }
 
@@ -369,6 +381,7 @@ func TestDefaultSponsorID(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    false,
+			doPacketManipulation: false,
 		})
 }
 
@@ -388,6 +401,7 @@ func TestDenyTrafficRules(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    false,
+			doPacketManipulation: false,
 		})
 }
 
@@ -407,6 +421,7 @@ func TestOmitAuthorization(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    false,
+			doPacketManipulation: false,
 		})
 }
 
@@ -426,6 +441,7 @@ func TestNoAuthorization(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    false,
+			doPacketManipulation: false,
 		})
 }
 
@@ -445,6 +461,7 @@ func TestUnusedAuthorization(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    false,
+			doPacketManipulation: false,
 		})
 }
 
@@ -464,6 +481,7 @@ func TestTCPOnlySLOK(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    false,
+			doPacketManipulation: false,
 		})
 }
 
@@ -483,6 +501,7 @@ func TestUDPOnlySLOK(t *testing.T) {
 			forceLivenessTest:    false,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    false,
+			doPacketManipulation: false,
 		})
 }
 
@@ -502,6 +521,7 @@ func TestLivenessTest(t *testing.T) {
 			forceLivenessTest:    true,
 			doPruneServerEntries: false,
 			doDanglingTCPConn:    false,
+			doPacketManipulation: false,
 		})
 }
 
@@ -521,6 +541,7 @@ func TestPruneServerEntries(t *testing.T) {
 			forceLivenessTest:    true,
 			doPruneServerEntries: true,
 			doDanglingTCPConn:    false,
+			doPacketManipulation: false,
 		})
 }
 
@@ -539,6 +560,7 @@ type runServerConfig struct {
 	forceLivenessTest    bool
 	doPruneServerEntries bool
 	doDanglingTCPConn    bool
+	doPacketManipulation bool
 }
 
 var (
@@ -706,6 +728,8 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
 	// Allow port forwards to local test web server.
 	serverConfig["AllowBogons"] = true
 
+	serverConfig["RunPacketManipulator"] = runConfig.doPacketManipulation
+
 	serverConfigJSON, _ = json.Marshal(serverConfig)
 
 	serverTunnelLog := make(chan map[string]interface{}, 1)
@@ -1171,11 +1195,16 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
 
 	expectClientBPFField := psiphon.ClientBPFEnabled() && doClientTactics
 	expectServerBPFField := ServerBPFEnabled() && doServerTactics
+	expectServerPacketManipulationField := runConfig.doPacketManipulation
 
 	select {
 	case logFields := <-serverTunnelLog:
 		err := checkExpectedServerTunnelLogFields(
-			runConfig, expectClientBPFField, expectServerBPFField, logFields)
+			runConfig,
+			expectClientBPFField,
+			expectServerBPFField,
+			expectServerPacketManipulationField,
+			logFields)
 		if err != nil {
 			t.Fatalf("invalid server tunnel log fields: %s", err)
 		}
@@ -1186,7 +1215,9 @@ func runServer(t *testing.T, runConfig *runServerConfig) {
 	if expectUniqueUser {
 		select {
 		case logFields := <-uniqueUserLog:
-			err := checkExpectedUniqueUserLogFields(runConfig, logFields)
+			err := checkExpectedUniqueUserLogFields(
+				runConfig,
+				logFields)
 			if err != nil {
 				t.Fatalf("invalid unique user log fields: %s", err)
 			}
@@ -1209,6 +1240,7 @@ func checkExpectedServerTunnelLogFields(
 	runConfig *runServerConfig,
 	expectClientBPFField bool,
 	expectServerBPFField bool,
+	expectServerPacketManipulationField bool,
 	fields map[string]interface{}) error {
 
 	// Limitations:
@@ -1385,6 +1417,15 @@ func checkExpectedServerTunnelLogFields(
 		}
 	}
 
+	if expectServerPacketManipulationField {
+		name := "server_packet_manipulation"
+		if fields[name] == nil {
+			return fmt.Errorf("missing expected field '%s'", name)
+		} else if fmt.Sprintf("%s", fields[name]) != "test-packetman-spec" {
+			return fmt.Errorf("unexpected field value %s: '%s'", name, fields[name])
+		}
+	}
+
 	if fields["network_type"].(string) != testNetworkType {
 		return fmt.Errorf("unexpected network_type '%s'", fields["network_type"])
 	}
@@ -1951,7 +1992,10 @@ func paveTacticsConfigFile(
             "Name" : "test-client-bpf",
               "Instructions" : [
                 {"Op": "RetConstant", "Args": {"Val": 65535}}]},
-          "BPFClientTCPProbability" : 1.0
+          "BPFClientTCPProbability" : 1.0,
+          "ServerPacketManipulationSpecs" : [{"Name": "test-packetman-spec", "PacketSpecs": [["TCP-flags S"]]}],
+          "ServerPacketManipulationProbability" : 1.0,
+          "ServerProtocolPacketManipulations": {"All" : ["test-packetman-spec"]}
         }
       },
       "FilteredTactics" : [

psiphon/server/tunnelServer.go

@@ -1058,7 +1058,8 @@ func (sshServer *sshServer) handleClient(
 	}
 
 	serverPacketManipulation := ""
-	if protocol.TunnelProtocolMayUseServerPacketManipulation(tunnelProtocol) {
+	if sshServer.support.Config.RunPacketManipulator &&
+		protocol.TunnelProtocolMayUseServerPacketManipulation(tunnelProtocol) {
 
 		// A meekConn has synthetic address values, including the original client
 		// address in cases where the client uses an upstream proxy to connect to