Add HTTP transforms

psiphon/common/parameters/parameters.go

@@ -239,6 +239,7 @@ const (
 	ReplayAPIRequestPadding                          = "ReplayAPIRequestPadding"
 	ReplayHoldOffTunnel                              = "ReplayHoldOffTunnel"
 	ReplayResolveParameters                          = "ReplayResolveParameters"
+	ReplayHTTPTransformerParameters                  = "ReplayHTTPTransformerParameters"
 	APIRequestUpstreamPaddingMinBytes                = "APIRequestUpstreamPaddingMinBytes"
 	APIRequestUpstreamPaddingMaxBytes                = "APIRequestUpstreamPaddingMaxBytes"
 	APIRequestDownstreamPaddingMinBytes              = "APIRequestDownstreamPaddingMinBytes"
@@ -320,6 +321,12 @@ const (
 	DNSResolverIncludeEDNS0Probability               = "DNSResolverIncludeEDNS0Probability"
 	DNSResolverCacheExtensionInitialTTL              = "DNSResolverCacheExtensionInitialTTL"
 	DNSResolverCacheExtensionVerifiedTTL             = "DNSResolverCacheExtensionVerifiedTTL"
+	DirectHTTPProtocolTransformSpecs                 = "DirectHTTPProtocolTransformSpecs"
+	DirectHTTPProtocolTransformScopedSpecNames       = "DirectHTTPProtocolTransformScopedSpecNames"
+	DirectHTTPProtocolTransformProbability           = "DirectHTTPProtocolTransformProbability"
+	FrontedHTTPProtocolTransformSpecs                = "FrontedHTTPProtocolTransformSpecs"
+	FrontedHTTPProtocolTransformScopedSpecNames      = "FrontedHTTPProtocolTransformScopedSpecNames"
+	FrontedHTTPProtocolTransformProbability          = "FrontedHTTPProtocolTransformProbability"
 )
 
 const (
@@ -574,6 +581,7 @@ var defaultParameters = map[string]struct {
 	ReplayAPIRequestPadding:                {value: true},
 	ReplayHoldOffTunnel:                    {value: true},
 	ReplayResolveParameters:                {value: true},
+	ReplayHTTPTransformerParameters:        {value: true},
 
 	APIRequestUpstreamPaddingMinBytes:   {value: 0, minimum: 0},
 	APIRequestUpstreamPaddingMaxBytes:   {value: 1024, minimum: 0},
@@ -675,6 +683,13 @@ var defaultParameters = map[string]struct {
 	DNSResolverIncludeEDNS0Probability:          {value: 0.0, minimum: 0.0},
 	DNSResolverCacheExtensionInitialTTL:         {value: time.Duration(0), minimum: time.Duration(0)},
 	DNSResolverCacheExtensionVerifiedTTL:        {value: time.Duration(0), minimum: time.Duration(0)},
+
+	DirectHTTPProtocolTransformSpecs:            {value: transforms.Specs{}},
+	DirectHTTPProtocolTransformScopedSpecNames:  {value: transforms.ScopedSpecNames{}},
+	DirectHTTPProtocolTransformProbability:      {value: 0.0, minimum: 0.0},
+	FrontedHTTPProtocolTransformSpecs:           {value: transforms.Specs{}},
+	FrontedHTTPProtocolTransformScopedSpecNames: {value: transforms.ScopedSpecNames{}},
+	FrontedHTTPProtocolTransformProbability:     {value: 0.0, minimum: 0.0},
 }
 
 // IsServerSideOnly indicates if the parameter specified by name is used
@@ -856,6 +871,22 @@ func (p *Parameters) Set(
 	dnsResolverProtocolTransformSpecs, _ :=
 		dnsResolverProtocolTransformSpecsValue.(transforms.Specs)
 
+	directHttpProtocolTransformSpecsValue, err := getAppliedValue(
+		DirectHTTPProtocolTransformSpecs, parameters, applyParameters)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+	directHttpProtocolTransformSpecs, _ :=
+		directHttpProtocolTransformSpecsValue.(transforms.Specs)
+
+	frontedHttpProtocolTransformSpecsValue, err := getAppliedValue(
+		FrontedHTTPProtocolTransformSpecs, parameters, applyParameters)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+	frontedHttpProtocolTransformSpecs, _ :=
+		frontedHttpProtocolTransformSpecsValue.(transforms.Specs)
+
 	for i := 0; i < len(applyParameters); i++ {
 
 		count := 0
@@ -1031,6 +1062,10 @@ func (p *Parameters) Set(
 				var specs transforms.Specs
 				if name == DNSResolverProtocolTransformScopedSpecNames {
 					specs = dnsResolverProtocolTransformSpecs
+				} else if name == DirectHTTPProtocolTransformScopedSpecNames {
+					specs = directHttpProtocolTransformSpecs
+				} else if name == FrontedHTTPProtocolTransformScopedSpecNames {
+					specs = frontedHttpProtocolTransformSpecs
 				}
 
 				err := v.Validate(specs)

psiphon/common/transforms/httpTransformer.go

@@ -0,0 +1,249 @@
+/*
+ * Copyright (c) 2023, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package transforms
+
+import (
+	"bytes"
+	"context"
+	"math"
+	"net"
+	"net/textproto"
+	"strconv"
+
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/errors"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng"
+)
+
+type HTTPTransformerParameters struct {
+	// ProtocolTransformName specifies the name associated with
+	// ProtocolTransformSpec and is used for metrics.
+	ProtocolTransformName string
+
+	// ProtocolTransformSpec specifies a transform to apply to the HTTP request.
+	// See: "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/transforms".
+	//
+	// HTTP transforms include strategies discovered by the Geneva team,
+	// https://geneva.cs.umd.edu.
+	ProtocolTransformSpec Spec
+
+	// ProtocolTransformSeed specifies the seed to use for generating random
+	// data in the ProtocolTransformSpec transform. To replay a transform,
+	// specify the same seed.
+	ProtocolTransformSeed *prng.Seed
+}
+
+const (
+	// httpTransformerReadHeader HTTPTransformer is waiting to finish reading
+	// the next HTTP request header.
+	httpTransformerReadHeader = 0
+	// httpTransformerReadWriteBody HTTPTransformer is waiting to finish reading
+	// and writing the current HTTP request body.
+	httpTransformerReadWriteBody = 1
+)
+
+// HTTPTransformer wraps a net.Conn, intercepting Write calls and applying the
+// specified protocol transform.
+//
+// The HTTP request to be written (input to the Write) is converted to a
+// string, transformed, and converted back to binary and then actually written
+// to the underlying net.Conn.
+//
+// HTTPTransformer is not safe for concurrent use.
+type HTTPTransformer struct {
+	transform Spec
+	seed      *prng.Seed
+
+	// state is the HTTPTransformer state. Possible values are
+	// httpTransformerReadingHeader and httpTransformerReadingBody.
+	state int64
+	// b is the accumulated bytes of the current HTTP request.
+	b []byte
+	// remain is the number of remaining HTTP request body bytes to read into b.
+	remain uint64
+
+	net.Conn
+}
+
+// Warning: Does not handle chunked encoding and multiple HTTP
+// requests written in a single Write(). Must be called synchronously.
+func (t *HTTPTransformer) Write(b []byte) (int, error) {
+
+	if t.state == httpTransformerReadHeader {
+
+		t.b = append(t.b, b...)
+
+		// Wait until the entire HTTP request header has been read. Must check
+		// all accumulated bytes incase the "\r\n\r\n" separator is written over
+		// multiple Write() calls; from reading the net/http code the entire
+		// HTTP request is written in a single Write() call.
+
+		sep := []byte("\r\n\r\n")
+
+		headerBodyLines := bytes.SplitN(t.b, sep, 2) // split header and body
+
+		if len(headerBodyLines) > 1 {
+
+			// read Content-Length before applying transform
+
+			var headerLines [][]byte
+
+			lines := bytes.Split(headerBodyLines[0], []byte("\r\n"))
+			if len(lines) > 1 {
+				// skip request line, e.g. "GET /foo HTTP/1.1"
+				headerLines = lines[1:]
+			}
+
+			var cl []byte
+			contentLengthHeader := []byte("Content-Length:")
+
+			for _, header := range headerLines {
+
+				if bytes.HasPrefix(header, contentLengthHeader) {
+
+					cl = textproto.TrimBytes(header[len(contentLengthHeader):])
+					break
+				}
+			}
+			if len(cl) == 0 {
+				// Either Content-Length header missing or Content-Length
+				// header value is empty, e.g. "Content-Length: ".
+				// b buffered in t.b
+				return len(b), errors.TraceNew("Content-Length missing")
+			}
+
+			n, err := strconv.ParseUint(string(cl), 10, 63)
+			if err != nil {
+				return 0, errors.Trace(err)
+			}
+
+			t.remain = n
+
+			// transform and write header
+
+			headerLen := len(headerBodyLines[0]) + len(sep)
+			header := t.b[:headerLen]
+
+			if t.transform != nil {
+				newHeaderS, err := t.transform.Apply(t.seed, string(header))
+				if err != nil {
+					return 0, errors.Trace(err)
+				}
+
+				newHeader := []byte(newHeaderS)
+
+				// only allocate new slice if header length changed
+				if len(newHeader) == len(header) {
+					copy(t.b[:len(header)], newHeader)
+				} else {
+					t.b = append(newHeader, t.b[len(header):]...)
+				}
+
+				header = newHeader
+			}
+
+			if math.MaxUint64-t.remain < uint64(len(header)) {
+				return 0, errors.TraceNew("t.remain + uint64(len(header)) overflows")
+			}
+			t.remain += uint64(len(header))
+
+			err = t.writeBuffer()
+
+			if t.remain > 0 {
+				// Entire request, header and body, has been written. Return to
+				// waiting for next HTTP request header to arrive.
+				t.state = httpTransformerReadWriteBody
+			}
+
+			if err != nil {
+				// b buffered in t.b
+				return len(b), errors.Trace(err)
+			}
+		}
+
+		// b buffered in t.b
+		return len(b), nil
+	}
+
+	// HTTP request header has been transformed. Write any remaining bytes of
+	// HTTP request header and then write HTTP request body.
+
+	// Must write buffered bytes first, in-order, to write bytes to underlying
+	// Conn in the same order they were received in.
+	err := t.writeBuffer()
+	if err != nil {
+		// b not written or buffered
+		return 0, errors.Trace(err)
+	}
+
+	n, err := t.Conn.Write(b)
+
+	if uint64(n) > t.remain {
+		return 0, errors.TraceNew("t.remain - uint64(n) underflows")
+	}
+
+	t.remain -= uint64(n)
+
+	if t.remain <= 0 {
+		// Entire request, header and body, has been written. Return to
+		// waiting for next HTTP request header to arrive.
+		t.state = httpTransformerReadHeader
+		t.remain = 0
+	}
+
+	return n, errors.Trace(err)
+}
+
+func (t *HTTPTransformer) writeBuffer() error {
+	for len(t.b) > 0 {
+		n, err := t.Conn.Write(t.b)
+
+		if uint64(n) > t.remain {
+			return errors.TraceNew("t.remain - uint64(n) underflows")
+		}
+
+		t.remain -= uint64(n)
+
+		if n == len(t.b) {
+			t.b = nil
+		} else {
+			t.b = t.b[n:]
+		}
+
+		if err != nil {
+			return errors.Trace(err)
+		}
+	}
+	return nil
+}
+
+func WrapDialerWithHTTPTransformer(dialer common.Dialer, params *HTTPTransformerParameters) common.Dialer {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		conn, err := dialer(ctx, network, addr)
+		if err != nil {
+			return nil, errors.Trace(err)
+		}
+		return &HTTPTransformer{
+			Conn:      conn,
+			transform: params.ProtocolTransformSpec,
+			seed:      params.ProtocolTransformSeed,
+		}, nil
+	}
+}

psiphon/common/transforms/httpTransformer_test.go

@@ -0,0 +1,391 @@
+/*
+ * Copyright (c) 2023, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package transforms
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng"
+)
+
+func TestHTTPTransformerHTTPRequest(t *testing.T) {
+
+	type test struct {
+		name           string
+		input          string
+		wantOutput     string
+		wantError      error
+		chunkSize      int
+		transform      Spec
+		connWriteLimit int
+		connWriteErrs  []error
+	}
+
+	tests := []test{
+		{
+			name:       "no transform",
+			input:      "HTTP 1.1\r\nContent-Length: 4\r\n\r\nabcd",
+			wantOutput: "HTTP 1.1\r\nContent-Length: 4\r\n\r\nabcd",
+			chunkSize:  1,
+		},
+		{
+			name:           "no transform with partial write and errors",
+			input:          "HTTP 1.1\r\nContent-Length: 4\r\n\r\nabcd",
+			wantOutput:     "HTTP 1.1\r\nContent-Length: 4\r\n\r\nabcd",
+			chunkSize:      1,
+			connWriteLimit: 1,
+			connWriteErrs:  []error{errors.New("err1"), errors.New("err2")},
+		},
+		{
+			name:       "transform not applied to body",
+			input:      "HTTP 1.1\r\nContent-Length: 4\r\n\r\nabcd",
+			wantOutput: "HTTP 1.1\r\nContent-Length: 4\r\n\r\nabcd",
+			chunkSize:  1,
+			transform:  Spec{[2]string{"abcd", "efgh"}},
+		},
+		{
+			name:      "Content-Length missing",
+			input:     "HTTP 1.1\r\n\r\nabcd",
+			wantError: errors.New("Content-Length missing"),
+			chunkSize: 1,
+		},
+		{
+			name:      "Content-Length overflow",
+			input:     fmt.Sprintf("HTTP 1.1\r\nContent-Length: %d\r\n\r\nabcd", uint64(math.MaxUint64)),
+			wantError: errors.New("strconv.ParseUint: parsing \"18446744073709551615\": value out of range"),
+			chunkSize: 1,
+		},
+		{
+			name:       "no transform",
+			input:      "HTTP 1.1\r\nContent-Length: 4\r\n\r\nabcd",
+			wantOutput: "HTTP 1.1\r\nContent-Length: 4\r\n\r\nabcd",
+			chunkSize:  1,
+		},
+		{
+			name:       "incorrect Content-Length header value",
+			input:      "HTTP 1.1\r\nContent-Length: 3\r\n\r\nabcd",
+			wantOutput: "HTTP 1.1\r\nContent-Length: 3\r\n\r\nabc",
+			chunkSize:  1,
+		},
+		{
+			name:       "single HTTP request written in a single write",
+			input:      "HTTP 1.1\r\nContent-Length: 4\r\n\r\nabcd",
+			wantOutput: "HTTP 1.1\r\nContent-Length: 4\r\n\r\nabcd",
+			chunkSize:  999,
+		},
+		{
+			name:       "transform",
+			input:      "POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nabcd",
+			wantOutput: "POST / HTTP/1.1\r\nContent-Length: 100\r\n\r\nabcd",
+			chunkSize:  1,
+			transform:  Spec{[2]string{"4", "100"}},
+		},
+		{
+			name:           "transform with partial write and errors in header write",
+			input:          "POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nabcd",
+			wantOutput:     "POST / HTTP/1.1\r\nContent-Length: 100\r\n\r\nabcd",
+			chunkSize:      1,
+			transform:      Spec{[2]string{"4", "100"}},
+			connWriteLimit: 1,
+			connWriteErrs:  []error{errors.New("err1"), errors.New("err2")},
+		},
+		{
+			name:           "transform with chunk write and errors in body write",
+			input:          "POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nabcd",
+			wantOutput:     "POST / HTTP/1.1\r\nContent-Length: 100\r\n\r\nabcd",
+			chunkSize:      39,
+			transform:      Spec{[2]string{"4", "100"}},
+			connWriteLimit: 1,
+			connWriteErrs:  []error{errors.New("err1"), errors.New("err2"), errors.New("err3")},
+		},
+		// Multiple HTTP requests written in a single write not supported so an
+		// error is expected.
+		{
+			name:       "multiple HTTP requests written in a single write",
+			input:      "HTTP 1.1\r\nContent-Length: 4\r\n\r\nabcdHTTP 1.1\r\nContent-Length: 2\r\n\r\n12",
+			wantOutput: "HTTP 1.1\r\nContent-Length: 4\r\n\r\nabcdHTTP 1.1\r\nContent-Length: 2\r\n\r\n12",
+			chunkSize:  999,
+			wantError:  errors.New("t.remain - uint64(n) underflows"),
+		},
+		// Multiple HTTP requests written in a single write not supported so an
+		// error is expected because a write will occur where it contains both
+		// the end of the previous HTTP request and the start of a new one.
+		{
+			name:       "multiple HTTP requests written in chunks",
+			input:      "HTTP 1.1\r\nContent-Length: 4\r\n\r\nabcdHTTP 1.1\r\nContent-Length: 2\r\n\r\n12",
+			wantOutput: "HTTP 1.1\r\nContent-Length: 4\r\n\r\nabcdHTTP 1.1\r\nContent-Length: 2\r\n\r\n12",
+			chunkSize:  3,
+			wantError:  errors.New("t.remain - uint64(n) underflows"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			seed, err := prng.NewSeed()
+			if err != nil {
+				t.Fatalf("prng.NewSeed failed %v", err)
+			}
+
+			conn := testConn{
+				writeLimit: tt.connWriteLimit,
+				writeErrs:  tt.connWriteErrs,
+			}
+
+			transformer := &HTTPTransformer{
+				transform: tt.transform,
+				seed:      seed,
+				Conn:      &conn,
+			}
+
+			remain := []byte(tt.input)
+
+			// Write input bytes to transformer in chunks and then check
+			// output.
+			for {
+				if len(remain) == 0 {
+					break
+				}
+
+				var b []byte
+				if len(remain) < tt.chunkSize {
+					b = remain
+				} else {
+					b = remain[:tt.chunkSize]
+				}
+
+				expectedErr := len(conn.writeErrs) > 0
+
+				var n int
+				n, err = transformer.Write(b)
+				if err != nil {
+					if expectedErr {
+						// reset err
+						err = nil
+					} else {
+						// err checked outside loop
+						break
+					}
+				}
+
+				remain = remain[n:]
+			}
+			if tt.wantError == nil {
+				if err != nil {
+					t.Fatalf("unexpected error %v", err)
+				}
+			} else {
+				// tt.wantError != nil
+				if err == nil {
+					t.Fatalf("expected error %v", tt.wantError)
+				} else if !strings.Contains(err.Error(), tt.wantError.Error()) {
+					t.Fatalf("expected error %v got %v", tt.wantError, err)
+				}
+			}
+			if tt.wantError == nil && string(conn.b) != tt.wantOutput {
+				t.Fatalf("expected \"%s\" of len %d but got \"%s\" of len %d", escapeNewlines(tt.wantOutput), len(tt.wantOutput), escapeNewlines(string(conn.b)), len(conn.b))
+			}
+		})
+	}
+}
+
+func TestHTTPTransformerHTTPServer(t *testing.T) {
+
+	type test struct {
+		name      string
+		request   func(string) *http.Request
+		wantBody  string
+		transform Spec
+	}
+
+	tests := []test{
+		{
+			name:      "request body truncated",
+			transform: Spec{[2]string{"Content-Length: 4", "Content-Length: 3"}},
+			request: func(addr string) *http.Request {
+
+				body := bytes.NewReader([]byte("abcd"))
+
+				req, err := http.NewRequest("POST", "http://"+addr, body)
+				if err != nil {
+					panic(err)
+				}
+
+				return req
+			},
+			wantBody: "abc",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			seed, err := prng.NewSeed()
+			if err != nil {
+				t.Fatalf("prng.NewSeed failed %v", err)
+			}
+
+			params := &HTTPTransformerParameters{
+				ProtocolTransformName: "spec",
+				ProtocolTransformSpec: tt.transform,
+				ProtocolTransformSeed: seed,
+			}
+
+			dialer := func(ctx context.Context, network, address string) (net.Conn, error) {
+				return net.Dial(network, address)
+			}
+
+			httpTransport := &http.Transport{
+				DialContext: WrapDialerWithHTTPTransformer(dialer, params),
+			}
+
+			type serverRequest struct {
+				req  *http.Request
+				body []byte
+			}
+
+			serverReq := make(chan *serverRequest, 1)
+
+			http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+				b, err := io.ReadAll(r.Body)
+				if err != nil {
+					w.WriteHeader(http.StatusInternalServerError)
+				}
+				go func() {
+					serverReq <- &serverRequest{
+						req:  r,
+						body: b,
+					}
+					close(serverReq)
+				}()
+			})
+
+			s := &http.Server{
+				Addr: "127.0.0.1:8080",
+			}
+
+			go func() {
+				s.ListenAndServe()
+			}()
+
+			client := http.Client{
+				Transport: httpTransport,
+				Timeout:   2 * time.Second,
+			}
+
+			req := tt.request(s.Addr)
+
+			resp, err := client.Do(req)
+
+			// first shutdown server, then check err
+			shutdownErr := s.Shutdown(context.Background())
+			if shutdownErr != nil {
+				t.Fatalf("s.Shutdown failed %v", shutdownErr)
+			}
+
+			if err != nil {
+				t.Fatalf("client.Do failed %v", err)
+			}
+
+			if resp.StatusCode != http.StatusOK {
+				t.Fatalf("expected 200 but got %d", resp.StatusCode)
+			}
+
+			r := <-serverReq
+
+			if tt.wantBody != string(r.body) {
+				t.Fatalf("expected body %s but got %s", tt.wantBody, string(r.body))
+			}
+		})
+	}
+}
+
+func escapeNewlines(s string) string {
+	s = strings.ReplaceAll(s, "\n", "\\n")
+	s = strings.ReplaceAll(s, "\r", "\\r")
+	return s
+}
+
+type testConn struct {
+	// b is the accumulated bytes from Write() calls.
+	b []byte
+	// writeLimit is the max number of bytes that will be written in a Write()
+	// call.
+	writeLimit int
+	// writeErrs are returned from Write() calls in order. If empty, then a nil
+	// error is returned.
+	writeErrs []error
+}
+
+func (c *testConn) Read(b []byte) (n int, err error) {
+	return 0, nil
+}
+
+func (c *testConn) Write(b []byte) (n int, err error) {
+
+	if len(c.writeErrs) > 0 {
+		err = c.writeErrs[0]
+		c.writeErrs = c.writeErrs[1:]
+	}
+
+	if c.writeLimit != 0 && c.writeLimit < len(b) {
+		c.b = append(c.b, b[:c.writeLimit]...)
+		n = c.writeLimit
+		return
+	}
+
+	c.b = append(c.b, b...)
+	n = len(b)
+	return
+}
+
+func (c *testConn) Close() error {
+	return nil
+}
+
+func (c *testConn) LocalAddr() net.Addr {
+	return nil
+}
+
+func (c *testConn) RemoteAddr() net.Addr {
+	return nil
+}
+
+func (c *testConn) SetDeadline(t time.Time) error {
+	return nil
+}
+
+func (c *testConn) SetReadDeadline(t time.Time) error {
+	return nil
+}
+
+func (c *testConn) SetWriteDeadline(t time.Time) error {
+	return nil
+}

psiphon/common/transforms/transforms.go

@@ -45,7 +45,7 @@ const (
 // data may be retained in the transformed data.
 //
 // For example, with the transform [2]string{"([a-b])", "\\$\\
-// {1\\}"c}, substrings consisting of the characters 'a' and 'b' will be
+// {1\\}c"}, substrings consisting of the characters 'a' and 'b' will be
 // transformed into the same substring with a single character 'c' appended.
 type Spec [][2]string
 

psiphon/config.go

@@ -813,6 +813,13 @@ type Config struct {
 	DNSResolverCacheExtensionInitialTTLMilliseconds  *int
 	DNSResolverCacheExtensionVerifiedTTLMilliseconds *int
 
+	DirectHTTPProtocolTransformSpecs            transforms.Specs
+	DirectHTTPProtocolTransformScopedSpecNames  transforms.ScopedSpecNames
+	DirectHTTPProtocolTransformProbability      *float64
+	FrontedHTTPProtocolTransformSpecs           transforms.Specs
+	FrontedHTTPProtocolTransformScopedSpecNames transforms.ScopedSpecNames
+	FrontedHTTPProtocolTransformProbability     *float64
+
 	// params is the active parameters.Parameters with defaults, config values,
 	// and, optionally, tactics applied.
 	//
@@ -1895,6 +1902,30 @@ func (config *Config) makeConfigParameters() map[string]interface{} {
 		applyParameters[parameters.DNSResolverCacheExtensionVerifiedTTL] = fmt.Sprintf("%dms", *config.DNSResolverCacheExtensionVerifiedTTLMilliseconds)
 	}
 
+	if config.DirectHTTPProtocolTransformSpecs != nil {
+		applyParameters[parameters.DirectHTTPProtocolTransformSpecs] = config.DirectHTTPProtocolTransformSpecs
+	}
+
+	if config.DirectHTTPProtocolTransformScopedSpecNames != nil {
+		applyParameters[parameters.DirectHTTPProtocolTransformScopedSpecNames] = config.DirectHTTPProtocolTransformScopedSpecNames
+	}
+
+	if config.DirectHTTPProtocolTransformProbability != nil {
+		applyParameters[parameters.DirectHTTPProtocolTransformProbability] = *config.DirectHTTPProtocolTransformProbability
+	}
+
+	if config.FrontedHTTPProtocolTransformSpecs != nil {
+		applyParameters[parameters.FrontedHTTPProtocolTransformSpecs] = config.FrontedHTTPProtocolTransformSpecs
+	}
+
+	if config.FrontedHTTPProtocolTransformScopedSpecNames != nil {
+		applyParameters[parameters.FrontedHTTPProtocolTransformScopedSpecNames] = config.FrontedHTTPProtocolTransformScopedSpecNames
+	}
+
+	if config.FrontedHTTPProtocolTransformProbability != nil {
+		applyParameters[parameters.FrontedHTTPProtocolTransformProbability] = *config.FrontedHTTPProtocolTransformProbability
+	}
+
 	// When adding new config dial parameters that may override tactics, also
 	// update setDialParametersHash.
 
@@ -2303,6 +2334,44 @@ func (config *Config) setDialParametersHash() {
 		binary.Write(hash, binary.LittleEndian, int64(*config.DNSResolverCacheExtensionVerifiedTTLMilliseconds))
 	}
 
+	if config.DirectHTTPProtocolTransformSpecs != nil {
+		hash.Write([]byte("DirectHTTPProtocolTransformSpecs"))
+		encodedDirectHTTPProtocolTransformSpecs, _ :=
+			json.Marshal(config.DirectHTTPProtocolTransformSpecs)
+		hash.Write(encodedDirectHTTPProtocolTransformSpecs)
+	}
+
+	if config.DirectHTTPProtocolTransformScopedSpecNames != nil {
+		hash.Write([]byte(""))
+		encodedDirectHTTPProtocolTransformScopedSpecNames, _ :=
+			json.Marshal(config.DirectHTTPProtocolTransformScopedSpecNames)
+		hash.Write(encodedDirectHTTPProtocolTransformScopedSpecNames)
+	}
+
+	if config.DirectHTTPProtocolTransformProbability != nil {
+		hash.Write([]byte("DirectHTTPProtocolTransformProbability"))
+		binary.Write(hash, binary.LittleEndian, *config.DirectHTTPProtocolTransformProbability)
+	}
+
+	if config.FrontedHTTPProtocolTransformSpecs != nil {
+		hash.Write([]byte("FrontedHTTPProtocolTransformSpecs"))
+		encodedFrontedHTTPProtocolTransformSpecs, _ :=
+			json.Marshal(config.FrontedHTTPProtocolTransformSpecs)
+		hash.Write(encodedFrontedHTTPProtocolTransformSpecs)
+	}
+
+	if config.FrontedHTTPProtocolTransformScopedSpecNames != nil {
+		hash.Write([]byte(""))
+		encodedFrontedHTTPProtocolTransformScopedSpecNames, _ :=
+			json.Marshal(config.FrontedHTTPProtocolTransformScopedSpecNames)
+		hash.Write(encodedFrontedHTTPProtocolTransformScopedSpecNames)
+	}
+
+	if config.FrontedHTTPProtocolTransformProbability != nil {
+		hash.Write([]byte("FrontedHTTPProtocolTransformProbability"))
+		binary.Write(hash, binary.LittleEndian, *config.FrontedHTTPProtocolTransformProbability)
+	}
+
 	config.dialParametersHash = hash.Sum(nil)
 }
 

psiphon/dialParameters.go

@@ -38,6 +38,7 @@ import (
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/protocol"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/resolver"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/transforms"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/values"
 	utls "github.com/refraction-networking/utls"
 	regen "github.com/zach-klippenstein/goregen"
@@ -143,6 +144,8 @@ type DialParameters struct {
 	resolver          *resolver.Resolver `json:"-"`
 	ResolveParameters *resolver.ResolveParameters
 
+	HTTPTransformerParameters *transforms.HTTPTransformerParameters
+
 	dialConfig *DialConfig `json:"-"`
 	meekConfig *MeekConfig `json:"-"`
 }
@@ -196,6 +199,7 @@ func MakeDialParameters(
 	replayAPIRequestPadding := p.Bool(parameters.ReplayAPIRequestPadding)
 	replayHoldOffTunnel := p.Bool(parameters.ReplayHoldOffTunnel)
 	replayResolveParameters := p.Bool(parameters.ReplayResolveParameters)
+	replayHTTPTransformerParameters := p.Bool(parameters.ReplayHTTPTransformerParameters)
 
 	// Check for existing dial parameters for this server/network ID.
 
@@ -763,6 +767,22 @@ func MakeDialParameters(
 
 	}
 
+	if (!isReplay || !replayHTTPTransformerParameters) && protocol.TunnelProtocolUsesMeekHTTP(dialParams.TunnelProtocol) {
+
+		isFronted := protocol.TunnelProtocolUsesFrontedMeek(dialParams.TunnelProtocol)
+
+		params, err := makeHTTPTransformerParameters(config.GetParameters().Get(), serverEntry.FrontingProviderID, isFronted)
+		if err != nil {
+			return nil, errors.Trace(err)
+		}
+
+		if params.ProtocolTransformSpec != nil {
+			dialParams.HTTPTransformerParameters = params
+		} else {
+			dialParams.HTTPTransformerParameters = nil
+		}
+	}
+
 	// Set dial address fields. This portion of configuration is
 	// deterministic, given the parameters established or replayed so far.
 
@@ -958,6 +978,7 @@ func MakeDialParameters(
 			MeekObfuscatedKey:             serverEntry.MeekObfuscatedKey,
 			MeekObfuscatorPaddingSeed:     dialParams.MeekObfuscatorPaddingSeed,
 			NetworkLatencyMultiplier:      dialParams.NetworkLatencyMultiplier,
+			HTTPTransformerParameters:     dialParams.HTTPTransformerParameters,
 		}
 
 		// Use an asynchronous callback to record the resolved IP address when
@@ -1066,11 +1087,11 @@ func (dialParams *DialParameters) GetTLSVersionForMetrics() string {
 // There are two concerns regarding which dial parameter fields are safe to
 // exchange:
 //
-// - Unlike signed server entries, there's no independent trust anchor
-//   that can certify that the exchange data is valid.
+//   - Unlike signed server entries, there's no independent trust anchor
+//     that can certify that the exchange data is valid.
 //
-// - While users should only perform the exchange with trusted peers,
-//   the user's trust in their peer may be misplaced.
+//   - While users should only perform the exchange with trusted peers,
+//     the user's trust in their peer may be misplaced.
 //
 // This presents the possibility of attack such as the peer sending dial
 // parameters that could be used to trace/monitor/flag the importer; or
@@ -1355,3 +1376,60 @@ func selectHostName(
 
 	return hostName
 }
+
+// makeHTTPTransformerParameters generates HTTPTransformerParameters using the
+// input tactics parameters and optional frontingProviderID context.
+func makeHTTPTransformerParameters(p parameters.ParametersAccessor,
+	frontingProviderID string, isFronted bool) (*transforms.HTTPTransformerParameters, error) {
+
+	params := transforms.HTTPTransformerParameters{}
+
+	// Select an HTTP transform. If the request is fronted, HTTP request
+	// transforms are "scoped" by fronting provider ID. Otherwise, a transform
+	// from the default scope (transforms.SCOPE_ANY == "") is selected.
+
+	var specsKey string
+	var scopedSpecsNamesKey string
+
+	useTransform := false
+	scope := transforms.SCOPE_ANY
+
+	if isFronted {
+		if p.WeightedCoinFlip(parameters.FrontedHTTPProtocolTransformProbability) {
+			useTransform = true
+			scope = frontingProviderID
+			specsKey = parameters.FrontedHTTPProtocolTransformSpecs
+			scopedSpecsNamesKey = parameters.FrontedHTTPProtocolTransformScopedSpecNames
+		}
+	} else {
+		// unfronted
+		if p.WeightedCoinFlip(parameters.DirectHTTPProtocolTransformProbability) {
+			useTransform = true
+			specsKey = parameters.DirectHTTPProtocolTransformSpecs
+			scopedSpecsNamesKey = parameters.DirectHTTPProtocolTransformScopedSpecNames
+		}
+	}
+
+	if useTransform {
+
+		specs := p.ProtocolTransformSpecs(
+			specsKey)
+		scopedSpecNames := p.ProtocolTransformScopedSpecNames(
+			scopedSpecsNamesKey)
+
+		name, spec := specs.Select(scope, scopedSpecNames)
+
+		if spec != nil {
+			params.ProtocolTransformName = name
+			params.ProtocolTransformSpec = spec
+			var err error
+			// transform seed generated
+			params.ProtocolTransformSeed, err = prng.NewSeed()
+			if err != nil {
+				return nil, errors.Trace(err)
+			}
+		}
+	}
+
+	return &params, nil
+}

psiphon/dialParameters_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/parameters"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/protocol"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/transforms"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/values"
 )
 
@@ -89,6 +90,9 @@ func runDialParametersAndReplay(t *testing.T, tunnelProtocol string) {
 	applyParameters[parameters.HoldOffTunnelFrontingProviderIDs] = []string{frontingProviderID}
 	applyParameters[parameters.HoldOffTunnelProbability] = 1.0
 	applyParameters[parameters.DNSResolverAlternateServers] = []string{"127.0.0.1", "127.0.0.2", "127.0.0.3"}
+	applyParameters[parameters.DirectHTTPProtocolTransformProbability] = 1.0
+	applyParameters[parameters.DirectHTTPProtocolTransformSpecs] = transforms.Specs{"spec": transforms.Spec{{"", ""}}}
+	applyParameters[parameters.DirectHTTPProtocolTransformScopedSpecNames] = transforms.ScopedSpecNames{"": {"spec"}}
 	err = clientConfig.SetParameters("tag1", false, applyParameters)
 	if err != nil {
 		t.Fatalf("SetParameters failed: %s", err)
@@ -356,6 +360,12 @@ func runDialParametersAndReplay(t *testing.T, tunnelProtocol string) {
 		t.Fatalf("mismatching ResolveParameters fields")
 	}
 
+	if (replayDialParams.HTTPTransformerParameters == nil) != (dialParams.HTTPTransformerParameters == nil) ||
+		(replayDialParams.HTTPTransformerParameters != nil &&
+			!reflect.DeepEqual(replayDialParams.HTTPTransformerParameters, dialParams.HTTPTransformerParameters)) {
+		t.Fatalf("mismatching HTTPTransformerParameters fields")
+	}
+
 	// Test: no replay after change tactics
 
 	applyParameters[parameters.ReplayDialParametersTTL] = "1s"
@@ -702,3 +712,97 @@ func makeMockServerEntries(
 
 	return serverEntries
 }
+
+func TestMakeHTTPTransformerParameters(t *testing.T) {
+
+	type test struct {
+		name                  string
+		frontingProviderID    string
+		isFronted             bool
+		paramValues           map[string]interface{}
+		expectedTransformName string
+		expectedTransformSpec transforms.Spec
+	}
+
+	tests := []test{
+		{
+			name:               "unfronted",
+			frontingProviderID: "",
+			isFronted:          false,
+			paramValues: map[string]interface{}{
+				"DirectHTTPProtocolTransformProbability": 1,
+				"DirectHTTPProtocolTransformSpecs": transforms.Specs{
+					"spec1": {{"A", "B"}},
+				},
+				"DirectHTTPProtocolTransformScopedSpecNames": transforms.ScopedSpecNames{
+					"": {"spec1"},
+				},
+			},
+			expectedTransformName: "spec1",
+			expectedTransformSpec: [][2]string{{"A", "B"}},
+		},
+		{
+			name:               "fronted",
+			frontingProviderID: "frontingProvider",
+			isFronted:          true,
+			paramValues: map[string]interface{}{
+				"FrontedHTTPProtocolTransformProbability": 1,
+				"FrontedHTTPProtocolTransformSpecs": transforms.Specs{
+					"spec1": {{"A", "B"}},
+				},
+				"FrontedHTTPProtocolTransformScopedSpecNames": transforms.ScopedSpecNames{
+					"frontingProvider": {"spec1"},
+				},
+			},
+			expectedTransformName: "spec1",
+			expectedTransformSpec: [][2]string{{"A", "B"}},
+		},
+		{
+			name:               "no transform, coinflip false",
+			frontingProviderID: "frontingProvider",
+			isFronted:          false,
+			paramValues: map[string]interface{}{
+				"DirectHTTPProtocolTransformProbability": 0,
+				"DirectHTTPProtocolTransformSpecs": transforms.Specs{
+					"spec1": {{"A", "B"}},
+				},
+				"DirectHTTPProtocolTransformScopedSpecNames": transforms.ScopedSpecNames{
+					"frontingProvider": {"spec1"},
+				},
+			},
+			expectedTransformName: "",
+			expectedTransformSpec: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			params, err := parameters.NewParameters(nil)
+			if err != nil {
+				t.Fatalf("parameters.NewParameters failed %v", err)
+			}
+
+			_, err = params.Set("", false, tt.paramValues)
+			if err != nil {
+				t.Fatalf("params.Set failed %v", err)
+			}
+
+			httpTransformerParams, err := makeHTTPTransformerParameters(params.Get(), tt.frontingProviderID, tt.isFronted)
+			if err != nil {
+				t.Fatalf("MakeHTTPTransformerParameters failed %v", err)
+			}
+			if httpTransformerParams.ProtocolTransformName != tt.expectedTransformName {
+				t.Fatalf("expected ProtocolTransformName \"%s\" but got \"%s\"", tt.expectedTransformName, httpTransformerParams.ProtocolTransformName)
+			}
+			if !reflect.DeepEqual(httpTransformerParams.ProtocolTransformSpec, tt.expectedTransformSpec) {
+				t.Fatalf("expected ProtocolTransformSpec %v but got %v", tt.expectedTransformSpec, httpTransformerParams.ProtocolTransformSpec)
+			}
+			if httpTransformerParams.ProtocolTransformSpec != nil {
+				if httpTransformerParams.ProtocolTransformSeed == nil {
+					t.Fatalf("expected non-nil seed")
+				}
+			}
+		})
+	}
+}

psiphon/meekConn.go

@@ -44,6 +44,7 @@ import (
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/protocol"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/quic"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/transforms"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/values"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/upstreamproxy"
 	"golang.org/x/crypto/nacl/box"
@@ -188,6 +189,10 @@ type MeekConfig struct {
 	MeekCookieEncryptionPublicKey string
 	MeekObfuscatedKey             string
 	MeekObfuscatorPaddingSeed     *prng.Seed
+
+	// HTTPTransformerParameters specifies an HTTP transformer to apply to the
+	// meek connection if it uses HTTP.
+	HTTPTransformerParameters *transforms.HTTPTransformerParameters
 }
 
 // MeekConn is a network connection that tunnels net.Conn flows over HTTP and supports
@@ -585,6 +590,15 @@ func DialMeek(
 			}
 		}
 
+		if protocol.TunnelProtocolUsesMeekHTTP(meekConfig.ClientTunnelProtocol) {
+			// Only apply transformer if it will perform a transform; otherwise
+			// applying a no-op transform will incur an unnecessary performance
+			// cost.
+			if meekConfig.HTTPTransformerParameters != nil && meekConfig.HTTPTransformerParameters.ProtocolTransformSpec != nil {
+				dialer = transforms.WrapDialerWithHTTPTransformer(dialer, meekConfig.HTTPTransformerParameters)
+			}
+		}
+
 		httpTransport := &http.Transport{
 			Proxy:       proxyUrl,
 			DialContext: dialer,

psiphon/notice.go

@@ -579,6 +579,12 @@ func noticeWithDialParameters(noticeType string, dialParams *DialParameters, pos
 			}
 		}
 
+		if dialParams.HTTPTransformerParameters != nil {
+			if dialParams.HTTPTransformerParameters.ProtocolTransformSpec != nil {
+				args = append(args, "HTTPTransform", dialParams.HTTPTransformerParameters.ProtocolTransformName)
+			}
+		}
+
 		if dialParams.DialConnMetrics != nil {
 			metrics := dialParams.DialConnMetrics.GetMetrics()
 			for name, value := range metrics {

psiphon/server/api.go

@@ -925,6 +925,7 @@ var baseDialParams = []requestParamSpec{
 	{"dns_preferred", isAnyString, requestParamOptional},
 	{"dns_transform", isAnyString, requestParamOptional},
 	{"dns_attempt", isIntString, requestParamOptional | requestParamLogStringAsInt},
+	{"http_transform", isAnyString, requestParamOptional},
 }
 
 // baseSessionAndDialParams adds baseDialParams to baseSessionParams.

psiphon/serverApi.go

@@ -1104,6 +1104,12 @@ func getBaseAPIParameters(
 					params["dns_transform"] = dialParams.ResolveParameters.ProtocolTransformName
 				}
 
+				if dialParams.HTTPTransformerParameters != nil {
+					if dialParams.HTTPTransformerParameters.ProtocolTransformSpec != nil {
+						params["http_transform"] = dialParams.HTTPTransformerParameters.ProtocolTransformName
+					}
+				}
+
 				params["dns_attempt"] = strconv.Itoa(
 					dialParams.ResolveParameters.GetFirstAttemptWithAnswer())
 			}