Merge remote-tracking branch 'upstream/master'

.gitignore

@@ -8,6 +8,7 @@ psiphon.boltdb
 # Exclude compiled tunnel core binaries
 ConsoleClient/ConsoleClient
 ConsoleClient/bin
+AndroidLibrary/psi.aar
 
 # Compiled Object files, Static and Dynamic libs (Shared Objects)
 *.o

.travis.yml

@@ -1,6 +1,6 @@
 language: go
 go:
-- 1.5
+- 1.6
 addons:
   apt_packages:
     - libx11-dev
@@ -9,6 +9,12 @@ install:
 - go get -t -d -v ./... && go build -v ./...
 script:
 - go test -v ./...
+- cd psiphon
+- go test -v -covermode=count -coverprofile=coverage.out
+- $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci -repotoken $COVERALLS_TOKEN
 before_install:
-- openssl aes-256-cbc -K $encrypted_9e40808ea1e2_key -iv $encrypted_9e40808ea1e2_iv
+- go get github.com/axw/gocov/gocov
+- go get github.com/mattn/goveralls
+- if ! go get github.com/golang/tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi
+- openssl aes-256-cbc -K $encrypted_bf83b4ab4874_key -iv $encrypted_bf83b4ab4874_iv
   -in psiphon/controller_test.config.enc -out psiphon/controller_test.config -d

AndroidLibrary/Dockerfile

@@ -0,0 +1,97 @@
+# Dockerfile to build an image with the local version of psiphon-tunnel-core.
+#
+# See README.md for usage instructions.
+
+FROM ubuntu:latest
+
+# Install system-level dependencies.
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -y && apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    curl \
+    git \
+    openjdk-7-jdk \
+    pkg-config \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+# Install Go.
+ENV GOVERSION=go1.6 GOROOT=/usr/local/go GOPATH=/go PATH=$PATH:/usr/local/go/bin:/go/bin CGO_ENABLED=1
+
+RUN curl -L https://storage.googleapis.com/golang/$GOVERSION.linux-amd64.tar.gz -o /tmp/go.tar.gz \
+  && tar -C /usr/local -xzf /tmp/go.tar.gz \
+  && rm /tmp/go.tar.gz \
+  && echo $GOVERSION > $GOROOT/VERSION
+
+# Setup Android Environment.
+ENV ANDROID_NDK_ROOT=/android-ndk ANDROID_HOME=/android-sdk-linux
+
+# Setup Android NDK
+RUN cd /tmp \
+  && curl -L http://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin -o /tmp/android-ndk.bin \
+  && chmod a+x /tmp/android-ndk.bin \
+  && /tmp/android-ndk.bin \
+  && rm /tmp/android-ndk.bin \
+  && ln -s $(find /tmp -type d -name 'android-ndk-*') /android-ndk
+
+# Setup Android SDK.
+RUN curl -L http://dl.google.com/android/android-sdk_r24.4.1-linux.tgz -o /tmp/android-sdk.tgz \
+  && tar -C / -xzf /tmp/android-sdk.tgz \
+  && rm /tmp/android-sdk.tgz \
+  && (while true; do echo 'y'; sleep 2; done) | $ANDROID_HOME/tools/android update sdk --no-ui --filter platform,platform-tool,tool
+
+# Setup OpenSSL libray.
+ENV OPENSSL_VERSION=1.0.1p
+ENV PKG_CONFIG_PATH=/tmp/openssl/openssl-$OPENSSL_VERSION
+ENV CGO_CFLAGS="-I $PKG_CONFIG_PATH/include" CGO_LDFLAGS="-L $PKG_CONFIG_PATH -lssl -lcrypto"
+
+RUN mkdir -p /tmp/openssl \
+  && curl -L https://github.com/Psiphon-Labs/psiphon-tunnel-core/raw/master/openssl/openssl-$OPENSSL_VERSION.tar.gz -o /tmp/openssl.tar.gz \
+  && tar -C /tmp/openssl -xzf /tmp/openssl.tar.gz \
+  && rm /tmp/openssl.tar.gz \
+  && curl -L https://github.com/Psiphon-Labs/psiphon-tunnel-core/raw/master/openssl/setenv-android.sh -o /tmp/setenv-android.sh \
+  && /bin/bash -c "\
+    source /tmp/setenv-android.sh \
+    && cd /tmp/openssl/openssl-$OPENSSL_VERSION \
+    && perl -pi -e 's/install: all install_docs install_sw/install: install_docs install_sw/g' Makefile.org \
+    && ./config \
+      no-shared \
+      no-ssl2 \
+      no-ssl3 \
+      no-comp \
+      no-hw \
+      no-md2 \
+      no-md4 \
+      no-rc2 \
+      no-rc5 \
+      no-krb5 \
+      no-ripemd160 \
+      no-idea \
+      no-gost \
+      no-camellia \
+      no-seed \
+      no-3des \
+      no-heartbeats \
+      --openssldir=./ssl \
+    && perl -pi -e 's/-O3/-Os -mfloat-abi=softfp/g' Makefile \
+    && make depend \
+    && make all"
+
+# Install Pinned Gomobile
+#  - Ordered last to allow use of previously cached layers when changing revisions
+ENV GOMOBILE_PINNED_REV=320ec40f6328971c405979b804e20d5f3c86770c
+RUN mkdir -p $GOPATH/pkg/gomobile/dl \
+  && cd $GOPATH/pkg/gomobile/dl \
+  && curl -O https://dl.google.com/go/mobile/gomobile-ndk-r10e-linux-x86_64.tar.gz \
+  && curl -O https://dl.google.com/go/mobile/gomobile-openal-soft-1.16.0.1.tar.gz \
+  && mkdir -p $GOPATH/src/golang.org/x \
+  && cd $GOPATH/src/golang.org/x \
+  && git clone https://github.com/golang/mobile \
+  && cd mobile \
+  && git checkout -b pinned $GOMOBILE_PINNED_REV \
+  && echo "master: $(git rev-parse master)\npinned: $(git rev-parse pinned)" | tee $GOROOT/MOBILE \
+  && go install golang.org/x/mobile/cmd/gomobile \
+  && gomobile init -v
+
+WORKDIR $GOPATH/src

AndroidLibrary/README.md

@@ -1,36 +1,60 @@
-Psiphon Library for Android README
-================================================================================
+##Psiphon Android Library README
 
-Overview
---------------------------------------------------------------------------------
+###Overview
 
 Psiphon Library for Android enables you to easily embed Psiphon in your Android
 app. The Psiphon Library for Android is implemented in Go and follows the standard
 conventions for using a Go library in an Android app.
 
-Status
---------------------------------------------------------------------------------
+###Building with Docker
 
-* Pre-release
+Note that you may need to use `sudo docker` below, depending on your OS.
 
-Building From Source
---------------------------------------------------------------------------------
+#####Create the build image:
 
-Follow Go Android documentation:
-* [gomobile documentation](https://godoc.org/golang.org/x/mobile/cmd/gomobile)
-* Requires Go 1.5 or later.
-* Build command: `gomobile bind -target=android github.com/Psiphon-Labs/psiphon-tunnel-core/AndroidLibrary/psi`
-  * Record build version info, as described [here](https://github.com/Psiphon-Labs/psiphon-tunnel-core/blob/master/README.md#setup), by passing a `-ldflags` argument to `gomobile bind`.
-* Output: `psi.aar`
+  1. Run the command: `docker build --no-cache=true -t psiandroid .` (this may take some time to complete)
+  2. Once completed, verify that you see an image named `psiandroid` when running: `docker images`
 
-Using
---------------------------------------------------------------------------------
+#####Run the build:
 
-1. Build `psi.aar` from source or use the [binary release](https://github.com/Psiphon-Labs/psiphon-tunnel-core/releases)
-1. Add `psi.aar` to your Android Studio project as described in the [gomobile documentation](https://godoc.org/golang.org/x/mobile/cmd/gomobile)
-1. Example usage in [TunneledWebView sample app](../SampleApps/TunneledWebView/README.md)
+  *Ensure that the command below is run from within the `AndroidLibrary` directory*
 
-Limitations
---------------------------------------------------------------------------------
+  ```bash
+  cd .. && \
+    docker run \
+    --rm \
+    -v $(pwd):/go/src/github.com/Psiphon-Labs/psiphon-tunnel-core \
+    psiandroid \
+    /bin/bash -c 'source /tmp/setenv-android.sh && cd /go/src/github.com/Psiphon-Labs/psiphon-tunnel-core/AndroidLibrary && ./make.bash' \
+  ; cd -
+  ```
+When that command completes, the compiled `.aar` file (suitable for use in an Android Studio project) will be located in the current directory (it will likely be owned by root, so be sure to `chown` to an appropriate user).
 
-* Only supports one concurrent instance of Psiphon.
+###Building without Docker (from source)
+
+#####Prerequisites:
+
+ - The `build-essential` package (on Debian based systems - or its equivalent for your platform)
+ - Go 1.5 or later
+ - Full JDK
+ - Android NDK
+ - Android SDK
+ - OpenSSL (tested against the version [here](../openssl))
+  - Follow its [README](../openssl/README.md) to prepare the environment before you follow the steps below
+
+#####Steps:
+
+ 1. Follow Go Android documentation ([gomobile documentation](https://godoc.org/golang.org/x/mobile/cmd/gomobile))
+ - Build command: `gomobile bind -target=android github.com/Psiphon-Labs/psiphon-tunnel-core/AndroidLibrary/psi`
+  - Record build version info, as described [here](../README.md#setup), by passing a `-ldflags` argument to `gomobile bind`.
+  - Output: `psi.aar`
+
+###Using the Library
+
+ 1. Build `psi.aar` from via the docker container, from source, or use the [binary release](https://github.com/Psiphon-Labs/psiphon-tunnel-core/releases)
+ 2. Add `psi.aar` to your Android Studio project as described in the [gomobile documentation](https://godoc.org/golang.org/x/mobile/cmd/gomobile)
+ 3. Example usage in [TunneledWebView sample app](../SampleApps/TunneledWebView/README.md)
+
+#####Limitations
+
+ - Only supports one concurrent instance of Psiphon.

AndroidLibrary/make.bash

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [ ! -f make.bash ]; then
+  echo "make.bash must be run from $GOPATH/src/github.com/Psiphon-Labs/psiphon-tunnel-core/AndroidLibrary"
+  exit 1
+fi
+
+GOOS=arm go get -d -v github.com/Psiphon-Inc/openssl
+GOOS=arm go get -d -v ./...
+if [ $? != 0 ]; then
+  echo "..'go get' failed, exiting"
+  exit $?
+fi
+
+BUILDDATE=$(date --iso-8601=seconds)
+BUILDREPO=$(git config --get remote.origin.url)
+BUILDREV=$(git rev-parse --short HEAD)
+GOVERSION=$(go version | perl -ne '/go version (.*?) / && print $1')
+GOMOBILEVERSION=$(gomobile version | perl -ne '/gomobile version (.*?) / && print $1')
+
+LDFLAGS="\
+-X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildDate=$BUILDDATE \
+-X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildRepo=$BUILDREPO \
+-X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildRev=$BUILDREV \
+-X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.goVersion=$GOVERSION \
+-X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.gomobileVersion=$GOMOBILEVERSION \
+"
+
+echo "Variables for ldflags:"
+echo " Build date: ${BUILDDATE}"
+echo " Build repo: ${BUILDREPO}"
+echo " Build revision: ${BUILDREV}"
+echo " Go version: ${GOVERSION}"
+echo " Gomobile version: ${GOMOBILEVERSION}"
+echo ""
+
+gomobile bind -v -target=android/arm -ldflags="$LDFLAGS" github.com/Psiphon-Labs/psiphon-tunnel-core/AndroidLibrary/psi
+if [ $? != 0 ]; then
+  echo "..'gomobile bind' failed, exiting"
+  exit $?
+fi
+
+echo "Done"

AndroidLibrary/psi/psi.go

@@ -68,6 +68,8 @@ func Start(
 			provider.Notice(string(notice))
 		}))
 
+	psiphon.EmitNoticeBuildInfo()
+
 	// TODO: should following errors be Notices?
 
 	err = psiphon.InitDataStore(config)
@@ -75,7 +77,10 @@ func Start(
 		return fmt.Errorf("error initializing datastore: %s", err)
 	}
 
-	serverEntries, err := psiphon.DecodeAndValidateServerEntryList(embeddedServerEntryList)
+	serverEntries, err := psiphon.DecodeAndValidateServerEntryList(
+		embeddedServerEntryList,
+		psiphon.GetCurrentTimestamp(),
+		psiphon.SERVER_ENTRY_SOURCE_EMBEDDED)
 	if err != nil {
 		return fmt.Errorf("error decoding embedded server entry list: %s", err)
 	}

ConsoleClient/Dockerfile

@@ -2,48 +2,89 @@
 #
 # See README.md for usage instructions.
 
-FROM ubuntu:15.04
-
-ENV GOVERSION=go1.5.3
+FROM ubuntu:latest
 
 # Install system-level dependencies.
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get -y install build-essential curl git mercurial upx gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 mingw-w64 gcc-multilib pkg-config
+RUN apt-get update -y && apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    curl \
+    gcc-mingw-w64-i686 \
+    gcc-mingw-w64-x86-64 \
+    gcc-multilib \
+    git \
+    mingw-w64 \
+    mercurial \
+    pkg-config \
+    upx \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
 
 # Install Go.
-ENV GOROOT=/usr/local/go GOPATH=/go
-ENV PATH=$PATH:$GOROOT/bin:$GOPATH/bin
-
-RUN curl -L https://storage.googleapis.com/golang/$GOVERSION.linux-amd64.tar.gz -o /tmp/go.tar.gz && \
-  tar -C /usr/local -xzf /tmp/go.tar.gz && \
-  rm /tmp/go.tar.gz && \
-  echo $GOVERSION > $GOROOT/VERSION
+ENV GOVERSION=go1.6 GOROOT=/usr/local/go GOPATH=/go PATH=$PATH:/usr/local/go/bin:/go/bin CGO_ENABLED=1
 
-ENV CGO_ENABLED=1
+RUN curl -L https://storage.googleapis.com/golang/$GOVERSION.linux-amd64.tar.gz -o /tmp/go.tar.gz \
+   && tar -C /usr/local -xzf /tmp/go.tar.gz \
+   && rm /tmp/go.tar.gz \
+   && echo $GOVERSION > $GOROOT/VERSION
 
-# Get go dependencies
-RUN go get github.com/mitchellh/gox && go get github.com/pwaller/goupx
+# Get external Go dependencies.
+RUN go get github.com/mitchellh/gox \
+    && go get github.com/pwaller/goupx
 
-# Setup paths for static OpenSSL libray
+# Setup OpenSSL libray.
 ENV OPENSSL_VERSION=1.0.1p
+ENV PKG_CONFIG_PATH_32=/tmp/openssl/32/openssl-$OPENSSL_VERSION PKG_CONFIG_PATH_64=/tmp/openssl/64/openssl-$OPENSSL_VERSION
+
+RUN mkdir -p /tmp/openssl/32 \
+ && mkdir -p /tmp/openssl/64 \
+ && curl -L https://github.com/Psiphon-Labs/psiphon-tunnel-core/raw/master/openssl/openssl-$OPENSSL_VERSION.tar.gz -o /tmp/openssl.tar.gz \
+ && tar -C /tmp/openssl/32 -xzf /tmp/openssl.tar.gz \
+ && tar -C /tmp/openssl/64 -xzf /tmp/openssl.tar.gz \
+ && rm /tmp/openssl.tar.gz
 
-RUN mkdir -p /tmp/openssl/32 && mkdir -p /tmp/openssl/64 && \
-      curl -L https://github.com/Psiphon-Labs/psiphon-tunnel-core/raw/master/openssl/openssl-$OPENSSL_VERSION.tar.gz -o /tmp/openssl.tar.gz && \
-      tar -C /tmp/openssl/32 -xzf /tmp/openssl.tar.gz && \
-      tar -C /tmp/openssl/64 -xzf /tmp/openssl.tar.gz
-
-ENV PKG_CONFIG_PATH_32=/tmp/openssl/32/openssl-$OPENSSL_VERSION
-RUN cd $PKG_CONFIG_PATH_32 && \
-      ./Configure --cross-compile-prefix=i686-w64-mingw32- mingw \
-      no-shared no-ssl2 no-ssl3 no-comp no-hw no-md2 no-md4 no-rc2 no-rc5 no-krb5 no-ripemd160 no-idea no-gost no-camellia no-seed no-3des no-heartbeats && \
-      make depend && \
-      make
-
-ENV PKG_CONFIG_PATH_64=/tmp/openssl/64/openssl-$OPENSSL_VERSION
-RUN cd $PKG_CONFIG_PATH_64 && \
-      ./Configure --cross-compile-prefix=x86_64-w64-mingw32- mingw64 \
-      no-shared no-ssl2 no-ssl3 no-comp no-hw no-md2 no-md4 no-rc2 no-rc5 no-krb5 no-ripemd160 no-idea no-gost no-camellia no-seed no-3des no-heartbeats && \
-      make depend && \
-      make
+RUN cd $PKG_CONFIG_PATH_32 \
+  && ./Configure --cross-compile-prefix=i686-w64-mingw32- mingw \
+    no-shared \
+    no-ssl2 \
+    no-ssl3 \
+    no-comp \
+    no-hw \
+    no-md2 \
+    no-md4 \
+    no-rc2 \
+    no-rc5 \
+    no-krb5 \
+    no-ripemd160 \
+    no-idea \
+    no-gost \
+    no-camellia \
+    no-seed \
+    no-3des \
+    no-heartbeats \
+  && make depend \
+  && make \
+  && cd $PKG_CONFIG_PATH_64 \
+  && ./Configure --cross-compile-prefix=x86_64-w64-mingw32- mingw64 \
+    no-shared \
+    no-ssl2 \
+    no-ssl3 \
+    no-comp \
+    no-hw \
+    no-md2 \
+    no-md4 \
+    no-rc2 \
+    no-rc5 \
+    no-krb5 \
+    no-ripemd160 \
+    no-idea \
+    no-gost \
+    no-camellia \
+    no-seed \
+    no-3des \
+    no-heartbeats \
+  && make depend \
+  && make
 
 WORKDIR $GOPATH/src

ConsoleClient/make.bash

@@ -12,11 +12,13 @@ BUILDINFOFILE="${EXE_BASENAME}_buildinfo.txt"
 BUILDDATE=$(date --iso-8601=seconds)
 BUILDREPO=$(git config --get remote.origin.url)
 BUILDREV=$(git rev-parse --short HEAD)
+GOVERSION=$(go version | perl -ne '/go version (.*?) / && print $1')
 
 LDFLAGS="\
--X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildDate $BUILDDATE \
--X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildRepo $BUILDREPO \
--X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildRev $BUILDREV \
+-X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildDate=$BUILDDATE \
+-X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildRepo=$BUILDREPO \
+-X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildRev=$BUILDREV \
+-X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.goVersion=$GOVERSION \
 "
 echo -e "${BUILDDATE}\n${BUILDREPO}\n${BUILDREV}\n" > $BUILDINFOFILE
 
@@ -24,6 +26,7 @@ echo "Variables for ldflags:"
 echo " Build date: ${BUILDDATE}"
 echo " Build repo: ${BUILDREPO}"
 echo " Build revision: ${BUILDREV}"
+echo " Go version: ${GOVERSION}"
 echo ""
 
 if [ ! -d bin ]; then
@@ -33,6 +36,10 @@ fi
 build_for_windows () {
   echo "...Getting project dependencies (via go get) for Windows. Parameter is: '$1'"
   GOOS=windows go get -d -v ./...
+  if [ $? != 0 ]; then
+    echo "....'go get' failed, exiting"
+    exit $?
+  fi
 
   if [ -z $1 ] || [ "$1" == "32" ]; then
     unset PKG_CONFIG_PATH
@@ -45,6 +52,13 @@ build_for_windows () {
     CGO_LDFLAGS="-L $PKG_CONFIG_PATH -L /usr/i686-w64-mingw32/lib/ -lssl -lcrypto -lwsock32 -lcrypt32 -lgdi32" \
     CC=/usr/bin/i686-w64-mingw32-gcc \
     gox -verbose -ldflags "$LDFLAGS" -osarch windows/386 -output bin/windows/${EXE_BASENAME}-i686
+    RETVAL=$?
+    echo ".....gox completed, exit code: $?"
+    if [ $RETVAL != 0 ]; then
+      echo ".....gox failed, exiting"
+      exit $RETVAL
+    fi
+    unset RETVAL
 
     ## We are finding that UPXing the full Windows Psiphon client produces better results if psiphon-tunnel-core.exe is not already UPX'd.
     echo "....No UPX for this build"
@@ -61,6 +75,12 @@ build_for_windows () {
     CGO_LDFLAGS="-L $PKG_CONFIG_PATH -L /usr/x86_64-w64-mingw32/lib/ -lssl -lcrypto -lwsock32 -lcrypt32 -lgdi32" \
     CC=/usr/bin/x86_64-w64-mingw32-gcc \
     gox -verbose -ldflags "$LDFLAGS" -osarch windows/amd64 -output bin/windows/${EXE_BASENAME}-x86_64
+    RETVAL=$?
+    if [ $RETVAL != 0 ]; then
+      echo ".....gox failed, exiting"
+      exit $RETVAL
+    fi
+    unset RETVAL
 
     # We are finding that UPXing the full Windows Psiphon client produces better results if psiphon-tunnel-core.exe is not already UPX'd.
     echo "....No UPX for this build"
@@ -70,25 +90,59 @@ build_for_windows () {
 build_for_linux () {
   echo "Getting project dependencies (via go get) for Linux. Parameter is: '$1'"
   GOOS=linux go get -d -v ./...
+  if [ $? != 0 ]; then
+    echo "...'go get' failed, exiting"
+    exit $?
+  fi
 
   if [ -z $1 ] || [ "$1" == "32" ]; then
     echo "...Building linux-i686"
     CFLAGS=-m32 gox -verbose -ldflags "$LDFLAGS" -osarch linux/386 -output bin/linux/${EXE_BASENAME}-i686
+    RETVAL=$?
+    if [ $RETVAL != 0 ]; then
+      echo ".....gox failed, exiting"
+      exit $RETVAL
+    fi
+    unset RETVAL
+
     echo "....UPX packaging output"
     goupx --best bin/linux/${EXE_BASENAME}-i686
+    RETVAL=$?
+    if [ $RETVAL != 0 ]; then
+      echo ".....goupx failed, exiting"
+      exit $RETVAL
+    fi
+    unset RETVAL
   fi
 
   if [ -z $1 ] || [ "$1" == "64" ]; then
     echo "...Building linux-x86_64"
     gox -verbose -ldflags "$LDFLAGS" -osarch linux/amd64 -output bin/linux/${EXE_BASENAME}-x86_64
+    RETVAL=$?
+    if [ $RETVAL != 0 ]; then
+      echo "....gox failed, exiting"
+      exit $RETVAL
+    fi
+    unset RETVAL
+
     echo "....UPX packaging output"
     goupx --best bin/linux/${EXE_BASENAME}-x86_64
+    RETVAL=$?
+    if [ $RETVAL != 0 ]; then
+      echo ".....goupx failed, exiting"
+      exit $RETVAL
+    fi
+    unset RETVAL
   fi
 }
 
 build_for_osx () {
   echo "Getting project dependencies (via go get) for OSX"
   GOOS=darwin go get -d -v ./...
+  if [ $? != 0 ]; then
+    echo "..'go get' failed, exiting"
+    exit $?
+  fi
 
   echo "Building darwin-x86_64..."
   echo "..Disabling CGO for this build"
@@ -102,26 +156,56 @@ case $TARGET in
   windows)
     echo "..Building for Windows"
     build_for_windows $2
+    exit $?
+
     ;;
   linux)
     echo "..Building for Linux"
     build_for_linux $2
+    exit $?
+
     ;;
   osx)
     echo "..Building for OSX"
     build_for_osx
+    exit $?
+
     ;;
   all)
     echo "..Building all"
     build_for_windows $2
+    if [ $? != 0 ]; then
+      exit $?
+    fi
+
     build_for_linux $2
+    if [ $? != 0 ]; then
+      exit $?
+    fi
+
     build_for_osx
+    if [ $? != 0 ]; then
+      exit $?
+    fi
+
     ;;
   *)
     echo "..No selection made, building all"
     build_for_windows $2
+    if [ $? != 0 ]; then
+      exit $?
+    fi
+
     build_for_linux $2
+    if [ $? != 0 ]; then
+      exit $?
+    fi
+
     build_for_osx
+    if [ $? != 0 ]; then
+      exit $?
+    fi
+
     ;;
 
 esac

ConsoleClient/psiphonClient.go

@@ -61,6 +61,8 @@ func main() {
 	}
 	psiphon.SetNoticeOutput(noticeWriter)
 
+	psiphon.EmitNoticeBuildInfo()
+
 	// Handle required config file parameter
 
 	if configFilename == "" {
@@ -135,7 +137,10 @@ func main() {
 				return
 			}
 			// TODO: stream embedded server list data? also, the cast makes an unnecessary copy of a large buffer?
-			serverEntries, err := psiphon.DecodeAndValidateServerEntryList(string(serverEntryList))
+			serverEntries, err := psiphon.DecodeAndValidateServerEntryList(
+				string(serverEntryList),
+				psiphon.GetCurrentTimestamp(),
+				psiphon.SERVER_ENTRY_SOURCE_EMBEDDED)
 			if err != nil {
 				psiphon.NoticeError("error decoding embedded server entry list file: %s", err)
 				return

README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/Psiphon-Labs/psiphon-tunnel-core.png)](https://travis-ci.org/Psiphon-Labs/psiphon-tunnel-core)
+[![Build Status](https://travis-ci.org/Psiphon-Labs/psiphon-tunnel-core.png)](https://travis-ci.org/Psiphon-Labs/psiphon-tunnel-core) [![Coverage Status](https://coveralls.io/repos/github/Psiphon-Labs/psiphon-tunnel-core/badge.svg?branch=master)](https://coveralls.io/github/Psiphon-Labs/psiphon-tunnel-core?branch=master)
 
 Psiphon 3 Tunnel Core README
 ================================================================================
@@ -13,7 +13,7 @@ This component does not include a UI and does not handle capturing or routing lo
 Status
 --------------------------------------------------------------------------------
 
-This project is currently at the proof-of-concept stage. Current production Psiphon client code is available at our [Psiphon 3 repository](https://bitbucket.org/psiphon/psiphon-circumvention-system).
+This project is in production and used as the tunneling engine in our Windows and Android clients, which are available at our [Psiphon 3 repository](https://bitbucket.org/psiphon/psiphon-circumvention-system).
 
 Setup
 --------------------------------------------------------------------------------

SampleApps/Psibot/app/src/main/java/ca/psiphon/PsiphonTunnel.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, Psiphon Inc.
+ * Copyright (c) 2015, Psiphon Inc.
  * All rights reserved.
  *
  * This program is free software: you can redistribute it and/or modify
@@ -27,6 +27,7 @@ import android.net.NetworkInfo;
 import android.net.VpnService;
 import android.os.Build;
 import android.os.ParcelFileDescriptor;
+import android.telephony.TelephonyManager;
 import android.util.Base64;
 
 import org.apache.http.conn.util.InetAddressUtils;
@@ -90,6 +91,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     }
 
     private final HostService mHostService;
+    private AtomicBoolean mVpnMode;
     private PrivateAddress mPrivateAddress;
     private AtomicReference<ParcelFileDescriptor> mTunFd;
     private AtomicInteger mLocalSocksProxyPort;
@@ -113,6 +115,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
 
     private PsiphonTunnel(HostService hostService) {
         mHostService = hostService;
+        mVpnMode = new AtomicBoolean(false);
         mTunFd = new AtomicReference<ParcelFileDescriptor>();
         mLocalSocksProxyPort = new AtomicInteger(0);
         mRoutingThroughTunnel = new AtomicBoolean(false);
@@ -148,10 +151,11 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         stopPsiphon();
         startPsiphon("");
     }
-
+    
     public synchronized void stop() {
         stopVpn();
         stopPsiphon();
+        mVpnMode.set(false);
         mLocalSocksProxyPort.set(0);
     }
 
@@ -164,7 +168,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     private final static int UDPGW_SERVER_PORT = 7300;
     private final static String DEFAULT_PRIMARY_DNS_SERVER = "8.8.4.4";
     private final static String DEFAULT_SECONDARY_DNS_SERVER = "8.8.8.8";
-
+    
     // Note: Atomic variables used for getting/setting local proxy port, routing flag, and
     // tun fd, as these functions may be called via PsiphonProvider callbacks. Do not use
     // synchronized functions as stop() is synchronized and a deadlock is possible as callbacks
@@ -173,6 +177,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     @TargetApi(Build.VERSION_CODES.ICE_CREAM_SANDWICH)
     private boolean startVpn() throws Exception {
 
+        mVpnMode.set(true);
         mPrivateAddress = selectPrivateAddress();
 
         Locale previousLocale = Locale.getDefault();
@@ -183,20 +188,21 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             Locale.setDefault(new Locale("en"));
 
             ParcelFileDescriptor tunFd =
-                    ((VpnService.Builder) mHostService.newVpnServiceBuilder())
-                            .setSession(mHostService.getAppName())
-                            .setMtu(VPN_INTERFACE_MTU)
-                            .addAddress(mPrivateAddress.mIpAddress, mPrivateAddress.mPrefixLength)
-                            .addRoute("0.0.0.0", 0)
-                            .addRoute(mPrivateAddress.mSubnet, mPrivateAddress.mPrefixLength)
-                            .addDnsServer(mPrivateAddress.mRouter)
-                            .establish();
+                ((VpnService.Builder) mHostService.newVpnServiceBuilder())
+                    .setSession(mHostService.getAppName())
+                    .setMtu(VPN_INTERFACE_MTU)
+                    .addAddress(mPrivateAddress.mIpAddress, mPrivateAddress.mPrefixLength)
+                    .addRoute("0.0.0.0", 0)
+                    .addRoute(mPrivateAddress.mSubnet, mPrivateAddress.mPrefixLength)
+                    .addDnsServer(mPrivateAddress.mRouter)
+                    .establish();
             if (tunFd == null) {
                 // As per http://developer.android.com/reference/android/net/VpnService.Builder.html#establish%28%29,
                 // this application is no longer prepared or was revoked.
                 return false;
             }
             mTunFd.set(tunFd);
+            mRoutingThroughTunnel.set(false);
 
             mHostService.onDiagnosticMessage("VPN established");
 
@@ -215,7 +221,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     }
 
     private boolean isVpnMode() {
-        return mTunFd.get() != null;
+        return mVpnMode.get();
     }
 
     private void setLocalSocksProxyPort(int port) {
@@ -226,10 +232,14 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         if (!mRoutingThroughTunnel.compareAndSet(false, true)) {
             return;
         }
+        ParcelFileDescriptor tunFd = mTunFd.getAndSet(null);
+        if (tunFd == null) {
+            return;
+        }
         String socksServerAddress = "127.0.0.1:" + Integer.toString(mLocalSocksProxyPort.get());
         String udpgwServerAddress = "127.0.0.1:" + Integer.toString(UDPGW_SERVER_PORT);
         startTun2Socks(
-                mTunFd.get(),
+                tunFd,
                 VPN_INTERFACE_MTU,
                 mPrivateAddress.mRouter,
                 VPN_INTERFACE_NETMASK,
@@ -243,6 +253,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     }
 
     private void stopVpn() {
+        stopTun2Socks();
         ParcelFileDescriptor tunFd = mTunFd.getAndSet(null);
         if (tunFd != null) {
             try {
@@ -250,10 +261,9 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             } catch (IOException e) {
             }
         }
-        waitStopTun2Socks();
         mRoutingThroughTunnel.set(false);
     }
-
+    
     //----------------------------------------------------------------------------------------------
     // PsiphonProvider (Core support) interface implementation
     //----------------------------------------------------------------------------------------------
@@ -311,10 +321,10 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         mHostService.onDiagnosticMessage("starting Psiphon library");
         try {
             Psi.Start(
-                    loadPsiphonConfig(mHostService.getContext()),
-                    embeddedServerEntries,
-                    this,
-                    isVpnMode());
+                loadPsiphonConfig(mHostService.getContext()),
+                embeddedServerEntries,
+                this,
+                isVpnMode());
         } catch (java.lang.Exception e) {
             throw new Exception("failed to start Psiphon library", e);
         }
@@ -333,7 +343,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         // Load settings from the raw resource JSON config file and
         // update as necessary. Then write JSON to disk for the Go client.
         JSONObject json = new JSONObject(mHostService.getPsiphonConfig());
-
+        
         // On Android, these directories must be set to the app private storage area.
         // The Psiphon library won't be able to use its current working directory
         // and the standard temporary directories do not exist.
@@ -360,7 +370,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         }
 
         json.put("UseIndistinguishableTLS", true);
-
+        
         if (Build.VERSION.SDK_INT < Build.VERSION_CODES.ICE_CREAM_SANDWICH) {
             json.put("UseTrustedCACertificatesForStockTLS", true);
         }
@@ -369,11 +379,13 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             // Also enable indistinguishable TLS for HTTPS requests that
             // require system CAs.
             json.put(
-                    "TrustedCACertificatesFilename",
-                    setupTrustedCertificates(mHostService.getContext()));
+                "TrustedCACertificatesFilename",
+                setupTrustedCertificates(mHostService.getContext()));
         } catch (Exception e) {
             mHostService.onDiagnosticMessage(e.getMessage());
         }
+        
+        json.put("DeviceRegion", getDeviceRegion(mHostService.getContext()));
 
         return json.toString();
     }
@@ -383,10 +395,10 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             // All notices are sent on as diagnostic messages
             // except those that may contain private user data.
             boolean diagnostic = true;
-
+            
             JSONObject notice = new JSONObject(noticeJSON);
             String noticeType = notice.getString("noticeType");
-
+            
             if (noticeType.equals("Tunnels")) {
                 int count = notice.getJSONObject("data").getInt("count");
                 if (count > 0) {
@@ -405,7 +417,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
                     regions.add(egressRegions.getString(i));
                 }
                 mHostService.onAvailableEgressRegions(regions);
-
+                
             } else if (noticeType.equals("SocksProxyPortInUse")) {
                 mHostService.onSocksProxyPortInUse(notice.getJSONObject("data").getInt("port"));
 
@@ -537,6 +549,24 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         }
     }
 
+    private static String getDeviceRegion(Context context) {
+        String region = "";
+        TelephonyManager telephonyManager = (TelephonyManager)context.getSystemService(Context.TELEPHONY_SERVICE);
+        if (telephonyManager != null) {
+            region = telephonyManager.getSimCountryIso();
+            if (region.length() == 0 && telephonyManager.getPhoneType() != TelephonyManager.PHONE_TYPE_CDMA) {
+                region = telephonyManager.getNetworkCountryIso();
+            }
+        }
+        if (region.length() == 0) {
+            Locale defaultLocale = Locale.getDefault();
+            if (defaultLocale != null) {
+                region = defaultLocale.getCountry();
+            }
+        }
+        return region.toUpperCase(Locale.US);
+    }
+    
     //----------------------------------------------------------------------------------------------
     // Tun2Socks
     //----------------------------------------------------------------------------------------------
@@ -550,11 +580,14 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             final String socksServerAddress,
             final String udpgwServerAddress,
             final boolean udpgwTransparentDNS) {
+        if (mTun2SocksThread != null) {
+            return;
+        }
         mTun2SocksThread = new Thread(new Runnable() {
             @Override
             public void run() {
                 runTun2Socks(
-                        vpnInterfaceFileDescriptor.getFd(),
+                        vpnInterfaceFileDescriptor.detachFd(),
                         vpnInterfaceMTU,
                         vpnIpAddress,
                         vpnNetMask,
@@ -567,10 +600,10 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         mHostService.onDiagnosticMessage("tun2socks started");
     }
 
-    private void waitStopTun2Socks() {
+    private void stopTun2Socks() {
         if (mTun2SocksThread != null) {
             try {
-                // Assumes mTunFd has been closed, which signals tun2socks to exit
+                terminateTun2Socks();
                 mTun2SocksThread.join();
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
@@ -593,6 +626,8 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             String socksServerAddress,
             String udpgwServerAddress,
             int udpgwTransparentDNS);
+    
+    private native static int terminateTun2Socks();
 
     static {
         System.loadLibrary("tun2socks");

SampleApps/TunneledWebView/app/src/main/java/ca/psiphon/PsiphonTunnel.java

@@ -24,6 +24,7 @@ import android.content.Context;
 import android.net.ConnectivityManager;
 import android.net.NetworkInfo;
 import android.os.Build;
+import android.telephony.TelephonyManager;
 import android.util.Base64;
 
 import org.json.JSONArray;
@@ -43,6 +44,7 @@ import java.security.cert.X509Certificate;
 import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.List;
+import java.util.Locale;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import go.psi.Psi;
@@ -224,6 +226,8 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             mTunneledApp.onDiagnosticMessage(e.getMessage());
         }
 
+        json.put("DeviceRegion", getDeviceRegion(mTunneledApp.getContext()));
+
         return json.toString();
     }
 
@@ -381,4 +385,22 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             throw new Exception(errorMessage, e);
         }
     }
+
+    private static String getDeviceRegion(Context context) {
+        String region = "";
+        TelephonyManager telephonyManager = (TelephonyManager)context.getSystemService(Context.TELEPHONY_SERVICE);
+        if (telephonyManager != null) {
+            region = telephonyManager.getSimCountryIso();
+            if (region.length() == 0 && telephonyManager.getPhoneType() != TelephonyManager.PHONE_TYPE_CDMA) {
+                region = telephonyManager.getNetworkCountryIso();
+            }
+        }
+        if (region.length() == 0) {
+            Locale defaultLocale = Locale.getDefault();
+            if (defaultLocale != null) {
+                region = defaultLocale.getCountry();
+            }
+        }
+        return region.toUpperCase(Locale.US);
+    }
 }

psiphon/TCPConn.go

@@ -61,6 +61,17 @@ func makeTCPDialer(config *DialConfig) func(network, addr string) (net.Conn, err
 		if err != nil {
 			return nil, ContextError(err)
 		}
+		// Note: when an upstream proxy is used, we don't know what IP address
+		// was resolved, by the proxy, for that destination.
+		if config.ResolvedIPCallback != nil && config.UpstreamProxyUrl == "" {
+			remoteAddr := conn.RemoteAddr()
+			if remoteAddr != nil {
+				host, _, err := net.SplitHostPort(conn.RemoteAddr().String())
+				if err == nil {
+					config.ResolvedIPCallback(host)
+				}
+			}
+		}
 		return conn, nil
 	}
 }
@@ -83,7 +94,7 @@ func interruptibleTCPDial(addr string, config *DialConfig) (*TCPConn, error) {
 	conn := &TCPConn{dialResult: make(chan error, 1)}
 
 	// Enable interruption
-	if !config.PendingConns.Add(conn) {
+	if config.PendingConns != nil && !config.PendingConns.Add(conn) {
 		return nil, ContextError(errors.New("pending connections already closed"))
 	}
 

psiphon/buildinfo.go

@@ -23,24 +23,32 @@ import "strings"
 
 /*
 These values should be filled in at build time using the `-X` option[1] to the
-Go linker (probably via `-ldflags` option to `go build` -- like `-ldflags "-X var1 abc -X var2 xyz"`).
+Go linker (probably via `-ldflags` option to `go build` -- like `-ldflags "-X var1=abc -X var2=xyz"`).
 [1]: http://golang.org/cmd/ld/
 Without those build flags, the build info in the notice will simply be empty strings.
 Suggestions for how to fill in the values will be given for each variable.
+Note that any passed value must contain no whitespace.
 */
-// -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildDate `date --iso-8601=seconds`
+// -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildDate=`date --iso-8601=seconds`
 var buildDate string
 
-// -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildRepo `git config --get remote.origin.url`
+// -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildRepo=`git config --get remote.origin.url`
 var buildRepo string
 
-// -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildRev `git rev-parse HEAD`
+// -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildRev=`git rev-parse --short HEAD`
 var buildRev string
 
-func NoticeBuildInfo() {
-	NoticeInfo(
-		"Built: %#v from %#v at rev %#v",
+// -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.goVersion=`go version | perl -ne '/go version (.*?) / && print $1'`
+var goVersion string
+
+// -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.gomobileVersion=`gomobile version | perl -ne '/gomobile version (.*?) / && print $1'`
+var gomobileVersion string
+
+func EmitNoticeBuildInfo() {
+	NoticeBuildInfo(
 		strings.TrimSpace(buildDate),
 		strings.TrimSpace(buildRepo),
-		strings.TrimSpace(buildRev))
+		strings.TrimSpace(buildRev),
+		strings.TrimSpace(goVersion),
+		strings.TrimSpace(gomobileVersion))
 }

psiphon/config.go

@@ -22,7 +22,9 @@ package psiphon
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"os"
+	"strconv"
 	"time"
 )
 
@@ -65,7 +67,8 @@ const (
 	PSIPHON_API_TUNNEL_STATS_MAX_COUNT             = 1000
 	FETCH_ROUTES_TIMEOUT                           = 1 * time.Minute
 	DOWNLOAD_UPGRADE_TIMEOUT                       = 15 * time.Minute
-	DOWNLOAD_UPGRADE_RETRY_PAUSE_PERIOD            = 5 * time.Second
+	DOWNLOAD_UPGRADE_RETRY_PERIOD                  = 5 * time.Second
+	DOWNLOAD_UPGRADE_STALE_PERIOD                  = 6 * time.Hour
 	IMPAIRED_PROTOCOL_CLASSIFICATION_DURATION      = 2 * time.Minute
 	IMPAIRED_PROTOCOL_CLASSIFICATION_THRESHOLD     = 3
 	TOTAL_BYTES_TRANSFERRED_NOTICE_PERIOD          = 5 * time.Minute
@@ -150,8 +153,8 @@ type Config struct {
 
 	// TunnelProtocol indicates which protocol to use. Valid values include:
 	// "SSH", "OSSH", "UNFRONTED-MEEK-OSSH", "UNFRONTED-MEEK-HTTPS-OSSH",
-	// "FRONTED-MEEK-OSSH". For the default, "", the best performing protocol
-	// is used.
+	// "FRONTED-MEEK-OSSH", "FRONTED-MEEK-HTTP-OSSH". For the default, "",
+	// the best performing protocol is used.
 	TunnelProtocol string
 
 	// EstablishTunnelTimeoutSeconds specifies a time limit after which to halt
@@ -208,6 +211,10 @@ type Config struct {
 	// This parameter is only applicable to library deployments.
 	DnsServerGetter DnsServerGetter
 
+	// HostNameTransformer is an interface that enables pluggable hostname
+	// transformation circumvention strategies.
+	HostNameTransformer HostNameTransformer
+
 	// TargetServerEntry is an encoded server entry. When specified, this server entry
 	// is used exclusively and all other known servers are ignored.
 	TargetServerEntry string
@@ -248,6 +255,13 @@ type Config struct {
 	// typically embedded in the client binary.
 	UpgradeDownloadUrl string
 
+	// UpgradeDownloadClientVersionHeader specifies the HTTP header name for the
+	// entity at UpgradeDownloadUrl which specifies the client version (an integer
+	// value). A HEAD request may be made to check the version number available at
+	// UpgradeDownloadUrl. UpgradeDownloadClientVersionHeader is required when
+	// UpgradeDownloadUrl is specified.
+	UpgradeDownloadClientVersionHeader string
+
 	// UpgradeDownloadFilename is the local target filename for an upgrade download.
 	// This parameter is required when UpgradeDownloadUrl is specified.
 	UpgradeDownloadFilename string
@@ -281,6 +295,20 @@ type Config struct {
 	// 1-2 minutes, when the tunnel is idle. If the SSH keepalive times out, the tunnel
 	// is considered to have failed.
 	DisablePeriodicSshKeepAlive bool
+
+	// DeviceRegion is the optional, reported region the host device is running in.
+	// This input value should be a ISO 3166-1 alpha-2 country code. The device region
+	// is reported to the server in the connected request and recorded for Psiphon
+	// stats.
+	// When provided, this value may be used, pre-connection, to select performance
+	// or circumvention optimization strategies for the given region.
+	DeviceRegion string
+
+	// EmitDiagnosticNotices indicates whether to output notices containing detailed
+	// information about the Psiphon session. As these notices may contain sensitive
+	// network information, they should not be insecurely distributed or displayed
+	// to users. Default is off.
+	EmitDiagnosticNotices bool
 }
 
 // LoadConfig parses and validates a JSON format Psiphon config JSON
@@ -292,6 +320,11 @@ func LoadConfig(configJson []byte) (*Config, error) {
 		return nil, ContextError(err)
 	}
 
+	// Do setEmitDiagnosticNotices first, to ensure config file errors are emitted.
+	if config.EmitDiagnosticNotices {
+		setEmitDiagnosticNotices(true)
+	}
+
 	// These fields are required; the rest are optional
 	if config.PropagationChannelId == "" {
 		return nil, ContextError(
@@ -313,6 +346,12 @@ func LoadConfig(configJson []byte) (*Config, error) {
 		config.ClientVersion = "0"
 	}
 
+	_, err = strconv.Atoi(config.ClientVersion)
+	if err != nil {
+		return nil, ContextError(
+			fmt.Errorf("invalid client version: %s", err))
+	}
+
 	if config.TunnelProtocol != "" {
 		if !Contains(SupportedTunnelProtocols, config.TunnelProtocol) {
 			return nil, ContextError(
@@ -345,5 +384,15 @@ func LoadConfig(configJson []byte) (*Config, error) {
 		return nil, ContextError(errors.New("DnsServerGetter interface must be set at runtime"))
 	}
 
+	if config.HostNameTransformer != nil {
+		return nil, ContextError(errors.New("HostNameTransformer interface must be set at runtime"))
+	}
+
+	if config.UpgradeDownloadUrl != "" &&
+		(config.UpgradeDownloadClientVersionHeader == "" || config.UpgradeDownloadFilename == "") {
+		return nil, ContextError(errors.New(
+			"UpgradeDownloadUrl requires UpgradeDownloadClientVersionHeader and UpgradeDownloadFilename"))
+	}
+
 	return &config, nil
 }

psiphon/controller.go

@@ -47,7 +47,6 @@ type Controller struct {
 	tunnels                        []*Tunnel
 	nextTunnel                     int
 	startedConnectedReporter       bool
-	startedUpgradeDownloader       bool
 	isEstablishing                 bool
 	establishWaitGroup             *sync.WaitGroup
 	stopEstablishingBroadcast      chan struct{}
@@ -57,6 +56,7 @@ type Controller struct {
 	untunneledDialConfig           *DialConfig
 	splitTunnelClassifier          *SplitTunnelClassifier
 	signalFetchRemoteServerList    chan struct{}
+	signalDownloadUpgrade          chan string
 	impairedProtocolClassification map[string]int
 	signalReportConnected          chan struct{}
 	serverAffinityDoneBroadcast    chan struct{}
@@ -73,16 +73,24 @@ func NewController(config *Config) (controller *Controller, err error) {
 	// Needed by regen, at least
 	rand.Seed(int64(time.Now().Nanosecond()))
 
+	// Supply a default HostNameTransformer
+	if config.HostNameTransformer == nil {
+		config.HostNameTransformer = &IdentityHostNameTransformer{}
+	}
+
 	// Generate a session ID for the Psiphon server API. This session ID is
 	// used across all tunnels established by the controller.
 	sessionId, err := MakeSessionId()
 	if err != nil {
 		return nil, ContextError(err)
 	}
+	NoticeSessionId(sessionId)
 
 	// untunneledPendingConns may be used to interrupt the fetch remote server list
 	// request and other untunneled connection establishments. BindToDevice may be
 	// used to exclude these requests and connection from VPN routing.
+	// TODO: fetch remote server list and untunneled upgrade download should remove
+	// their completed conns from untunneledPendingConns.
 	untunneledPendingConns := new(Conns)
 	untunneledDialConfig := &DialConfig{
 		UpstreamProxyUrl:              config.UpstreamProxyUrl,
@@ -91,6 +99,7 @@ func NewController(config *Config) (controller *Controller, err error) {
 		DnsServerGetter:               config.DnsServerGetter,
 		UseIndistinguishableTLS:       config.UseIndistinguishableTLS,
 		TrustedCACertificatesFilename: config.TrustedCACertificatesFilename,
+		DeviceRegion:                  config.DeviceRegion,
 	}
 
 	controller = &Controller{
@@ -109,7 +118,6 @@ func NewController(config *Config) (controller *Controller, err error) {
 		tunnels:                        make([]*Tunnel, 0),
 		establishedOnce:                false,
 		startedConnectedReporter:       false,
-		startedUpgradeDownloader:       false,
 		isEstablishing:                 false,
 		establishPendingConns:          new(Conns),
 		untunneledPendingConns:         untunneledPendingConns,
@@ -119,6 +127,7 @@ func NewController(config *Config) (controller *Controller, err error) {
 		// starting? Trade-off is potential back-to-back fetch remotes. As-is,
 		// establish will eventually signal another fetch remote.
 		signalFetchRemoteServerList: make(chan struct{}),
+		signalDownloadUpgrade:       make(chan string),
 		signalReportConnected:       make(chan struct{}),
 	}
 
@@ -137,7 +146,6 @@ func NewController(config *Config) (controller *Controller, err error) {
 // - a local SOCKS proxy that port forwards through the pool of tunnels
 // - a local HTTP proxy that port forwards through the pool of tunnels
 func (controller *Controller) Run(shutdownBroadcast <-chan struct{}) {
-	NoticeBuildInfo()
 	ReportAvailableRegions()
 
 	// Start components
@@ -168,6 +176,13 @@ func (controller *Controller) Run(shutdownBroadcast <-chan struct{}) {
 		go controller.remoteServerListFetcher()
 	}
 
+	if controller.config.UpgradeDownloadUrl != "" &&
+		controller.config.UpgradeDownloadFilename != "" {
+
+		controller.runWaitGroup.Add(1)
+		go controller.upgradeDownloader()
+	}
+
 	/// Note: the connected reporter isn't started until a tunnel is
 	// established
 
@@ -189,20 +204,28 @@ func (controller *Controller) Run(shutdownBroadcast <-chan struct{}) {
 	}
 
 	close(controller.shutdownBroadcast)
+
+	// Interrupts and stops establish workers blocking on
+	// tunnel establishment network operations.
 	controller.establishPendingConns.CloseAll()
-	controller.runWaitGroup.Wait()
 
-	// Stops untunneled connections, including fetch remote server list,
-	// split tunnel port forwards and also untunneled final stats requests.
-	// Note: there's a circular dependency with runWaitGroup.Wait() and
-	// untunneledPendingConns.CloseAll(): runWaitGroup depends on tunnels
-	// stopping which depends, in orderly shutdown, on final status requests
-	// completing. So this pending conns cancel comes too late to interrupt
-	// final status requests in the orderly shutdown case -- which is desired
-	// since we give those a short timeout and would prefer to not interrupt
-	// them.
+	// Interrupts and stops workers blocking on untunneled
+	// network operations. This includes fetch remote server
+	// list and untunneled uprade download.
+	// Note: this doesn't interrupt the final, untunneled status
+	// requests started in operateTunnel after shutdownBroadcast.
+	// This is by design -- we want to give these requests a short
+	// timer period to succeed and deliver stats. These particular
+	// requests opt out of untunneledPendingConns and use the
+	// PSIPHON_API_SHUTDOWN_SERVER_TIMEOUT timeout (see
+	// doUntunneledStatusRequest).
 	controller.untunneledPendingConns.CloseAll()
 
+	// Now with all workers signaled to stop and with all
+	// blocking network operations interrupted, wait for
+	// all workers to terminate.
+	controller.runWaitGroup.Wait()
+
 	controller.splitTunnelClassifier.Shutdown()
 
 	NoticeInfo("exiting controller")
@@ -365,63 +388,78 @@ func (controller *Controller) startOrSignalConnectedReporter() {
 // download. DownloadUpgrade() is resumable, so each attempt has potential for
 // getting closer to completion, even in conditions where the download or
 // tunnel is repeatedly interrupted.
-// Once the download is complete, the downloader exits and is not run again:
+// An upgrade download is triggered by either a handshake response indicating
+// that a new version is available; or after failing to connect, in which case
+// it's useful to check, out-of-band, for an upgrade with new circumvention
+// capabilities.
+// Once the download operation completes successfully, the downloader exits
+// and is not run again: either there is not a newer version, or the upgrade
+// has been downloaded and is ready to be applied.
 // We're assuming that the upgrade will be applied and the entire system
 // restarted before another upgrade is to be downloaded.
-func (controller *Controller) upgradeDownloader(clientUpgradeVersion string) {
+//
+// TODO: refactor upgrade downloader and remote server list fetcher to use
+// common code (including the resumable download routines).
+//
+func (controller *Controller) upgradeDownloader() {
 	defer controller.runWaitGroup.Done()
 
-loop:
-	for {
-		// Pick any active tunnel and make the next download attempt. No error
-		// is logged if there's no active tunnel, as that's not an unexpected condition.
-		tunnel := controller.getNextActiveTunnel()
-		if tunnel != nil {
-			err := DownloadUpgrade(controller.config, clientUpgradeVersion, tunnel)
-			if err == nil {
-				break loop
-			}
-			NoticeAlert("upgrade download failed: %s", err)
-		}
+	var lastDownloadTime time.Time
 
-		timeout := time.After(DOWNLOAD_UPGRADE_RETRY_PAUSE_PERIOD)
+downloadLoop:
+	for {
+		// Wait for a signal before downloading
+		var handshakeVersion string
 		select {
-		case <-timeout:
-			// Make another download attempt
+		case handshakeVersion = <-controller.signalDownloadUpgrade:
 		case <-controller.shutdownBroadcast:
-			break loop
+			break downloadLoop
 		}
-	}
 
-	NoticeInfo("exiting upgrade downloader")
-}
+		// Unless handshake is explicitly advertizing a new version, skip
+		// checking entirely when a recent download was successful.
+		if handshakeVersion == "" &&
+			time.Now().Before(lastDownloadTime.Add(DOWNLOAD_UPGRADE_STALE_PERIOD)) {
+			continue
+		}
 
-func (controller *Controller) startClientUpgradeDownloader(
-	serverContext *ServerContext) {
+	retryLoop:
+		for {
+			// Don't attempt to download while there is no network connectivity,
+			// to avoid alert notice noise.
+			if !WaitForNetworkConnectivity(
+				controller.config.NetworkConnectivityChecker,
+				controller.shutdownBroadcast) {
+				break downloadLoop
+			}
 
-	// serverContext is nil when DisableApi is set
-	if controller.config.DisableApi {
-		return
-	}
+			// Pick any active tunnel and make the next download attempt. If there's
+			// no active tunnel, the untunneledDialConfig will be used.
+			tunnel := controller.getNextActiveTunnel()
 
-	if controller.config.UpgradeDownloadUrl == "" ||
-		controller.config.UpgradeDownloadFilename == "" {
-		// No upgrade is desired
-		return
-	}
+			err := DownloadUpgrade(
+				controller.config,
+				handshakeVersion,
+				tunnel,
+				controller.untunneledDialConfig)
 
-	if serverContext.clientUpgradeVersion == "" {
-		// No upgrade is offered
-		return
-	}
+			if err == nil {
+				lastDownloadTime = time.Now()
+				break retryLoop
+			}
 
-	// Start the client upgrade downloaded after the first tunnel is established.
-	// Concurrency note: only the runTunnels goroutine may access startClientUpgradeDownloader.
-	if !controller.startedUpgradeDownloader {
-		controller.startedUpgradeDownloader = true
-		controller.runWaitGroup.Add(1)
-		go controller.upgradeDownloader(serverContext.clientUpgradeVersion)
+			NoticeAlert("failed to download upgrade: %s", err)
+
+			timeout := time.After(DOWNLOAD_UPGRADE_RETRY_PERIOD)
+			select {
+			case <-timeout:
+			case <-controller.shutdownBroadcast:
+				break downloadLoop
+			}
+		}
 	}
+
+	NoticeInfo("exiting upgrade downloader")
 }
 
 // runTunnels is the controller tunnel management main loop. It starts and stops
@@ -472,40 +510,70 @@ loop:
 				controller.startEstablishing()
 			}
 
-		// !TODO! design issue: might not be enough server entries with region/caps to ever fill tunnel slots
-		// solution(?) target MIN(CountServerEntries(region, protocol), TunnelPoolSize)
 		case establishedTunnel := <-controller.establishedTunnels:
-			tunnelCount, registered := controller.registerTunnel(establishedTunnel)
-			if registered {
-				NoticeActiveTunnel(establishedTunnel.serverEntry.IpAddress, establishedTunnel.protocol)
-
-				if tunnelCount == 1 {
-
-					// The split tunnel classifier is started once the first tunnel is
-					// established. This first tunnel is passed in to be used to make
-					// the routes data request.
-					// A long-running controller may run while the host device is present
-					// in different regions. In this case, we want the split tunnel logic
-					// to switch to routes for new regions and not classify traffic based
-					// on routes installed for older regions.
-					// We assume that when regions change, the host network will also
-					// change, and so all tunnels will fail and be re-established. Under
-					// that assumption, the classifier will be re-Start()-ed here when
-					// the region has changed.
-					controller.splitTunnelClassifier.Start(establishedTunnel)
-
-					// Signal a connected request on each 1st tunnel establishment. For
-					// multi-tunnels, the session is connected as long as at least one
-					// tunnel is established.
-					controller.startOrSignalConnectedReporter()
-
-					controller.startClientUpgradeDownloader(
-						establishedTunnel.serverContext)
+
+			if controller.isImpairedProtocol(establishedTunnel.protocol) {
+
+				NoticeAlert("established tunnel with impaired protocol: %s", establishedTunnel.protocol)
+
+				// Protocol was classified as impaired while this tunnel
+				// established, so discard.
+				controller.discardTunnel(establishedTunnel)
+
+				// Reset establish generator to stop producing tunnels
+				// with impaired protocols.
+				if controller.isEstablishing {
+					controller.stopEstablishing()
+					controller.startEstablishing()
 				}
+				break
+			}
 
-			} else {
+			tunnelCount, registered := controller.registerTunnel(establishedTunnel)
+			if !registered {
+				// Already fully established, so discard.
 				controller.discardTunnel(establishedTunnel)
+				break
+			}
+
+			NoticeActiveTunnel(establishedTunnel.serverEntry.IpAddress, establishedTunnel.protocol)
+
+			if tunnelCount == 1 {
+
+				// The split tunnel classifier is started once the first tunnel is
+				// established. This first tunnel is passed in to be used to make
+				// the routes data request.
+				// A long-running controller may run while the host device is present
+				// in different regions. In this case, we want the split tunnel logic
+				// to switch to routes for new regions and not classify traffic based
+				// on routes installed for older regions.
+				// We assume that when regions change, the host network will also
+				// change, and so all tunnels will fail and be re-established. Under
+				// that assumption, the classifier will be re-Start()-ed here when
+				// the region has changed.
+				controller.splitTunnelClassifier.Start(establishedTunnel)
+
+				// Signal a connected request on each 1st tunnel establishment. For
+				// multi-tunnels, the session is connected as long as at least one
+				// tunnel is established.
+				controller.startOrSignalConnectedReporter()
+
+				// If the handshake indicated that a new client version is available,
+				// trigger an upgrade download.
+				// Note: serverContext is nil when DisableApi is set
+				if establishedTunnel.serverContext != nil &&
+					establishedTunnel.serverContext.clientUpgradeVersion != "" {
+
+					handshakeVersion := establishedTunnel.serverContext.clientUpgradeVersion
+					select {
+					case controller.signalDownloadUpgrade <- handshakeVersion:
+					default:
+					}
+				}
 			}
+
+			// TODO: design issue -- might not be enough server entries with region/caps to ever fill tunnel slots;
+			// possible solution is establish target MIN(CountServerEntries(region, protocol), TunnelPoolSize)
 			if controller.isFullyEstablished() {
 				controller.stopEstablishing()
 			}
@@ -565,9 +633,7 @@ func (controller *Controller) classifyImpairedProtocol(failedTunnel *Tunnel) {
 //
 // Concurrency note: only the runTunnels() goroutine may call getImpairedProtocols
 func (controller *Controller) getImpairedProtocols() []string {
-	if len(controller.impairedProtocolClassification) > 0 {
-		NoticeInfo("impaired protocols: %+v", controller.impairedProtocolClassification)
-	}
+	NoticeImpairedProtocolClassification(controller.impairedProtocolClassification)
 	impairedProtocols := make([]string, 0)
 	for protocol, count := range controller.impairedProtocolClassification {
 		if count >= IMPAIRED_PROTOCOL_CLASSIFICATION_THRESHOLD {
@@ -577,6 +643,14 @@ func (controller *Controller) getImpairedProtocols() []string {
 	return impairedProtocols
 }
 
+// isImpairedProtocol checks if the specified protocol is classified as impaired.
+//
+// Concurrency note: only the runTunnels() goroutine may call isImpairedProtocol
+func (controller *Controller) isImpairedProtocol(protocol string) bool {
+	count, ok := controller.impairedProtocolClassification[protocol]
+	return ok && count >= IMPAIRED_PROTOCOL_CLASSIFICATION_THRESHOLD
+}
+
 // SignalTunnelFailure implements the TunnelOwner interface. This function
 // is called by Tunnel.operateTunnel when the tunnel has detected that it
 // has failed. The Controller will signal runTunnels to create a new
@@ -751,7 +825,7 @@ func (controller *Controller) Dial(
 		// relative to the outbound network.
 
 		if controller.splitTunnelClassifier.IsUntunneled(host) {
-			// !TODO! track downstreamConn and close it when the DialTCP conn closes, as with tunnel.Dial conns?
+			// TODO: track downstreamConn and close it when the DialTCP conn closes, as with tunnel.Dial conns?
 			return DialTCP(remoteAddr, controller.untunneledDialConfig)
 		}
 	}
@@ -943,6 +1017,14 @@ loop:
 		default:
 		}
 
+		// Trigger an out-of-band upgrade availability check and download.
+		// Since we may have failed to connect, we may benefit from upgrading
+		// to a new client version with new circumvention capabilities.
+		select {
+		case controller.signalDownloadUpgrade <- "":
+		default:
+		}
+
 		// After a complete iteration of candidate servers, pause before iterating again.
 		// This helps avoid some busy wait loop conditions, and also allows some time for
 		// network conditions to change. Also allows for fetch remote to complete,

psiphon/controller_test.go

@@ -20,33 +20,253 @@
 package psiphon
 
 import (
+	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
+
+	socks "github.com/Psiphon-Inc/goptlib"
 )
 
-func TestControllerRunSSH(t *testing.T) {
-	controllerRun(t, TUNNEL_PROTOCOL_SSH)
+func TestMain(m *testing.M) {
+	flag.Parse()
+	os.Remove(DATA_STORE_FILENAME)
+	initDisruptor()
+	setEmitDiagnosticNotices(true)
+	os.Exit(m.Run())
+}
+
+// Note: untunneled upgrade tests must execute before
+// the other tests to ensure no tunnel is established.
+// We need a way to reset the datastore after it's been
+// initialized in order to to clear out its data entries
+// and be able to arbitrarily order the tests.
+
+func TestUntunneledUpgradeDownload(t *testing.T) {
+	controllerRun(t,
+		&controllerRunConfig{
+			protocol:                 "",
+			clientIsLatestVersion:    false,
+			disableUntunneledUpgrade: false,
+			disableEstablishing:      true,
+			tunnelPoolSize:           1,
+			disruptNetwork:           false,
+			useHostNameTransformer:   false,
+			runDuration:              0,
+		})
+}
+
+func TestUntunneledResumableUpgradeDownload(t *testing.T) {
+	controllerRun(t,
+		&controllerRunConfig{
+			protocol:                 "",
+			clientIsLatestVersion:    false,
+			disableUntunneledUpgrade: false,
+			disableEstablishing:      true,
+			tunnelPoolSize:           1,
+			disruptNetwork:           true,
+			useHostNameTransformer:   false,
+			runDuration:              0,
+		})
+}
+
+func TestUntunneledUpgradeClientIsLatestVersion(t *testing.T) {
+	controllerRun(t,
+		&controllerRunConfig{
+			protocol:                 "",
+			clientIsLatestVersion:    true,
+			disableUntunneledUpgrade: false,
+			disableEstablishing:      true,
+			tunnelPoolSize:           1,
+			disruptNetwork:           false,
+			useHostNameTransformer:   false,
+			runDuration:              0,
+		})
+}
+
+func TestTunneledUpgradeClientIsLatestVersion(t *testing.T) {
+	controllerRun(t,
+		&controllerRunConfig{
+			protocol:                 "",
+			clientIsLatestVersion:    true,
+			disableUntunneledUpgrade: true,
+			disableEstablishing:      false,
+			tunnelPoolSize:           1,
+			disruptNetwork:           false,
+			useHostNameTransformer:   false,
+			runDuration:              0,
+		})
+}
+
+func TestImpairedProtocols(t *testing.T) {
+
+	// This test sets a tunnelPoolSize of 40 and runs
+	// the session for 1 minute with network disruption
+	// on. All 40 tunnels being disrupted every 10
+	// seconds (followed by ssh keep alive probe timeout)
+	// should be sufficient to trigger at least one
+	// impaired protocol classification.
+
+	controllerRun(t,
+		&controllerRunConfig{
+			protocol:                 "",
+			clientIsLatestVersion:    true,
+			disableUntunneledUpgrade: true,
+			disableEstablishing:      false,
+			tunnelPoolSize:           40,
+			disruptNetwork:           true,
+			useHostNameTransformer:   false,
+			runDuration:              1 * time.Minute,
+		})
+}
+
+func TestSSH(t *testing.T) {
+	controllerRun(t,
+		&controllerRunConfig{
+			protocol:                 TUNNEL_PROTOCOL_SSH,
+			clientIsLatestVersion:    false,
+			disableUntunneledUpgrade: true,
+			disableEstablishing:      false,
+			tunnelPoolSize:           1,
+			disruptNetwork:           false,
+			useHostNameTransformer:   false,
+			runDuration:              0,
+		})
+}
+
+func TestObfuscatedSSH(t *testing.T) {
+	controllerRun(t,
+		&controllerRunConfig{
+			protocol:                 TUNNEL_PROTOCOL_OBFUSCATED_SSH,
+			clientIsLatestVersion:    false,
+			disableUntunneledUpgrade: true,
+			disableEstablishing:      false,
+			tunnelPoolSize:           1,
+			disruptNetwork:           false,
+			useHostNameTransformer:   false,
+			runDuration:              0,
+		})
+}
+
+func TestUnfrontedMeek(t *testing.T) {
+	controllerRun(t,
+		&controllerRunConfig{
+			protocol:                 TUNNEL_PROTOCOL_UNFRONTED_MEEK,
+			clientIsLatestVersion:    false,
+			disableUntunneledUpgrade: true,
+			disableEstablishing:      false,
+			tunnelPoolSize:           1,
+			disruptNetwork:           false,
+			useHostNameTransformer:   false,
+			runDuration:              0,
+		})
+}
+
+func TestUnfrontedMeekWithTransformer(t *testing.T) {
+	controllerRun(t,
+		&controllerRunConfig{
+			protocol:                 TUNNEL_PROTOCOL_UNFRONTED_MEEK,
+			clientIsLatestVersion:    true,
+			disableUntunneledUpgrade: true,
+			disableEstablishing:      false,
+			tunnelPoolSize:           1,
+			disruptNetwork:           false,
+			useHostNameTransformer:   true,
+			runDuration:              0,
+		})
+}
+
+func TestFrontedMeek(t *testing.T) {
+	controllerRun(t,
+		&controllerRunConfig{
+			protocol:                 TUNNEL_PROTOCOL_FRONTED_MEEK,
+			clientIsLatestVersion:    false,
+			disableUntunneledUpgrade: true,
+			disableEstablishing:      false,
+			tunnelPoolSize:           1,
+			disruptNetwork:           false,
+			useHostNameTransformer:   false,
+			runDuration:              0,
+		})
+}
+
+func TestFrontedMeekWithTransformer(t *testing.T) {
+	controllerRun(t,
+		&controllerRunConfig{
+			protocol:                 TUNNEL_PROTOCOL_FRONTED_MEEK,
+			clientIsLatestVersion:    true,
+			disableUntunneledUpgrade: true,
+			disableEstablishing:      false,
+			tunnelPoolSize:           1,
+			disruptNetwork:           false,
+			useHostNameTransformer:   true,
+			runDuration:              0,
+		})
+}
+
+func TestFrontedMeekHTTP(t *testing.T) {
+	controllerRun(t,
+		&controllerRunConfig{
+			protocol:                 TUNNEL_PROTOCOL_FRONTED_MEEK_HTTP,
+			clientIsLatestVersion:    true,
+			disableUntunneledUpgrade: true,
+			disableEstablishing:      false,
+			tunnelPoolSize:           1,
+			disruptNetwork:           false,
+			useHostNameTransformer:   false,
+			runDuration:              0,
+		})
 }
 
-func TestControllerRunObfuscatedSSH(t *testing.T) {
-	controllerRun(t, TUNNEL_PROTOCOL_OBFUSCATED_SSH)
+func TestUnfrontedMeekHTTPS(t *testing.T) {
+	controllerRun(t,
+		&controllerRunConfig{
+			protocol:                 TUNNEL_PROTOCOL_UNFRONTED_MEEK_HTTPS,
+			clientIsLatestVersion:    false,
+			disableUntunneledUpgrade: true,
+			disableEstablishing:      false,
+			tunnelPoolSize:           1,
+			disruptNetwork:           false,
+			useHostNameTransformer:   false,
+			runDuration:              0,
+		})
 }
 
-func TestControllerRunUnfrontedMeek(t *testing.T) {
-	controllerRun(t, TUNNEL_PROTOCOL_UNFRONTED_MEEK)
+func TestUnfrontedMeekHTTPSWithTransformer(t *testing.T) {
+	controllerRun(t,
+		&controllerRunConfig{
+			protocol:                 TUNNEL_PROTOCOL_UNFRONTED_MEEK_HTTPS,
+			clientIsLatestVersion:    true,
+			disableUntunneledUpgrade: true,
+			disableEstablishing:      false,
+			tunnelPoolSize:           1,
+			disruptNetwork:           false,
+			useHostNameTransformer:   true,
+			runDuration:              0,
+		})
 }
 
-func TestControllerRunFrontedMeek(t *testing.T) {
-	controllerRun(t, TUNNEL_PROTOCOL_FRONTED_MEEK)
+type controllerRunConfig struct {
+	protocol                 string
+	clientIsLatestVersion    bool
+	disableUntunneledUpgrade bool
+	disableEstablishing      bool
+	tunnelPoolSize           int
+	disruptNetwork           bool
+	useHostNameTransformer   bool
+	runDuration              time.Duration
 }
 
-func controllerRun(t *testing.T, protocol string) {
+func controllerRun(t *testing.T, runConfig *controllerRunConfig) {
 
 	configFileContents, err := ioutil.ReadFile("controller_test.config")
 	if err != nil {
@@ -55,21 +275,46 @@ func controllerRun(t *testing.T, protocol string) {
 	}
 	config, err := LoadConfig(configFileContents)
 	if err != nil {
-		t.Errorf("error processing configuration file: %s", err)
-		t.FailNow()
+		t.Fatalf("error processing configuration file: %s", err)
+	}
+
+	if runConfig.clientIsLatestVersion {
+		config.ClientVersion = "999999999"
+	}
+
+	if runConfig.disableEstablishing {
+		// Clear remote server list so tunnel cannot be established.
+		// TODO: also delete all server entries in the datastore.
+		config.RemoteServerListUrl = ""
+	}
+
+	config.TunnelPoolSize = runConfig.tunnelPoolSize
+
+	if runConfig.disableUntunneledUpgrade {
+		// Disable untunneled upgrade downloader to ensure tunneled case is tested
+		config.UpgradeDownloadClientVersionHeader = ""
 	}
-	config.TunnelProtocol = protocol
+
+	if runConfig.disruptNetwork {
+		config.UpstreamProxyUrl = disruptorProxyURL
+	}
+
+	if runConfig.useHostNameTransformer {
+		config.HostNameTransformer = &TestHostNameTransformer{}
+	}
+
+	os.Remove(config.UpgradeDownloadFilename)
+
+	config.TunnelProtocol = runConfig.protocol
 
 	err = InitDataStore(config)
 	if err != nil {
-		t.Errorf("error initializing datastore: %s", err)
-		t.FailNow()
+		t.Fatalf("error initializing datastore: %s", err)
 	}
 
 	controller, err := NewController(config)
 	if err != nil {
-		t.Errorf("error creating controller: %s", err)
-		t.FailNow()
+		t.Fatalf("error creating controller: %s", err)
 	}
 
 	// Monitor notices for "Tunnels" with count > 1, the
@@ -80,6 +325,16 @@ func controllerRun(t *testing.T, protocol string) {
 	httpProxyPort := 0
 
 	tunnelEstablished := make(chan struct{}, 1)
+	upgradeDownloaded := make(chan struct{}, 1)
+	confirmedLatestVersion := make(chan struct{}, 1)
+
+	var clientUpgradeDownloadedBytesCount int32
+	var impairedProtocolCount int32
+	var impairedProtocolClassification = struct {
+		sync.RWMutex
+		classification map[string]int
+	}{classification: make(map[string]int)}
+
 	SetNoticeOutput(NewNoticeReceiver(
 		func(notice []byte) {
 			// TODO: log notices without logging server IPs:
@@ -89,22 +344,86 @@ func controllerRun(t *testing.T, protocol string) {
 				return
 			}
 			switch noticeType {
+
+			case "ListeningHttpProxyPort":
+
+				httpProxyPort = int(payload["port"].(float64))
+
+			case "ConnectingServer":
+
+				serverProtocol := payload["protocol"].(string)
+				if runConfig.protocol != "" && serverProtocol != runConfig.protocol {
+					// TODO: wrong goroutine for t.FatalNow()
+					t.Fatalf("wrong protocol selected: %s", serverProtocol)
+				}
+
 			case "Tunnels":
+
 				count := int(payload["count"].(float64))
 				if count > 0 {
-					select {
-					case tunnelEstablished <- *new(struct{}):
-					default:
+					if runConfig.disableEstablishing {
+						// TODO: wrong goroutine for t.FatalNow()
+						t.Fatalf("tunnel established unexpectedly")
+					} else {
+						select {
+						case tunnelEstablished <- *new(struct{}):
+						default:
+						}
 					}
 				}
-			case "ListeningHttpProxyPort":
-				httpProxyPort = int(payload["port"].(float64))
-			case "ConnectingServer":
-				serverProtocol := payload["protocol"]
-				if serverProtocol != protocol {
-					t.Errorf("wrong protocol selected: %s", serverProtocol)
-					t.FailNow()
+
+			case "ClientUpgradeDownloadedBytes":
+
+				atomic.AddInt32(&clientUpgradeDownloadedBytesCount, 1)
+				t.Logf("ClientUpgradeDownloadedBytes: %d", int(payload["bytes"].(float64)))
+
+			case "ClientUpgradeDownloaded":
+
+				select {
+				case upgradeDownloaded <- *new(struct{}):
+				default:
+				}
+
+			case "ClientIsLatestVersion":
+
+				select {
+				case confirmedLatestVersion <- *new(struct{}):
+				default:
+				}
+
+			case "ImpairedProtocolClassification":
+
+				classification := payload["classification"].(map[string]interface{})
+
+				impairedProtocolClassification.Lock()
+				impairedProtocolClassification.classification = make(map[string]int)
+				for k, v := range classification {
+					count := int(v.(float64))
+					if count >= IMPAIRED_PROTOCOL_CLASSIFICATION_THRESHOLD {
+						atomic.AddInt32(&impairedProtocolCount, 1)
+					}
+					impairedProtocolClassification.classification[k] = count
+				}
+				impairedProtocolClassification.Unlock()
+
+			case "ActiveTunnel":
+
+				serverProtocol := payload["protocol"].(string)
+
+				classification := make(map[string]int)
+				impairedProtocolClassification.RLock()
+				for k, v := range impairedProtocolClassification.classification {
+					classification[k] = v
+				}
+				impairedProtocolClassification.RUnlock()
+
+				count, ok := classification[serverProtocol]
+				if ok && count >= IMPAIRED_PROTOCOL_CLASSIFICATION_THRESHOLD {
+					// TODO: wrong goroutine for t.FatalNow()
+					t.Fatalf("unexpected tunnel using impaired protocol: %s, %+v",
+						serverProtocol, classification)
 				}
+
 			}
 		}))
 
@@ -118,44 +437,115 @@ func controllerRun(t *testing.T, protocol string) {
 		controller.Run(shutdownBroadcast)
 	}()
 
-	// Test: tunnel must be established within 60 seconds
+	defer func() {
+		// Test: shutdown must complete within 20 seconds
 
-	establishTimeout := time.NewTimer(60 * time.Second)
+		close(shutdownBroadcast)
 
-	select {
-	case <-tunnelEstablished:
+		shutdownTimeout := time.NewTimer(20 * time.Second)
+
+		shutdownOk := make(chan struct{}, 1)
+		go func() {
+			controllerWaitGroup.Wait()
+			shutdownOk <- *new(struct{})
+		}()
+
+		select {
+		case <-shutdownOk:
+		case <-shutdownTimeout.C:
+			t.Fatalf("controller shutdown timeout exceeded")
+		}
+	}()
+
+	if !runConfig.disableEstablishing {
+
+		// Test: tunnel must be established within 60 seconds
+
+		establishTimeout := time.NewTimer(60 * time.Second)
+
+		select {
+		case <-tunnelEstablished:
+
+		case <-establishTimeout.C:
+			t.Fatalf("tunnel establish timeout exceeded")
+		}
+
+		// Test: fetch website through tunnel
 
 		// Allow for known race condition described in NewHttpProxy():
 		time.Sleep(1 * time.Second)
 
-		// Test: fetch website through tunnel
-		fetchWebsite(t, httpProxyPort)
+		fetchAndVerifyWebsite(t, httpProxyPort)
 
-	case <-establishTimeout.C:
-		t.Errorf("tunnel establish timeout exceeded")
-		// ...continue with cleanup
-	}
+		// Test: run for duration, periodically using the tunnel to
+		// ensure failed tunnel detection, and ultimately hitting
+		// impaired protocol checks.
 
-	close(shutdownBroadcast)
+		startTime := time.Now()
 
-	// Test: shutdown must complete within 10 seconds
+		for {
 
-	shutdownTimeout := time.NewTimer(10 * time.Second)
+			time.Sleep(1 * time.Second)
+			useTunnel(t, httpProxyPort)
 
-	shutdownOk := make(chan struct{}, 1)
-	go func() {
-		controllerWaitGroup.Wait()
-		shutdownOk <- *new(struct{})
-	}()
+			if startTime.Add(runConfig.runDuration).Before(time.Now()) {
+				break
+			}
+		}
+
+		// Test: with disruptNetwork, impaired protocols should be exercised
+
+		if runConfig.runDuration > 0 && runConfig.disruptNetwork {
+			count := atomic.LoadInt32(&impairedProtocolCount)
+			if count <= 0 {
+				t.Fatalf("unexpected impaired protocol count: %d", count)
+			} else {
+				impairedProtocolClassification.RLock()
+				t.Logf("impaired protocol classification: %+v",
+					impairedProtocolClassification.classification)
+				impairedProtocolClassification.RUnlock()
+			}
+		}
+	}
+
+	// Test: upgrade check/download must be downloaded within 120 seconds
+
+	upgradeTimeout := time.NewTimer(120 * time.Second)
 
 	select {
-	case <-shutdownOk:
-	case <-shutdownTimeout.C:
-		t.Errorf("controller shutdown timeout exceeded")
+	case <-upgradeDownloaded:
+		// TODO: verify downloaded file
+		if runConfig.clientIsLatestVersion {
+			t.Fatalf("upgrade downloaded unexpectedly")
+		}
+
+		// Test: with disruptNetwork, must be multiple download progress notices
+
+		if runConfig.disruptNetwork {
+			count := atomic.LoadInt32(&clientUpgradeDownloadedBytesCount)
+			if count <= 1 {
+				t.Fatalf("unexpected upgrade download progress: %d", count)
+			}
+		}
+
+	case <-confirmedLatestVersion:
+		if !runConfig.clientIsLatestVersion {
+			t.Fatalf("confirmed latest version unexpectedly")
+		}
+
+	case <-upgradeTimeout.C:
+		t.Fatalf("upgrade download timeout exceeded")
 	}
 }
 
-func fetchWebsite(t *testing.T, httpProxyPort int) {
+type TestHostNameTransformer struct {
+}
+
+func (TestHostNameTransformer) TransformHostName(string) (string, bool) {
+	return "example.com", true
+}
+
+func fetchAndVerifyWebsite(t *testing.T, httpProxyPort int) {
 
 	testUrl := "https://raw.githubusercontent.com/Psiphon-Labs/psiphon-tunnel-core/master/LICENSE"
 	roundTripTimeout := 10 * time.Second
@@ -169,8 +559,7 @@ func fetchWebsite(t *testing.T, httpProxyPort int) {
 
 	proxyUrl, err := url.Parse(fmt.Sprintf("http://127.0.0.1:%d", httpProxyPort))
 	if err != nil {
-		t.Errorf("error initializing proxied HTTP request: %s", err)
-		t.FailNow()
+		t.Fatalf("error initializing proxied HTTP request: %s", err)
 	}
 
 	httpClient := &http.Client{
@@ -182,20 +571,17 @@ func fetchWebsite(t *testing.T, httpProxyPort int) {
 
 	response, err := httpClient.Get(testUrl)
 	if err != nil {
-		t.Errorf("error sending proxied HTTP request: %s", err)
-		t.FailNow()
+		t.Fatalf("error sending proxied HTTP request: %s", err)
 	}
 
 	body, err := ioutil.ReadAll(response.Body)
 	if err != nil {
-		t.Errorf("error reading proxied HTTP response: %s", err)
-		t.FailNow()
+		t.Fatalf("error reading proxied HTTP response: %s", err)
 	}
 	response.Body.Close()
 
 	if !checkResponse(string(body)) {
-		t.Errorf("unexpected proxied HTTP response")
-		t.FailNow()
+		t.Fatalf("unexpected proxied HTTP response")
 	}
 
 	// Test: use direct URL proxy
@@ -209,20 +595,17 @@ func fetchWebsite(t *testing.T, httpProxyPort int) {
 		fmt.Sprintf("http://127.0.0.1:%d/direct/%s",
 			httpProxyPort, url.QueryEscape(testUrl)))
 	if err != nil {
-		t.Errorf("error sending direct URL request: %s", err)
-		t.FailNow()
+		t.Fatalf("error sending direct URL request: %s", err)
 	}
 
 	body, err = ioutil.ReadAll(response.Body)
 	if err != nil {
-		t.Errorf("error reading direct URL response: %s", err)
-		t.FailNow()
+		t.Fatalf("error reading direct URL response: %s", err)
 	}
 	response.Body.Close()
 
 	if !checkResponse(string(body)) {
-		t.Errorf("unexpected direct URL response")
-		t.FailNow()
+		t.Fatalf("unexpected direct URL response")
 	}
 
 	// Test: use tunneled URL proxy
@@ -231,19 +614,92 @@ func fetchWebsite(t *testing.T, httpProxyPort int) {
 		fmt.Sprintf("http://127.0.0.1:%d/tunneled/%s",
 			httpProxyPort, url.QueryEscape(testUrl)))
 	if err != nil {
-		t.Errorf("error sending tunneled URL request: %s", err)
-		t.FailNow()
+		t.Fatalf("error sending tunneled URL request: %s", err)
 	}
 
 	body, err = ioutil.ReadAll(response.Body)
 	if err != nil {
-		t.Errorf("error reading tunneled URL response: %s", err)
-		t.FailNow()
+		t.Fatalf("error reading tunneled URL response: %s", err)
 	}
 	response.Body.Close()
 
 	if !checkResponse(string(body)) {
-		t.Errorf("unexpected tunneled URL response")
-		t.FailNow()
+		t.Fatalf("unexpected tunneled URL response")
+	}
+}
+
+func useTunnel(t *testing.T, httpProxyPort int) {
+
+	// No action on errors as the tunnel is expected to fail sometimes
+
+	testUrl := "https://psiphon3.com"
+	roundTripTimeout := 1 * time.Second
+	proxyUrl, err := url.Parse(fmt.Sprintf("http://127.0.0.1:%d", httpProxyPort))
+	if err != nil {
+		return
+	}
+	httpClient := &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyURL(proxyUrl),
+		},
+		Timeout: roundTripTimeout,
 	}
+	response, err := httpClient.Get(testUrl)
+	if err != nil {
+		return
+	}
+	response.Body.Close()
+}
+
+const disruptorProxyAddress = "127.0.0.1:2160"
+const disruptorProxyURL = "socks4a://" + disruptorProxyAddress
+const disruptorMaxConnectionBytes = 2000000
+const disruptorMaxConnectionTime = 10 * time.Second
+
+func initDisruptor() {
+
+	go func() {
+		listener, err := socks.ListenSocks("tcp", disruptorProxyAddress)
+		if err != nil {
+			fmt.Errorf("disruptor proxy listen error: %s", err)
+			return
+		}
+		for {
+			localConn, err := listener.AcceptSocks()
+			if err != nil {
+				fmt.Errorf("disruptor proxy accept error: %s", err)
+				return
+			}
+			go func() {
+				defer localConn.Close()
+				remoteConn, err := net.Dial("tcp", localConn.Req.Target)
+				if err != nil {
+					fmt.Errorf("disruptor proxy dial error: %s", err)
+					return
+				}
+				defer remoteConn.Close()
+				err = localConn.Grant(&net.TCPAddr{IP: net.ParseIP("0.0.0.0"), Port: 0})
+				if err != nil {
+					fmt.Errorf("disruptor proxy grant error: %s", err)
+					return
+				}
+
+				// Cut connection after disruptorMaxConnectionTime
+				time.AfterFunc(disruptorMaxConnectionTime, func() {
+					localConn.Close()
+					remoteConn.Close()
+				})
+
+				// Relay connection, but only up to disruptorMaxConnectionBytes
+				waitGroup := new(sync.WaitGroup)
+				waitGroup.Add(1)
+				go func() {
+					defer waitGroup.Done()
+					io.CopyN(localConn, remoteConn, disruptorMaxConnectionBytes)
+				}()
+				io.CopyN(remoteConn, localConn, disruptorMaxConnectionBytes)
+				waitGroup.Wait()
+			}()
+		}
+	}()
 }

psiphon/dataStore.go

@@ -396,7 +396,8 @@ func NewServerEntryIterator(config *Config) (iterator *ServerEntryIterator, err
 
 // newTargetServerEntryIterator is a helper for initializing the TargetServerEntry case
 func newTargetServerEntryIterator(config *Config) (iterator *ServerEntryIterator, err error) {
-	serverEntry, err := DecodeServerEntry(config.TargetServerEntry)
+	serverEntry, err := DecodeServerEntry(
+		config.TargetServerEntry, GetCurrentTimestamp(), SERVER_ENTRY_SOURCE_TARGET)
 	if err != nil {
 		return nil, err
 	}

psiphon/meekConn.go

@@ -59,6 +59,37 @@ const (
 	MEEK_ROUND_TRIP_TIMEOUT        = 20 * time.Second
 )
 
+// MeekConfig specifies the behavior of a MeekConn
+type MeekConfig struct {
+
+	// DialAddress is the actual network address to dial to establish a
+	// connection to the meek server. This may be either a fronted or
+	// direct address. The address must be in the form "host:port",
+	// where host may be a domain name or IP address.
+	DialAddress string
+
+	// UseHTTPS indicates whether to use HTTPS (true) or HTTP (false).
+	UseHTTPS bool
+
+	// SNIServerName is the value to place in the TLS SNI server_name
+	// field when HTTPS is used.
+	SNIServerName string
+
+	// HostHeader is the value to place in the HTTP request Host header.
+	HostHeader string
+
+	// TransformedHostName records whether a HostNameTransformer
+	// transformation is in effect. This value is used for stats reporting.
+	TransformedHostName bool
+
+	// The following values are used to create the obfuscated meek cookie.
+
+	PsiphonServerAddress          string
+	SessionID                     string
+	MeekCookieEncryptionPublicKey string
+	MeekObfuscatedKey             string
+}
+
 // MeekConn is a network connection that tunnels TCP over HTTP and supports "fronting". Meek sends
 // client->server flow in HTTP request bodies and receives server->client flow in HTTP response bodies.
 // Polling is used to achieve full duplex TCP.
@@ -71,8 +102,8 @@ const (
 // MeekConn also operates in unfronted mode, in which plain HTTP connections are made without routing
 // through a CDN.
 type MeekConn struct {
-	frontingAddress      string
 	url                  *url.URL
+	additionalHeaders    map[string]string
 	cookie               *http.Cookie
 	pendingConns         *Conns
 	transport            transporter
@@ -103,11 +134,9 @@ type transporter interface {
 // is spawned which will eventually start HTTP polling.
 // When frontingAddress is not "", fronting is used. This option assumes caller has
 // already checked server entry capabilities.
-// Fronting always uses HTTPS. Otherwise, HTTPS is optional.
 func DialMeek(
-	serverEntry *ServerEntry, sessionId string,
-	useHttps bool, frontingAddress string,
-	config *DialConfig) (meek *MeekConn, err error) {
+	meekConfig *MeekConfig,
+	dialConfig *DialConfig) (meek *MeekConn, err error) {
 
 	// Configure transport
 	// Note: MeekConn has its own PendingConns to manage the underlying HTTP transport connections,
@@ -117,20 +146,17 @@ func DialMeek(
 	pendingConns := new(Conns)
 
 	// Use a copy of DialConfig with the meek pendingConns
-	meekConfig := new(DialConfig)
-	*meekConfig = *config
-	meekConfig.PendingConns = pendingConns
+	meekDialConfig := new(DialConfig)
+	*meekDialConfig = *dialConfig
+	meekDialConfig.PendingConns = pendingConns
 
-	// host is both what is dialed and what ends up in the HTTP Host header
-	host := fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.MeekServerPort)
-	var dialer Dialer
-	var proxyUrl func(*http.Request) (*url.URL, error)
+	var transport transporter
 
-	if useHttps || frontingAddress != "" {
+	if meekConfig.UseHTTPS {
 		// Custom TLS dialer:
 		//
 		//  1. ignores the HTTP request address and uses the fronting domain
-		//  2. disables SNI -- SNI breaks fronting when used with CDNs that support SNI on the server side.
+		//  2. optionally disables SNI -- SNI breaks fronting when used with certain CDNs.
 		//  3. skips verifying the server cert.
 		//
 		// Reasoning for #3:
@@ -160,66 +186,81 @@ func DialMeek(
 		// exclusively connect to non-MiM CDNs); then the adversary kills the underlying TCP connection after
 		// some short period. This is mitigated by the "impaired" protocol classification mechanism.
 
-		customTLSConfig := &CustomTLSConfig{
-			Dial:                          NewTCPDialer(meekConfig),
-			Timeout:                       meekConfig.ConnectTimeout,
-			SendServerName:                false,
+		dialer := NewCustomTLSDialer(&CustomTLSConfig{
+			DialAddr:                      meekConfig.DialAddress,
+			Dial:                          NewTCPDialer(meekDialConfig),
+			Timeout:                       meekDialConfig.ConnectTimeout,
+			SNIServerName:                 meekConfig.SNIServerName,
 			SkipVerify:                    true,
-			UseIndistinguishableTLS:       config.UseIndistinguishableTLS,
-			TrustedCACertificatesFilename: config.TrustedCACertificatesFilename,
-		}
+			UseIndistinguishableTLS:       meekDialConfig.UseIndistinguishableTLS,
+			TrustedCACertificatesFilename: meekDialConfig.TrustedCACertificatesFilename,
+		})
 
-		if frontingAddress != "" {
-			// In this case, host is not what is dialed but is what ends up in the HTTP Host header
-			host = serverEntry.MeekFrontingHost
-			customTLSConfig.FrontingAddr = fmt.Sprintf("%s:%d", frontingAddress, 443)
+		transport = &http.Transport{
+			Dial: dialer,
+			ResponseHeaderTimeout: MEEK_ROUND_TRIP_TIMEOUT,
 		}
-
-		dialer = NewCustomTLSDialer(customTLSConfig)
-
 	} else {
 
-		if strings.HasPrefix(meekConfig.UpstreamProxyUrl, "http://") {
-			// For unfronted meek, we let the http.Transport handle proxying, as the
-			// target server hostname has to be in the HTTP request line. Also, in this
-			// case, we don't require the proxy to support CONNECT and so we can work
-			// through HTTP proxies that don't support it.
-			url, err := url.Parse(meekConfig.UpstreamProxyUrl)
+		// For HTTP meek, we let the http.Transport handle proxying. http.Transport will
+		// put the the HTTP server address in the HTTP request line. In this case, we can
+		// use an HTTP proxy that does not support CONNECT.
+		var proxyUrl func(*http.Request) (*url.URL, error)
+		if strings.HasPrefix(meekDialConfig.UpstreamProxyUrl, "http://") &&
+			(meekConfig.DialAddress == meekConfig.HostHeader ||
+				meekConfig.DialAddress == meekConfig.HostHeader+":80") {
+			url, err := url.Parse(meekDialConfig.UpstreamProxyUrl)
 			if err != nil {
 				return nil, ContextError(err)
 			}
 			proxyUrl = http.ProxyURL(url)
-			meekConfig.UpstreamProxyUrl = ""
+			meekDialConfig.UpstreamProxyUrl = ""
+		}
+
+		// dialer ignores address that http.Transport will pass in (derived from
+		// the HTTP request URL) and always dials meekConfig.DialAddress.
+		dialer := func(string, string) (net.Conn, error) {
+			return NewTCPDialer(meekDialConfig)("tcp", meekConfig.DialAddress)
 		}
 
-		dialer = NewTCPDialer(meekConfig)
+		httpTransport := &http.Transport{
+			Proxy: proxyUrl,
+			Dial:  dialer,
+			ResponseHeaderTimeout: MEEK_ROUND_TRIP_TIMEOUT,
+		}
+		if proxyUrl != nil {
+			// Wrap transport with a transport that can perform HTTP proxy auth negotiation
+			transport, err = upstreamproxy.NewProxyAuthTransport(httpTransport)
+			if err != nil {
+				return nil, ContextError(err)
+			}
+		} else {
+			transport = httpTransport
+		}
 	}
 
 	// Scheme is always "http". Otherwise http.Transport will try to do another TLS
 	// handshake inside the explicit TLS session (in fronting mode).
 	url := &url.URL{
 		Scheme: "http",
-		Host:   host,
+		Host:   meekConfig.HostHeader,
 		Path:   "/",
 	}
-	cookie, err := makeCookie(serverEntry, sessionId)
-	if err != nil {
-		return nil, ContextError(err)
-	}
-	httpTransport := &http.Transport{
-		Proxy: proxyUrl,
-		Dial:  dialer,
-		ResponseHeaderTimeout: MEEK_ROUND_TRIP_TIMEOUT,
-	}
-	var transport transporter
-	if proxyUrl != nil {
-		// Wrap transport with a transport that can perform HTTP proxy auth negotiation
-		transport, err = upstreamproxy.NewProxyAuthTransport(httpTransport)
+
+	var additionalHeaders map[string]string
+	if meekConfig.UseHTTPS {
+		host, _, err := net.SplitHostPort(meekConfig.DialAddress)
 		if err != nil {
 			return nil, ContextError(err)
 		}
-	} else {
-		transport = httpTransport
+		additionalHeaders = map[string]string{
+			"X-Psiphon-Fronting-Address": host,
+		}
+	}
+
+	cookie, err := makeCookie(meekConfig)
+	if err != nil {
+		return nil, ContextError(err)
 	}
 
 	// The main loop of a MeekConn is run in the relay() goroutine.
@@ -239,8 +280,8 @@ func DialMeek(
 	// Write() calls and relay() are synchronized in a similar way, using a single
 	// sendBuffer.
 	meek = &MeekConn{
-		frontingAddress:      frontingAddress,
 		url:                  url,
+		additionalHeaders:    additionalHeaders,
 		cookie:               cookie,
 		pendingConns:         pendingConns,
 		transport:            transport,
@@ -261,7 +302,7 @@ func DialMeek(
 	go meek.relay()
 
 	// Enable interruption
-	if !config.PendingConns.Add(meek) {
+	if !dialConfig.PendingConns.Add(meek) {
 		meek.Close()
 		return nil, ContextError(errors.New("pending connections already closed"))
 	}
@@ -495,16 +536,16 @@ func (meek *MeekConn) roundTrip(sendPayload []byte) (receivedPayload io.ReadClos
 		return nil, ContextError(err)
 	}
 
-	if meek.frontingAddress != "" && nil == net.ParseIP(meek.frontingAddress) {
-		request.Header.Set("X-Psiphon-Fronting-Address", meek.frontingAddress)
-	}
-
 	// Don't use the default user agent ("Go 1.1 package http").
 	// For now, just omit the header (net/http/request.go: "may be blank to not send the header").
 	request.Header.Set("User-Agent", "")
 
 	request.Header.Set("Content-Type", "application/octet-stream")
 
+	for name, value := range meek.additionalHeaders {
+		request.Header.Set(name, value)
+	}
+
 	request.AddCookie(meek.cookie)
 
 	// The retry mitigates intermittent failures between the client and front/server.
@@ -595,13 +636,13 @@ type meekCookieData struct {
 // all consequent HTTP requests
 // In unfronted meek mode, the cookie is visible over the adversary network, so the
 // cookie is encrypted and obfuscated.
-func makeCookie(serverEntry *ServerEntry, sessionId string) (cookie *http.Cookie, err error) {
+func makeCookie(meekConfig *MeekConfig) (cookie *http.Cookie, err error) {
 
 	// Make the JSON data
-	serverAddress := fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.SshObfuscatedPort)
+	serverAddress := meekConfig.PsiphonServerAddress
 	cookieData := &meekCookieData{
 		ServerAddress:       serverAddress,
-		SessionID:           sessionId,
+		SessionID:           meekConfig.SessionID,
 		MeekProtocolVersion: MEEK_PROTOCOL_VERSION,
 	}
 	serializedCookie, err := json.Marshal(cookieData)
@@ -618,7 +659,7 @@ func makeCookie(serverEntry *ServerEntry, sessionId string) (cookie *http.Cookie
 	// different messages if the messages are sent to two different public keys."
 	var nonce [24]byte
 	var publicKey [32]byte
-	decodedPublicKey, err := base64.StdEncoding.DecodeString(serverEntry.MeekCookieEncryptionPublicKey)
+	decodedPublicKey, err := base64.StdEncoding.DecodeString(meekConfig.MeekCookieEncryptionPublicKey)
 	if err != nil {
 		return nil, ContextError(err)
 	}
@@ -634,7 +675,7 @@ func makeCookie(serverEntry *ServerEntry, sessionId string) (cookie *http.Cookie
 
 	// Obfuscate the encrypted data
 	obfuscator, err := NewObfuscator(
-		&ObfuscatorConfig{Keyword: serverEntry.MeekObfuscatedKey, MaxPadding: MEEK_COOKIE_MAX_PADDING})
+		&ObfuscatorConfig{Keyword: meekConfig.MeekObfuscatedKey, MaxPadding: MEEK_COOKIE_MAX_PADDING})
 	if err != nil {
 		return nil, ContextError(err)
 	}

psiphon/net.go

@@ -59,6 +59,7 @@ type DialConfig struct {
 	// Dials may be interrupted using PendingConns.CloseAll(). Once instantiated,
 	// a conn is added to pendingConns before the network connect begins and
 	// removed from pendingConns once the connect succeeds or fails.
+	// May be nil.
 	PendingConns *Conns
 
 	// BindToDevice parameters are used to exclude connections and
@@ -83,11 +84,18 @@ type DialConfig struct {
 	// SSL_CTX_load_verify_locations.
 	// Only applies to UseIndistinguishableTLS connections.
 	TrustedCACertificatesFilename string
-}
 
-// DeviceBinder defines the interface to the external BindToDevice provider
-type DeviceBinder interface {
-	BindToDevice(fileDescriptor int) error
+	// DeviceRegion is the reported region the host device is running in.
+	// When set, this value may be used, pre-connection, to select performance
+	// or circumvention optimization strategies for the given region.
+	DeviceRegion string
+
+	// ResolvedIPCallback, when set, is called with the IP address that was
+	// dialed. This is either the specified IP address in the dial address,
+	// or the resolved IP address in the case where the dial address is a
+	// domain name.
+	// The callback may be invoked by a concurrent goroutine.
+	ResolvedIPCallback func(string)
 }
 
 // NetworkConnectivityChecker defines the interface to the external
@@ -97,12 +105,31 @@ type NetworkConnectivityChecker interface {
 	HasNetworkConnectivity() int
 }
 
+// DeviceBinder defines the interface to the external BindToDevice provider
+type DeviceBinder interface {
+	BindToDevice(fileDescriptor int) error
+}
+
 // DnsServerGetter defines the interface to the external GetDnsServer provider
 type DnsServerGetter interface {
 	GetPrimaryDnsServer() string
 	GetSecondaryDnsServer() string
 }
 
+// HostNameTransformer defines the interface for pluggable hostname
+// transformation circumvention strategies.
+type HostNameTransformer interface {
+	TransformHostName(hostname string) (string, bool)
+}
+
+// IdentityHostNameTransformer is the default HostNameTransformer, which
+// returns the hostname unchanged.
+type IdentityHostNameTransformer struct{}
+
+func (IdentityHostNameTransformer) TransformHostName(hostname string) (string, bool) {
+	return hostname, false
+}
+
 // TimeoutError implements the error interface
 type TimeoutError struct{}
 
@@ -264,26 +291,15 @@ func MakeUntunneledHttpsClient(
 	requestUrl string,
 	requestTimeout time.Duration) (*http.Client, string, error) {
 
-	dialer := NewCustomTLSDialer(
-		// Note: when verifyLegacyCertificate is not nil, some
-		// of the other CustomTLSConfig is overridden.
-		&CustomTLSConfig{
-			Dial: NewTCPDialer(dialConfig),
-			VerifyLegacyCertificate:       verifyLegacyCertificate,
-			SendServerName:                true,
-			SkipVerify:                    false,
-			UseIndistinguishableTLS:       dialConfig.UseIndistinguishableTLS,
-			TrustedCACertificatesFilename: dialConfig.TrustedCACertificatesFilename,
-		})
+	// Change the scheme to "http"; otherwise http.Transport will try to do
+	// another TLS handshake inside the explicit TLS session. Also need to
+	// force an explicit port, as the default for "http", 80, won't talk TLS.
 
 	urlComponents, err := url.Parse(requestUrl)
 	if err != nil {
 		return nil, "", ContextError(err)
 	}
 
-	// Change the scheme to "http"; otherwise http.Transport will try to do
-	// another TLS handshake inside the explicit TLS session. Also need to
-	// force an explicit port, as the default for "http", 80, won't talk TLS.
 	urlComponents.Scheme = "http"
 	host, port, err := net.SplitHostPort(urlComponents.Host)
 	if err != nil {
@@ -296,6 +312,21 @@ func MakeUntunneledHttpsClient(
 	}
 	urlComponents.Host = net.JoinHostPort(host, port)
 
+	// Note: IndistinguishableTLS mode doesn't support VerifyLegacyCertificate
+	useIndistinguishableTLS := dialConfig.UseIndistinguishableTLS && verifyLegacyCertificate == nil
+
+	dialer := NewCustomTLSDialer(
+		// Note: when verifyLegacyCertificate is not nil, some
+		// of the other CustomTLSConfig is overridden.
+		&CustomTLSConfig{
+			Dial: NewTCPDialer(dialConfig),
+			VerifyLegacyCertificate:       verifyLegacyCertificate,
+			SNIServerName:                 host,
+			SkipVerify:                    false,
+			UseIndistinguishableTLS:       useIndistinguishableTLS,
+			TrustedCACertificatesFilename: dialConfig.TrustedCACertificatesFilename,
+		})
+
 	transport := &http.Transport{
 		Dial: dialer,
 	}

psiphon/notice.go

@@ -29,11 +29,25 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 )
 
 var noticeLoggerMutex sync.Mutex
 var noticeLogger = log.New(os.Stderr, "", 0)
+var noticeLogDiagnostics = int32(0)
+
+func setEmitDiagnosticNotices(enable bool) {
+	if enable {
+		atomic.StoreInt32(&noticeLogDiagnostics, 1)
+	} else {
+		atomic.StoreInt32(&noticeLogDiagnostics, 0)
+	}
+}
+
+func getEmitDiagnoticNotices() bool {
+	return atomic.LoadInt32(&noticeLogDiagnostics) == 1
+}
 
 // SetNoticeOutput sets a target writer to receive notices. By default,
 // notices are written to stderr.
@@ -60,7 +74,12 @@ func SetNoticeOutput(output io.Writer) {
 }
 
 // outputNotice encodes a notice in JSON and writes it to the output writer.
-func outputNotice(noticeType string, showUser bool, args ...interface{}) {
+func outputNotice(noticeType string, isDiagnostic, showUser bool, args ...interface{}) {
+
+	if isDiagnostic && !getEmitDiagnoticNotices() {
+		return
+	}
+
 	obj := make(map[string]interface{})
 	noticeData := make(map[string]interface{})
 	obj["noticeType"] = noticeType
@@ -88,22 +107,22 @@ func outputNotice(noticeType string, showUser bool, args ...interface{}) {
 
 // NoticeInfo is an informational message
 func NoticeInfo(format string, args ...interface{}) {
-	outputNotice("Info", false, "message", fmt.Sprintf(format, args...))
+	outputNotice("Info", true, false, "message", fmt.Sprintf(format, args...))
 }
 
 // NoticeAlert is an alert message; typically a recoverable error condition
 func NoticeAlert(format string, args ...interface{}) {
-	outputNotice("Alert", false, "message", fmt.Sprintf(format, args...))
+	outputNotice("Alert", true, false, "message", fmt.Sprintf(format, args...))
 }
 
 // NoticeError is an error message; typically an unrecoverable error condition
 func NoticeError(format string, args ...interface{}) {
-	outputNotice("Error", true, "message", fmt.Sprintf(format, args...))
+	outputNotice("Error", true, false, "message", fmt.Sprintf(format, args...))
 }
 
 // NoticeCandidateServers is how many possible servers are available for the selected region and protocol
 func NoticeCandidateServers(region, protocol string, count int) {
-	outputNotice("CandidateServers", false, "region", region, "protocol", protocol, "count", count)
+	outputNotice("CandidateServers", false, false, "region", region, "protocol", protocol, "count", count)
 }
 
 // NoticeAvailableEgressRegions is what regions are available for egress from.
@@ -114,63 +133,95 @@ func NoticeAvailableEgressRegions(regions []string) {
 	repetitionMessage := strings.Join(sortedRegions, "")
 	outputRepetitiveNotice(
 		"AvailableEgressRegions", repetitionMessage, 0,
-		"AvailableEgressRegions", false, "regions", sortedRegions)
+		"AvailableEgressRegions", false, false, "regions", sortedRegions)
 }
 
 // NoticeConnectingServer is details on a connection attempt
-func NoticeConnectingServer(ipAddress, region, protocol, frontingAddress string) {
-	outputNotice("ConnectingServer", false, "ipAddress", ipAddress, "region",
-		region, "protocol", protocol, "frontingAddress", frontingAddress)
+func NoticeConnectingServer(ipAddress, region, protocol, directTCPDialAddress string, meekConfig *MeekConfig) {
+	if meekConfig == nil {
+		outputNotice("ConnectingServer", true, false,
+			"ipAddress", ipAddress,
+			"region", region,
+			"protocol", protocol,
+			"directTCPDialAddress", directTCPDialAddress)
+	} else {
+		outputNotice("ConnectingServer", true, false,
+			"ipAddress", ipAddress,
+			"region", region,
+			"protocol", protocol,
+			"meekDialAddress", meekConfig.DialAddress,
+			"meekUseHTTPS", meekConfig.UseHTTPS,
+			"meekSNIServerName", meekConfig.SNIServerName,
+			"meekHostHeader", meekConfig.HostHeader,
+			"meekTransformedHostName", meekConfig.TransformedHostName)
+	}
 }
 
 // NoticeActiveTunnel is a successful connection that is used as an active tunnel for port forwarding
 func NoticeActiveTunnel(ipAddress, protocol string) {
-	outputNotice("ActiveTunnel", false, "ipAddress", ipAddress, "protocol", protocol)
+	outputNotice("ActiveTunnel", true, false, "ipAddress", ipAddress, "protocol", protocol)
 }
 
 // NoticeSocksProxyPortInUse is a failure to use the configured LocalSocksProxyPort
 func NoticeSocksProxyPortInUse(port int) {
-	outputNotice("SocksProxyPortInUse", true, "port", port)
+	outputNotice("SocksProxyPortInUse", false, true, "port", port)
 }
 
 // NoticeListeningSocksProxyPort is the selected port for the listening local SOCKS proxy
 func NoticeListeningSocksProxyPort(port int) {
-	outputNotice("ListeningSocksProxyPort", false, "port", port)
+	outputNotice("ListeningSocksProxyPort", false, false, "port", port)
 }
 
 // NoticeSocksProxyPortInUse is a failure to use the configured LocalHttpProxyPort
 func NoticeHttpProxyPortInUse(port int) {
-	outputNotice("HttpProxyPortInUse", true, "port", port)
+	outputNotice("HttpProxyPortInUse", false, true, "port", port)
 }
 
 // NoticeListeningSocksProxyPort is the selected port for the listening local HTTP proxy
 func NoticeListeningHttpProxyPort(port int) {
-	outputNotice("ListeningHttpProxyPort", false, "port", port)
+	outputNotice("ListeningHttpProxyPort", false, false, "port", port)
 }
 
 // NoticeClientUpgradeAvailable is an available client upgrade, as per the handshake. The
 // client should download and install an upgrade.
 func NoticeClientUpgradeAvailable(version string) {
-	outputNotice("ClientUpgradeAvailable", false, "version", version)
+	outputNotice("ClientUpgradeAvailable", false, false, "version", version)
+}
+
+// NoticeClientIsLatestVersion reports that an upgrade check was made and the client
+// is already the latest version. availableVersion is the version available for download,
+// if known.
+func NoticeClientIsLatestVersion(availableVersion string) {
+	outputNotice("ClientIsLatestVersion", false, false, "availableVersion", availableVersion)
 }
 
 // NoticeClientUpgradeAvailable is a sponsor homepage, as per the handshake. The client
 // should display the sponsor's homepage.
 func NoticeHomepage(url string) {
-	outputNotice("Homepage", false, "url", url)
+	outputNotice("Homepage", false, false, "url", url)
 }
 
 // NoticeClientRegion is the client's region, as determined by the server and
 // reported to the client in the handshake.
 func NoticeClientRegion(region string) {
-	outputNotice("ClientRegion", false, "region", region)
+	outputNotice("ClientRegion", true, false, "region", region)
 }
 
 // NoticeTunnels is how many active tunnels are available. The client should use this to
 // determine connecting/unexpected disconnect state transitions. When count is 0, the core is
 // disconnected; when count > 1, the core is connected.
 func NoticeTunnels(count int) {
-	outputNotice("Tunnels", false, "count", count)
+	outputNotice("Tunnels", false, false, "count", count)
+}
+
+// NoticeSessionId is the session ID used across all tunnels established by the controller.
+func NoticeSessionId(sessionId string) {
+	outputNotice("SessionId", true, false, "sessionId", sessionId)
+}
+
+func NoticeImpairedProtocolClassification(impairedProtocolClassification map[string]int) {
+	outputNotice("ImpairedProtocolClassification", false, false,
+		"classification", impairedProtocolClassification)
 }
 
 // NoticeUntunneled indicates than an address has been classified as untunneled and is being
@@ -180,38 +231,53 @@ func NoticeTunnels(count int) {
 // users, not for diagnostics logs.
 //
 func NoticeUntunneled(address string) {
-	outputNotice("Untunneled", true, "address", address)
+	outputNotice("Untunneled", false, true, "address", address)
 }
 
 // NoticeSplitTunnelRegion reports that split tunnel is on for the given region.
 func NoticeSplitTunnelRegion(region string) {
-	outputNotice("SplitTunnelRegion", true, "region", region)
+	outputNotice("SplitTunnelRegion", false, true, "region", region)
 }
 
 // NoticeUpstreamProxyError reports an error when connecting to an upstream proxy. The
 // user may have input, for example, an incorrect address or incorrect credentials.
 func NoticeUpstreamProxyError(err error) {
-	outputNotice("UpstreamProxyError", true, "message", err.Error())
+	outputNotice("UpstreamProxyError", false, true, "message", err.Error())
+}
+
+// NoticeClientUpgradeDownloadedBytes reports client upgrade download progress.
+func NoticeClientUpgradeDownloadedBytes(bytes int64) {
+	outputNotice("ClientUpgradeDownloadedBytes", true, false, "bytes", bytes)
 }
 
 // NoticeClientUpgradeDownloaded indicates that a client upgrade download
 // is complete and available at the destination specified.
 func NoticeClientUpgradeDownloaded(filename string) {
-	outputNotice("ClientUpgradeDownloaded", false, "filename", filename)
+	outputNotice("ClientUpgradeDownloaded", false, false, "filename", filename)
 }
 
 // NoticeBytesTransferred reports how many tunneled bytes have been
 // transferred since the last NoticeBytesTransferred, for the tunnel
 // to the server at ipAddress.
 func NoticeBytesTransferred(ipAddress string, sent, received int64) {
-	outputNotice("BytesTransferred", false, "ipAddress", ipAddress, "sent", sent, "received", received)
+	if getEmitDiagnoticNotices() {
+		outputNotice("BytesTransferred", true, false, "ipAddress", ipAddress, "sent", sent, "received", received)
+	} else {
+		// This case keeps the EmitBytesTransferred and EmitDiagnosticNotices config options independent
+		outputNotice("BytesTransferred", false, false, "sent", sent, "received", received)
+	}
 }
 
 // NoticeTotalBytesTransferred reports how many tunneled bytes have been
 // transferred in total up to this point, for the tunnel to the server
 // at ipAddress.
 func NoticeTotalBytesTransferred(ipAddress string, sent, received int64) {
-	outputNotice("TotalBytesTransferred", false, "ipAddress", ipAddress, "sent", sent, "received", received)
+	if getEmitDiagnoticNotices() {
+		outputNotice("TotalBytesTransferred", true, false, "ipAddress", ipAddress, "sent", sent, "received", received)
+	} else {
+		// This case keeps the EmitBytesTransferred and EmitDiagnosticNotices config options independent
+		outputNotice("TotalBytesTransferred", false, false, "sent", sent, "received", received)
+	}
 }
 
 // NoticeLocalProxyError reports a local proxy error message. Repetitive
@@ -231,7 +297,28 @@ func NoticeLocalProxyError(proxyType string, err error) {
 
 	outputRepetitiveNotice(
 		"LocalProxyError"+proxyType, repetitionMessage, 1,
-		"LocalProxyError", false, "message", err.Error())
+		"LocalProxyError", true, false, "message", err.Error())
+}
+
+// NoticeConnectedMeekStats reports extra network details for a meek tunnel connection.
+func NoticeConnectedMeekStats(ipAddress string, meekStats *MeekStats) {
+	outputNotice("NoticeConnectedMeekStats", true, false,
+		"ipAddress", ipAddress,
+		"dialAddress", meekStats.DialAddress,
+		"resolvedIPAddress", meekStats.ResolvedIPAddress,
+		"sniServerName", meekStats.SNIServerName,
+		"hostHeader", meekStats.HostHeader,
+		"transformedHostName", meekStats.TransformedHostName)
+}
+
+// NoticeBuildInfo reports build version info.
+func NoticeBuildInfo(buildDate, buildRepo, buildRev, goVersion, gomobileVersion string) {
+	outputNotice("NoticeBuildInfo", false, false,
+		"buildDate", buildDate,
+		"buildRepo", buildRepo,
+		"buildRev", buildRev,
+		"goVersion", goVersion,
+		"gomobileVersion", gomobileVersion)
 }
 
 type repetitiveNoticeState struct {
@@ -248,7 +335,7 @@ var repetitiveNoticeStates = make(map[string]*repetitiveNoticeState)
 // until the repetitionMessage differs.
 func outputRepetitiveNotice(
 	repetitionKey, repetitionMessage string, repeatLimit int,
-	noticeType string, showUser bool, args ...interface{}) {
+	noticeType string, isDiagnostic, showUser bool, args ...interface{}) {
 
 	repetitiveNoticeMutex.Lock()
 	defer repetitiveNoticeMutex.Unlock()
@@ -274,7 +361,7 @@ func outputRepetitiveNotice(
 		if state.repeats > 0 {
 			args = append(args, "repeats", state.repeats)
 		}
-		outputNotice(noticeType, showUser, args...)
+		outputNotice(noticeType, isDiagnostic, showUser, args...)
 	}
 }
 

psiphon/opensslConn.go

@@ -96,10 +96,15 @@ func newOpenSSLConn(rawConn net.Conn, hostname string, config *CustomTLSConfig)
 		return nil, ContextError(err)
 	}
 
-	if config.SendServerName {
-		err = conn.SetTlsExtHostName(hostname)
-		if err != nil {
-			return nil, ContextError(err)
+	if config.SNIServerName != "" {
+		// Explicitly exclude IPs:
+		// - "Literal IPv4 and IPv6 addresses are not permitted": https://tools.ietf.org/html/rfc6066#page-6.
+		// - OpenSSL does not appear to enforce this rule itself.
+		if net.ParseIP(config.SNIServerName) == nil {
+			err = conn.SetTlsExtHostName(config.SNIServerName)
+			if err != nil {
+				return nil, ContextError(err)
+			}
 		}
 	}
 

psiphon/remoteServerList.go

@@ -86,7 +86,10 @@ func FetchRemoteServerList(config *Config, dialConfig *DialConfig) (err error) {
 		return ContextError(err)
 	}
 
-	serverEntries, err := DecodeAndValidateServerEntryList(remoteServerList)
+	serverEntries, err := DecodeAndValidateServerEntryList(
+		remoteServerList,
+		GetCurrentTimestamp(),
+		SERVER_ENTRY_SOURCE_REMOTE)
 	if err != nil {
 		return ContextError(err)
 	}

psiphon/serverApi.go

@@ -51,6 +51,15 @@ type ServerContext struct {
 	serverHandshakeTimestamp string
 }
 
+// MeekStats holds extra stats that are only gathered for meek tunnels.
+type MeekStats struct {
+	DialAddress         string
+	ResolvedIPAddress   string
+	SNIServerName       string
+	HostHeader          string
+	TransformedHostName bool
+}
+
 // nextTunnelNumber is a monotonically increasing number assigned to each
 // successive tunnel connection. The sessionId and tunnelNumber together
 // form a globally unique identifier for tunnels, which is used for
@@ -152,11 +161,18 @@ func (serverContext *ServerContext) doHandshakeRequest() error {
 	var decodedServerEntries []*ServerEntry
 
 	// Store discovered server entries
+	// We use the server's time, as it's available here, for the server entry
+	// timestamp since this is more reliable than the client time.
 	for _, encodedServerEntry := range handshakeConfig.EncodedServerList {
-		serverEntry, err := DecodeServerEntry(encodedServerEntry)
+
+		serverEntry, err := DecodeServerEntry(
+			encodedServerEntry,
+			TruncateTimestampToHour(handshakeConfig.ServerTimestamp),
+			SERVER_ENTRY_SOURCE_DISCOVERY)
 		if err != nil {
 			return ContextError(err)
 		}
+
 		err = ValidateServerEntry(serverEntry)
 		if err != nil {
 			// Skip this entry and continue with the next one
@@ -183,6 +199,8 @@ func (serverContext *ServerContext) doHandshakeRequest() error {
 	serverContext.clientUpgradeVersion = handshakeConfig.UpgradeClientVersion
 	if handshakeConfig.UpgradeClientVersion != "" {
 		NoticeClientUpgradeAvailable(handshakeConfig.UpgradeClientVersion)
+	} else {
+		NoticeClientIsLatestVersion("")
 	}
 
 	var regexpsNotices []string
@@ -406,13 +424,21 @@ func doUntunneledStatusRequest(
 		return ContextError(err)
 	}
 
+	dialConfig := tunnel.untunneledDialConfig
+
 	timeout := PSIPHON_API_SERVER_TIMEOUT
 	if isShutdown {
 		timeout = PSIPHON_API_SHUTDOWN_SERVER_TIMEOUT
+
+		// Use a copy of DialConfig without pendingConns. This ensures
+		// this request isn't interrupted/canceled. This measure should
+		// be used only with the very short PSIPHON_API_SHUTDOWN_SERVER_TIMEOUT.
+		dialConfig = new(DialConfig)
+		*dialConfig = *tunnel.untunneledDialConfig
 	}
 
 	httpClient, requestUrl, err := MakeUntunneledHttpsClient(
-		tunnel.untunneledDialConfig,
+		dialConfig,
 		certificate,
 		url,
 		timeout)
@@ -599,6 +625,59 @@ func makeBaseRequestUrl(tunnel *Tunnel, port, sessionId string) string {
 	requestUrl.WriteString(tunnel.config.ClientPlatform)
 	requestUrl.WriteString("&tunnel_whole_device=")
 	requestUrl.WriteString(strconv.Itoa(tunnel.config.TunnelWholeDevice))
+
+	// The following parameters may be blank and must
+	// not be sent to the server if blank.
+
+	if tunnel.config.DeviceRegion != "" {
+		requestUrl.WriteString("&device_region=")
+		requestUrl.WriteString(tunnel.config.DeviceRegion)
+	}
+	if tunnel.meekStats != nil {
+		if tunnel.meekStats.DialAddress != "" {
+			requestUrl.WriteString("&meek_dial_address=")
+			requestUrl.WriteString(tunnel.meekStats.DialAddress)
+		}
+		if tunnel.meekStats.ResolvedIPAddress != "" {
+			requestUrl.WriteString("&meek_resolved_ip_address=")
+			requestUrl.WriteString(tunnel.meekStats.ResolvedIPAddress)
+		}
+		if tunnel.meekStats.SNIServerName != "" {
+			requestUrl.WriteString("&meek_sni_server_name=")
+			requestUrl.WriteString(tunnel.meekStats.SNIServerName)
+		}
+		if tunnel.meekStats.HostHeader != "" {
+			requestUrl.WriteString("&meek_host_header=")
+			requestUrl.WriteString(tunnel.meekStats.HostHeader)
+		}
+		requestUrl.WriteString("&meek_transformed_host_name=")
+		if tunnel.meekStats.TransformedHostName {
+			requestUrl.WriteString("1")
+		} else {
+			requestUrl.WriteString("0")
+		}
+	}
+
+	if tunnel.serverEntry.Region != "" {
+		requestUrl.WriteString("&server_entry_region=")
+		requestUrl.WriteString(tunnel.serverEntry.Region)
+	}
+
+	if tunnel.serverEntry.LocalSource != "" {
+		requestUrl.WriteString("&server_entry_source=")
+		requestUrl.WriteString(tunnel.serverEntry.LocalSource)
+	}
+
+	// As with last_connected, this timestamp stat, which may be
+	// a precise handshake request server timestamp, is truncated
+	// to hour granularity to avoid introducing a reconstructable
+	// cross-session user trace into server logs.
+	localServerEntryTimestamp := TruncateTimestampToHour(tunnel.serverEntry.LocalTimestamp)
+	if localServerEntryTimestamp != "" {
+		requestUrl.WriteString("&server_entry_timestamp=")
+		requestUrl.WriteString(localServerEntryTimestamp)
+	}
+
 	return requestUrl.String()
 }
 

psiphon/serverEntry.go

@@ -35,10 +35,12 @@ const (
 	TUNNEL_PROTOCOL_UNFRONTED_MEEK       = "UNFRONTED-MEEK-OSSH"
 	TUNNEL_PROTOCOL_UNFRONTED_MEEK_HTTPS = "UNFRONTED-MEEK-HTTPS-OSSH"
 	TUNNEL_PROTOCOL_FRONTED_MEEK         = "FRONTED-MEEK-OSSH"
+	TUNNEL_PROTOCOL_FRONTED_MEEK_HTTP    = "FRONTED-MEEK-HTTP-OSSH"
 )
 
 var SupportedTunnelProtocols = []string{
 	TUNNEL_PROTOCOL_FRONTED_MEEK,
+	TUNNEL_PROTOCOL_FRONTED_MEEK_HTTP,
 	TUNNEL_PROTOCOL_UNFRONTED_MEEK,
 	TUNNEL_PROTOCOL_UNFRONTED_MEEK_HTTPS,
 	TUNNEL_PROTOCOL_OBFUSCATED_SSH,
@@ -46,8 +48,8 @@ var SupportedTunnelProtocols = []string{
 }
 
 // ServerEntry represents a Psiphon server. It contains information
-// about how to estalish a tunnel connection to the server through
-// several protocols. ServerEntry are JSON records downloaded from
+// about how to establish a tunnel connection to the server through
+// several protocols. Server entries are JSON records downloaded from
 // various sources.
 type ServerEntry struct {
 	IpAddress                     string   `json:"ipAddress"`
@@ -66,11 +68,28 @@ type ServerEntry struct {
 	MeekCookieEncryptionPublicKey string   `json:"meekCookieEncryptionPublicKey"`
 	MeekObfuscatedKey             string   `json:"meekObfuscatedKey"`
 	MeekFrontingHost              string   `json:"meekFrontingHost"`
+	MeekFrontingHosts             []string `json:"meekFrontingHosts"`
 	MeekFrontingDomain            string   `json:"meekFrontingDomain"`
 	MeekFrontingAddresses         []string `json:"meekFrontingAddresses"`
 	MeekFrontingAddressesRegex    string   `json:"meekFrontingAddressesRegex"`
+	MeekFrontingDisableSNI        bool     `json:"meekFrontingDisableSNI"`
+
+	// These local fields are not expected to be present in downloaded server
+	// entries. They are added by the client to record and report stats about
+	// how and when server entries are obtained.
+	LocalSource    string `json:"localSource"`
+	LocalTimestamp string `json:"localTimestamp"`
 }
 
+type ServerEntrySource string
+
+const (
+	SERVER_ENTRY_SOURCE_EMBEDDED  ServerEntrySource = "EMBEDDED"
+	SERVER_ENTRY_SOURCE_REMOTE    ServerEntrySource = "REMOTE"
+	SERVER_ENTRY_SOURCE_DISCOVERY ServerEntrySource = "DISCOVERY"
+	SERVER_ENTRY_SOURCE_TARGET    ServerEntrySource = "TARGET"
+)
+
 // SupportsProtocol returns true if and only if the ServerEntry has
 // the necessary capability to support the specified tunnel protocol.
 func (serverEntry *ServerEntry) SupportsProtocol(protocol string) bool {
@@ -127,22 +146,39 @@ func (serverEntry *ServerEntry) GetDirectWebRequestPorts() []string {
 
 // DecodeServerEntry extracts server entries from the encoding
 // used by remote server lists and Psiphon server handshake requests.
-func DecodeServerEntry(encodedServerEntry string) (serverEntry *ServerEntry, err error) {
+//
+// The resulting ServerEntry.LocalSource is populated with serverEntrySource,
+// which should be one of SERVER_ENTRY_SOURCE_EMBEDDED, SERVER_ENTRY_SOURCE_REMOTE,
+// SERVER_ENTRY_SOURCE_DISCOVERY, SERVER_ENTRY_SOURCE_TARGET.
+// ServerEntry.LocalTimestamp is populated with the provided timestamp, which
+// should be a RFC 3339 formatted string. These local fields are stored with the
+// server entry and reported to the server as stats (a coarse granularity timestamp
+// is reported).
+func DecodeServerEntry(
+	encodedServerEntry, timestamp string,
+	serverEntrySource ServerEntrySource) (serverEntry *ServerEntry, err error) {
+
 	hexDecodedServerEntry, err := hex.DecodeString(encodedServerEntry)
 	if err != nil {
 		return nil, ContextError(err)
 	}
+
 	// Skip past legacy format (4 space delimited fields) and just parse the JSON config
 	fields := bytes.SplitN(hexDecodedServerEntry, []byte(" "), 5)
 	if len(fields) != 5 {
 		return nil, ContextError(errors.New("invalid encoded server entry"))
 	}
+
 	serverEntry = new(ServerEntry)
 	err = json.Unmarshal(fields[4], &serverEntry)
 	if err != nil {
 		return nil, ContextError(err)
 	}
 
+	// NOTE: if the source JSON happens to have values in these fields, they get clobbered.
+	serverEntry.LocalSource = string(serverEntrySource)
+	serverEntry.LocalTimestamp = timestamp
+
 	return serverEntry, nil
 }
 
@@ -166,7 +202,11 @@ func ValidateServerEntry(serverEntry *ServerEntry) error {
 // DecodeAndValidateServerEntryList extracts server entries from the list encoding
 // used by remote server lists and Psiphon server handshake requests.
 // Each server entry is validated and invalid entries are skipped.
-func DecodeAndValidateServerEntryList(encodedServerEntryList string) (serverEntries []*ServerEntry, err error) {
+// See DecodeServerEntry for note on serverEntrySource/timestamp.
+func DecodeAndValidateServerEntryList(
+	encodedServerEntryList, timestamp string,
+	serverEntrySource ServerEntrySource) (serverEntries []*ServerEntry, err error) {
+
 	serverEntries = make([]*ServerEntry, 0)
 	for _, encodedServerEntry := range strings.Split(encodedServerEntryList, "\n") {
 		if len(encodedServerEntry) == 0 {
@@ -174,7 +214,7 @@ func DecodeAndValidateServerEntryList(encodedServerEntryList string) (serverEntr
 		}
 
 		// TODO: skip this entry and continue if can't decode?
-		serverEntry, err := DecodeServerEntry(encodedServerEntry)
+		serverEntry, err := DecodeServerEntry(encodedServerEntry, timestamp, serverEntrySource)
 		if err != nil {
 			return nil, ContextError(err)
 		}

psiphon/serverEntry_test.go

@@ -40,7 +40,8 @@ func TestDecodeAndValidateServerEntryList(t *testing.T) {
 		hex.EncodeToString([]byte(_INVALID_WINDOWS_REGISTRY_LEGACY_SERVER_ENTRY)) + "\n" +
 		hex.EncodeToString([]byte(_INVALID_MALFORMED_IP_ADDRESS_SERVER_ENTRY))
 
-	serverEntries, err := DecodeAndValidateServerEntryList(testEncodedServerEntryList)
+	serverEntries, err := DecodeAndValidateServerEntryList(
+		testEncodedServerEntryList, GetCurrentTimestamp(), SERVER_ENTRY_SOURCE_EMBEDDED)
 	if err != nil {
 		t.Error(err.Error())
 		t.FailNow()
@@ -62,7 +63,8 @@ func TestInvalidServerEntries(t *testing.T) {
 
 	for _, testCase := range testCases {
 		encodedServerEntry := hex.EncodeToString([]byte(testCase))
-		serverEntry, err := DecodeServerEntry(encodedServerEntry)
+		serverEntry, err := DecodeServerEntry(
+			encodedServerEntry, GetCurrentTimestamp(), SERVER_ENTRY_SOURCE_EMBEDDED)
 		if err != nil {
 			t.Error(err.Error())
 		}

psiphon/tlsDialer.go

@@ -91,12 +91,14 @@ type CustomTLSConfig struct {
 	// connection dial and TLS handshake.
 	Timeout time.Duration
 
-	// FrontingAddr overrides the "addr" input to Dial when specified
-	FrontingAddr string
+	// DialAddr overrides the "addr" input to Dial when specified
+	DialAddr string
 
-	// SendServerName specifies whether to use SNI
-	// (tlsdialer functionality)
-	SendServerName bool
+	// SNIServerName specifies the value to set in the SNI
+	// server_name field. When blank, SNI is omitted. Note that
+	// underlying TLS code also automatically omits SNI when
+	// the server_name is an IP address.
+	SNIServerName string
 
 	// SkipVerify completely disables server certificate verification.
 	SkipVerify bool
@@ -151,8 +153,8 @@ func CustomTLSDial(network, addr string, config *CustomTLSConfig) (net.Conn, err
 	}
 
 	dialAddr := addr
-	if config.FrontingAddr != "" {
-		dialAddr = config.FrontingAddr
+	if config.DialAddr != "" {
+		dialAddr = config.DialAddr
 	}
 
 	rawConn, err := config.Dial(network, dialAddr)
@@ -172,10 +174,13 @@ func CustomTLSDial(network, addr string, config *CustomTLSConfig) (net.Conn, err
 		tlsConfig.InsecureSkipVerify = true
 	}
 
-	if config.SendServerName && config.VerifyLegacyCertificate == nil {
+	if config.SNIServerName != "" && config.VerifyLegacyCertificate == nil {
 		// Set the ServerName and rely on the usual logic in
-		// tls.Conn.Handshake() to do its verification
-		tlsConfig.ServerName = hostname
+		// tls.Conn.Handshake() to do its verification.
+		// Note: Go TLS will automatically omit this ServerName when it's an IP address
+		if net.ParseIP(hostname) == nil {
+			tlsConfig.ServerName = config.SNIServerName
+		}
 	} else {
 		// No SNI.
 		// Disable verification in tls.Conn.Handshake().  We'll verify manually

psiphon/transferstats/transferstats_test.go

@@ -36,8 +36,11 @@ const (
 	_SERVER_ID = "myserverid"
 )
 
+var nextServerID = 0
+
 type StatsTestSuite struct {
 	suite.Suite
+	serverID   string
 	httpClient *http.Client
 }
 
@@ -47,9 +50,11 @@ func TestStatsTestSuite(t *testing.T) {
 
 func (suite *StatsTestSuite) SetupTest() {
 	re := make(Regexps, 0)
+	suite.serverID = fmt.Sprintf("%s-%d", _SERVER_ID, nextServerID)
+	nextServerID++
 	suite.httpClient = &http.Client{
 		Transport: &http.Transport{
-			Dial: makeStatsDialer(_SERVER_ID, &re),
+			Dial: makeStatsDialer(suite.serverID, &re),
 		},
 	}
 }
@@ -91,23 +96,20 @@ func (suite *StatsTestSuite) Test_StatsConn() {
 	resp, err = suite.httpClient.Get("https://example.org/index.html")
 	suite.Nil(err, "basic HTTPS requests should succeed")
 	resp.Body.Close()
-
-	// Clear out stats
-	_ = TakeOutStatsForServer(_SERVER_ID)
 }
 
 func (suite *StatsTestSuite) Test_TakeOutStatsForServer() {
 
 	zeroPayload := &AccumulatedStats{hostnameToStats: make(map[string]*hostStats)}
 
-	payload := TakeOutStatsForServer(_SERVER_ID)
+	payload := TakeOutStatsForServer(suite.serverID)
 	suite.Equal(payload, zeroPayload, "should get zero stats before any traffic")
 
 	resp, err := suite.httpClient.Get("http://example.com/index.html")
 	suite.Nil(err, "need successful http to proceed with tests")
 	resp.Body.Close()
 
-	payload = TakeOutStatsForServer(_SERVER_ID)
+	payload = TakeOutStatsForServer(suite.serverID)
 	suite.NotNil(payload, "should receive valid payload for valid server ID")
 
 	payloadJSON, err := json.Marshal(payload)
@@ -116,7 +118,7 @@ func (suite *StatsTestSuite) Test_TakeOutStatsForServer() {
 	suite.Nil(err, "payload JSON should parse successfully")
 
 	// After we retrieve the stats for a server, they should be cleared out of the tracked stats
-	payload = TakeOutStatsForServer(_SERVER_ID)
+	payload = TakeOutStatsForServer(suite.serverID)
 	suite.Equal(payload, zeroPayload, "after retrieving stats for a server, there should be zero stats (until more data goes through)")
 }
 
@@ -125,19 +127,19 @@ func (suite *StatsTestSuite) Test_PutBackStatsForServer() {
 	suite.Nil(err, "need successful http to proceed with tests")
 	resp.Body.Close()
 
-	payloadToPutBack := TakeOutStatsForServer(_SERVER_ID)
+	payloadToPutBack := TakeOutStatsForServer(suite.serverID)
 	suite.NotNil(payloadToPutBack, "should receive valid payload for valid server ID")
 
 	zeroPayload := &AccumulatedStats{hostnameToStats: make(map[string]*hostStats)}
 
-	payload := TakeOutStatsForServer(_SERVER_ID)
+	payload := TakeOutStatsForServer(suite.serverID)
 	suite.Equal(payload, zeroPayload, "should be zero stats after getting them")
 
-	PutBackStatsForServer(_SERVER_ID, payloadToPutBack)
+	PutBackStatsForServer(suite.serverID, payloadToPutBack)
 	// PutBack is asynchronous, so we'll need to wait a moment for it to do its thing
 	<-time.After(100 * time.Millisecond)
 
-	payload = TakeOutStatsForServer(_SERVER_ID)
+	payload = TakeOutStatsForServer(suite.serverID)
 	suite.NotEqual(payload, zeroPayload, "stats should be re-added after putting back")
 	suite.Equal(payload, payloadToPutBack, "stats should be the same as after the first retrieval")
 }
@@ -196,7 +198,7 @@ func (suite *StatsTestSuite) Test_Regex() {
 
 	suite.httpClient = &http.Client{
 		Transport: &http.Transport{
-			Dial: makeStatsDialer(_SERVER_ID, regexps),
+			Dial: makeStatsDialer(suite.serverID, regexps),
 		},
 	}
 
@@ -220,7 +222,7 @@ func (suite *StatsTestSuite) Test_Regex() {
 		suite.Nil(err)
 		resp.Body.Close()
 
-		payload := TakeOutStatsForServer(_SERVER_ID)
+		payload := TakeOutStatsForServer(suite.serverID)
 		suite.NotNil(payload, "should get stats because we made HTTP reqs; %s", scheme)
 
 		expectedHostnames := mapset.NewSet()
@@ -251,7 +253,7 @@ func (suite *StatsTestSuite) Test_getTLSHostname() {
 
 	// TODO: Talk to a local TCP server instead of spamming example.com
 
-	dialer := makeStatsDialer(_SERVER_ID, nil)
+	dialer := makeStatsDialer(suite.serverID, nil)
 
 	// Data too short
 	conn, err := dialer("tcp", "example.com:80")

psiphon/tunnel.go

@@ -28,6 +28,7 @@ import (
 	"io"
 	"net"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	regen "github.com/Psiphon-Inc/goregen"
@@ -77,6 +78,7 @@ type Tunnel struct {
 	signalPortForwardFailure chan struct{}
 	totalPortForwardFailures int
 	startTime                time.Time
+	meekStats                *MeekStats
 }
 
 // EstablishTunnel first makes a network transport connection to the
@@ -103,7 +105,7 @@ func EstablishTunnel(
 	}
 
 	// Build transport layers and establish SSH connection
-	conn, sshClient, err := dialSsh(
+	conn, sshClient, meekStats, err := dialSsh(
 		config, pendingConns, serverEntry, selectedProtocol, sessionId)
 	if err != nil {
 		return nil, ContextError(err)
@@ -132,6 +134,7 @@ func EstablishTunnel(
 		// A buffer allows at least one signal to be sent even when the receiver is
 		// not listening. Senders should not block.
 		signalPortForwardFailure: make(chan struct{}, 1),
+		meekStats:                meekStats,
 	}
 
 	// Create a new Psiphon API server context for this tunnel. This includes
@@ -331,73 +334,181 @@ func selectProtocol(config *Config, serverEntry *ServerEntry) (selectedProtocol
 	return selectedProtocol, nil
 }
 
-// dialSsh is a helper that builds the transport layers and establishes the SSH connection
-func dialSsh(
+// selectFrontingParameters is a helper which selects/generates meek fronting
+// parameters where the server entry provides multiple options or patterns.
+func selectFrontingParameters(
+	serverEntry *ServerEntry) (frontingAddress, frontingHost string, err error) {
+
+	if len(serverEntry.MeekFrontingAddressesRegex) > 0 {
+
+		// Generate a front address based on the regex.
+
+		frontingAddress, err = regen.Generate(serverEntry.MeekFrontingAddressesRegex)
+		if err != nil {
+			return "", "", ContextError(err)
+		}
+	} else {
+
+		// Randomly select, for this connection attempt, one front address for
+		// fronting-capable servers.
+
+		if len(serverEntry.MeekFrontingAddresses) == 0 {
+			return "", "", ContextError(errors.New("MeekFrontingAddresses is empty"))
+		}
+		index, err := MakeSecureRandomInt(len(serverEntry.MeekFrontingAddresses))
+		if err != nil {
+			return "", "", ContextError(err)
+		}
+		frontingAddress = serverEntry.MeekFrontingAddresses[index]
+	}
+
+	if len(serverEntry.MeekFrontingHosts) > 0 {
+		index, err := MakeSecureRandomInt(len(serverEntry.MeekFrontingHosts))
+		if err != nil {
+			return "", "", ContextError(err)
+		}
+		frontingHost = serverEntry.MeekFrontingHosts[index]
+	} else {
+		// Backwards compatibility case
+		frontingHost = serverEntry.MeekFrontingHost
+	}
+
+	return
+}
+
+// initMeekConfig is a helper that creates a MeekConfig suitable for the
+// selected meek tunnel protocol.
+func initMeekConfig(
 	config *Config,
-	pendingConns *Conns,
 	serverEntry *ServerEntry,
 	selectedProtocol,
-	sessionId string) (conn net.Conn, sshClient *ssh.Client, err error) {
+	sessionId string) (*MeekConfig, error) {
+
+	// The meek protocol always uses OSSH
+	psiphonServerAddress := fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.SshObfuscatedPort)
+
+	var dialAddress string
+	useHTTPS := false
+	var SNIServerName, hostHeader string
+	transformedHostName := false
 
-	// The meek protocols tunnel obfuscated SSH. Obfuscated SSH is layered on top of SSH.
-	// So depending on which protocol is used, multiple layers are initialized.
-	port := 0
-	useMeek := false
-	useMeekHttps := false
-	useFronting := false
-	useObfuscatedSsh := false
 	switch selectedProtocol {
 	case TUNNEL_PROTOCOL_FRONTED_MEEK:
-		useMeek = true
-		useMeekHttps = true
-		useFronting = true
-		useObfuscatedSsh = true
+		frontingAddress, frontingHost, err := selectFrontingParameters(serverEntry)
+		if err != nil {
+			return nil, ContextError(err)
+		}
+		dialAddress = fmt.Sprintf("%s:443", frontingAddress)
+		useHTTPS = true
+		if !serverEntry.MeekFrontingDisableSNI {
+			SNIServerName, transformedHostName =
+				config.HostNameTransformer.TransformHostName(frontingAddress)
+		}
+		hostHeader = frontingHost
+
+	case TUNNEL_PROTOCOL_FRONTED_MEEK_HTTP:
+		frontingAddress, frontingHost, err := selectFrontingParameters(serverEntry)
+		if err != nil {
+			return nil, ContextError(err)
+		}
+		dialAddress = fmt.Sprintf("%s:80", frontingAddress)
+		hostHeader = frontingHost
+
 	case TUNNEL_PROTOCOL_UNFRONTED_MEEK:
-		useMeek = true
-		useObfuscatedSsh = true
-		port = serverEntry.SshObfuscatedPort
+		dialAddress = fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.MeekServerPort)
+		hostname := serverEntry.IpAddress
+		hostname, transformedHostName = config.HostNameTransformer.TransformHostName(hostname)
+		if serverEntry.MeekServerPort == 80 {
+			hostHeader = hostname
+		} else {
+			hostHeader = fmt.Sprintf("%s:%d", hostname, serverEntry.MeekServerPort)
+		}
+
 	case TUNNEL_PROTOCOL_UNFRONTED_MEEK_HTTPS:
-		useMeek = true
-		useMeekHttps = true
-		useObfuscatedSsh = true
-		port = serverEntry.SshObfuscatedPort
-	case TUNNEL_PROTOCOL_OBFUSCATED_SSH:
-		useObfuscatedSsh = true
-		port = serverEntry.SshObfuscatedPort
-	case TUNNEL_PROTOCOL_SSH:
-		port = serverEntry.SshPort
+		dialAddress = fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.MeekServerPort)
+		useHTTPS = true
+		SNIServerName, transformedHostName =
+			config.HostNameTransformer.TransformHostName(serverEntry.IpAddress)
+		if serverEntry.MeekServerPort == 443 {
+			hostHeader = serverEntry.IpAddress
+		} else {
+			hostHeader = fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.MeekServerPort)
+		}
+
+	default:
+		return nil, ContextError(errors.New("unexpected selectedProtocol"))
 	}
 
-	frontingAddress := ""
-	if useFronting {
-		if len(serverEntry.MeekFrontingAddressesRegex) > 0 {
+	// The unnderlying TLS will automatically disable SNI for IP address server name
+	// values; we have this explicit check here so we record the correct value for stats.
+	if net.ParseIP(SNIServerName) != nil {
+		SNIServerName = ""
+	}
 
-			// Generate a front address based on the regex.
+	return &MeekConfig{
+		DialAddress:                   dialAddress,
+		UseHTTPS:                      useHTTPS,
+		SNIServerName:                 SNIServerName,
+		HostHeader:                    hostHeader,
+		TransformedHostName:           transformedHostName,
+		PsiphonServerAddress:          psiphonServerAddress,
+		SessionID:                     sessionId,
+		MeekCookieEncryptionPublicKey: serverEntry.MeekCookieEncryptionPublicKey,
+		MeekObfuscatedKey:             serverEntry.MeekObfuscatedKey,
+	}, nil
+}
 
-			frontingAddress, err = regen.Generate(serverEntry.MeekFrontingAddressesRegex)
-			if err != nil {
-				return nil, nil, ContextError(err)
-			}
-		} else {
+// dialSsh is a helper that builds the transport layers and establishes the SSH connection.
+// When a meek protocols is selected, additional MeekStats are recorded and returned.
+func dialSsh(
+	config *Config,
+	pendingConns *Conns,
+	serverEntry *ServerEntry,
+	selectedProtocol,
+	sessionId string) (
+	conn net.Conn, sshClient *ssh.Client, meekStats *MeekStats, err error) {
 
-			// Randomly select, for this connection attempt, one front address for
-			// fronting-capable servers.
+	// The meek protocols tunnel obfuscated SSH. Obfuscated SSH is layered on top of SSH.
+	// So depending on which protocol is used, multiple layers are initialized.
 
-			if len(serverEntry.MeekFrontingAddresses) == 0 {
-				return nil, nil, ContextError(errors.New("MeekFrontingAddresses is empty"))
-			}
-			index, err := MakeSecureRandomInt(len(serverEntry.MeekFrontingAddresses))
-			if err != nil {
-				return nil, nil, ContextError(err)
-			}
-			frontingAddress = serverEntry.MeekFrontingAddresses[index]
+	useObfuscatedSsh := false
+	var directTCPDialAddress string
+	var meekConfig *MeekConfig
+
+	switch selectedProtocol {
+	case TUNNEL_PROTOCOL_OBFUSCATED_SSH:
+		useObfuscatedSsh = true
+		directTCPDialAddress = fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.SshObfuscatedPort)
+
+	case TUNNEL_PROTOCOL_SSH:
+		directTCPDialAddress = fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.SshPort)
+
+	default:
+		useObfuscatedSsh = true
+		meekConfig, err = initMeekConfig(config, serverEntry, selectedProtocol, sessionId)
+		if err != nil {
+			return nil, nil, nil, ContextError(err)
 		}
 	}
+
 	NoticeConnectingServer(
 		serverEntry.IpAddress,
 		serverEntry.Region,
 		selectedProtocol,
-		frontingAddress)
+		directTCPDialAddress,
+		meekConfig)
+
+	// Use an asynchronous callback to record the resolved IP address when
+	// dialing a domain name. Note that DialMeek doesn't immediately
+	// establish any HTTPS connections, so the resolved IP address won't be
+	// reported until during/after ssh session establishment (the ssh traffic
+	// is meek payload). So don't Load() the IP address value until after that
+	// has completed to ensure a result.
+	var resolvedIPAddress atomic.Value
+	resolvedIPAddress.Store("")
+	setResolvedIPAddress := func(IPAddress string) {
+		resolvedIPAddress.Store(IPAddress)
+	}
 
 	// Create the base transport: meek or direct connection
 	dialConfig := &DialConfig{
@@ -408,16 +519,18 @@ func dialSsh(
 		DnsServerGetter:               config.DnsServerGetter,
 		UseIndistinguishableTLS:       config.UseIndistinguishableTLS,
 		TrustedCACertificatesFilename: config.TrustedCACertificatesFilename,
+		DeviceRegion:                  config.DeviceRegion,
+		ResolvedIPCallback:            setResolvedIPAddress,
 	}
-	if useMeek {
-		conn, err = DialMeek(serverEntry, sessionId, useMeekHttps, frontingAddress, dialConfig)
+	if meekConfig != nil {
+		conn, err = DialMeek(meekConfig, dialConfig)
 		if err != nil {
-			return nil, nil, ContextError(err)
+			return nil, nil, nil, ContextError(err)
 		}
 	} else {
-		conn, err = DialTCP(fmt.Sprintf("%s:%d", serverEntry.IpAddress, port), dialConfig)
+		conn, err = DialTCP(directTCPDialAddress, dialConfig)
 		if err != nil {
-			return nil, nil, ContextError(err)
+			return nil, nil, nil, ContextError(err)
 		}
 	}
 
@@ -435,14 +548,14 @@ func dialSsh(
 	if useObfuscatedSsh {
 		sshConn, err = NewObfuscatedSshConn(conn, serverEntry.SshObfuscatedKey)
 		if err != nil {
-			return nil, nil, ContextError(err)
+			return nil, nil, nil, ContextError(err)
 		}
 	}
 
 	// Now establish the SSH session over the sshConn transport
 	expectedPublicKey, err := base64.StdEncoding.DecodeString(serverEntry.SshHostKey)
 	if err != nil {
-		return nil, nil, ContextError(err)
+		return nil, nil, nil, ContextError(err)
 	}
 	sshCertChecker := &ssh.CertChecker{
 		HostKeyFallback: func(addr string, remote net.Addr, publicKey ssh.PublicKey) error {
@@ -458,7 +571,7 @@ func dialSsh(
 			SshPassword string `json:"SshPassword"`
 		}{sessionId, serverEntry.SshPassword})
 	if err != nil {
-		return nil, nil, ContextError(err)
+		return nil, nil, nil, ContextError(err)
 	}
 	sshClientConfig := &ssh.ClientConfig{
 		User: serverEntry.SshUsername,
@@ -502,10 +615,22 @@ func dialSsh(
 
 	result := <-resultChannel
 	if result.err != nil {
-		return nil, nil, ContextError(result.err)
+		return nil, nil, nil, ContextError(result.err)
+	}
+
+	if meekConfig != nil {
+		meekStats = &MeekStats{
+			DialAddress:         meekConfig.DialAddress,
+			ResolvedIPAddress:   resolvedIPAddress.Load().(string),
+			SNIServerName:       meekConfig.SNIServerName,
+			HostHeader:          meekConfig.HostHeader,
+			TransformedHostName: meekConfig.TransformedHostName,
+		}
+
+		NoticeConnectedMeekStats(serverEntry.IpAddress, meekStats)
 	}
 
-	return conn, result.sshClient, nil
+	return conn, result.sshClient, meekStats, nil
 }
 
 // operateTunnel monitors the health of the tunnel and performs
@@ -744,10 +869,11 @@ func (tunnel *Tunnel) operateTunnel(tunnelOwner TunnelOwner) {
 	// everything must be wrapped up quickly. Also, we still have a working
 	// tunnel. So we first attempt a tunneled status request (with a short
 	// timeout) and then attempt, synchronously -- otherwise the Contoller's
-	// untunneledPendingConns.CloseAll() will immediately interrupt untunneled
-	// requests -- untunneled requests (also with short timeouts).
-	// Note that this depends on the order of untunneledPendingConns.CloseAll()
-	// coming after tunnel.Close(): see note in Controller.Run().
+	// runWaitGroup.Wait() will return while a request is still in progress
+	// -- untunneled requests (also with short timeouts). Note that in this
+	// case the untunneled request will opt out of untunneledPendingConns so
+	// that it's not inadvertently canceled by the Controller shutdown
+	// sequence (see doUntunneledStatusRequest).
 	//
 	// If the tunnel has failed, the Controller may continue working. We want
 	// to re-establish as soon as possible (so don't want to block on status

psiphon/upgradeDownload.go

@@ -26,32 +26,120 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"strconv"
 )
 
-// DownloadUpgrade performs a tunneled, resumable download of client upgrade files.
+// DownloadUpgrade performs a resumable download of client upgrade files.
+//
 // While downloading/resuming, a temporary file is used. Once the download is complete,
 // a notice is issued and the upgrade is available at the destination specified in
 // config.UpgradeDownloadFilename.
-// NOTE: this code does not check that any existing file at config.UpgradeDownloadFilename
-// is actually the version specified in clientUpgradeVersion.
-func DownloadUpgrade(config *Config, clientUpgradeVersion string, tunnel *Tunnel) error {
+//
+// The upgrade download may be either tunneled or untunneled. As the untunneled case may
+// happen with no handshake request response, the downloader cannot rely on having the
+// upgrade_client_version output from handshake and instead this logic performs a
+// comparison between the config.ClientVersion and the client version recorded in the
+// remote entity's UpgradeDownloadClientVersionHeader. A HEAD request is made to check the
+// version before proceeding with a full download.
+//
+// NOTE: This code does not check that any existing file at config.UpgradeDownloadFilename
+// is actually the version specified in handshakeVersion.
+//
+// TODO: This logic requires the outer client to *omit* config.UpgradeDownloadFilename
+// when there's already a downloaded upgrade pending. Because the outer client currently
+// handles the authenticated package phase, and because the outer client deletes the
+// intermediate files (including config.UpgradeDownloadFilename), if the outer client
+// does not omit config.UpgradeDownloadFilename then the new version will be downloaded
+// repeatedly. Implement a new scheme where tunnel core does the authenticated package phase
+// and tracks the the output by version number so that (a) tunnel core knows when it's not
+// necessary to re-download; (b) newer upgrades will be downloaded even when an older
+// upgrade is still pending install by the outer client.
+func DownloadUpgrade(
+	config *Config,
+	handshakeVersion string,
+	tunnel *Tunnel,
+	untunneledDialConfig *DialConfig) error {
 
 	// Check if complete file already downloaded
+
 	if _, err := os.Stat(config.UpgradeDownloadFilename); err == nil {
 		NoticeClientUpgradeDownloaded(config.UpgradeDownloadFilename)
 		return nil
 	}
 
-	httpClient, err := MakeTunneledHttpClient(config, tunnel, DOWNLOAD_UPGRADE_TIMEOUT)
-	if err != nil {
-		return ContextError(err)
+	requestUrl := config.UpgradeDownloadUrl
+	var httpClient *http.Client
+	var err error
+
+	// Select tunneled or untunneled configuration
+
+	if tunnel != nil {
+		httpClient, err = MakeTunneledHttpClient(config, tunnel, DOWNLOAD_UPGRADE_TIMEOUT)
+		if err != nil {
+			return ContextError(err)
+		}
+	} else {
+		httpClient, requestUrl, err = MakeUntunneledHttpsClient(
+			untunneledDialConfig, nil, requestUrl, DOWNLOAD_UPGRADE_TIMEOUT)
+		if err != nil {
+			return ContextError(err)
+		}
 	}
 
+	// If no handshake version is supplied, make an initial HEAD request
+	// to get the current version from the version header.
+
+	availableClientVersion := handshakeVersion
+	if availableClientVersion == "" {
+		request, err := http.NewRequest("HEAD", requestUrl, nil)
+		if err != nil {
+			return ContextError(err)
+		}
+		response, err := httpClient.Do(request)
+		if err == nil && response.StatusCode != http.StatusOK {
+			response.Body.Close()
+			err = fmt.Errorf("unexpected response status code: %d", response.StatusCode)
+		}
+		if err != nil {
+			return ContextError(err)
+		}
+		defer response.Body.Close()
+
+		currentClientVersion, err := strconv.Atoi(config.ClientVersion)
+		if err != nil {
+			return ContextError(err)
+		}
+
+		// Note: if the header is missing, Header.Get returns "" and then
+		// strconv.Atoi returns a parse error.
+		availableClientVersion := response.Header.Get(config.UpgradeDownloadClientVersionHeader)
+		checkAvailableClientVersion, err := strconv.Atoi(availableClientVersion)
+		if err != nil {
+			// If the header is missing or malformed, we can't determine the available
+			// version number. This is unexpected; but if it happens, it's likely due
+			// to a server-side configuration issue. In this one case, we don't
+			// return an error so that we don't go into a rapid retry loop making
+			// ineffective HEAD requests (the client may still signal an upgrade
+			// download later in the session).
+			NoticeAlert(
+				"failed to download upgrade: invalid %s header value %s: %s",
+				config.UpgradeDownloadClientVersionHeader, availableClientVersion, err)
+			return nil
+		}
+
+		if currentClientVersion >= checkAvailableClientVersion {
+			NoticeClientIsLatestVersion(availableClientVersion)
+			return nil
+		}
+	}
+
+	// Proceed with full download
+
 	partialFilename := fmt.Sprintf(
-		"%s.%s.part", config.UpgradeDownloadFilename, clientUpgradeVersion)
+		"%s.%s.part", config.UpgradeDownloadFilename, availableClientVersion)
 
 	partialETagFilename := fmt.Sprintf(
-		"%s.%s.part.etag", config.UpgradeDownloadFilename, clientUpgradeVersion)
+		"%s.%s.part.etag", config.UpgradeDownloadFilename, availableClientVersion)
 
 	file, err := os.OpenFile(partialFilename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
 	if err != nil {
@@ -85,7 +173,7 @@ func DownloadUpgrade(config *Config, clientUpgradeVersion string, tunnel *Tunnel
 
 	}
 
-	request, err := http.NewRequest("GET", config.UpgradeDownloadUrl, nil)
+	request, err := http.NewRequest("GET", requestUrl, nil)
 	if err != nil {
 		return ContextError(err)
 	}
@@ -131,12 +219,13 @@ func DownloadUpgrade(config *Config, clientUpgradeVersion string, tunnel *Tunnel
 	// A partial download occurs when this copy is interrupted. The io.Copy
 	// will fail, leaving a partial download in place (.part and .part.etag).
 	n, err := io.Copy(NewSyncFileWriter(file), response.Body)
+
+	NoticeClientUpgradeDownloadedBytes(n)
+
 	if err != nil {
 		return ContextError(err)
 	}
 
-	NoticeInfo("client upgrade downloaded bytes: %d", n)
-
 	// Ensure the file is flushed to disk. The deferred close
 	// will be a noop when this succeeds.
 	err = file.Close()

psiphon/upstreamproxy/proxy_socks4.go

@@ -107,6 +107,7 @@ func (s *socks4Proxy) Dial(network, addr string) (net.Conn, error) {
 		// try to put an invalid IP into DSTIP field and
 		// append domain name terminated by '\x00' at the end of request
 		ip = net.IPv4(0, 0, 0, 1)
+		domainDest = hostStr
 	}
 	ip4 := ip.To4()
 	if ip4 == nil {

psiphon/utils.go

@@ -46,6 +46,15 @@ func Contains(list []string, target string) bool {
 	return false
 }
 
+// FlipCoin is a helper function that randomly
+// returns true or false. If the underlying random
+// number generator fails, FlipCoin still returns
+// a result.
+func FlipCoin() bool {
+	randomInt, _ := MakeSecureRandomInt(2)
+	return randomInt == 1
+}
+
 // MakeSecureRandomInt is a helper function that wraps
 // MakeSecureRandomInt64.
 func MakeSecureRandomInt(max int) (int, error) {
@@ -212,3 +221,21 @@ func (writer *SyncFileWriter) Write(p []byte) (n int, err error) {
 	}
 	return
 }
+
+// GetCurrentTimestamp returns the current time in UTC as
+// an RFC 3339 formatted string.
+func GetCurrentTimestamp() string {
+	return time.Now().UTC().Format(time.RFC3339)
+}
+
+// TruncateTimestampToHour truncates an RFC 3339 formatted string
+// to hour granularity. If the input is not a valid format, the
+// result is "".
+func TruncateTimestampToHour(timestamp string) string {
+	t, err := time.Parse(time.RFC3339, timestamp)
+	if err != nil {
+		NoticeAlert("failed to truncate timestamp: %s", err)
+		return ""
+	}
+	return t.Truncate(1 * time.Hour).Format(time.RFC3339)
+}