Merge branch 'master' of github.com:adam-p/psiphon-tunnel-core

.gitignore

@@ -3,6 +3,11 @@ psiphon_config
 psiphon.config
 controller_test.config
 psiphon.db*
+psiphon.boltdb
+
+# Exclude compiled tunnel core binaries
+ConsoleClient/ConsoleClient
+ConsoleClient/bin
 
 # Compiled Object files, Static and Dynamic libs (Shared Objects)
 *.o

AndroidLibrary/README.md

@@ -28,18 +28,7 @@ Using
 
 1. Build `psi.aar` from source or use the [binary release](https://github.com/Psiphon-Labs/psiphon-tunnel-core/releases)
 1. Add `psi.aar` to your Android Studio project as described in the [gomobile documentation](https://godoc.org/golang.org/x/mobile/cmd/gomobile)
-1. Example usage in [Psibot sample app](../SampleApps/Psibot/README.md)
-
-See sample API usage in [Psibot's PsiphonVpn.java](../SampleApps/Psibot/app/src/main/java/ca/psiphon/PsiphonVpn.java). Uses `gobind` conventions for data passing.
-
-1. Embed a [config file](../README.md#setup)
-1. Call `Go.init(getApplicationContext());` in `Application.onCreate()`
-1. Extend `Psi.Listener.Stub` to receive messages in `Message(String line)`
-1. Call `Psi.Start(configFile, Psi.Listener)` to start Psiphon. Catch `Exception` to receive errors.
-1. Call `Psi.Stop()` to stop Psiphon.
-1. Sample shows how to monitor messages and detect which proxy ports to use and when the tunnel is active.
-
-NOTE: may add more explicit interface for state change events.
+1. Example usage in [TunneledWebView sample app](../SampleApps/TunneledWebView/README.md)
 
 Limitations
 --------------------------------------------------------------------------------

AndroidLibrary/psi/psi.go

@@ -35,7 +35,8 @@ type PsiphonProvider interface {
 	Notice(noticeJSON string)
 	HasNetworkConnectivity() int
 	BindToDevice(fileDescriptor int) error
-	GetDnsServer() string
+	GetPrimaryDnsServer() string
+	GetSecondaryDnsServer() string
 }
 
 var controller *psiphon.Controller

ConsoleClient/.gitignore

@@ -0,0 +1 @@
+bin

ConsoleClient/Dockerfile

@@ -2,36 +2,48 @@
 #
 # See README.md for usage instructions.
 
-FROM ubuntu:12.04
+FROM ubuntu:15.04
 
-ENV GOVERSION=go1.4.1
+ENV GOVERSION=go1.5.3
 
 # Install system-level dependencies.
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-  apt-get -y install build-essential python-software-properties bzip2 unzip curl \
-    git subversion mercurial bzr \
-    upx gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 gcc-multilib
+RUN apt-get update && apt-get -y install build-essential curl git mercurial upx gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 mingw-w64 gcc-multilib pkg-config
 
 # Install Go.
-ENV GOROOT=/go \
-  GOPATH=/
-ENV PATH=$PATH:$GOROOT/bin
-RUN echo "INSTALLING GO" && \
-  curl -L https://github.com/golang/go/archive/$GOVERSION.zip -o /tmp/go.zip && \
-  unzip /tmp/go.zip && \
-  rm /tmp/go.zip && \
-  mv /go-$GOVERSION $GOROOT && \
-  echo $GOVERSION > $GOROOT/VERSION && \
-  cd $GOROOT/src && \
-  ./all.bash
+ENV GOROOT=/usr/local/go GOPATH=/go
+ENV PATH=$PATH:$GOROOT/bin:$GOPATH/bin
+
+RUN curl -L https://storage.googleapis.com/golang/$GOVERSION.linux-amd64.tar.gz -o /tmp/go.tar.gz && \
+  tar -C /usr/local -xzf /tmp/go.tar.gz && \
+  rm /tmp/go.tar.gz && \
+  echo $GOVERSION > $GOROOT/VERSION
 
 ENV CGO_ENABLED=1
-RUN go get github.com/mitchellh/gox && \
-  go get github.com/inconshreveable/gonative && \
-  mkdir -p /usr/local/gonative && \
-  cd /usr/local/gonative && \
-  gonative build
-ENV PATH=/usr/local/gonative/go/bin:$PATH
+
+# Get go dependencies
+RUN go get github.com/mitchellh/gox && go get github.com/pwaller/goupx
+
+# Setup paths for static OpenSSL libray
+ENV OPENSSL_VERSION=1.0.1p
+
+RUN mkdir -p /tmp/openssl/32 && mkdir -p /tmp/openssl/64 && \
+      curl -L https://github.com/Psiphon-Labs/psiphon-tunnel-core/raw/master/openssl/openssl-$OPENSSL_VERSION.tar.gz -o /tmp/openssl.tar.gz && \
+      tar -C /tmp/openssl/32 -xzf /tmp/openssl.tar.gz && \
+      tar -C /tmp/openssl/64 -xzf /tmp/openssl.tar.gz
+
+ENV PKG_CONFIG_PATH_32=/tmp/openssl/32/openssl-$OPENSSL_VERSION
+RUN cd $PKG_CONFIG_PATH_32 && \
+      ./Configure --cross-compile-prefix=i686-w64-mingw32- mingw \
+      no-shared no-ssl2 no-ssl3 no-comp no-hw no-md2 no-md4 no-rc2 no-rc5 no-krb5 no-ripemd160 no-idea no-gost no-camellia no-seed no-3des no-heartbeats && \
+      make depend && \
+      make
+
+ENV PKG_CONFIG_PATH_64=/tmp/openssl/64/openssl-$OPENSSL_VERSION
+RUN cd $PKG_CONFIG_PATH_64 && \
+      ./Configure --cross-compile-prefix=x86_64-w64-mingw32- mingw64 \
+      no-shared no-ssl2 no-ssl3 no-comp no-hw no-md2 no-md4 no-rc2 no-rc5 no-krb5 no-ripemd160 no-idea no-gost no-camellia no-seed no-3des no-heartbeats && \
+      make depend && \
+      make
 
 WORKDIR $GOPATH/src

ConsoleClient/README.md

@@ -1,6 +1,47 @@
-Psiphon Console Client README
-================================================================================
+##Psiphon Console Client README
 
-### Setup
+###Building with Docker
 
-See: https://github.com/Psiphon-Labs/psiphon-tunnel-core#setup
+Note that you may need to use `sudo docker` below, depending on your OS.
+
+#####Create the build image:
+  1. Run the command: `docker build --no-cache=true -t psiclient .` (this may take some time to complete)
+  2. Once completed, verify that you see an image named `psiclient` when running: `docker images`
+
+#####Run the build:
+  *Ensure that the command below is run from within the `ConsoleClient` directory*
+
+  ```bash
+  cd .. && \
+    docker run \
+    --rm \
+    -v $(pwd):/go/src/github.com/Psiphon-Labs/psiphon-tunnel-core \
+    psiclient \
+    /bin/bash -c 'cd /go/src/github.com/Psiphon-Labs/psiphon-tunnel-core/ConsoleClient && ./make.bash all' \
+  ; cd -
+  ```
+This command can also be modified by:
+ - replacing `all` with `windows`, `linux`, or `osx` as the first parameter to `make.bash` (as in `...&& ./make.bash windows`) to only build binaries for the operating system of choice
+   - if `windows` or `linux` is specified as the first parameter, the second parameter can be passed as either `32` or `64` (as in `...&& ./make.bash windows 32`)to limit the builds to just one or the other (no second parameter means both will build)
+
+When that command completes, the compiled binaries will be located in the `bin` directory (`./bin`, and everything under it will likely be owned by root, so be sure to `chown` to an appropriate user) under the current directory. The structure will be:
+  ```
+  bin
+  ├── darwin
+  │   └── psiphon-tunnel-core-x86_64
+  ├── linux
+  │   └── psiphon-tunnel-core-i686
+  │   └── psiphon-tunnel-core-x86_64
+  └── windows
+      └── psiphon-tunnel-core-i686.exe
+      └── psiphon-tunnel-core-x86_64.exe
+
+  ```
+
+### Building without Docker
+
+See the [main README build section](../README.md#build)
+
+### Creating a configuration file
+
+See the [main README configuration section](../README.md#configure)

ConsoleClient/make.bash

@@ -1,49 +1,129 @@
 #!/usr/bin/env bash
 
 set -e
-set -exv # verbose output for testing
 
 if [ ! -f make.bash ]; then
-  echo 'make.bash must be run from $GOPATH/src/github.com/Psiphon-Labs/psiphon-tunnel-core/ConsoleClient'
+  echo "make.bash must be run from $GOPATH/src/github.com/Psiphon-Labs/psiphon-tunnel-core/ConsoleClient"
   exit 1
 fi
 
-CGO_ENABLED=1
-
-# Make sure we have our dependencies
-echo -e "go-getting dependencies...\n"
-go get -d -v ./...
-
 EXE_BASENAME="psiphon-tunnel-core"
 BUILDINFOFILE="${EXE_BASENAME}_buildinfo.txt"
 BUILDDATE=$(date --iso-8601=seconds)
 BUILDREPO=$(git config --get remote.origin.url)
-BUILDREV=$(git rev-parse HEAD)
+BUILDREV=$(git rev-parse --short HEAD)
+
 LDFLAGS="\
 -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildDate $BUILDDATE \
 -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildRepo $BUILDREPO \
 -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon.buildRev $BUILDREV \
 "
 echo -e "${BUILDDATE}\n${BUILDREPO}\n${BUILDREV}\n" > $BUILDINFOFILE
-echo -e "LDFLAGS=$LDFLAGS\n"
-
-echo -e "\nBuilding windows-386..."
-CC=/usr/bin/i686-w64-mingw32-gcc \
-  gox -verbose -ldflags "$LDFLAGS" -osarch windows/386 -output windows_386_${EXE_BASENAME}
-# We are finding that UPXing the full Windows Psiphon client produces better results
-# if psiphon-tunnel-core.exe is not already UPX'd.
-#upx --best windows_386_${EXE_BASENAME}.exe
-
-echo -e "\nBuilding windows-amd64..."
-CC=/usr/bin/x86_64-w64-mingw32-gcc \
-  gox -verbose -ldflags "$LDFLAGS" -osarch windows/amd64 -output windows_amd64_${EXE_BASENAME}
-upx --best windows_amd64_${EXE_BASENAME}.exe
-
-echo -e "\nBuilding linux-amd64..."
-gox -verbose -ldflags "$LDFLAGS" -osarch linux/amd64 -output linux_amd64_${EXE_BASENAME}
-upx --best linux_amd64_${EXE_BASENAME}
-
-echo -e "\nBuilding linux-386..."
-CFLAGS=-m32 \
-  gox -verbose -ldflags "$LDFLAGS" -osarch linux/386 -output linux_386_${EXE_BASENAME}
-upx --best linux_386_${EXE_BASENAME}
+
+echo "Variables for ldflags:"
+echo " Build date: ${BUILDDATE}"
+echo " Build repo: ${BUILDREPO}"
+echo " Build revision: ${BUILDREV}"
+echo ""
+
+if [ ! -d bin ]; then
+  mkdir bin
+fi
+
+build_for_windows () {
+  echo "...Getting project dependencies (via go get) for Windows. Parameter is: '$1'"
+  GOOS=windows go get -d -v ./...
+
+  if [ -z $1 ] || [ "$1" == "32" ]; then
+    unset PKG_CONFIG_PATH
+    export PKG_CONFIG_PATH=$PKG_CONFIG_PATH_32
+
+    echo "...Building windows-i686"
+    echo "....PKG_CONFIG_PATH=$PKG_CONFIG_PATH"
+
+    CGO_CFLAGS="-I $PKG_CONFIG_PATH/include/" \
+    CGO_LDFLAGS="-L $PKG_CONFIG_PATH -L /usr/i686-w64-mingw32/lib/ -lssl -lcrypto -lwsock32 -lcrypt32 -lgdi32" \
+    CC=/usr/bin/i686-w64-mingw32-gcc \
+    gox -verbose -ldflags "$LDFLAGS" -osarch windows/386 -output bin/windows/${EXE_BASENAME}-i686
+
+    ## We are finding that UPXing the full Windows Psiphon client produces better results if psiphon-tunnel-core.exe is not already UPX'd.
+    echo "....No UPX for this build"
+  fi
+
+  if [ -z $1 ] || [ "$1" == "64" ]; then
+    unset PKG_CONFIG_PATH
+    export PKG_CONFIG_PATH=$PKG_CONFIG_PATH_64
+
+    echo "...Building windows-x86_64"
+    echo "....PKG_CONFIG_PATH=$PKG_CONFIG_PATH"
+
+    CGO_CFLAGS="-I $PKG_CONFIG_PATH/include/" \
+    CGO_LDFLAGS="-L $PKG_CONFIG_PATH -L /usr/x86_64-w64-mingw32/lib/ -lssl -lcrypto -lwsock32 -lcrypt32 -lgdi32" \
+    CC=/usr/bin/x86_64-w64-mingw32-gcc \
+    gox -verbose -ldflags "$LDFLAGS" -osarch windows/amd64 -output bin/windows/${EXE_BASENAME}-x86_64
+
+    # We are finding that UPXing the full Windows Psiphon client produces better results if psiphon-tunnel-core.exe is not already UPX'd.
+    echo "....No UPX for this build"
+  fi
+}
+
+build_for_linux () {
+  echo "Getting project dependencies (via go get) for Linux. Parameter is: '$1'"
+  GOOS=linux go get -d -v ./...
+
+  if [ -z $1 ] || [ "$1" == "32" ]; then
+    echo "...Building linux-i686"
+    CFLAGS=-m32 gox -verbose -ldflags "$LDFLAGS" -osarch linux/386 -output bin/linux/${EXE_BASENAME}-i686
+    echo "....UPX packaging output"
+    goupx --best bin/linux/${EXE_BASENAME}-i686
+  fi
+
+  if [ -z $1 ] || [ "$1" == "64" ]; then
+    echo "...Building linux-x86_64"
+    gox -verbose -ldflags "$LDFLAGS" -osarch linux/amd64 -output bin/linux/${EXE_BASENAME}-x86_64
+    echo "....UPX packaging output"
+    goupx --best bin/linux/${EXE_BASENAME}-x86_64
+  fi
+}
+
+build_for_osx () {
+  echo "Getting project dependencies (via go get) for OSX"
+  GOOS=darwin go get -d -v ./...
+
+  echo "Building darwin-x86_64..."
+  echo "..Disabling CGO for this build"
+  CGO_ENABLED=0 gox -verbose -ldflags "$LDFLAGS" -osarch darwin/amd64 -output bin/darwin/${EXE_BASENAME}-x86_64
+  # Darwin binaries don't seem to be UPXable when built this way
+  echo "..No UPX for this build"
+}
+
+TARGET=$1
+case $TARGET in
+  windows)
+    echo "..Building for Windows"
+    build_for_windows $2
+    ;;
+  linux)
+    echo "..Building for Linux"
+    build_for_linux $2
+    ;;
+  osx)
+    echo "..Building for OSX"
+    build_for_osx
+    ;;
+  all)
+    echo "..Building all"
+    build_for_windows $2
+    build_for_linux $2
+    build_for_osx
+    ;;
+  *)
+    echo "..No selection made, building all"
+    build_for_windows $2
+    build_for_linux $2
+    build_for_osx
+    ;;
+
+esac
+
+echo "Done"

README.md

@@ -18,6 +18,8 @@ This project is currently at the proof-of-concept stage. Current production Psip
 Setup
 --------------------------------------------------------------------------------
 
+#### Build
+
 * Go 1.5 (or higher) is required.
 * This project builds and runs on recent versions of Windows, Linux, and Mac OS X.
 * Note that the `psiphon` package is imported using the absolute path `github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon`; without further local configuration, `go` will use this version of the code and not the local copy in the repository.
@@ -35,7 +37,10 @@ Setup
     "
     ```
 
-* Run `./ConsoleClient --config psiphon.config` where the config file looks like this:
+#### Configure
+
+ * Configuration files are standard text files containing a valid JSON object. Example:
+
 
   <!--BEGIN-SAMPLE-CONFIG-->
   ```
@@ -48,10 +53,23 @@ Setup
   ```
   <!--END-SAMPLE-CONFIG-->
 
-* Config file parameters are [documented here](https://godoc.org/github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon#Config).
-* Replace each `<placeholder>` with a value from your Psiphon network. The Psiphon server-side stack is open source and can be found in our  [Psiphon 3 repository](https://bitbucket.org/psiphon/psiphon-circumvention-system). If you would like to use the Psiphon Inc. network, contact <[email protected]>.
+*Note: The lines `<!--BEGIN-SAMPLE-CONFIG-->` and `<--END-SAMPLE-CONFIG-->` (visible in the raw Markdown) are used by the [config test](psiphon/config_test.go). Do not remove them.*
+
+* All config file parameters are [documented here](https://godoc.org/github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon#Config).
+* Replace each `<placeholder>` with a value from your Psiphon server. The Psiphon server-side stack is open source and can be found in our [Psiphon 3 repository](https://bitbucket.org/psiphon/psiphon-circumvention-system).
+
+
+#### Run
+
+* Run `./ConsoleClient --config psiphon.config` where `psiphon.config` is created as described in the [Configure](#configure) section above
+
+
+Other Platforms
+--------------------------------------------------------------------------------
+
 * The project builds and runs on Android. See the [AndroidLibrary README](AndroidLibrary/README.md) for more information about building the Go component, and the [AndroidApp README](AndroidApp/README.md) for a sample Android app that uses it.
 
+
 Licensing
 --------------------------------------------------------------------------------
 

SampleApps/Psibot/.idea/gradle.xml

@@ -5,7 +5,7 @@
       <GradleProjectSettings>
         <option name="distributionType" value="LOCAL" />
         <option name="externalProjectPath" value="$PROJECT_DIR$" />
-        <option name="gradleHome" value="$APPLICATION_HOME_DIR$/gradle/gradle-2.4" />
+        <option name="gradleHome" value="$APPLICATION_HOME_DIR$/gradle/gradle-2.8" />
         <option name="gradleJvm" value="1.8" />
         <option name="modules">
           <set>

SampleApps/Psibot/README.md

@@ -4,7 +4,7 @@ Psibot README
 Overview
 --------------------------------------------------------------------------------
 
-Psibot is a sample app that demonstrates embedding the Psiphon Go client in
+Psibot is a sample app that demonstrates embedding the Psiphon Library in
 an Android app. Psibot uses the Android VpnService API to route all device
 traffic through tun2socks and in turn through Psiphon.
 
@@ -21,8 +21,6 @@ Native libraries
 Psiphon Android Library and config file
 --------------------------------------------------------------------------------
 
-Uses the [Psiphon Android Library](../AndroidLibrary/README.md).
+Uses the [Psiphon Android Library](../../AndroidLibrary/README.md).
 
 * `app/src/main/res/raw/psiphon_config_stub` and its placeholder values must be replaced with `app\src\main\res\raw\psiphon_config` and valid configuration values.
-
-* Install the Android Library shared object binary at `app/src/main/jniLibs/armeabi-v7a/libgojni.so`.

SampleApps/Psibot/app/app.iml

@@ -65,30 +65,23 @@
       <sourceFolder url="file://$MODULE_DIR$/src/androidTest/jni" isTestSource="true" />
       <sourceFolder url="file://$MODULE_DIR$/src/androidTest/rs" isTestSource="true" />
       <excludeFolder url="file://$MODULE_DIR$/build/intermediates/assets" />
-      <excludeFolder url="file://$MODULE_DIR$/build/intermediates/bundles" />
       <excludeFolder url="file://$MODULE_DIR$/build/intermediates/classes" />
-      <excludeFolder url="file://$MODULE_DIR$/build/intermediates/coverage-instrumented-classes" />
       <excludeFolder url="file://$MODULE_DIR$/build/intermediates/dependency-cache" />
       <excludeFolder url="file://$MODULE_DIR$/build/intermediates/dex" />
-      <excludeFolder url="file://$MODULE_DIR$/build/intermediates/dex-cache" />
-      <excludeFolder url="file://$MODULE_DIR$/build/intermediates/exploded-aar/go.psi/psi/0.0.8/jars" />
+      <excludeFolder url="file://$MODULE_DIR$/build/intermediates/exploded-aar/go.psi/psi/0.0.10/jars" />
       <excludeFolder url="file://$MODULE_DIR$/build/intermediates/incremental" />
-      <excludeFolder url="file://$MODULE_DIR$/build/intermediates/jacoco" />
-      <excludeFolder url="file://$MODULE_DIR$/build/intermediates/javaResources" />
-      <excludeFolder url="file://$MODULE_DIR$/build/intermediates/libs" />
-      <excludeFolder url="file://$MODULE_DIR$/build/intermediates/lint" />
       <excludeFolder url="file://$MODULE_DIR$/build/intermediates/manifests" />
-      <excludeFolder url="file://$MODULE_DIR$/build/intermediates/ndk" />
       <excludeFolder url="file://$MODULE_DIR$/build/intermediates/pre-dexed" />
-      <excludeFolder url="file://$MODULE_DIR$/build/intermediates/proguard" />
       <excludeFolder url="file://$MODULE_DIR$/build/intermediates/res" />
+      <excludeFolder url="file://$MODULE_DIR$/build/intermediates/resources" />
       <excludeFolder url="file://$MODULE_DIR$/build/intermediates/rs" />
       <excludeFolder url="file://$MODULE_DIR$/build/intermediates/symbols" />
+      <excludeFolder url="file://$MODULE_DIR$/build/intermediates/tmp" />
       <excludeFolder url="file://$MODULE_DIR$/build/outputs" />
       <excludeFolder url="file://$MODULE_DIR$/build/tmp" />
     </content>
     <orderEntry type="jdk" jdkName="Android API 21 Platform" jdkType="Android SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
-    <orderEntry type="library" exported="" name="psi-0.0.8" level="project" />
+    <orderEntry type="library" exported="" name="psi-0.0.10" level="project" />
   </component>
 </module>

SampleApps/Psibot/app/build.gradle

@@ -27,5 +27,5 @@ repositories {
 
 dependencies {
     compile fileTree(dir: 'libs', include: ['*.jar'])
-    compile 'go.psi:psi:0.0.8@aar'
+    compile 'go.psi:psi:0.0.10@aar'
 }

SampleApps/Psibot/app/src/main/java/ca/psiphon/PsiphonTunnel.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Psiphon Inc.
+ * Copyright (c) 2016, Psiphon Inc.
  * All rights reserved.
  *
  * This program is free software: you can redistribute it and/or modify
@@ -35,6 +35,7 @@ import org.json.JSONException;
 import org.json.JSONObject;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
@@ -57,6 +58,8 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 
 import go.psi.Psi;
 
@@ -65,8 +68,8 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     public interface HostService {
         public String getAppName();
         public Context getContext();
-        public VpnService getVpnService();
-        public VpnService.Builder newVpnServiceBuilder();
+        public Object getVpnService(); // Object must be a VpnService (Android < 4 cannot reference this class name)
+        public Object newVpnServiceBuilder(); // Object must be a VpnService.Builder (Android < 4 cannot reference this class name)
         public String getPsiphonConfig();
         public void onDiagnosticMessage(String message);
         public void onAvailableEgressRegions(List<String> regions);
@@ -88,9 +91,9 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
 
     private final HostService mHostService;
     private PrivateAddress mPrivateAddress;
-    private ParcelFileDescriptor mTunFd;
-    private int mLocalSocksProxyPort;
-    private boolean mRoutingThroughTunnel;
+    private AtomicReference<ParcelFileDescriptor> mTunFd;
+    private AtomicInteger mLocalSocksProxyPort;
+    private AtomicBoolean mRoutingThroughTunnel;
     private Thread mTun2SocksThread;
     private AtomicBoolean mIsWaitingForNetworkConnectivity;
 
@@ -98,7 +101,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     // go.psi.Psi and tun2socks implementations each contain global state.
     private static PsiphonTunnel mPsiphonTunnel;
 
-    public static synchronized PsiphonTunnel newPsiphonVpn(HostService hostService) {
+    public static synchronized PsiphonTunnel newPsiphonTunnel(HostService hostService) {
         if (mPsiphonTunnel != null) {
             mPsiphonTunnel.stop();
         }
@@ -110,8 +113,9 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
 
     private PsiphonTunnel(HostService hostService) {
         mHostService = hostService;
-        mLocalSocksProxyPort = 0;
-        mRoutingThroughTunnel = false;
+        mTunFd = new AtomicReference<ParcelFileDescriptor>();
+        mLocalSocksProxyPort = new AtomicInteger(0);
+        mRoutingThroughTunnel = new AtomicBoolean(false);
         mIsWaitingForNetworkConnectivity = new AtomicBoolean(false);
     }
 
@@ -148,7 +152,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     public synchronized void stop() {
         stopVpn();
         stopPsiphon();
-        mLocalSocksProxyPort = 0;
+        mLocalSocksProxyPort.set(0);
     }
 
     //----------------------------------------------------------------------------------------------
@@ -158,7 +162,13 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     private final static String VPN_INTERFACE_NETMASK = "255.255.255.0";
     private final static int VPN_INTERFACE_MTU = 1500;
     private final static int UDPGW_SERVER_PORT = 7300;
-    private final static String DEFAULT_DNS_SERVER = "8.8.4.4";
+    private final static String DEFAULT_PRIMARY_DNS_SERVER = "8.8.4.4";
+    private final static String DEFAULT_SECONDARY_DNS_SERVER = "8.8.8.8";
+
+    // Note: Atomic variables used for getting/setting local proxy port, routing flag, and
+    // tun fd, as these functions may be called via PsiphonProvider callbacks. Do not use
+    // synchronized functions as stop() is synchronized and a deadlock is possible as callbacks
+    // can be called while stop holds the lock.
 
     @TargetApi(Build.VERSION_CODES.ICE_CREAM_SANDWICH)
     private boolean startVpn() throws Exception {
@@ -172,19 +182,22 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             // Workaround for https://code.google.com/p/android/issues/detail?id=61096
             Locale.setDefault(new Locale("en"));
 
-            mTunFd = mHostService.newVpnServiceBuilder()
-                    .setSession(mHostService.getAppName())
-                    .setMtu(VPN_INTERFACE_MTU)
-                    .addAddress(mPrivateAddress.mIpAddress, mPrivateAddress.mPrefixLength)
-                    .addRoute("0.0.0.0", 0)
-                    .addRoute(mPrivateAddress.mSubnet, mPrivateAddress.mPrefixLength)
-                    .addDnsServer(mPrivateAddress.mRouter)
-                    .establish();
-            if (mTunFd == null) {
+            ParcelFileDescriptor tunFd =
+                    ((VpnService.Builder) mHostService.newVpnServiceBuilder())
+                            .setSession(mHostService.getAppName())
+                            .setMtu(VPN_INTERFACE_MTU)
+                            .addAddress(mPrivateAddress.mIpAddress, mPrivateAddress.mPrefixLength)
+                            .addRoute("0.0.0.0", 0)
+                            .addRoute(mPrivateAddress.mSubnet, mPrivateAddress.mPrefixLength)
+                            .addDnsServer(mPrivateAddress.mRouter)
+                            .establish();
+            if (tunFd == null) {
                 // As per http://developer.android.com/reference/android/net/VpnService.Builder.html#establish%28%29,
                 // this application is no longer prepared or was revoked.
                 return false;
             }
+            mTunFd.set(tunFd);
+
             mHostService.onDiagnosticMessage("VPN established");
 
         } catch(IllegalArgumentException e) {
@@ -201,19 +214,22 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         return true;
     }
 
-    private synchronized void setLocalSocksProxyPort(int port) {
-        mLocalSocksProxyPort = port;
+    private boolean isVpnMode() {
+        return mTunFd.get() != null;
     }
 
-    private synchronized void routeThroughTunnel() {
-        if (mRoutingThroughTunnel) {
+    private void setLocalSocksProxyPort(int port) {
+        mLocalSocksProxyPort.set(port);
+    }
+
+    private void routeThroughTunnel() {
+        if (!mRoutingThroughTunnel.compareAndSet(false, true)) {
             return;
         }
-        mRoutingThroughTunnel = true;
-        String socksServerAddress = "127.0.0.1:" + Integer.toString(mLocalSocksProxyPort);
+        String socksServerAddress = "127.0.0.1:" + Integer.toString(mLocalSocksProxyPort.get());
         String udpgwServerAddress = "127.0.0.1:" + Integer.toString(UDPGW_SERVER_PORT);
         startTun2Socks(
-                mTunFd,
+                mTunFd.get(),
                 VPN_INTERFACE_MTU,
                 mPrivateAddress.mRouter,
                 VPN_INTERFACE_NETMASK,
@@ -227,17 +243,17 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     }
 
     private void stopVpn() {
-        if (mTunFd != null) {
+        ParcelFileDescriptor tunFd = mTunFd.getAndSet(null);
+        if (tunFd != null) {
             try {
-                mTunFd.close();
+                tunFd.close();
             } catch (IOException e) {
             }
-            mTunFd = null;
         }
         waitStopTun2Socks();
-        mRoutingThroughTunnel = false;
+        mRoutingThroughTunnel.set(false);
     }
-    
+
     //----------------------------------------------------------------------------------------------
     // PsiphonProvider (Core support) interface implementation
     //----------------------------------------------------------------------------------------------
@@ -250,7 +266,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     @TargetApi(Build.VERSION_CODES.ICE_CREAM_SANDWICH)
     @Override
     public void BindToDevice(long fileDescriptor) throws Exception {
-        if (!mHostService.getVpnService().protect((int)fileDescriptor)) {
+        if (!((VpnService)mHostService.getVpnService()).protect((int)fileDescriptor)) {
             throw new Exception("protect socket failed");
         }
     }
@@ -270,17 +286,22 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
     }
 
     @Override
-    public String GetDnsServer() {
+    public String GetPrimaryDnsServer() {
         String dnsResolver = null;
         try {
-            dnsResolver = getFirstActiveNetworkDnsResolver(mHostService.getVpnService());
+            dnsResolver = getFirstActiveNetworkDnsResolver(mHostService.getContext());
         } catch (Exception e) {
             mHostService.onDiagnosticMessage("failed to get active network DNS resolver: " + e.getMessage());
-            dnsResolver = DEFAULT_DNS_SERVER;
+            dnsResolver = DEFAULT_PRIMARY_DNS_SERVER;
         }
         return dnsResolver;
     }
 
+    @Override
+    public String GetSecondaryDnsServer() {
+        return DEFAULT_SECONDARY_DNS_SERVER;
+    }
+
     //----------------------------------------------------------------------------------------------
     // Psiphon Tunnel Core
     //----------------------------------------------------------------------------------------------
@@ -289,12 +310,11 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         stopPsiphon();
         mHostService.onDiagnosticMessage("starting Psiphon library");
         try {
-            boolean isVpnMode = (mTunFd != null);
             Psi.Start(
-                loadPsiphonConfig(mHostService.getContext(), isVpnMode),
-                embeddedServerEntries,
-                this,
-                isVpnMode);
+                    loadPsiphonConfig(mHostService.getContext()),
+                    embeddedServerEntries,
+                    this,
+                    isVpnMode());
         } catch (java.lang.Exception e) {
             throw new Exception("failed to start Psiphon library", e);
         }
@@ -307,13 +327,13 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         mHostService.onDiagnosticMessage("Psiphon library stopped");
     }
 
-    private String loadPsiphonConfig(Context context, boolean isVpnMode)
+    private String loadPsiphonConfig(Context context)
             throws IOException, JSONException {
 
         // Load settings from the raw resource JSON config file and
         // update as necessary. Then write JSON to disk for the Go client.
         JSONObject json = new JSONObject(mHostService.getPsiphonConfig());
-        
+
         // On Android, these directories must be set to the app private storage area.
         // The Psiphon library won't be able to use its current working directory
         // and the standard temporary directories do not exist.
@@ -327,11 +347,11 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         json.put("EstablishTunnelTimeoutSeconds", 0);
 
         // This parameter is for stats reporting
-        json.put("TunnelWholeDevice", isVpnMode ? 1 : 0);
+        json.put("TunnelWholeDevice", isVpnMode() ? 1 : 0);
 
         json.put("EmitBytesTransferred", true);
 
-        if (mLocalSocksProxyPort != 0) {
+        if (mLocalSocksProxyPort.get() != 0) {
             // When mLocalSocksProxyPort is set, tun2socks is already configured
             // to use that port value. So we force use of the same port.
             // A side-effect of this is that changing the SOCKS port preference
@@ -341,12 +361,16 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
 
         json.put("UseIndistinguishableTLS", true);
 
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.ICE_CREAM_SANDWICH) {
+            json.put("UseTrustedCACertificatesForStockTLS", true);
+        }
+
         try {
             // Also enable indistinguishable TLS for HTTPS requests that
             // require system CAs.
             json.put(
-                "TrustedCACertificatesFilename",
-                setupTrustedCertificates(mHostService.getContext()));
+                    "TrustedCACertificatesFilename",
+                    setupTrustedCertificates(mHostService.getContext()));
         } catch (Exception e) {
             mHostService.onDiagnosticMessage(e.getMessage());
         }
@@ -359,14 +383,16 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             // All notices are sent on as diagnostic messages
             // except those that may contain private user data.
             boolean diagnostic = true;
-            
+
             JSONObject notice = new JSONObject(noticeJSON);
             String noticeType = notice.getString("noticeType");
-            
+
             if (noticeType.equals("Tunnels")) {
                 int count = notice.getJSONObject("data").getInt("count");
                 if (count > 0) {
-                    routeThroughTunnel();
+                    if (isVpnMode()) {
+                        routeThroughTunnel();
+                    }
                     mHostService.onConnected();
                 } else {
                     mHostService.onConnecting();
@@ -379,7 +405,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
                     regions.add(egressRegions.getString(i));
                 }
                 mHostService.onAvailableEgressRegions(regions);
-                
+
             } else if (noticeType.equals("SocksProxyPortInUse")) {
                 mHostService.onSocksProxyPortInUse(notice.getJSONObject("data").getInt("port"));
 
@@ -399,7 +425,7 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
                 mHostService.onUpstreamProxyError(notice.getJSONObject("data").getString("message"));
 
             } else if (noticeType.equals("ClientUpgradeDownloaded")) {
-                mHostService.onHomepage(notice.getJSONObject("data").getString("filename"));
+                mHostService.onClientUpgradeDownloaded(notice.getJSONObject("data").getString("filename"));
 
             } else if (noticeType.equals("Homepage")) {
                 mHostService.onHomepage(notice.getJSONObject("data").getString("url"));
@@ -459,8 +485,21 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
             try {
                 output = new PrintStream(new FileOutputStream(file));
 
-                KeyStore keyStore = KeyStore.getInstance("AndroidCAStore");
-                keyStore.load(null, null);
+                KeyStore keyStore;
+                if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_CREAM_SANDWICH) {
+                    keyStore = KeyStore.getInstance("AndroidCAStore");
+                    keyStore.load(null, null);
+                } else {
+                    keyStore = KeyStore.getInstance("BKS");
+                    FileInputStream inputStream = new FileInputStream("/etc/security/cacerts.bks");
+                    try {
+                        keyStore.load(inputStream, "changeit".toCharArray());
+                    } finally {
+                        if (inputStream != null) {
+                            inputStream.close();
+                        }
+                    }
+                }
 
                 Enumeration<String> aliases = keyStore.aliases();
                 while (aliases.hasMoreElements()) {

SampleApps/Psibot/app/src/main/java/ca/psiphon/psibot/Service.java

@@ -44,7 +44,7 @@ public class Service extends VpnService
 
     @Override
     public void onCreate() {
-        mPsiphonTunnel = PsiphonTunnel.newPsiphonVpn(this);
+        mPsiphonTunnel = PsiphonTunnel.newPsiphonTunnel(this);
         startForeground(R.string.foregroundServiceNotificationId, makeForegroundNotification());
         try {
             if (!mPsiphonTunnel.startRouting()) {

SampleApps/TunneledWebView/.gitignore

@@ -0,0 +1,9 @@
+*.iml
+*.aar
+.gradle
+/local.properties
+/.idea/workspace.xml
+/.idea/libraries
+.DS_Store
+/build
+/captures

SampleApps/TunneledWebView/.idea/.name

@@ -0,0 +1 @@
+TunneledWebView

SampleApps/TunneledWebView/.idea/compiler.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CompilerConfiguration">
+    <option name="DEFAULT_COMPILER" value="Javac" />
+    <resourceExtensions />
+    <wildcardResourcePatterns>
+      <entry name="!?*.java" />
+      <entry name="!?*.form" />
+      <entry name="!?*.class" />
+      <entry name="!?*.groovy" />
+      <entry name="!?*.scala" />
+      <entry name="!?*.flex" />
+      <entry name="!?*.kt" />
+      <entry name="!?*.clj" />
+    </wildcardResourcePatterns>
+    <annotationProcessing>
+      <profile default="true" name="Default" enabled="false">
+        <processorPath useClasspath="true" />
+      </profile>
+    </annotationProcessing>
+  </component>
+</project>

SampleApps/TunneledWebView/.idea/copyright/profiles_settings.xml

@@ -0,0 +1,3 @@
+<component name="CopyrightManager">
+  <settings default="" />
+</component>

SampleApps/TunneledWebView/.idea/gradle.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="GradleSettings">
+    <option name="linkedExternalProjectsSettings">
+      <GradleProjectSettings>
+        <option name="distributionType" value="LOCAL" />
+        <option name="externalProjectPath" value="$PROJECT_DIR$" />
+        <option name="gradleHome" value="$APPLICATION_HOME_DIR$/gradle/gradle-2.4" />
+        <option name="gradleJvm" value="1.8" />
+        <option name="modules">
+          <set>
+            <option value="$PROJECT_DIR$" />
+            <option value="$PROJECT_DIR$/app" />
+          </set>
+        </option>
+      </GradleProjectSettings>
+    </option>
+  </component>
+</project>

SampleApps/TunneledWebView/.idea/misc.xml

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="EntryPointsManager">
+    <entry_points version="2.0" />
+  </component>
+  <component name="NullableNotNullManager">
+    <option name="myDefaultNullable" value="android.support.annotation.Nullable" />
+    <option name="myDefaultNotNull" value="android.support.annotation.NonNull" />
+    <option name="myNullables">
+      <value>
+        <list size="4">
+          <item index="0" class="java.lang.String" itemvalue="org.jetbrains.annotations.Nullable" />
+          <item index="1" class="java.lang.String" itemvalue="javax.annotation.Nullable" />
+          <item index="2" class="java.lang.String" itemvalue="edu.umd.cs.findbugs.annotations.Nullable" />
+          <item index="3" class="java.lang.String" itemvalue="android.support.annotation.Nullable" />
+        </list>
+      </value>
+    </option>
+    <option name="myNotNulls">
+      <value>
+        <list size="4">
+          <item index="0" class="java.lang.String" itemvalue="org.jetbrains.annotations.NotNull" />
+          <item index="1" class="java.lang.String" itemvalue="javax.annotation.Nonnull" />
+          <item index="2" class="java.lang.String" itemvalue="edu.umd.cs.findbugs.annotations.NonNull" />
+          <item index="3" class="java.lang.String" itemvalue="android.support.annotation.NonNull" />
+        </list>
+      </value>
+    </option>
+  </component>
+  <component name="ProjectLevelVcsManager" settingsEditedManually="false">
+    <OptionsSetting value="true" id="Add" />
+    <OptionsSetting value="true" id="Remove" />
+    <OptionsSetting value="true" id="Checkout" />
+    <OptionsSetting value="true" id="Update" />
+    <OptionsSetting value="true" id="Status" />
+    <OptionsSetting value="true" id="Edit" />
+    <ConfirmationsSetting value="0" id="Add" />
+    <ConfirmationsSetting value="0" id="Remove" />
+  </component>
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_1_7" assert-keyword="true" jdk-15="true" project-jdk-name="1.8" project-jdk-type="JavaSDK">
+    <output url="file://$PROJECT_DIR$/build/classes" />
+  </component>
+  <component name="ProjectType">
+    <option name="id" value="Android" />
+  </component>
+</project>

SampleApps/TunneledWebView/.idea/modules.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/TunneledWebView.iml" filepath="$PROJECT_DIR$/TunneledWebView.iml" />
+      <module fileurl="file://$PROJECT_DIR$/app/app.iml" filepath="$PROJECT_DIR$/app/app.iml" />
+    </modules>
+  </component>
+</project>

SampleApps/TunneledWebView/.idea/runConfigurations.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="RunConfigurationProducerService">
+    <option name="ignoredProducers">
+      <set>
+        <option value="org.jetbrains.plugins.gradle.execution.test.runner.AllInPackageGradleConfigurationProducer" />
+        <option value="org.jetbrains.plugins.gradle.execution.test.runner.TestClassGradleConfigurationProducer" />
+        <option value="org.jetbrains.plugins.gradle.execution.test.runner.TestMethodGradleConfigurationProducer" />
+      </set>
+    </option>
+  </component>
+</project>

SampleApps/TunneledWebView/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="" />
+  </component>
+</project>

SampleApps/TunneledWebView/README.md

@@ -0,0 +1,159 @@
+TunneledWebView README
+================================================================================
+
+Overview
+--------------------------------------------------------------------------------
+
+TunneledWebView is a sample app that demonstrates embedding the Psiphon Library in
+an Android app. TunneledWebView proxies a WebView through the Psiphon tunnel.
+
+Integration
+--------------------------------------------------------------------------------
+
+Uses the [Psiphon Android Library](../../AndroidLibrary/README.md).
+
+Integration is illustrated in the main activity source file in the sample app. Here are the key parts.
+
+```Java
+
+/*
+ * Copyright (c) 2016, Psiphon Inc.
+ * All rights reserved.
+ */
+ 
+package ca.psiphon.tunneledwebview;
+
+// ...
+
+import ca.psiphon.PsiphonTunnel;
+
+//----------------------------------------------------------------------------------------------
+// TunneledWebView
+//
+// This sample app demonstrates tunneling a WebView through the
+// Psiphon Library. This app's main activity shows a log of
+// events and a WebView that is loaded once Psiphon is connected.
+//
+// The flow is as follows:
+//
+// - The Psiphon tunnel is started in onResume(). PsiphonTunnel.start()
+//   is an asynchronous call that returns immediately.
+//
+// - Once Psiphon has selected a local HTTP proxy listening port, the
+//   onListeningHttpProxyPort() callback is called. This app records the
+//   port to use for tunneling traffic.
+//
+// - Once Psiphon has established a tunnel, the onConnected() callback
+//   is called. This app now loads the WebView, after setting its proxy
+//   to point to Psiphon's local HTTP proxy.
+//
+// To adapt this sample into your own app:
+//
+// - Embed a Psiphon config file in app/src/main/res/raw/psiphon_config.
+//
+// - Add the Psiphon Library AAR module as a dependency (see this app's
+//   project settings; to build this sample project, you need to drop
+//   psi-0.0.10.aar into app/libs).
+//
+// - Use app/src/main/java/ca/psiphon/PsiphonTunnel.java, which provides
+//   a higher-level wrapper around the Psiphon Library module. This file
+//   shows how to use PsiphonTunnel and PsiphonTunnel.TunneledApp.
+//
+//----------------------------------------------------------------------------------------------
+
+public class MainActivity extends ActionBarActivity
+        implements PsiphonTunnel.TunneledApp {
+
+// ...
+
+    @Override
+    protected void onCreate(Bundle savedInstanceState) {
+
+        // ...
+
+        mPsiphonTunnel = PsiphonTunnel.newPsiphonTunnel(this);
+    }
+
+    @Override
+    protected void onResume() {
+        super.onResume();
+
+        // NOTE: for demonstration purposes, this sample app
+        // restarts Psiphon in onPause/onResume. Since it may take some
+        // time to connect, it's generally recommended to keep
+        // Psiphon running, so start/stop in onCreate/onDestroy or
+        // even consider running a background Service.
+
+        if (!mPsiphonTunnel.start("")) {
+            logMessage("failed to start Psiphon");
+        }
+    }
+
+    @Override
+    protected void onPause() {
+        super.onPause();
+
+        // NOTE: stop() can block for a few seconds, so it's generally
+        // recommended to run PsiphonTunnel.start()/stop() in a background
+        // thread and signal the thread appropriately.
+
+        mPsiphonTunnel.stop();
+    }
+
+    private void setHttpProxyPort(int port) {
+
+        // NOTE: here we record the Psiphon proxy port for subsequent
+        // use in tunneling app traffic. In this sample app, we will
+        // use WebViewProxySettings.setLocalProxy to tunnel a WebView
+        // through Psiphon. By default, the local proxy port is selected
+        // dynamically, so it's important to record and use the correct
+        // port number.
+
+        mLocalHttpProxyPort.set(port);
+    }
+
+    private void loadWebView() {
+
+        // NOTE: functions called via PsiphonTunnel.TunneledApp may be
+        // called on background threads. It's important to ensure that
+        // these threads are not blocked and that UI functions are not
+        // called directly from these threads. Here we use runOnUiThread
+        // to handle this.
+
+        runOnUiThread(new Runnable() {
+            public void run() {
+                WebViewProxySettings.setLocalProxy(
+                        MainActivity.this, mLocalHttpProxyPort.get());
+                mWebView.loadUrl("https://ipinfo.io/");
+            }
+        });
+    }
+
+    // ...
+
+    //----------------------------------------------------------------------------------------------
+    // PsiphonTunnel.TunneledApp implementation
+    //
+    // NOTE: these are callbacks from the Psiphon Library
+    //----------------------------------------------------------------------------------------------
+
+    // ...
+
+    @Override
+    public void onListeningSocksProxyPort(int port) {
+        logMessage("local SOCKS proxy listening on port: " + Integer.toString(port));
+    }
+
+    // ...
+
+    @Override
+    public void onConnected() {
+        logMessage("connected");
+        loadWebView();
+    }
+
+    // ...
+
+}
+
+```

SampleApps/TunneledWebView/app/.gitignore

@@ -0,0 +1 @@
+/build

SampleApps/TunneledWebView/app/build.gradle

@@ -0,0 +1,33 @@
+apply plugin: 'com.android.application'
+
+android {
+    compileSdkVersion 21
+    buildToolsVersion "21.1.2"
+
+    defaultConfig {
+        applicationId "ca.psiphon.tunneledwebview"
+        minSdkVersion 15
+        targetSdkVersion 21
+        versionCode 1
+        versionName "1.0"
+    }
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+        }
+    }
+}
+
+repositories {
+    flatDir {
+        dirs 'libs'
+    }
+}
+
+dependencies {
+    compile fileTree(dir: 'libs', include: ['*.jar'])
+    testCompile 'junit:junit:4.12'
+    compile 'com.android.support:appcompat-v7:21.0.3'
+    compile 'go.psi:psi:0.0.10@aar'
+}

SampleApps/TunneledWebView/app/proguard-rules.pro

@@ -0,0 +1,17 @@
+# Add project specific ProGuard rules here.
+# By default, the flags in this file are appended to flags specified
+# in /Users/user/Code/AndroidStudioSDK/tools/proguard/proguard-android.txt
+# You can edit the include path and order by changing the proguardFiles
+# directive in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# Add any project specific keep options here:
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}

SampleApps/TunneledWebView/app/src/androidTest/java/ca/psiphon/tunneledwebview/ApplicationTest.java

@@ -0,0 +1,13 @@
+package ca.psiphon.tunneledwebview;
+
+import android.app.Application;
+import android.test.ApplicationTestCase;
+
+/**
+ * <a href="http://d.android.com/tools/testing/testing_android.html">Testing Fundamentals</a>
+ */
+public class ApplicationTest extends ApplicationTestCase<Application> {
+    public ApplicationTest() {
+        super(Application.class);
+    }
+}

SampleApps/TunneledWebView/app/src/main/AndroidManifest.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="ca.psiphon.tunneledwebview">
+
+    <application
+        android:allowBackup="true"
+        android:icon="@mipmap/ic_launcher"
+        android:label="@string/app_name"
+        android:supportsRtl="true"
+        android:theme="@style/AppTheme">
+        <activity android:name=".MainActivity">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+
+    </application>
+
+    <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
+
+</manifest>

SampleApps/TunneledWebView/app/src/main/java/ca/psiphon/PsiphonTunnel.java

@@ -0,0 +1,384 @@
+/*
+ * Copyright (c) 2016, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package ca.psiphon;
+
+import android.annotation.TargetApi;
+import android.content.Context;
+import android.net.ConnectivityManager;
+import android.net.NetworkInfo;
+import android.os.Build;
+import android.util.Base64;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import go.psi.Psi;
+
+public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
+
+    public interface TunneledApp {
+        Context getContext();
+        String getPsiphonConfig();
+        void onDiagnosticMessage(String message);
+        void onAvailableEgressRegions(List<String> regions);
+        void onSocksProxyPortInUse(int port);
+        void onHttpProxyPortInUse(int port);
+        void onListeningSocksProxyPort(int port);
+        void onListeningHttpProxyPort(int port);
+        void onUpstreamProxyError(String message);
+        void onConnecting();
+        void onConnected();
+        void onHomepage(String url);
+        void onClientRegion(String region);
+        void onClientUpgradeDownloaded(String filename);
+        void onSplitTunnelRegion(String region);
+        void onUntunneledAddress(String address);
+        void onBytesTransferred(long sent, long received);
+        void onStartedWaitingForNetworkConnectivity();
+    }
+
+    private final TunneledApp mTunneledApp;
+    private AtomicBoolean mIsWaitingForNetworkConnectivity;
+
+    // Only one PsiphonVpn instance may exist at a time, as the underlying
+    // go.psi.Psi contains global state.
+    private static PsiphonTunnel mPsiphonTunnel;
+
+    public static synchronized PsiphonTunnel newPsiphonTunnel(TunneledApp tunneledApp) {
+        if (mPsiphonTunnel != null) {
+            mPsiphonTunnel.stop();
+        }
+        // Load the native go code embedded in psi.aar
+        System.loadLibrary("gojni");
+        mPsiphonTunnel = new PsiphonTunnel(tunneledApp);
+        return mPsiphonTunnel;
+    }
+
+    private PsiphonTunnel(TunneledApp tunneledApp) {
+        mTunneledApp = tunneledApp;
+        mIsWaitingForNetworkConnectivity = new AtomicBoolean(false);
+    }
+
+    public Object clone() throws CloneNotSupportedException {
+        throw new CloneNotSupportedException();
+    }
+
+    //----------------------------------------------------------------------------------------------
+    // Public API
+    //----------------------------------------------------------------------------------------------
+
+    public synchronized boolean start(String embeddedServerEntries) {
+        return startPsiphon(embeddedServerEntries);
+    }
+
+    public synchronized void stop() {
+        stopPsiphon();
+    }
+
+    //----------------------------------------------------------------------------------------------
+    // PsiphonProvider (Core support) interface implementation
+    //----------------------------------------------------------------------------------------------
+
+    @Override
+    public void Notice(String noticeJSON) {
+        handlePsiphonNotice(noticeJSON);
+    }
+
+    @TargetApi(Build.VERSION_CODES.ICE_CREAM_SANDWICH)
+    @Override
+    public void BindToDevice(long fileDescriptor) throws Exception {
+        // This PsiphonProvider function is only called in TunnelWholeDevice mode
+        throw new Exception("BindToDevice not supported");
+    }
+
+    @Override
+    public long HasNetworkConnectivity() {
+        boolean hasConnectivity = hasNetworkConnectivity(mTunneledApp.getContext());
+        boolean wasWaitingForNetworkConnectivity = mIsWaitingForNetworkConnectivity.getAndSet(!hasConnectivity);
+        if (!hasConnectivity && !wasWaitingForNetworkConnectivity) {
+            // HasNetworkConnectivity may be called many times, but only call
+            // onStartedWaitingForNetworkConnectivity once per loss of connectivity,
+            // so the HostService may log a single message.
+            mTunneledApp.onStartedWaitingForNetworkConnectivity();
+        }
+        // TODO: change to bool return value once gobind supports that type
+        return hasConnectivity ? 1 : 0;
+    }
+
+    private static boolean hasNetworkConnectivity(Context context) {
+        ConnectivityManager connectivityManager =
+                (ConnectivityManager)context.getSystemService(Context.CONNECTIVITY_SERVICE);
+        NetworkInfo networkInfo = connectivityManager.getActiveNetworkInfo();
+        return networkInfo != null && networkInfo.isConnected();
+    }
+
+    @Override
+    public String GetPrimaryDnsServer() {
+        // This PsiphonProvider function is only called in TunnelWholeDevice mode
+        return "";
+    }
+
+    @Override
+    public String GetSecondaryDnsServer() {
+        // This PsiphonProvider function is only called in TunnelWholeDevice mode
+        return "";
+    }
+
+    //----------------------------------------------------------------------------------------------
+    // Psiphon Tunnel Core
+    //----------------------------------------------------------------------------------------------
+
+    private boolean startPsiphon(String embeddedServerEntries) {
+        stopPsiphon();
+        mTunneledApp.onDiagnosticMessage("starting Psiphon library");
+        try {
+            Psi.Start(
+                    loadPsiphonConfig(mTunneledApp.getContext()),
+                    embeddedServerEntries,
+                    this,
+                    false);
+        } catch (java.lang.Exception e) {
+            mTunneledApp.onDiagnosticMessage("failed to start Psiphon library: " + e.getMessage());
+            return false;
+        }
+        mTunneledApp.onDiagnosticMessage("Psiphon library started");
+        return true;
+    }
+
+    private void stopPsiphon() {
+        mTunneledApp.onDiagnosticMessage("stopping Psiphon library");
+        Psi.Stop();
+        mTunneledApp.onDiagnosticMessage("Psiphon library stopped");
+    }
+
+    private String loadPsiphonConfig(Context context)
+            throws IOException, JSONException {
+
+        // Load settings from the raw resource JSON config file and
+        // update as necessary. Then write JSON to disk for the Go client.
+        JSONObject json = new JSONObject(mTunneledApp.getPsiphonConfig());
+
+        // On Android, these directories must be set to the app private storage area.
+        // The Psiphon library won't be able to use its current working directory
+        // and the standard temporary directories do not exist.
+        json.put("DataStoreDirectory", context.getFilesDir());
+        json.put("DataStoreTempDirectory", context.getCacheDir());
+
+        // Note: onConnecting/onConnected logic assumes 1 tunnel connection
+        json.put("TunnelPoolSize", 1);
+
+        // Continue to run indefinitely until connected
+        json.put("EstablishTunnelTimeoutSeconds", 0);
+
+        // This parameter is for stats reporting
+        json.put("TunnelWholeDevice", 0);
+
+        json.put("EmitBytesTransferred", true);
+
+        json.put("UseIndistinguishableTLS", true);
+
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.ICE_CREAM_SANDWICH) {
+            json.put("UseTrustedCACertificatesForStockTLS", true);
+        }
+
+        try {
+            // Also enable indistinguishable TLS for HTTPS requests that
+            // require system CAs.
+            json.put(
+                    "TrustedCACertificatesFilename",
+                    setupTrustedCertificates(mTunneledApp.getContext()));
+        } catch (Exception e) {
+            mTunneledApp.onDiagnosticMessage(e.getMessage());
+        }
+
+        return json.toString();
+    }
+
+    private void handlePsiphonNotice(String noticeJSON) {
+        try {
+            // All notices are sent on as diagnostic messages
+            // except those that may contain private user data.
+            boolean diagnostic = true;
+
+            JSONObject notice = new JSONObject(noticeJSON);
+            String noticeType = notice.getString("noticeType");
+
+            if (noticeType.equals("Tunnels")) {
+                int count = notice.getJSONObject("data").getInt("count");
+                if (count > 0) {
+                    mTunneledApp.onConnected();
+                } else {
+                    mTunneledApp.onConnecting();
+                }
+
+            } else if (noticeType.equals("AvailableEgressRegions")) {
+                JSONArray egressRegions = notice.getJSONObject("data").getJSONArray("regions");
+                ArrayList<String> regions = new ArrayList<String>();
+                for (int i=0; i<egressRegions.length(); i++) {
+                    regions.add(egressRegions.getString(i));
+                }
+                mTunneledApp.onAvailableEgressRegions(regions);
+
+            } else if (noticeType.equals("SocksProxyPortInUse")) {
+                mTunneledApp.onSocksProxyPortInUse(notice.getJSONObject("data").getInt("port"));
+
+            } else if (noticeType.equals("HttpProxyPortInUse")) {
+                mTunneledApp.onHttpProxyPortInUse(notice.getJSONObject("data").getInt("port"));
+
+            } else if (noticeType.equals("ListeningSocksProxyPort")) {
+                int port = notice.getJSONObject("data").getInt("port");
+                mTunneledApp.onListeningSocksProxyPort(port);
+
+            } else if (noticeType.equals("ListeningHttpProxyPort")) {
+                int port = notice.getJSONObject("data").getInt("port");
+                mTunneledApp.onListeningHttpProxyPort(port);
+
+            } else if (noticeType.equals("UpstreamProxyError")) {
+                mTunneledApp.onUpstreamProxyError(notice.getJSONObject("data").getString("message"));
+
+            } else if (noticeType.equals("ClientUpgradeDownloaded")) {
+                mTunneledApp.onClientUpgradeDownloaded(notice.getJSONObject("data").getString("filename"));
+
+            } else if (noticeType.equals("Homepage")) {
+                mTunneledApp.onHomepage(notice.getJSONObject("data").getString("url"));
+
+            } else if (noticeType.equals("ClientRegion")) {
+                mTunneledApp.onClientRegion(notice.getJSONObject("data").getString("region"));
+
+            } else if (noticeType.equals("SplitTunnelRegion")) {
+                mTunneledApp.onSplitTunnelRegion(notice.getJSONObject("data").getString("region"));
+
+            } else if (noticeType.equals("UntunneledAddress")) {
+                mTunneledApp.onUntunneledAddress(notice.getJSONObject("data").getString("address"));
+
+            } else if (noticeType.equals("BytesTransferred")) {
+                diagnostic = false;
+                JSONObject data = notice.getJSONObject("data");
+                mTunneledApp.onBytesTransferred(data.getLong("sent"), data.getLong("received"));
+            }
+
+            if (diagnostic) {
+                String diagnosticMessage = noticeType + ": " + notice.getJSONObject("data").toString();
+                mTunneledApp.onDiagnosticMessage(diagnosticMessage);
+            }
+
+        } catch (JSONException e) {
+            // Ignore notice
+        }
+    }
+
+    private String setupTrustedCertificates(Context context) throws Exception {
+
+        // Copy the Android system CA store to a local, private cert bundle file.
+        //
+        // This results in a file that can be passed to SSL_CTX_load_verify_locations
+        // for use with OpenSSL modes in tunnel-core.
+        // https://www.openssl.org/docs/manmaster/ssl/SSL_CTX_load_verify_locations.html
+        //
+        // TODO: to use the path mode of load_verify_locations would require emulating
+        // the filename scheme used by c_rehash:
+        // https://www.openssl.org/docs/manmaster/apps/c_rehash.html
+        // http://stackoverflow.com/questions/19237167/the-new-subject-hash-openssl-algorithm-differs
+
+        File directory = context.getDir("PsiphonCAStore", Context.MODE_PRIVATE);
+
+        final String errorMessage = "copy AndroidCAStore failed";
+        try {
+
+            File file = new File(directory, "certs.dat");
+
+            // Pave a fresh copy on every run, which ensures we're not using old certs.
+            // Note: assumes KeyStore doesn't return revoked certs.
+            //
+            // TODO: this takes under 1 second, but should we avoid repaving every time?
+            file.delete();
+
+            PrintStream output = null;
+            try {
+                output = new PrintStream(new FileOutputStream(file));
+
+                KeyStore keyStore;
+                if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_CREAM_SANDWICH) {
+                    keyStore = KeyStore.getInstance("AndroidCAStore");
+                    keyStore.load(null, null);
+                } else {
+                    keyStore = KeyStore.getInstance("BKS");
+                    FileInputStream inputStream = new FileInputStream("/etc/security/cacerts.bks");
+                    try {
+                        keyStore.load(inputStream, "changeit".toCharArray());
+                    } finally {
+                        if (inputStream != null) {
+                            inputStream.close();
+                        }
+                    }
+                }
+
+                Enumeration<String> aliases = keyStore.aliases();
+                while (aliases.hasMoreElements()) {
+                    String alias = aliases.nextElement();
+                    X509Certificate cert = (X509Certificate) keyStore.getCertificate(alias);
+
+                    output.println("-----BEGIN CERTIFICATE-----");
+                    String pemCert = new String(Base64.encode(cert.getEncoded(), Base64.NO_WRAP), "UTF-8");
+                    // OpenSSL appears to reject the default linebreaking done by Base64.encode,
+                    // so we manually linebreak every 64 characters
+                    for (int i = 0; i < pemCert.length() ; i+= 64) {
+                        output.println(pemCert.substring(i, Math.min(i + 64, pemCert.length())));
+                    }
+                    output.println("-----END CERTIFICATE-----");
+                }
+
+                mTunneledApp.onDiagnosticMessage("prepared PsiphonCAStore");
+
+                return file.getAbsolutePath();
+
+            } finally {
+                if (output != null) {
+                    output.close();
+                }
+            }
+
+        } catch (KeyStoreException e) {
+            throw new Exception(errorMessage, e);
+        } catch (NoSuchAlgorithmException e) {
+            throw new Exception(errorMessage, e);
+        } catch (CertificateException e) {
+            throw new Exception(errorMessage, e);
+        } catch (IOException e) {
+            throw new Exception(errorMessage, e);
+        }
+    }
+}

SampleApps/TunneledWebView/app/src/main/java/ca/psiphon/tunneledwebview/MainActivity.java

@@ -0,0 +1,296 @@
+/*
+ * Copyright (c) 2016, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package ca.psiphon.tunneledwebview;
+
+import android.content.Context;
+import android.os.Bundle;
+import android.support.v7.app.ActionBarActivity;
+import android.webkit.WebView;
+import android.widget.ArrayAdapter;
+import android.widget.ListView;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import ca.psiphon.PsiphonTunnel;
+
+//----------------------------------------------------------------------------------------------
+// TunneledWebView
+//
+// This sample app demonstrates tunneling a WebView through the
+// Psiphon Library. This app's main activity shows a log of
+// events and a WebView that is loaded once Psiphon is connected.
+//
+// The flow is as follows:
+//
+// - The Psiphon tunnel is started in onResume(). PsiphonTunnel.start()
+//   is an asynchronous call that returns immediately.
+//
+// - Once Psiphon has selected a local HTTP proxy listening port, the
+//   onListeningHttpProxyPort() callback is called. This app records the
+//   port to use for tunneling traffic.
+//
+// - Once Psiphon has established a tunnel, the onConnected() callback
+//   is called. This app now loads the WebView, after setting its proxy
+//   to point to Psiphon's local HTTP proxy.
+//
+// To adapt this sample into your own app:
+//
+// - Embed a Psiphon config file in app/src/main/res/raw/psiphon_config.
+//
+// - Add the Psiphon Library AAR module as a dependency (see this app's
+//   project settings; to build this sample project, you need to drop
+//   psi-0.0.10.aar into app/libs).
+//
+// - Use app/src/main/java/ca/psiphon/PsiphonTunnel.java, which provides
+//   a higher-level wrapper around the Psiphon Library module. This file
+//   shows how to use PsiphonTunnel and PsiphonTunnel.TunneledApp.
+//
+//----------------------------------------------------------------------------------------------
+
+public class MainActivity extends ActionBarActivity
+        implements PsiphonTunnel.TunneledApp {
+
+    private ListView mListView;
+    private WebView mWebView;
+
+    private ArrayAdapter<String> mLogMessages;
+    private AtomicInteger mLocalHttpProxyPort;
+
+    private PsiphonTunnel mPsiphonTunnel;
+
+    @Override
+    protected void onCreate(Bundle savedInstanceState) {
+        super.onCreate(savedInstanceState);
+        setContentView(R.layout.activity_main);
+
+        mListView = (ListView)findViewById(R.id.listView);
+        mWebView = (WebView)findViewById(R.id.webView);
+
+        mLogMessages = new ArrayAdapter<String>(
+                this, R.layout.log_message, R.id.logMessageTextView);
+
+        mListView.setAdapter(mLogMessages);
+
+        mLocalHttpProxyPort = new AtomicInteger(0);
+
+        mPsiphonTunnel = PsiphonTunnel.newPsiphonTunnel(this);
+    }
+
+    @Override
+    protected void onResume() {
+        super.onResume();
+
+        // NOTE: for demonstration purposes, this sample app
+        // restarts Psiphon in onPause/onResume. Since it may take some
+        // time to connect, it's generally recommended to keep
+        // Psiphon running, so start/stop in onCreate/onDestroy or
+        // even consider running a background Service.
+
+        if (!mPsiphonTunnel.start("")) {
+            logMessage("failed to start Psiphon");
+        }
+    }
+
+    @Override
+    protected void onPause() {
+        super.onPause();
+
+        // NOTE: stop() can block for a few seconds, so it's generally
+        // recommended to run PsiphonTunnel.start()/stop() in a background
+        // thread and signal the thread appropriately.
+
+        mPsiphonTunnel.stop();
+    }
+
+    private void setHttpProxyPort(int port) {
+
+        // NOTE: here we record the Psiphon proxy port for subsequent
+        // use in tunneling app traffic. In this sample app, we will
+        // use WebViewProxySettings.setLocalProxy to tunnel a WebView
+        // through Psiphon. By default, the local proxy port is selected
+        // dynamically, so it's important to record and use the correct
+        // port number.
+
+        mLocalHttpProxyPort.set(port);
+    }
+
+    private void loadWebView() {
+
+        // NOTE: functions called via PsiphonTunnel.TunneledApp may be
+        // called on background threads. It's important to ensure that
+        // these threads are not blocked and that UI functions are not
+        // called directly from these threads. Here we use runOnUiThread
+        // to handle this.
+
+        runOnUiThread(new Runnable() {
+            public void run() {
+                WebViewProxySettings.setLocalProxy(
+                        MainActivity.this, mLocalHttpProxyPort.get());
+                mWebView.loadUrl("https://ipinfo.io/");
+            }
+        });
+    }
+
+    private void logMessage(final String message) {
+        runOnUiThread(new Runnable() {
+            public void run() {
+                mLogMessages.add(message);
+                mListView.setSelection(mLogMessages.getCount() - 1);
+            }
+        });
+    }
+
+    //----------------------------------------------------------------------------------------------
+    // PsiphonTunnel.TunneledApp implementation
+    //
+    // NOTE: these are callbacks from the Psiphon Library
+    //----------------------------------------------------------------------------------------------
+
+    @Override
+    public Context getContext() {
+        return this;
+    }
+
+    @Override
+    public String getPsiphonConfig() {
+        try {
+            JSONObject config = new JSONObject(
+                    readInputStreamToString(
+                            getResources().openRawResource(R.raw.psiphon_config)));
+
+            return config.toString();
+
+        } catch (IOException e) {
+            logMessage("error loading Psiphon config: " + e.getMessage());
+        } catch (JSONException e) {
+            logMessage("error loading Psiphon config: " + e.getMessage());
+        }
+        return "";
+    }
+
+    @Override
+    public void onDiagnosticMessage(String message) {
+        android.util.Log.i(getString(R.string.app_name), message);
+        logMessage(message);
+    }
+
+    @Override
+    public void onAvailableEgressRegions(List<String> regions) {
+        for (String region : regions) {
+            logMessage("available egress region: " + region);
+        }
+    }
+
+    @Override
+    public void onSocksProxyPortInUse(int port) {
+        logMessage("local SOCKS proxy port in use: " + Integer.toString(port));
+    }
+
+    @Override
+    public void onHttpProxyPortInUse(int port) {
+        logMessage("local HTTP proxy port in use: " + Integer.toString(port));
+    }
+
+    @Override
+    public void onListeningSocksProxyPort(int port) {
+        logMessage("local SOCKS proxy listening on port: " + Integer.toString(port));
+    }
+
+    @Override
+    public void onListeningHttpProxyPort(int port) {
+        logMessage("local HTTP proxy listening on port: " + Integer.toString(port));
+        setHttpProxyPort(port);
+    }
+
+    @Override
+    public void onUpstreamProxyError(String message) {
+        logMessage("upstream proxy error: " + message);
+    }
+
+    @Override
+    public void onConnecting() {
+        logMessage("connecting...");
+    }
+
+    @Override
+    public void onConnected() {
+        logMessage("connected");
+        loadWebView();
+    }
+
+    @Override
+    public void onHomepage(String url) {
+        logMessage("home page: " + url);
+    }
+
+    @Override
+    public void onClientUpgradeDownloaded(String filename) {
+        logMessage("client upgrade downloaded");
+    }
+
+    @Override
+    public void onSplitTunnelRegion(String region) {
+        logMessage("split tunnel region: " + region);
+    }
+
+    @Override
+    public void onUntunneledAddress(String address) {
+        logMessage("untunneled address: " + address);
+    }
+
+    @Override
+    public void onBytesTransferred(long sent, long received) {
+        logMessage("bytes sent: " + Long.toString(sent));
+        logMessage("bytes received: " + Long.toString(received));
+    }
+
+    @Override
+    public void onStartedWaitingForNetworkConnectivity() {
+        logMessage("waiting for network connectivity...");
+    }
+
+    @Override
+    public void onClientRegion(String region) {
+        logMessage("client region: " + region);
+    }
+
+    private static String readInputStreamToString(InputStream inputStream) throws IOException {
+        return new String(readInputStreamToBytes(inputStream), "UTF-8");
+    }
+
+    private static byte[] readInputStreamToBytes(InputStream inputStream) throws IOException {
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        int readCount;
+        byte[] buffer = new byte[16384];
+        while ((readCount = inputStream.read(buffer, 0, buffer.length)) != -1) {
+            outputStream.write(buffer, 0, readCount);
+        }
+        outputStream.flush();
+        inputStream.close();
+        return outputStream.toByteArray();
+    }
+}

SampleApps/TunneledWebView/app/src/main/java/ca/psiphon/tunneledwebview/WebViewProxySettings.java

@@ -0,0 +1,313 @@
+/*
+ * Copyright (c) 2013, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package ca.psiphon.tunneledwebview;
+
+import android.annotation.TargetApi;
+import android.content.Context;
+import android.content.Intent;
+import android.net.Proxy;
+import android.os.Build;
+import android.os.Parcelable;
+import android.util.ArrayMap;
+
+import org.apache.http.HttpHost;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+public class WebViewProxySettings 
+{
+
+    public static void setLocalProxy(Context ctx, int port)
+    {
+        setProxy(ctx, "localhost", port);
+    }
+    
+    /* 
+    Proxy setting code taken directly from Orweb, with some modifications.
+    (...And some of the Orweb code was taken from an earlier version of our code.)
+    See: https://github.com/guardianproject/Orweb/blob/master/src/org/torproject/android/OrbotHelper.java#L39
+    Note that we tried and abandoned doing feature detection by trying the 
+    newer (>= ICS) proxy setting, catching, and then failing over to the older
+    approach. The problem was that on Android 3.0, an exception would be thrown
+    *in another thread*, so we couldn't catch it and the whole app would force-close.
+    Orweb has always been doing an explicit version check, and it seems to work,
+    so we're so going to switch to that approach.
+    */
+    public static boolean setProxy (Context ctx, String host, int port)
+    {
+        boolean worked = false;
+
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.ICE_CREAM_SANDWICH)
+        {
+            worked = setWebkitProxyGingerbread(ctx, host, port);
+        }
+        else if (Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT)
+        {
+            worked = setWebkitProxyICS(ctx, host, port);
+        }
+        else if (Build.VERSION.SDK_INT == Build.VERSION_CODES.KITKAT)
+        {
+            worked = setWebkitProxyKitKat(ctx.getApplicationContext(), host, port);
+        }
+        else
+        {
+            worked = setWebkitProxyLollipop(ctx.getApplicationContext(), host, port);
+        }
+        
+        return worked;
+    }
+
+    private static boolean setWebkitProxyGingerbread(Context ctx, String host, int port)
+    {
+        try
+        {
+            Object requestQueueObject = getRequestQueue(ctx);
+            if (requestQueueObject != null) {
+                //Create Proxy config object and set it into request Q
+                HttpHost httpHost = new HttpHost(host, port, "http");   
+                setDeclaredField(requestQueueObject, "mProxyHost", httpHost);
+                
+                return true;
+            }
+        }
+        catch (Throwable e)
+        {
+            // Failed. Fall through to false return.
+        }
+        
+        return false;
+    }
+    
+    @SuppressWarnings("rawtypes")
+    private static boolean setWebkitProxyICS(Context ctx, String host, int port)
+    {
+        try 
+        {
+            Class webViewCoreClass = Class.forName("android.webkit.WebViewCore");
+           
+            Class proxyPropertiesClass = Class.forName("android.net.ProxyProperties");
+            if (webViewCoreClass != null && proxyPropertiesClass != null) 
+            {
+                Method m = webViewCoreClass.getDeclaredMethod("sendStaticMessage", Integer.TYPE, Object.class);
+                Constructor c = proxyPropertiesClass.getConstructor(String.class, Integer.TYPE, String.class);
+                
+                if (m != null && c != null)
+                {
+                    m.setAccessible(true);
+                    c.setAccessible(true);
+                    Object properties = c.newInstance(host, port, null);
+                
+                    // android.webkit.WebViewCore.EventHub.PROXY_CHANGED = 193;
+                    m.invoke(null, 193, properties);
+                    return true;
+                }
+            }
+        }
+        catch (Exception e) 
+        {
+        }
+        catch (Error e) 
+        {
+        }
+        
+        return false;
+    }
+
+    // http://stackoverflow.com/questions/19979578/android-webview-set-proxy-programatically-kitkat
+    // http://src.chromium.org/viewvc/chrome/trunk/src/net/android/java/src/org/chromium/net/ProxyChangeListener.java
+    @TargetApi(Build.VERSION_CODES.KITKAT)
+    @SuppressWarnings("rawtypes")
+    private static boolean setWebkitProxyKitKat(Context appContext, String host, int port)
+    {
+        System.setProperty("http.proxyHost", host);
+        System.setProperty("http.proxyPort", port + "");
+        System.setProperty("https.proxyHost", host);
+        System.setProperty("https.proxyPort", port + "");
+        try
+        {
+            Class applicationClass = Class.forName("android.app.Application");
+            Field loadedApkField = applicationClass.getDeclaredField("mLoadedApk");
+            loadedApkField.setAccessible(true);
+            Object loadedApk = loadedApkField.get(appContext);
+            Class loadedApkClass = Class.forName("android.app.LoadedApk");
+            Field receiversField = loadedApkClass.getDeclaredField("mReceivers");
+            receiversField.setAccessible(true);
+            ArrayMap receivers = (ArrayMap) receiversField.get(loadedApk);
+            for (Object receiverMap : receivers.values())
+            {
+                for (Object receiver : ((ArrayMap) receiverMap).keySet())
+                {
+                    Class receiverClass = receiver.getClass();
+                    if (receiverClass.getName().contains("ProxyChangeListener"))
+                    {
+                        Method onReceiveMethod = receiverClass.getDeclaredMethod("onReceive", Context.class, Intent.class);
+                        Intent intent = new Intent(Proxy.PROXY_CHANGE_ACTION);
+
+                        final String CLASS_NAME = "android.net.ProxyProperties";
+                        Class proxyPropertiesClass = Class.forName(CLASS_NAME);
+                        Constructor constructor = proxyPropertiesClass.getConstructor(String.class, Integer.TYPE, String.class);
+                        constructor.setAccessible(true);
+                        Object proxyProperties = constructor.newInstance(host, port, null);
+                        intent.putExtra("proxy", (Parcelable) proxyProperties);
+
+                        onReceiveMethod.invoke(receiver, appContext, intent);
+                    }
+                }
+            }
+            return true;
+        }
+        catch (ClassNotFoundException e)
+        {
+        }
+        catch (NoSuchFieldException e)
+        {
+        }
+        catch (IllegalAccessException e)
+        {
+        }
+        catch (IllegalArgumentException e)
+        {
+        }
+        catch (NoSuchMethodException e)
+        {
+        }
+        catch (InvocationTargetException e)
+        {
+        }
+        catch (InstantiationException e)
+        {
+        }
+        return false;
+    }
+
+    // http://stackanswers.com/questions/25272393/android-webview-set-proxy-programmatically-on-android-l
+    @TargetApi(Build.VERSION_CODES.KITKAT) // for android.util.ArrayMap methods
+    @SuppressWarnings("rawtypes")
+    private static boolean setWebkitProxyLollipop(Context appContext, String host, int port)
+    {
+        System.setProperty("http.proxyHost", host);
+        System.setProperty("http.proxyPort", port + "");
+        System.setProperty("https.proxyHost", host);
+        System.setProperty("https.proxyPort", port + "");
+        try {
+            Class applictionClass = Class.forName("android.app.Application");
+            Field mLoadedApkField = applictionClass.getDeclaredField("mLoadedApk");
+            mLoadedApkField.setAccessible(true);
+            Object mloadedApk = mLoadedApkField.get(appContext);
+            Class loadedApkClass = Class.forName("android.app.LoadedApk");
+            Field mReceiversField = loadedApkClass.getDeclaredField("mReceivers");
+            mReceiversField.setAccessible(true);
+            ArrayMap receivers = (ArrayMap) mReceiversField.get(mloadedApk);
+            for (Object receiverMap : receivers.values())
+            {
+                for (Object receiver : ((ArrayMap) receiverMap).keySet())
+                {
+                    Class clazz = receiver.getClass();
+                    if (clazz.getName().contains("ProxyChangeListener"))
+                    {
+                        Method onReceiveMethod = clazz.getDeclaredMethod("onReceive", Context.class, Intent.class);
+                        Intent intent = new Intent(Proxy.PROXY_CHANGE_ACTION);
+                        onReceiveMethod.invoke(receiver, appContext, intent);
+                    }
+                }
+            }
+            return true;
+        }
+        catch (ClassNotFoundException e)
+        {
+        }
+        catch (NoSuchFieldException e)
+        {
+        }
+        catch (IllegalAccessException e)
+        {
+        }
+        catch (NoSuchMethodException e)
+        {
+        }
+        catch (InvocationTargetException e)
+        {
+        }
+        return false;
+     }
+    
+    @SuppressWarnings("rawtypes")
+    private static Object GetNetworkInstance(Context ctx) throws ClassNotFoundException
+    {
+        Class networkClass = Class.forName("android.webkit.Network");
+        return networkClass;
+    }
+    
+    private static Object getRequestQueue(Context ctx) throws Exception 
+    {
+        Object ret = null;
+        Object networkClass = GetNetworkInstance(ctx);
+        if (networkClass != null) 
+        {
+            Object networkObj = invokeMethod(networkClass, "getInstance", new Object[]{ctx}, Context.class);
+            if (networkObj != null) 
+            {
+                ret = getDeclaredField(networkObj, "mRequestQueue");
+            }
+        }
+        return ret;
+    }
+
+    private static Object getDeclaredField(Object obj, String name)
+            throws SecurityException, NoSuchFieldException,
+            IllegalArgumentException, IllegalAccessException 
+    {
+        Field f = obj.getClass().getDeclaredField(name);
+        f.setAccessible(true);
+        Object out = f.get(obj);
+        return out;
+    }
+
+    private static void setDeclaredField(Object obj, String name, Object value)
+            throws SecurityException, NoSuchFieldException,
+            IllegalArgumentException, IllegalAccessException 
+    {
+        Field f = obj.getClass().getDeclaredField(name);
+        f.setAccessible(true);
+        f.set(obj, value);
+    }
+
+    @SuppressWarnings("rawtypes")
+    private static Object invokeMethod(Object object, String methodName, Object[] params, Class... types) throws Exception 
+    {
+        Object out = null;
+        Class c = object instanceof Class ? (Class) object : object.getClass();
+        
+        if (types != null) 
+        {
+            Method method = c.getMethod(methodName, types);
+            out = method.invoke(object, params);
+        } 
+        else 
+        {
+            Method method = c.getMethod(methodName);
+            out = method.invoke(object);
+        }
+        return out;
+    }
+}

SampleApps/TunneledWebView/app/src/main/res/layout/activity_main.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="utf-8"?>
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    android:orientation="vertical"
+    android:paddingBottom="@dimen/activity_vertical_margin"
+    android:paddingLeft="@dimen/activity_horizontal_margin"
+    android:paddingRight="@dimen/activity_horizontal_margin"
+    android:paddingTop="@dimen/activity_vertical_margin"
+    tools:context="ca.psiphon.tunneledwebview.MainActivity">
+
+    <LinearLayout
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        android:layout_weight="1">
+        <WebView
+            android:id="@+id/webView"
+            android:layout_width="match_parent"
+            android:layout_height="match_parent">
+        </WebView>
+    </LinearLayout>
+
+    <View
+        android:layout_width="match_parent"
+        android:layout_height="10dp">
+    </View>
+
+    <LinearLayout
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        android:layout_weight="2">
+        <ListView
+            android:id="@+id/listView"
+            android:layout_width="match_parent"
+            android:layout_height="match_parent">
+        </ListView>
+    </LinearLayout>
+
+</LinearLayout>

SampleApps/TunneledWebView/app/src/main/res/layout/log_message.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="utf-8"?>
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:orientation="vertical"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent">
+
+    <TextView xmlns:android="http://schemas.android.com/apk/res/android"
+        android:id="@+id/logMessageTextView"
+        android:layout_width="fill_parent"
+        android:layout_height="wrap_content"
+        android:padding="1dp"
+        android:textSize="5sp" >
+    </TextView>
+
+</LinearLayout>

SampleApps/TunneledWebView/app/src/main/res/raw/psiphon_config_stub

@@ -0,0 +1,17 @@
+{
+    "PropagationChannelId" : "<placeholder>",
+    "SponsorId" : "<placeholder>",
+    "RemoteServerListUrl" : "<placeholder>",
+    "RemoteServerListSignaturePublicKey" : "<placeholder>",
+    "DataStoreDirectory" : "",
+    "DataStoreTempDirectory" : "",
+    "LogFilename" : "",
+    "LocalHttpProxyPort" : 0,
+    "LocalSocksProxyPort" : 0,
+    "EgressRegion" : "",
+    "TunnelProtocol" : "",
+    "ConnectionWorkerPoolSize" : 10,
+    "TunnelPoolSize" : 1,
+    "PortForwardFailureThreshold" : 10,
+    "UpstreamProxyUrl" : ""
+}

SampleApps/TunneledWebView/app/src/main/res/values-w820dp/dimens.xml

@@ -0,0 +1,6 @@
+<resources>
+    <!-- Example customization of dimensions originally defined in res/values/dimens.xml
+         (such as screen margins) for screens with more than 820dp of available width. This
+         would include 7" and 10" devices in landscape (~960dp and ~1280dp respectively). -->
+    <dimen name="activity_horizontal_margin">64dp</dimen>
+</resources>

SampleApps/TunneledWebView/app/src/main/res/values/dimens.xml

@@ -0,0 +1,5 @@
+<resources>
+    <!-- Default screen margins, per the Android Design guidelines. -->
+    <dimen name="activity_horizontal_margin">16dp</dimen>
+    <dimen name="activity_vertical_margin">16dp</dimen>
+</resources>

SampleApps/TunneledWebView/app/src/main/res/values/strings.xml

@@ -0,0 +1,3 @@
+<resources>
+    <string name="app_name">TunneledWebView</string>
+</resources>

SampleApps/TunneledWebView/app/src/main/res/values/styles.xml

@@ -0,0 +1,8 @@
+<resources>
+
+    <!-- Base application theme. -->
+    <style name="AppTheme" parent="Theme.AppCompat.Light.DarkActionBar">
+        <!-- Customize your theme here. -->
+    </style>
+
+</resources>

SampleApps/TunneledWebView/app/src/test/java/ca/psiphon/tunneledwebview/ExampleUnitTest.java

@@ -0,0 +1,15 @@
+package ca.psiphon.tunneledwebview;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * To work on unit tests, switch the Test Artifact in the Build Variants view.
+ */
+public class ExampleUnitTest {
+    @Test
+    public void addition_isCorrect() throws Exception {
+        assertEquals(4, 2 + 2);
+    }
+}

SampleApps/TunneledWebView/build.gradle

@@ -0,0 +1,24 @@
+// Top-level build file where you can add configuration options common to all sub-projects/modules.
+
+buildscript {
+    repositories {
+        jcenter()
+    }
+    dependencies {
+        classpath 'com.android.tools.build:gradle:1.3.0'
+
+        // NOTE: Do not place your application dependencies here; they belong
+        // in the individual module build.gradle files
+    }
+}
+
+allprojects {
+    repositories {
+        jcenter()
+    }
+}
+
+task clean(type: Delete) {
+    delete rootProject.buildDir
+}
+

SampleApps/TunneledWebView/gradle.properties

@@ -0,0 +1,18 @@
+# Project-wide Gradle settings.
+
+# IDE (e.g. Android Studio) users:
+# Gradle settings configured through the IDE *will override*
+# any settings specified in this file.
+
+# For more details on how to configure your build environment visit
+# http://www.gradle.org/docs/current/userguide/build_environment.html
+
+# Specifies the JVM arguments used for the daemon process.
+# The setting is particularly useful for tweaking memory settings.
+# Default value: -Xmx10248m -XX:MaxPermSize=256m
+# org.gradle.jvmargs=-Xmx2048m -XX:MaxPermSize=512m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8
+
+# When configured, Gradle will run in incubating parallel mode.
+# This option should only be used with decoupled projects. More details, visit
+# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
+# org.gradle.parallel=true

SampleApps/TunneledWebView/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Thu Jan 21 13:26:45 EST 2016
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.4-all.zip

SampleApps/TunneledWebView/gradlew

@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn ( ) {
+    echo "$*"
+}
+
+die ( ) {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+esac
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched.
+if $cygwin ; then
+    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
+fi
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >&-
+APP_HOME="`pwd -P`"
+cd "$SAVED" >&-
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
+function splitJvmOpts() {
+    JVM_OPTS=("$@")
+}
+eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
+JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+
+exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"

SampleApps/TunneledWebView/gradlew.bat

@@ -0,0 +1,90 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windowz variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+if "%@eval[2+2]" == "4" goto 4NT_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+goto execute
+
+:4NT_args
+@rem Get arguments from the 4NT Shell from JP Software
+set CMD_LINE_ARGS=%$
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

SampleApps/TunneledWebView/settings.gradle

@@ -0,0 +1 @@
+include ':app'

psiphon/LookupIP.go

@@ -37,7 +37,20 @@ import (
 // to the specified DNS resolver.
 func LookupIP(host string, config *DialConfig) (addrs []net.IP, err error) {
 	if config.DeviceBinder != nil {
-		return bindLookupIP(host, config)
+		addrs, err = bindLookupIP(host, config.DnsServerGetter.GetPrimaryDnsServer(), config)
+		if err == nil {
+			if len(addrs) == 0 {
+				err = errors.New("empty address list")
+			} else {
+				return addrs, err
+			}
+		}
+		NoticeAlert("retry resolve host %s: %s", host, err)
+		dnsServer := config.DnsServerGetter.GetSecondaryDnsServer()
+		if dnsServer == "" {
+			return addrs, err
+		}
+		return bindLookupIP(host, dnsServer, config)
 	}
 	return net.LookupIP(host)
 }
@@ -46,7 +59,7 @@ func LookupIP(host string, config *DialConfig) (addrs []net.IP, err error) {
 // To implement socket device binding, the lower-level syscall APIs are used.
 // The sequence of syscalls in this implementation are taken from:
 // https://code.google.com/p/go/issues/detail?id=6966
-func bindLookupIP(host string, config *DialConfig) (addrs []net.IP, err error) {
+func bindLookupIP(host, dnsServer string, config *DialConfig) (addrs []net.IP, err error) {
 
 	// When the input host is an IP address, echo it back
 	ipAddr := net.ParseIP(host)
@@ -65,8 +78,8 @@ func bindLookupIP(host string, config *DialConfig) (addrs []net.IP, err error) {
 		return nil, ContextError(fmt.Errorf("BindToDevice failed: %s", err))
 	}
 
-	// config.DnsServerGetter.GetDnsServer must return an IP address
-	ipAddr = net.ParseIP(config.DnsServerGetter.GetDnsServer())
+	// config.DnsServerGetter.GetDnsServers() must return IP addresses
+	ipAddr = net.ParseIP(dnsServer)
 	if ipAddr == nil {
 		return nil, ContextError(errors.New("invalid IP address"))
 	}
@@ -95,8 +108,6 @@ func bindLookupIP(host string, config *DialConfig) (addrs []net.IP, err error) {
 		conn.SetWriteDeadline(time.Now().Add(config.ConnectTimeout))
 	}
 
-	// TODO: make conn interruptible?
-
 	addrs, _, err = ResolveIP(host, conn)
 	return
 }

psiphon/config.go

@@ -29,11 +29,12 @@ import (
 // TODO: allow all params to be configured
 
 const (
-	DATA_STORE_FILENAME                            = "psiphon.db"
+	LEGACY_DATA_STORE_FILENAME                     = "psiphon.db"
+	DATA_STORE_FILENAME                            = "psiphon.boltdb"
 	CONNECTION_WORKER_POOL_SIZE                    = 10
 	TUNNEL_POOL_SIZE                               = 1
 	TUNNEL_CONNECT_TIMEOUT                         = 20 * time.Second
-	TUNNEL_OPERATE_SHUTDOWN_TIMEOUT                = 500 * time.Millisecond
+	TUNNEL_OPERATE_SHUTDOWN_TIMEOUT                = 1 * time.Second
 	TUNNEL_PORT_FORWARD_DIAL_TIMEOUT               = 10 * time.Second
 	TUNNEL_SSH_KEEP_ALIVE_PAYLOAD_MAX_BYTES        = 256
 	TUNNEL_SSH_KEEP_ALIVE_PERIOD_MIN               = 60 * time.Second
@@ -45,6 +46,7 @@ const (
 	ESTABLISH_TUNNEL_TIMEOUT_SECONDS               = 300
 	ESTABLISH_TUNNEL_WORK_TIME                     = 60 * time.Second
 	ESTABLISH_TUNNEL_PAUSE_PERIOD                  = 5 * time.Second
+	ESTABLISH_TUNNEL_SERVER_AFFINITY_GRACE_PERIOD  = 1 * time.Second
 	HTTP_PROXY_ORIGIN_SERVER_TIMEOUT               = 15 * time.Second
 	HTTP_PROXY_MAX_IDLE_CONNECTIONS_PER_HOST       = 50
 	FETCH_REMOTE_SERVER_LIST_TIMEOUT               = 30 * time.Second
@@ -52,11 +54,15 @@ const (
 	FETCH_REMOTE_SERVER_LIST_STALE_PERIOD          = 6 * time.Hour
 	PSIPHON_API_CLIENT_SESSION_ID_LENGTH           = 16
 	PSIPHON_API_SERVER_TIMEOUT                     = 20 * time.Second
+	PSIPHON_API_SHUTDOWN_SERVER_TIMEOUT            = 1 * time.Second
 	PSIPHON_API_STATUS_REQUEST_PERIOD_MIN          = 5 * time.Minute
 	PSIPHON_API_STATUS_REQUEST_PERIOD_MAX          = 10 * time.Minute
+	PSIPHON_API_STATUS_REQUEST_SHORT_PERIOD_MIN    = 5 * time.Second
+	PSIPHON_API_STATUS_REQUEST_SHORT_PERIOD_MAX    = 10 * time.Second
 	PSIPHON_API_STATUS_REQUEST_PADDING_MAX_BYTES   = 256
 	PSIPHON_API_CONNECTED_REQUEST_PERIOD           = 24 * time.Hour
 	PSIPHON_API_CONNECTED_REQUEST_RETRY_PERIOD     = 5 * time.Second
+	PSIPHON_API_TUNNEL_STATS_MAX_COUNT             = 1000
 	FETCH_ROUTES_TIMEOUT                           = 1 * time.Minute
 	DOWNLOAD_UPGRADE_TIMEOUT                       = 15 * time.Minute
 	DOWNLOAD_UPGRADE_RETRY_PAUSE_PERIOD            = 5 * time.Second
@@ -79,6 +85,11 @@ type Config struct {
 	// DataStoreDirectory is the directory in which to store the persistent
 	// database, which contains information such as server entries.
 	// By default, current working directory.
+	//
+	// Warning: If the datastore file, DataStoreDirectory/DATA_STORE_FILENAME,
+	// exists but fails to open for any reason (checksum error, unexpected file
+	// format, etc.) it will be deleted in order to pave a new datastore and
+	// continue running.
 	DataStoreDirectory string
 
 	// DataStoreTempDirectory is the directory in which to store temporary
@@ -118,6 +129,9 @@ type Config struct {
 	// automatic updates.
 	// This value is supplied by and depends on the Psiphon Network, and is
 	// typically embedded in the client binary.
+	// Note that sending a ClientPlatform string which includes "windows"
+	// (case insensitive) and a ClientVersion of <= 44 will cause an
+	// error in processing the response to DoConnectedRequest calls.
 	ClientVersion string
 
 	// ClientPlatform is the client platform ("Windows", "Android", etc.) that
@@ -135,8 +149,9 @@ type Config struct {
 	EgressRegion string
 
 	// TunnelProtocol indicates which protocol to use. Valid values include:
-	// "SSH", "OSSH", "UNFRONTED-MEEK-OSSH", "FRONTED-MEEK-OSSH". For the default,
-	// "", the best performing protocol is used.
+	// "SSH", "OSSH", "UNFRONTED-MEEK-OSSH", "UNFRONTED-MEEK-HTTPS-OSSH",
+	// "FRONTED-MEEK-OSSH". For the default, "", the best performing protocol
+	// is used.
 	TunnelProtocol string
 
 	// EstablishTunnelTimeoutSeconds specifies a time limit after which to halt
@@ -242,10 +257,20 @@ type Config struct {
 	EmitBytesTransferred bool
 
 	// UseIndistinguishableTLS enables use of an alternative TLS stack with a less
-	// distinct fingerprint (ClientHello content) than the stock Go TLS. This
+	// distinct fingerprint (ClientHello content) than the stock Go TLS.
+	// UseIndistinguishableTLS only applies to untunneled TLS connections. This
 	// parameter is only supported on platforms built with OpenSSL.
+	// Requires TrustedCACertificatesFilename to be set.
 	UseIndistinguishableTLS bool
 
+	// UseTrustedCACertificates toggles use of the trusted CA certs, specified
+	// in TrustedCACertificatesFilename, for tunneled TLS connections that expect
+	// server certificates signed with public certificate authorities (currently,
+	// only upgrade downloads). This option is used with stock Go TLS in cases where
+	// Go may fail to obtain a list of root CAs from the operating system.
+	// Requires TrustedCACertificatesFilename to be set.
+	UseTrustedCACertificatesForStockTLS bool
+
 	// TrustedCACertificatesFilename specifies a file containing trusted CA certs.
 	// The file contents should be compatible with OpenSSL's SSL_CTX_load_verify_locations.
 	// When specified, this enables use of indistinguishable TLS for HTTPS requests

psiphon/controller.go

@@ -51,7 +51,7 @@ type Controller struct {
 	isEstablishing                 bool
 	establishWaitGroup             *sync.WaitGroup
 	stopEstablishingBroadcast      chan struct{}
-	candidateServerEntries         chan *ServerEntry
+	candidateServerEntries         chan *candidateServerEntry
 	establishPendingConns          *Conns
 	untunneledPendingConns         *Conns
 	untunneledDialConfig           *DialConfig
@@ -59,6 +59,12 @@ type Controller struct {
 	signalFetchRemoteServerList    chan struct{}
 	impairedProtocolClassification map[string]int
 	signalReportConnected          chan struct{}
+	serverAffinityDoneBroadcast    chan struct{}
+}
+
+type candidateServerEntry struct {
+	serverEntry               *ServerEntry
+	isServerAffinityCandidate bool
 }
 
 // NewController initializes a new controller.
@@ -184,9 +190,19 @@ func (controller *Controller) Run(shutdownBroadcast <-chan struct{}) {
 
 	close(controller.shutdownBroadcast)
 	controller.establishPendingConns.CloseAll()
-	controller.untunneledPendingConns.CloseAll()
 	controller.runWaitGroup.Wait()
 
+	// Stops untunneled connections, including fetch remote server list,
+	// split tunnel port forwards and also untunneled final stats requests.
+	// Note: there's a circular dependency with runWaitGroup.Wait() and
+	// untunneledPendingConns.CloseAll(): runWaitGroup depends on tunnels
+	// stopping which depends, in orderly shutdown, on final status requests
+	// completing. So this pending conns cancel comes too late to interrupt
+	// final status requests in the orderly shutdown case -- which is desired
+	// since we give those a short timeout and would prefer to not interrupt
+	// them.
+	controller.untunneledPendingConns.CloseAll()
+
 	controller.splitTunnelClassifier.Shutdown()
 
 	NoticeInfo("exiting controller")
@@ -296,7 +312,7 @@ loop:
 		reported := false
 		tunnel := controller.getNextActiveTunnel()
 		if tunnel != nil {
-			err := tunnel.session.DoConnectedRequest()
+			err := tunnel.serverContext.DoConnectedRequest()
 			if err == nil {
 				reported = true
 			} else {
@@ -365,7 +381,7 @@ loop:
 			if err == nil {
 				break loop
 			}
-			NoticeAlert("upgrade download failed: ", err)
+			NoticeAlert("upgrade download failed: %s", err)
 		}
 
 		timeout := time.After(DOWNLOAD_UPGRADE_RETRY_PAUSE_PERIOD)
@@ -380,8 +396,10 @@ loop:
 	NoticeInfo("exiting upgrade downloader")
 }
 
-func (controller *Controller) startClientUpgradeDownloader(session *Session) {
-	// session is nil when DisableApi is set
+func (controller *Controller) startClientUpgradeDownloader(
+	serverContext *ServerContext) {
+
+	// serverContext is nil when DisableApi is set
 	if controller.config.DisableApi {
 		return
 	}
@@ -392,7 +410,7 @@ func (controller *Controller) startClientUpgradeDownloader(session *Session) {
 		return
 	}
 
-	if session.clientUpgradeVersion == "" {
+	if serverContext.clientUpgradeVersion == "" {
 		// No upgrade is offered
 		return
 	}
@@ -402,7 +420,7 @@ func (controller *Controller) startClientUpgradeDownloader(session *Session) {
 	if !controller.startedUpgradeDownloader {
 		controller.startedUpgradeDownloader = true
 		controller.runWaitGroup.Add(1)
-		go controller.upgradeDownloader(session.clientUpgradeVersion)
+		go controller.upgradeDownloader(serverContext.clientUpgradeVersion)
 	}
 }
 
@@ -439,7 +457,7 @@ loop:
 			// establishPendingConns; this causes the pendingConns.Add() within
 			// interruptibleTCPDial to succeed instead of aborting, and the result
 			// is that it's possible for establish goroutines to run all the way through
-			// NewSession before being discarded... delaying shutdown.
+			// NewServerContext before being discarded... delaying shutdown.
 			select {
 			case <-controller.shutdownBroadcast:
 				break loop
@@ -481,7 +499,8 @@ loop:
 					// tunnel is established.
 					controller.startOrSignalConnectedReporter()
 
-					controller.startClientUpgradeDownloader(establishedTunnel.session)
+					controller.startClientUpgradeDownloader(
+						establishedTunnel.serverContext)
 				}
 
 			} else {
@@ -516,7 +535,7 @@ loop:
 
 // classifyImpairedProtocol tracks "impaired" protocol classifications for failed
 // tunnels. A protocol is classified as impaired if a tunnel using that protocol
-// fails, repeatedly, shortly after the start of the session. During tunnel
+// fails, repeatedly, shortly after the start of the connection. During tunnel
 // establishment, impaired protocols are briefly skipped.
 //
 // One purpose of this measure is to defend against an attack where the adversary,
@@ -527,7 +546,7 @@ loop:
 //
 // Concurrency note: only the runTunnels() goroutine may call classifyImpairedProtocol
 func (controller *Controller) classifyImpairedProtocol(failedTunnel *Tunnel) {
-	if failedTunnel.sessionStartTime.Add(IMPAIRED_PROTOCOL_CLASSIFICATION_DURATION).After(time.Now()) {
+	if failedTunnel.startTime.Add(IMPAIRED_PROTOCOL_CLASSIFICATION_DURATION).After(time.Now()) {
 		controller.impairedProtocolClassification[failedTunnel.protocol] += 1
 	} else {
 		controller.impairedProtocolClassification[failedTunnel.protocol] = 0
@@ -581,7 +600,7 @@ func (controller *Controller) discardTunnel(tunnel *Tunnel) {
 	// discarded tunnel before fully active tunnels. Can a discarded tunnel
 	// be promoted (since it connects), but with lower rank than all active
 	// tunnels?
-	tunnel.Close()
+	tunnel.Close(true)
 }
 
 // registerTunnel adds the connected tunnel to the pool of active tunnels
@@ -605,6 +624,14 @@ func (controller *Controller) registerTunnel(tunnel *Tunnel) (int, bool) {
 	controller.tunnels = append(controller.tunnels, tunnel)
 	NoticeTunnels(len(controller.tunnels))
 
+	// Promote this successful tunnel to first rank so it's one
+	// of the first candidates next time establish runs.
+	// Connecting to a TargetServerEntry does not change the
+	// ranking.
+	if controller.config.TargetServerEntry == "" {
+		PromoteServerEntry(tunnel.serverEntry.IpAddress)
+	}
+
 	return len(controller.tunnels), true
 }
 
@@ -640,7 +667,7 @@ func (controller *Controller) terminateTunnel(tunnel *Tunnel) {
 			if controller.nextTunnel >= len(controller.tunnels) {
 				controller.nextTunnel = 0
 			}
-			activeTunnel.Close()
+			activeTunnel.Close(false)
 			NoticeTunnels(len(controller.tunnels))
 			break
 		}
@@ -661,7 +688,7 @@ func (controller *Controller) terminateAllTunnels() {
 		tunnel := activeTunnel
 		go func() {
 			defer closeWaitGroup.Done()
-			tunnel.Close()
+			tunnel.Close(false)
 		}()
 	}
 	closeWaitGroup.Wait()
@@ -748,9 +775,36 @@ func (controller *Controller) startEstablishing() {
 	controller.isEstablishing = true
 	controller.establishWaitGroup = new(sync.WaitGroup)
 	controller.stopEstablishingBroadcast = make(chan struct{})
-	controller.candidateServerEntries = make(chan *ServerEntry)
+	controller.candidateServerEntries = make(chan *candidateServerEntry)
 	controller.establishPendingConns.Reset()
 
+	// The server affinity mechanism attempts to favor the previously
+	// used server when reconnecting. This is beneficial for user
+	// applications which expect consistency in user IP address (for
+	// example, a web site which prompts for additional user
+	// authentication when the IP address changes).
+	//
+	// Only the very first server, as determined by
+	// datastore.PromoteServerEntry(), is the server affinity candidate.
+	// Concurrent connections attempts to many servers are launched
+	// without delay, in case the affinity server connection fails.
+	// While the affinity server connection is outstanding, when any
+	// other connection is established, there is a short grace period
+	// delay before delivering the established tunnel; this allows some
+	// time for the affinity server connection to succeed first.
+	// When the affinity server connection fails, any other established
+	// tunnel is registered without delay.
+	//
+	// Note: the establishTunnelWorker that receives the affinity
+	// candidate is solely resonsible for closing
+	// controller.serverAffinityDoneBroadcast.
+	//
+	// Note: if config.EgressRegion or config.TunnelProtocol has changed
+	// since the top server was promoted, the first server may not actually
+	// be the last connected server.
+	// TODO: should not favor the first server in this case
+	controller.serverAffinityDoneBroadcast = make(chan struct{})
+
 	for i := 0; i < controller.config.ConnectionWorkerPoolSize; i++ {
 		controller.establishWaitGroup.Add(1)
 		go controller.establishTunnelWorker()
@@ -781,6 +835,7 @@ func (controller *Controller) stopEstablishing() {
 	controller.establishWaitGroup = nil
 	controller.stopEstablishingBroadcast = nil
 	controller.candidateServerEntries = nil
+	controller.serverAffinityDoneBroadcast = nil
 }
 
 // establishCandidateGenerator populates the candidate queue with server entries
@@ -798,6 +853,14 @@ func (controller *Controller) establishCandidateGenerator(impairedProtocols []st
 	}
 	defer iterator.Close()
 
+	isServerAffinityCandidate := true
+
+	// TODO: reconcile server affinity scheme with multi-tunnel mode
+	if controller.config.TunnelPoolSize > 1 {
+		isServerAffinityCandidate = false
+		close(controller.serverAffinityDoneBroadcast)
+	}
+
 loop:
 	// Repeat until stopped
 	for i := 0; ; i++ {
@@ -827,7 +890,7 @@ loop:
 			// first iteration of the ESTABLISH_TUNNEL_WORK_TIME
 			// loop since (a) one iteration should be sufficient to
 			// evade the attack; (b) there's a good chance of false
-			// positives (such as short session durations due to network
+			// positives (such as short tunnel durations due to network
 			// hopping on a mobile device).
 			// Impaired protocols logic is not applied when
 			// config.TunnelProtocol is specified.
@@ -843,11 +906,16 @@ loop:
 				}
 			}
 
+			// Note: there must be only one server affinity candidate, as it
+			// closes the serverAffinityDoneBroadcast channel.
+			candidate := &candidateServerEntry{serverEntry, isServerAffinityCandidate}
+			isServerAffinityCandidate = false
+
 			// TODO: here we could generate multiple candidates from the
 			// server entry when there are many MeekFrontingAddresses.
 
 			select {
-			case controller.candidateServerEntries <- serverEntry:
+			case controller.candidateServerEntries <- candidate:
 			case <-controller.stopEstablishingBroadcast:
 				break loop
 			case <-controller.shutdownBroadcast:
@@ -901,7 +969,7 @@ loop:
 func (controller *Controller) establishTunnelWorker() {
 	defer controller.establishWaitGroup.Done()
 loop:
-	for serverEntry := range controller.candidateServerEntries {
+	for candidateServerEntry := range controller.candidateServerEntries {
 		// Note: don't receive from candidateServerEntries and stopEstablishingBroadcast
 		// in the same select, since we want to prioritize receiving the stop signal
 		if controller.isStopEstablishingBroadcast() {
@@ -909,26 +977,44 @@ loop:
 		}
 
 		// There may already be a tunnel to this candidate. If so, skip it.
-		if controller.isActiveTunnelServerEntry(serverEntry) {
+		if controller.isActiveTunnelServerEntry(candidateServerEntry.serverEntry) {
 			continue
 		}
 
 		tunnel, err := EstablishTunnel(
 			controller.config,
+			controller.untunneledDialConfig,
 			controller.sessionId,
 			controller.establishPendingConns,
-			serverEntry,
+			candidateServerEntry.serverEntry,
 			controller) // TunnelOwner
 		if err != nil {
+
+			// Unblock other candidates immediately when
+			// server affinity candidate fails.
+			if candidateServerEntry.isServerAffinityCandidate {
+				close(controller.serverAffinityDoneBroadcast)
+			}
+
 			// Before emitting error, check if establish interrupted, in which
 			// case the error is noise.
 			if controller.isStopEstablishingBroadcast() {
 				break loop
 			}
-			NoticeInfo("failed to connect to %s: %s", serverEntry.IpAddress, err)
+			NoticeInfo("failed to connect to %s: %s", candidateServerEntry.serverEntry.IpAddress, err)
 			continue
 		}
 
+		// Block for server affinity grace period before delivering.
+		if !candidateServerEntry.isServerAffinityCandidate {
+			timer := time.NewTimer(ESTABLISH_TUNNEL_SERVER_AFFINITY_GRACE_PERIOD)
+			select {
+			case <-timer.C:
+			case <-controller.serverAffinityDoneBroadcast:
+			case <-controller.stopEstablishingBroadcast:
+			}
+		}
+
 		// Deliver established tunnel.
 		// Don't block. Assumes the receiver has a buffer large enough for
 		// the number of desired tunnels. If there's no room, the tunnel must
@@ -938,6 +1024,12 @@ loop:
 		default:
 			controller.discardTunnel(tunnel)
 		}
+
+		// Unblock other candidates only after delivering when
+		// server affinity candidate succeeds.
+		if candidateServerEntry.isServerAffinityCandidate {
+			close(controller.serverAffinityDoneBroadcast)
+		}
 	}
 	NoticeInfo("stopped establish worker")
 }

psiphon/dataStore.go

@@ -1,5 +1,3 @@
-// +build windows
-
 /*
  * Copyright (c) 2015, Psiphon Inc.
  * All rights reserved.
@@ -22,24 +20,45 @@
 package psiphon
 
 import (
-	"database/sql"
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"math/rand"
+	"os"
 	"path/filepath"
 	"strings"
 	"sync"
 	"time"
 
-	sqlite3 "github.com/Psiphon-Inc/go-sqlite3"
+	"github.com/Psiphon-Inc/bolt"
 )
 
+// The BoltDB dataStore implementation is an alternative to the sqlite3-based
+// implementation in dataStore.go. Both implementations have the same interface.
+//
+// BoltDB is pure Go, and is intended to be used in cases where we have trouble
+// building sqlite3/CGO (e.g., currently go mobile due to
+// https://github.com/mattn/go-sqlite3/issues/201), and perhaps ultimately as
+// the primary dataStore implementation.
+//
 type dataStore struct {
 	init sync.Once
-	db   *sql.DB
+	db   *bolt.DB
 }
 
+const (
+	serverEntriesBucket         = "serverEntries"
+	rankedServerEntriesBucket   = "rankedServerEntries"
+	rankedServerEntriesKey      = "rankedServerEntries"
+	splitTunnelRouteETagsBucket = "splitTunnelRouteETags"
+	splitTunnelRouteDataBucket  = "splitTunnelRouteData"
+	urlETagsBucket              = "urlETags"
+	keyValueBucket              = "keyValues"
+	tunnelStatsBucket           = "tunnelStats"
+	rankedServerEntryCount      = 100
+)
+
 var singleton dataStore
 
 // InitDataStore initializes the singleton instance of dataStore. This
@@ -52,58 +71,73 @@ var singleton dataStore
 // have been replaced by checkInitDataStore() to assert that Init was called.
 func InitDataStore(config *Config) (err error) {
 	singleton.init.Do(func() {
+		// Need to gather the list of migratable server entries before
+		// initializing the boltdb store (as prepareMigrationEntries
+		// checks for the existence of the bolt db file)
+		migratableServerEntries := prepareMigrationEntries(config)
+
 		filename := filepath.Join(config.DataStoreDirectory, DATA_STORE_FILENAME)
-		var db *sql.DB
-		db, err = sql.Open(
-			"sqlite3",
-			fmt.Sprintf("file:%s?cache=private&mode=rwc", filename))
+		var db *bolt.DB
+		db, err = bolt.Open(filename, 0600, &bolt.Options{Timeout: 1 * time.Second})
+
+		// The datastore file may be corrupt, so attempt to delete and try again
+		if err != nil {
+			NoticeAlert("retry on initDataStore error: %s", err)
+			os.Remove(filename)
+			db, err = bolt.Open(filename, 0600, &bolt.Options{Timeout: 1 * time.Second})
+		}
+
 		if err != nil {
 			// Note: intending to set the err return value for InitDataStore
 			err = fmt.Errorf("initDataStore failed to open database: %s", err)
 			return
 		}
-		initialization := "pragma journal_mode=WAL;\n"
-		if config.DataStoreTempDirectory != "" {
-			// On some platforms (e.g., Android), the standard temporary directories expected
-			// by sqlite (see unixGetTempname in aggregate sqlite3.c) may not be present.
-			// In that case, sqlite tries to use the current working directory; but this may
-			// be "/" (again, on Android) which is not writable.
-			// Instead of setting the process current working directory from this library,
-			// use the deprecated temp_store_directory pragma to force use of a specified
-			// temporary directory: https://www.sqlite.org/pragma.html#pragma_temp_store_directory.
-			// TODO: is there another way to restrict writing of temporary files? E.g. temp_store=3?
-			initialization += fmt.Sprintf(
-				"pragma temp_store_directory=\"%s\";\n", config.DataStoreTempDirectory)
-		}
-		initialization += `
-        create table if not exists serverEntry
-            (id text not null primary key,
-             rank integer not null unique,
-             region text not null,
-             data blob not null);
-        create index if not exists idx_serverEntry_region on serverEntry(region);
-        create table if not exists serverEntryProtocol
-            (serverEntryId text not null,
-             protocol text not null,
-             primary key (serverEntryId, protocol));
-        create table if not exists splitTunnelRoutes
-            (region text not null primary key,
-             etag text not null,
-             data blob not null);
-        create table if not exists urlETags
-            (url text not null primary key,
-             etag text not null);
-        create table if not exists keyValue
-            (key text not null primary key,
-             value text not null);
-        `
-		_, err = db.Exec(initialization)
+
+		err = db.Update(func(tx *bolt.Tx) error {
+			requiredBuckets := []string{
+				serverEntriesBucket,
+				rankedServerEntriesBucket,
+				splitTunnelRouteETagsBucket,
+				splitTunnelRouteDataBucket,
+				urlETagsBucket,
+				keyValueBucket,
+				tunnelStatsBucket,
+			}
+			for _, bucket := range requiredBuckets {
+				_, err := tx.CreateBucketIfNotExists([]byte(bucket))
+				if err != nil {
+					return err
+				}
+			}
+			return nil
+		})
 		if err != nil {
-			err = fmt.Errorf("initDataStore failed to initialize: %s", err)
+			err = fmt.Errorf("initDataStore failed to create buckets: %s", err)
 			return
 		}
+
+		// Run consistency checks on datastore and emit errors for diagnostics purposes
+		// We assume this will complete quickly for typical size Psiphon datastores.
+		db.View(func(tx *bolt.Tx) error {
+			err := <-tx.Check()
+			if err != nil {
+				NoticeAlert("boltdb Check(): %s", err)
+			}
+			return nil
+		})
+
 		singleton.db = db
+
+		// The migrateServerEntries function requires the data store is
+		// initialized prior to execution so that migrated entries can be stored
+
+		if len(migratableServerEntries) > 0 {
+			migrateEntries(migratableServerEntries, filepath.Join(config.DataStoreDirectory, LEGACY_DATA_STORE_FILENAME))
+		}
+
+		resetAllTunnelStatsToUnreported()
 	})
+
 	return err
 }
 
@@ -113,71 +147,17 @@ func checkInitDataStore() {
 	}
 }
 
-func canRetry(err error) bool {
-	sqlError, ok := err.(sqlite3.Error)
-	return ok && (sqlError.Code == sqlite3.ErrBusy ||
-		sqlError.Code == sqlite3.ErrLocked ||
-		sqlError.ExtendedCode == sqlite3.ErrLockedSharedCache ||
-		sqlError.ExtendedCode == sqlite3.ErrBusySnapshot)
-}
-
-// transactionWithRetry will retry a write transaction if sqlite3
-// reports a table is locked by another writer.
-func transactionWithRetry(updater func(*sql.Tx) error) error {
-	checkInitDataStore()
-	for i := 0; i < 10; i++ {
-		if i > 0 {
-			// Delay on retry
-			time.Sleep(100)
-		}
-		transaction, err := singleton.db.Begin()
-		if err != nil {
-			return ContextError(err)
-		}
-		err = updater(transaction)
-		if err != nil {
-			transaction.Rollback()
-			if canRetry(err) {
-				continue
-			}
-			return ContextError(err)
-		}
-		err = transaction.Commit()
-		if err != nil {
-			transaction.Rollback()
-			if canRetry(err) {
-				continue
-			}
-			return ContextError(err)
-		}
-		return nil
-	}
-	return ContextError(errors.New("retries exhausted"))
-}
-
-// serverEntryExists returns true if a serverEntry with the
-// given ipAddress id already exists.
-func serverEntryExists(transaction *sql.Tx, ipAddress string) (bool, error) {
-	query := "select count(*) from serverEntry where id  = ?;"
-	var count int
-	err := singleton.db.QueryRow(query, ipAddress).Scan(&count)
-	if err != nil {
-		return false, ContextError(err)
-	}
-	return count > 0, nil
-}
-
 // StoreServerEntry adds the server entry to the data store.
 // A newly stored (or re-stored) server entry is assigned the next-to-top
 // rank for iteration order (the previous top ranked entry is promoted). The
 // purpose of inserting at next-to-top is to keep the last selected server
-// as the top ranked server. Note, server candidates are iterated in decending
-// rank order, so the largest rank is top rank.
+// as the top ranked server.
 // When replaceIfExists is true, an existing server entry record is
 // overwritten; otherwise, the existing record is unchanged.
 // If the server entry data is malformed, an alert notice is issued and
 // the entry is skipped; no error is returned.
 func StoreServerEntry(serverEntry *ServerEntry, replaceIfExists bool) error {
+	checkInitDataStore()
 
 	// Server entries should already be validated before this point,
 	// so instead of skipping we fail with an error.
@@ -186,63 +166,60 @@ func StoreServerEntry(serverEntry *ServerEntry, replaceIfExists bool) error {
 		return ContextError(errors.New("invalid server entry"))
 	}
 
-	return transactionWithRetry(func(transaction *sql.Tx) error {
-		serverEntryExists, err := serverEntryExists(transaction, serverEntry.IpAddress)
-		if err != nil {
-			return ContextError(err)
+	// BoltDB implementation note:
+	// For simplicity, we don't maintain indexes on server entry
+	// region or supported protocols. Instead, we perform full-bucket
+	// scans with a filter. With a small enough database (thousands or
+	// even tens of thousand of server entries) and common enough
+	// values (e.g., many servers support all protocols), performance
+	// is expected to be acceptable.
+
+	err = singleton.db.Update(func(tx *bolt.Tx) error {
+
+		serverEntries := tx.Bucket([]byte(serverEntriesBucket))
+
+		// Check not only that the entry exists, but is valid. This
+		// will replace in the rare case where the data is corrupt.
+		existingServerEntryValid := false
+		existingData := serverEntries.Get([]byte(serverEntry.IpAddress))
+		if existingData != nil {
+			existingServerEntry := new(ServerEntry)
+			if json.Unmarshal(existingData, existingServerEntry) == nil {
+				existingServerEntryValid = true
+			}
 		}
-		if serverEntryExists && !replaceIfExists {
+
+		if existingServerEntryValid && !replaceIfExists {
 			// Disabling this notice, for now, as it generates too much noise
 			// in diagnostics with clients that always submit embedded servers
 			// to the core on each run.
 			// NoticeInfo("ignored update for server %s", serverEntry.IpAddress)
 			return nil
 		}
-		_, err = transaction.Exec(`
-            update serverEntry set rank = rank + 1
-                where id = (select id from serverEntry order by rank desc limit 1);
-            `)
-		if err != nil {
-			// Note: ContextError() would break canRetry()
-			return err
-		}
+
 		data, err := json.Marshal(serverEntry)
 		if err != nil {
 			return ContextError(err)
 		}
-		_, err = transaction.Exec(`
-            insert or replace into serverEntry (id, rank, region, data)
-            values (?, (select coalesce(max(rank)-1, 0) from serverEntry), ?, ?);
-            `, serverEntry.IpAddress, serverEntry.Region, data)
+		err = serverEntries.Put([]byte(serverEntry.IpAddress), data)
 		if err != nil {
-			return err
+			return ContextError(err)
 		}
-		_, err = transaction.Exec(`
-            delete from serverEntryProtocol where serverEntryId = ?;
-            `, serverEntry.IpAddress)
+
+		err = insertRankedServerEntry(tx, serverEntry.IpAddress, 1)
 		if err != nil {
-			return err
-		}
-		for _, protocol := range SupportedTunnelProtocols {
-			// Note: for meek, the capabilities are FRONTED-MEEK and UNFRONTED-MEEK
-			// and the additonal OSSH service is assumed to be available internally.
-			requiredCapability := strings.TrimSuffix(protocol, "-OSSH")
-			if Contains(serverEntry.Capabilities, requiredCapability) {
-				_, err = transaction.Exec(`
-                    insert into serverEntryProtocol (serverEntryId, protocol)
-                    values (?, ?);
-                    `, serverEntry.IpAddress, protocol)
-				if err != nil {
-					return err
-				}
-			}
-		}
-		// TODO: post notice after commit
-		if !serverEntryExists {
-			NoticeInfo("updated server %s", serverEntry.IpAddress)
+			return ContextError(err)
 		}
+
+		NoticeInfo("updated server %s", serverEntry.IpAddress)
+
 		return nil
 	})
+	if err != nil {
+		return ContextError(err)
+	}
+
+	return nil
 }
 
 // StoreServerEntries shuffles and stores a list of server entries.
@@ -250,6 +227,7 @@ func StoreServerEntry(serverEntry *ServerEntry, replaceIfExists bool) error {
 // load balancing.
 // There is an independent transaction for each entry insert/update.
 func StoreServerEntries(serverEntries []*ServerEntry, replaceIfExists bool) error {
+	checkInitDataStore()
 
 	for index := len(serverEntries) - 1; index > 0; index-- {
 		swapIndex := rand.Intn(index + 1)
@@ -275,18 +253,110 @@ func StoreServerEntries(serverEntries []*ServerEntry, replaceIfExists bool) erro
 // iterated in decending rank order, so this server entry will be
 // the first candidate in a subsequent tunnel establishment.
 func PromoteServerEntry(ipAddress string) error {
-	return transactionWithRetry(func(transaction *sql.Tx) error {
-		_, err := transaction.Exec(`
-            update serverEntry
-            set rank = (select MAX(rank)+1 from serverEntry)
-            where id = ?;
-            `, ipAddress)
-		if err != nil {
-			// Note: ContextError() would break canRetry()
-			return err
+	checkInitDataStore()
+
+	err := singleton.db.Update(func(tx *bolt.Tx) error {
+
+		// Ensure the corresponding entry exists before
+		// inserting into rank.
+		bucket := tx.Bucket([]byte(serverEntriesBucket))
+		data := bucket.Get([]byte(ipAddress))
+		if data == nil {
+			NoticeAlert(
+				"PromoteServerEntry: ignoring unknown server entry: %s",
+				ipAddress)
+			return nil
 		}
-		return nil
+
+		return insertRankedServerEntry(tx, ipAddress, 0)
 	})
+
+	if err != nil {
+		return ContextError(err)
+	}
+	return nil
+}
+
+func getRankedServerEntries(tx *bolt.Tx) ([]string, error) {
+	bucket := tx.Bucket([]byte(rankedServerEntriesBucket))
+	data := bucket.Get([]byte(rankedServerEntriesKey))
+
+	if data == nil {
+		return []string{}, nil
+	}
+
+	rankedServerEntries := make([]string, 0)
+	err := json.Unmarshal(data, &rankedServerEntries)
+	if err != nil {
+		return nil, ContextError(err)
+	}
+	return rankedServerEntries, nil
+}
+
+func setRankedServerEntries(tx *bolt.Tx, rankedServerEntries []string) error {
+	data, err := json.Marshal(rankedServerEntries)
+	if err != nil {
+		return ContextError(err)
+	}
+
+	bucket := tx.Bucket([]byte(rankedServerEntriesBucket))
+	err = bucket.Put([]byte(rankedServerEntriesKey), data)
+	if err != nil {
+		return ContextError(err)
+	}
+
+	return nil
+}
+
+func insertRankedServerEntry(tx *bolt.Tx, serverEntryId string, position int) error {
+	rankedServerEntries, err := getRankedServerEntries(tx)
+	if err != nil {
+		return ContextError(err)
+	}
+
+	// BoltDB implementation note:
+	// For simplicity, we store the ranked server ids in an array serialized to
+	// a single key value. To ensure this value doesn't grow without bound,
+	// it's capped at rankedServerEntryCount. For now, this cap should be large
+	// enough to meet the shuffleHeadLength = config.TunnelPoolSize criteria, for
+	// any reasonable configuration of config.TunnelPoolSize.
+
+	// Using: https://github.com/golang/go/wiki/SliceTricks
+
+	// When serverEntryId is already ranked, remove it first to avoid duplicates
+
+	for i, rankedServerEntryId := range rankedServerEntries {
+		if rankedServerEntryId == serverEntryId {
+			rankedServerEntries = append(
+				rankedServerEntries[:i], rankedServerEntries[i+1:]...)
+			break
+		}
+	}
+
+	// SliceTricks insert, with length cap enforced
+
+	if len(rankedServerEntries) < rankedServerEntryCount {
+		rankedServerEntries = append(rankedServerEntries, "")
+	}
+	if position >= len(rankedServerEntries) {
+		position = len(rankedServerEntries) - 1
+	}
+	copy(rankedServerEntries[position+1:], rankedServerEntries[position:])
+	rankedServerEntries[position] = serverEntryId
+
+	err = setRankedServerEntries(tx, rankedServerEntries)
+	if err != nil {
+		return ContextError(err)
+	}
+
+	return nil
+}
+
+func serverEntrySupportsProtocol(serverEntry *ServerEntry, protocol string) bool {
+	// Note: for meek, the capabilities are FRONTED-MEEK and UNFRONTED-MEEK
+	// and the additonal OSSH service is assumed to be available internally.
+	requiredCapability := strings.TrimSuffix(protocol, "-OSSH")
+	return Contains(serverEntry.Capabilities, requiredCapability)
 }
 
 // ServerEntryIterator is used to iterate over
@@ -295,14 +365,14 @@ type ServerEntryIterator struct {
 	region                      string
 	protocol                    string
 	shuffleHeadLength           int
-	transaction                 *sql.Tx
-	cursor                      *sql.Rows
+	serverEntryIds              []string
+	serverEntryIndex            int
 	isTargetServerEntryIterator bool
 	hasNextTargetServerEntry    bool
 	targetServerEntry           *ServerEntry
 }
 
-// NewServerEntryIterator creates a new NewServerEntryIterator
+// NewServerEntryIterator creates a new ServerEntryIterator
 func NewServerEntryIterator(config *Config) (iterator *ServerEntryIterator, err error) {
 
 	// When configured, this target server entry is the only candidate
@@ -362,54 +432,69 @@ func (iterator *ServerEntryIterator) Reset() error {
 	count := CountServerEntries(iterator.region, iterator.protocol)
 	NoticeCandidateServers(iterator.region, iterator.protocol, count)
 
-	transaction, err := singleton.db.Begin()
-	if err != nil {
-		return ContextError(err)
-	}
-	var cursor *sql.Rows
-
 	// This query implements the Psiphon server candidate selection
 	// algorithm: the first TunnelPoolSize server candidates are in rank
 	// (priority) order, to favor previously successful servers; then the
 	// remaining long tail is shuffled to raise up less recent candidates.
 
-	whereClause, whereParams := makeServerEntryWhereClause(
-		iterator.region, iterator.protocol, nil)
-	headLength := iterator.shuffleHeadLength
-	queryFormat := `
-		select data from serverEntry %s
-		order by case
-		when rank > coalesce((select rank from serverEntry %s order by rank desc limit ?, 1), -1) then rank
-		else abs(random())%%((select rank from serverEntry %s order by rank desc limit ?, 1))
-		end desc;`
-	query := fmt.Sprintf(queryFormat, whereClause, whereClause, whereClause)
-	params := make([]interface{}, 0)
-	params = append(params, whereParams...)
-	params = append(params, whereParams...)
-	params = append(params, headLength)
-	params = append(params, whereParams...)
-	params = append(params, headLength)
-
-	cursor, err = transaction.Query(query, params...)
+	// BoltDB implementation note:
+	// We don't keep a transaction open for the duration of the iterator
+	// because this would expose the following semantics to consumer code:
+	//
+	//     Read-only transactions and read-write transactions ... generally
+	//     shouldn't be opened simultaneously in the same goroutine. This can
+	//     cause a deadlock as the read-write transaction needs to periodically
+	//     re-map the data file but it cannot do so while a read-only
+	//     transaction is open.
+	//     (https://github.com/boltdb/bolt)
+	//
+	// So the underlying serverEntriesBucket could change after the serverEntryIds
+	// list is built.
+
+	var serverEntryIds []string
+
+	err := singleton.db.View(func(tx *bolt.Tx) error {
+		var err error
+		serverEntryIds, err = getRankedServerEntries(tx)
+		if err != nil {
+			return err
+		}
+
+		skipServerEntryIds := make(map[string]bool)
+		for _, serverEntryId := range serverEntryIds {
+			skipServerEntryIds[serverEntryId] = true
+		}
+
+		bucket := tx.Bucket([]byte(serverEntriesBucket))
+		cursor := bucket.Cursor()
+		for key, _ := cursor.Last(); key != nil; key, _ = cursor.Prev() {
+			serverEntryId := string(key)
+			if _, ok := skipServerEntryIds[serverEntryId]; ok {
+				continue
+			}
+			serverEntryIds = append(serverEntryIds, serverEntryId)
+		}
+		return nil
+	})
 	if err != nil {
-		transaction.Rollback()
 		return ContextError(err)
 	}
-	iterator.transaction = transaction
-	iterator.cursor = cursor
+
+	for i := len(serverEntryIds) - 1; i > iterator.shuffleHeadLength-1; i-- {
+		j := rand.Intn(i+1-iterator.shuffleHeadLength) + iterator.shuffleHeadLength
+		serverEntryIds[i], serverEntryIds[j] = serverEntryIds[j], serverEntryIds[i]
+	}
+
+	iterator.serverEntryIds = serverEntryIds
+	iterator.serverEntryIndex = 0
+
 	return nil
 }
 
 // Close cleans up resources associated with a ServerEntryIterator.
 func (iterator *ServerEntryIterator) Close() {
-	if iterator.cursor != nil {
-		iterator.cursor.Close()
-	}
-	iterator.cursor = nil
-	if iterator.transaction != nil {
-		iterator.transaction.Rollback()
-	}
-	iterator.transaction = nil
+	iterator.serverEntryIds = nil
+	iterator.serverEntryIndex = 0
 }
 
 // Next returns the next server entry, by rank, for a ServerEntryIterator.
@@ -429,24 +514,55 @@ func (iterator *ServerEntryIterator) Next() (serverEntry *ServerEntry, err error
 		return nil, nil
 	}
 
-	if !iterator.cursor.Next() {
-		err = iterator.cursor.Err()
+	// There are no region/protocol indexes for the server entries bucket.
+	// Loop until we have the next server entry that matches the iterator
+	// filter requirements.
+	for {
+		if iterator.serverEntryIndex >= len(iterator.serverEntryIds) {
+			// There is no next item
+			return nil, nil
+		}
+
+		serverEntryId := iterator.serverEntryIds[iterator.serverEntryIndex]
+		iterator.serverEntryIndex += 1
+
+		var data []byte
+		err = singleton.db.View(func(tx *bolt.Tx) error {
+			bucket := tx.Bucket([]byte(serverEntriesBucket))
+			value := bucket.Get([]byte(serverEntryId))
+			if value != nil {
+				// Must make a copy as slice is only valid within transaction.
+				data = make([]byte, len(value))
+				copy(data, value)
+			}
+			return nil
+		})
 		if err != nil {
 			return nil, ContextError(err)
 		}
-		// There is no next item
-		return nil, nil
-	}
 
-	var data []byte
-	err = iterator.cursor.Scan(&data)
-	if err != nil {
-		return nil, ContextError(err)
-	}
-	serverEntry = new(ServerEntry)
-	err = json.Unmarshal(data, serverEntry)
-	if err != nil {
-		return nil, ContextError(err)
+		if data == nil {
+			// In case of data corruption or a bug causing this condition,
+			// do not stop iterating.
+			NoticeAlert("ServerEntryIterator.Next: unexpected missing server entry: %s", serverEntryId)
+			continue
+		}
+
+		serverEntry = new(ServerEntry)
+		err = json.Unmarshal(data, serverEntry)
+		if err != nil {
+			// In case of data corruption or a bug causing this condition,
+			// do not stop iterating.
+			NoticeAlert("ServerEntryIterator.Next: %s", ContextError(err))
+			continue
+		}
+
+		// Check filter requirements
+		if (iterator.region == "" || serverEntry.Region == iterator.region) &&
+			(iterator.protocol == "" || serverEntrySupportsProtocol(serverEntry, iterator.protocol)) {
+
+			break
+		}
 	}
 
 	return MakeCompatibleServerEntry(serverEntry), nil
@@ -465,123 +581,95 @@ func MakeCompatibleServerEntry(serverEntry *ServerEntry) *ServerEntry {
 	return serverEntry
 }
 
-func makeServerEntryWhereClause(
-	region, protocol string, excludeIds []string) (whereClause string, whereParams []interface{}) {
-	whereClause = ""
-	whereParams = make([]interface{}, 0)
-	if region != "" {
-		whereClause += " where region = ?"
-		whereParams = append(whereParams, region)
-	}
-	if protocol != "" {
-		if len(whereClause) > 0 {
-			whereClause += " and"
-		} else {
-			whereClause += " where"
-		}
-		whereClause +=
-			" exists (select 1 from serverEntryProtocol where protocol = ? and serverEntryId = serverEntry.id)"
-		whereParams = append(whereParams, protocol)
-	}
-	if len(excludeIds) > 0 {
-		if len(whereClause) > 0 {
-			whereClause += " and"
-		} else {
-			whereClause += " where"
-		}
-		whereClause += " id in ("
-		for index, id := range excludeIds {
-			if index > 0 {
-				whereClause += ", "
+func scanServerEntries(scanner func(*ServerEntry)) error {
+	err := singleton.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(serverEntriesBucket))
+		cursor := bucket.Cursor()
+
+		for key, value := cursor.First(); key != nil; key, value = cursor.Next() {
+			serverEntry := new(ServerEntry)
+			err := json.Unmarshal(value, serverEntry)
+			if err != nil {
+				// In case of data corruption or a bug causing this condition,
+				// do not stop iterating.
+				NoticeAlert("scanServerEntries: %s", ContextError(err))
+				continue
 			}
-			whereClause += "?"
-			whereParams = append(whereParams, id)
+			scanner(serverEntry)
 		}
-		whereClause += ")"
+
+		return nil
+	})
+
+	if err != nil {
+		return ContextError(err)
 	}
-	return whereClause, whereParams
+
+	return nil
 }
 
 // CountServerEntries returns a count of stored servers for the
 // specified region and protocol.
 func CountServerEntries(region, protocol string) int {
 	checkInitDataStore()
-	var count int
-	whereClause, whereParams := makeServerEntryWhereClause(region, protocol, nil)
-	query := "select count(*) from serverEntry" + whereClause
-	err := singleton.db.QueryRow(query, whereParams...).Scan(&count)
+
+	count := 0
+	err := scanServerEntries(func(serverEntry *ServerEntry) {
+		if (region == "" || serverEntry.Region == region) &&
+			(protocol == "" || serverEntrySupportsProtocol(serverEntry, protocol)) {
+			count += 1
+		}
+	})
 
 	if err != nil {
 		NoticeAlert("CountServerEntries failed: %s", err)
 		return 0
 	}
 
-	if region == "" {
-		region = "(any)"
-	}
-	if protocol == "" {
-		protocol = "(any)"
-	}
-	NoticeInfo("servers for region %s and protocol %s: %d",
-		region, protocol, count)
-
 	return count
 }
 
 // ReportAvailableRegions prints a notice with the available egress regions.
+// Note that this report ignores config.TunnelProtocol.
 func ReportAvailableRegions() {
 	checkInitDataStore()
 
-	// TODO: For consistency, regions-per-protocol should be used
+	regions := make(map[string]bool)
+	err := scanServerEntries(func(serverEntry *ServerEntry) {
+		regions[serverEntry.Region] = true
+	})
 
-	rows, err := singleton.db.Query("select distinct(region) from serverEntry;")
 	if err != nil {
-		NoticeAlert("failed to query data store for available regions: %s", ContextError(err))
+		NoticeAlert("ReportAvailableRegions failed: %s", err)
 		return
 	}
-	defer rows.Close()
-
-	var regions []string
-
-	for rows.Next() {
-		var region string
-		err = rows.Scan(&region)
-		if err != nil {
-			NoticeAlert("failed to retrieve available regions from data store: %s", ContextError(err))
-			return
-		}
 
+	regionList := make([]string, 0, len(regions))
+	for region, _ := range regions {
 		// Some server entries do not have a region, but it makes no sense to return
 		// an empty string as an "available region".
 		if region != "" {
-			regions = append(regions, region)
+			regionList = append(regionList, region)
 		}
 	}
 
-	NoticeAvailableEgressRegions(regions)
+	NoticeAvailableEgressRegions(regionList)
 }
 
 // GetServerEntryIpAddresses returns an array containing
 // all stored server IP addresses.
 func GetServerEntryIpAddresses() (ipAddresses []string, err error) {
 	checkInitDataStore()
+
 	ipAddresses = make([]string, 0)
-	rows, err := singleton.db.Query("select id from serverEntry;")
+	err = scanServerEntries(func(serverEntry *ServerEntry) {
+		ipAddresses = append(ipAddresses, serverEntry.IpAddress)
+	})
+
 	if err != nil {
 		return nil, ContextError(err)
 	}
-	defer rows.Close()
-	for rows.Next() {
-		var ipAddress string
-		err = rows.Scan(&ipAddress)
-		if err != nil {
-			return nil, ContextError(err)
-		}
-		ipAddresses = append(ipAddresses, ipAddress)
-	}
-	if err = rows.Err(); err != nil {
-		return nil, ContextError(err)
-	}
+
 	return ipAddresses, nil
 }
 
@@ -589,28 +677,34 @@ func GetServerEntryIpAddresses() (ipAddresses []string, err error) {
 // the given region. The associated etag is also stored and
 // used to make efficient web requests for updates to the data.
 func SetSplitTunnelRoutes(region, etag string, data []byte) error {
-	return transactionWithRetry(func(transaction *sql.Tx) error {
-		_, err := transaction.Exec(`
-            insert or replace into splitTunnelRoutes (region, etag, data)
-            values (?, ?, ?);
-            `, region, etag, data)
-		if err != nil {
-			// Note: ContextError() would break canRetry()
-			return err
-		}
-		return nil
+	checkInitDataStore()
+
+	err := singleton.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(splitTunnelRouteETagsBucket))
+		err := bucket.Put([]byte(region), []byte(etag))
+
+		bucket = tx.Bucket([]byte(splitTunnelRouteDataBucket))
+		err = bucket.Put([]byte(region), data)
+		return err
 	})
+
+	if err != nil {
+		return ContextError(err)
+	}
+	return nil
 }
 
 // GetSplitTunnelRoutesETag retrieves the etag for cached routes
 // data for the specified region. If not found, it returns an empty string value.
 func GetSplitTunnelRoutesETag(region string) (etag string, err error) {
 	checkInitDataStore()
-	rows := singleton.db.QueryRow("select etag from splitTunnelRoutes where region = ?;", region)
-	err = rows.Scan(&etag)
-	if err == sql.ErrNoRows {
-		return "", nil
-	}
+
+	err = singleton.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(splitTunnelRouteETagsBucket))
+		etag = string(bucket.Get([]byte(region)))
+		return nil
+	})
+
 	if err != nil {
 		return "", ContextError(err)
 	}
@@ -621,11 +715,18 @@ func GetSplitTunnelRoutesETag(region string) (etag string, err error) {
 // for the specified region. If not found, it returns a nil value.
 func GetSplitTunnelRoutesData(region string) (data []byte, err error) {
 	checkInitDataStore()
-	rows := singleton.db.QueryRow("select data from splitTunnelRoutes where region = ?;", region)
-	err = rows.Scan(&data)
-	if err == sql.ErrNoRows {
-		return nil, nil
-	}
+
+	err = singleton.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(splitTunnelRouteDataBucket))
+		value := bucket.Get([]byte(region))
+		if value != nil {
+			// Must make a copy as slice is only valid within transaction.
+			data = make([]byte, len(value))
+			copy(data, value)
+		}
+		return nil
+	})
+
 	if err != nil {
 		return nil, ContextError(err)
 	}
@@ -636,28 +737,31 @@ func GetSplitTunnelRoutesData(region string) (data []byte, err error) {
 // Note: input URL is treated as a string, and is not
 // encoded or decoded or otherwise canonicalized.
 func SetUrlETag(url, etag string) error {
-	return transactionWithRetry(func(transaction *sql.Tx) error {
-		_, err := transaction.Exec(`
-            insert or replace into urlETags (url, etag)
-            values (?, ?);
-            `, url, etag)
-		if err != nil {
-			// Note: ContextError() would break canRetry()
-			return err
-		}
-		return nil
+	checkInitDataStore()
+
+	err := singleton.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(urlETagsBucket))
+		err := bucket.Put([]byte(url), []byte(etag))
+		return err
 	})
+
+	if err != nil {
+		return ContextError(err)
+	}
+	return nil
 }
 
 // GetUrlETag retrieves a previously stored an ETag for the
 // specfied URL. If not found, it returns an empty string value.
 func GetUrlETag(url string) (etag string, err error) {
 	checkInitDataStore()
-	rows := singleton.db.QueryRow("select etag from urlETags where url = ?;", url)
-	err = rows.Scan(&etag)
-	if err == sql.ErrNoRows {
-		return "", nil
-	}
+
+	err = singleton.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(urlETagsBucket))
+		etag = string(bucket.Get([]byte(url)))
+		return nil
+	})
+
 	if err != nil {
 		return "", ContextError(err)
 	}
@@ -666,30 +770,226 @@ func GetUrlETag(url string) (etag string, err error) {
 
 // SetKeyValue stores a key/value pair.
 func SetKeyValue(key, value string) error {
-	return transactionWithRetry(func(transaction *sql.Tx) error {
-		_, err := transaction.Exec(`
-            insert or replace into keyValue (key, value)
-            values (?, ?);
-            `, key, value)
-		if err != nil {
-			// Note: ContextError() would break canRetry()
-			return err
-		}
-		return nil
+	checkInitDataStore()
+
+	err := singleton.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(keyValueBucket))
+		err := bucket.Put([]byte(key), []byte(value))
+		return err
 	})
+
+	if err != nil {
+		return ContextError(err)
+	}
+	return nil
 }
 
 // GetKeyValue retrieves the value for a given key. If not found,
 // it returns an empty string value.
 func GetKeyValue(key string) (value string, err error) {
 	checkInitDataStore()
-	rows := singleton.db.QueryRow("select value from keyValue where key = ?;", key)
-	err = rows.Scan(&value)
-	if err == sql.ErrNoRows {
-		return "", nil
-	}
+
+	err = singleton.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(keyValueBucket))
+		value = string(bucket.Get([]byte(key)))
+		return nil
+	})
+
 	if err != nil {
 		return "", ContextError(err)
 	}
 	return value, nil
 }
+
+// Tunnel stats records in the tunnelStatsStateUnreported
+// state are available for take out.
+// Records in the tunnelStatsStateReporting have been
+// taken out and are pending either deleting (for a
+// successful request) or change to StateUnreported (for
+// a failed request).
+// All tunnel stats records are reverted to StateUnreported
+// when the datastore is initialized at start up.
+
+var tunnelStatsStateUnreported = []byte("0")
+var tunnelStatsStateReporting = []byte("1")
+
+// StoreTunnelStats adds a new tunnel stats record, which is
+// set to StateUnreported and is an immediate candidate for
+// reporting.
+// tunnelStats is a JSON byte array containing fields as
+// required by the Psiphon server API (see RecordTunnelStats).
+// It's assumed that the JSON value contains enough unique
+// information for the value to function as a key in the
+// key/value datastore. This assumption is currently satisfied
+// by the fields sessionId + tunnelNumber.
+func StoreTunnelStats(tunnelStats []byte) error {
+	checkInitDataStore()
+
+	err := singleton.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(tunnelStatsBucket))
+		err := bucket.Put(tunnelStats, tunnelStatsStateUnreported)
+		return err
+	})
+
+	if err != nil {
+		return ContextError(err)
+	}
+	return nil
+}
+
+// CountUnreportedTunnelStats returns the number of tunnel
+// stats records in StateUnreported.
+func CountUnreportedTunnelStats() int {
+	checkInitDataStore()
+
+	unreported := 0
+
+	err := singleton.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(tunnelStatsBucket))
+		cursor := bucket.Cursor()
+		for key, value := cursor.First(); key != nil; key, value = cursor.Next() {
+			if 0 == bytes.Compare(value, tunnelStatsStateUnreported) {
+				unreported++
+				break
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		NoticeAlert("CountUnreportedTunnelStats failed: %s", err)
+		return 0
+	}
+
+	return unreported
+}
+
+// TakeOutUnreportedTunnelStats returns up to maxCount tunnel
+// stats records that are in StateUnreported. The records are set
+// to StateReporting. If the records are successfully reported,
+// clear them with ClearReportedTunnelStats. If the records are
+// not successfully reported, restore them with
+// PutBackUnreportedTunnelStats.
+func TakeOutUnreportedTunnelStats(maxCount int) ([][]byte, error) {
+	checkInitDataStore()
+
+	tunnelStats := make([][]byte, 0)
+
+	err := singleton.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(tunnelStatsBucket))
+		cursor := bucket.Cursor()
+		for key, value := cursor.First(); key != nil; key, value = cursor.Next() {
+
+			// Perform a test JSON unmarshaling. In case of data corruption or a bug,
+			// skip the record.
+			var jsonData interface{}
+			err := json.Unmarshal(key, &jsonData)
+			if err != nil {
+				NoticeAlert(
+					"Invalid key in TakeOutUnreportedTunnelStats: %s: %s",
+					string(key), err)
+				continue
+			}
+
+			if 0 == bytes.Compare(value, tunnelStatsStateUnreported) {
+				// Must make a copy as slice is only valid within transaction.
+				data := make([]byte, len(key))
+				copy(data, key)
+				tunnelStats = append(tunnelStats, data)
+				if len(tunnelStats) >= maxCount {
+					break
+				}
+			}
+		}
+		for _, key := range tunnelStats {
+			err := bucket.Put(key, tunnelStatsStateReporting)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, ContextError(err)
+	}
+	return tunnelStats, nil
+}
+
+// PutBackUnreportedTunnelStats restores a list of tunnel
+// stats records to StateUnreported.
+func PutBackUnreportedTunnelStats(tunnelStats [][]byte) error {
+	checkInitDataStore()
+
+	err := singleton.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(tunnelStatsBucket))
+		for _, key := range tunnelStats {
+			err := bucket.Put(key, tunnelStatsStateUnreported)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return ContextError(err)
+	}
+	return nil
+}
+
+// ClearReportedTunnelStats deletes a list of tunnel
+// stats records that were succesdfully reported.
+func ClearReportedTunnelStats(tunnelStats [][]byte) error {
+	checkInitDataStore()
+
+	err := singleton.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(tunnelStatsBucket))
+		for _, key := range tunnelStats {
+			err := bucket.Delete(key)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return ContextError(err)
+	}
+	return nil
+}
+
+// resetAllTunnelStatsToUnreported sets all tunnel
+// stats records to StateUnreported. This reset is called
+// when the datastore is initialized at start up, as we do
+// not know if tunnel records in StateReporting were reported
+// or not.
+func resetAllTunnelStatsToUnreported() error {
+	checkInitDataStore()
+
+	err := singleton.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(tunnelStatsBucket))
+		resetKeys := make([][]byte, 0)
+		cursor := bucket.Cursor()
+		for key, _ := cursor.First(); key != nil; key, _ = cursor.Next() {
+			resetKeys = append(resetKeys, key)
+		}
+		// TODO: data mutation is done outside cursor. Is this
+		// strictly necessary in this case?
+		// https://godoc.org/github.com/boltdb/bolt#Cursor
+		for _, key := range resetKeys {
+			err := bucket.Put(key, tunnelStatsStateUnreported)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return ContextError(err)
+	}
+	return nil
+}

psiphon/dataStore_alt.go

@@ -1,719 +0,0 @@
-// +build !windows
-
-/*
- * Copyright (c) 2015, Psiphon Inc.
- * All rights reserved.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- */
-
-package psiphon
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"math/rand"
-	"path/filepath"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/Psiphon-Inc/bolt"
-)
-
-// The BoltDB dataStore implementation is an alternative to the sqlite3-based
-// implementation in dataStore.go. Both implementations have the same interface.
-//
-// BoltDB is pure Go, and is intended to be used in cases where we have trouble
-// building sqlite3/CGO (e.g., currently go mobile due to
-// https://github.com/mattn/go-sqlite3/issues/201), and perhaps ultimately as
-// the primary dataStore implementation.
-//
-type dataStore struct {
-	init sync.Once
-	db   *bolt.DB
-}
-
-const (
-	serverEntriesBucket         = "serverEntries"
-	rankedServerEntriesBucket   = "rankedServerEntries"
-	rankedServerEntriesKey      = "rankedServerEntries"
-	splitTunnelRouteETagsBucket = "splitTunnelRouteETags"
-	splitTunnelRouteDataBucket  = "splitTunnelRouteData"
-	urlETagsBucket              = "urlETags"
-	keyValueBucket              = "keyValues"
-	rankedServerEntryCount      = 100
-)
-
-var singleton dataStore
-
-// InitDataStore initializes the singleton instance of dataStore. This
-// function uses a sync.Once and is safe for use by concurrent goroutines.
-// The underlying sql.DB connection pool is also safe.
-//
-// Note: the sync.Once was more useful when initDataStore was private and
-// called on-demand by the public functions below. Now we require an explicit
-// InitDataStore() call with the filename passed in. The on-demand calls
-// have been replaced by checkInitDataStore() to assert that Init was called.
-func InitDataStore(config *Config) (err error) {
-	singleton.init.Do(func() {
-		filename := filepath.Join(config.DataStoreDirectory, DATA_STORE_FILENAME)
-		var db *bolt.DB
-		db, err = bolt.Open(filename, 0600, &bolt.Options{Timeout: 1 * time.Second})
-		if err != nil {
-			// Note: intending to set the err return value for InitDataStore
-			err = fmt.Errorf("initDataStore failed to open database: %s", err)
-			return
-		}
-
-		err = db.Update(func(tx *bolt.Tx) error {
-			requiredBuckets := []string{
-				serverEntriesBucket,
-				rankedServerEntriesBucket,
-				splitTunnelRouteETagsBucket,
-				splitTunnelRouteDataBucket,
-				urlETagsBucket,
-				keyValueBucket,
-			}
-			for _, bucket := range requiredBuckets {
-				_, err := tx.CreateBucketIfNotExists([]byte(bucket))
-				if err != nil {
-					return err
-				}
-			}
-			return nil
-		})
-		if err != nil {
-			err = fmt.Errorf("initDataStore failed to create buckets: %s", err)
-			return
-		}
-
-		singleton.db = db
-	})
-	return err
-}
-
-func checkInitDataStore() {
-	if singleton.db == nil {
-		panic("checkInitDataStore: datastore not initialized")
-	}
-}
-
-// StoreServerEntry adds the server entry to the data store.
-// A newly stored (or re-stored) server entry is assigned the next-to-top
-// rank for iteration order (the previous top ranked entry is promoted). The
-// purpose of inserting at next-to-top is to keep the last selected server
-// as the top ranked server.
-// When replaceIfExists is true, an existing server entry record is
-// overwritten; otherwise, the existing record is unchanged.
-// If the server entry data is malformed, an alert notice is issued and
-// the entry is skipped; no error is returned.
-func StoreServerEntry(serverEntry *ServerEntry, replaceIfExists bool) error {
-	checkInitDataStore()
-
-	// Server entries should already be validated before this point,
-	// so instead of skipping we fail with an error.
-	err := ValidateServerEntry(serverEntry)
-	if err != nil {
-		return ContextError(errors.New("invalid server entry"))
-	}
-
-	// BoltDB implementation note:
-	// For simplicity, we don't maintain indexes on server entry
-	// region or supported protocols. Instead, we perform full-bucket
-	// scans with a filter. With a small enough database (thousands or
-	// even tens of thousand of server entries) and common enough
-	// values (e.g., many servers support all protocols), performance
-	// is expected to be acceptable.
-
-	serverEntryExists := false
-	err = singleton.db.Update(func(tx *bolt.Tx) error {
-
-		serverEntries := tx.Bucket([]byte(serverEntriesBucket))
-		serverEntryExists = (serverEntries.Get([]byte(serverEntry.IpAddress)) != nil)
-
-		if serverEntryExists && !replaceIfExists {
-			// Disabling this notice, for now, as it generates too much noise
-			// in diagnostics with clients that always submit embedded servers
-			// to the core on each run.
-			// NoticeInfo("ignored update for server %s", serverEntry.IpAddress)
-			return nil
-		}
-
-		data, err := json.Marshal(serverEntry)
-		if err != nil {
-			return ContextError(err)
-		}
-		err = serverEntries.Put([]byte(serverEntry.IpAddress), data)
-		if err != nil {
-			return ContextError(err)
-		}
-
-		err = insertRankedServerEntry(tx, serverEntry.IpAddress, 1)
-		if err != nil {
-			return ContextError(err)
-		}
-
-		return nil
-	})
-	if err != nil {
-		return ContextError(err)
-	}
-
-	if !serverEntryExists {
-		NoticeInfo("updated server %s", serverEntry.IpAddress)
-	}
-	return nil
-}
-
-// StoreServerEntries shuffles and stores a list of server entries.
-// Shuffling is performed on imported server entrues as part of client-side
-// load balancing.
-// There is an independent transaction for each entry insert/update.
-func StoreServerEntries(serverEntries []*ServerEntry, replaceIfExists bool) error {
-	checkInitDataStore()
-
-	for index := len(serverEntries) - 1; index > 0; index-- {
-		swapIndex := rand.Intn(index + 1)
-		serverEntries[index], serverEntries[swapIndex] = serverEntries[swapIndex], serverEntries[index]
-	}
-
-	for _, serverEntry := range serverEntries {
-		err := StoreServerEntry(serverEntry, replaceIfExists)
-		if err != nil {
-			return ContextError(err)
-		}
-	}
-
-	// Since there has possibly been a significant change in the server entries,
-	// take this opportunity to update the available egress regions.
-	ReportAvailableRegions()
-
-	return nil
-}
-
-// PromoteServerEntry assigns the top rank (one more than current
-// max rank) to the specified server entry. Server candidates are
-// iterated in decending rank order, so this server entry will be
-// the first candidate in a subsequent tunnel establishment.
-func PromoteServerEntry(ipAddress string) error {
-	checkInitDataStore()
-
-	err := singleton.db.Update(func(tx *bolt.Tx) error {
-		return insertRankedServerEntry(tx, ipAddress, 0)
-	})
-
-	if err != nil {
-		return ContextError(err)
-	}
-	return nil
-}
-
-func getRankedServerEntries(tx *bolt.Tx) ([]string, error) {
-	bucket := tx.Bucket([]byte(rankedServerEntriesBucket))
-	data := bucket.Get([]byte(rankedServerEntriesKey))
-
-	if data == nil {
-		return []string{}, nil
-	}
-
-	rankedServerEntries := make([]string, 0)
-	err := json.Unmarshal(data, &rankedServerEntries)
-	if err != nil {
-		return nil, ContextError(err)
-	}
-	return rankedServerEntries, nil
-}
-
-func setRankedServerEntries(tx *bolt.Tx, rankedServerEntries []string) error {
-	data, err := json.Marshal(rankedServerEntries)
-	if err != nil {
-		return ContextError(err)
-	}
-
-	bucket := tx.Bucket([]byte(rankedServerEntriesBucket))
-	err = bucket.Put([]byte(rankedServerEntriesKey), data)
-	if err != nil {
-		return ContextError(err)
-	}
-
-	return nil
-}
-
-func insertRankedServerEntry(tx *bolt.Tx, serverEntryId string, position int) error {
-	rankedServerEntries, err := getRankedServerEntries(tx)
-	if err != nil {
-		return ContextError(err)
-	}
-
-	// BoltDB implementation note:
-	// For simplicity, we store the ranked server ids in an array serialized to
-	// a single key value. To ensure this value doesn't grow without bound,
-	// it's capped at rankedServerEntryCount. For now, this cap should be large
-	// enough to meet the shuffleHeadLength = config.TunnelPoolSize criteria, for
-	// any reasonable configuration of config.TunnelPoolSize.
-
-	if position >= len(rankedServerEntries) {
-		rankedServerEntries = append(rankedServerEntries, serverEntryId)
-	} else {
-		end := len(rankedServerEntries)
-		if end+1 > rankedServerEntryCount {
-			end = rankedServerEntryCount
-		}
-		// insert: https://github.com/golang/go/wiki/SliceTricks
-		rankedServerEntries = append(
-			rankedServerEntries[:position],
-			append([]string{serverEntryId},
-				rankedServerEntries[position:end]...)...)
-	}
-
-	err = setRankedServerEntries(tx, rankedServerEntries)
-	if err != nil {
-		return ContextError(err)
-	}
-
-	return nil
-}
-
-func serverEntrySupportsProtocol(serverEntry *ServerEntry, protocol string) bool {
-	// Note: for meek, the capabilities are FRONTED-MEEK and UNFRONTED-MEEK
-	// and the additonal OSSH service is assumed to be available internally.
-	requiredCapability := strings.TrimSuffix(protocol, "-OSSH")
-	return Contains(serverEntry.Capabilities, requiredCapability)
-}
-
-// ServerEntryIterator is used to iterate over
-// stored server entries in rank order.
-type ServerEntryIterator struct {
-	region                      string
-	protocol                    string
-	shuffleHeadLength           int
-	serverEntryIds              []string
-	serverEntryIndex            int
-	isTargetServerEntryIterator bool
-	hasNextTargetServerEntry    bool
-	targetServerEntry           *ServerEntry
-}
-
-// NewServerEntryIterator creates a new ServerEntryIterator
-func NewServerEntryIterator(config *Config) (iterator *ServerEntryIterator, err error) {
-
-	// When configured, this target server entry is the only candidate
-	if config.TargetServerEntry != "" {
-		return newTargetServerEntryIterator(config)
-	}
-
-	checkInitDataStore()
-	iterator = &ServerEntryIterator{
-		region:                      config.EgressRegion,
-		protocol:                    config.TunnelProtocol,
-		shuffleHeadLength:           config.TunnelPoolSize,
-		isTargetServerEntryIterator: false,
-	}
-	err = iterator.Reset()
-	if err != nil {
-		return nil, err
-	}
-	return iterator, nil
-}
-
-// newTargetServerEntryIterator is a helper for initializing the TargetServerEntry case
-func newTargetServerEntryIterator(config *Config) (iterator *ServerEntryIterator, err error) {
-	serverEntry, err := DecodeServerEntry(config.TargetServerEntry)
-	if err != nil {
-		return nil, err
-	}
-	if config.EgressRegion != "" && serverEntry.Region != config.EgressRegion {
-		return nil, errors.New("TargetServerEntry does not support EgressRegion")
-	}
-	if config.TunnelProtocol != "" {
-		// Note: same capability/protocol mapping as in StoreServerEntry
-		requiredCapability := strings.TrimSuffix(config.TunnelProtocol, "-OSSH")
-		if !Contains(serverEntry.Capabilities, requiredCapability) {
-			return nil, errors.New("TargetServerEntry does not support TunnelProtocol")
-		}
-	}
-	iterator = &ServerEntryIterator{
-		isTargetServerEntryIterator: true,
-		hasNextTargetServerEntry:    true,
-		targetServerEntry:           serverEntry,
-	}
-	NoticeInfo("using TargetServerEntry: %s", serverEntry.IpAddress)
-	return iterator, nil
-}
-
-// Reset a NewServerEntryIterator to the start of its cycle. The next
-// call to Next will return the first server entry.
-func (iterator *ServerEntryIterator) Reset() error {
-	iterator.Close()
-
-	if iterator.isTargetServerEntryIterator {
-		iterator.hasNextTargetServerEntry = true
-		return nil
-	}
-
-	count := CountServerEntries(iterator.region, iterator.protocol)
-	NoticeCandidateServers(iterator.region, iterator.protocol, count)
-
-	// This query implements the Psiphon server candidate selection
-	// algorithm: the first TunnelPoolSize server candidates are in rank
-	// (priority) order, to favor previously successful servers; then the
-	// remaining long tail is shuffled to raise up less recent candidates.
-
-	// BoltDB implementation note:
-	// We don't keep a transaction open for the duration of the iterator
-	// because this would expose the following semantics to consumer code:
-	//
-	//     Read-only transactions and read-write transactions ... generally
-	//     shouldn't be opened simultaneously in the same goroutine. This can
-	//     cause a deadlock as the read-write transaction needs to periodically
-	//     re-map the data file but it cannot do so while a read-only
-	//     transaction is open.
-	//     (https://github.com/boltdb/bolt)
-	//
-	// So the uderlying serverEntriesBucket could change after the serverEntryIds
-	// list is built.
-
-	var serverEntryIds []string
-
-	err := singleton.db.View(func(tx *bolt.Tx) error {
-		var err error
-		serverEntryIds, err = getRankedServerEntries(tx)
-		if err != nil {
-			return err
-		}
-
-		skipServerEntryIds := make(map[string]bool)
-		for _, serverEntryId := range serverEntryIds {
-			skipServerEntryIds[serverEntryId] = true
-		}
-
-		bucket := tx.Bucket([]byte(serverEntriesBucket))
-		cursor := bucket.Cursor()
-		for key, _ := cursor.Last(); key != nil; key, _ = cursor.Prev() {
-			serverEntryId := string(key)
-			if _, ok := skipServerEntryIds[serverEntryId]; ok {
-				continue
-			}
-			serverEntryIds = append(serverEntryIds, serverEntryId)
-		}
-		return nil
-	})
-	if err != nil {
-		return ContextError(err)
-	}
-
-	for i := len(serverEntryIds) - 1; i > iterator.shuffleHeadLength-1; i-- {
-		j := rand.Intn(i)
-		serverEntryIds[i], serverEntryIds[j] = serverEntryIds[j], serverEntryIds[i]
-	}
-
-	iterator.serverEntryIds = serverEntryIds
-	iterator.serverEntryIndex = 0
-
-	return nil
-}
-
-// Close cleans up resources associated with a ServerEntryIterator.
-func (iterator *ServerEntryIterator) Close() {
-	iterator.serverEntryIds = nil
-	iterator.serverEntryIndex = 0
-}
-
-// Next returns the next server entry, by rank, for a ServerEntryIterator.
-// Returns nil with no error when there is no next item.
-func (iterator *ServerEntryIterator) Next() (serverEntry *ServerEntry, err error) {
-	defer func() {
-		if err != nil {
-			iterator.Close()
-		}
-	}()
-
-	if iterator.isTargetServerEntryIterator {
-		if iterator.hasNextTargetServerEntry {
-			iterator.hasNextTargetServerEntry = false
-			return MakeCompatibleServerEntry(iterator.targetServerEntry), nil
-		}
-		return nil, nil
-	}
-
-	// There are no region/protocol indexes for the server entries bucket.
-	// Loop until we have the next server entry that matches the iterator
-	// filter requirements.
-	for {
-		if iterator.serverEntryIndex >= len(iterator.serverEntryIds) {
-			// There is no next item
-			return nil, nil
-		}
-
-		serverEntryId := iterator.serverEntryIds[iterator.serverEntryIndex]
-		iterator.serverEntryIndex += 1
-
-		var data []byte
-		err = singleton.db.View(func(tx *bolt.Tx) error {
-			bucket := tx.Bucket([]byte(serverEntriesBucket))
-			data = bucket.Get([]byte(serverEntryId))
-			return nil
-		})
-		if err != nil {
-			return nil, ContextError(err)
-		}
-
-		if data == nil {
-			return nil, ContextError(
-				fmt.Errorf("Unexpected missing server entry: %s", serverEntryId))
-		}
-
-		serverEntry = new(ServerEntry)
-		err = json.Unmarshal(data, serverEntry)
-		if err != nil {
-			return nil, ContextError(err)
-		}
-
-		if (iterator.region == "" || serverEntry.Region == iterator.region) &&
-			(iterator.protocol == "" || serverEntrySupportsProtocol(serverEntry, iterator.protocol)) {
-
-			break
-		}
-	}
-
-	return MakeCompatibleServerEntry(serverEntry), nil
-}
-
-// MakeCompatibleServerEntry provides backwards compatibility with old server entries
-// which have a single meekFrontingDomain and not a meekFrontingAddresses array.
-// By copying this one meekFrontingDomain into meekFrontingAddresses, this client effectively
-// uses that single value as legacy clients do.
-func MakeCompatibleServerEntry(serverEntry *ServerEntry) *ServerEntry {
-	if len(serverEntry.MeekFrontingAddresses) == 0 && serverEntry.MeekFrontingDomain != "" {
-		serverEntry.MeekFrontingAddresses =
-			append(serverEntry.MeekFrontingAddresses, serverEntry.MeekFrontingDomain)
-	}
-
-	return serverEntry
-}
-
-func scanServerEntries(scanner func(*ServerEntry)) error {
-	err := singleton.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(serverEntriesBucket))
-		cursor := bucket.Cursor()
-
-		for key, value := cursor.First(); key != nil; key, value = cursor.Next() {
-			serverEntry := new(ServerEntry)
-			err := json.Unmarshal(value, serverEntry)
-			if err != nil {
-				return err
-			}
-			scanner(serverEntry)
-		}
-
-		return nil
-	})
-
-	if err != nil {
-		return ContextError(err)
-	}
-
-	return nil
-}
-
-// CountServerEntries returns a count of stored servers for the
-// specified region and protocol.
-func CountServerEntries(region, protocol string) int {
-	checkInitDataStore()
-
-	count := 0
-	err := scanServerEntries(func(serverEntry *ServerEntry) {
-		if (region == "" || serverEntry.Region == region) &&
-			(protocol == "" || serverEntrySupportsProtocol(serverEntry, protocol)) {
-			count += 1
-		}
-	})
-
-	if err != nil {
-		NoticeAlert("CountServerEntries failed: %s", err)
-		return 0
-	}
-
-	return count
-}
-
-// ReportAvailableRegions prints a notice with the available egress regions.
-// Note that this report ignores config.TunnelProtocol.
-func ReportAvailableRegions() {
-	checkInitDataStore()
-
-	regions := make(map[string]bool)
-	err := scanServerEntries(func(serverEntry *ServerEntry) {
-		regions[serverEntry.Region] = true
-	})
-
-	if err != nil {
-		NoticeAlert("ReportAvailableRegions failed: %s", err)
-		return
-	}
-
-	regionList := make([]string, 0, len(regions))
-	for region, _ := range regions {
-		// Some server entries do not have a region, but it makes no sense to return
-		// an empty string as an "available region".
-		if region != "" {
-			regionList = append(regionList, region)
-		}
-	}
-
-	NoticeAvailableEgressRegions(regionList)
-}
-
-// GetServerEntryIpAddresses returns an array containing
-// all stored server IP addresses.
-func GetServerEntryIpAddresses() (ipAddresses []string, err error) {
-	checkInitDataStore()
-
-	ipAddresses = make([]string, 0)
-	err = scanServerEntries(func(serverEntry *ServerEntry) {
-		ipAddresses = append(ipAddresses, serverEntry.IpAddress)
-	})
-
-	if err != nil {
-		return nil, ContextError(err)
-	}
-
-	return ipAddresses, nil
-}
-
-// SetSplitTunnelRoutes updates the cached routes data for
-// the given region. The associated etag is also stored and
-// used to make efficient web requests for updates to the data.
-func SetSplitTunnelRoutes(region, etag string, data []byte) error {
-	checkInitDataStore()
-
-	err := singleton.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(splitTunnelRouteETagsBucket))
-		err := bucket.Put([]byte(region), []byte(etag))
-
-		bucket = tx.Bucket([]byte(splitTunnelRouteDataBucket))
-		err = bucket.Put([]byte(region), data)
-		return err
-	})
-
-	if err != nil {
-		return ContextError(err)
-	}
-	return nil
-}
-
-// GetSplitTunnelRoutesETag retrieves the etag for cached routes
-// data for the specified region. If not found, it returns an empty string value.
-func GetSplitTunnelRoutesETag(region string) (etag string, err error) {
-	checkInitDataStore()
-
-	err = singleton.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(splitTunnelRouteETagsBucket))
-		etag = string(bucket.Get([]byte(region)))
-		return nil
-	})
-
-	if err != nil {
-		return "", ContextError(err)
-	}
-	return etag, nil
-}
-
-// GetSplitTunnelRoutesData retrieves the cached routes data
-// for the specified region. If not found, it returns a nil value.
-func GetSplitTunnelRoutesData(region string) (data []byte, err error) {
-	checkInitDataStore()
-
-	err = singleton.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(splitTunnelRouteDataBucket))
-		data = bucket.Get([]byte(region))
-		return nil
-	})
-
-	if err != nil {
-		return nil, ContextError(err)
-	}
-	return data, nil
-}
-
-// SetUrlETag stores an ETag for the specfied URL.
-// Note: input URL is treated as a string, and is not
-// encoded or decoded or otherwise canonicalized.
-func SetUrlETag(url, etag string) error {
-	checkInitDataStore()
-
-	err := singleton.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(urlETagsBucket))
-		err := bucket.Put([]byte(url), []byte(etag))
-		return err
-	})
-
-	if err != nil {
-		return ContextError(err)
-	}
-	return nil
-}
-
-// GetUrlETag retrieves a previously stored an ETag for the
-// specfied URL. If not found, it returns an empty string value.
-func GetUrlETag(url string) (etag string, err error) {
-	checkInitDataStore()
-
-	err = singleton.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(urlETagsBucket))
-		etag = string(bucket.Get([]byte(url)))
-		return nil
-	})
-
-	if err != nil {
-		return "", ContextError(err)
-	}
-	return etag, nil
-}
-
-// SetKeyValue stores a key/value pair.
-func SetKeyValue(key, value string) error {
-	checkInitDataStore()
-
-	err := singleton.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(keyValueBucket))
-		err := bucket.Put([]byte(key), []byte(value))
-		return err
-	})
-
-	if err != nil {
-		return ContextError(err)
-	}
-	return nil
-}
-
-// GetKeyValue retrieves the value for a given key. If not found,
-// it returns an empty string value.
-func GetKeyValue(key string) (value string, err error) {
-	checkInitDataStore()
-
-	err = singleton.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(keyValueBucket))
-		value = string(bucket.Get([]byte(key)))
-		return nil
-	})
-
-	if err != nil {
-		return "", ContextError(err)
-	}
-	return value, nil
-}

psiphon/meekConn.go

@@ -30,6 +30,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"strings"
 	"sync"
 	"time"
 
@@ -102,9 +103,11 @@ type transporter interface {
 // is spawned which will eventually start HTTP polling.
 // When frontingAddress is not "", fronting is used. This option assumes caller has
 // already checked server entry capabilities.
+// Fronting always uses HTTPS. Otherwise, HTTPS is optional.
 func DialMeek(
 	serverEntry *ServerEntry, sessionId string,
-	frontingAddress string, config *DialConfig) (meek *MeekConn, err error) {
+	useHttps bool, frontingAddress string,
+	config *DialConfig) (meek *MeekConn, err error) {
 
 	// Configure transport
 	// Note: MeekConn has its own PendingConns to manage the underlying HTTP transport connections,
@@ -118,14 +121,12 @@ func DialMeek(
 	*meekConfig = *config
 	meekConfig.PendingConns = pendingConns
 
-	var host string
+	// host is both what is dialed and what ends up in the HTTP Host header
+	host := fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.MeekServerPort)
 	var dialer Dialer
 	var proxyUrl func(*http.Request) (*url.URL, error)
 
-	if frontingAddress != "" {
-		// In this case, host is not what is dialed but is what ends up in the HTTP Host header
-		host = serverEntry.MeekFrontingHost
-
+	if useHttps || frontingAddress != "" {
 		// Custom TLS dialer:
 		//
 		//  1. ignores the HTTP request address and uses the fronting domain
@@ -157,24 +158,28 @@ func DialMeek(
 		// classify Psiphon traffic on some CDNs but not others) may throttle non-MiM CDNs so that our server
 		// selection always chooses tunnels to the MiM CDN (without any server cert verification, we won't
 		// exclusively connect to non-MiM CDNs); then the adversary kills the underlying TCP connection after
-		// some short period. This is similar to the "unidentified protocol" attack outlined in selectProtocol().
-		// A similar weighted selection defense may be appropriate.
-
-		dialer = NewCustomTLSDialer(
-			&CustomTLSConfig{
-				Dial:                          NewTCPDialer(meekConfig),
-				Timeout:                       meekConfig.ConnectTimeout,
-				FrontingAddr:                  fmt.Sprintf("%s:%d", frontingAddress, 443),
-				SendServerName:                false,
-				SkipVerify:                    true,
-				UseIndistinguishableTLS:       config.UseIndistinguishableTLS,
-				TrustedCACertificatesFilename: config.TrustedCACertificatesFilename,
-			})
+		// some short period. This is mitigated by the "impaired" protocol classification mechanism.
+
+		customTLSConfig := &CustomTLSConfig{
+			Dial:                          NewTCPDialer(meekConfig),
+			Timeout:                       meekConfig.ConnectTimeout,
+			SendServerName:                false,
+			SkipVerify:                    true,
+			UseIndistinguishableTLS:       config.UseIndistinguishableTLS,
+			TrustedCACertificatesFilename: config.TrustedCACertificatesFilename,
+		}
+
+		if frontingAddress != "" {
+			// In this case, host is not what is dialed but is what ends up in the HTTP Host header
+			host = serverEntry.MeekFrontingHost
+			customTLSConfig.FrontingAddr = fmt.Sprintf("%s:%d", frontingAddress, 443)
+		}
+
+		dialer = NewCustomTLSDialer(customTLSConfig)
+
 	} else {
-		// In the unfronted case, host is both what is dialed and what ends up in the HTTP Host header
-		host = fmt.Sprintf("%s:%d", serverEntry.IpAddress, serverEntry.MeekServerPort)
 
-		if meekConfig.UpstreamProxyUrl != "" {
+		if strings.HasPrefix(meekConfig.UpstreamProxyUrl, "http://") {
 			// For unfronted meek, we let the http.Transport handle proxying, as the
 			// target server hostname has to be in the HTTP request line. Also, in this
 			// case, we don't require the proxy to support CONNECT and so we can work
@@ -446,7 +451,7 @@ func (meek *MeekConn) relay() {
 		} else {
 			interval = time.Duration(float64(interval) * POLL_INTERNAL_MULTIPLIER)
 			if interval >= MAX_POLL_INTERVAL {
-				interval = MIN_POLL_INTERVAL
+				interval = MAX_POLL_INTERVAL
 			}
 		}
 	}

psiphon/migrateDataStore.go

@@ -0,0 +1,31 @@
+// +build !windows
+
+/*
+ * Copyright (c) 2015, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package psiphon
+
+// Stub function to return an empty list for non-Windows builds
+func prepareMigrationEntries(config *Config) []*ServerEntry {
+	return nil
+}
+
+// Stub function to return immediately for non-Windows builds
+func migrateEntries(serverEntries []*ServerEntry, legacyDataStoreFilename string) {
+}

psiphon/migrateDataStore_windows.go

@@ -0,0 +1,243 @@
+/*
+ * Copyright (c) 2015, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package psiphon
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	_ "github.com/Psiphon-Inc/go-sqlite3"
+)
+
+var legacyDb *sql.DB
+
+func prepareMigrationEntries(config *Config) []*ServerEntry {
+	var migratableServerEntries []*ServerEntry
+
+	// If DATA_STORE_FILENAME does not exist on disk
+	if _, err := os.Stat(filepath.Join(config.DataStoreDirectory, DATA_STORE_FILENAME)); os.IsNotExist(err) {
+		// If LEGACY_DATA_STORE_FILENAME exists on disk
+		if _, err := os.Stat(filepath.Join(config.DataStoreDirectory, LEGACY_DATA_STORE_FILENAME)); err == nil {
+
+			legacyDb, err = sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=private&mode=rwc", filepath.Join(config.DataStoreDirectory, LEGACY_DATA_STORE_FILENAME)))
+			defer legacyDb.Close()
+
+			if err != nil {
+				NoticeAlert("prepareMigrationEntries: sql.Open failed: %s", err)
+				return nil
+			}
+
+			initialization := "pragma journal_mode=WAL;\n"
+			_, err = legacyDb.Exec(initialization)
+			if err != nil {
+				NoticeAlert("prepareMigrationEntries: sql.DB.Exec failed: %s", err)
+				return nil
+			}
+
+			iterator, err := newlegacyServerEntryIterator(config)
+			if err != nil {
+				NoticeAlert("prepareMigrationEntries: newlegacyServerEntryIterator failed: %s", err)
+				return nil
+			}
+			defer iterator.Close()
+
+			for {
+				serverEntry, err := iterator.Next()
+				if err != nil {
+					NoticeAlert("prepareMigrationEntries: legacyServerEntryIterator.Next failed: %s", err)
+					break
+				}
+				if serverEntry == nil {
+					break
+				}
+
+				migratableServerEntries = append(migratableServerEntries, serverEntry)
+			}
+			NoticeInfo("%d server entries prepared for data store migration", len(migratableServerEntries))
+		}
+	}
+
+	return migratableServerEntries
+}
+
+// migrateEntries calls the BoltDB data store method to shuffle
+// and store an array of server entries (StoreServerEntries)
+// Failing to migrate entries, or delete the legacy file is never fatal
+func migrateEntries(serverEntries []*ServerEntry, legacyDataStoreFilename string) {
+	checkInitDataStore()
+
+	err := StoreServerEntries(serverEntries, false)
+	if err != nil {
+		NoticeAlert("migrateEntries: StoreServerEntries failed: %s", err)
+	} else {
+		// Retain server affinity from old datastore by taking the first
+		// array element (previous top ranked server) and promoting it
+		// to the top rank before the server selection process begins
+		err = PromoteServerEntry(serverEntries[0].IpAddress)
+		if err != nil {
+			NoticeAlert("migrateEntries: PromoteServerEntry failed: %s", err)
+		}
+
+		NoticeAlert("%d server entries successfully migrated to new data store", len(serverEntries))
+	}
+
+	err = os.Remove(legacyDataStoreFilename)
+	if err != nil {
+		NoticeAlert("migrateEntries: failed to delete legacy data store file '%s': %s", legacyDataStoreFilename, err)
+	}
+
+	return
+}
+
+// This code is copied from the dataStore.go code used to operate the legacy
+// SQLite datastore. The word "legacy" was added to all of the method names to avoid
+// namespace conflicts with the methods used to operate the BoltDB datastore
+
+// legacyServerEntryIterator is used to iterate over
+// stored server entries in rank order.
+type legacyServerEntryIterator struct {
+	shuffleHeadLength int
+	transaction       *sql.Tx
+	cursor            *sql.Rows
+}
+
+// newLegacyServerEntryIterator creates a new legacyServerEntryIterator
+func newlegacyServerEntryIterator(config *Config) (iterator *legacyServerEntryIterator, err error) {
+
+	iterator = &legacyServerEntryIterator{
+		shuffleHeadLength: config.TunnelPoolSize,
+	}
+	err = iterator.Reset()
+	if err != nil {
+		return nil, err
+	}
+	return iterator, nil
+}
+
+// Close cleans up resources associated with a legacyServerEntryIterator.
+func (iterator *legacyServerEntryIterator) Close() {
+	if iterator.cursor != nil {
+		iterator.cursor.Close()
+	}
+	iterator.cursor = nil
+	if iterator.transaction != nil {
+		iterator.transaction.Rollback()
+	}
+	iterator.transaction = nil
+}
+
+// Next returns the next server entry, by rank, for a legacyServerEntryIterator.
+// Returns nil with no error when there is no next item.
+func (iterator *legacyServerEntryIterator) Next() (serverEntry *ServerEntry, err error) {
+	defer func() {
+		if err != nil {
+			iterator.Close()
+		}
+	}()
+
+	if !iterator.cursor.Next() {
+		err = iterator.cursor.Err()
+		if err != nil {
+			return nil, ContextError(err)
+		}
+		// There is no next item
+		return nil, nil
+	}
+
+	var data []byte
+	err = iterator.cursor.Scan(&data)
+	if err != nil {
+		return nil, ContextError(err)
+	}
+	serverEntry = new(ServerEntry)
+	err = json.Unmarshal(data, serverEntry)
+	if err != nil {
+		return nil, ContextError(err)
+	}
+
+	return MakeCompatibleServerEntry(serverEntry), nil
+}
+
+// Reset a NewlegacyServerEntryIterator to the start of its cycle. The next
+// call to Next will return the first server entry.
+func (iterator *legacyServerEntryIterator) Reset() error {
+	iterator.Close()
+
+	transaction, err := legacyDb.Begin()
+	if err != nil {
+		return ContextError(err)
+	}
+	var cursor *sql.Rows
+
+	// This query implements the Psiphon server candidate selection
+	// algorithm: the first TunnelPoolSize server candidates are in rank
+	// (priority) order, to favor previously successful servers; then the
+	// remaining long tail is shuffled to raise up less recent candidates.
+
+	whereClause, whereParams := makeServerEntryWhereClause(nil)
+	headLength := iterator.shuffleHeadLength
+	queryFormat := `
+		select data from serverEntry %s
+		order by case
+		when rank > coalesce((select rank from serverEntry %s order by rank desc limit ?, 1), -1) then rank
+		else abs(random())%%((select rank from serverEntry %s order by rank desc limit ?, 1))
+		end desc;`
+	query := fmt.Sprintf(queryFormat, whereClause, whereClause, whereClause)
+	params := make([]interface{}, 0)
+	params = append(params, whereParams...)
+	params = append(params, whereParams...)
+	params = append(params, headLength)
+	params = append(params, whereParams...)
+	params = append(params, headLength)
+
+	cursor, err = transaction.Query(query, params...)
+	if err != nil {
+		transaction.Rollback()
+		return ContextError(err)
+	}
+	iterator.transaction = transaction
+	iterator.cursor = cursor
+	return nil
+}
+
+func makeServerEntryWhereClause(excludeIds []string) (whereClause string, whereParams []interface{}) {
+	whereClause = ""
+	whereParams = make([]interface{}, 0)
+	if len(excludeIds) > 0 {
+		if len(whereClause) > 0 {
+			whereClause += " and"
+		} else {
+			whereClause += " where"
+		}
+		whereClause += " id in ("
+		for index, id := range excludeIds {
+			if index > 0 {
+				whereClause += ", "
+			}
+			whereClause += "?"
+			whereParams = append(whereParams, id)
+		}
+		whereClause += ")"
+	}
+	return whereClause, whereParams
+}

psiphon/net.go

@@ -20,9 +20,15 @@
 package psiphon
 
 import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net"
+	"net/http"
+	"net/url"
 	"reflect"
 	"sync"
 	"time"
@@ -93,7 +99,8 @@ type NetworkConnectivityChecker interface {
 
 // DnsServerGetter defines the interface to the external GetDnsServer provider
 type DnsServerGetter interface {
-	GetDnsServer() string
+	GetPrimaryDnsServer() string
+	GetSecondaryDnsServer() string
 }
 
 // TimeoutError implements the error interface
@@ -241,3 +248,101 @@ func ResolveIP(host string, conn net.Conn) (addrs []net.IP, ttls []time.Duration
 	}
 	return addrs, ttls, nil
 }
+
+// MakeUntunneledHttpsClient returns a net/http.Client which is
+// configured to use custom dialing features -- including BindToDevice,
+// UseIndistinguishableTLS, etc. -- for a specific HTTPS request URL.
+// If verifyLegacyCertificate is not nil, it's used for certificate
+// verification.
+// Because UseIndistinguishableTLS requires a hack to work with
+// net/http, MakeUntunneledHttpClient may return a modified request URL
+// to be used. Callers should always use this return value to make
+// requests, not the input value.
+func MakeUntunneledHttpsClient(
+	dialConfig *DialConfig,
+	verifyLegacyCertificate *x509.Certificate,
+	requestUrl string,
+	requestTimeout time.Duration) (*http.Client, string, error) {
+
+	dialer := NewCustomTLSDialer(
+		// Note: when verifyLegacyCertificate is not nil, some
+		// of the other CustomTLSConfig is overridden.
+		&CustomTLSConfig{
+			Dial: NewTCPDialer(dialConfig),
+			VerifyLegacyCertificate:       verifyLegacyCertificate,
+			SendServerName:                true,
+			SkipVerify:                    false,
+			UseIndistinguishableTLS:       dialConfig.UseIndistinguishableTLS,
+			TrustedCACertificatesFilename: dialConfig.TrustedCACertificatesFilename,
+		})
+
+	urlComponents, err := url.Parse(requestUrl)
+	if err != nil {
+		return nil, "", ContextError(err)
+	}
+
+	// Change the scheme to "http"; otherwise http.Transport will try to do
+	// another TLS handshake inside the explicit TLS session. Also need to
+	// force an explicit port, as the default for "http", 80, won't talk TLS.
+	urlComponents.Scheme = "http"
+	host, port, err := net.SplitHostPort(urlComponents.Host)
+	if err != nil {
+		// Assume there's no port
+		host = urlComponents.Host
+		port = ""
+	}
+	if port == "" {
+		port = "443"
+	}
+	urlComponents.Host = net.JoinHostPort(host, port)
+
+	transport := &http.Transport{
+		Dial: dialer,
+	}
+	httpClient := &http.Client{
+		Timeout:   requestTimeout,
+		Transport: transport,
+	}
+
+	return httpClient, urlComponents.String(), nil
+}
+
+// MakeTunneledHttpClient returns a net/http.Client which is
+// configured to use custom dialing features including tunneled
+// dialing and, optionally, UseTrustedCACertificatesForStockTLS.
+// Unlike MakeUntunneledHttpsClient and makePsiphonHttpsClient,
+// This http.Client uses stock TLS and no scheme transformation
+// hack is required.
+func MakeTunneledHttpClient(
+	config *Config,
+	tunnel *Tunnel,
+	requestTimeout time.Duration) (*http.Client, error) {
+
+	tunneledDialer := func(_, addr string) (conn net.Conn, err error) {
+		return tunnel.sshClient.Dial("tcp", addr)
+	}
+
+	transport := &http.Transport{
+		Dial: tunneledDialer,
+		ResponseHeaderTimeout: requestTimeout,
+	}
+
+	if config.UseTrustedCACertificatesForStockTLS {
+		if config.TrustedCACertificatesFilename == "" {
+			return nil, ContextError(errors.New(
+				"UseTrustedCACertificatesForStockTLS requires TrustedCACertificatesFilename"))
+		}
+		rootCAs := x509.NewCertPool()
+		certData, err := ioutil.ReadFile(config.TrustedCACertificatesFilename)
+		if err != nil {
+			return nil, ContextError(err)
+		}
+		rootCAs.AppendCertsFromPEM(certData)
+		transport.TLSClientConfig = &tls.Config{RootCAs: rootCAs}
+	}
+
+	return &http.Client{
+		Transport: transport,
+		Timeout:   requestTimeout,
+	}, nil
+}

psiphon/notice.go

@@ -91,12 +91,12 @@ func NoticeInfo(format string, args ...interface{}) {
 	outputNotice("Info", false, "message", fmt.Sprintf(format, args...))
 }
 
-// NoticeInfo is an alert message; typically a recoverable error condition
+// NoticeAlert is an alert message; typically a recoverable error condition
 func NoticeAlert(format string, args ...interface{}) {
 	outputNotice("Alert", false, "message", fmt.Sprintf(format, args...))
 }
 
-// NoticeInfo is an error message; typically an unrecoverable error condition
+// NoticeError is an error message; typically an unrecoverable error condition
 func NoticeError(format string, args ...interface{}) {
 	outputNotice("Error", true, "message", fmt.Sprintf(format, args...))
 }

psiphon/opensslConn.go

@@ -1,4 +1,4 @@
-// +build android
+// +build android windows
 
 /*
  * Copyright (c) 2015, Psiphon Inc.

psiphon/opensslConn_unsupported.go

@@ -1,4 +1,4 @@
-// +build !android
+// +build !android,!windows
 
 /*
  * Copyright (c) 2015, Psiphon Inc.

psiphon/remoteServerList.go

@@ -23,9 +23,7 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"net"
 	"net/http"
-	"net/url"
 )
 
 // FetchRemoteServerList downloads a remote server list JSON record from
@@ -42,46 +40,13 @@ func FetchRemoteServerList(config *Config, dialConfig *DialConfig) (err error) {
 		return ContextError(errors.New("remote server list signature public key blank"))
 	}
 
-	dialer := NewTCPDialer(dialConfig)
-
-	// When the URL is HTTPS, use the custom TLS dialer with the
-	// UseIndistinguishableTLS option.
-	// TODO: refactor into helper function
-	requestUrl, err := url.Parse(config.RemoteServerListUrl)
+	httpClient, requestUrl, err := MakeUntunneledHttpsClient(
+		dialConfig, nil, config.RemoteServerListUrl, FETCH_REMOTE_SERVER_LIST_TIMEOUT)
 	if err != nil {
 		return ContextError(err)
 	}
-	if requestUrl.Scheme == "https" {
-		dialer = NewCustomTLSDialer(
-			&CustomTLSConfig{
-				Dial:                          dialer,
-				SendServerName:                true,
-				SkipVerify:                    false,
-				UseIndistinguishableTLS:       config.UseIndistinguishableTLS,
-				TrustedCACertificatesFilename: config.TrustedCACertificatesFilename,
-			})
-
-		// Change the scheme to "http"; otherwise http.Transport will try to do
-		// another TLS handshake inside the explicit TLS session. Also need to
-		// force the port to 443,as the default for "http", 80, won't talk TLS.
-		requestUrl.Scheme = "http"
-		host, _, err := net.SplitHostPort(requestUrl.Host)
-		if err != nil {
-			// Assume there's no port
-			host = requestUrl.Host
-		}
-		requestUrl.Host = net.JoinHostPort(host, "443")
-	}
-
-	transport := &http.Transport{
-		Dial: dialer,
-	}
-	httpClient := http.Client{
-		Timeout:   FETCH_REMOTE_SERVER_LIST_TIMEOUT,
-		Transport: transport,
-	}
 
-	request, err := http.NewRequest("GET", requestUrl.String(), nil)
+	request, err := http.NewRequest("GET", requestUrl, nil)
 	if err != nil {
 		return ContextError(err)
 	}

psiphon/serverApi.go

@@ -31,29 +31,39 @@ import (
 	"net"
 	"net/http"
 	"strconv"
+	"sync/atomic"
 
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/transferstats"
 )
 
-// Session is a utility struct which holds all of the data associated
-// with a Psiphon session. In addition to the established tunnel, this
-// includes the session ID (used for Psiphon API requests) and a http
+// ServerContext is a utility struct which holds all of the data associated
+// with a Psiphon server connection. In addition to the established tunnel, this
+// includes data associated with Psiphon API requests and a persistent http
 // client configured to make tunneled Psiphon API requests.
-type Session struct {
-	sessionId            string
-	baseRequestUrl       string
-	psiphonHttpsClient   *http.Client
-	statsRegexps         *transferstats.Regexps
-	clientRegion         string
-	clientUpgradeVersion string
+type ServerContext struct {
+	sessionId                string
+	tunnelNumber             int64
+	baseRequestUrl           string
+	psiphonHttpsClient       *http.Client
+	statsRegexps             *transferstats.Regexps
+	clientRegion             string
+	clientUpgradeVersion     string
+	serverHandshakeTimestamp string
 }
 
-// MakeSessionId creates a new session ID. Making the session ID is not done
-// in NewSession because:
-// (1) the transport needs to send the ID in the SSH credentials before the tunnel
-//     is established and NewSession performs a handshake on an established tunnel.
-// (2) the same session ID is used across multi-tunnel controller runs, where each
-//     tunnel has its own Session instance.
+// nextTunnelNumber is a monotonically increasing number assigned to each
+// successive tunnel connection. The sessionId and tunnelNumber together
+// form a globally unique identifier for tunnels, which is used for
+// stats. Note that the number is increasing but not necessarily
+// consecutive for each active tunnel in session.
+var nextTunnelNumber int64
+
+// MakeSessionId creates a new session ID. The same session ID is used across
+// multi-tunnel controller runs, where each tunnel has its own ServerContext
+// instance.
+// In server-side stats, we now consider a "session" to be the lifetime of the
+// Controller (e.g., the user's commanded start and stop) and we measure this
+// duration as well as the duration of each tunnel within the session.
 func MakeSessionId() (sessionId string, err error) {
 	randomId, err := MakeSecureRandomBytes(PSIPHON_API_CLIENT_SESSION_ID_LENGTH)
 	if err != nil {
@@ -62,108 +72,35 @@ func MakeSessionId() (sessionId string, err error) {
 	return hex.EncodeToString(randomId), nil
 }
 
-// NewSession makes the tunnelled handshake request to the
-// Psiphon server and returns a Session struct, initialized with the
-// session ID, for use with subsequent Psiphon server API requests (e.g.,
-// periodic connected and status requests).
-func NewSession(config *Config, tunnel *Tunnel, sessionId string) (session *Session, err error) {
+// NewServerContext makes the tunnelled handshake request to the Psiphon server
+// and returns a ServerContext struct for use with subsequent Psiphon server API
+// requests (e.g., periodic connected and status requests).
+func NewServerContext(tunnel *Tunnel, sessionId string) (*ServerContext, error) {
 
 	psiphonHttpsClient, err := makePsiphonHttpsClient(tunnel)
 	if err != nil {
 		return nil, ContextError(err)
 	}
-	session = &Session{
+
+	serverContext := &ServerContext{
 		sessionId:          sessionId,
-		baseRequestUrl:     makeBaseRequestUrl(config, tunnel, sessionId),
+		tunnelNumber:       atomic.AddInt64(&nextTunnelNumber, 1),
+		baseRequestUrl:     makeBaseRequestUrl(tunnel, "", sessionId),
 		psiphonHttpsClient: psiphonHttpsClient,
 	}
 
-	err = session.doHandshakeRequest()
+	err = serverContext.doHandshakeRequest()
 	if err != nil {
 		return nil, ContextError(err)
 	}
 
-	return session, nil
-}
-
-// DoConnectedRequest performs the connected API request. This request is
-// used for statistics. The server returns a last_connected token for
-// the client to store and send next time it connects. This token is
-// a timestamp (using the server clock, and should be rounded to the
-// nearest hour) which is used to determine when a connection represents
-// a unique user for a time period.
-func (session *Session) DoConnectedRequest() error {
-	const DATA_STORE_LAST_CONNECTED_KEY = "lastConnected"
-	lastConnected, err := GetKeyValue(DATA_STORE_LAST_CONNECTED_KEY)
-	if err != nil {
-		return ContextError(err)
-	}
-	if lastConnected == "" {
-		lastConnected = "None"
-	}
-	url := session.buildRequestUrl(
-		"connected",
-		&ExtraParam{"session_id", session.sessionId},
-		&ExtraParam{"last_connected", lastConnected})
-	responseBody, err := session.doGetRequest(url)
-	if err != nil {
-		return ContextError(err)
-	}
-
-	var response struct {
-		ConnectedTimestamp string `json:"connected_timestamp"`
-	}
-	err = json.Unmarshal(responseBody, &response)
-	if err != nil {
-		return ContextError(err)
-	}
-
-	err = SetKeyValue(DATA_STORE_LAST_CONNECTED_KEY, response.ConnectedTimestamp)
-	if err != nil {
-		return ContextError(err)
-	}
-	return nil
-}
-
-// StatsRegexps gets the Regexps used for the statistics for this tunnel.
-func (session *Session) StatsRegexps() *transferstats.Regexps {
-	return session.statsRegexps
-}
-
-// DoStatusRequest makes a /status request to the server, sending session stats.
-func (session *Session) DoStatusRequest(statsPayload json.Marshaler) error {
-	statsPayloadJSON, err := json.Marshal(statsPayload)
-	if err != nil {
-		return ContextError(err)
-	}
-
-	// Add a random amount of padding to help prevent stats updates from being
-	// a predictable size (which often happens when the connection is quiet).
-	padding := MakeSecureRandomPadding(0, PSIPHON_API_STATUS_REQUEST_PADDING_MAX_BYTES)
-
-	// "connected" is a legacy parameter. This client does not report when
-	// it has disconnected.
-
-	url := session.buildRequestUrl(
-		"status",
-		&ExtraParam{"session_id", session.sessionId},
-		&ExtraParam{"connected", "1"},
-		// TODO: base64 encoding of padding means the padding
-		// size is not exactly [0, PADDING_MAX_BYTES]
-		&ExtraParam{"padding", base64.StdEncoding.EncodeToString(padding)})
-
-	err = session.doPostRequest(url, "application/json", bytes.NewReader(statsPayloadJSON))
-	if err != nil {
-		return ContextError(err)
-	}
-
-	return nil
+	return serverContext, nil
 }
 
 // doHandshakeRequest performs the handshake API request. The handshake
 // returns upgrade info, newly discovered server entries -- which are
 // stored -- and sponsor info (home pages, stat regexes).
-func (session *Session) doHandshakeRequest() error {
+func (serverContext *ServerContext) doHandshakeRequest() error {
 	extraParams := make([]*ExtraParam, 0)
 	serverEntryIpAddresses, err := GetServerEntryIpAddresses()
 	if err != nil {
@@ -174,8 +111,8 @@ func (session *Session) doHandshakeRequest() error {
 	for _, ipAddress := range serverEntryIpAddresses {
 		extraParams = append(extraParams, &ExtraParam{"known_server", ipAddress})
 	}
-	url := session.buildRequestUrl("handshake", extraParams...)
-	responseBody, err := session.doGetRequest(url)
+	url := buildRequestUrl(serverContext.baseRequestUrl, "handshake", extraParams...)
+	responseBody, err := serverContext.doGetRequest(url)
 	if err != nil {
 		return ContextError(err)
 	}
@@ -202,14 +139,15 @@ func (session *Session) doHandshakeRequest() error {
 		HttpsRequestRegexes  []map[string]string `json:"https_request_regexes"`
 		EncodedServerList    []string            `json:"encoded_server_list"`
 		ClientRegion         string              `json:"client_region"`
+		ServerTimestamp      string              `json:"server_timestamp"`
 	}
 	err = json.Unmarshal(configLine, &handshakeConfig)
 	if err != nil {
 		return ContextError(err)
 	}
 
-	session.clientRegion = handshakeConfig.ClientRegion
-	NoticeClientRegion(session.clientRegion)
+	serverContext.clientRegion = handshakeConfig.ClientRegion
+	NoticeClientRegion(serverContext.clientRegion)
 
 	var decodedServerEntries []*ServerEntry
 
@@ -242,13 +180,13 @@ func (session *Session) doHandshakeRequest() error {
 		NoticeHomepage(homepage)
 	}
 
-	session.clientUpgradeVersion = handshakeConfig.UpgradeClientVersion
+	serverContext.clientUpgradeVersion = handshakeConfig.UpgradeClientVersion
 	if handshakeConfig.UpgradeClientVersion != "" {
 		NoticeClientUpgradeAvailable(handshakeConfig.UpgradeClientVersion)
 	}
 
 	var regexpsNotices []string
-	session.statsRegexps, regexpsNotices = transferstats.MakeRegexps(
+	serverContext.statsRegexps, regexpsNotices = transferstats.MakeRegexps(
 		handshakeConfig.PageViewRegexes,
 		handshakeConfig.HttpsRequestRegexes)
 
@@ -256,15 +194,345 @@ func (session *Session) doHandshakeRequest() error {
 		NoticeAlert(notice)
 	}
 
+	serverContext.serverHandshakeTimestamp = handshakeConfig.ServerTimestamp
+
+	return nil
+}
+
+// DoConnectedRequest performs the connected API request. This request is
+// used for statistics. The server returns a last_connected token for
+// the client to store and send next time it connects. This token is
+// a timestamp (using the server clock, and should be rounded to the
+// nearest hour) which is used to determine when a connection represents
+// a unique user for a time period.
+func (serverContext *ServerContext) DoConnectedRequest() error {
+	const DATA_STORE_LAST_CONNECTED_KEY = "lastConnected"
+	lastConnected, err := GetKeyValue(DATA_STORE_LAST_CONNECTED_KEY)
+	if err != nil {
+		return ContextError(err)
+	}
+	if lastConnected == "" {
+		lastConnected = "None"
+	}
+	url := buildRequestUrl(
+		serverContext.baseRequestUrl,
+		"connected",
+		&ExtraParam{"session_id", serverContext.sessionId},
+		&ExtraParam{"last_connected", lastConnected})
+	responseBody, err := serverContext.doGetRequest(url)
+	if err != nil {
+		return ContextError(err)
+	}
+
+	var response struct {
+		ConnectedTimestamp string `json:"connected_timestamp"`
+	}
+	err = json.Unmarshal(responseBody, &response)
+	if err != nil {
+		return ContextError(err)
+	}
+
+	err = SetKeyValue(DATA_STORE_LAST_CONNECTED_KEY, response.ConnectedTimestamp)
+	if err != nil {
+		return ContextError(err)
+	}
 	return nil
 }
 
+// StatsRegexps gets the Regexps used for the statistics for this tunnel.
+func (serverContext *ServerContext) StatsRegexps() *transferstats.Regexps {
+	return serverContext.statsRegexps
+}
+
+// DoStatusRequest makes a /status request to the server, sending session stats.
+func (serverContext *ServerContext) DoStatusRequest(tunnel *Tunnel) error {
+
+	url := makeStatusRequestUrl(serverContext.sessionId, serverContext.baseRequestUrl, true)
+
+	payload, payloadInfo, err := makeStatusRequestPayload(tunnel.serverEntry.IpAddress)
+	if err != nil {
+		return ContextError(err)
+	}
+
+	err = serverContext.doPostRequest(url, "application/json", bytes.NewReader(payload))
+	if err != nil {
+
+		// Resend the transfer stats and tunnel stats later
+		// Note: potential duplicate reports if the server received and processed
+		// the request but the client failed to receive the response.
+		putBackStatusRequestPayload(payloadInfo)
+
+		return ContextError(err)
+	}
+	confirmStatusRequestPayload(payloadInfo)
+
+	return nil
+}
+
+func makeStatusRequestUrl(sessionId, baseRequestUrl string, isTunneled bool) string {
+
+	// Add a random amount of padding to help prevent stats updates from being
+	// a predictable size (which often happens when the connection is quiet).
+	padding := MakeSecureRandomPadding(0, PSIPHON_API_STATUS_REQUEST_PADDING_MAX_BYTES)
+
+	// Legacy clients set "connected" to "0" when disconnecting, and this value
+	// is used to calculate session duration estimates. This is now superseded
+	// by explicit tunnel stats duration reporting.
+	// The legacy method of reconstructing session durations is not compatible
+	// with this client's connected request retries and asynchronous final
+	// status request attempts. So we simply set this "connected" flag to reflect
+	// whether the request is sent tunneled or not.
+
+	connected := "1"
+	if !isTunneled {
+		connected = "0"
+	}
+
+	return buildRequestUrl(
+		baseRequestUrl,
+		"status",
+		&ExtraParam{"session_id", sessionId},
+		&ExtraParam{"connected", connected},
+		// TODO: base64 encoding of padding means the padding
+		// size is not exactly [0, PADDING_MAX_BYTES]
+		&ExtraParam{"padding", base64.StdEncoding.EncodeToString(padding)})
+}
+
+// statusRequestPayloadInfo is a temporary structure for data used to
+// either "clear" or "put back" status request payload data depending
+// on whether or not the request succeeded.
+type statusRequestPayloadInfo struct {
+	serverId      string
+	transferStats *transferstats.AccumulatedStats
+	tunnelStats   [][]byte
+}
+
+func makeStatusRequestPayload(
+	serverId string) ([]byte, *statusRequestPayloadInfo, error) {
+
+	transferStats := transferstats.TakeOutStatsForServer(serverId)
+	tunnelStats, err := TakeOutUnreportedTunnelStats(
+		PSIPHON_API_TUNNEL_STATS_MAX_COUNT)
+	if err != nil {
+		NoticeAlert(
+			"TakeOutUnreportedTunnelStats failed: %s", ContextError(err))
+		tunnelStats = nil
+		// Proceed with transferStats only
+	}
+	payloadInfo := &statusRequestPayloadInfo{
+		serverId, transferStats, tunnelStats}
+
+	payload := make(map[string]interface{})
+
+	hostBytes, bytesTransferred := transferStats.GetStatsForStatusRequest()
+	payload["host_bytes"] = hostBytes
+	payload["bytes_transferred"] = bytesTransferred
+
+	// We're not recording these fields, but the server requires them.
+	payload["page_views"] = make([]string, 0)
+	payload["https_requests"] = make([]string, 0)
+
+	// Tunnel stats records are already in JSON format
+	jsonTunnelStats := make([]json.RawMessage, len(tunnelStats))
+	for i, tunnelStatsRecord := range tunnelStats {
+		jsonTunnelStats[i] = json.RawMessage(tunnelStatsRecord)
+	}
+	payload["tunnel_stats"] = jsonTunnelStats
+
+	jsonPayload, err := json.Marshal(payload)
+	if err != nil {
+
+		// Send the transfer stats and tunnel stats later
+		putBackStatusRequestPayload(payloadInfo)
+
+		return nil, nil, ContextError(err)
+	}
+
+	return jsonPayload, payloadInfo, nil
+}
+
+func putBackStatusRequestPayload(payloadInfo *statusRequestPayloadInfo) {
+	transferstats.PutBackStatsForServer(
+		payloadInfo.serverId, payloadInfo.transferStats)
+	err := PutBackUnreportedTunnelStats(payloadInfo.tunnelStats)
+	if err != nil {
+		// These tunnel stats records won't be resent under after a
+		// datastore re-initialization.
+		NoticeAlert(
+			"PutBackUnreportedTunnelStats failed: %s", ContextError(err))
+	}
+}
+
+func confirmStatusRequestPayload(payloadInfo *statusRequestPayloadInfo) {
+	err := ClearReportedTunnelStats(payloadInfo.tunnelStats)
+	if err != nil {
+		// These tunnel stats records may be resent.
+		NoticeAlert(
+			"ClearReportedTunnelStats failed: %s", ContextError(err))
+	}
+}
+
+// TryUntunneledStatusRequest makes direct connections to the specified
+// server (if supported) in an attempt to send useful bytes transferred
+// and tunnel duration stats after a tunnel has alreay failed.
+// The tunnel is assumed to be closed, but its config, protocol, and
+// context values must still be valid.
+// TryUntunneledStatusRequest emits notices detailing failed attempts.
+func TryUntunneledStatusRequest(tunnel *Tunnel, isShutdown bool) error {
+
+	for _, port := range tunnel.serverEntry.GetDirectWebRequestPorts() {
+		err := doUntunneledStatusRequest(tunnel, port, isShutdown)
+		if err == nil {
+			return nil
+		}
+		NoticeAlert("doUntunneledStatusRequest failed for %s:%s: %s",
+			tunnel.serverEntry.IpAddress, port, err)
+	}
+
+	return errors.New("all attempts failed")
+}
+
+// doUntunneledStatusRequest attempts an untunneled stratus request.
+func doUntunneledStatusRequest(
+	tunnel *Tunnel, port string, isShutdown bool) error {
+
+	url := makeStatusRequestUrl(
+		tunnel.serverContext.sessionId,
+		makeBaseRequestUrl(tunnel, port, tunnel.serverContext.sessionId),
+		false)
+
+	certificate, err := DecodeCertificate(tunnel.serverEntry.WebServerCertificate)
+	if err != nil {
+		return ContextError(err)
+	}
+
+	timeout := PSIPHON_API_SERVER_TIMEOUT
+	if isShutdown {
+		timeout = PSIPHON_API_SHUTDOWN_SERVER_TIMEOUT
+	}
+
+	httpClient, requestUrl, err := MakeUntunneledHttpsClient(
+		tunnel.untunneledDialConfig,
+		certificate,
+		url,
+		timeout)
+	if err != nil {
+		return ContextError(err)
+	}
+
+	payload, payloadInfo, err := makeStatusRequestPayload(tunnel.serverEntry.IpAddress)
+	if err != nil {
+		return ContextError(err)
+	}
+
+	bodyType := "application/json"
+	body := bytes.NewReader(payload)
+
+	response, err := httpClient.Post(requestUrl, bodyType, body)
+	if err == nil && response.StatusCode != http.StatusOK {
+		response.Body.Close()
+		err = fmt.Errorf("HTTP POST request failed with response code: %d", response.StatusCode)
+	}
+	if err != nil {
+
+		// Resend the transfer stats and tunnel stats later
+		// Note: potential duplicate reports if the server received and processed
+		// the request but the client failed to receive the response.
+		putBackStatusRequestPayload(payloadInfo)
+
+		// Trim this error since it may include long URLs
+		return ContextError(TrimError(err))
+	}
+	confirmStatusRequestPayload(payloadInfo)
+	response.Body.Close()
+
+	return nil
+}
+
+// RecordTunnelStats records a tunnel duration and bytes
+// sent and received for subsequent reporting and quality
+// analysis.
+//
+// Tunnel durations are precisely measured client-side
+// and reported in status requests. As the duration is
+// not determined until the tunnel is closed, tunnel
+// stats records are stored in the persistent datastore
+// and reported via subsequent status requests sent to any
+// Psiphon server.
+//
+// Since the status request that reports a tunnel stats
+// record is not necessarily handled by the same server, the
+// tunnel stats records include the original server ID.
+//
+// Other fields that may change between tunnel stats recording
+// and reporting include client geo data, propagation channel,
+// sponsor ID, client version. These are not stored in the
+// datastore (client region, in particular, since that would
+// create an on-disk record of user location).
+// TODO: the server could encrypt, with a nonce and key unknown to
+// the client, a blob containing this data; return it in the
+// handshake response; and the client could store and later report
+// this blob with its tunnel stats records.
+//
+// Multiple "status" requests may be in flight at once (due
+// to multi-tunnel, asynchronous final status retry, and
+// aggressive status requests for pre-registered tunnels),
+// To avoid duplicate reporting, tunnel stats records are
+// "taken-out" by a status request and then "put back" in
+// case the request fails.
+//
+// Note: since tunnel stats records have a globally unique
+// identifier (sessionId + tunnelNumber), we could tolerate
+// duplicate reporting and filter our duplicates on the
+// server-side. Permitting duplicate reporting could increase
+// the velocity of reporting (for example, both the asynchronous
+// untunneled final status requests and the post-connected
+// immediate startus requests could try to report the same tunnel
+// stats).
+// Duplicate reporting may also occur when a server receives and
+// processes a status request but the client fails to receive
+// the response.
+func RecordTunnelStats(
+	sessionId string,
+	tunnelNumber int64,
+	tunnelServerIpAddress string,
+	serverHandshakeTimestamp, duration string,
+	totalBytesSent, totalBytesReceived int64) error {
+
+	tunnelStats := struct {
+		SessionId                string `json:"session_id"`
+		TunnelNumber             int64  `json:"tunnel_number"`
+		TunnelServerIpAddress    string `json:"tunnel_server_ip_address"`
+		ServerHandshakeTimestamp string `json:"server_handshake_timestamp"`
+		Duration                 string `json:"duration"`
+		TotalBytesSent           int64  `json:"total_bytes_sent"`
+		TotalBytesReceived       int64  `json:"total_bytes_received"`
+	}{
+		sessionId,
+		tunnelNumber,
+		tunnelServerIpAddress,
+		serverHandshakeTimestamp,
+		duration,
+		totalBytesSent,
+		totalBytesReceived,
+	}
+
+	tunnelStatsJson, err := json.Marshal(tunnelStats)
+	if err != nil {
+		return ContextError(err)
+	}
+
+	return StoreTunnelStats(tunnelStatsJson)
+}
+
 // doGetRequest makes a tunneled HTTPS request and returns the response body.
-func (session *Session) doGetRequest(requestUrl string) (responseBody []byte, err error) {
-	response, err := session.psiphonHttpsClient.Get(requestUrl)
+func (serverContext *ServerContext) doGetRequest(
+	requestUrl string) (responseBody []byte, err error) {
+
+	response, err := serverContext.psiphonHttpsClient.Get(requestUrl)
 	if err == nil && response.StatusCode != http.StatusOK {
 		response.Body.Close()
-		err = fmt.Errorf("unexpected response status code: %d", response.StatusCode)
+		err = fmt.Errorf("HTTP GET request failed with response code: %d", response.StatusCode)
 	}
 	if err != nil {
 		// Trim this error since it may include long URLs
@@ -275,41 +543,42 @@ func (session *Session) doGetRequest(requestUrl string) (responseBody []byte, er
 	if err != nil {
 		return nil, ContextError(err)
 	}
-	if response.StatusCode != http.StatusOK {
-		return nil, ContextError(fmt.Errorf("HTTP GET request failed with response code: %d", response.StatusCode))
-	}
 	return body, nil
 }
 
 // doPostRequest makes a tunneled HTTPS POST request.
-func (session *Session) doPostRequest(requestUrl string, bodyType string, body io.Reader) (err error) {
-	response, err := session.psiphonHttpsClient.Post(requestUrl, bodyType, body)
+func (serverContext *ServerContext) doPostRequest(
+	requestUrl string, bodyType string, body io.Reader) (err error) {
+
+	response, err := serverContext.psiphonHttpsClient.Post(requestUrl, bodyType, body)
 	if err == nil && response.StatusCode != http.StatusOK {
 		response.Body.Close()
-		err = fmt.Errorf("unexpected response status code: %d", response.StatusCode)
+		err = fmt.Errorf("HTTP POST request failed with response code: %d", response.StatusCode)
 	}
 	if err != nil {
 		// Trim this error since it may include long URLs
 		return ContextError(TrimError(err))
 	}
 	response.Body.Close()
-	if response.StatusCode != http.StatusOK {
-		return ContextError(fmt.Errorf("HTTP POST request failed with response code: %d", response.StatusCode))
-	}
-	return
+	return nil
 }
 
 // makeBaseRequestUrl makes a URL containing all the common parameters
 // that are included with Psiphon API requests. These common parameters
 // are used for statistics.
-func makeBaseRequestUrl(config *Config, tunnel *Tunnel, sessionId string) string {
+func makeBaseRequestUrl(tunnel *Tunnel, port, sessionId string) string {
 	var requestUrl bytes.Buffer
+
+	if port == "" {
+		port = tunnel.serverEntry.WebServerPort
+	}
+
 	// Note: don't prefix with HTTPS scheme, see comment in doGetRequest.
 	// e.g., don't do this: requestUrl.WriteString("https://")
 	requestUrl.WriteString("http://")
 	requestUrl.WriteString(tunnel.serverEntry.IpAddress)
 	requestUrl.WriteString(":")
-	requestUrl.WriteString(tunnel.serverEntry.WebServerPort)
+	requestUrl.WriteString(port)
 	requestUrl.WriteString("/")
 	// Placeholder for the path component of a request
 	requestUrl.WriteString("%s")
@@ -318,18 +587,18 @@ func makeBaseRequestUrl(config *Config, tunnel *Tunnel, sessionId string) string
 	requestUrl.WriteString("&server_secret=")
 	requestUrl.WriteString(tunnel.serverEntry.WebServerSecret)
 	requestUrl.WriteString("&propagation_channel_id=")
-	requestUrl.WriteString(config.PropagationChannelId)
+	requestUrl.WriteString(tunnel.config.PropagationChannelId)
 	requestUrl.WriteString("&sponsor_id=")
-	requestUrl.WriteString(config.SponsorId)
+	requestUrl.WriteString(tunnel.config.SponsorId)
 	requestUrl.WriteString("&client_version=")
-	requestUrl.WriteString(config.ClientVersion)
+	requestUrl.WriteString(tunnel.config.ClientVersion)
 	// TODO: client_tunnel_core_version
 	requestUrl.WriteString("&relay_protocol=")
 	requestUrl.WriteString(tunnel.protocol)
 	requestUrl.WriteString("&client_platform=")
-	requestUrl.WriteString(config.ClientPlatform)
+	requestUrl.WriteString(tunnel.config.ClientPlatform)
 	requestUrl.WriteString("&tunnel_whole_device=")
-	requestUrl.WriteString(strconv.Itoa(config.TunnelWholeDevice))
+	requestUrl.WriteString(strconv.Itoa(tunnel.config.TunnelWholeDevice))
 	return requestUrl.String()
 }
 
@@ -337,9 +606,9 @@ type ExtraParam struct{ name, value string }
 
 // buildRequestUrl makes a URL for an API request. The URL includes the
 // base request URL and any extra parameters for the specific request.
-func (session *Session) buildRequestUrl(path string, extraParams ...*ExtraParam) string {
+func buildRequestUrl(baseRequestUrl, path string, extraParams ...*ExtraParam) string {
 	var requestUrl bytes.Buffer
-	requestUrl.WriteString(fmt.Sprintf(session.baseRequestUrl, path))
+	requestUrl.WriteString(fmt.Sprintf(baseRequestUrl, path))
 	for _, extraParam := range extraParams {
 		requestUrl.WriteString("&")
 		requestUrl.WriteString(extraParam.name)
@@ -349,7 +618,7 @@ func (session *Session) buildRequestUrl(path string, extraParams ...*ExtraParam)
 	return requestUrl.String()
 }
 
-// makeHttpsClient creates a Psiphon HTTPS client that tunnels requests and which validates
+// makePsiphonHttpsClient creates a Psiphon HTTPS client that tunnels requests and which validates
 // the web server using the Psiphon server entry web server certificate.
 // This is not a general purpose HTTPS client.
 // As the custom dialer makes an explicit TLS connection, URLs submitted to the returned
@@ -361,6 +630,7 @@ func makePsiphonHttpsClient(tunnel *Tunnel) (httpsClient *http.Client, err error
 		return nil, ContextError(err)
 	}
 	tunneledDialer := func(_, addr string) (conn net.Conn, err error) {
+		// TODO: check tunnel.isClosed, and apply TUNNEL_PORT_FORWARD_DIAL_TIMEOUT as in Tunnel.Dial?
 		return tunnel.sshClient.Dial("tcp", addr)
 	}
 	dialer := NewCustomTLSDialer(

psiphon/serverEntry.go

@@ -30,15 +30,17 @@ import (
 )
 
 const (
-	TUNNEL_PROTOCOL_SSH            = "SSH"
-	TUNNEL_PROTOCOL_OBFUSCATED_SSH = "OSSH"
-	TUNNEL_PROTOCOL_UNFRONTED_MEEK = "UNFRONTED-MEEK-OSSH"
-	TUNNEL_PROTOCOL_FRONTED_MEEK   = "FRONTED-MEEK-OSSH"
+	TUNNEL_PROTOCOL_SSH                  = "SSH"
+	TUNNEL_PROTOCOL_OBFUSCATED_SSH       = "OSSH"
+	TUNNEL_PROTOCOL_UNFRONTED_MEEK       = "UNFRONTED-MEEK-OSSH"
+	TUNNEL_PROTOCOL_UNFRONTED_MEEK_HTTPS = "UNFRONTED-MEEK-HTTPS-OSSH"
+	TUNNEL_PROTOCOL_FRONTED_MEEK         = "FRONTED-MEEK-OSSH"
 )
 
 var SupportedTunnelProtocols = []string{
 	TUNNEL_PROTOCOL_FRONTED_MEEK,
 	TUNNEL_PROTOCOL_UNFRONTED_MEEK,
+	TUNNEL_PROTOCOL_UNFRONTED_MEEK_HTTPS,
 	TUNNEL_PROTOCOL_OBFUSCATED_SSH,
 	TUNNEL_PROTOCOL_SSH,
 }
@@ -109,6 +111,20 @@ func (serverEntry *ServerEntry) DisableImpairedProtocols(impairedProtocols []str
 	serverEntry.Capabilities = capabilities
 }
 
+func (serverEntry *ServerEntry) GetDirectWebRequestPorts() []string {
+	ports := make([]string, 0)
+	if Contains(serverEntry.Capabilities, "handshake") {
+		// Server-side configuration quirk: there's a port forward from
+		// port 443 to the web server, which we can try, except on servers
+		// running FRONTED_MEEK, which listens on port 443.
+		if !serverEntry.SupportsProtocol(TUNNEL_PROTOCOL_FRONTED_MEEK) {
+			ports = append(ports, "443")
+		}
+		ports = append(ports, serverEntry.WebServerPort)
+	}
+	return ports
+}
+
 // DecodeServerEntry extracts server entries from the encoding
 // used by remote server lists and Psiphon server handshake requests.
 func DecodeServerEntry(encodedServerEntry string) (serverEntry *ServerEntry, err error) {

psiphon/splitTunnel.go

@@ -114,12 +114,12 @@ func (classifier *SplitTunnelClassifier) Start(fetchRoutesTunnel *Tunnel) {
 		return
 	}
 
-	if fetchRoutesTunnel.session == nil {
-		// Tunnel has no session
+	if fetchRoutesTunnel.serverContext == nil {
+		// Tunnel has no serverContext
 		return
 	}
 
-	if fetchRoutesTunnel.session.clientRegion == "" {
+	if fetchRoutesTunnel.serverContext.clientRegion == "" {
 		// Split tunnel region is unknown
 		return
 	}
@@ -207,7 +207,7 @@ func (classifier *SplitTunnelClassifier) setRoutes(tunnel *Tunnel) {
 		return
 	}
 
-	NoticeSplitTunnelRegion(tunnel.session.clientRegion)
+	NoticeSplitTunnelRegion(tunnel.serverContext.clientRegion)
 }
 
 // getRoutes makes a web request to download fresh routes data for the
@@ -216,13 +216,13 @@ func (classifier *SplitTunnelClassifier) setRoutes(tunnel *Tunnel) {
 // fails and cached routes data is present, that cached data is returned.
 func (classifier *SplitTunnelClassifier) getRoutes(tunnel *Tunnel) (routesData []byte, err error) {
 
-	url := fmt.Sprintf(classifier.fetchRoutesUrlFormat, tunnel.session.clientRegion)
+	url := fmt.Sprintf(classifier.fetchRoutesUrlFormat, tunnel.serverContext.clientRegion)
 	request, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return nil, ContextError(err)
 	}
 
-	etag, err := GetSplitTunnelRoutesETag(tunnel.session.clientRegion)
+	etag, err := GetSplitTunnelRoutesETag(tunnel.serverContext.clientRegion)
 	if err != nil {
 		return nil, ContextError(err)
 	}
@@ -310,7 +310,7 @@ func (classifier *SplitTunnelClassifier) getRoutes(tunnel *Tunnel) (routesData [
 	if !useCachedRoutes {
 		etag := response.Header.Get("ETag")
 		if etag != "" {
-			err := SetSplitTunnelRoutes(tunnel.session.clientRegion, etag, routesData)
+			err := SetSplitTunnelRoutes(tunnel.serverContext.clientRegion, etag, routesData)
 			if err != nil {
 				NoticeAlert("failed to cache split tunnel routes: %s", ContextError(err))
 				// Proceed with fetched data, even when we can't cache it
@@ -319,7 +319,7 @@ func (classifier *SplitTunnelClassifier) getRoutes(tunnel *Tunnel) (routesData [
 	}
 
 	if useCachedRoutes {
-		routesData, err = GetSplitTunnelRoutesData(tunnel.session.clientRegion)
+		routesData, err = GetSplitTunnelRoutesData(tunnel.serverContext.clientRegion)
 		if err != nil {
 			return nil, ContextError(err)
 		}

psiphon/transferstats/collector.go

@@ -20,7 +20,6 @@
 package transferstats
 
 import (
-	"encoding/json"
 	"sync"
 )
 
@@ -40,21 +39,39 @@ type hostStats struct {
 	numBytesReceived int64
 }
 
-func newHostStats() *hostStats {
-	return &hostStats{}
+// AccumulatedStats holds the Psiphon Server API status request data for a
+// given server. To accommodate status requests that may fail, and be retried,
+// the TakeOutStatsForServer/PutBackStatsForServer procedure allows the requester
+// to check out stats for reporting and merge back stats for a later retry.
+type AccumulatedStats struct {
+	hostnameToStats map[string]*hostStats
 }
 
-// serverStats holds per-server stats.
-type serverStats struct {
-	hostnameToStats    map[string]*hostStats
-	totalBytesSent     int64
-	totalBytesReceived int64
-}
+// GetStatsForStatusRequest summarizes AccumulatedStats data as
+// required for the Psiphon Server API status request.
+func (stats AccumulatedStats) GetStatsForStatusRequest() (map[string]int64, int64) {
 
-func newServerStats() *serverStats {
-	return &serverStats{
-		hostnameToStats: make(map[string]*hostStats),
+	hostBytes := make(map[string]int64)
+	bytesTransferred := int64(0)
+
+	for hostname, hostStats := range stats.hostnameToStats {
+		totalBytes := hostStats.numBytesReceived + hostStats.numBytesSent
+		bytesTransferred += totalBytes
+		hostBytes[hostname] = totalBytes
 	}
+
+	return hostBytes, bytesTransferred
+}
+
+// serverStats holds per-server stats.
+// accumulatedStats data is payload for the Psiphon status request
+// which is accessed via TakeOut/PutBack.
+// recentBytes data is for tunnel monitoring which is accessed via
+// ReportRecentBytesTransferredForServer.
+type serverStats struct {
+	accumulatedStats    *AccumulatedStats
+	recentBytesSent     int64
+	recentBytesReceived int64
 }
 
 // allStats is the root object that holds stats for all servers and all hosts,
@@ -73,10 +90,9 @@ type statsUpdate struct {
 }
 
 // recordStats makes sure the given stats update is added to the global
-// collection. Guaranteed to not block.
-// Callers of this function should assume that it "takes control" of the
-// statsUpdate object.
-func recordStat(stat *statsUpdate) {
+// collection. recentBytes are not adjusted when isPutBack is true,
+// as recentBytes aren't subject to TakeOut/PutBack.
+func recordStat(stat *statsUpdate, isPutBack bool) {
 	allStats.statsMutex.Lock()
 	defer allStats.statsMutex.Unlock()
 
@@ -86,51 +102,31 @@ func recordStat(stat *statsUpdate) {
 
 	storedServerStats := allStats.serverIDtoStats[stat.serverID]
 	if storedServerStats == nil {
-		storedServerStats = newServerStats()
+		storedServerStats = &serverStats{
+			accumulatedStats: &AccumulatedStats{
+				hostnameToStats: make(map[string]*hostStats)}}
 		allStats.serverIDtoStats[stat.serverID] = storedServerStats
 	}
 
-	storedHostStats := storedServerStats.hostnameToStats[stat.hostname]
+	storedHostStats := storedServerStats.accumulatedStats.hostnameToStats[stat.hostname]
 	if storedHostStats == nil {
-		storedHostStats = newHostStats()
-		storedServerStats.hostnameToStats[stat.hostname] = storedHostStats
+		storedHostStats = &hostStats{}
+		storedServerStats.accumulatedStats.hostnameToStats[stat.hostname] = storedHostStats
 	}
 
-	storedServerStats.totalBytesSent += stat.numBytesSent
-	storedServerStats.totalBytesReceived += stat.numBytesReceived
-
 	storedHostStats.numBytesSent += stat.numBytesSent
 	storedHostStats.numBytesReceived += stat.numBytesReceived
 
-	//fmt.Println("server:", stat.serverID, "host:", stat.hostname, "sent:", storedHostStats.numBytesSent, "received:", storedHostStats.numBytesReceived)
-}
-
-// Implement the json.Marshaler interface
-func (ss serverStats) MarshalJSON() ([]byte, error) {
-	out := make(map[string]interface{})
-
-	hostBytes := make(map[string]int64)
-	bytesTransferred := int64(0)
-
-	for hostname, hostStats := range ss.hostnameToStats {
-		totalBytes := hostStats.numBytesReceived + hostStats.numBytesSent
-		bytesTransferred += totalBytes
-		hostBytes[hostname] = totalBytes
+	if !isPutBack {
+		storedServerStats.recentBytesSent += stat.numBytesSent
+		storedServerStats.recentBytesReceived += stat.numBytesReceived
 	}
-
-	out["bytes_transferred"] = bytesTransferred
-	out["host_bytes"] = hostBytes
-
-	// We're not using these fields, but the server requires them
-	out["page_views"] = make([]string, 0)
-	out["https_requests"] = make([]string, 0)
-
-	return json.Marshal(out)
 }
 
-// GetBytesTransferredForServer returns total bytes sent and received since
-// the last call to GetBytesTransferredForServer.
-func GetBytesTransferredForServer(serverID string) (sent, received int64) {
+// ReportRecentBytesTransferredForServer returns bytes sent and received since
+// the last call to ReportRecentBytesTransferredForServer. The accumulated sent
+// and received are reset to 0 by this call.
+func ReportRecentBytesTransferredForServer(serverID string) (sent, received int64) {
 	allStats.statsMutex.Lock()
 	defer allStats.statsMutex.Unlock()
 
@@ -140,37 +136,49 @@ func GetBytesTransferredForServer(serverID string) (sent, received int64) {
 		return
 	}
 
-	sent = stats.totalBytesSent
-	received = stats.totalBytesReceived
+	sent = stats.recentBytesSent
+	received = stats.recentBytesReceived
 
-	stats.totalBytesSent = 0
-	stats.totalBytesReceived = 0
+	stats.recentBytesSent = 0
+	stats.recentBytesReceived = 0
 
 	return
 }
 
-// GetForServer returns the json-able stats package for the given server.
-func GetForServer(serverID string) (payload *serverStats) {
+// TakeOutStatsForServer borrows the AccumulatedStats for the specified
+// server. When we fail to report these stats, resubmit them with
+// PutBackStatsForServer. Stats will continue to be accumulated between
+// TakeOut and PutBack calls. The recentBytes values are unaffected by
+// TakeOut/PutBack. Returns empty stats if the serverID is not found.
+func TakeOutStatsForServer(serverID string) (accumulatedStats *AccumulatedStats) {
 	allStats.statsMutex.Lock()
 	defer allStats.statsMutex.Unlock()
 
-	payload = allStats.serverIDtoStats[serverID]
-	if payload == nil {
-		payload = newServerStats()
+	newAccumulatedStats := &AccumulatedStats{
+		hostnameToStats: make(map[string]*hostStats)}
+
+	// Note: for an existing serverStats, only the accumulatedStats is
+	// affected; the recentBytes fields are not changed.
+	serverStats := allStats.serverIDtoStats[serverID]
+	if serverStats != nil {
+		accumulatedStats = serverStats.accumulatedStats
+		serverStats.accumulatedStats = newAccumulatedStats
+	} else {
+		accumulatedStats = newAccumulatedStats
 	}
-	delete(allStats.serverIDtoStats, serverID)
 	return
 }
 
-// PutBack re-adds a set of server stats to the collection.
-func PutBack(serverID string, ss *serverStats) {
-	for hostname, hoststats := range ss.hostnameToStats {
+// PutBackStatsForServer re-adds a set of server stats to the collection.
+func PutBackStatsForServer(serverID string, accumulatedStats *AccumulatedStats) {
+	for hostname, hoststats := range accumulatedStats.hostnameToStats {
 		recordStat(
 			&statsUpdate{
 				serverID:         serverID,
 				hostname:         hostname,
 				numBytesSent:     hoststats.numBytesSent,
 				numBytesReceived: hoststats.numBytesReceived,
-			})
+			},
+			true)
 	}
 }

psiphon/transferstats/conn.go

@@ -47,8 +47,8 @@ type Conn struct {
 }
 
 // NewConn creates a Conn. serverID can be anything that uniquely
-// identifies the server; it will be passed to GetForServer() when retrieving
-// the accumulated stats.
+// identifies the server; it will be passed to TakeOutStatsForServer() when
+// retrieving the accumulated stats.
 func NewConn(nextConn net.Conn, serverID string, regexps *Regexps) *Conn {
 	return &Conn{
 		Conn:           nextConn,
@@ -85,7 +85,8 @@ func (conn *Conn) Write(buffer []byte) (n int, err error) {
 			conn.serverID,
 			conn.hostname,
 			int64(n),
-			0})
+			0},
+			false)
 	}
 
 	return
@@ -108,7 +109,8 @@ func (conn *Conn) Read(buffer []byte) (n int, err error) {
 		conn.serverID,
 		hostname,
 		0,
-		int64(n)})
+		int64(n)},
+		false)
 
 	return
 }

psiphon/transferstats/transferstats_test.go

@@ -91,20 +91,23 @@ func (suite *StatsTestSuite) Test_StatsConn() {
 	resp, err = suite.httpClient.Get("https://example.org/index.html")
 	suite.Nil(err, "basic HTTPS requests should succeed")
 	resp.Body.Close()
+
+	// Clear out stats
+	_ = TakeOutStatsForServer(_SERVER_ID)
 }
 
-func (suite *StatsTestSuite) Test_GetForServer() {
+func (suite *StatsTestSuite) Test_TakeOutStatsForServer() {
 
-	zeroPayload := newServerStats()
+	zeroPayload := &AccumulatedStats{hostnameToStats: make(map[string]*hostStats)}
 
-	payload := GetForServer(_SERVER_ID)
+	payload := TakeOutStatsForServer(_SERVER_ID)
 	suite.Equal(payload, zeroPayload, "should get zero stats before any traffic")
 
 	resp, err := suite.httpClient.Get("http://example.com/index.html")
 	suite.Nil(err, "need successful http to proceed with tests")
 	resp.Body.Close()
 
-	payload = GetForServer(_SERVER_ID)
+	payload = TakeOutStatsForServer(_SERVER_ID)
 	suite.NotNil(payload, "should receive valid payload for valid server ID")
 
 	payloadJSON, err := json.Marshal(payload)
@@ -113,28 +116,28 @@ func (suite *StatsTestSuite) Test_GetForServer() {
 	suite.Nil(err, "payload JSON should parse successfully")
 
 	// After we retrieve the stats for a server, they should be cleared out of the tracked stats
-	payload = GetForServer(_SERVER_ID)
+	payload = TakeOutStatsForServer(_SERVER_ID)
 	suite.Equal(payload, zeroPayload, "after retrieving stats for a server, there should be zero stats (until more data goes through)")
 }
 
-func (suite *StatsTestSuite) Test_PutBack() {
+func (suite *StatsTestSuite) Test_PutBackStatsForServer() {
 	resp, err := suite.httpClient.Get("http://example.com/index.html")
 	suite.Nil(err, "need successful http to proceed with tests")
 	resp.Body.Close()
 
-	payloadToPutBack := GetForServer(_SERVER_ID)
+	payloadToPutBack := TakeOutStatsForServer(_SERVER_ID)
 	suite.NotNil(payloadToPutBack, "should receive valid payload for valid server ID")
 
-	zeroPayload := newServerStats()
+	zeroPayload := &AccumulatedStats{hostnameToStats: make(map[string]*hostStats)}
 
-	payload := GetForServer(_SERVER_ID)
+	payload := TakeOutStatsForServer(_SERVER_ID)
 	suite.Equal(payload, zeroPayload, "should be zero stats after getting them")
 
-	PutBack(_SERVER_ID, payloadToPutBack)
+	PutBackStatsForServer(_SERVER_ID, payloadToPutBack)
 	// PutBack is asynchronous, so we'll need to wait a moment for it to do its thing
 	<-time.After(100 * time.Millisecond)
 
-	payload = GetForServer(_SERVER_ID)
+	payload = TakeOutStatsForServer(_SERVER_ID)
 	suite.NotEqual(payload, zeroPayload, "stats should be re-added after putting back")
 	suite.Equal(payload, payloadToPutBack, "stats should be the same as after the first retrieval")
 }
@@ -217,7 +220,7 @@ func (suite *StatsTestSuite) Test_Regex() {
 		suite.Nil(err)
 		resp.Body.Close()
 
-		payload := GetForServer(_SERVER_ID)
+		payload := TakeOutStatsForServer(_SERVER_ID)
 		suite.NotNil(payload, "should get stats because we made HTTP reqs; %s", scheme)
 
 		expectedHostnames := mapset.NewSet()

psiphon/tunnel.go

@@ -63,9 +63,12 @@ type TunnelOwner interface {
 // and an SSH session built on top of that transport.
 type Tunnel struct {
 	mutex                    *sync.Mutex
+	config                   *Config
+	untunneledDialConfig     *DialConfig
+	isDiscarded              bool
 	isClosed                 bool
 	serverEntry              *ServerEntry
-	session                  *Session
+	serverContext            *ServerContext
 	protocol                 string
 	conn                     net.Conn
 	sshClient                *ssh.Client
@@ -73,7 +76,7 @@ type Tunnel struct {
 	shutdownOperateBroadcast chan struct{}
 	signalPortForwardFailure chan struct{}
 	totalPortForwardFailures int
-	sessionStartTime         time.Time
+	startTime                time.Time
 }
 
 // EstablishTunnel first makes a network transport connection to the
@@ -85,8 +88,10 @@ type Tunnel struct {
 // HTTP (meek protocol).
 // When requiredProtocol is not blank, that protocol is used. Otherwise,
 // the a random supported protocol is used.
+// untunneledDialConfig is used for untunneled final status requests.
 func EstablishTunnel(
 	config *Config,
+	untunneledDialConfig *DialConfig,
 	sessionId string,
 	pendingConns *Conns,
 	serverEntry *ServerEntry,
@@ -115,6 +120,8 @@ func EstablishTunnel(
 	// The tunnel is now connected
 	tunnel = &Tunnel{
 		mutex:                    new(sync.Mutex),
+		config:                   config,
+		untunneledDialConfig:     untunneledDialConfig,
 		isClosed:                 false,
 		serverEntry:              serverEntry,
 		protocol:                 selectedProtocol,
@@ -127,41 +134,39 @@ func EstablishTunnel(
 		signalPortForwardFailure: make(chan struct{}, 1),
 	}
 
-	// Create a new Psiphon API session for this tunnel. This includes performing
-	// a handshake request. If the handshake fails, this establishment fails.
-	//
-	// TODO: as long as the servers are not enforcing that a client perform a handshake,
-	// proceed with this tunnel as long as at least one previous handhake succeeded?
-	//
+	// Create a new Psiphon API server context for this tunnel. This includes
+	// performing a handshake request. If the handshake fails, this establishment
+	// fails.
 	if !config.DisableApi {
-		NoticeInfo("starting session for %s", tunnel.serverEntry.IpAddress)
-		tunnel.session, err = NewSession(config, tunnel, sessionId)
+		NoticeInfo("starting server context for %s", tunnel.serverEntry.IpAddress)
+		tunnel.serverContext, err = NewServerContext(tunnel, sessionId)
 		if err != nil {
-			return nil, ContextError(fmt.Errorf("error starting session for %s: %s", tunnel.serverEntry.IpAddress, err))
+			return nil, ContextError(
+				fmt.Errorf("error starting server context for %s: %s",
+					tunnel.serverEntry.IpAddress, err))
 		}
 	}
 
-	tunnel.sessionStartTime = time.Now()
+	tunnel.startTime = time.Now()
 
 	// Now that network operations are complete, cancel interruptibility
 	pendingConns.Remove(conn)
 
-	// Promote this successful tunnel to first rank so it's one
-	// of the first candidates next time establish runs.
-	PromoteServerEntry(tunnel.serverEntry.IpAddress)
-
 	// Spawn the operateTunnel goroutine, which monitors the tunnel and handles periodic stats updates.
 	tunnel.operateWaitGroup.Add(1)
-	go tunnel.operateTunnel(config, tunnelOwner)
+	go tunnel.operateTunnel(tunnelOwner)
 
 	return tunnel, nil
 }
 
 // Close stops operating the tunnel and closes the underlying connection.
 // Supports multiple and/or concurrent calls to Close().
-func (tunnel *Tunnel) Close() {
+// When isDicarded is set, operateTunnel will not attempt to send final
+// status requests.
+func (tunnel *Tunnel) Close(isDiscarded bool) {
 
 	tunnel.mutex.Lock()
+	tunnel.isDiscarded = isDiscarded
 	isClosed := tunnel.isClosed
 	tunnel.isClosed = true
 	tunnel.mutex.Unlock()
@@ -185,6 +190,13 @@ func (tunnel *Tunnel) Close() {
 	}
 }
 
+// IsDiscarded returns the tunnel's discarded flag.
+func (tunnel *Tunnel) IsDiscarded() bool {
+	tunnel.mutex.Lock()
+	defer tunnel.mutex.Unlock()
+	return tunnel.isDiscarded
+}
+
 // Dial establishes a port forward connection through the tunnel
 // This Dial doesn't support split tunnel, so alwaysTunnel is not referenced
 func (tunnel *Tunnel) Dial(
@@ -226,12 +238,12 @@ func (tunnel *Tunnel) Dial(
 		tunnel:         tunnel,
 		downstreamConn: downstreamConn}
 
-	// Tunnel does not have a session when DisableApi is set. We still use
+	// Tunnel does not have a serverContext when DisableApi is set. We still use
 	// transferstats.Conn to count bytes transferred for monitoring tunnel
 	// quality.
 	var regexps *transferstats.Regexps
-	if tunnel.session != nil {
-		regexps = tunnel.session.StatsRegexps()
+	if tunnel.serverContext != nil {
+		regexps = tunnel.serverContext.StatsRegexps()
 	}
 	conn = transferstats.NewConn(conn, tunnel.serverEntry.IpAddress, regexps)
 
@@ -242,7 +254,7 @@ func (tunnel *Tunnel) Dial(
 // This will terminate the tunnel.
 func (tunnel *Tunnel) SignalComponentFailure() {
 	NoticeAlert("tunnel received component failure signal")
-	tunnel.Close()
+	tunnel.Close(false)
 }
 
 // TunneledConn implements net.Conn and wraps a port foward connection.
@@ -331,17 +343,24 @@ func dialSsh(
 	// So depending on which protocol is used, multiple layers are initialized.
 	port := 0
 	useMeek := false
+	useMeekHttps := false
 	useFronting := false
 	useObfuscatedSsh := false
 	switch selectedProtocol {
 	case TUNNEL_PROTOCOL_FRONTED_MEEK:
 		useMeek = true
+		useMeekHttps = true
 		useFronting = true
 		useObfuscatedSsh = true
 	case TUNNEL_PROTOCOL_UNFRONTED_MEEK:
 		useMeek = true
 		useObfuscatedSsh = true
 		port = serverEntry.SshObfuscatedPort
+	case TUNNEL_PROTOCOL_UNFRONTED_MEEK_HTTPS:
+		useMeek = true
+		useMeekHttps = true
+		useObfuscatedSsh = true
+		port = serverEntry.SshObfuscatedPort
 	case TUNNEL_PROTOCOL_OBFUSCATED_SSH:
 		useObfuscatedSsh = true
 		port = serverEntry.SshObfuscatedPort
@@ -391,7 +410,7 @@ func dialSsh(
 		TrustedCACertificatesFilename: config.TrustedCACertificatesFilename,
 	}
 	if useMeek {
-		conn, err = DialMeek(serverEntry, sessionId, frontingAddress, dialConfig)
+		conn, err = DialMeek(serverEntry, sessionId, useMeekHttps, frontingAddress, dialConfig)
 		if err != nil {
 			return nil, nil, ContextError(err)
 		}
@@ -535,7 +554,7 @@ func dialSsh(
 // TODO: change "recently active" to include having received any
 // SSH protocol messages from the server, not just user payload?
 //
-func (tunnel *Tunnel) operateTunnel(config *Config, tunnelOwner TunnelOwner) {
+func (tunnel *Tunnel) operateTunnel(tunnelOwner TunnelOwner) {
 	defer tunnel.operateWaitGroup.Done()
 
 	lastBytesReceivedTime := time.Now()
@@ -544,11 +563,6 @@ func (tunnel *Tunnel) operateTunnel(config *Config, tunnelOwner TunnelOwner) {
 	totalSent := int64(0)
 	totalReceived := int64(0)
 
-	// Always emit a final NoticeTotalBytesTransferred
-	defer func() {
-		NoticeTotalBytesTransferred(tunnel.serverEntry.IpAddress, totalSent, totalReceived)
-	}()
-
 	noticeBytesTransferredTicker := time.NewTicker(1 * time.Second)
 	defer noticeBytesTransferredTicker.Stop()
 
@@ -564,6 +578,22 @@ func (tunnel *Tunnel) operateTunnel(config *Config, tunnelOwner TunnelOwner) {
 	statsTimer := time.NewTimer(nextStatusRequestPeriod())
 	defer statsTimer.Stop()
 
+	// Schedule an immediate status request to deliver any unreported
+	// tunnel stats.
+	// Note: this may not be effective when there's an outstanding
+	// asynchronous untunneled final status request is holding the
+	// tunnel stats records. It may also conflict with other
+	// tunnel candidates which attempt to send an immediate request
+	// before being discarded. For now, we mitigate this with a short,
+	// random delay.
+	unreported := CountUnreportedTunnelStats()
+	if unreported > 0 {
+		NoticeInfo("Unreported tunnel stats: %d", unreported)
+		statsTimer.Reset(MakeRandomPeriod(
+			PSIPHON_API_STATUS_REQUEST_SHORT_PERIOD_MIN,
+			PSIPHON_API_STATUS_REQUEST_SHORT_PERIOD_MAX))
+	}
+
 	nextSshKeepAlivePeriod := func() time.Duration {
 		return MakeRandomPeriod(
 			TUNNEL_SSH_KEEP_ALIVE_PERIOD_MIN,
@@ -572,7 +602,7 @@ func (tunnel *Tunnel) operateTunnel(config *Config, tunnelOwner TunnelOwner) {
 
 	// TODO: don't initialize timer when config.DisablePeriodicSshKeepAlive is set
 	sshKeepAliveTimer := time.NewTimer(nextSshKeepAlivePeriod())
-	if config.DisablePeriodicSshKeepAlive {
+	if tunnel.config.DisablePeriodicSshKeepAlive {
 		sshKeepAliveTimer.Stop()
 	} else {
 		defer sshKeepAliveTimer.Stop()
@@ -580,13 +610,10 @@ func (tunnel *Tunnel) operateTunnel(config *Config, tunnelOwner TunnelOwner) {
 
 	// Perform network requests in separate goroutines so as not to block
 	// other operations.
-	// Note: defer LIFO dependency: channels to be closed before Wait()
 	requestsWaitGroup := new(sync.WaitGroup)
-	defer requestsWaitGroup.Wait()
 
 	requestsWaitGroup.Add(1)
 	signalStatusRequest := make(chan struct{})
-	defer close(signalStatusRequest)
 	go func() {
 		defer requestsWaitGroup.Done()
 		for _ = range signalStatusRequest {
@@ -597,7 +624,6 @@ func (tunnel *Tunnel) operateTunnel(config *Config, tunnelOwner TunnelOwner) {
 	requestsWaitGroup.Add(1)
 	signalSshKeepAlive := make(chan time.Duration)
 	sshKeepAliveError := make(chan error, 1)
-	defer close(signalSshKeepAlive)
 	go func() {
 		defer requestsWaitGroup.Done()
 		for timeout := range signalSshKeepAlive {
@@ -611,11 +637,12 @@ func (tunnel *Tunnel) operateTunnel(config *Config, tunnelOwner TunnelOwner) {
 		}
 	}()
 
+	shutdown := false
 	var err error
-	for err == nil {
+	for !shutdown && err == nil {
 		select {
 		case <-noticeBytesTransferredTicker.C:
-			sent, received := transferstats.GetBytesTransferredForServer(
+			sent, received := transferstats.ReportRecentBytesTransferredForServer(
 				tunnel.serverEntry.IpAddress)
 
 			if received > 0 {
@@ -631,7 +658,7 @@ func (tunnel *Tunnel) operateTunnel(config *Config, tunnelOwner TunnelOwner) {
 			}
 
 			// Only emit the frequent BytesTransferred notice when tunnel is not idle.
-			if config.EmitBytesTransferred && (sent > 0 || received > 0) {
+			if tunnel.config.EmitBytesTransferred && (sent > 0 || received > 0) {
 				NoticeBytesTransferred(tunnel.serverEntry.IpAddress, sent, received)
 			}
 
@@ -663,22 +690,82 @@ func (tunnel *Tunnel) operateTunnel(config *Config, tunnelOwner TunnelOwner) {
 				default:
 				}
 			}
-			if !config.DisablePeriodicSshKeepAlive {
+			if !tunnel.config.DisablePeriodicSshKeepAlive {
 				sshKeepAliveTimer.Reset(nextSshKeepAlivePeriod())
 			}
 
 		case err = <-sshKeepAliveError:
 
 		case <-tunnel.shutdownOperateBroadcast:
-			// Attempt to send any remaining stats
-			sendStats(tunnel)
-			NoticeInfo("shutdown operate tunnel")
-			return
+			shutdown = true
 		}
 	}
 
-	if err != nil {
+	close(signalSshKeepAlive)
+	close(signalStatusRequest)
+	requestsWaitGroup.Wait()
+
+	// Capture bytes transferred since the last noticeBytesTransferredTicker tick
+	sent, received := transferstats.ReportRecentBytesTransferredForServer(tunnel.serverEntry.IpAddress)
+	totalSent += sent
+	totalReceived += received
+
+	// Always emit a final NoticeTotalBytesTransferred
+	NoticeTotalBytesTransferred(tunnel.serverEntry.IpAddress, totalSent, totalReceived)
+
+	// The stats for this tunnel will be reported via the next successful
+	// status request.
+	// Note: Since client clocks are unreliable, we use the server's reported
+	// timestamp in the handshake response as the tunnel start time. This time
+	// will be slightly earlier than the actual tunnel activation time, as the
+	// client has to receive and parse the response and activate the tunnel.
+	if !tunnel.IsDiscarded() {
+		err := RecordTunnelStats(
+			tunnel.serverContext.sessionId,
+			tunnel.serverContext.tunnelNumber,
+			tunnel.serverEntry.IpAddress,
+			tunnel.serverContext.serverHandshakeTimestamp,
+			fmt.Sprintf("%d", time.Now().Sub(tunnel.startTime)),
+			totalSent,
+			totalReceived)
+		if err != nil {
+			NoticeAlert("RecordTunnelStats failed: %s", ContextError(err))
+		}
+	}
+
+	// Final status request notes:
+	//
+	// It's highly desirable to send a final status request in order to report
+	// domain bytes transferred stats as well as to report tunnel stats as
+	// soon as possible. For this reason, we attempt untunneled requests when
+	// the tunneled request isn't possible or has failed.
+	//
+	// In an orderly shutdown (err == nil), the Controller is stopping and
+	// everything must be wrapped up quickly. Also, we still have a working
+	// tunnel. So we first attempt a tunneled status request (with a short
+	// timeout) and then attempt, synchronously -- otherwise the Contoller's
+	// untunneledPendingConns.CloseAll() will immediately interrupt untunneled
+	// requests -- untunneled requests (also with short timeouts).
+	// Note that this depends on the order of untunneledPendingConns.CloseAll()
+	// coming after tunnel.Close(): see note in Controller.Run().
+	//
+	// If the tunnel has failed, the Controller may continue working. We want
+	// to re-establish as soon as possible (so don't want to block on status
+	// requests, even for a second). We may have a long time to attempt
+	// untunneled requests in the background. And there is no tunnel through
+	// which to attempt tunneled requests. So we spawn a goroutine to run the
+	// untunneled requests, which are allowed a longer timeout. These requests
+	// will be interrupted by the Controller's untunneledPendingConns.CloseAll()
+	// in the case of a shutdown.
+
+	if err == nil {
+		NoticeInfo("shutdown operate tunnel")
+		if !sendStats(tunnel) {
+			sendUntunneledStats(tunnel, true)
+		}
+	} else {
 		NoticeAlert("operate tunnel error for %s: %s", tunnel.serverEntry.IpAddress, err)
+		go sendUntunneledStats(tunnel, false)
 		tunnelOwner.SignalTunnelFailure(tunnel)
 	}
 }
@@ -712,17 +799,43 @@ func sendSshKeepAlive(
 }
 
 // sendStats is a helper for sending session stats to the server.
-func sendStats(tunnel *Tunnel) {
+func sendStats(tunnel *Tunnel) bool {
 
-	// Tunnel does not have a session when DisableApi is set
-	if tunnel.session == nil {
-		return
+	// Tunnel does not have a serverContext when DisableApi is set
+	if tunnel.serverContext == nil {
+		return true
 	}
 
-	payload := transferstats.GetForServer(tunnel.serverEntry.IpAddress)
-	err := tunnel.session.DoStatusRequest(payload)
+	// Skip when tunnel is discarded
+	if tunnel.IsDiscarded() {
+		return true
+	}
+
+	err := tunnel.serverContext.DoStatusRequest(tunnel)
 	if err != nil {
 		NoticeAlert("DoStatusRequest failed for %s: %s", tunnel.serverEntry.IpAddress, err)
-		transferstats.PutBack(tunnel.serverEntry.IpAddress, payload)
+	}
+
+	return err == nil
+}
+
+// sendUntunnelStats sends final status requests directly to Psiphon
+// servers after the tunnel has already failed. This is an attempt
+// to retain useful bytes transferred stats.
+func sendUntunneledStats(tunnel *Tunnel, isShutdown bool) {
+
+	// Tunnel does not have a serverContext when DisableApi is set
+	if tunnel.serverContext == nil {
+		return
+	}
+
+	// Skip when tunnel is discarded
+	if tunnel.IsDiscarded() {
+		return
+	}
+
+	err := TryUntunneledStatusRequest(tunnel, isShutdown)
+	if err != nil {
+		NoticeAlert("TryUntunneledStatusRequest failed for %s: %s", tunnel.serverEntry.IpAddress, err)
 	}
 }

psiphon/upgradeDownload.go

@@ -20,9 +20,10 @@
 package psiphon
 
 import (
+	"errors"
 	"fmt"
 	"io"
-	"net"
+	"io/ioutil"
 	"net/http"
 	"os"
 )
@@ -41,9 +42,17 @@ func DownloadUpgrade(config *Config, clientUpgradeVersion string, tunnel *Tunnel
 		return nil
 	}
 
+	httpClient, err := MakeTunneledHttpClient(config, tunnel, DOWNLOAD_UPGRADE_TIMEOUT)
+	if err != nil {
+		return ContextError(err)
+	}
+
 	partialFilename := fmt.Sprintf(
 		"%s.%s.part", config.UpgradeDownloadFilename, clientUpgradeVersion)
 
+	partialETagFilename := fmt.Sprintf(
+		"%s.%s.part.etag", config.UpgradeDownloadFilename, clientUpgradeVersion)
+
 	file, err := os.OpenFile(partialFilename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
 	if err != nil {
 		return ContextError(err)
@@ -55,32 +64,50 @@ func DownloadUpgrade(config *Config, clientUpgradeVersion string, tunnel *Tunnel
 		return ContextError(err)
 	}
 
+	// A partial download should have an ETag which is to be sent with the
+	// Range request to ensure that the source object is the same as the
+	// one that is partially downloaded.
+	var partialETag []byte
+	if fileInfo.Size() > 0 {
+
+		partialETag, err = ioutil.ReadFile(partialETagFilename)
+
+		// When the ETag can't be loaded, delete the partial download. To keep the
+		// code simple, there is no immediate, inline retry here, on the assumption
+		// that the controller's upgradeDownloader will shortly call DownloadUpgrade
+		// again.
+		if err != nil {
+			os.Remove(partialFilename)
+			os.Remove(partialETagFilename)
+			return ContextError(
+				fmt.Errorf("failed to load partial download ETag: %s", err))
+		}
+
+	}
+
 	request, err := http.NewRequest("GET", config.UpgradeDownloadUrl, nil)
 	if err != nil {
 		return ContextError(err)
 	}
 	request.Header.Add("Range", fmt.Sprintf("bytes=%d-", fileInfo.Size()))
 
-	tunneledDialer := func(_, addr string) (conn net.Conn, err error) {
-		return tunnel.sshClient.Dial("tcp", addr)
-	}
-	transport := &http.Transport{
-		Dial: tunneledDialer,
-		ResponseHeaderTimeout: DOWNLOAD_UPGRADE_TIMEOUT,
-	}
-	httpClient := &http.Client{
-		Transport: transport,
-		Timeout:   DOWNLOAD_UPGRADE_TIMEOUT,
+	// Note: not using If-Range, since not all remote server list host servers
+	// support it. Using If-Match means we need to check for status code 412
+	// and reset when the ETag has changed since the last partial download.
+	if partialETag != nil {
+		request.Header.Add("If-Match", string(partialETag))
 	}
 
 	response, err := httpClient.Do(request)
 
 	// The resumeable download may ask for bytes past the resource range
 	// since it doesn't store the "completed download" state. In this case,
-	// the HTTP server returns 416. Otherwise, we expect 206.
+	// the HTTP server returns 416. Otherwise, we expect 206. We may also
+	// receive 412 on ETag mismatch.
 	if err == nil &&
 		(response.StatusCode != http.StatusPartialContent &&
-			response.StatusCode != http.StatusRequestedRangeNotSatisfiable) {
+			response.StatusCode != http.StatusRequestedRangeNotSatisfiable &&
+			response.StatusCode != http.StatusPreconditionFailed) {
 		response.Body.Close()
 		err = fmt.Errorf("unexpected response status code: %d", response.StatusCode)
 	}
@@ -89,6 +116,20 @@ func DownloadUpgrade(config *Config, clientUpgradeVersion string, tunnel *Tunnel
 	}
 	defer response.Body.Close()
 
+	if response.StatusCode == http.StatusPreconditionFailed {
+		// When the ETag no longer matches, delete the partial download. As above,
+		// simply failing and relying on the controller's upgradeDownloader retry.
+		os.Remove(partialFilename)
+		os.Remove(partialETagFilename)
+		return ContextError(errors.New("partial download ETag mismatch"))
+	}
+
+	// Not making failure to write ETag file fatal, in case the entire download
+	// succeeds in this one request.
+	ioutil.WriteFile(partialETagFilename, []byte(response.Header.Get("ETag")), 0600)
+
+	// A partial download occurs when this copy is interrupted. The io.Copy
+	// will fail, leaving a partial download in place (.part and .part.etag).
 	n, err := io.Copy(NewSyncFileWriter(file), response.Body)
 	if err != nil {
 		return ContextError(err)
@@ -108,6 +149,8 @@ func DownloadUpgrade(config *Config, clientUpgradeVersion string, tunnel *Tunnel
 		return ContextError(err)
 	}
 
+	os.Remove(partialETagFilename)
+
 	NoticeClientUpgradeDownloaded(config.UpgradeDownloadFilename)
 
 	return nil

psiphon/upstreamproxy/proxy_socks4.go

@@ -96,13 +96,17 @@ func (s *socks4Proxy) Dial(network, addr string) (net.Conn, error) {
 	}
 
 	// Deal with the destination address/string.
-	ipStr, portStr, err := net.SplitHostPort(addr)
+	hostStr, portStr, err := net.SplitHostPort(addr)
+	domainDest := ""
 	if err != nil {
 		return nil, proxyError(fmt.Errorf("parsing destination address: %v", err))
 	}
-	ip := net.ParseIP(ipStr)
+	ip := net.ParseIP(hostStr)
 	if ip == nil {
-		return nil, proxyError(fmt.Errorf("failed to parse destination IP"))
+		// hostStr is not representing an IP, probably a domain name
+		// try to put an invalid IP into DSTIP field and
+		// append domain name terminated by '\x00' at the end of request
+		ip = net.IPv4(0, 0, 0, 1)
 	}
 	ip4 := ip.To4()
 	if ip4 == nil {
@@ -133,6 +137,10 @@ func (s *socks4Proxy) Dial(network, addr string) (net.Conn, error) {
 		req = append(req, s.username...)
 	}
 	req = append(req, socks4Null)
+	if domainDest != "" {
+		req = append(req, domainDest...)
+		req = append(req, socks4Null)
+	}
 	_, err = c.Write(req)
 	if err != nil {
 		c.Close()
@@ -176,6 +184,5 @@ func socks4ErrorToString(code byte) string {
 }
 
 func init() {
-	// Despite the scheme name, this really is SOCKS4.
 	proxy.RegisterDialerType("socks4a", newSOCKS4)
 }