Merge branch 'master' of https://github.com/Psiphon-Labs/psiphon-tunnel-core

AndroidLibrary/Dockerfile

@@ -1,10 +1,12 @@
 # Dockerfile to build an image with the local version of psiphon-tunnel-core.
 #
-#  > docker build -t psigobuild $GOPATH/src/github.com/Psiphon-Labs/psiphon-tunnel-core/AndroidApp
-#  > docker run -it --rm -v $GOPATH/src:/src psigobuild
+# See README.md for usage instructions.
 
+# TODO: Switch to debian:testing
 FROM ubuntu:12.04
 
+ENV GOVERSION=go1.4
+
 # Install system-level dependencies.
 ENV DEBIAN_FRONTEND=noninteractive
 RUN echo "debconf shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections && \
@@ -54,7 +56,7 @@ ENV GOROOT=/go \
   GOPATH=/
 ENV PATH=$PATH:$GOROOT/bin
 RUN echo "INSTALLING GO" && \
-  curl -L https://github.com/golang/go/archive/master.zip -o /tmp/go.zip && \
+  curl -L https://github.com/golang/go/archive/$GOVERSION.zip -o /tmp/go.zip && \
   unzip /tmp/go.zip && \
   rm /tmp/go.zip && \
   mv /go-master $GOROOT && \

AndroidLibrary/README.md

@@ -48,20 +48,22 @@ Follow Go Android documentation:
 
 ### Building with Docker
 
+Note that you may need to use `sudo docker` below, depending on your OS.
+
 Create the build image:
 
 ```bash
 # While in the same directory as the Dockerfile...
-$ sudo docker build -t psibuild .
+$ docker build --no-cache=true -t psigoandroid .
 # That will take a long time to complete.
-# After it's done, you'll have an image called "psibuild". Check with...
-$ sudo docker images
+# After it's done, you'll have an image called "psigoandroid". Check with...
+$ docker images
 ```
 
 To do the build:
 
 ```bash
-$ sudo docker run -v $GOPATH/src:/src psibuild /bin/bash -c 'cd /src/github.com/Psiphon-Labs/psiphon-tunnel-core/AndroidLibrary && ./make.bash'
+$ docker run --rm -v $GOPATH/src:/src psigoandroid /bin/bash -c 'cd /src/github.com/Psiphon-Labs/psiphon-tunnel-core/AndroidLibrary && ./make.bash'
 ```
 
 When that command completes, the compiled library will be located at `libs/armeabi-v7a/libgojni.so`.

AndroidLibrary/make.bash

@@ -7,8 +7,6 @@ if [ ! -f make.bash ]; then
   exit 1
 fi
 
-ANDROID_APP=$PWD
-
 # Make sure we have our dependencies
 echo 'go-getting dependencies...'
 go get -d -v ./...

ConsoleClient/Dockerfile

@@ -0,0 +1,37 @@
+# Dockerfile to build an image with the local version of psiphon-tunnel-core.
+#
+# See README.md for usage instructions.
+
+FROM debian:testing
+
+ENV GOVERSION=go1.4
+
+# Install system-level dependencies.
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+  apt-get -y install build-essential python-software-properties bzip2 unzip curl \
+    git subversion mercurial bzr \
+    upx gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 gcc-multilib
+
+# Install Go.
+ENV GOROOT=/go \
+  GOPATH=/
+ENV PATH=$PATH:$GOROOT/bin
+RUN echo "INSTALLING GO" && \
+  curl -L https://github.com/golang/go/archive/$GOVERSION.zip -o /tmp/go.zip && \
+  unzip /tmp/go.zip && \
+  rm /tmp/go.zip && \
+  mv /go-$GOVERSION $GOROOT && \
+  echo $GOVERSION > $GOROOT/VERSION && \
+  cd $GOROOT/src && \
+  ./all.bash
+
+ENV CGO_ENABLED=1
+RUN go get github.com/mitchellh/gox && \
+  go get github.com/inconshreveable/gonative && \
+  mkdir -p /usr/local/gonative && \
+  cd /usr/local/gonative && \
+  gonative build
+ENV PATH=/usr/local/gonative/go/bin:$PATH
+
+WORKDIR $GOPATH/src

ConsoleClient/README.md

@@ -0,0 +1,24 @@
+Psiphon Console Client README
+================================================================================
+
+### Building with Docker
+
+Note that you may need to use `sudo docker` below, depending on your OS.
+
+Create the build image:
+
+```bash
+# While in the same directory as the Dockerfile...
+$ docker build --no-cache=true -t psigoconsole .
+# That will take a long time to complete.
+# After it's done, you'll have an image called "psigoconsole". Check with...
+$ docker images
+```
+
+To do the build:
+
+```bash
+$ docker run --rm -v $GOPATH/src:/src psigoconsole /bin/bash -c 'cd /src/github.com/Psiphon-Labs/psiphon-tunnel-core/ConsoleClient && ./make.bash'
+```
+
+When that command completes, the compiled library will be located at `windows_386/psiphon-tunnel-core.exe`.

ConsoleClient/make.bash

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [ ! -f make.bash ]; then
+  echo 'make.bash must be run from $GOPATH/src/github.com/Psiphon-Labs/psiphon-tunnel-core/ConsoleClient'
+  exit 1
+fi
+
+# Make sure we have our dependencies
+echo 'go-getting dependencies...'
+go get -d -v ./...
+
+CGO_ENABLED=1
+
+echo 'Building windows-386...'
+CC=/usr/bin/i686-w64-mingw32-gcc \
+  gox -verbose -osarch windows/386 -output windows_386_psiphon-tunnel-core
+upx --best windows_386_psiphon-tunnel-core.exe
+
+echo 'Building windows-amd64...'
+CC=/usr/bin/x86_64-w64-mingw32-gcc \
+  gox -verbose -osarch windows/amd64 -output windows_amd64_psiphon-tunnel-core
+upx --best windows_amd64_psiphon-tunnel-core.exe
+
+echo 'Building linux-amd64...'
+gox -verbose -osarch linux/amd64 -output linux_amd64_psiphon-tunnel-core
+upx --best linux_amd64_psiphon-tunnel-core
+
+echo 'Building linux-386...'
+CFLAGS=-m32 \
+  gox -verbose -osarch linux/386 -output linux_386_psiphon-tunnel-core
+upx --best linux_386_psiphon-tunnel-core

psiphon/serverApi.go

@@ -26,6 +26,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/transferstats"
 	"io"
 	"io/ioutil"
 	"net"
@@ -41,7 +42,7 @@ type Session struct {
 	sessionId          string
 	baseRequestUrl     string
 	psiphonHttpsClient *http.Client
-	statsRegexps       *Regexps
+	statsRegexps       *transferstats.Regexps
 	statsServerId      string
 }
 
@@ -128,7 +129,7 @@ func (session *Session) StatsServerID() string {
 }
 
 // StatsRegexps gets the Regexps used for the statistics for this tunnel.
-func (session *Session) StatsRegexps() *Regexps {
+func (session *Session) StatsRegexps() *transferstats.Regexps {
 	return session.statsRegexps
 }
 
@@ -235,9 +236,15 @@ func (session *Session) doHandshakeRequest() error {
 		NoticeClientUpgradeAvailable(handshakeConfig.UpgradeClientVersion)
 	}
 
-	session.statsRegexps = MakeRegexps(
+	var regexpsNotices []string
+	session.statsRegexps, regexpsNotices = transferstats.MakeRegexps(
 		handshakeConfig.PageViewRegexes,
 		handshakeConfig.HttpsRequestRegexes)
+
+	for _, notice := range regexpsNotices {
+		NoticeAlert(notice)
+	}
+
 	return nil
 }
 

psiphon/transferstats/README.md

@@ -0,0 +1,15 @@
+`transferstats` Package
+=======================
+
+This provides a `net.Conn` interface implementation that can be put in a chain
+of connections and used to collect transfer statistics for the network traffic
+passing through it.
+
+Total bytes transferred is recorded, as well as per-hostname bytes transferred
+stats for HTTP and HTTPS traffic (as long as the HTTPS traffic contains [SNI]
+information). Which hostnames are recorded is specified by a set of regular
+expressions.
+
+[SNI]: https://en.wikipedia.org/wiki/Server_Name_Indication
+
+(TODO: More info.)

psiphon/stats_collector.go → psiphon/transferstats/collector.go

@@ -17,7 +17,7 @@
  *
  */
 
-package psiphon
+package transferstats
 
 import (
 	"encoding/json"
@@ -116,9 +116,6 @@ func (ss serverStats) MarshalJSON() ([]byte, error) {
 	out["bytes_transferred"] = bytesTransferred
 	out["host_bytes"] = hostBytes
 
-	noticeJSON, _ := json.Marshal(out)
-	NoticeInfo("sending stats: %s", noticeJSON)
-
 	// We're not using these fields, but the server requires them
 	out["page_views"] = make([]string, 0)
 	out["https_requests"] = make([]string, 0)

psiphon/stats_conn.go → psiphon/transferstats/conn.go

@@ -19,7 +19,7 @@
 
 // Package stats counts and keeps track of session stats. These are per-domain
 // bytes transferred and total bytes transferred.
-package psiphon
+package transferstats
 
 /*
 Assumption: The same connection will not be used to access different hostnames
@@ -35,9 +35,9 @@ import (
 	"sync/atomic"
 )
 
-// StatsConn is to be used as an intermediate link in a chain of net.Conn objects.
+// Conn is to be used as an intermediate link in a chain of net.Conn objects.
 // It inspects requests and responses and derives stats from them.
-type StatsConn struct {
+type Conn struct {
 	net.Conn
 	serverID       string
 	firstWrite     int32
@@ -46,11 +46,11 @@ type StatsConn struct {
 	regexps        *Regexps
 }
 
-// NewStatsConn creates a StatsConn. serverID can be anything that uniquely
+// NewConn creates a Conn. serverID can be anything that uniquely
 // identifies the server; it will be passed to GetForServer() when retrieving
 // the accumulated stats.
-func NewStatsConn(nextConn net.Conn, serverID string, regexps *Regexps) *StatsConn {
-	return &StatsConn{
+func NewConn(nextConn net.Conn, serverID string, regexps *Regexps) *Conn {
+	return &Conn{
 		Conn:           nextConn,
 		serverID:       serverID,
 		firstWrite:     1,
@@ -61,7 +61,7 @@ func NewStatsConn(nextConn net.Conn, serverID string, regexps *Regexps) *StatsCo
 
 // Write is called when requests are being written out through the tunnel to
 // the remote server.
-func (conn *StatsConn) Write(buffer []byte) (n int, err error) {
+func (conn *Conn) Write(buffer []byte) (n int, err error) {
 	// First pass the data down the chain.
 	n, err = conn.Conn.Write(buffer)
 
@@ -92,7 +92,7 @@ func (conn *StatsConn) Write(buffer []byte) (n int, err error) {
 }
 
 // Read is called when responses to requests are being read from the remote server.
-func (conn *StatsConn) Read(buffer []byte) (n int, err error) {
+func (conn *Conn) Read(buffer []byte) (n int, err error) {
 	n, err = conn.Conn.Read(buffer)
 
 	var hostname string

psiphon/stats_hostname.go → psiphon/transferstats/hostname.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, Psiphon Inc.
+ * Copyright (c) 2015, Psiphon Inc.
  * All rights reserved.
  *
  * This program is free software: you can redistribute it and/or modify
@@ -17,7 +17,7 @@
  *
  */
 
-package psiphon
+package transferstats
 
 import (
 	"bufio"

psiphon/stats_regexp.go → psiphon/transferstats/regexp.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, Psiphon Inc.
+ * Copyright (c) 2015, Psiphon Inc.
  * All rights reserved.
  *
  * This program is free software: you can redistribute it and/or modify
@@ -17,9 +17,12 @@
  *
  */
 
-package psiphon
+package transferstats
 
-import "regexp"
+import (
+	"fmt"
+	"regexp"
+)
 
 type regexpReplace struct {
 	regexp  *regexp.Regexp
@@ -32,33 +35,36 @@ type Regexps []regexpReplace
 
 // MakeRegexps takes the raw string-map form of the regex-replace pairs
 // returned by the server handshake and turns them into a usable object.
-func MakeRegexps(pageViewRegexes, httpsRequestRegexes []map[string]string) *Regexps {
-	regexps := make(Regexps, 0)
+func MakeRegexps(pageViewRegexes, httpsRequestRegexes []map[string]string) (regexps *Regexps, notices []string) {
+	regexpsSlice := make(Regexps, 0)
+	notices = make([]string, 0)
 
 	// We aren't doing page view stats anymore, so we won't process those regexps.
 	for _, rr := range httpsRequestRegexes {
 		regexString := rr["regex"]
 		if regexString == "" {
-			NoticeAlert("MakeRegexps: empty regex")
+			notices = append(notices, "MakeRegexps: empty regex")
 			continue
 		}
 
 		replace := rr["replace"]
 		if replace == "" {
-			NoticeAlert("MakeRegexps: empty replace")
+			notices = append(notices, "MakeRegexps: empty replace")
 			continue
 		}
 
 		regex, err := regexp.Compile(regexString)
 		if err != nil {
-			NoticeAlert("MakeRegexps: failed to compile regex: %s: %s", regexString, err)
+			notices = append(notices, fmt.Sprintf("MakeRegexps: failed to compile regex: %s: %s", regexString, err))
 			continue
 		}
 
-		regexps = append(regexps, regexpReplace{regex, replace})
+		regexpsSlice = append(regexpsSlice, regexpReplace{regex, replace})
 	}
 
-	return &regexps
+	regexps = &regexpsSlice
+
+	return
 }
 
 // regexHostname processes hostname through the given regexps and returns the

psiphon/stats_test.go → psiphon/transferstats/transferstats_test.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, Psiphon Inc.
+ * Copyright (c) 2015, Psiphon Inc.
  * All rights reserved.
  *
  * This program is free software: you can redistribute it and/or modify
@@ -17,7 +17,7 @@
  *
  */
 
-package psiphon
+package transferstats
 
 import (
 	"encoding/json"
@@ -77,7 +77,7 @@ func makeStatsDialer(serverID string, regexps *Regexps) func(network, addr strin
 			return
 		}
 
-		conn = NewStatsConn(subConn, serverID, regexps)
+		conn = NewConn(subConn, serverID, regexps)
 		err = nil
 		return
 	}
@@ -149,9 +149,10 @@ func (suite *StatsTestSuite) Test_MakeRegexps() {
 	httpsRequestRegexes[1]["regex"] = `^.*example\.org$`
 	httpsRequestRegexes[1]["replace"] = "replacement"
 
-	regexps := MakeRegexps(pageViewRegexes, httpsRequestRegexes)
+	regexps, notices := MakeRegexps(pageViewRegexes, httpsRequestRegexes)
 	suite.NotNil(regexps, "should return a valid object")
 	suite.Len(*regexps, 2, "should only have processed httpsRequestRegexes")
+	suite.Len(notices, 0, "should return no notices")
 
 	//
 	// Test some bad regexps
@@ -159,21 +160,24 @@ func (suite *StatsTestSuite) Test_MakeRegexps() {
 
 	httpsRequestRegexes[0]["regex"] = ""
 	httpsRequestRegexes[0]["replace"] = "$1"
-	regexps = MakeRegexps(pageViewRegexes, httpsRequestRegexes)
+	regexps, notices = MakeRegexps(pageViewRegexes, httpsRequestRegexes)
 	suite.NotNil(regexps, "should return a valid object")
 	suite.Len(*regexps, 1, "should have discarded one regexp")
+	suite.Len(notices, 1, "should have returned one notice")
 
 	httpsRequestRegexes[0]["regex"] = `^[a-z0-9\.]*\.(example\.com)$`
 	httpsRequestRegexes[0]["replace"] = ""
-	regexps = MakeRegexps(pageViewRegexes, httpsRequestRegexes)
+	regexps, notices = MakeRegexps(pageViewRegexes, httpsRequestRegexes)
 	suite.NotNil(regexps, "should return a valid object")
 	suite.Len(*regexps, 1, "should have discarded one regexp")
+	suite.Len(notices, 1, "should have returned one notice")
 
 	httpsRequestRegexes[0]["regex"] = `^[a-z0-9\.]*\.(example\.com$` // missing closing paren
 	httpsRequestRegexes[0]["replace"] = "$1"
-	regexps = MakeRegexps(pageViewRegexes, httpsRequestRegexes)
+	regexps, notices = MakeRegexps(pageViewRegexes, httpsRequestRegexes)
 	suite.NotNil(regexps, "should return a valid object")
 	suite.Len(*regexps, 1, "should have discarded one regexp")
+	suite.Len(notices, 1, "should have returned one notice")
 }
 
 func (suite *StatsTestSuite) Test_Regex() {
@@ -184,7 +188,7 @@ func (suite *StatsTestSuite) Test_Regex() {
 	httpsRequestRegexes[0]["replace"] = "$1"
 	httpsRequestRegexes[1]["regex"] = `^.*example\.org$`
 	httpsRequestRegexes[1]["replace"] = "replacement"
-	regexps := MakeRegexps(pageViewRegexes, httpsRequestRegexes)
+	regexps, _ := MakeRegexps(pageViewRegexes, httpsRequestRegexes)
 
 	suite.httpClient = &http.Client{
 		Transport: &http.Transport{

psiphon/tunnel.go

@@ -31,6 +31,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/transferstats"
 	"golang.org/x/crypto/ssh"
 )
 
@@ -200,7 +201,7 @@ func (tunnel *Tunnel) Dial(remoteAddr string) (conn net.Conn, err error) {
 
 	// Tunnel does not have a session when DisableApi is set
 	if tunnel.session != nil {
-		conn = NewStatsConn(
+		conn = transferstats.NewConn(
 			conn, tunnel.session.StatsServerID(), tunnel.session.StatsRegexps())
 	}
 
@@ -509,12 +510,12 @@ func sendStats(tunnel *Tunnel) {
 		return
 	}
 
-	payload := GetForServer(tunnel.serverEntry.IpAddress)
+	payload := transferstats.GetForServer(tunnel.serverEntry.IpAddress)
 	if payload != nil {
 		err := tunnel.session.DoStatusRequest(payload)
 		if err != nil {
 			NoticeAlert("DoStatusRequest failed for %s: %s", tunnel.serverEntry.IpAddress, err)
-			PutBack(tunnel.serverEntry.IpAddress, payload)
+			transferstats.PutBack(tunnel.serverEntry.IpAddress, payload)
 		}
 	}
 }