Moved the stats code into a sub-package called transferstats.

psiphon/serverApi.go

@@ -26,6 +26,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/transferstats"
 	"io"
 	"io/ioutil"
 	"net"
@@ -41,7 +42,7 @@ type Session struct {
 	sessionId          string
 	baseRequestUrl     string
 	psiphonHttpsClient *http.Client
-	statsRegexps       *Regexps
+	statsRegexps       *transferstats.Regexps
 	statsServerId      string
 }
 
@@ -128,7 +129,7 @@ func (session *Session) StatsServerID() string {
 }
 
 // StatsRegexps gets the Regexps used for the statistics for this tunnel.
-func (session *Session) StatsRegexps() *Regexps {
+func (session *Session) StatsRegexps() *transferstats.Regexps {
 	return session.statsRegexps
 }
 
@@ -235,9 +236,15 @@ func (session *Session) doHandshakeRequest() error {
 		NoticeClientUpgradeAvailable(handshakeConfig.UpgradeClientVersion)
 	}
 
-	session.statsRegexps = MakeRegexps(
+	var regexpsNotices []string
+	session.statsRegexps, regexpsNotices = transferstats.MakeRegexps(
 		handshakeConfig.PageViewRegexes,
 		handshakeConfig.HttpsRequestRegexes)
+
+	for _, notice := range regexpsNotices {
+		NoticeAlert(notice)
+	}
+
 	return nil
 }
 

psiphon/transferstats/README.md

@@ -0,0 +1,15 @@
+`transferstats` Package
+=======================
+
+This provides a `net.Conn` interface implementation that can be put in a chain
+of connections and used to collect transfer statistics for the network traffic
+passing through it.
+
+Total bytes transferred is recorded, as well as per-hostname bytes transferred
+stats for HTTP and HTTPS traffic (as long as the HTTPS traffic contains [SNI]
+information). Which hostnames are recorded is specified by a set of regular
+expressions.
+
+[SNI]: https://en.wikipedia.org/wiki/Server_Name_Indication
+
+(TODO: More info.)

psiphon/stats_collector.go → psiphon/transferstats/collector.go

@@ -17,7 +17,7 @@
  *
  */
 
-package psiphon
+package transferstats
 
 import (
 	"encoding/json"
@@ -116,9 +116,6 @@ func (ss serverStats) MarshalJSON() ([]byte, error) {
 	out["bytes_transferred"] = bytesTransferred
 	out["host_bytes"] = hostBytes
 
-	noticeJSON, _ := json.Marshal(out)
-	NoticeInfo("sending stats: %s", noticeJSON)
-
 	// We're not using these fields, but the server requires them
 	out["page_views"] = make([]string, 0)
 	out["https_requests"] = make([]string, 0)

psiphon/stats_conn.go → psiphon/transferstats/conn.go

@@ -19,7 +19,7 @@
 
 // Package stats counts and keeps track of session stats. These are per-domain
 // bytes transferred and total bytes transferred.
-package psiphon
+package transferstats
 
 /*
 Assumption: The same connection will not be used to access different hostnames
@@ -35,9 +35,9 @@ import (
 	"sync/atomic"
 )
 
-// StatsConn is to be used as an intermediate link in a chain of net.Conn objects.
+// Conn is to be used as an intermediate link in a chain of net.Conn objects.
 // It inspects requests and responses and derives stats from them.
-type StatsConn struct {
+type Conn struct {
 	net.Conn
 	serverID       string
 	firstWrite     int32
@@ -46,11 +46,11 @@ type StatsConn struct {
 	regexps        *Regexps
 }
 
-// NewStatsConn creates a StatsConn. serverID can be anything that uniquely
+// NewConn creates a Conn. serverID can be anything that uniquely
 // identifies the server; it will be passed to GetForServer() when retrieving
 // the accumulated stats.
-func NewStatsConn(nextConn net.Conn, serverID string, regexps *Regexps) *StatsConn {
-	return &StatsConn{
+func NewConn(nextConn net.Conn, serverID string, regexps *Regexps) *Conn {
+	return &Conn{
 		Conn:           nextConn,
 		serverID:       serverID,
 		firstWrite:     1,
@@ -61,7 +61,7 @@ func NewStatsConn(nextConn net.Conn, serverID string, regexps *Regexps) *StatsCo
 
 // Write is called when requests are being written out through the tunnel to
 // the remote server.
-func (conn *StatsConn) Write(buffer []byte) (n int, err error) {
+func (conn *Conn) Write(buffer []byte) (n int, err error) {
 	// First pass the data down the chain.
 	n, err = conn.Conn.Write(buffer)
 
@@ -92,7 +92,7 @@ func (conn *StatsConn) Write(buffer []byte) (n int, err error) {
 }
 
 // Read is called when responses to requests are being read from the remote server.
-func (conn *StatsConn) Read(buffer []byte) (n int, err error) {
+func (conn *Conn) Read(buffer []byte) (n int, err error) {
 	n, err = conn.Conn.Read(buffer)
 
 	var hostname string

psiphon/stats_hostname.go → psiphon/transferstats/hostname.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, Psiphon Inc.
+ * Copyright (c) 2015, Psiphon Inc.
  * All rights reserved.
  *
  * This program is free software: you can redistribute it and/or modify
@@ -17,7 +17,7 @@
  *
  */
 
-package psiphon
+package transferstats
 
 import (
 	"bufio"

psiphon/stats_regexp.go → psiphon/transferstats/regexp.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, Psiphon Inc.
+ * Copyright (c) 2015, Psiphon Inc.
  * All rights reserved.
  *
  * This program is free software: you can redistribute it and/or modify
@@ -17,9 +17,12 @@
  *
  */
 
-package psiphon
+package transferstats
 
-import "regexp"
+import (
+	"fmt"
+	"regexp"
+)
 
 type regexpReplace struct {
 	regexp  *regexp.Regexp
@@ -32,33 +35,36 @@ type Regexps []regexpReplace
 
 // MakeRegexps takes the raw string-map form of the regex-replace pairs
 // returned by the server handshake and turns them into a usable object.
-func MakeRegexps(pageViewRegexes, httpsRequestRegexes []map[string]string) *Regexps {
-	regexps := make(Regexps, 0)
+func MakeRegexps(pageViewRegexes, httpsRequestRegexes []map[string]string) (regexps *Regexps, notices []string) {
+	regexpsSlice := make(Regexps, 0)
+	notices = make([]string, 0)
 
 	// We aren't doing page view stats anymore, so we won't process those regexps.
 	for _, rr := range httpsRequestRegexes {
 		regexString := rr["regex"]
 		if regexString == "" {
-			NoticeAlert("MakeRegexps: empty regex")
+			notices = append(notices, "MakeRegexps: empty regex")
 			continue
 		}
 
 		replace := rr["replace"]
 		if replace == "" {
-			NoticeAlert("MakeRegexps: empty replace")
+			notices = append(notices, "MakeRegexps: empty replace")
 			continue
 		}
 
 		regex, err := regexp.Compile(regexString)
 		if err != nil {
-			NoticeAlert("MakeRegexps: failed to compile regex: %s: %s", regexString, err)
+			notices = append(notices, fmt.Sprintf("MakeRegexps: failed to compile regex: %s: %s", regexString, err))
 			continue
 		}
 
-		regexps = append(regexps, regexpReplace{regex, replace})
+		regexpsSlice = append(regexpsSlice, regexpReplace{regex, replace})
 	}
 
-	return &regexps
+	regexps = &regexpsSlice
+
+	return
 }
 
 // regexHostname processes hostname through the given regexps and returns the

psiphon/stats_test.go → psiphon/transferstats/transferstats_test.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, Psiphon Inc.
+ * Copyright (c) 2015, Psiphon Inc.
  * All rights reserved.
  *
  * This program is free software: you can redistribute it and/or modify
@@ -17,7 +17,7 @@
  *
  */
 
-package psiphon
+package transferstats
 
 import (
 	"encoding/json"
@@ -77,7 +77,7 @@ func makeStatsDialer(serverID string, regexps *Regexps) func(network, addr strin
 			return
 		}
 
-		conn = NewStatsConn(subConn, serverID, regexps)
+		conn = NewConn(subConn, serverID, regexps)
 		err = nil
 		return
 	}
@@ -149,9 +149,10 @@ func (suite *StatsTestSuite) Test_MakeRegexps() {
 	httpsRequestRegexes[1]["regex"] = `^.*example\.org$`
 	httpsRequestRegexes[1]["replace"] = "replacement"
 
-	regexps := MakeRegexps(pageViewRegexes, httpsRequestRegexes)
+	regexps, notices := MakeRegexps(pageViewRegexes, httpsRequestRegexes)
 	suite.NotNil(regexps, "should return a valid object")
 	suite.Len(*regexps, 2, "should only have processed httpsRequestRegexes")
+	suite.Len(notices, 0, "should return no notices")
 
 	//
 	// Test some bad regexps
@@ -159,21 +160,24 @@ func (suite *StatsTestSuite) Test_MakeRegexps() {
 
 	httpsRequestRegexes[0]["regex"] = ""
 	httpsRequestRegexes[0]["replace"] = "$1"
-	regexps = MakeRegexps(pageViewRegexes, httpsRequestRegexes)
+	regexps, notices = MakeRegexps(pageViewRegexes, httpsRequestRegexes)
 	suite.NotNil(regexps, "should return a valid object")
 	suite.Len(*regexps, 1, "should have discarded one regexp")
+	suite.Len(notices, 1, "should have returned one notice")
 
 	httpsRequestRegexes[0]["regex"] = `^[a-z0-9\.]*\.(example\.com)$`
 	httpsRequestRegexes[0]["replace"] = ""
-	regexps = MakeRegexps(pageViewRegexes, httpsRequestRegexes)
+	regexps, notices = MakeRegexps(pageViewRegexes, httpsRequestRegexes)
 	suite.NotNil(regexps, "should return a valid object")
 	suite.Len(*regexps, 1, "should have discarded one regexp")
+	suite.Len(notices, 1, "should have returned one notice")
 
 	httpsRequestRegexes[0]["regex"] = `^[a-z0-9\.]*\.(example\.com$` // missing closing paren
 	httpsRequestRegexes[0]["replace"] = "$1"
-	regexps = MakeRegexps(pageViewRegexes, httpsRequestRegexes)
+	regexps, notices = MakeRegexps(pageViewRegexes, httpsRequestRegexes)
 	suite.NotNil(regexps, "should return a valid object")
 	suite.Len(*regexps, 1, "should have discarded one regexp")
+	suite.Len(notices, 1, "should have returned one notice")
 }
 
 func (suite *StatsTestSuite) Test_Regex() {
@@ -184,7 +188,7 @@ func (suite *StatsTestSuite) Test_Regex() {
 	httpsRequestRegexes[0]["replace"] = "$1"
 	httpsRequestRegexes[1]["regex"] = `^.*example\.org$`
 	httpsRequestRegexes[1]["replace"] = "replacement"
-	regexps := MakeRegexps(pageViewRegexes, httpsRequestRegexes)
+	regexps, _ := MakeRegexps(pageViewRegexes, httpsRequestRegexes)
 
 	suite.httpClient = &http.Client{
 		Transport: &http.Transport{

psiphon/tunnel.go

@@ -31,6 +31,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/transferstats"
 	"golang.org/x/crypto/ssh"
 )
 
@@ -200,7 +201,7 @@ func (tunnel *Tunnel) Dial(remoteAddr string) (conn net.Conn, err error) {
 
 	// Tunnel does not have a session when DisableApi is set
 	if tunnel.session != nil {
-		conn = NewStatsConn(
+		conn = transferstats.NewConn(
 			conn, tunnel.session.StatsServerID(), tunnel.session.StatsRegexps())
 	}
 
@@ -509,12 +510,12 @@ func sendStats(tunnel *Tunnel) {
 		return
 	}
 
-	payload := GetForServer(tunnel.serverEntry.IpAddress)
+	payload := transferstats.GetForServer(tunnel.serverEntry.IpAddress)
 	if payload != nil {
 		err := tunnel.session.DoStatusRequest(payload)
 		if err != nil {
 			NoticeAlert("DoStatusRequest failed for %s: %s", tunnel.serverEntry.IpAddress, err)
-			PutBack(tunnel.serverEntry.IpAddress, payload)
+			transferstats.PutBack(tunnel.serverEntry.IpAddress, payload)
 		}
 	}
 }