Clarify use of page_view_regexes/https_request_regexes

- Additional comment noting that https_request_regexes
  are used as hostname regexes for both TLS and HTTP.

- Don't reference obsolete page_view_regexes at all
  beyond the handshake protocol.

- The handshake protocol is unchanged for backwards/legacy
  compatibility.

psiphon/server/api.go

@@ -320,6 +320,10 @@ func handshakeAPIRequestHandler(
 		}
 	}
 
+	// PageViewRegexes is obsolete and not used by any tunnel-core clients. In
+	// the JSON response, return an empty array instead of null for legacy
+	// clients.
+
 	handshakeResponse := protocol.HandshakeResponse{
 		SSHSessionID:             sessionID,
 		Homepages:                db.GetRandomizedHomepages(sponsorID, geoIPData.Country, geoIPData.ASN, isMobile),

psiphon/serverApi.go

@@ -278,9 +278,15 @@ func (serverContext *ServerContext) doHandshakeRequest(
 
 	if !ignoreStatsRegexps {
 
+		// The handshake returns page_view_regexes and https_request_regexes.
+		// page_view_regexes is obsolete and not used. https_request_regexes, which
+		// are actually host/domain name regexes, are used for host/domain name
+		// bytes transferred metrics: tunneled traffic TLS SNI server names and HTTP
+		// Host header host names are matched against these regexes to select flows
+		// for bytes transferred counting.
+
 		var regexpsNotices []string
 		serverContext.statsRegexps, regexpsNotices = transferstats.MakeRegexps(
-			handshakeResponse.PageViewRegexes,
 			handshakeResponse.HttpsRequestRegexes)
 
 		for _, notice := range regexpsNotices {

psiphon/transferstats/regexp.go

@@ -35,12 +35,11 @@ type Regexps []regexpReplace
 
 // MakeRegexps takes the raw string-map form of the regex-replace pairs
 // returned by the server handshake and turns them into a usable object.
-func MakeRegexps(pageViewRegexes, httpsRequestRegexes []map[string]string) (regexps *Regexps, notices []string) {
+func MakeRegexps(hostnameRegexes []map[string]string) (regexps *Regexps, notices []string) {
 	regexpsSlice := make(Regexps, 0)
 	notices = make([]string, 0)
 
-	// We aren't doing page view stats anymore, so we won't process those regexps.
-	for _, rr := range httpsRequestRegexes {
+	for _, rr := range hostnameRegexes {
 		regexString := rr["regex"]
 		if regexString == "" {
 			notices = append(notices, "MakeRegexps: empty regex")

psiphon/transferstats/transferstats_test.go

@@ -171,42 +171,38 @@ func (suite *StatsTestSuite) Test_NoRegexes() {
 }
 
 func (suite *StatsTestSuite) Test_MakeRegexps() {
-	pageViewRegexes := []map[string]string{make(map[string]string)}
-	pageViewRegexes[0]["regex"] = `(^http://[a-z0-9\.]*\.example\.[a-z\.]*)/.*`
-	pageViewRegexes[0]["replace"] = "$1"
+	hostnameRegexes := []map[string]string{make(map[string]string), make(map[string]string)}
+	hostnameRegexes[0]["regex"] = `^[a-z0-9\.]*\.(example\.com)$`
+	hostnameRegexes[0]["replace"] = "$1"
+	hostnameRegexes[1]["regex"] = `^.*example\.org$`
+	hostnameRegexes[1]["replace"] = "replacement"
 
-	httpsRequestRegexes := []map[string]string{make(map[string]string), make(map[string]string)}
-	httpsRequestRegexes[0]["regex"] = `^[a-z0-9\.]*\.(example\.com)$`
-	httpsRequestRegexes[0]["replace"] = "$1"
-	httpsRequestRegexes[1]["regex"] = `^.*example\.org$`
-	httpsRequestRegexes[1]["replace"] = "replacement"
-
-	regexps, notices := MakeRegexps(pageViewRegexes, httpsRequestRegexes)
+	regexps, notices := MakeRegexps(hostnameRegexes)
 	suite.NotNil(regexps, "should return a valid object")
-	suite.Len(*regexps, 2, "should only have processed httpsRequestRegexes")
+	suite.Len(*regexps, 2, "should only have processed hostnameRegexes")
 	suite.Len(notices, 0, "should return no notices")
 
 	//
 	// Test some bad regexps
 	//
 
-	httpsRequestRegexes[0]["regex"] = ""
-	httpsRequestRegexes[0]["replace"] = "$1"
-	regexps, notices = MakeRegexps(pageViewRegexes, httpsRequestRegexes)
+	hostnameRegexes[0]["regex"] = ""
+	hostnameRegexes[0]["replace"] = "$1"
+	regexps, notices = MakeRegexps(hostnameRegexes)
 	suite.NotNil(regexps, "should return a valid object")
 	suite.Len(*regexps, 1, "should have discarded one regexp")
 	suite.Len(notices, 1, "should have returned one notice")
 
-	httpsRequestRegexes[0]["regex"] = `^[a-z0-9\.]*\.(example\.com)$`
-	httpsRequestRegexes[0]["replace"] = ""
-	regexps, notices = MakeRegexps(pageViewRegexes, httpsRequestRegexes)
+	hostnameRegexes[0]["regex"] = `^[a-z0-9\.]*\.(example\.com)$`
+	hostnameRegexes[0]["replace"] = ""
+	regexps, notices = MakeRegexps(hostnameRegexes)
 	suite.NotNil(regexps, "should return a valid object")
 	suite.Len(*regexps, 1, "should have discarded one regexp")
 	suite.Len(notices, 1, "should have returned one notice")
 
-	httpsRequestRegexes[0]["regex"] = `^[a-z0-9\.]*\.(example\.com$` // missing closing paren
-	httpsRequestRegexes[0]["replace"] = "$1"
-	regexps, notices = MakeRegexps(pageViewRegexes, httpsRequestRegexes)
+	hostnameRegexes[0]["regex"] = `^[a-z0-9\.]*\.(example\.com$` // missing closing paren
+	hostnameRegexes[0]["replace"] = "$1"
+	regexps, notices = MakeRegexps(hostnameRegexes)
 	suite.NotNil(regexps, "should return a valid object")
 	suite.Len(*regexps, 1, "should have discarded one regexp")
 	suite.Len(notices, 1, "should have returned one notice")
@@ -214,13 +210,12 @@ func (suite *StatsTestSuite) Test_MakeRegexps() {
 
 func (suite *StatsTestSuite) Test_Regex() {
 	// We'll make a new client with actual regexps.
-	pageViewRegexes := make([]map[string]string, 0)
-	httpsRequestRegexes := []map[string]string{make(map[string]string), make(map[string]string)}
-	httpsRequestRegexes[0]["regex"] = `^[a-z0-9\.]*\.(example\.com)$`
-	httpsRequestRegexes[0]["replace"] = "$1"
-	httpsRequestRegexes[1]["regex"] = `^.*example\.org$`
-	httpsRequestRegexes[1]["replace"] = "replacement"
-	regexps, _ := MakeRegexps(pageViewRegexes, httpsRequestRegexes)
+	hostnameRegexes := []map[string]string{make(map[string]string), make(map[string]string)}
+	hostnameRegexes[0]["regex"] = `^[a-z0-9\.]*\.(example\.com)$`
+	hostnameRegexes[0]["replace"] = "$1"
+	hostnameRegexes[1]["regex"] = `^.*example\.org$`
+	hostnameRegexes[1]["replace"] = "replacement"
+	regexps, _ := MakeRegexps(hostnameRegexes)
 
 	suite.httpClient = &http.Client{
 		Transport: &http.Transport{