Rename "tunnel durations" to "tunnel stats"

psiphon/config.go

@@ -61,7 +61,7 @@ const (
 	PSIPHON_API_STATUS_REQUEST_PADDING_MAX_BYTES   = 256
 	PSIPHON_API_CONNECTED_REQUEST_PERIOD           = 24 * time.Hour
 	PSIPHON_API_CONNECTED_REQUEST_RETRY_PERIOD     = 5 * time.Second
-	PSIPHON_API_TUNNEL_DURATIONS_MAX_COUNT         = 1000
+	PSIPHON_API_TUNNEL_STATS_MAX_COUNT             = 1000
 	FETCH_ROUTES_TIMEOUT                           = 1 * time.Minute
 	DOWNLOAD_UPGRADE_TIMEOUT                       = 15 * time.Minute
 	DOWNLOAD_UPGRADE_RETRY_PAUSE_PERIOD            = 5 * time.Second

psiphon/controller.go

@@ -775,8 +775,7 @@ func (controller *Controller) startEstablishing() {
 	controller.establishPendingConns.Reset()
 
 	// The server affinity mechanism attempts to favor the previously
-	// used server when reconnecting. This is useful for server-side
-	// session duration stats calculation and also beneficial for user
+	// used server when reconnecting. This is beneficial for user
 	// applications which expect consistency in user IP address (for
 	// example, a web site which prompts for additional user
 	// authentication when the IP address changes).

psiphon/dataStore_alt.go

@@ -56,7 +56,7 @@ const (
 	splitTunnelRouteDataBucket  = "splitTunnelRouteData"
 	urlETagsBucket              = "urlETags"
 	keyValueBucket              = "keyValues"
-	tunnelDurationsBucket       = "tunnelDurations"
+	tunnelStatsBucket           = "tunnelStats"
 	rankedServerEntryCount      = 100
 )
 
@@ -89,7 +89,7 @@ func InitDataStore(config *Config) (err error) {
 				splitTunnelRouteDataBucket,
 				urlETagsBucket,
 				keyValueBucket,
-				tunnelDurationsBucket,
+				tunnelStatsBucket,
 			}
 			for _, bucket := range requiredBuckets {
 				_, err := tx.CreateBucketIfNotExists([]byte(bucket))
@@ -106,7 +106,7 @@ func InitDataStore(config *Config) (err error) {
 
 		singleton.db = db
 
-		resetAllTunnelDurationsToUnreported()
+		resetAllTunnelStatsToUnreported()
 	})
 	return err
 }
@@ -723,33 +723,33 @@ func GetKeyValue(key string) (value string, err error) {
 	return value, nil
 }
 
-// Tunnel duration records in the tunnelDurationStateUnreported
+// Tunnel stats records in the tunnelStatsStateUnreported
 // state are available for take out.
-// Records in the tunnelDurationStateReporting have been
+// Records in the tunnelStatsStateReporting have been
 // taken out and are pending either deleting (for a
 // successful request) or change to StateUnreported (for
 // a failed request).
-// All tunnel durations are reverted to StateUnreported when
-// the datastore is initialized at start up.
+// All tunnel stats records are reverted to StateUnreported
+// when the datastore is initialized at start up.
 
-var tunnelDurationStateUnreported = []byte("0")
-var tunnelDurationStateReporting = []byte("1")
+var tunnelStatsStateUnreported = []byte("0")
+var tunnelStatsStateReporting = []byte("1")
 
-// StoreTunnelDuration adds a new tunnel duration, which is
+// StoreTunnelStats adds a new tunnel stats record, which is
 // set to StateUnreported and is an immediate candidate for
 // reporting.
-// tunnelDuration is a JSON byte array containing fields as
-// required by the Psiphon server API (see RecordTunnelDuration).
+// tunnelStats is a JSON byte array containing fields as
+// required by the Psiphon server API (see RecordTunnelStats).
 // It's assumed that the JSON value contains enough unique
 // information for the value to function as a key in the
 // key/value datastore. This assumption is currently satisfied
 // by the fields sessionId + tunnelNumber.
-func StoreTunnelDuration(tunnelDuration []byte) error {
+func StoreTunnelStats(tunnelStats []byte) error {
 	checkInitDataStore()
 
 	err := singleton.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(tunnelDurationsBucket))
-		err := bucket.Put(tunnelDuration, tunnelDurationStateUnreported)
+		bucket := tx.Bucket([]byte(tunnelStatsBucket))
+		err := bucket.Put(tunnelStats, tunnelStatsStateUnreported)
 		return err
 	})
 
@@ -759,18 +759,18 @@ func StoreTunnelDuration(tunnelDuration []byte) error {
 	return nil
 }
 
-// CountUnreportedTunnelDurations returns the number of tunnel
-// duration records in StateUnreported.
-func CountUnreportedTunnelDurations() int {
+// CountUnreportedTunnelStats returns the number of tunnel
+// stats records in StateUnreported.
+func CountUnreportedTunnelStats() int {
 	checkInitDataStore()
 
 	unreported := 0
 
 	err := singleton.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(tunnelDurationsBucket))
+		bucket := tx.Bucket([]byte(tunnelStatsBucket))
 		cursor := bucket.Cursor()
 		for key, value := cursor.First(); key != nil; key, value = cursor.Next() {
-			if 0 == bytes.Compare(value, tunnelDurationStateUnreported) {
+			if 0 == bytes.Compare(value, tunnelStatsStateUnreported) {
 				unreported++
 				break
 			}
@@ -779,35 +779,35 @@ func CountUnreportedTunnelDurations() int {
 	})
 
 	if err != nil {
-		NoticeAlert("CountUnreportedTunnelDurations failed: %s", err)
+		NoticeAlert("CountUnreportedTunnelStats failed: %s", err)
 		return 0
 	}
 
 	return unreported
 }
 
-// TakeOutUnreportedTunnelDurations returns up to maxCount tunnel
-// durations that are in StateUnreported. The records are set to
-// StateReporting. If the records are successfully reported,
-// clear them with ClearReportedTunnelDurations. If the records are
+// TakeOutUnreportedTunnelStats returns up to maxCount tunnel
+// stats records that are in StateUnreported. The records are set
+// to StateReporting. If the records are successfully reported,
+// clear them with ClearReportedTunnelStats. If the records are
 // not successfully reported, restore them with
-// PutBackUnreportedTunnelDurations.
-func TakeOutUnreportedTunnelDurations(maxCount int) ([][]byte, error) {
+// PutBackUnreportedTunnelStats.
+func TakeOutUnreportedTunnelStats(maxCount int) ([][]byte, error) {
 	checkInitDataStore()
 
-	tunnelDurations := make([][]byte, 0)
+	tunnelStats := make([][]byte, 0)
 
 	err := singleton.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(tunnelDurationsBucket))
+		bucket := tx.Bucket([]byte(tunnelStatsBucket))
 		cursor := bucket.Cursor()
 		for key, value := cursor.First(); key != nil; key, value = cursor.Next() {
-			if 0 == bytes.Compare(value, tunnelDurationStateUnreported) {
-				err := bucket.Put(key, tunnelDurationStateReporting)
+			if 0 == bytes.Compare(value, tunnelStatsStateUnreported) {
+				err := bucket.Put(key, tunnelStatsStateReporting)
 				if err != nil {
 					return err
 				}
-				tunnelDurations = append(tunnelDurations, key)
-				if len(tunnelDurations) >= maxCount {
+				tunnelStats = append(tunnelStats, key)
+				if len(tunnelStats) >= maxCount {
 					break
 				}
 			}
@@ -818,18 +818,18 @@ func TakeOutUnreportedTunnelDurations(maxCount int) ([][]byte, error) {
 	if err != nil {
 		return nil, ContextError(err)
 	}
-	return tunnelDurations, nil
+	return tunnelStats, nil
 }
 
-// PutBackUnreportedTunnelDurations restores a list of tunnel
-// durations to StateUnreported.
-func PutBackUnreportedTunnelDurations(tunnelDurations [][]byte) error {
+// PutBackUnreportedTunnelStats restores a list of tunnel
+// stats records to StateUnreported.
+func PutBackUnreportedTunnelStats(tunnelStats [][]byte) error {
 	checkInitDataStore()
 
 	err := singleton.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(tunnelDurationsBucket))
-		for _, tunnelDuration := range tunnelDurations {
-			err := bucket.Put(tunnelDuration, tunnelDurationStateUnreported)
+		bucket := tx.Bucket([]byte(tunnelStatsBucket))
+		for _, key := range tunnelStats {
+			err := bucket.Put(key, tunnelStatsStateUnreported)
 			if err != nil {
 				return err
 			}
@@ -843,15 +843,15 @@ func PutBackUnreportedTunnelDurations(tunnelDurations [][]byte) error {
 	return nil
 }
 
-// ClearReportedTunnelDurations deletes a list of tunnel
-// durations that were succesdfully reported.
-func ClearReportedTunnelDurations(tunnelDurations [][]byte) error {
+// ClearReportedTunnelStats deletes a list of tunnel
+// stats records that were succesdfully reported.
+func ClearReportedTunnelStats(tunnelStats [][]byte) error {
 	checkInitDataStore()
 
 	err := singleton.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(tunnelDurationsBucket))
-		for _, tunnelDuration := range tunnelDurations {
-			err := bucket.Delete(tunnelDuration)
+		bucket := tx.Bucket([]byte(tunnelStatsBucket))
+		for _, key := range tunnelStats {
+			err := bucket.Delete(key)
 			if err != nil {
 				return err
 			}
@@ -865,19 +865,19 @@ func ClearReportedTunnelDurations(tunnelDurations [][]byte) error {
 	return nil
 }
 
-// resetAllTunnelDurationsToUnreported sets all tunnel
-// duration records to StateUnreported. This reset is called
+// resetAllTunnelStatsToUnreported sets all tunnel
+// stats records to StateUnreported. This reset is called
 // when the datastore is initialized at start up, as we do
 // not know if tunnel records in StateReporting were reported
 // or not.
-func resetAllTunnelDurationsToUnreported() error {
+func resetAllTunnelStatsToUnreported() error {
 	checkInitDataStore()
 
 	err := singleton.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(tunnelDurationsBucket))
+		bucket := tx.Bucket([]byte(tunnelStatsBucket))
 		cursor := bucket.Cursor()
 		for key, _ := cursor.First(); key != nil; key, _ = cursor.Next() {
-			err := bucket.Put(key, tunnelDurationStateUnreported)
+			err := bucket.Put(key, tunnelStatsStateUnreported)
 			if err != nil {
 				return err
 			}

psiphon/serverApi.go

@@ -257,7 +257,7 @@ func (serverContext *ServerContext) DoStatusRequest(tunnel *Tunnel) error {
 	err = serverContext.doPostRequest(url, "application/json", bytes.NewReader(payload))
 	if err != nil {
 
-		// Resend the transfer stats and tunnel durations later
+		// Resend the transfer stats and tunnel stats later
 		// Note: potential duplicate reports if the server received and processed
 		// the request but the client failed to receive the response.
 		putBackStatusRequestPayload(payloadInfo)
@@ -277,7 +277,7 @@ func makeStatusRequestUrl(sessionId, baseRequestUrl string, isTunneled bool) str
 
 	// Legacy clients set "connected" to "0" when disconnecting, and this value
 	// is used to calculate session duration estimates. This is now superseded
-	// by explicit tunnel duration reporting.
+	// by explicit tunnel stats duration reporting.
 	// The legacy method of reconstructing session durations is not compatible
 	// with this client's connected request retries and asynchronous final
 	// status request attempts. So we simply set this "connected" flag to reflect
@@ -302,25 +302,25 @@ func makeStatusRequestUrl(sessionId, baseRequestUrl string, isTunneled bool) str
 // either "clear" or "put back" status request payload data depending
 // on whether or not the request succeeded.
 type statusRequestPayloadInfo struct {
-	serverId        string
-	transferStats   *transferstats.ServerStats
-	tunnelDurations [][]byte
+	serverId      string
+	transferStats *transferstats.ServerStats
+	tunnelStats   [][]byte
 }
 
 func makeStatusRequestPayload(
 	serverId string) ([]byte, *statusRequestPayloadInfo, error) {
 
 	transferStats := transferstats.GetForServer(serverId)
-	tunnelDurations, err := TakeOutUnreportedTunnelDurations(
-		PSIPHON_API_TUNNEL_DURATIONS_MAX_COUNT)
+	tunnelStats, err := TakeOutUnreportedTunnelStats(
+		PSIPHON_API_TUNNEL_STATS_MAX_COUNT)
 	if err != nil {
 		NoticeAlert(
-			"TakeOutUnreportedTunnelDurations failed: %s", ContextError(err))
-		tunnelDurations = nil
+			"TakeOutUnreportedTunnelStats failed: %s", ContextError(err))
+		tunnelStats = nil
 		// Proceed with transferStats only
 	}
 	payloadInfo := &statusRequestPayloadInfo{
-		serverId, transferStats, tunnelDurations}
+		serverId, transferStats, tunnelStats}
 
 	payload := make(map[string]interface{})
 
@@ -332,17 +332,17 @@ func makeStatusRequestPayload(
 	payload["page_views"] = make([]string, 0)
 	payload["https_requests"] = make([]string, 0)
 
-	// Tunnel duration records are already in JSON format
-	jsonTunnelDurations := make([]json.RawMessage, len(tunnelDurations))
-	for i, tunnelDuration := range tunnelDurations {
-		jsonTunnelDurations[i] = json.RawMessage(tunnelDuration)
+	// Tunnel stats records are already in JSON format
+	jsonTunnelStats := make([]json.RawMessage, len(tunnelStats))
+	for i, tunnelStatsRecord := range tunnelStats {
+		jsonTunnelStats[i] = json.RawMessage(tunnelStatsRecord)
 	}
-	payload["tunnel_durations"] = jsonTunnelDurations
+	payload["tunnel_stats"] = jsonTunnelStats
 
 	jsonPayload, err := json.Marshal(payload)
 	if err != nil {
 
-		// Send the transfer stats and tunnel durations later
+		// Send the transfer stats and tunnel stats later
 		putBackStatusRequestPayload(payloadInfo)
 
 		return nil, nil, ContextError(err)
@@ -353,21 +353,21 @@ func makeStatusRequestPayload(
 
 func putBackStatusRequestPayload(payloadInfo *statusRequestPayloadInfo) {
 	transferstats.PutBack(payloadInfo.serverId, payloadInfo.transferStats)
-	err := PutBackUnreportedTunnelDurations(payloadInfo.tunnelDurations)
+	err := PutBackUnreportedTunnelStats(payloadInfo.tunnelStats)
 	if err != nil {
-		// These tunnel duration records won't be resent under after a
+		// These tunnel stats records won't be resent under after a
 		// datastore re-initialization.
 		NoticeAlert(
-			"PutBackUnreportedTunnelDurations failed: %s", ContextError(err))
+			"PutBackUnreportedTunnelStats failed: %s", ContextError(err))
 	}
 }
 
 func confirmStatusRequestPayload(payloadInfo *statusRequestPayloadInfo) {
-	err := ClearReportedTunnelDurations(payloadInfo.tunnelDurations)
+	err := ClearReportedTunnelStats(payloadInfo.tunnelStats)
 	if err != nil {
-		// These tunnel duration records may be resent.
+		// These tunnel stats records may be resent.
 		NoticeAlert(
-			"ClearReportedTunnelDurations failed: %s", ContextError(err))
+			"ClearReportedTunnelStats failed: %s", ContextError(err))
 	}
 }
 
@@ -434,7 +434,7 @@ func doUntunneledStatusRequest(
 	}
 	if err != nil {
 
-		// Resend the transfer stats and tunnel durations later
+		// Resend the transfer stats and tunnel stats later
 		// Note: potential duplicate reports if the server received and processed
 		// the request but the client failed to receive the response.
 		putBackStatusRequestPayload(payloadInfo)
@@ -448,59 +448,60 @@ func doUntunneledStatusRequest(
 	return nil
 }
 
-// RecordTunnelDuration records a tunnel duration for
-// subsequent reporting.
+// RecordTunnelStats records a tunnel duration and bytes
+// sent and received for subsequent reporting and quality
+// analysis.
 //
 // Tunnel durations are precisely measured client-side
 // and reported in status requests. As the duration is
 // not determined until the tunnel is closed, tunnel
-// duration records are stored in the persistent datastore
+// stats records are stored in the persistent datastore
 // and reported via subsequent status requests sent to any
 // Psiphon server.
 //
-// Since the status request that reports a tunnel duration
-// is not necessarily handled by the same server, the
-// tunnel duration records include the original server ID.
+// Since the status request that reports a tunnel stats
+// record is not necessarily handled by the same server, the
+// tunnel stats records include the original server ID.
 //
-// Other fields that may change between duration recording and
-// duration reporting include client geo data, propagation channel,
+// Other fields that may change between tunnel stats recording
+// and reporting include client geo data, propagation channel,
 // sponsor ID, client version. These are not stored in the
 // datastore (client region, in particular, since that would
 // create an on-disk record of user location).
 // TODO: the server could encrypt, with a nonce and key unknown to
 // the client, a blob containing this data; return it in the
 // handshake response; and the client could store and later report
-// this blob with its tunnel duration records.
+// this blob with its tunnel stats records.
 //
 // Multiple "status" requests may be in flight at once (due
 // to multi-tunnel, asynchronous final status retry, and
 // aggressive status requests for pre-registered tunnels),
-// To avoid duplicate reporting, tunnel duration records are
+// To avoid duplicate reporting, tunnel stats records are
 // "taken-out" by a status request and then "put back" in
 // case the request fails.
 //
-// Note: since tunnel duration records have a globally unique
-// identifier (sessionId + tunnelNumber), we could permit
+// Note: since tunnel stats records have a globally unique
+// identifier (sessionId + tunnelNumber), we could tolerate
 // duplicate reporting and filter our duplicates on the
 // server-side. Permitting duplicate reporting could increase
 // the velocity of reporting (for example, both the asynchronous
 // untunneled final status requests and the post-connected
 // immediate startus requests could try to report the same tunnel
-// durations).
+// stats).
 // Duplicate reporting may also occur when a server receives and
 // processes a status request but the client fails to receive
 // the response.
-func RecordTunnelDuration(
+func RecordTunnelStats(
 	sessionId string,
 	tunnelNumber int64,
-	serverId string,
+	tunnelServerIpAddress string,
 	serverHandshakeTimestamp, duration string,
 	totalBytesSent, totalBytesReceived int64) error {
 
-	tunnelDuration := struct {
+	tunnelStats := struct {
 		SessionId                string `json:"session_id"`
 		TunnelNumber             int64  `json:"tunnel_number"`
-		ServerId                 string `json:"server_id"`
+		TunnelServerIpAddress    string `json:"tunnel_server_ip_address"`
 		ServerHandshakeTimestamp string `json:"server_handshake_timestamp"`
 		Duration                 string `json:"duration"`
 		TotalBytesSent           int64  `json:"total_bytes_sent"`
@@ -508,19 +509,19 @@ func RecordTunnelDuration(
 	}{
 		sessionId,
 		tunnelNumber,
-		serverId,
+		tunnelServerIpAddress,
 		serverHandshakeTimestamp,
 		duration,
 		totalBytesSent,
 		totalBytesReceived,
 	}
 
-	tunnelDurationJson, err := json.Marshal(tunnelDuration)
+	tunnelStatsJson, err := json.Marshal(tunnelStats)
 	if err != nil {
 		return ContextError(err)
 	}
 
-	return StoreTunnelDuration(tunnelDurationJson)
+	return StoreTunnelStats(tunnelStatsJson)
 }
 
 // doGetRequest makes a tunneled HTTPS request and returns the response body.

psiphon/tunnel.go

@@ -577,16 +577,16 @@ func (tunnel *Tunnel) operateTunnel(tunnelOwner TunnelOwner) {
 	defer statsTimer.Stop()
 
 	// Schedule an immediate status request to deliver any unreported
-	// tunnelDurations.
+	// tunnel stats.
 	// Note: this may not be effective when there's an outstanding
 	// asynchronous untunneled final status request is holding the
-	// tunnel duration records. It may also conflict with other
+	// tunnel stats records. It may also conflict with other
 	// tunnel candidates which attempt to send an immediate request
 	// before being discarded. For now, we mitigate this with a short,
 	// random delay.
-	unreported := CountUnreportedTunnelDurations()
+	unreported := CountUnreportedTunnelStats()
 	if unreported > 0 {
-		NoticeInfo("Unreported tunnel durations: %d", unreported)
+		NoticeInfo("Unreported tunnel stats: %d", unreported)
 		statsTimer.Reset(MakeRandomPeriod(
 			PSIPHON_API_STATUS_REQUEST_SHORT_PERIOD_MIN,
 			PSIPHON_API_STATUS_REQUEST_SHORT_PERIOD_MAX))
@@ -703,14 +703,14 @@ func (tunnel *Tunnel) operateTunnel(tunnelOwner TunnelOwner) {
 	close(signalStatusRequest)
 	requestsWaitGroup.Wait()
 
-	// This tunnel duration will be reported via the next successful status
-	// request.
+	// The stats for this tunnel will be reported via the next successful
+	// status request.
 	// Note: Since client clocks are unreliable, we use the server's reported
 	// timestamp in the handshake response as the tunnel start time. This time
 	// will be slightly earlier than the actual tunnel activation time, as the
 	// client has to receive and parse the response and activate the tunnel.
 	if !tunnel.IsDiscarded() {
-		err := RecordTunnelDuration(
+		err := RecordTunnelStats(
 			tunnel.serverContext.sessionId,
 			tunnel.serverContext.tunnelNumber,
 			tunnel.serverEntry.IpAddress,
@@ -719,14 +719,14 @@ func (tunnel *Tunnel) operateTunnel(tunnelOwner TunnelOwner) {
 			totalSent,
 			totalReceived)
 		if err != nil {
-			NoticeAlert("RecordTunnelDuration failed: %s", ContextError(err))
+			NoticeAlert("RecordTunnelStats failed: %s", ContextError(err))
 		}
 	}
 
 	// Final status request notes:
 	//
 	// It's highly desirable to send a final status request in order to report
-	// domain bytes transferred stats as well as to report tunnel duration as
+	// domain bytes transferred stats as well as to report tunnel stats as
 	// soon as possible. For this reason, we attempt untunneled requests when
 	// the tunneled request isn't possible or has failed.
 	//