Integrated split tunnel feature.

* Implemented configuration and logic flow.
* Internal implementation incomplete.
* Server-side changes incomplete.

psiphon/config.go

@@ -55,6 +55,7 @@ const (
 	PSIPHON_API_STATUS_REQUEST_PADDING_MAX_BYTES = 256
 	PSIPHON_API_CONNECTED_REQUEST_PERIOD         = 24 * time.Hour
 	PSIPHON_API_CONNECTED_REQUEST_RETRY_PERIOD   = 5 * time.Second
+	FETCH_ROUTES_TIMEOUT                         = 10 * time.Second
 )
 
 // To distinguish omitted timeout params from explicit 0 value timeout
@@ -87,6 +88,8 @@ type Config struct {
 	TargetServerEntry                  string
 	DisableApi                         bool
 	DisableRemoteServerListFetcher     bool
+	SplitTunnelRoutesUrlFormat         string
+	SplitTunnelDnsServer               string
 }
 
 // LoadConfig parses and validates a JSON format Psiphon config JSON

psiphon/controller.go

@@ -51,7 +51,9 @@ type Controller struct {
 	stopEstablishingBroadcast chan struct{}
 	candidateServerEntries    chan *ServerEntry
 	establishPendingConns     *Conns
-	fetchRemotePendingConns   *Conns
+	untunneledPendingConns    *Conns
+	untunneledDialConfig      *DialConfig
+	splitTunnelClassifier     *SplitTunnelClassifier
 }
 
 // NewController initializes a new controller.
@@ -64,6 +66,17 @@ func NewController(config *Config) (controller *Controller, err error) {
 		return nil, ContextError(err)
 	}
 
+	// untunneledPendingConns may be used to interrupt the fetch remote server list
+	// request and other untunneled connection establishments. BindToDevice may be
+	// used to exclude these requests and connection from VPN routing.
+	untunneledPendingConns := new(Conns)
+	untunneledDialConfig := &DialConfig{
+		UpstreamHttpProxyAddress: config.UpstreamHttpProxyAddress,
+		PendingConns:             untunneledPendingConns,
+		DeviceBinder:             config.DeviceBinder,
+		DnsServerGetter:          config.DnsServerGetter,
+	}
+
 	return &Controller{
 		config:    config,
 		sessionId: sessionId,
@@ -82,7 +95,9 @@ func NewController(config *Config) (controller *Controller, err error) {
 		startedConnectedReporter: false,
 		isEstablishing:           false,
 		establishPendingConns:    new(Conns),
-		fetchRemotePendingConns:  new(Conns),
+		untunneledPendingConns:   untunneledPendingConns,
+		untunneledDialConfig:     untunneledDialConfig,
+		splitTunnelClassifier:    NewSplitTunnelClassifier(config, untunneledDialConfig),
 	}, nil
 }
 
@@ -142,9 +157,11 @@ func (controller *Controller) Run(shutdownBroadcast <-chan struct{}) {
 
 	close(controller.shutdownBroadcast)
 	controller.establishPendingConns.CloseAll()
-	controller.fetchRemotePendingConns.CloseAll()
+	controller.untunneledPendingConns.CloseAll()
 	controller.runWaitGroup.Wait()
 
+	controller.splitTunnelClassifier.Shutdown()
+
 	NoticeInfo("exiting controller")
 }
 
@@ -172,7 +189,7 @@ loop:
 		}
 
 		err := FetchRemoteServerList(
-			controller.config, controller.fetchRemotePendingConns)
+			controller.config, controller.untunneledDialConfig)
 
 		var duration time.Duration
 		if err != nil {
@@ -386,6 +403,23 @@ func (controller *Controller) registerTunnel(tunnel *Tunnel) bool {
 	controller.establishedOnce = true
 	controller.tunnels = append(controller.tunnels, tunnel)
 	NoticeTunnels(len(controller.tunnels))
+
+	// The split tunnel classifier is started once the first tunnel is
+	// established. This first tunnel is passed in to be used to make
+	// the routes data request.
+	// A long-running controller may run while the host device is present
+	// in different regions. In this case, we want the split tunnel logic
+	// to switch to routes for new regions and not classify traffic based
+	// on routes installed for older regions.
+	// We assume that when regions change, the host network will also
+	// change, and so all tunnels will fail and be re-established. Under
+	// that assumption, the classifier will be re-Start()-ed here when
+	// the region has changed.
+	if len(controller.tunnels) == 1 {
+		controller.splitTunnelClassifier.Start(
+			tunnel, controller.untunneledDialConfig)
+	}
+
 	return true
 }
 
@@ -465,7 +499,7 @@ func (controller *Controller) getNextActiveTunnel() (tunnel *Tunnel) {
 	return nil
 }
 
-// isActiveTunnelServerEntries is used to check if there's already
+// isActiveTunnelServerEntry is used to check if there's already
 // an existing tunnel to a candidate server.
 func (controller *Controller) isActiveTunnelServerEntry(serverEntry *ServerEntry) bool {
 	controller.tunnelMutex.Lock()
@@ -487,6 +521,11 @@ func (controller *Controller) Dial(remoteAddr string, downstreamConn net.Conn) (
 		return nil, ContextError(errors.New("no active tunnels"))
 	}
 
+	if controller.splitTunnelClassifier.IsUntunneled(remoteAddr) {
+		// !TODO! track downstreamConn and close it when the DialTCP conn closes, as with tunnel.Dial conns?
+		return DialTCP(remoteAddr, controller.untunneledDialConfig)
+	}
+
 	tunneledConn, err := tunnel.Dial(remoteAddr, downstreamConn)
 	if err != nil {
 		return nil, ContextError(err)

psiphon/dataStore.go

@@ -83,6 +83,10 @@ func InitDataStore(config *Config) (err error) {
             (serverEntryId text not null,
              protocol text not null,
              primary key (serverEntryId, protocol));
+        create table if not exists splitTunnelRoutes
+            (region text not null primary key,
+             etag text not null,
+             data blob not null);
         create table if not exists keyValue
             (key text not null primary key,
              value text not null);
@@ -520,6 +524,50 @@ func GetServerEntryIpAddresses() (ipAddresses []string, err error) {
 	return ipAddresses, nil
 }
 
+// SetSplitTunnelRoutes updates the cached routes data for
+// the given region. The associated etag is also stored and
+// used to make efficient web requests for updates to the data.
+func SetSplitTunnelRoutes(region, etag string, data []byte) error {
+	return transactionWithRetry(func(transaction *sql.Tx) error {
+		_, err := transaction.Exec(`
+            insert or replace into splitTunnelRoutes (region, etag, data)
+            values (?, ?. ?);
+            `, region, etag, data)
+		if err != nil {
+			// Note: ContextError() would break canRetry()
+			return err
+		}
+		return nil
+	})
+}
+
+// GetSplitTunnelRoutesETag retrieves the etag for cached routes
+// data for the specified region. If not found, it returns an empty string value.
+func GetSplitTunnelRoutesETag(region string) (etag string, err error) {
+	checkInitDataStore()
+	rows := singleton.db.QueryRow("select etag from splitTunnelRoutes where region = ?;", region)
+	err = rows.Scan(&etag)
+	if err == sql.ErrNoRows {
+		return "", nil
+	}
+	if err != nil {
+		return "", ContextError(err)
+	}
+	return etag, nil
+}
+
+// GetSplitTunnelRoutesData retrieves the cached routes data
+// for the specified region. It returns an error if not found.
+func GetSplitTunnelRoutesData(region string) (data []byte, err error) {
+	checkInitDataStore()
+	rows := singleton.db.QueryRow("select data from splitTunnelRoutes where region = ?;", region)
+	err = rows.Scan(&data)
+	if err != nil {
+		return nil, ContextError(err)
+	}
+	return data, nil
+}
+
 // SetKeyValue stores a key/value pair.
 func SetKeyValue(key, value string) error {
 	return transactionWithRetry(func(transaction *sql.Tx) error {

psiphon/remoteServerList.go

@@ -44,17 +44,9 @@ type RemoteServerList struct {
 // config.RemoteServerListUrl; validates its digital signature using the
 // public key config.RemoteServerListSignaturePublicKey; and parses the
 // data field into ServerEntry records.
-func FetchRemoteServerList(config *Config, pendingConns *Conns) (err error) {
+func FetchRemoteServerList(config *Config, dialConfig *DialConfig) (err error) {
 	NoticeInfo("fetching remote server list")
 
-	// Note: pendingConns may be used to interrupt the fetch remote server list
-	// request. BindToDevice may be used to exclude requests from VPN routing.
-	dialConfig := &DialConfig{
-		UpstreamHttpProxyAddress: config.UpstreamHttpProxyAddress,
-		PendingConns:             pendingConns,
-		DeviceBinder:             config.DeviceBinder,
-		DnsServerGetter:          config.DnsServerGetter,
-	}
 	transport := &http.Transport{
 		Dial: NewTCPDialer(dialConfig),
 	}

psiphon/serverApi.go

@@ -44,6 +44,7 @@ type Session struct {
 	psiphonHttpsClient *http.Client
 	statsRegexps       *transferstats.Regexps
 	statsServerId      string
+	clientRegion       string
 }
 
 // MakeSessionId creates a new session ID. Making the session ID is not done
@@ -204,12 +205,15 @@ func (session *Session) doHandshakeRequest() error {
 		PageViewRegexes      []map[string]string `json:"page_view_regexes"`
 		HttpsRequestRegexes  []map[string]string `json:"https_request_regexes"`
 		EncodedServerList    []string            `json:"encoded_server_list"`
+		ClientRegion         string              `json:"region"`
 	}
 	err = json.Unmarshal(configLine, &handshakeConfig)
 	if err != nil {
 		return ContextError(err)
 	}
 
+	session.clientRegion = handshakeConfig.ClientRegion
+
 	// Store discovered server entries
 	for _, encodedServerEntry := range handshakeConfig.EncodedServerList {
 		serverEntry, err := DecodeServerEntry(encodedServerEntry)

psiphon/splitTunnel.go

@@ -0,0 +1,239 @@
+/*
+ * Copyright (c) 2015, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package psiphon
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"sync"
+)
+
+// SplitTunnelClassifier determines whether a network destination
+// should be accessed through a tunnel or accessed directly.
+//
+// The classifier uses tables of IP address data, routes data,
+// to determine if a given IP is to be tunneled or not. If presented
+// with a hostname, the classifier performs a tunneled (uncensored)
+// DNS request to first determine the IP address for that hostname;
+// then a classification is made based on the IP address.
+//
+// Classification is by geographical region (country code). When the
+// split tunnel feature is configured to be on, and if the IP
+// address is within the user's region, it may be accessed untunneled.
+// Otherwise, the IP address must be accessed through a tunnel. The
+// user's current region is revealed to a Tunnel via the Psiphon server
+// API handshake.
+//
+// When a Tunnel has a blank region (e.g., when DisableApi is set and
+// the tunnel registers without performing a handshake) then no routes
+// data is set and all IP addresses are classified as requiring tunneling.
+//
+// Split tunnel is made on a best effort basis. After the classifier is
+// started, but before routes data is available for the given region,
+// all IP addresses will be classified as requiring tunneling.
+//
+// Routes data is fetched asynchronously after Start() is called. Routes
+// data is cached in the data store so it need not be downloaded in full
+// when fresh data is in the cache.
+type SplitTunnelClassifier struct {
+	mutex                sync.RWMutex
+	fetchRoutesUrlFormat string
+	dnsServerAddress     string
+	dnsDialConfig        *DialConfig
+	fetchRoutesWaitGroup *sync.WaitGroup
+	isRoutesSet          bool
+}
+
+func NewSplitTunnelClassifier(config *Config, dnsDialConfig *DialConfig) *SplitTunnelClassifier {
+	return &SplitTunnelClassifier{
+		fetchRoutesUrlFormat: config.SplitTunnelRoutesUrlFormat,
+		dnsServerAddress:     config.SplitTunnelDnsServer,
+		dnsDialConfig:        dnsDialConfig,
+		fetchRoutesWaitGroup: new(sync.WaitGroup),
+		isRoutesSet:          false,
+	}
+}
+
+// Start resets the state of the classifier. In the default state,
+// all IP addresses are classified as requiring tunneling. With
+// sufficient configuration and region info, this function starts
+// a goroutine to asynchronously fetch and install the routes data.
+func (classifier *SplitTunnelClassifier) Start(
+	fetchRoutesTunnel *Tunnel, dnsDialConfig *DialConfig) {
+
+	classifier.mutex.Lock()
+	defer classifier.mutex.Unlock()
+
+	classifier.isRoutesSet = false
+
+	if classifier.dnsServerAddress == "" ||
+		classifier.fetchRoutesUrlFormat == "" {
+		// Split tunnel capability is not configured
+		return
+	}
+
+	if fetchRoutesTunnel.session.clientRegion == "" {
+		// Split tunnel region is unknown
+		return
+	}
+
+	go classifier.setRoutes(fetchRoutesTunnel)
+}
+
+// Shutdown waits until the background setRoutes() goroutine is finished.
+// There is no explicit shutdown signal sent to setRoutes() -- instead
+// we assume that in an overall shutdown situation, the tunnel used for
+// network access in setRoutes() is closed and network events won't delay
+// the completion of the goroutine.
+func (classifier *SplitTunnelClassifier) Shutdown() {
+	classifier.mutex.Lock()
+	defer classifier.mutex.Unlock()
+
+	if classifier.fetchRoutesWaitGroup != nil {
+		classifier.fetchRoutesWaitGroup.Wait()
+		classifier.fetchRoutesWaitGroup = nil
+		classifier.isRoutesSet = false
+	}
+}
+
+// IsUntunneled takes a destination hostname or IP address and determines
+// if it should be accessed through a tunnel. When a hostname is presented, it
+// is first resolved to an IP address which can be matched against the routes data.
+// Multiple goroutines may invoke RequiresTunnel simultaneously. A multi-reader
+// lock is used to enable concurrent access.
+func (classifier *SplitTunnelClassifier) IsUntunneled(targetAddress string) bool {
+	classifier.mutex.RLock()
+	defer classifier.mutex.RUnlock()
+
+	if !classifier.isRoutesSet {
+		return false
+	}
+
+	// ***TODO***: implementation
+
+	return false
+}
+
+// setRoutes is a background routine that fetches routes data and installs it,
+// which sets the isRoutesSet flag, indicating that IP addresses may now be classified.
+func (classifier *SplitTunnelClassifier) setRoutes(tunnel *Tunnel) {
+	defer classifier.fetchRoutesWaitGroup.Done()
+
+	// Note: a possible optimization is to install cached routes
+	// before making the request. That would ensure some split
+	// tunneling for the duration of the request.
+
+	routesData, err := classifier.getRoutes(tunnel)
+	if err != nil {
+		NoticeAlert("failed to get split tunnel routes: %s", err)
+		return
+	}
+
+	err = classifier.installRoutes(routesData)
+	if err != nil {
+		NoticeAlert("failed to install split tunnel routes: %s", err)
+		return
+	}
+}
+
+// getRoutes makes a web request to download fresh routes data for the
+// given region, as indicated by the tunnel. It uses web caching, If-None-Match/ETag,
+// to save downloading known routes data repeatedly. If the web request
+// fails and cached routes data is present, that cached data is returned.
+func (classifier *SplitTunnelClassifier) getRoutes(tunnel *Tunnel) (routesData []byte, err error) {
+
+	url := fmt.Sprintf(classifier.fetchRoutesUrlFormat, tunnel.session.clientRegion)
+	request, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, ContextError(err)
+	}
+
+	etag, err := GetSplitTunnelRoutesETag(tunnel.session.clientRegion)
+	if err != nil {
+		return nil, ContextError(err)
+	}
+	if etag != "" {
+		request.Header.Add("If-None-Match", etag)
+	}
+
+	tunneledDialer := func(_, addr string) (conn net.Conn, err error) {
+		return tunnel.sshClient.Dial("tcp", addr)
+	}
+	transport := &http.Transport{
+		Dial: tunneledDialer,
+		ResponseHeaderTimeout: FETCH_ROUTES_TIMEOUT,
+	}
+	httpClient := &http.Client{
+		Transport: transport,
+		Timeout:   FETCH_ROUTES_TIMEOUT,
+	}
+
+	useCachedRoutes := false
+
+	response, err := httpClient.Do(request)
+	if err != nil {
+		NoticeAlert("failed to request split tunnel routes: %s", ContextError(err))
+		useCachedRoutes = true
+	} else {
+		defer response.Body.Close()
+		if response.StatusCode == http.StatusNotModified {
+			useCachedRoutes = true
+		} else {
+			routesData, err = ioutil.ReadAll(response.Body)
+			if err != nil {
+				NoticeAlert("failed to read split tunnel routes: %s", ContextError(err))
+				useCachedRoutes = true
+			} else {
+				etag := response.Header.Get("ETag")
+				if etag != "" {
+					err := SetSplitTunnelRoutes(tunnel.session.clientRegion, etag, routesData)
+					if err != nil {
+						NoticeAlert("failed to cache split tunnel routes: %s", ContextError(err))
+						// Proceed with fetched data, even when we can't cache it
+					}
+				}
+			}
+		}
+	}
+
+	if useCachedRoutes {
+		routesData, err = GetSplitTunnelRoutesData(tunnel.session.clientRegion)
+		if err != nil {
+			return nil, ContextError(err)
+		}
+	}
+
+	return routesData, nil
+}
+
+// installRoutes parses the raw routes data and creates data structures
+// for fast in-memory classification.
+func (classifier *SplitTunnelClassifier) installRoutes(routesData []byte) (err error) {
+	classifier.mutex.Lock()
+	defer classifier.mutex.Unlock()
+
+	// ***TODO***: implementation
+
+	classifier.isRoutesSet = true
+
+	return nil
+}