Merge remote-tracking branch 'upstream/master'

.travis.yml

@@ -2,9 +2,11 @@ language: go
 go:
 - 1.4
 - tip
+addons:
+  apt_packages:
+    - libx11-dev
+    - libgles2-mesa-dev
 install:
-- sudo apt-get install libx11-dev
-- sudo apt-get install libgles2-mesa-dev
 - go get -t -d -v ./... && go build -v ./...
 script:
 - go test -v ./...

README.md

@@ -29,8 +29,8 @@ Setup
     {
         "PropagationChannelId" : "<placeholder>",
         "SponsorId" : "<placeholder>",
-        "RemoteServerListUrl" : "<placeholder>",
-        "RemoteServerListSignaturePublicKey" : "<placeholder>",
+        "RemoteServerListUrl" : "",
+        "RemoteServerListSignaturePublicKey" : "",
         "DataStoreDirectory" : "",
         "DataStoreTempDirectory" : "",
         "LogFilename" : "",

psiphon/controller.go

@@ -116,6 +116,7 @@ func NewController(config *Config) (controller *Controller, err error) {
 func (controller *Controller) Run(shutdownBroadcast <-chan struct{}) {
 	NoticeBuildInfo()
 	NoticeCoreVersion(VERSION)
+	ReportAvailableRegions()
 
 	// Start components
 
@@ -126,7 +127,8 @@ func (controller *Controller) Run(shutdownBroadcast <-chan struct{}) {
 	}
 	defer socksProxy.Close()
 
-	httpProxy, err := NewHttpProxy(controller.config, controller)
+	httpProxy, err := NewHttpProxy(
+		controller.config, controller.untunneledDialConfig, controller)
 	if err != nil {
 		NoticeAlert("error initializing local HTTP proxy: %s", err)
 		return

psiphon/controller_test.go

@@ -20,7 +20,11 @@
 package psiphon
 
 import (
+	"fmt"
 	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -69,19 +73,32 @@ func controllerRun(t *testing.T, protocol string) {
 	}
 
 	// Monitor notices for "Tunnels" with count > 1, the
-	// indication of tunnel establishment success
+	// indication of tunnel establishment success.
+	// Also record the selected HTTP proxy port to use
+	// when fetching websites through the tunnel.
+
+	httpProxyPort := 0
 
 	tunnelEstablished := make(chan struct{}, 1)
 	SetNoticeOutput(NewNoticeReceiver(
 		func(notice []byte) {
 			// TODO: log notices without logging server IPs:
 			// fmt.Fprintf(os.Stderr, "%s\n", string(notice))
-			count, ok := GetNoticeTunnels(notice)
-			if ok && count > 0 {
-				select {
-				case tunnelEstablished <- *new(struct{}):
-				default:
+			noticeType, payload, err := GetNotice(notice)
+			if err != nil {
+				return
+			}
+			switch noticeType {
+			case "Tunnels":
+				count := int(payload["count"].(float64))
+				if count > 0 {
+					select {
+					case tunnelEstablished <- *new(struct{}):
+					default:
+					}
 				}
+			case "ListeningHttpProxyPort":
+				httpProxyPort = int(payload["port"].(float64))
 			}
 		}))
 
@@ -101,8 +118,12 @@ func controllerRun(t *testing.T, protocol string) {
 
 	select {
 	case <-tunnelEstablished:
+		// Test: fetch website through tunnel
+		fetchWebsite(t, httpProxyPort)
+
 	case <-establishTimeout.C:
 		t.Errorf("tunnel establish timeout exceeded")
+		// ...continue with cleanup
 	}
 
 	close(shutdownBroadcast)
@@ -123,3 +144,96 @@ func controllerRun(t *testing.T, protocol string) {
 		t.Errorf("controller shutdown timeout exceeded")
 	}
 }
+
+func fetchWebsite(t *testing.T, httpProxyPort int) {
+
+	testUrl := "https://raw.githubusercontent.com/Psiphon-Labs/psiphon-tunnel-core/master/LICENSE"
+	roundTripTimeout := 10 * time.Second
+	expectedResponsePrefix := "                    GNU GENERAL PUBLIC LICENSE"
+	expectedResponseSize := 35148
+	checkResponse := func(responseBody string) bool {
+		return strings.HasPrefix(responseBody, expectedResponsePrefix) && len(responseBody) == expectedResponseSize
+	}
+
+	// Test: use HTTP proxy
+
+	proxyUrl, err := url.Parse(fmt.Sprintf("http://127.0.0.1:%d", httpProxyPort))
+	if err != nil {
+		t.Errorf("error initializing proxied HTTP request: %s", err)
+		t.FailNow()
+	}
+
+	httpClient := &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyURL(proxyUrl),
+		},
+		Timeout: roundTripTimeout,
+	}
+
+	response, err := httpClient.Get(testUrl)
+	if err != nil {
+		t.Errorf("error sending proxied HTTP request: %s", err)
+		t.FailNow()
+	}
+
+	body, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		t.Errorf("error reading proxied HTTP response: %s", err)
+		t.FailNow()
+	}
+	response.Body.Close()
+
+	if !checkResponse(string(body)) {
+		t.Errorf("unexpected proxied HTTP response")
+		t.FailNow()
+	}
+
+	// Test: use direct URL proxy
+
+	httpClient = &http.Client{
+		Transport: http.DefaultTransport,
+		Timeout:   roundTripTimeout,
+	}
+
+	response, err = httpClient.Get(
+		fmt.Sprintf("http://127.0.0.1:%d/direct/%s",
+			httpProxyPort, url.QueryEscape(testUrl)))
+	if err != nil {
+		t.Errorf("error sending direct URL request: %s", err)
+		t.FailNow()
+	}
+
+	body, err = ioutil.ReadAll(response.Body)
+	if err != nil {
+		t.Errorf("error reading direct URL response: %s", err)
+		t.FailNow()
+	}
+	response.Body.Close()
+
+	if !checkResponse(string(body)) {
+		t.Errorf("unexpected direct URL response")
+		t.FailNow()
+	}
+
+	// Test: use tunneled URL proxy
+
+	response, err = httpClient.Get(
+		fmt.Sprintf("http://127.0.0.1:%d/tunneled/%s",
+			httpProxyPort, url.QueryEscape(testUrl)))
+	if err != nil {
+		t.Errorf("error sending tunneled URL request: %s", err)
+		t.FailNow()
+	}
+
+	body, err = ioutil.ReadAll(response.Body)
+	if err != nil {
+		t.Errorf("error reading tunneled URL response: %s", err)
+		t.FailNow()
+	}
+	response.Body.Close()
+
+	if !checkResponse(string(body)) {
+		t.Errorf("unexpected tunneled URL response")
+		t.FailNow()
+	}
+}

psiphon/dataStore.go

@@ -79,6 +79,7 @@ func InitDataStore(config *Config) (err error) {
              rank integer not null unique,
              region text not null,
              data blob not null);
+        create index if not exists idx_serverEntry_region on serverEntry(region);
         create table if not exists serverEntryProtocol
             (serverEntryId text not null,
              protocol text not null,
@@ -256,6 +257,11 @@ func StoreServerEntries(serverEntries []*ServerEntry, replaceIfExists bool) erro
 			return ContextError(err)
 		}
 	}
+
+	// Since there has possibly been a significant change in the server entries,
+	// take this opportunity to update the available egress regions.
+	ReportAvailableRegions()
+
 	return nil
 }
 
@@ -517,6 +523,39 @@ func CountServerEntries(region, protocol string) int {
 	return count
 }
 
+// ReportAvailableRegions prints a notice with the available egress regions.
+func ReportAvailableRegions() {
+	checkInitDataStore()
+
+	// TODO: For consistency, regions-per-protocol should be used
+
+	rows, err := singleton.db.Query("select distinct(region) from serverEntry;")
+	if err != nil {
+		NoticeAlert("failed to query data store for available regions: %s", ContextError(err))
+		return
+	}
+	defer rows.Close()
+
+	var regions []string
+
+	for rows.Next() {
+		var region string
+		err = rows.Scan(&region)
+		if err != nil {
+			NoticeAlert("failed to retrieve available regions from data store: %s", ContextError(err))
+			return
+		}
+
+		// Some server entries do not have a region, but it makes no sense to return
+		// an empty string as an "available region".
+		if (region != "") {
+			regions = append(regions, region)
+		}
+	}
+
+	NoticeAvailableEgressRegions(regions)
+}
+
 // GetServerEntryIpAddresses returns an array containing
 // all stored server IP addresses.
 func GetServerEntryIpAddresses() (ipAddresses []string, err error) {

psiphon/httpProxy.go

@@ -25,22 +25,51 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"net/url"
+	"strings"
 	"sync"
 )
 
-// HttpProxy is a HTTP server that relays HTTP requests through
-// the tunnel SSH client.
+// HttpProxy is a HTTP server that relays HTTP requests through the Psiphon tunnel.
+// It includes support for HTTP CONNECT.
+//
+// This proxy also offers a "URL proxy" mode that relays requests for HTTP or HTTPS
+// or URLs specified in the proxy request path. This mode relays either through the
+// Psiphon tunnel, or directly.
+//
+// An example use case for tunneled URL proxy relays is to craft proxied URLs to pass to
+// components that don't support HTTP or SOCKS proxy settings. For example, the
+// Android Media Player (http://developer.android.com/reference/android/media/MediaPlayer.html).
+// To make the Media Player use the Psiphon tunnel, construct a URL such as:
+// "http://127.0.0.1:<proxy-port>/tunneled/<origin media URL>"; and pass this to the player.
+// TODO: add ICY protocol to support certain streaming media (e.g., https://gist.github.com/tulskiy/1008126)
+//
+// An example use case for direct, untunneled, relaying is to make use of Go's TLS
+// stack for HTTPS requests in cases where the native TLS stack is lacking (e.g.,
+// WinHTTP on Windows XP). The URL for direct relaying is:
+// "http://127.0.0.1:<proxy-port>/direct/<origin URL>".
+//
+// Origin URLs must include the scheme prefix ("http://" or "https://") and must be
+// URL encoded.
+//
 type HttpProxy struct {
 	tunneler               Tunneler
 	listener               net.Listener
 	serveWaitGroup         *sync.WaitGroup
-	httpRelay              *http.Transport
+	httpTunneledRelay      *http.Transport
+	httpTunneledClient     *http.Client
+	httpDirectRelay        *http.Transport
+	httpDirectClient       *http.Client
 	openConns              *Conns
 	stopListeningBroadcast chan struct{}
 }
 
 // NewHttpProxy initializes and runs a new HTTP proxy server.
-func NewHttpProxy(config *Config, tunneler Tunneler) (proxy *HttpProxy, err error) {
+func NewHttpProxy(
+	config *Config,
+	untunneledDialConfig *DialConfig,
+	tunneler Tunneler) (proxy *HttpProxy, err error) {
+
 	listener, err := net.Listen(
 		"tcp", fmt.Sprintf("127.0.0.1:%d", config.LocalHttpProxyPort))
 	if err != nil {
@@ -49,6 +78,7 @@ func NewHttpProxy(config *Config, tunneler Tunneler) (proxy *HttpProxy, err erro
 		}
 		return nil, ContextError(err)
 	}
+
 	tunneledDialer := func(_, addr string) (conn net.Conn, err error) {
 		// downstreamConn is not set in this case, as there is not a fixed
 		// association between a downstream client connection and a particular
@@ -56,17 +86,38 @@ func NewHttpProxy(config *Config, tunneler Tunneler) (proxy *HttpProxy, err erro
 		// TODO: connect timeout?
 		return tunneler.Dial(addr, false, nil)
 	}
-	// TODO: also use http.Client, with its Timeout field?
-	transport := &http.Transport{
-		Dial:                  tunneledDialer,
+	httpTunneledRelay := &http.Transport{
+		Dial:                tunneledDialer,
+		MaxIdleConnsPerHost: HTTP_PROXY_MAX_IDLE_CONNECTIONS_PER_HOST,
+	}
+	httpTunneledClient := &http.Client{
+		Transport: httpTunneledRelay,
+		Jar:       nil, // TODO: cookie support for URL proxy?
+		Timeout:   HTTP_PROXY_ORIGIN_SERVER_TIMEOUT,
+	}
+
+	directDialer := func(_, addr string) (conn net.Conn, err error) {
+		return DialTCP(addr, untunneledDialConfig)
+	}
+	httpDirectRelay := &http.Transport{
+		Dial:                  directDialer,
 		MaxIdleConnsPerHost:   HTTP_PROXY_MAX_IDLE_CONNECTIONS_PER_HOST,
 		ResponseHeaderTimeout: HTTP_PROXY_ORIGIN_SERVER_TIMEOUT,
 	}
+	httpDirectClient := &http.Client{
+		Transport: httpDirectRelay,
+		Jar:       nil,
+		Timeout:   HTTP_PROXY_ORIGIN_SERVER_TIMEOUT,
+	}
+
 	proxy = &HttpProxy{
 		tunneler:               tunneler,
 		listener:               listener,
 		serveWaitGroup:         new(sync.WaitGroup),
-		httpRelay:              transport,
+		httpTunneledRelay:      httpTunneledRelay,
+		httpTunneledClient:     httpTunneledClient,
+		httpDirectRelay:        httpDirectRelay,
+		httpDirectClient:       httpDirectClient,
 		openConns:              new(Conns),
 		stopListeningBroadcast: make(chan struct{}),
 	}
@@ -85,7 +136,8 @@ func (proxy *HttpProxy) Close() {
 	proxy.openConns.CloseAll()
 	// Close idle proxy->origin persistent connections
 	// TODO: also close active connections
-	proxy.httpRelay.CloseIdleConnections()
+	proxy.httpTunneledRelay.CloseIdleConnections()
+	proxy.httpDirectRelay.CloseIdleConnections()
 }
 
 // ServeHTTP receives HTTP requests and proxies them. CONNECT requests
@@ -119,15 +171,89 @@ func (proxy *HttpProxy) ServeHTTP(responseWriter http.ResponseWriter, request *h
 				NoticeAlert("%s", ContextError(err))
 			}
 		}()
+	} else if request.URL.IsAbs() {
+		proxy.httpProxyHandler(responseWriter, request)
+	} else {
+		proxy.urlProxyHandler(responseWriter, request)
+	}
+}
+
+func (proxy *HttpProxy) httpConnectHandler(localConn net.Conn, target string) (err error) {
+	defer localConn.Close()
+	defer proxy.openConns.Remove(localConn)
+	proxy.openConns.Add(localConn)
+	// Setting downstreamConn so localConn.Close() will be called when remoteConn.Close() is called.
+	// This ensures that the downstream client (e.g., web browser) doesn't keep waiting on the
+	// open connection for data which will never arrive.
+	remoteConn, err := proxy.tunneler.Dial(target, false, localConn)
+	if err != nil {
+		return ContextError(err)
+	}
+	defer remoteConn.Close()
+	_, err = localConn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n"))
+	if err != nil {
+		return ContextError(err)
+	}
+	Relay(localConn, remoteConn)
+	return nil
+}
+
+func (proxy *HttpProxy) httpProxyHandler(responseWriter http.ResponseWriter, request *http.Request) {
+	relayHttpRequest(proxy.httpTunneledClient, request, responseWriter)
+}
+
+const (
+	URL_PROXY_TUNNELED_REQUEST_PATH = "/tunneled/"
+	URL_PROXY_DIRECT_REQUEST_PATH   = "/direct/"
+)
+
+func (proxy *HttpProxy) urlProxyHandler(responseWriter http.ResponseWriter, request *http.Request) {
+
+	var client *http.Client
+	var originUrl string
+	var err error
+
+	// Request URL should be "/tunneled/<origin URL>" or  "/direct/<origin URL>" and the
+	// origin URL must be URL encoded.
+	switch {
+	case strings.HasPrefix(request.URL.Path, URL_PROXY_TUNNELED_REQUEST_PATH):
+		originUrl, err = url.QueryUnescape(request.URL.Path[len(URL_PROXY_TUNNELED_REQUEST_PATH):])
+		client = proxy.httpTunneledClient
+	case strings.HasPrefix(request.URL.Path, URL_PROXY_DIRECT_REQUEST_PATH):
+		originUrl, err = url.QueryUnescape(request.URL.Path[len(URL_PROXY_DIRECT_REQUEST_PATH):])
+		client = proxy.httpDirectClient
+	default:
+		err = errors.New("missing origin URL")
+	}
+	if err != nil {
+		NoticeAlert("%s", ContextError(err))
+		forceClose(responseWriter)
+		return
+	}
+
+	// Origin URL must be well-formed, absolute, and have a scheme of  "http" or "https"
+	url, err := url.ParseRequestURI(originUrl)
+	if err != nil {
+		NoticeAlert("%s", ContextError(err))
+		forceClose(responseWriter)
 		return
 	}
-	if !request.URL.IsAbs() {
-		NoticeAlert("%s", ContextError(errors.New("no domain in request URL")))
-		http.Error(responseWriter, "", http.StatusInternalServerError)
+	if !url.IsAbs() || (url.Scheme != "http" && url.Scheme != "https") {
+		NoticeAlert("invalid origin URL")
+		forceClose(responseWriter)
 		return
 	}
 
-	// Transform request struct before using as input to relayed request
+	// Transform received request to directly reference the origin URL
+	request.Host = url.Host
+	request.URL = url
+
+	relayHttpRequest(client, request, responseWriter)
+}
+
+func relayHttpRequest(client *http.Client, request *http.Request, responseWriter http.ResponseWriter) {
+
+	// Transform received request struct before using as input to relayed request
 	request.Close = false
 	request.RequestURI = ""
 	for _, key := range hopHeaders {
@@ -135,7 +261,8 @@ func (proxy *HttpProxy) ServeHTTP(responseWriter http.ResponseWriter, request *h
 	}
 
 	// Relay the HTTP request and get the response
-	response, err := proxy.httpRelay.RoundTrip(request)
+	//response, err := relay.RoundTrip(request)
+	response, err := client.Do(request)
 	if err != nil {
 		NoticeAlert("%s", ContextError(err))
 		forceClose(responseWriter)
@@ -192,26 +319,6 @@ var hopHeaders = []string{
 	"Upgrade",
 }
 
-func (proxy *HttpProxy) httpConnectHandler(localConn net.Conn, target string) (err error) {
-	defer localConn.Close()
-	defer proxy.openConns.Remove(localConn)
-	proxy.openConns.Add(localConn)
-	// Setting downstreamConn so localConn.Close() will be called when remoteConn.Close() is called.
-	// This ensures that the downstream client (e.g., web browser) doesn't keep waiting on the
-	// open connection for data which will never arrive.
-	remoteConn, err := proxy.tunneler.Dial(target, false, localConn)
-	if err != nil {
-		return ContextError(err)
-	}
-	defer remoteConn.Close()
-	_, err = localConn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n"))
-	if err != nil {
-		return ContextError(err)
-	}
-	Relay(localConn, remoteConn)
-	return nil
-}
-
 // httpConnStateCallback is called by http.Server when the state of a local->proxy
 // connection changes. Open connections are tracked so that all local->proxy persistent
 // connections can be closed by HttpProxy.Close()

psiphon/notice.go

@@ -109,6 +109,11 @@ func NoticeCandidateServers(region, protocol string, count int) {
 	outputNotice("CandidateServers", false, "region", region, "protocol", protocol, "count", count)
 }
 
+// NoticeAvailableEgressRegions is what regions are available for egress from
+func NoticeAvailableEgressRegions(regions []string) {
+	outputNotice("AvailableEgressRegions", false, "regions", regions)
+}
+
 // NoticeConnectingServer is details on a connection attempt
 func NoticeConnectingServer(ipAddress, region, protocol, frontingAddress string) {
 	outputNotice("ConnectingServer", false, "ipAddress", ipAddress, "region",
@@ -186,24 +191,22 @@ type noticeObject struct {
 	Timestamp  string          `json:"timestamp"`
 }
 
-// GetNoticeTunnels receives a JSON encoded object and attempts to parse it as a Notice.
-// When the object is a Notice of type Tunnels, the count payload is returned.
-func GetNoticeTunnels(notice []byte) (count int, ok bool) {
+// GetNotice receives a JSON encoded object and attempts to parse it as a Notice.
+// The type is returned as a string and the payload as a generic map.
+func GetNotice(notice []byte) (
+	noticeType string, payload map[string]interface{}, err error) {
+
 	var object noticeObject
-	if json.Unmarshal(notice, &object) != nil {
-		return 0, false
-	}
-	if object.NoticeType != "Tunnels" {
-		return 0, false
-	}
-	type tunnelsPayload struct {
-		Count int `json:"count"`
+	err = json.Unmarshal(notice, &object)
+	if err != nil {
+		return "", nil, err
 	}
-	var payload tunnelsPayload
-	if json.Unmarshal(object.Data, &payload) != nil {
-		return 0, false
+	var objectPayload interface{}
+	err = json.Unmarshal(object.Data, &objectPayload)
+	if err != nil {
+		return "", nil, err
 	}
-	return payload.Count, true
+	return object.NoticeType, objectPayload.(map[string]interface{}), nil
 }
 
 // NoticeReceiver consumes a notice input stream and invokes a callback function

psiphon/serverApi.go

@@ -109,14 +109,16 @@ func (session *Session) DoConnectedRequest() error {
 	if err != nil {
 		return ContextError(err)
 	}
+
 	var response struct {
-		connectedTimestamp string `json:connected_timestamp`
+		ConnectedTimestamp string `json:"connected_timestamp"`
 	}
 	err = json.Unmarshal(responseBody, &response)
 	if err != nil {
 		return ContextError(err)
 	}
-	err = SetKeyValue(DATA_STORE_LAST_CONNECTED_KEY, response.connectedTimestamp)
+
+	err = SetKeyValue(DATA_STORE_LAST_CONNECTED_KEY, response.ConnectedTimestamp)
 	if err != nil {
 		return ContextError(err)
 	}
@@ -214,6 +216,8 @@ func (session *Session) doHandshakeRequest() error {
 
 	session.clientRegion = handshakeConfig.ClientRegion
 
+	var decodedServerEntries []*ServerEntry
+
 	// Store discovered server entries
 	for _, encodedServerEntry := range handshakeConfig.EncodedServerList {
 		serverEntry, err := DecodeServerEntry(encodedServerEntry)
@@ -225,10 +229,16 @@ func (session *Session) doHandshakeRequest() error {
 			// Skip this entry and continue with the next one
 			continue
 		}
-		err = StoreServerEntry(serverEntry, true)
-		if err != nil {
-			return ContextError(err)
-		}
+
+		decodedServerEntries = append(decodedServerEntries, serverEntry)
+	}
+
+	// The reason we are storing the entire array of server entries at once rather
+	// than one at a time is that some desirable side-effects get triggered by
+	// StoreServerEntries that don't get triggered by StoreServerEntry.
+	err = StoreServerEntries(decodedServerEntries, true)
+	if err != nil {
+		return ContextError(err)
 	}
 
 	// TODO: formally communicate the sponsor and upgrade info to an