* GeoIP session cache for web transport API requests

* Add test case forcing use of web transport API requests

Server/main.go

@@ -121,9 +121,12 @@ func main() {
 
 		configFileContents, serverEntryFileContents, err :=
 			server.GenerateConfig(
-				serverIPaddress,
-				generateWebServerPort,
-				tunnelProtocolPorts)
+				&server.GenerateConfigParams{
+					ServerIPAddress:      serverIPaddress,
+					EnableSSHAPIRequests: true,
+					WebServerPort:        generateWebServerPort,
+					TunnelProtocolPorts:  tunnelProtocolPorts,
+				})
 		if err != nil {
 			fmt.Printf("generate failed: %s\n", err)
 			os.Exit(1)

psiphon/server/config.go

@@ -57,6 +57,7 @@ const (
 	REDIS_POOL_MAX_IDLE                   = 50
 	REDIS_POOL_MAX_ACTIVE                 = 1000
 	REDIS_POOL_IDLE_TIMEOUT               = 5 * time.Minute
+	GEOIP_SESSION_CACHE_TTL               = 60 * time.Minute
 )
 
 // TODO: break config into sections (sub-structs)
@@ -447,6 +448,15 @@ func LoadConfig(configJSONs [][]byte) (*Config, error) {
 	return &config, nil
 }
 
+// GenerateConfigParams specifies customizations to be applied to
+// a generated server config.
+type GenerateConfigParams struct {
+	ServerIPAddress      string
+	WebServerPort        int
+	EnableSSHAPIRequests bool
+	TunnelProtocolPorts  map[string]int
+}
+
 // GenerateConfig creates a new Psiphon server config. It returns a JSON
 // encoded config and a client-compatible "server entry" for the server. It
 // generates all necessary secrets and key material, which are emitted in
@@ -454,51 +464,57 @@ func LoadConfig(configJSONs [][]byte) (*Config, error) {
 // GenerateConfig uses sample values for many fields. The intention is for
 // a generated config to be used for testing or as a template for production
 // setup, not to generate production-ready configurations.
-func GenerateConfig(
-	serverIPaddress string,
-	webServerPort int,
-	tunnelProtocolPorts map[string]int) ([]byte, []byte, error) {
+func GenerateConfig(params *GenerateConfigParams) ([]byte, []byte, error) {
 
 	// Input validation
 
-	if net.ParseIP(serverIPaddress) == nil {
+	if net.ParseIP(params.ServerIPAddress) == nil {
 		return nil, nil, psiphon.ContextError(errors.New("invalid IP address"))
 	}
 
-	if len(tunnelProtocolPorts) == 0 {
+	if len(params.TunnelProtocolPorts) == 0 {
 		return nil, nil, psiphon.ContextError(errors.New("no tunnel protocols"))
 	}
 
 	usedPort := make(map[int]bool)
-	if webServerPort != 0 {
-		usedPort[webServerPort] = true
+	if params.WebServerPort != 0 {
+		usedPort[params.WebServerPort] = true
 	}
 
-	for protocol, port := range tunnelProtocolPorts {
+	usingMeek := false
+
+	for protocol, port := range params.TunnelProtocolPorts {
+
 		if !psiphon.Contains(psiphon.SupportedTunnelProtocols, protocol) {
 			return nil, nil, psiphon.ContextError(errors.New("invalid tunnel protocol"))
 		}
+
 		if usedPort[port] {
 			return nil, nil, psiphon.ContextError(errors.New("duplicate listening port"))
 		}
 		usedPort[port] = true
+
+		if psiphon.TunnelProtocolUsesMeekHTTP(protocol) ||
+			psiphon.TunnelProtocolUsesMeekHTTPS(protocol) {
+			usingMeek = true
+		}
 	}
 
 	// Web server config
 
-	webServerSecret, err := psiphon.MakeRandomStringHex(WEB_SERVER_SECRET_BYTE_LENGTH)
-	if err != nil {
-		return nil, nil, psiphon.ContextError(err)
-	}
+	var webServerSecret, webServerCertificate, webServerPrivateKey string
 
-	webServerCertificate, webServerPrivateKey, err := GenerateWebServerCertificate("")
-	if err != nil {
-		return nil, nil, psiphon.ContextError(err)
-	}
+	if params.WebServerPort != 0 {
+		var err error
+		webServerSecret, err = psiphon.MakeRandomStringHex(WEB_SERVER_SECRET_BYTE_LENGTH)
+		if err != nil {
+			return nil, nil, psiphon.ContextError(err)
+		}
 
-	discoveryValueHMACKey, err := psiphon.MakeRandomStringBase64(DISCOVERY_VALUE_KEY_BYTE_LENGTH)
-	if err != nil {
-		return nil, nil, psiphon.ContextError(err)
+		webServerCertificate, webServerPrivateKey, err = GenerateWebServerCertificate("")
+		if err != nil {
+			return nil, nil, psiphon.ContextError(err)
+		}
 	}
 
 	// SSH config
@@ -547,13 +563,27 @@ func GenerateConfig(
 
 	// Meek config
 
-	meekCookieEncryptionPublicKey, meekCookieEncryptionPrivateKey, err :=
-		box.GenerateKey(rand.Reader)
-	if err != nil {
-		return nil, nil, psiphon.ContextError(err)
+	var meekCookieEncryptionPublicKey, meekCookieEncryptionPrivateKey, meekObfuscatedKey string
+
+	if usingMeek {
+		rawMeekCookieEncryptionPublicKey, rawMeekCookieEncryptionPrivateKey, err :=
+			box.GenerateKey(rand.Reader)
+		if err != nil {
+			return nil, nil, psiphon.ContextError(err)
+		}
+
+		meekCookieEncryptionPublicKey = base64.StdEncoding.EncodeToString(rawMeekCookieEncryptionPublicKey[:])
+		meekCookieEncryptionPrivateKey = base64.StdEncoding.EncodeToString(rawMeekCookieEncryptionPrivateKey[:])
+
+		meekObfuscatedKey, err = psiphon.MakeRandomStringHex(SSH_OBFUSCATED_KEY_BYTE_LENGTH)
+		if err != nil {
+			return nil, nil, psiphon.ContextError(err)
+		}
 	}
 
-	meekObfuscatedKey, err := psiphon.MakeRandomStringHex(SSH_OBFUSCATED_KEY_BYTE_LENGTH)
+	// Other config
+
+	discoveryValueHMACKey, err := psiphon.MakeRandomStringBase64(DISCOVERY_VALUE_KEY_BYTE_LENGTH)
 	if err != nil {
 		return nil, nil, psiphon.ContextError(err)
 	}
@@ -570,9 +600,9 @@ func GenerateConfig(
 		Fail2BanFormat:                 "Authentication failure for psiphon-client from %s",
 		GeoIPDatabaseFilename:          "",
 		HostID:                         "example-host-id",
-		ServerIPAddress:                serverIPaddress,
+		ServerIPAddress:                params.ServerIPAddress,
 		DiscoveryValueHMACKey:          discoveryValueHMACKey,
-		WebServerPort:                  webServerPort,
+		WebServerPort:                  params.WebServerPort,
 		WebServerSecret:                webServerSecret,
 		WebServerCertificate:           webServerCertificate,
 		WebServerPrivateKey:            webServerPrivateKey,
@@ -581,11 +611,11 @@ func GenerateConfig(
 		SSHUserName:                    sshUserName,
 		SSHPassword:                    sshPassword,
 		ObfuscatedSSHKey:               obfuscatedSSHKey,
-		TunnelProtocolPorts:            tunnelProtocolPorts,
+		TunnelProtocolPorts:            params.TunnelProtocolPorts,
 		RedisServerAddress:             "",
 		UDPForwardDNSServerAddress:     "8.8.8.8:53",
 		UDPInterceptUdpgwServerAddress: "127.0.0.1:7300",
-		MeekCookieEncryptionPrivateKey: base64.StdEncoding.EncodeToString(meekCookieEncryptionPrivateKey[:]),
+		MeekCookieEncryptionPrivateKey: meekCookieEncryptionPrivateKey,
 		MeekObfuscatedKey:              meekObfuscatedKey,
 		MeekCertificateCommonName:      "www.example.org",
 		MeekProhibitedHeaders:          nil,
@@ -618,34 +648,44 @@ func GenerateConfig(
 	lines := strings.Split(webServerCertificate, "\n")
 	strippedWebServerCertificate := strings.Join(lines[1:len(lines)-2], "")
 
-	capabilities := []string{psiphon.CAPABILITY_SSH_API_REQUESTS}
+	capabilities := []string{}
 
-	if webServerPort != 0 {
+	if params.EnableSSHAPIRequests {
+		capabilities = append(capabilities, psiphon.CAPABILITY_SSH_API_REQUESTS)
+	}
+
+	if params.WebServerPort != 0 {
 		capabilities = append(capabilities, psiphon.CAPABILITY_UNTUNNELED_WEB_API_REQUESTS)
 	}
 
-	for protocol, _ := range tunnelProtocolPorts {
+	for protocol, _ := range params.TunnelProtocolPorts {
 		capabilities = append(capabilities, psiphon.GetCapability(protocol))
 	}
 
-	sshPort := tunnelProtocolPorts["SSH"]
-	obfuscatedSSHPort := tunnelProtocolPorts["OSSH"]
+	sshPort := params.TunnelProtocolPorts["SSH"]
+	obfuscatedSSHPort := params.TunnelProtocolPorts["OSSH"]
 
 	// Meek port limitations
 	// - fronted meek protocols are hard-wired in the client to be port 443 or 80.
 	// - only one other meek port may be specified.
-	meekPort := tunnelProtocolPorts["UNFRONTED-MEEK-OSSH"]
+	meekPort := params.TunnelProtocolPorts["UNFRONTED-MEEK-OSSH"]
 	if meekPort == 0 {
-		meekPort = tunnelProtocolPorts["UNFRONTED-MEEK-HTTPS-OSSH"]
+		meekPort = params.TunnelProtocolPorts["UNFRONTED-MEEK-HTTPS-OSSH"]
 	}
 
 	// Note: fronting params are a stub; this server entry will exercise
 	// client and server fronting code paths, but not actually traverse
 	// a fronting hop.
 
+	serverEntryWebServerPort := ""
+
+	if params.WebServerPort != 0 {
+		serverEntryWebServerPort = fmt.Sprintf("%d", params.WebServerPort)
+	}
+
 	serverEntry := &psiphon.ServerEntry{
-		IpAddress:                     serverIPaddress,
-		WebServerPort:                 fmt.Sprintf("%d", webServerPort),
+		IpAddress:                     params.ServerIPAddress,
+		WebServerPort:                 serverEntryWebServerPort,
 		WebServerSecret:               webServerSecret,
 		WebServerCertificate:          strippedWebServerCertificate,
 		SshPort:                       sshPort,
@@ -657,10 +697,10 @@ func GenerateConfig(
 		Capabilities:                  capabilities,
 		Region:                        "US",
 		MeekServerPort:                meekPort,
-		MeekCookieEncryptionPublicKey: base64.StdEncoding.EncodeToString(meekCookieEncryptionPublicKey[:]),
+		MeekCookieEncryptionPublicKey: meekCookieEncryptionPublicKey,
 		MeekObfuscatedKey:             meekObfuscatedKey,
-		MeekFrontingHosts:             []string{serverIPaddress},
-		MeekFrontingAddresses:         []string{serverIPaddress},
+		MeekFrontingHosts:             []string{params.ServerIPAddress},
+		MeekFrontingAddresses:         []string{params.ServerIPAddress},
 		MeekFrontingDisableSNI:        false,
 	}
 

psiphon/server/geoip.go

@@ -23,7 +23,9 @@ import (
 	"crypto/hmac"
 	"crypto/sha256"
 	"net"
+	"time"
 
+	cache "github.com/Psiphon-Inc/go-cache"
 	maxminddb "github.com/Psiphon-Inc/maxminddb-golang"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon"
 )
@@ -97,6 +99,24 @@ func GeoIPLookup(ipAddress string) GeoIPData {
 	return result
 }
 
+func SetGeoIPSessionCache(sessionID string, geoIPData GeoIPData) {
+	if geoIPSessionCache == nil {
+		return
+	}
+	geoIPSessionCache.Set(sessionID, geoIPData, cache.DefaultExpiration)
+}
+
+func GetGeoIPSessionCache(sessionID string) GeoIPData {
+	if geoIPSessionCache == nil {
+		return NewGeoIPData()
+	}
+	geoIPData, found := geoIPSessionCache.Get(sessionID)
+	if !found {
+		return NewGeoIPData()
+	}
+	return geoIPData.(GeoIPData)
+}
+
 // calculateDiscoveryValue derives a value from the client IP address to be
 // used as input in the server discovery algorithm. Since we do not explicitly
 // store the client IP address, we must derive the value here and store it for
@@ -115,6 +135,7 @@ func calculateDiscoveryValue(ipAddress string) int {
 }
 
 var geoIPReader *maxminddb.Reader
+var geoIPSessionCache *cache.Cache
 var discoveryValueHMACKey string
 
 // InitGeoIP opens a GeoIP2/GeoLite2 MaxMind database and prepares
@@ -124,11 +145,15 @@ func InitGeoIP(config *Config) error {
 	discoveryValueHMACKey = config.DiscoveryValueHMACKey
 
 	if config.GeoIPDatabaseFilename != "" {
+
 		var err error
 		geoIPReader, err = maxminddb.Open(config.GeoIPDatabaseFilename)
 		if err != nil {
 			return psiphon.ContextError(err)
 		}
+
+		geoIPSessionCache = cache.New(GEOIP_SESSION_CACHE_TTL, 1*time.Minute)
+
 		log.WithContext().Info("GeoIP initialized")
 	}
 

psiphon/server/server_test.go

@@ -45,29 +45,61 @@ func TestMain(m *testing.M) {
 // hard-wired to except running on privileged ports 80 and 443.
 
 func TestSSH(t *testing.T) {
-	runServer(t, "SSH")
+	runServer(t,
+		&runServerConfig{
+			tunnelProtocol:       "SSH",
+			enableSSHAPIRequests: true,
+		})
 }
 
 func TestOSSH(t *testing.T) {
-	runServer(t, "OSSH")
+	runServer(t,
+		&runServerConfig{
+			tunnelProtocol:       "OSSH",
+			enableSSHAPIRequests: true,
+		})
 }
 
 func TestUnfrontedMeek(t *testing.T) {
-	runServer(t, "UNFRONTED-MEEK-OSSH")
+	runServer(t,
+		&runServerConfig{
+			tunnelProtocol:       "UNFRONTED-MEEK-OSSH",
+			enableSSHAPIRequests: true,
+		})
 }
 
 func TestUnfrontedMeekHTTPS(t *testing.T) {
-	runServer(t, "UNFRONTED-MEEK-HTTPS-OSSH")
+	runServer(t,
+		&runServerConfig{
+			tunnelProtocol:       "UNFRONTED-MEEK-HTTPS-OSSH",
+			enableSSHAPIRequests: true,
+		})
 }
 
-func runServer(t *testing.T, tunnelProtocol string) {
+func TestWebTransportAPIRequests(t *testing.T) {
+	runServer(t,
+		&runServerConfig{
+			tunnelProtocol:       "OSSH",
+			enableSSHAPIRequests: false,
+		})
+}
+
+type runServerConfig struct {
+	tunnelProtocol       string
+	enableSSHAPIRequests bool
+}
+
+func runServer(t *testing.T, runConfig *runServerConfig) {
 
 	// create a server
 
 	serverConfigFileContents, serverEntryFileContents, err := GenerateConfig(
-		"127.0.0.1",
-		8000,
-		map[string]int{tunnelProtocol: 4000})
+		&GenerateConfigParams{
+			ServerIPAddress:      "127.0.0.1",
+			EnableSSHAPIRequests: runConfig.enableSSHAPIRequests,
+			WebServerPort:        8000,
+			TunnelProtocolPorts:  map[string]int{runConfig.tunnelProtocol: 4000},
+		})
 	if err != nil {
 		t.Fatalf("error generating server config: %s", err)
 	}
@@ -134,7 +166,7 @@ func runServer(t *testing.T, tunnelProtocol string) {
 	clientConfig.DisableRemoteServerListFetcher = true
 	clientConfig.EstablishTunnelPausePeriodSeconds = &establishTunnelPausePeriodSeconds
 	clientConfig.TargetServerEntry = string(serverEntryFileContents)
-	clientConfig.TunnelProtocol = tunnelProtocol
+	clientConfig.TunnelProtocol = runConfig.tunnelProtocol
 	clientConfig.LocalHttpProxyPort = localHTTPProxyPort
 
 	err = psiphon.InitDataStore(clientConfig)
@@ -152,7 +184,7 @@ func runServer(t *testing.T, tunnelProtocol string) {
 	psiphon.SetNoticeOutput(psiphon.NewNoticeReceiver(
 		func(notice []byte) {
 
-			fmt.Printf("%s\n", string(notice))
+			//fmt.Printf("%s\n", string(notice))
 
 			noticeType, payload, err := psiphon.GetNotice(notice)
 			if err != nil {

psiphon/server/services.go

@@ -58,12 +58,6 @@ func RunServices(encodedConfigs [][]byte) error {
 		return psiphon.ContextError(err)
 	}
 
-	psinetDatabase, err := NewPsinetDatabase(config.PsinetDatabaseFilename)
-	if err != nil {
-		log.WithContextFields(LogFields{"error": err}).Error("init PsinetDatabase failed")
-		return psiphon.ContextError(err)
-	}
-
 	if config.UseRedis() {
 		err = InitRedis(config)
 		if err != nil {
@@ -72,6 +66,12 @@ func RunServices(encodedConfigs [][]byte) error {
 		}
 	}
 
+	psinetDatabase, err := NewPsinetDatabase(config.PsinetDatabaseFilename)
+	if err != nil {
+		log.WithContextFields(LogFields{"error": err}).Error("init PsinetDatabase failed")
+		return psiphon.ContextError(err)
+	}
+
 	waitGroup := new(sync.WaitGroup)
 	shutdownBroadcast := make(chan struct{})
 	errors := make(chan error)

psiphon/server/tunnelServer.go

@@ -543,6 +543,10 @@ func (sshClient *sshClient) passwordCallback(conn ssh.ConnMetadata, password []b
 	geoIPData := sshClient.geoIPData
 	sshClient.Unlock()
 
+	// Store the GeoIP data associated with the session ID. This makes the GeoIP data
+	// available to the web server for web transport Psiphon API requests.
+	SetGeoIPSessionCache(psiphonSessionID, geoIPData)
+
 	if sshClient.sshServer.config.UseRedis() {
 		err = UpdateRedisForLegacyPsiWeb(psiphonSessionID, geoIPData)
 		if err != nil {

psiphon/server/webServer.go

@@ -181,9 +181,13 @@ func convertHTTPRequestToAPIRequest(
 
 func (webServer *webServer) lookupGeoIPData(params requestJSONObject) GeoIPData {
 
-	// TODO: implement
+	clientSessionID, err := getStringRequestParam(params, "client_session_id")
+	if err != nil {
+		// Not all clients send this parameter
+		return NewGeoIPData()
+	}
 
-	return NewGeoIPData()
+	return GetGeoIPSessionCache(clientSessionID)
 }
 
 func (webServer *webServer) handshakeHandler(w http.ResponseWriter, r *http.Request) {