Merge pull request #398 from amirkhan5/master

Updated mobile library enabling streaming embedded server entries

.gitignore

@@ -54,3 +54,6 @@ build/
 
 # Visual Studio Code
 .vscode/
+
+# Intellij IDEs
+.idea/

MobileLibrary/iOS/PsiphonTunnel/PsiphonTunnel/PsiphonTunnel.h

@@ -119,16 +119,25 @@ typedef NS_ENUM(NSInteger, PsiphonConnectionState)
 
 /*!
  Called when the tunnel is starting to get the initial server entries (typically embedded in the app) that will be used to bootstrap the Psiphon tunnel connection. This value is in a particular format and will be supplied by Psiphon Inc.
+ If getEmbeddedServerEntriesPath is also implemented, it will take precedence over this method, unless getEmbeddedServerEntriesPath returns NULL or an empty string.
  @return  Pre-existing server entries to use when attempting to connect to a server. Must return an empty string if there are no embedded server entries. Must return NULL if there is an error and the tunnel starting should abort.
  */
 - (NSString * _Nullable)getEmbeddedServerEntries;
 
+
 //
 // Optional delegate methods. Note that some of these are probably necessary for
 // for a functioning app to implement, for example `onConnected`.
 //
 @optional
 
+/*!
+  Called when the tunnel is starting to get the initial server entries (typically embedded in the app) that will be used to bootstrap the Psiphon tunnel connection. This value is in a particular format and will be supplied by Psiphon Inc.
+  If this method is implemented, it takes precedence over getEmbeddedServerEntries, and getEmbeddedServerEntries will not be called unless this method returns NULL or an empty string.
+  @return Optional path where embedded server entries file is located. This file should be accessible by the Network Extension.
+ */
+- (NSString * _Nullable)getEmbeddedServerEntriesPath;
+
 /*!
  Gets runtime errors info that may be useful for debugging.
  @param message  The diagnostic message string.

MobileLibrary/iOS/PsiphonTunnel/PsiphonTunnel/PsiphonTunnel.m

@@ -115,14 +115,28 @@
             return FALSE;
         }
 
+        __block NSString *embeddedServerEntriesPath = nil;
         __block NSString *embeddedServerEntries = nil;
-        dispatch_sync(self->callbackQueue, ^{
-            embeddedServerEntries = [self.tunneledAppDelegate getEmbeddedServerEntries];
-        });
-
-        if (embeddedServerEntries == nil) {
-            [self logMessage:@"Error getting embedded server entries from delegate"];
-            return FALSE;
+        
+        // getEmbeddedServerEntriesPath is optional in the protocol
+        if ([self.tunneledAppDelegate respondsToSelector:@selector(getEmbeddedServerEntriesPath)]) {
+            dispatch_sync(self->callbackQueue, ^{
+                embeddedServerEntriesPath = [self.tunneledAppDelegate getEmbeddedServerEntriesPath];
+            });
+        }
+        
+        // If getEmbeddedServerEntriesPath returns nil or empty string,
+        // call getEmbeddedServerEntries
+        if (embeddedServerEntriesPath == nil || [embeddedServerEntriesPath length] == 0) {
+            
+            dispatch_sync(self->callbackQueue, ^{
+                embeddedServerEntries = [self.tunneledAppDelegate getEmbeddedServerEntries];
+            });
+            
+            if (embeddedServerEntries == nil) {
+                [self logMessage:@"Error getting embedded server entries from delegate"];
+                return FALSE;
+            }
         }
 
         [self changeConnectionStateTo:PsiphonConnectionStateConnecting evenIfSameState:NO];
@@ -133,6 +147,7 @@
             BOOL res = GoPsiStart(
                            configStr,
                            embeddedServerEntries,
+                           embeddedServerEntriesPath,
                            self,
                            self->tunnelWholeDevice, // useDeviceBinder
                            useIPv6Synthesizer,

MobileLibrary/psi/psi.go

@@ -32,6 +32,7 @@ import (
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/protocol"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/tun"
+	"os"
 )
 
 type PsiphonProvider interface {
@@ -49,7 +50,8 @@ var shutdownBroadcast chan struct{}
 var controllerWaitGroup *sync.WaitGroup
 
 func Start(
-	configJson, embeddedServerEntryList string,
+	configJson, embeddedServerEntryList,
+	embeddedServerEntryListPath string,
 	provider PsiphonProvider,
 	useDeviceBinder bool, useIPv6Synthesizer bool) error {
 
@@ -89,16 +91,10 @@ func Start(
 		return fmt.Errorf("error initializing datastore: %s", err)
 	}
 
-	serverEntries, err := protocol.DecodeServerEntryList(
-		embeddedServerEntryList,
-		common.GetCurrentTimestamp(),
-		protocol.SERVER_ENTRY_SOURCE_EMBEDDED)
+	// Stores list of server entries.
+	err = storeServerEntries(embeddedServerEntryListPath, embeddedServerEntryList)
 	if err != nil {
-		return fmt.Errorf("error decoding embedded server entry list: %s", err)
-	}
-	err = psiphon.StoreServerEntries(serverEntries, false)
-	if err != nil {
-		return fmt.Errorf("error storing embedded server entry list: %s", err)
+		return err
 	}
 
 	controller, err = psiphon.NewController(config)
@@ -166,3 +162,39 @@ func GetPacketTunnelDNSResolverIPv4Address() string {
 func GetPacketTunnelDNSResolverIPv6Address() string {
 	return tun.GetTransparentDNSResolverIPv6Address().String()
 }
+
+// Helper function to store a list of server entries.
+// if embeddedServerEntryListPath is not empty, embeddedServerEntryList will be ignored.
+func storeServerEntries(embeddedServerEntryListPath, embeddedServerEntryList string) error {
+
+	// if embeddedServerEntryListPath is not empty, ignore embeddedServerEntryList.
+	if embeddedServerEntryListPath != "" {
+
+		serverEntriesFile, err := os.Open(embeddedServerEntryListPath)
+		if err != nil {
+			return fmt.Errorf("failed to read remote server list: %s", common.ContextError(err))
+		}
+		defer serverEntriesFile.Close()
+
+		err = psiphon.StreamingStoreServerEntriesWithIOReader(serverEntriesFile, protocol.SERVER_ENTRY_SOURCE_EMBEDDED)
+		if err != nil {
+			return fmt.Errorf("failed to store common remote server list: %s", common.ContextError(err))
+		}
+
+	} else {
+
+		serverEntries, err := protocol.DecodeServerEntryList(
+			embeddedServerEntryList,
+			common.GetCurrentTimestamp(),
+			protocol.SERVER_ENTRY_SOURCE_EMBEDDED)
+		if err != nil {
+			return fmt.Errorf("error decoding embedded server entry list: %s", err)
+		}
+		err = psiphon.StoreServerEntries(serverEntries, false)
+		if err != nil {
+			return fmt.Errorf("error storing embedded server entry list: %s", err)
+		}
+	}
+
+	return nil
+}

psiphon/dataStore.go

@@ -24,6 +24,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -268,6 +269,25 @@ func StoreServerEntries(serverEntries []*protocol.ServerEntry, replaceIfExists b
 	return nil
 }
 
+// StreamingStoreServerEntriesWithIOReader is a wrapper around StreamingStoreServerEntries
+// an io.Reader is passed instead of an instance of StreamingServerEntryDecoder
+func StreamingStoreServerEntriesWithIOReader(serverListReader io.Reader, serverEntrySource string) error {
+
+	serverEntries := protocol.NewStreamingServerEntryDecoder(
+		serverListReader,
+		common.GetCurrentTimestamp(),
+		serverEntrySource)
+
+	// TODO: record stats for newly discovered servers
+
+	err := StreamingStoreServerEntries(serverEntries, true)
+	if err != nil {
+		return common.ContextError(err)
+	}
+
+	return nil
+}
+
 // StreamingStoreServerEntries stores a list of server entries.
 // There is an independent transaction for each entry insert/update.
 func StreamingStoreServerEntries(

psiphon/remoteServerList.go

@@ -23,7 +23,6 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"time"
 
@@ -78,7 +77,7 @@ func FetchCommonRemoteServerList(
 	}
 	defer serverListPayload.Close()
 
-	err = streamingStoreServerEntries(serverListPayload, protocol.SERVER_ENTRY_SOURCE_REMOTE)
+	err = StreamingStoreServerEntriesWithIOReader(serverListPayload, protocol.SERVER_ENTRY_SOURCE_REMOTE)
 	if err != nil {
 		return fmt.Errorf("failed to store common remote server list: %s", common.ContextError(err))
 	}
@@ -378,20 +377,3 @@ func storeServerEntries(serverList, serverEntrySource string) error {
 
 	return nil
 }
-
-func streamingStoreServerEntries(serverListReader io.Reader, serverEntrySource string) error {
-
-	serverEntries := protocol.NewStreamingServerEntryDecoder(
-		serverListReader,
-		common.GetCurrentTimestamp(),
-		serverEntrySource)
-
-	// TODO: record stats for newly discovered servers
-
-	err := StreamingStoreServerEntries(serverEntries, true)
-	if err != nil {
-		return common.ContextError(err)
-	}
-
-	return nil
-}