Merge branch 'master' into staging-client

ConsoleClient/privatePlugins.go

@@ -22,5 +22,6 @@
 package main
 
 import (
+	_ "github.com/Psiphon-Inc/psiphon-tunnel-core-private-plugins/client_plugins"
 	_ "github.com/Psiphon-Inc/psiphon-tunnel-core-private-plugins/common_plugins"
 )

MobileLibrary/Android/PsiphonTunnel/PsiphonTunnel.java

@@ -335,6 +335,9 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
         return DEFAULT_SECONDARY_DNS_SERVER;
     }
 
+    @Override
+    public String IPv6Synthesize(String IPv4Addr) { return IPv4Addr; }
+
     //----------------------------------------------------------------------------------------------
     // Psiphon Tunnel Core
     //----------------------------------------------------------------------------------------------
@@ -347,7 +350,8 @@ public class PsiphonTunnel extends Psi.PsiphonProvider.Stub {
                     loadPsiphonConfig(mHostService.getContext()),
                     embeddedServerEntries,
                     this,
-                    isVpnMode());
+                    isVpnMode(),
+                    false /* Do not use IPv6 synthesizer for android */);
         } catch (java.lang.Exception e) {
             throw new Exception("failed to start Psiphon library", e);
         }

MobileLibrary/iOS/PsiphonTunnel/PsiphonTunnel.xcodeproj/project.pbxproj

@@ -7,6 +7,8 @@
 	objects = {
 
 /* Begin PBXBuildFile section */
+		4E89F7FE1E2ED3CE00005F4C /* LookupIPv6.c in Sources */ = {isa = PBXBuildFile; fileRef = 4E89F7FC1E2ED3CE00005F4C /* LookupIPv6.c */; };
+		4E89F7FF1E2ED3CE00005F4C /* LookupIPv6.h in Headers */ = {isa = PBXBuildFile; fileRef = 4E89F7FD1E2ED3CE00005F4C /* LookupIPv6.h */; };
 		660E0B7A1E2D6EB6002BF5D4 /* Psi in Frameworks */ = {isa = PBXBuildFile; fileRef = 660E0B791E2D6EB6002BF5D4 /* Psi */; };
 		662659271DD270E900872F6C /* Reachability.h in Headers */ = {isa = PBXBuildFile; fileRef = 662659251DD270E900872F6C /* Reachability.h */; settings = {ATTRIBUTES = (Public, ); }; };
 		662659281DD270E900872F6C /* Reachability.m in Sources */ = {isa = PBXBuildFile; fileRef = 662659261DD270E900872F6C /* Reachability.m */; };
@@ -15,6 +17,7 @@
 		6685BDCD1E2E88A200F0E414 /* Psi-meta.h in Headers */ = {isa = PBXBuildFile; fileRef = 6685BDCC1E2E88A200F0E414 /* Psi-meta.h */; };
 		6685BDD41E2EBB1000F0E414 /* GoPsi.objc.h in Headers */ = {isa = PBXBuildFile; fileRef = 6685BDD21E2EBB1000F0E414 /* GoPsi.objc.h */; };
 		6685BDD51E2EBB1000F0E414 /* Universe.objc.h in Headers */ = {isa = PBXBuildFile; fileRef = 6685BDD31E2EBB1000F0E414 /* Universe.objc.h */; };
+		6685BDD91E300AC200F0E414 /* strip-frameworks.sh in Resources */ = {isa = PBXBuildFile; fileRef = 6685BDD81E300AC200F0E414 /* strip-frameworks.sh */; };
 		66BDB02A1DA6BFCC0079384C /* PsiphonTunnel.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 66BDB0201DA6BFCC0079384C /* PsiphonTunnel.framework */; };
 		66BDB02F1DA6BFCC0079384C /* PsiphonTunnelTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 66BDB02E1DA6BFCC0079384C /* PsiphonTunnelTests.m */; };
 		66BDB0311DA6BFCC0079384C /* PsiphonTunnel.h in Headers */ = {isa = PBXBuildFile; fileRef = 66BDB0231DA6BFCC0079384C /* PsiphonTunnel.h */; settings = {ATTRIBUTES = (Public, ); }; };
@@ -60,6 +63,8 @@
 /* End PBXCopyFilesBuildPhase section */
 
 /* Begin PBXFileReference section */
+		4E89F7FC1E2ED3CE00005F4C /* LookupIPv6.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = LookupIPv6.c; sourceTree = "<group>"; };
+		4E89F7FD1E2ED3CE00005F4C /* LookupIPv6.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = LookupIPv6.h; sourceTree = "<group>"; };
 		660E0B791E2D6EB6002BF5D4 /* Psi */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = Psi; path = PsiphonTunnel/Psi.framework/Versions/A/Psi; sourceTree = "<group>"; };
 		662659251DD270E900872F6C /* Reachability.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Reachability.h; sourceTree = "<group>"; };
 		662659261DD270E900872F6C /* Reachability.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = Reachability.m; sourceTree = "<group>"; };
@@ -68,6 +73,7 @@
 		6685BDCC1E2E88A200F0E414 /* Psi-meta.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = "Psi-meta.h"; path = "PsiphonTunnel/Psi-meta.h"; sourceTree = "<group>"; };
 		6685BDD21E2EBB1000F0E414 /* GoPsi.objc.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = GoPsi.objc.h; path = PsiphonTunnel/Psi.framework/Versions/A/Headers/GoPsi.objc.h; sourceTree = "<group>"; };
 		6685BDD31E2EBB1000F0E414 /* Universe.objc.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = Universe.objc.h; path = PsiphonTunnel/Psi.framework/Versions/A/Headers/Universe.objc.h; sourceTree = "<group>"; };
+		6685BDD81E300AC200F0E414 /* strip-frameworks.sh */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; name = "strip-frameworks.sh"; path = "scripts/strip-frameworks.sh"; sourceTree = "<group>"; };
 		66BDB0201DA6BFCC0079384C /* PsiphonTunnel.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = PsiphonTunnel.framework; sourceTree = BUILT_PRODUCTS_DIR; };
 		66BDB0231DA6BFCC0079384C /* PsiphonTunnel.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PsiphonTunnel.h; sourceTree = "<group>"; };
 		66BDB0241DA6BFCC0079384C /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
@@ -118,6 +124,8 @@
 			children = (
 				662659251DD270E900872F6C /* Reachability.h */,
 				662659261DD270E900872F6C /* Reachability.m */,
+				4E89F7FC1E2ED3CE00005F4C /* LookupIPv6.c */,
+				4E89F7FD1E2ED3CE00005F4C /* LookupIPv6.h */,
 			);
 			path = Reachability;
 			sourceTree = "<group>";
@@ -179,6 +187,7 @@
 		66BDB03C1DA6C7940079384C /* Resources */ = {
 			isa = PBXGroup;
 			children = (
+				6685BDD81E300AC200F0E414 /* strip-frameworks.sh */,
 				66BDB03D1DA6C79E0079384C /* rootCAs.txt */,
 			);
 			name = Resources;
@@ -214,6 +223,7 @@
 			buildActionMask = 2147483647;
 			files = (
 				6685BDCB1E2E882800F0E414 /* ref.h in Headers */,
+				4E89F7FF1E2ED3CE00005F4C /* LookupIPv6.h in Headers */,
 				662659271DD270E900872F6C /* Reachability.h in Headers */,
 				66BDB05D1DC26CCC0079384C /* SBJson4StreamParser.h in Headers */,
 				6685BDD41E2EBB1000F0E414 /* GoPsi.objc.h in Headers */,
@@ -316,6 +326,7 @@
 			buildActionMask = 2147483647;
 			files = (
 				66BDB03E1DA6C79E0079384C /* rootCAs.txt in Resources */,
+				6685BDD91E300AC200F0E414 /* strip-frameworks.sh in Resources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -337,6 +348,7 @@
 				66BDB0641DC26CCC0079384C /* SBJson4StreamWriter.m in Sources */,
 				66BDB0661DC26CCC0079384C /* SBJson4StreamWriterState.m in Sources */,
 				66BDB05C1DC26CCC0079384C /* SBJson4Parser.m in Sources */,
+				4E89F7FE1E2ED3CE00005F4C /* LookupIPv6.c in Sources */,
 				66BDB0681DC26CCC0079384C /* SBJson4Writer.m in Sources */,
 				66BDB0621DC26CCC0079384C /* SBJson4StreamTokeniser.m in Sources */,
 				66BDB0441DA6C7DD0079384C /* PsiphonTunnel.m in Sources */,

MobileLibrary/iOS/PsiphonTunnel/PsiphonTunnel/PsiphonTunnel.h

@@ -64,11 +64,8 @@ FOUNDATION_EXPORT const unsigned char PsiphonTunnelVersionString[];
  - `ClientPlatform`: Should not be set by most library consumers.
  - `UpstreamProxyUrl`
  - `EmitDiagnosticNotices`
- - `LocalHttpProxyPort` // TODO: Should this be set-able for iOS?
- - `LocalSocksProxyPort` // TODO: Should this be set-able for iOS?
  - `EgressRegion`
  - `EstablishTunnelTimeoutSeconds`
- - `TunnelWholeDevice`: For stats purposes, but must be accurate. Defaults to 0 (false).
  - Should only be set if the Psiphon library is handling upgrade downloading (which it usually is _not_):
    - `UpgradeDownloadUrl`
    - `UpgradeDownloadClientVersionHeader`
@@ -82,6 +79,11 @@ FOUNDATION_EXPORT const unsigned char PsiphonTunnelVersionString[];
    - `PsiphonApiServerTimeoutSeconds`
    - `FetchRoutesTimeoutSeconds`
    - `HttpProxyOriginServerTimeoutSeconds`
+ - Fields which should only be set by Psiphon proper:
+   - `LocalHttpProxyPort`
+   - `LocalSocksProxyPort`
+   - `TunnelWholeDevice`: For stats purposes, but must be accurate. Defaults to 0 (false).
+
  @endcode
 
  @note All other config fields must not be set.
@@ -203,34 +205,31 @@ FOUNDATION_EXPORT const unsigned char PsiphonTunnelVersionString[];
  */
 - (void)onBytesTransferred:(int64_t)sent :(int64_t)received;
 
-// TODO: Only applicable to Psiphon proper?
 /*!
  Called when tunnel-core discovers a home page associated with this client.
  If there are no home pages, it will not be called. May be called more than
  once, for multiple home pages.
+ Note: This is probably only applicable to Psiphon Inc.'s apps.
  @param url  The URL of the home page.
  Swift: @code func onHomepage(_ url: String) @endcode
  */
 - (void)onHomepage:(NSString * _Nonnull)url;
 
-// TODO: Only applicable to Psiphon proper?
 /*!
  Called if the current version of the client is the latest (i.e., there is no upgrade available).
+ Note: This is probably only applicable to Psiphon Inc.'s apps.
  Swift: @code func onClientIsLatestVersion() @endcode
  */
 - (void)onClientIsLatestVersion;
 
-// TODO: Only applicable to Psiphon proper?
 /*!
  Called when a client upgrade has been downloaded.
  @param filename  The name of the file containing the upgrade.
+ Note: This is probably only applicable to Psiphon Inc.'s apps.
  Swift: @code func onClientUpgradeDownloaded(_ filename: String) @endcode
  */
 - (void)onClientUpgradeDownloaded:(NSString * _Nonnull)filename;
 
-// TODO: Applies to iOS?
-//func onClientVerificationRequired(nonce: String, ttlSeconds: Int, resetCache: Bool)
-
 @end
 
 /*!
@@ -261,17 +260,13 @@ FOUNDATION_EXPORT const unsigned char PsiphonTunnelVersionString[];
  Upload a feedback package to Psiphon Inc. The app collects feedback and diagnostics information in a particular format, then calls this function to upload it for later investigation.
  @note The key, server, path, and headers must be provided by Psiphon Inc.
  @param feedbackJson  The feedback and diagnostics data to upload.
- @param connectionConfigJson  This function may create a tunnel to perform the upload, and this configuration is used to create that tunnel.
  @param b64EncodedPublicKey  The key that will be used to encrypt the payload before uploading.
- @param uploadServer  The server to which the data will be uploaded.
- @param uploadPath  The path on the server to which the data will be loaded.
+ @param uploadServer  The server and path to which the data will be uploaded.
  @param uploadServerHeaders  The request headers that will be used when uploading.
  */
-+ (void)sendFeedback:(NSString * _Nonnull)feedbackJson
-    connectionConfig:(NSString * _Nonnull)connectionConfigJson
+- (void)sendFeedback:(NSString * _Nonnull)feedbackJson
            publicKey:(NSString * _Nonnull)b64EncodedPublicKey
         uploadServer:(NSString * _Nonnull)uploadServer
-          uploadPath:(NSString * _Nonnull)uploadPath
  uploadServerHeaders:(NSString * _Nonnull)uploadServerHeaders;
 
 @end

MobileLibrary/iOS/PsiphonTunnel/PsiphonTunnel/PsiphonTunnel.m

@@ -19,6 +19,7 @@
 
 #import <CoreTelephony/CTTelephonyNetworkInfo.h>
 #import <CoreTelephony/CTCarrier.h>
+#import "LookupIPv6.h"
 #import "Psi-meta.h"
 #import "PsiphonTunnel.h"
 #import "json-framework/SBJson4.h"
@@ -61,6 +62,9 @@
 
         // Not supported on iOS.
         const BOOL useDeviceBinder = FALSE;
+
+        // Must always use IPv6Synthesizer for iOS
+        const BOOL useIPv6Synthesizer = TRUE;
         
         NSString *configStr = [self getConfig];
         if (configStr == nil) {
@@ -75,6 +79,7 @@
                            embeddedServerEntries,
                            self,
                            useDeviceBinder,
+                           useIPv6Synthesizer,
                            &e);
             
             [self logMessage:[NSString stringWithFormat: @"GoPsiStart: %@", res ? @"TRUE" : @"FALSE"]];
@@ -103,13 +108,15 @@
 }
 
 // See comment in header.
-+ (void)sendFeedback:(NSString * _Nonnull)feedbackJson
-    connectionConfig:(NSString * _Nonnull)connectionConfigJson
+- (void)sendFeedback:(NSString * _Nonnull)feedbackJson
            publicKey:(NSString * _Nonnull)b64EncodedPublicKey
         uploadServer:(NSString * _Nonnull)uploadServer
-          uploadPath:(NSString * _Nonnull)uploadPath
  uploadServerHeaders:(NSString * _Nonnull)uploadServerHeaders {
-    GoPsiSendFeedback(connectionConfigJson, feedbackJson, b64EncodedPublicKey, uploadServer, uploadPath, uploadServerHeaders);
+    NSString *connectionConfigJson = [self getConfig];
+    if (connectionConfigJson == nil) {
+       [self logMessage:@"Error getting config for feedback upload"];
+    }
+    GoPsiSendFeedback(connectionConfigJson, feedbackJson, b64EncodedPublicKey, uploadServer, @"", uploadServerHeaders);
 }
 
 
@@ -273,13 +280,10 @@
     // * UpgradeDownloadFilename
     // * timeout fields
     
-    // TODO: Is LocalSocksProxyPort relevant for iOS?
-    
     //
     // Fill in the rest of the values.
     //
     
-    // TODO: Should be configurable?
     config[@"EmitBytesTransferred"] = [NSNumber numberWithBool:TRUE];
 
     config[@"DeviceRegion"] = [PsiphonTunnel getDeviceRegion];
@@ -302,7 +306,6 @@
     // Some of them require default values.
     //
     
-    // TODO: After updating tunnel-core in the framework, verify that this value is getting through to Kibana.
     if (config[@"ClientPlatform"] == nil) {
         config[@"ClientPlatform"] = @"iOS-Library";
     }
@@ -554,6 +557,17 @@
     return (netstat != NotReachable) ? 1 : 0;
 }
 
+- (NSString *)iPv6Synthesize:(NSString *)IPv4Addr {
+    // This function is called to synthesize an ipv6 address from an ipv4 one on a DNS64/NAT64 network
+    char *result = getIPv6ForIPv4([IPv4Addr UTF8String]);
+    if (result != NULL) {
+        NSString *IPv6Addr = [NSString stringWithUTF8String:result];
+        free(result);
+        return IPv6Addr;
+    }
+    return @"";
+}
+
 - (void)notice:(NSString *)noticeJSON {
     [self handlePsiphonNotice:noticeJSON];
 }

MobileLibrary/iOS/PsiphonTunnel/PsiphonTunnel/Reachability/LookupIPv6.c

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2016, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "LookupIPv6.h"
+
+#include <arpa/inet.h>
+#include <err.h>
+#include <netdb.h>
+#include <stdlib.h>
+#include <string.h>
+
+char *getIPv6ForIPv4(const char *ipv4_str) {
+    char *ipv6_str = NULL;
+    struct addrinfo hints, *res, *res0;
+    int error;
+    
+    memset(&hints, 0, sizeof(hints));
+    hints.ai_family = AF_INET6;
+    hints.ai_socktype = SOCK_STREAM;
+    hints.ai_flags = AI_DEFAULT;
+    error = getaddrinfo(ipv4_str, NULL, &hints, &res0);
+    if (error) {
+        /* NOTREACHED */
+        return NULL;
+    }
+
+    for (res = res0; res; res = res->ai_next) {
+        if (res->ai_family == AF_INET6) {
+            struct sockaddr_in6 *sockaddr = (struct sockaddr_in6*)res->ai_addr;
+            ipv6_str = (char *)malloc(sizeof(char)*(INET6_ADDRSTRLEN)); // INET6_ADDRSTRLEN includes null terminating character
+            inet_ntop(AF_INET6, &(sockaddr->sin6_addr), ipv6_str, INET6_ADDRSTRLEN);
+            break;
+        }
+    }
+    
+    freeaddrinfo(res0);
+    return ipv6_str;
+}

MobileLibrary/iOS/PsiphonTunnel/PsiphonTunnel/Reachability/LookupIPv6.h

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2016, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef LookupIPv6_h
+#define LookupIPv6_h
+
+char *getIPv6ForIPv4(const char *ipv4_str);
+
+#endif /* LookupIPv6_h */

MobileLibrary/iOS/PsiphonTunnel/PsiphonTunnel/Reachability/README.md

@@ -0,0 +1,5 @@
+# Reachability
+
+This code is from: 
+
+https://developer.apple.com/library/content/samplecode/Reachability/Introduction/Intro.html

MobileLibrary/iOS/PsiphonTunnel/PsiphonTunnel/Reachability/Reachability.h

@@ -1,102 +1,64 @@
 /*
- Copyright (c) 2011, Tony Million.
- All rights reserved.
+ Copyright (C) 2016 Apple Inc. All Rights Reserved.
+ See LICENSE.txt for this sample’s licensing information
  
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- 1. Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
- 
- 2. Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- POSSIBILITY OF SUCH DAMAGE.
+ Abstract:
+ Basic demonstration of how to use the SystemConfiguration Reachablity APIs.
  */
 
 #import <Foundation/Foundation.h>
 #import <SystemConfiguration/SystemConfiguration.h>
+#import <netinet/in.h>
 
-//! Project version number for MacOSReachability.
-FOUNDATION_EXPORT double ReachabilityVersionNumber;
-
-//! Project version string for MacOSReachability.
-FOUNDATION_EXPORT const unsigned char ReachabilityVersionString[];
 
-/**
- * Create NS_ENUM macro if it does not exist on the targeted version of iOS or OS X.
- *
- * @see http://nshipster.com/ns_enum-ns_options/
- **/
-#ifndef NS_ENUM
-#define NS_ENUM(_type, _name) enum _name : _type _name; enum _name : _type
-#endif
+typedef enum : NSInteger {
+	NotReachable = 0,
+	ReachableViaWiFi,
+	ReachableViaWWAN
+} NetworkStatus;
 
-extern NSString *const kReachabilityChangedNotification;
+#pragma mark IPv6 Support
+//Reachability fully support IPv6.  For full details, see ReadMe.md.
 
-typedef NS_ENUM(NSInteger, NetworkStatus) {
-    // Apple NetworkStatus Compatible Names.
-    NotReachable = 0,
-    ReachableViaWiFi = 2,
-    ReachableViaWWAN = 1
-};
 
-@class Reachability;
-
-typedef void (^NetworkReachable)(Reachability * reachability);
-typedef void (^NetworkUnreachable)(Reachability * reachability);
-typedef void (^NetworkReachability)(Reachability * reachability, SCNetworkConnectionFlags flags);
+extern NSString *kReachabilityChangedNotification;
 
 
 @interface Reachability : NSObject
 
-@property (nonatomic, copy) NetworkReachable    reachableBlock;
-@property (nonatomic, copy) NetworkUnreachable  unreachableBlock;
-@property (nonatomic, copy) NetworkReachability reachabilityBlock;
-
-@property (nonatomic, assign) BOOL reachableOnWWAN;
+/*!
+ * Use to check the reachability of a given host name.
+ */
++ (instancetype)reachabilityWithHostName:(NSString *)hostName;
 
+/*!
+ * Use to check the reachability of a given IP address.
+ */
++ (instancetype)reachabilityWithAddress:(const struct sockaddr *)hostAddress;
 
-+(instancetype)reachabilityWithHostname:(NSString*)hostname;
-// This is identical to the function above, but is here to maintain
-//compatibility with Apples original code. (see .m)
-+(instancetype)reachabilityWithHostName:(NSString*)hostname;
-+(instancetype)reachabilityForInternetConnection;
-+(instancetype)reachabilityWithAddress:(void *)hostAddress;
-+(instancetype)reachabilityForLocalWiFi;
+/*!
+ * Checks whether the default route is available. Should be used by applications that do not connect to a particular host.
+ */
++ (instancetype)reachabilityForInternetConnection;
 
--(instancetype)initWithReachabilityRef:(SCNetworkReachabilityRef)ref;
 
--(BOOL)startNotifier;
--(void)stopNotifier;
+#pragma mark reachabilityForLocalWiFi
+//reachabilityForLocalWiFi has been removed from the sample.  See ReadMe.md for more information.
+//+ (instancetype)reachabilityForLocalWiFi;
 
--(BOOL)isReachable;
--(BOOL)isReachableViaWWAN;
--(BOOL)isReachableViaWiFi;
+/*!
+ * Start listening for reachability notifications on the current run loop.
+ */
+- (BOOL)startNotifier;
+- (void)stopNotifier;
 
-// WWAN may be available, but not active until a connection has been established.
-// WiFi may require a connection for VPN on Demand.
--(BOOL)isConnectionRequired; // Identical DDG variant.
--(BOOL)connectionRequired; // Apple's routine.
-// Dynamic, on demand connection?
--(BOOL)isConnectionOnDemand;
-// Is user intervention required?
--(BOOL)isInterventionRequired;
+- (NetworkStatus)currentReachabilityStatus;
 
--(NetworkStatus)currentReachabilityStatus;
--(SCNetworkReachabilityFlags)reachabilityFlags;
--(NSString*)currentReachabilityString;
--(NSString*)currentReachabilityFlags;
+/*!
+ * WWAN may be available, but not active until a connection has been established. WiFi may require a connection for VPN on Demand.
+ */
+- (BOOL)connectionRequired;
 
 @end
+
+

MobileLibrary/iOS/PsiphonTunnel/PsiphonTunnel/Reachability/Reachability.m

@@ -1,481 +1,242 @@
 /*
- Copyright (c) 2011, Tony Million.
- All rights reserved.
+ Copyright (C) 2016 Apple Inc. All Rights Reserved.
+ See LICENSE.txt for this sample’s licensing information
  
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- 1. Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
- 
- 2. Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
- LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- POSSIBILITY OF SUCH DAMAGE.
+ Abstract:
+ Basic demonstration of how to use the SystemConfiguration Reachablity APIs.
  */
 
-#import "Reachability.h"
-
-#import <sys/socket.h>
-#import <netinet/in.h>
-#import <netinet6/in6.h>
 #import <arpa/inet.h>
 #import <ifaddrs.h>
 #import <netdb.h>
+#import <sys/socket.h>
+#import <netinet/in.h>
 
+#import <CoreFoundation/CoreFoundation.h>
 
-NSString *const kReachabilityChangedNotification = @"kReachabilityChangedNotification";
+#import "Reachability.h"
 
+#pragma mark IPv6 Support
+//Reachability fully support IPv6.  For full details, see ReadMe.md.
 
-@interface Reachability ()
 
-@property (nonatomic, assign) SCNetworkReachabilityRef  reachabilityRef;
-@property (nonatomic, strong) dispatch_queue_t          reachabilitySerialQueue;
-@property (nonatomic, strong) id                        reachabilityObject;
+NSString *kReachabilityChangedNotification = @"kNetworkReachabilityChangedNotification";
 
--(void)reachabilityChanged:(SCNetworkReachabilityFlags)flags;
--(BOOL)isReachableWithFlags:(SCNetworkReachabilityFlags)flags;
 
-@end
+#pragma mark - Supporting functions
 
+#define kShouldPrintReachabilityFlags 0
 
-static NSString *reachabilityFlags(SCNetworkReachabilityFlags flags)
+static void PrintReachabilityFlags(SCNetworkReachabilityFlags flags, const char* comment)
 {
-    return [NSString stringWithFormat:@"%c%c %c%c%c%c%c%c%c",
-#if	TARGET_OS_IPHONE
-            (flags & kSCNetworkReachabilityFlagsIsWWAN)               ? 'W' : '-',
-#else
-            'X',
-#endif
-            (flags & kSCNetworkReachabilityFlagsReachable)            ? 'R' : '-',
-            (flags & kSCNetworkReachabilityFlagsConnectionRequired)   ? 'c' : '-',
-            (flags & kSCNetworkReachabilityFlagsTransientConnection)  ? 't' : '-',
-            (flags & kSCNetworkReachabilityFlagsInterventionRequired) ? 'i' : '-',
-            (flags & kSCNetworkReachabilityFlagsConnectionOnTraffic)  ? 'C' : '-',
-            (flags & kSCNetworkReachabilityFlagsConnectionOnDemand)   ? 'D' : '-',
-            (flags & kSCNetworkReachabilityFlagsIsLocalAddress)       ? 'l' : '-',
-            (flags & kSCNetworkReachabilityFlagsIsDirect)             ? 'd' : '-'];
-}
+#if kShouldPrintReachabilityFlags
 
-// Start listening for reachability notifications on the current run loop
-static void TMReachabilityCallback(SCNetworkReachabilityRef target, SCNetworkReachabilityFlags flags, void* info)
-{
-#pragma unused (target)
-    
-    Reachability *reachability = ((__bridge Reachability*)info);
-    
-    // We probably don't need an autoreleasepool here, as GCD docs state each queue has its own autorelease pool,
-    // but what the heck eh?
-    @autoreleasepool
-    {
-        [reachability reachabilityChanged:flags];
-    }
-}
+    NSLog(@"Reachability Flag Status: %c%c %c%c%c%c%c%c%c %s\n",
+          (flags & kSCNetworkReachabilityFlagsIsWWAN)				? 'W' : '-',
+          (flags & kSCNetworkReachabilityFlagsReachable)            ? 'R' : '-',
 
-
-@implementation Reachability
-
-#pragma mark - Class Constructor Methods
-
-+(instancetype)reachabilityWithHostName:(NSString*)hostname
-{
-    return [Reachability reachabilityWithHostname:hostname];
+          (flags & kSCNetworkReachabilityFlagsTransientConnection)  ? 't' : '-',
+          (flags & kSCNetworkReachabilityFlagsConnectionRequired)   ? 'c' : '-',
+          (flags & kSCNetworkReachabilityFlagsConnectionOnTraffic)  ? 'C' : '-',
+          (flags & kSCNetworkReachabilityFlagsInterventionRequired) ? 'i' : '-',
+          (flags & kSCNetworkReachabilityFlagsConnectionOnDemand)   ? 'D' : '-',
+          (flags & kSCNetworkReachabilityFlagsIsLocalAddress)       ? 'l' : '-',
+          (flags & kSCNetworkReachabilityFlagsIsDirect)             ? 'd' : '-',
+          comment
+          );
+#endif
 }
 
-+(instancetype)reachabilityWithHostname:(NSString*)hostname
-{
-    SCNetworkReachabilityRef ref = SCNetworkReachabilityCreateWithName(NULL, [hostname UTF8String]);
-    if (ref)
-    {
-        id reachability = [[self alloc] initWithReachabilityRef:ref];
-        
-        CFRelease(ref);
-        
-        return reachability;
-    }
-    
-    return nil;
-}
 
-+(instancetype)reachabilityWithAddress:(void *)hostAddress
+static void ReachabilityCallback(SCNetworkReachabilityRef target, SCNetworkReachabilityFlags flags, void* info)
 {
-    SCNetworkReachabilityRef ref = SCNetworkReachabilityCreateWithAddress(kCFAllocatorDefault, (const struct sockaddr*)hostAddress);
-    if (ref)
-    {
-        id reachability = [[self alloc] initWithReachabilityRef:ref];
-        
-        CFRelease(ref);
-        
-        return reachability;
-    }
-    
-    return nil;
-}
+#pragma unused (target, flags)
+	NSCAssert(info != NULL, @"info was NULL in ReachabilityCallback");
+	NSCAssert([(__bridge NSObject*) info isKindOfClass: [Reachability class]], @"info was wrong class in ReachabilityCallback");
 
-+(instancetype)reachabilityForInternetConnection
-{
-    struct sockaddr_in zeroAddress;
-    bzero(&zeroAddress, sizeof(zeroAddress));
-    zeroAddress.sin_len = sizeof(zeroAddress);
-    zeroAddress.sin_family = AF_INET;
-    
-    return [self reachabilityWithAddress:&zeroAddress];
-}
-
-+(instancetype)reachabilityForLocalWiFi
-{
-    struct sockaddr_in localWifiAddress;
-    bzero(&localWifiAddress, sizeof(localWifiAddress));
-    localWifiAddress.sin_len            = sizeof(localWifiAddress);
-    localWifiAddress.sin_family         = AF_INET;
-    // IN_LINKLOCALNETNUM is defined in <netinet/in.h> as 169.254.0.0
-    localWifiAddress.sin_addr.s_addr    = htonl(IN_LINKLOCALNETNUM);
-    
-    return [self reachabilityWithAddress:&localWifiAddress];
+    Reachability* noteObject = (__bridge Reachability *)info;
+    // Post a notification to notify the client that the network reachability changed.
+    [[NSNotificationCenter defaultCenter] postNotificationName: kReachabilityChangedNotification object: noteObject];
 }
 
 
-// Initialization methods
+#pragma mark - Reachability implementation
 
--(instancetype)initWithReachabilityRef:(SCNetworkReachabilityRef)ref
+@implementation Reachability
 {
-    self = [super init];
-    if (self != nil)
-    {
-        self.reachableOnWWAN = YES;
-        self.reachabilityRef = ref;
-        
-        CFRetain(self.reachabilityRef);
-        
-        // We need to create a serial queue.
-        // We allocate this once for the lifetime of the notifier.
-        
-        self.reachabilitySerialQueue = dispatch_queue_create("com.tonymillion.reachability", NULL);
-    }
-    
-    return self;
+	SCNetworkReachabilityRef _reachabilityRef;
 }
 
--(void)dealloc
++ (instancetype)reachabilityWithHostName:(NSString *)hostName
 {
-    [self stopNotifier];
-    
-    if(self.reachabilityRef)
-    {
-        CFRelease(self.reachabilityRef);
-        self.reachabilityRef = nil;
-    }
-    
-    self.reachableBlock          = nil;
-    self.unreachableBlock        = nil;
-    self.reachabilityBlock       = nil;
-    self.reachabilitySerialQueue = nil;
+	Reachability* returnValue = NULL;
+	SCNetworkReachabilityRef reachability = SCNetworkReachabilityCreateWithName(NULL, [hostName UTF8String]);
+	if (reachability != NULL)
+	{
+		returnValue= [[self alloc] init];
+		if (returnValue != NULL)
+		{
+			returnValue->_reachabilityRef = reachability;
+		}
+        else {
+            CFRelease(reachability);
+        }
+	}
+	return returnValue;
 }
 
-#pragma mark - Notifier Methods
 
-// Notifier
-// NOTE: This uses GCD to trigger the blocks - they *WILL NOT* be called on THE MAIN THREAD
-// - In other words DO NOT DO ANY UI UPDATES IN THE BLOCKS.
-//   INSTEAD USE dispatch_async(dispatch_get_main_queue(), ^{UISTUFF}) (or dispatch_sync if you want)
-
--(BOOL)startNotifier
++ (instancetype)reachabilityWithAddress:(const struct sockaddr *)hostAddress
 {
-    // allow start notifier to be called multiple times
-    if(self.reachabilityObject && (self.reachabilityObject == self))
-    {
-        return YES;
-    }
-    
-    
-    SCNetworkReachabilityContext    context = { 0, NULL, NULL, NULL, NULL };
-    context.info = (__bridge void *)self;
-    
-    if(SCNetworkReachabilitySetCallback(self.reachabilityRef, TMReachabilityCallback, &context))
-    {
-        // Set it as our reachability queue, which will retain the queue
-        if(SCNetworkReachabilitySetDispatchQueue(self.reachabilityRef, self.reachabilitySerialQueue))
-        {
-            // this should do a retain on ourself, so as long as we're in notifier mode we shouldn't disappear out from under ourselves
-            // woah
-            self.reachabilityObject = self;
-            return YES;
-        }
-        else
-        {
-#ifdef DEBUG
-            NSLog(@"SCNetworkReachabilitySetDispatchQueue() failed: %s", SCErrorString(SCError()));
-#endif
-            
-            // UH OH - FAILURE - stop any callbacks!
-            SCNetworkReachabilitySetCallback(self.reachabilityRef, NULL, NULL);
+	SCNetworkReachabilityRef reachability = SCNetworkReachabilityCreateWithAddress(kCFAllocatorDefault, hostAddress);
+
+	Reachability* returnValue = NULL;
+
+	if (reachability != NULL)
+	{
+		returnValue = [[self alloc] init];
+		if (returnValue != NULL)
+		{
+			returnValue->_reachabilityRef = reachability;
+		}
+        else {
+            CFRelease(reachability);
         }
-    }
-    else
-    {
-#ifdef DEBUG
-        NSLog(@"SCNetworkReachabilitySetCallback() failed: %s", SCErrorString(SCError()));
-#endif
-    }
-    
-    // if we get here we fail at the internet
-    self.reachabilityObject = nil;
-    return NO;
+	}
+	return returnValue;
 }
 
--(void)stopNotifier
+
++ (instancetype)reachabilityForInternetConnection
 {
-    // First stop, any callbacks!
-    SCNetworkReachabilitySetCallback(self.reachabilityRef, NULL, NULL);
+	struct sockaddr_in zeroAddress;
+	bzero(&zeroAddress, sizeof(zeroAddress));
+	zeroAddress.sin_len = sizeof(zeroAddress);
+	zeroAddress.sin_family = AF_INET;
     
-    // Unregister target from the GCD serial dispatch queue.
-    SCNetworkReachabilitySetDispatchQueue(self.reachabilityRef, NULL);
-    
-    self.reachabilityObject = nil;
+    return [self reachabilityWithAddress: (const struct sockaddr *) &zeroAddress];
 }
 
-#pragma mark - reachability tests
+#pragma mark reachabilityForLocalWiFi
+//reachabilityForLocalWiFi has been removed from the sample.  See ReadMe.md for more information.
+//+ (instancetype)reachabilityForLocalWiFi
 
-// This is for the case where you flick the airplane mode;
-// you end up getting something like this:
-//Reachability: WR ct-----
-//Reachability: -- -------
-//Reachability: WR ct-----
-//Reachability: -- -------
-// We treat this as 4 UNREACHABLE triggers - really apple should do better than this
 
-#define testcase (kSCNetworkReachabilityFlagsConnectionRequired | kSCNetworkReachabilityFlagsTransientConnection)
 
--(BOOL)isReachableWithFlags:(SCNetworkReachabilityFlags)flags
-{
-    BOOL connectionUP = YES;
-    
-    if(!(flags & kSCNetworkReachabilityFlagsReachable))
-        connectionUP = NO;
-    
-    if( (flags & testcase) == testcase )
-        connectionUP = NO;
-    
-#if	TARGET_OS_IPHONE
-    if(flags & kSCNetworkReachabilityFlagsIsWWAN)
-    {
-        // We're on 3G.
-        if(!self.reachableOnWWAN)
-        {
-            // We don't want to connect when on 3G.
-            connectionUP = NO;
-        }
-    }
-#endif
-    
-    return connectionUP;
-}
+#pragma mark - Start and stop notifier
 
--(BOOL)isReachable
+- (BOOL)startNotifier
 {
-    SCNetworkReachabilityFlags flags;
-    
-    if(!SCNetworkReachabilityGetFlags(self.reachabilityRef, &flags))
-        return NO;
-    
-    return [self isReachableWithFlags:flags];
-}
+	BOOL returnValue = NO;
+	SCNetworkReachabilityContext context = {0, (__bridge void *)(self), NULL, NULL, NULL};
 
--(BOOL)isReachableViaWWAN
-{
-#if	TARGET_OS_IPHONE
+	if (SCNetworkReachabilitySetCallback(_reachabilityRef, ReachabilityCallback, &context))
+	{
+		if (SCNetworkReachabilityScheduleWithRunLoop(_reachabilityRef, CFRunLoopGetCurrent(), kCFRunLoopDefaultMode))
+		{
+			returnValue = YES;
+		}
+	}
     
-    SCNetworkReachabilityFlags flags = 0;
-    
-    if(SCNetworkReachabilityGetFlags(self.reachabilityRef, &flags))
-    {
-        // Check we're REACHABLE
-        if(flags & kSCNetworkReachabilityFlagsReachable)
-        {
-            // Now, check we're on WWAN
-            if(flags & kSCNetworkReachabilityFlagsIsWWAN)
-            {
-                return YES;
-            }
-        }
-    }
-#endif
-    
-    return NO;
+	return returnValue;
 }
 
--(BOOL)isReachableViaWiFi
+
+- (void)stopNotifier
 {
-    SCNetworkReachabilityFlags flags = 0;
-    
-    if(SCNetworkReachabilityGetFlags(self.reachabilityRef, &flags))
-    {
-        // Check we're reachable
-        if((flags & kSCNetworkReachabilityFlagsReachable))
-        {
-#if	TARGET_OS_IPHONE
-            // Check we're NOT on WWAN
-            if((flags & kSCNetworkReachabilityFlagsIsWWAN))
-            {
-                return NO;
-            }
-#endif
-            return YES;
-        }
-    }
-    
-    return NO;
+	if (_reachabilityRef != NULL)
+	{
+		SCNetworkReachabilityUnscheduleFromRunLoop(_reachabilityRef, CFRunLoopGetCurrent(), kCFRunLoopDefaultMode);
+	}
 }
 
 
-// WWAN may be available, but not active until a connection has been established.
-// WiFi may require a connection for VPN on Demand.
--(BOOL)isConnectionRequired
+- (void)dealloc
 {
-    return [self connectionRequired];
+	[self stopNotifier];
+	if (_reachabilityRef != NULL)
+	{
+		CFRelease(_reachabilityRef);
+	}
 }
 
--(BOOL)connectionRequired
-{
-    SCNetworkReachabilityFlags flags;
-    
-    if(SCNetworkReachabilityGetFlags(self.reachabilityRef, &flags))
-    {
-        return (flags & kSCNetworkReachabilityFlagsConnectionRequired);
-    }
-    
-    return NO;
-}
 
-// Dynamic, on demand connection?
--(BOOL)isConnectionOnDemand
-{
-    SCNetworkReachabilityFlags flags;
-    
-    if (SCNetworkReachabilityGetFlags(self.reachabilityRef, &flags))
-    {
-        return ((flags & kSCNetworkReachabilityFlagsConnectionRequired) &&
-                (flags & (kSCNetworkReachabilityFlagsConnectionOnTraffic | kSCNetworkReachabilityFlagsConnectionOnDemand)));
-    }
-    
-    return NO;
-}
+#pragma mark - Network Flag Handling
 
-// Is user intervention required?
--(BOOL)isInterventionRequired
+- (NetworkStatus)networkStatusForFlags:(SCNetworkReachabilityFlags)flags
 {
-    SCNetworkReachabilityFlags flags;
-    
-    if (SCNetworkReachabilityGetFlags(self.reachabilityRef, &flags))
-    {
-        return ((flags & kSCNetworkReachabilityFlagsConnectionRequired) &&
-                (flags & kSCNetworkReachabilityFlagsInterventionRequired));
-    }
-    
-    return NO;
-}
+	PrintReachabilityFlags(flags, "networkStatusForFlags");
+	if ((flags & kSCNetworkReachabilityFlagsReachable) == 0)
+	{
+		// The target host is not reachable.
+		return NotReachable;
+	}
 
+    NetworkStatus returnValue = NotReachable;
 
-#pragma mark - reachability status stuff
+	if ((flags & kSCNetworkReachabilityFlagsConnectionRequired) == 0)
+	{
+		/*
+         If the target host is reachable and no connection is required then we'll assume (for now) that you're on Wi-Fi...
+         */
+		returnValue = ReachableViaWiFi;
+	}
 
--(NetworkStatus)currentReachabilityStatus
-{
-    if([self isReachable])
-    {
-        if([self isReachableViaWiFi])
-            return ReachableViaWiFi;
-        
-#if	TARGET_OS_IPHONE
-        return ReachableViaWWAN;
-#endif
-    }
-    
-    return NotReachable;
-}
+	if ((((flags & kSCNetworkReachabilityFlagsConnectionOnDemand ) != 0) ||
+        (flags & kSCNetworkReachabilityFlagsConnectionOnTraffic) != 0))
+	{
+        /*
+         ... and the connection is on-demand (or on-traffic) if the calling application is using the CFSocketStream or higher APIs...
+         */
 
--(SCNetworkReachabilityFlags)reachabilityFlags
-{
-    SCNetworkReachabilityFlags flags = 0;
-    
-    if(SCNetworkReachabilityGetFlags(self.reachabilityRef, &flags))
-    {
-        return flags;
+        if ((flags & kSCNetworkReachabilityFlagsInterventionRequired) == 0)
+        {
+            /*
+             ... and no [user] intervention is needed...
+             */
+            returnValue = ReachableViaWiFi;
+        }
     }
-    
-    return 0;
-}
 
--(NSString*)currentReachabilityString
-{
-    NetworkStatus temp = [self currentReachabilityStatus];
+	if ((flags & kSCNetworkReachabilityFlagsIsWWAN) == kSCNetworkReachabilityFlagsIsWWAN)
+	{
+		/*
+         ... but WWAN connections are OK if the calling application is using the CFNetwork APIs.
+         */
+		returnValue = ReachableViaWWAN;
+	}
     
-    if(temp == ReachableViaWWAN)
-    {
-        // Updated for the fact that we have CDMA phones now!
-        return NSLocalizedString(@"Cellular", @"");
-    }
-    if (temp == ReachableViaWiFi)
-    {
-        return NSLocalizedString(@"WiFi", @"");
-    }
-    
-    return NSLocalizedString(@"No Connection", @"");
+	return returnValue;
 }
 
--(NSString*)currentReachabilityFlags
+
+- (BOOL)connectionRequired
 {
-    return reachabilityFlags([self reachabilityFlags]);
+	NSAssert(_reachabilityRef != NULL, @"connectionRequired called with NULL reachabilityRef");
+	SCNetworkReachabilityFlags flags;
+
+	if (SCNetworkReachabilityGetFlags(_reachabilityRef, &flags))
+	{
+		return (flags & kSCNetworkReachabilityFlagsConnectionRequired);
+	}
+
+    return NO;
 }
 
-#pragma mark - Callback function calls this method
 
--(void)reachabilityChanged:(SCNetworkReachabilityFlags)flags
+- (NetworkStatus)currentReachabilityStatus
 {
-    if([self isReachableWithFlags:flags])
-    {
-        if(self.reachableBlock)
-        {
-            self.reachableBlock(self);
-        }
-    }
-    else
-    {
-        if(self.unreachableBlock)
-        {
-            self.unreachableBlock(self);
-        }
-    }
+	NSAssert(_reachabilityRef != NULL, @"currentNetworkStatus called with NULL SCNetworkReachabilityRef");
+	NetworkStatus returnValue = NotReachable;
+	SCNetworkReachabilityFlags flags;
     
-    if(self.reachabilityBlock)
-    {
-        self.reachabilityBlock(self, flags);
-    }
+	if (SCNetworkReachabilityGetFlags(_reachabilityRef, &flags))
+	{
+        returnValue = [self networkStatusForFlags:flags];
+	}
     
-    // this makes sure the change notification happens on the MAIN THREAD
-    dispatch_async(dispatch_get_main_queue(), ^{
-        [[NSNotificationCenter defaultCenter] postNotificationName:kReachabilityChangedNotification
-                                                            object:self];
-    });
+	return returnValue;
 }
 
-#pragma mark - Debug Description
-
-- (NSString *) description
-{
-    NSString *description = [NSString stringWithFormat:@"<%@: %#x (%@)>",
-                             NSStringFromClass([self class]), (unsigned int) self, [self currentReachabilityFlags]];
-    return description;
-}
 
 @end

MobileLibrary/iOS/PsiphonTunnel/scripts/strip-frameworks.sh

@@ -0,0 +1,72 @@
+################################################################################
+#
+# Copyright 2015 Realm Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+# This script strips all non-valid architectures from dynamic libraries in
+# the application's `Frameworks` directory.
+#
+# The following environment variables are required:
+#
+# BUILT_PRODUCTS_DIR
+# FRAMEWORKS_FOLDER_PATH
+# VALID_ARCHS
+# EXPANDED_CODE_SIGN_IDENTITY
+
+
+# Signs a framework with the provided identity
+code_sign() {
+  # Use the current code_sign_identitiy
+  echo "Code Signing $1 with Identity ${EXPANDED_CODE_SIGN_IDENTITY_NAME}"
+  echo "/usr/bin/codesign --force --sign ${EXPANDED_CODE_SIGN_IDENTITY} --preserve-metadata=identifier,entitlements $1"
+  /usr/bin/codesign --force --sign ${EXPANDED_CODE_SIGN_IDENTITY} --preserve-metadata=identifier,entitlements "$1"
+}
+
+# Set working directory to product’s embedded frameworks 
+cd "${BUILT_PRODUCTS_DIR}/${FRAMEWORKS_FOLDER_PATH}"
+
+if [ "$ACTION" = "install" ]; then
+  echo "Copy .bcsymbolmap files to .xcarchive"
+  find . -name '*.bcsymbolmap' -type f -exec mv {} "${CONFIGURATION_BUILD_DIR}" \;
+else
+  # Delete *.bcsymbolmap files from framework bundle unless archiving
+  find . -name '*.bcsymbolmap' -type f -exec rm -rf "{}" +\;
+fi
+
+echo "Stripping frameworks"
+
+for file in $(find . -type f -perm +111); do
+  # Skip non-dynamic libraries
+  if ! [[ "$(file "$file")" == *"dynamically linked shared library"* ]]; then
+    continue
+  fi
+  # Get architectures for current file
+  archs="$(lipo -info "${file}" | rev | cut -d ':' -f1 | rev)"
+  stripped=""
+  for arch in $archs; do
+    if ! [[ "${VALID_ARCHS}" == *"$arch"* ]]; then
+      # Strip non-valid architectures in-place
+      lipo -remove "$arch" -output "$file" "$file" || exit 1
+      stripped="$stripped $arch"
+    fi
+  done
+  if [[ "$stripped" != "" ]]; then
+    echo "Stripped $file of architectures:$stripped"
+    if [ "${CODE_SIGNING_REQUIRED}" == "YES" ]; then
+      code_sign "${file}"
+    fi
+  fi
+done

MobileLibrary/iOS/SampleApps/TunneledWebRequest/TunneledWebRequest.xcodeproj/project.pbxproj

@@ -157,6 +157,7 @@
 				662658E71DCB8CF300872F6C /* Frameworks */,
 				662658E81DCB8CF300872F6C /* Resources */,
 				662659221DCBC8CB00872F6C /* CopyFiles */,
+				6685BDD71E300A7800F0E414 /* ShellScript */,
 			);
 			buildRules = (
 			);
@@ -280,6 +281,22 @@
 		};
 /* End PBXResourcesBuildPhase section */
 
+/* Begin PBXShellScriptBuildPhase section */
+		6685BDD71E300A7800F0E414 /* ShellScript */ = {
+			isa = PBXShellScriptBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+			);
+			inputPaths = (
+			);
+			outputPaths = (
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+			shellPath = /bin/sh;
+			shellScript = "bash \"${BUILT_PRODUCTS_DIR}/${FRAMEWORKS_FOLDER_PATH}/PsiphonTunnel.framework/strip-frameworks.sh\"";
+		};
+/* End PBXShellScriptBuildPhase section */
+
 /* Begin PBXSourcesBuildPhase section */
 		662658E61DCB8CF300872F6C /* Sources */ = {
 			isa = PBXSourcesBuildPhase;

MobileLibrary/iOS/USAGE.md

@@ -32,7 +32,15 @@ This code is a canonical guide for integrating the Library.
 
 5. In the "Build Settings" for the target, click the `+` at the top, then "Add User-Defined Setting". Name the new setting `STRIP_BITCODE_FROM_COPIED_FILES` and set it to `NO`.
 
-6. In target Build Phases, add a "Copy Files" phase. Set "Destination" to "Frameworks". Add `PsiphonTunnel.framework` to the list. Ensure "Code Sign on Copy" is checked.
+6. In the "Build Phases" for the target, add a "Copy Files" phase. Set "Destination" to "Frameworks". Add `PsiphonTunnel.framework` to the list. Ensure "Code Sign on Copy" is checked.
+
+7. In the "Build Phases" for the target, add a "Run Script" phase. Set the script contents to:
+
+   ```no-highlight
+   bash "${BUILT_PRODUCTS_DIR}/${FRAMEWORKS_FOLDER_PATH}/PsiphonTunnel.framework/strip-frameworks.sh"
+   ```
+
+   This step is required to work around an [App Store submission validation bug](http://www.openradar.me/23681704) that rejects apps containing a framework with simulator slices.
 
 ## Compiling and testing
 

MobileLibrary/iOS/build-psiphon-framework.sh

@@ -1,5 +1,15 @@
 #!/usr/bin/env bash
 
+# This script takes one optional argument: 'private', if private plugins should
+# be used. It should be omitted if private plugins are not desired.
+if [[ $1 == "private" ]]; then
+  FORCE_PRIVATE_PLUGINS=true
+  echo "TRUE"
+else
+  FORCE_PRIVATE_PLUGINS=false
+  echo "FALSE"
+fi
+
 # -x echos commands. -u exits if an unintialized variable is used.
 # -e exits if a command returns an error.
 set -x -u -e
@@ -31,6 +41,7 @@ FRAMEWORK_BINARY="${INTERMEDIATE_OUPUT_DIR}/${INTERMEDIATE_OUPUT_FILE}/Versions/
 # The "OPENSSL" tag enables support of OpenSSL for use by IndistinguishableTLS.
 
 PRIVATE_PLUGINS_TAG="PRIVATE_PLUGINS"
+if [[ ${FORCE_PRIVATE_PLUGINS} == true ]]; then PRIVATE_PLUGINS_TAG="PRIVATE_PLUGINS"; fi
 BUILD_TAGS="OPENSSL IOS ${PRIVATE_PLUGINS_TAG}"
 
 LIBSSL=${BASE_DIR}/OpenSSL-for-iPhone/lib/libssl.a
@@ -55,7 +66,7 @@ export PATH=${GOPATH}/bin:${PATH}
 rm -rf ${GOPATH}
 
 # When updating the pinned rev, you will have to manually delete go-ios-build
-GOMOBILE_PINNED_REV=c4d780faeb85123ee32b88e84fd022739ed8c124
+GOMOBILE_PINNED_REV=a0f998b2d8c7ee81ddbead9202dd5e0184a998ad
 GOMOBILE_PATH=${GOPATH}/src/golang.org/x/mobile/cmd/gomobile
 
 TUNNEL_CORE_SRC_DIR=${GOPATH}/src/github.com/Psiphon-Labs/psiphon-tunnel-core

MobileLibrary/psi/privatePlugins.go

@@ -22,5 +22,6 @@
 package psi
 
 import (
+	_ "github.com/Psiphon-Inc/psiphon-tunnel-core-private-plugins/client_plugins"
 	_ "github.com/Psiphon-Inc/psiphon-tunnel-core-private-plugins/common_plugins"
 )

MobileLibrary/psi/psi.go

@@ -37,6 +37,7 @@ type PsiphonProvider interface {
 	Notice(noticeJSON string)
 	HasNetworkConnectivity() int
 	BindToDevice(fileDescriptor int) error
+	IPv6Synthesize(IPv4Addr string) string
 	GetPrimaryDnsServer() string
 	GetSecondaryDnsServer() string
 }
@@ -49,7 +50,7 @@ var controllerWaitGroup *sync.WaitGroup
 func Start(
 	configJson, embeddedServerEntryList string,
 	provider PsiphonProvider,
-	useDeviceBinder bool) error {
+	useDeviceBinder bool, useIPv6Synthesizer bool) error {
 
 	controllerMutex.Lock()
 	defer controllerMutex.Unlock()
@@ -69,6 +70,10 @@ func Start(
 		config.DnsServerGetter = provider
 	}
 
+	if useIPv6Synthesizer {
+		config.IPv6Synthesizer = provider
+	}
+
 	psiphon.SetNoticeOutput(psiphon.NewNoticeReceiver(
 		func(notice []byte) {
 			provider.Notice(string(notice))
@@ -142,8 +147,8 @@ func SetClientVerificationPayload(clientVerificationPayload string) {
 func SendFeedback(configJson, diagnosticsJson, b64EncodedPublicKey, uploadServer, uploadPath, uploadServerHeaders string) {
 	err := psiphon.SendFeedback(configJson, diagnosticsJson, b64EncodedPublicKey, uploadServer, uploadPath, uploadServerHeaders)
 	if err != nil {
-		psiphon.NoticeAlert("failed to upload feedback: %s", err)
+		psiphon.NoticeAlert("Failed to upload feedback: %s", err)
 	} else {
-		psiphon.NoticeInfo("feedback uploaded successfully")
+		psiphon.NoticeInfo("Feedback uploaded successfully")
 	}
 }

Server/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:latest
+FROM alpine:3.4
 
 MAINTAINER Psiphon Inc. <info@psiphon.ca>
 LABEL Description="Alpine Linux based Psiphon Tunnel-Core Server" Vendor="Psiphon Inc." Version="1.0"

Server/Dockerfile-binary-builder

@@ -1,4 +1,4 @@
-FROM alpine:latest
+FROM alpine:3.4
 
 ENV GOLANG_VERSION 1.7.3
 ENV GOLANG_SRC_URL https://golang.org/dl/go$GOLANG_VERSION.src.tar.gz

psiphon/LookupIP.go

@@ -69,7 +69,28 @@ func bindLookupIP(host, dnsServer string, config *DialConfig) (addrs []net.IP, e
 		return []net.IP{ipAddr}, nil
 	}
 
-	socketFd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, 0)
+	// config.DnsServerGetter.GetDnsServers() must return IP addresses
+	ipAddr = net.ParseIP(dnsServer)
+	if ipAddr == nil {
+		return nil, common.ContextError(errors.New("invalid IP address"))
+	}
+
+	var ipv4 [4]byte
+	var ipv6 [16]byte
+	var domain int
+
+	// Get address type (IPv4 or IPv6)
+	if ipAddr.To4() != nil {
+		copy(ipv4[:], ipAddr.To4())
+		domain = syscall.AF_INET
+	} else if ipAddr.To16() != nil {
+		copy(ipv6[:], ipAddr.To16())
+		domain = syscall.AF_INET6
+	} else {
+		return nil, common.ContextError(fmt.Errorf("Got invalid IP address for dns server: %s", ipAddr.String()))
+	}
+
+	socketFd, err := syscall.Socket(domain, syscall.SOCK_DGRAM, 0)
 	if err != nil {
 		return nil, common.ContextError(err)
 	}
@@ -80,18 +101,15 @@ func bindLookupIP(host, dnsServer string, config *DialConfig) (addrs []net.IP, e
 		return nil, common.ContextError(fmt.Errorf("BindToDevice failed: %s", err))
 	}
 
-	// config.DnsServerGetter.GetDnsServers() must return IP addresses
-	ipAddr = net.ParseIP(dnsServer)
-	if ipAddr == nil {
-		return nil, common.ContextError(errors.New("invalid IP address"))
-	}
-
-	// TODO: IPv6 support
-	var ip [4]byte
-	copy(ip[:], ipAddr.To4())
-	sockAddr := syscall.SockaddrInet4{Addr: ip, Port: DNS_PORT}
+	// Connect socket to the server's IP address
 	// Note: no timeout or interrupt for this connect, as it's a datagram socket
-	err = syscall.Connect(socketFd, &sockAddr)
+	if domain == syscall.AF_INET {
+		sockAddr := syscall.SockaddrInet4{Addr: ipv4, Port: DNS_PORT}
+		err = syscall.Connect(socketFd, &sockAddr)
+	} else if domain == syscall.AF_INET6 {
+		sockAddr := syscall.SockaddrInet6{Addr: ipv6, Port: DNS_PORT}
+		err = syscall.Connect(socketFd, &sockAddr)
+	}
 	if err != nil {
 		return nil, common.ContextError(err)
 	}

psiphon/TCPConn.go

@@ -104,6 +104,30 @@ func interruptibleTCPDial(addr string, config *DialConfig) (*TCPConn, error) {
 	// when tcpDial, amoung other things, when makes a blocking syscall.Connect()
 	// call.
 	go func() {
+		if config.IPv6Synthesizer != nil {
+			// Synthesize an IPv6 address from an IPv4 one
+			// This is for compatibility on DNS64/NAT64 networks
+			host, port, err := net.SplitHostPort(addr)
+			if err != nil {
+				select {
+				case conn.dialResult <- err:
+				default:
+				}
+				return
+			}
+			ip := net.ParseIP(host)
+			if ip != nil && ip.To4() != nil {
+				synthesizedAddr := config.IPv6Synthesizer.IPv6Synthesize(host)
+				// If IPv6Synthesize fails we will try dialing with the
+				// original IPv4 address instead of logging an error. If
+				// the address is unreachable an error will be emitted
+				// from tcpDial.
+				if synthesizedAddr != "" {
+					addr = net.JoinHostPort(synthesizedAddr, port)
+				}
+			}
+		}
+
 		var netConn net.Conn
 		var err error
 		if config.UpstreamProxyUrl != "" {
@@ -152,11 +176,14 @@ func proxiedTcpDial(
 	dialer := func(network, addr string) (net.Conn, error) {
 		return tcpDial(addr, config, dialResult)
 	}
+
+	dialHeaders, _ := common.UserAgentIfUnset(config.UpstreamProxyCustomHeaders)
+
 	upstreamDialer := upstreamproxy.NewProxyDialFunc(
 		&upstreamproxy.UpstreamProxyConfig{
 			ForwardDialFunc: dialer,
 			ProxyURIString:  config.UpstreamProxyUrl,
-			CustomHeaders:   config.UpstreamProxyCustomHeaders,
+			CustomHeaders:   dialHeaders,
 		})
 	netConn, err := upstreamDialer("tcp", addr)
 	if _, ok := err.(*upstreamproxy.Error); ok {

psiphon/TCPConn_bind.go

@@ -78,12 +78,24 @@ func tcpDial(addr string, config *DialConfig, dialResult chan error) (net.Conn,
 		return nil, common.ContextError(err)
 	}
 
-	// TODO: IPv6 support
-	var ip [4]byte
-	copy(ip[:], ipAddrs[index].To4())
+	var ipv4 [4]byte
+	var ipv6 [16]byte
+	var domain int
+	ipAddr := ipAddrs[index]
+
+	// Get address type (IPv4 or IPv6)
+	if ipAddr != nil && ipAddr.To4() != nil {
+		copy(ipv4[:], ipAddr.To4())
+		domain = syscall.AF_INET
+	} else if ipAddr != nil && ipAddr.To16() != nil {
+		copy(ipv6[:], ipAddr.To16())
+		domain = syscall.AF_INET6
+	} else {
+		return nil, common.ContextError(fmt.Errorf("Got invalid IP address: %s", ipAddr.String()))
+	}
 
 	// Create a socket and bind to device, when configured to do so
-	socketFd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, 0)
+	socketFd, err := syscall.Socket(domain, syscall.SOCK_STREAM, 0)
 	if err != nil {
 		return nil, common.ContextError(err)
 	}
@@ -100,8 +112,14 @@ func tcpDial(addr string, config *DialConfig, dialResult chan error) (net.Conn,
 		}
 	}
 
-	sockAddr := syscall.SockaddrInet4{Addr: ip, Port: port}
-	err = syscall.Connect(socketFd, &sockAddr)
+	// Connect socket to the server's IP address
+	if domain == syscall.AF_INET {
+		sockAddr := syscall.SockaddrInet4{Addr: ipv4, Port: port}
+		err = syscall.Connect(socketFd, &sockAddr)
+	} else if domain == syscall.AF_INET6 {
+		sockAddr := syscall.SockaddrInet6{Addr: ipv6, Port: port}
+		err = syscall.Connect(socketFd, &sockAddr)
+	}
 	if err != nil {
 		syscall.Close(socketFd)
 		return nil, common.ContextError(err)

psiphon/common/osl/osl.go

@@ -43,6 +43,7 @@ import (
 	"net/url"
 	"path"
 	"path/filepath"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -535,7 +536,7 @@ func (state *ClientSeedState) issueSLOKs() {
 			slok, ok := state.scheme.derivedSLOKCache[*ref]
 			state.scheme.derivedSLOKCacheMutex.RUnlock()
 			if !ok {
-				slok = deriveSLOK(state.scheme, ref)
+				slok = state.scheme.deriveSLOK(ref)
 				state.scheme.derivedSLOKCacheMutex.Lock()
 				state.scheme.derivedSLOKCache[*ref] = slok
 				state.scheme.derivedSLOKCacheMutex.Unlock()
@@ -569,31 +570,6 @@ func getSLOKTime(seedPeriodNanoseconds int64) int64 {
 	return time.Now().UTC().Truncate(time.Duration(seedPeriodNanoseconds)).UnixNano()
 }
 
-// deriveSLOK produces SLOK secret keys and IDs using HKDF-Expand
-// defined in https://tools.ietf.org/html/rfc5869.
-func deriveSLOK(
-	scheme *Scheme, ref *slokReference) *SLOK {
-
-	timeBytes := make([]byte, 8)
-	binary.LittleEndian.PutUint64(timeBytes, uint64(ref.Time.UnixNano()))
-
-	key := deriveKeyHKDF(
-		scheme.MasterKey,
-		[]byte(ref.PropagationChannelID),
-		[]byte(ref.SeedSpecID),
-		timeBytes)
-
-	// TODO: is ID derivation cryptographically sound?
-	id := deriveKeyHKDF(
-		scheme.MasterKey,
-		key)
-
-	return &SLOK{
-		ID:  id,
-		Key: key,
-	}
-}
-
 // GetSeedPayload issues any pending SLOKs and returns the accumulated
 // SLOKs for a given client. psiphond will calls this when it receives
 // signalIssueSLOKs which is the trigger to check for new SLOKs.
@@ -631,6 +607,44 @@ func (state *ClientSeedState) ClearSeedPayload() {
 	state.payloadSLOKs = nil
 }
 
+// deriveSLOK produces SLOK secret keys and IDs using HKDF-Expand
+// defined in https://tools.ietf.org/html/rfc5869.
+func (scheme *Scheme) deriveSLOK(ref *slokReference) *SLOK {
+
+	timeBytes := make([]byte, 8)
+	binary.LittleEndian.PutUint64(timeBytes, uint64(ref.Time.UnixNano()))
+
+	key := deriveKeyHKDF(
+		scheme.MasterKey,
+		[]byte(ref.PropagationChannelID),
+		[]byte(ref.SeedSpecID),
+		timeBytes)
+
+	// TODO: is ID derivation cryptographically sound?
+	id := deriveKeyHKDF(
+		scheme.MasterKey,
+		key)
+
+	return &SLOK{
+		ID:  id,
+		Key: key,
+	}
+}
+
+// GetOSLDuration returns the total time duration of an OSL,
+// which is a function of the scheme's SeedPeriodNanoSeconds,
+// the duration of a single SLOK, and the scheme's SeedPeriodKeySplits,
+// the number of SLOKs associated with an OSL.
+func (scheme *Scheme) GetOSLDuration() time.Duration {
+	slokTimePeriodsPerOSL := 1
+	for _, keySplit := range scheme.SeedPeriodKeySplits {
+		slokTimePeriodsPerOSL *= keySplit.Total
+	}
+
+	return time.Duration(
+		int64(slokTimePeriodsPerOSL) * scheme.SeedPeriodNanoseconds)
+}
+
 // PaveFile describes an OSL data file to be paved to an out-of-band
 // distribution drop site. There are two types of files: a registry,
 // which describes how to assemble keys for OSLs, and the encrypted
@@ -677,6 +691,16 @@ type KeyShares struct {
 	KeyShares   []*KeyShares
 }
 
+type PaveLogInfo struct {
+	FileName             string
+	SchemeIndex          int
+	PropagationChannelID string
+	OSLID                string
+	OSLTime              time.Time
+	OSLDuration          time.Duration
+	ServerEntryCount     int
+}
+
 // Pave creates the full set of OSL files, for all schemes in the
 // configuration, to be dropped in an out-of-band distribution site.
 // Only OSLs for the propagation channel ID associated with the
@@ -686,14 +710,14 @@ type KeyShares struct {
 // the client functions GetRegistryURL and GetOSLFileURL.
 //
 // Pave returns a pave file for the entire registry of all OSLs from
-// epoch. It only returns pave files for OSLs referenced in
-// paveServerEntries. paveServerEntries is a list of maps, one for each
-// scheme, from the first SLOK time period identifying an OSL to a
-// payload to encrypt and pave.
-// The registry file spec MD5 checksum values are populated only for
-// OSLs referenced in paveServerEntries. To ensure a registry is fully
-// populated with hashes for skipping redownloading, all OSLs should
-// be paved.
+// epoch to endTime, and a pave file for each OSL. paveServerEntries is
+// a map from hex-encoded OSL IDs to server entries to pave into that OSL.
+// When entries are found, OSL will contain those entries, newline
+// seperated. Otherwise the OSL will still be issued, but be empty.
+//
+// As OSLs outside the epoch-endTime range will no longer appear in
+// the registry, Pave is intended to be used to create the full set
+// of OSLs for a distribution site; i.e., not incrementally.
 //
 // Automation is responsible for consistently distributing server entries
 // to OSLs in the case where OSLs are repaved in subsequent calls.
@@ -702,8 +726,8 @@ func (config *Config) Pave(
 	propagationChannelID string,
 	signingPublicKey string,
 	signingPrivateKey string,
-	paveServerEntries []map[time.Time]string,
-	logCallback func(int, time.Time, string)) ([]*PaveFile, error) {
+	paveServerEntries map[string][]string,
+	logCallback func(*PaveLogInfo)) ([]*PaveFile, error) {
 
 	config.ReloadableFile.RLock()
 	defer config.ReloadableFile.RUnlock()
@@ -712,19 +736,13 @@ func (config *Config) Pave(
 
 	registry := &Registry{}
 
-	if len(paveServerEntries) != len(config.Schemes) {
-		return nil, common.ContextError(errors.New("invalid paveServerEntries"))
-	}
-
 	for schemeIndex, scheme := range config.Schemes {
+		if common.Contains(scheme.PropagationChannelIDs, propagationChannelID) {
 
-		slokTimePeriodsPerOSL := 1
-		for _, keySplit := range scheme.SeedPeriodKeySplits {
-			slokTimePeriodsPerOSL *= keySplit.Total
-		}
+			oslDuration := scheme.GetOSLDuration()
 
-		if common.Contains(scheme.PropagationChannelIDs, propagationChannelID) {
 			oslTime := scheme.epoch
+
 			for !oslTime.After(endTime) {
 
 				firstSLOKTime := oslTime
@@ -734,43 +752,52 @@ func (config *Config) Pave(
 					return nil, common.ContextError(err)
 				}
 
-				registry.FileSpecs = append(registry.FileSpecs, fileSpec)
+				hexEncodedOSLID := hex.EncodeToString(fileSpec.ID)
 
-				serverEntries, ok := paveServerEntries[schemeIndex][oslTime]
-				if ok {
+				registry.FileSpecs = append(registry.FileSpecs, fileSpec)
 
-					signedServerEntries, err := common.WriteAuthenticatedDataPackage(
-						serverEntries,
-						signingPublicKey,
-						signingPrivateKey)
-					if err != nil {
-						return nil, common.ContextError(err)
-					}
+				serverEntryCount := len(paveServerEntries[hexEncodedOSLID])
 
-					boxedServerEntries, err := box(fileKey, common.Compress(signedServerEntries))
-					if err != nil {
-						return nil, common.ContextError(err)
-					}
+				// serverEntries will be "" when nothing is found in paveServerEntries
+				serverEntries := strings.Join(paveServerEntries[hexEncodedOSLID], "\n")
 
-					md5sum := md5.Sum(boxedServerEntries)
-					fileSpec.MD5Sum = md5sum[:]
+				signedServerEntries, err := common.WriteAuthenticatedDataPackage(
+					serverEntries,
+					signingPublicKey,
+					signingPrivateKey)
+				if err != nil {
+					return nil, common.ContextError(err)
+				}
 
-					fileName := fmt.Sprintf(
-						OSL_FILENAME_FORMAT, hex.EncodeToString(fileSpec.ID))
+				boxedServerEntries, err := box(fileKey, common.Compress(signedServerEntries))
+				if err != nil {
+					return nil, common.ContextError(err)
+				}
 
-					paveFiles = append(paveFiles, &PaveFile{
-						Name:     fileName,
-						Contents: boxedServerEntries,
+				md5sum := md5.Sum(boxedServerEntries)
+				fileSpec.MD5Sum = md5sum[:]
+
+				fileName := fmt.Sprintf(
+					OSL_FILENAME_FORMAT, hexEncodedOSLID)
+
+				paveFiles = append(paveFiles, &PaveFile{
+					Name:     fileName,
+					Contents: boxedServerEntries,
+				})
+
+				if logCallback != nil {
+					logCallback(&PaveLogInfo{
+						FileName:             fileName,
+						SchemeIndex:          schemeIndex,
+						PropagationChannelID: propagationChannelID,
+						OSLID:                hexEncodedOSLID,
+						OSLTime:              oslTime,
+						OSLDuration:          oslDuration,
+						ServerEntryCount:     serverEntryCount,
 					})
-
-					if logCallback != nil {
-						logCallback(schemeIndex, oslTime, fileName)
-					}
 				}
 
-				oslTime = oslTime.Add(
-					time.Duration(
-						int64(slokTimePeriodsPerOSL) * scheme.SeedPeriodNanoseconds))
+				oslTime = oslTime.Add(oslDuration)
 			}
 		}
 	}
@@ -811,7 +838,7 @@ func makeOSLFileSpec(
 		SeedSpecID:           string(scheme.SeedSpecs[0].ID),
 		Time:                 firstSLOKTime,
 	}
-	firstSLOK := deriveSLOK(scheme, ref)
+	firstSLOK := scheme.deriveSLOK(ref)
 	oslID := firstSLOK.ID
 
 	fileKey, err := common.MakeSecureRandomBytes(KEY_LENGTH_BYTES)
@@ -922,7 +949,7 @@ func divideKeyWithSeedSpecSLOKs(
 			SeedSpecID:           string(seedSpec.ID),
 			Time:                 *nextSLOKTime,
 		}
-		slok := deriveSLOK(scheme, ref)
+		slok := scheme.deriveSLOK(ref)
 
 		boxedShare, err := box(slok.Key, shares[index])
 		if err != nil {

psiphon/common/osl/osl_test.go

@@ -21,6 +21,7 @@ package osl
 
 import (
 	"encoding/base64"
+	"encoding/hex"
 	"fmt"
 	"net"
 	"testing"
@@ -317,31 +318,25 @@ func TestOSL(t *testing.T) {
 
 			// Dummy server entry payloads will be the OSL ID, which the following
 			// tests use to verify that the correct OSL file decrypts successfully.
-			paveServerEntries := make([]map[time.Time]string, len(config.Schemes))
-			for schemeIndex, scheme := range config.Schemes {
+			paveServerEntries := make(map[string][]string)
+			for _, scheme := range config.Schemes {
 
-				paveServerEntries[schemeIndex] = make(map[time.Time]string)
-
-				slokTimePeriodsPerOSL := 1
-				for _, keySplit := range scheme.SeedPeriodKeySplits {
-					slokTimePeriodsPerOSL *= keySplit.Total
-				}
+				oslDuration := scheme.GetOSLDuration()
 
 				oslTime := scheme.epoch
 				for oslTime.Before(endTime) {
+
 					firstSLOKRef := &slokReference{
 						PropagationChannelID: propagationChannelID,
 						SeedSpecID:           string(scheme.SeedSpecs[0].ID),
 						Time:                 oslTime,
 					}
-					firstSLOK := deriveSLOK(scheme, firstSLOKRef)
+					firstSLOK := scheme.deriveSLOK(firstSLOKRef)
 					oslID := firstSLOK.ID
-					paveServerEntries[schemeIndex][oslTime] =
-						base64.StdEncoding.EncodeToString(oslID)
+					paveServerEntries[hex.EncodeToString(oslID)] =
+						[]string{base64.StdEncoding.EncodeToString(oslID)}
 
-					oslTime = oslTime.Add(
-						time.Duration(
-							int64(slokTimePeriodsPerOSL) * scheme.SeedPeriodNanoseconds))
+					oslTime = oslTime.Add(oslDuration)
 				}
 			}
 
@@ -492,8 +487,7 @@ func TestOSL(t *testing.T) {
 			for _, timePeriod := range testCase.issueSLOKTimePeriods {
 				for _, seedSpecIndex := range testCase.issueSLOKSeedSpecIndexes {
 
-					slok := deriveSLOK(
-						testCase.scheme,
+					slok := testCase.scheme.deriveSLOK(
 						&slokReference{
 							PropagationChannelID: testCase.propagationChannelID,
 							SeedSpecID:           string(testCase.scheme.SeedSpecs[seedSpecIndex].ID),

psiphon/common/osl/paver/main.go

@@ -22,6 +22,7 @@ package main
 import (
 	"crypto/x509"
 	"encoding/base64"
+	"encoding/json"
 	"encoding/pem"
 	"flag"
 	"fmt"
@@ -36,22 +37,33 @@ import (
 func main() {
 
 	var configFilename string
-	flag.StringVar(&configFilename, "config", "", "OSL configuration file")
+	flag.StringVar(&configFilename, "config", "", "OSL configuration filename")
 
 	var offset time.Duration
-	flag.DurationVar(&offset, "offset", 0, "pave OSL start time (offset from now)")
+	flag.DurationVar(
+		&offset, "offset", 0,
+		"pave OSL start time (offset from now); default, 0, selects earliest epoch")
 
 	var period time.Duration
-	flag.DurationVar(&period, "period", 0, "pave OSL total period (starting from offset)")
+	flag.DurationVar(
+		&period, "period", 0,
+		"pave OSL total period (starting from offset); default, 0, selects at least one OSL period from now for all schemes")
 
 	var signingKeyPairFilename string
-	flag.StringVar(&signingKeyPairFilename, "key", "", "signing public key pair")
+	flag.StringVar(&signingKeyPairFilename, "key", "", "signing public key pair filename")
+
+	var payloadFilename string
+	flag.StringVar(&payloadFilename, "payload", "", "server entries to pave into OSLs")
 
 	var destinationDirectory string
-	flag.StringVar(&destinationDirectory, "output", "", "destination directory for output files")
+	flag.StringVar(
+		&destinationDirectory, "output", "",
+		"destination directory for output files; when omitted, no files are written (dry run mode)")
 
 	flag.Parse()
 
+	// load config
+
 	configJSON, err := ioutil.ReadFile(configFilename)
 	if err != nil {
 		fmt.Printf("failed loading configuration file: %s\n", err)
@@ -64,6 +76,8 @@ func main() {
 		os.Exit(1)
 	}
 
+	// load key pair
+
 	keyPairPEM, err := ioutil.ReadFile(signingKeyPairFilename)
 	if err != nil {
 		fmt.Printf("failed loading signing public key pair file: %s\n", err)
@@ -97,47 +111,84 @@ func main() {
 	signingPublicKey := base64.StdEncoding.EncodeToString(publicKeyBytes)
 	signingPrivateKey := base64.StdEncoding.EncodeToString(privateKeyBytes)
 
-	paveTime := time.Now().UTC()
-	startTime := paveTime.Add(offset)
-	endTime := startTime.Add(period)
-
-	schemeOSLTimePeriods := make(map[int]time.Duration)
-	for index, scheme := range config.Schemes {
-		slokTimePeriodsPerOSL := 1
-		for _, keySplit := range scheme.SeedPeriodKeySplits {
-			slokTimePeriodsPerOSL *= keySplit.Total
+	// load payload
+
+	paveServerEntries := make(map[string][]string)
+
+	pavedPayloadOSLID := make(map[string]bool)
+
+	if payloadFilename != "" {
+		payloadJSON, err := ioutil.ReadFile(payloadFilename)
+		if err != nil {
+			fmt.Printf("failed loading payload file: %s\n", err)
+			os.Exit(1)
 		}
-		schemeOSLTimePeriods[index] =
-			time.Duration(scheme.SeedPeriodNanoseconds * int64(slokTimePeriodsPerOSL))
-	}
 
-	allPropagationChannelIDs := make(map[string][]int)
-	for index, scheme := range config.Schemes {
-		for _, propagationChannelID := range scheme.PropagationChannelIDs {
-			allPropagationChannelIDs[propagationChannelID] =
-				append(allPropagationChannelIDs[propagationChannelID], index)
+		var payload []*struct {
+			OSLIDs      []string
+			ServerEntry string
 		}
-	}
 
-	for propagationChannelID, schemeIndexes := range allPropagationChannelIDs {
+		err = json.Unmarshal(payloadJSON, &payload)
+		if err != nil {
+			fmt.Printf("failed unmarshaling payload file: %s\n", err)
+			os.Exit(1)
+		}
 
-		paveServerEntries := make([]map[time.Time]string, len(config.Schemes))
+		for _, item := range payload {
+			for _, oslID := range item.OSLIDs {
+				paveServerEntries[oslID] = append(
+					paveServerEntries[oslID], item.ServerEntry)
+				pavedPayloadOSLID[oslID] = false
+			}
+		}
+	}
 
-		for _, index := range schemeIndexes {
+	// determine pave time range
 
-			paveServerEntries[index] = make(map[time.Time]string)
+	paveTime := time.Now().UTC()
 
-			oslTime, _ := time.Parse(time.RFC3339, config.Schemes[index].Epoch)
-			for !oslTime.After(endTime) {
-				if !oslTime.Before(startTime) {
-					paveServerEntries[index][oslTime] = ""
-				}
-				oslTime = oslTime.Add(schemeOSLTimePeriods[index])
+	var startTime, endTime time.Time
+
+	if offset != 0 {
+		startTime = paveTime.Add(offset)
+	} else {
+		// Default to the earliest scheme epoch.
+		startTime = paveTime
+		for _, scheme := range config.Schemes {
+			epoch, _ := time.Parse(time.RFC3339, scheme.Epoch)
+			if epoch.Before(startTime) {
+				startTime = epoch
 			}
+		}
+	}
 
-			fmt.Printf("Paving propagation channel %s, scheme #%d, [%s - %s], %s\n",
-				propagationChannelID, index, startTime, endTime, schemeOSLTimePeriods[index])
+	if period != 0 {
+		endTime = startTime.Add(period)
+	} else {
+		// Default to at least one OSL period after "now",
+		// considering all schemes.
+		endTime = paveTime
+		for _, scheme := range config.Schemes {
+			oslDuration := scheme.GetOSLDuration()
+			if endTime.Add(oslDuration).After(endTime) {
+				endTime = endTime.Add(oslDuration)
+			}
 		}
+	}
+
+	// build list of all participating propagation channel IDs
+
+	allPropagationChannelIDs := make(map[string]bool)
+	for _, scheme := range config.Schemes {
+		for _, propagationChannelID := range scheme.PropagationChannelIDs {
+			allPropagationChannelIDs[propagationChannelID] = true
+		}
+	}
+
+	// pave a directory for each propagation channel
+
+	for propagationChannelID, _ := range allPropagationChannelIDs {
 
 		paveFiles, err := config.Pave(
 			endTime,
@@ -145,29 +196,58 @@ func main() {
 			signingPublicKey,
 			signingPrivateKey,
 			paveServerEntries,
-			func(schemeIndex int, oslTime time.Time, fileName string) {
-				fmt.Printf("\tPaved scheme %d %s: %s\n", schemeIndex, oslTime, fileName)
+			func(logInfo *osl.PaveLogInfo) {
+				pavedPayloadOSLID[logInfo.OSLID] = true
+				fmt.Printf(
+					"paved %s: scheme %d, propagation channel ID %s, "+
+						"OSL time %s, OSL duration %s, server entries: %d\n",
+					logInfo.FileName,
+					logInfo.SchemeIndex,
+					logInfo.PropagationChannelID,
+					logInfo.OSLTime,
+					logInfo.OSLDuration,
+					logInfo.ServerEntryCount)
 			})
 		if err != nil {
 			fmt.Printf("failed paving: %s\n", err)
 			os.Exit(1)
 		}
 
-		directory := filepath.Join(destinationDirectory, propagationChannelID)
+		if destinationDirectory != "" {
 
-		err = os.MkdirAll(directory, 0755)
-		if err != nil {
-			fmt.Printf("failed creating output directory: %s\n", err)
-			os.Exit(1)
-		}
+			directory := filepath.Join(destinationDirectory, propagationChannelID)
 
-		for _, paveFile := range paveFiles {
-			filename := filepath.Join(directory, paveFile.Name)
-			err = ioutil.WriteFile(filename, paveFile.Contents, 0755)
+			err = os.MkdirAll(directory, 0755)
 			if err != nil {
-				fmt.Printf("error writing output file: %s\n", err)
+				fmt.Printf("failed creating output directory: %s\n", err)
 				os.Exit(1)
 			}
+
+			for _, paveFile := range paveFiles {
+				filename := filepath.Join(directory, paveFile.Name)
+				err = ioutil.WriteFile(filename, paveFile.Contents, 0755)
+				if err != nil {
+					fmt.Printf("error writing output file: %s\n", err)
+					os.Exit(1)
+				}
+			}
+		}
+	}
+
+	// fail if payload contains OSL IDs not in the config and time range
+
+	unknown := false
+	for oslID, paved := range pavedPayloadOSLID {
+		if !paved {
+			fmt.Printf(
+				"ignored %d server entries for unknown OSL ID: %s\n",
+				len(paveServerEntries[oslID]),
+				oslID)
+			unknown = true
 		}
 	}
+	if unknown {
+		fmt.Printf("payload contains unknown OSL IDs\n")
+		os.Exit(1)
+	}
 }

psiphon/common/userAgentPicker.go

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2017, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package common
+
+import (
+	"net/http"
+	"sync/atomic"
+)
+
+var registeredUserAgentPicker atomic.Value
+
+func RegisterUserAgentPicker(generator func() string) {
+	registeredUserAgentPicker.Store(generator)
+}
+
+func pickUserAgent() string {
+	generator := registeredUserAgentPicker.Load()
+	if generator != nil {
+		return generator.(func() string)()
+	}
+	return ""
+}
+
+// UserAgentIfUnset returns an http.Header object and a boolean
+// representing whether or not its User-Agent header was modified.
+// Any modifications are made to a copy of the original header map
+func UserAgentIfUnset(h http.Header) (http.Header, bool) {
+	var dialHeaders http.Header
+
+	if _, ok := h["User-Agent"]; !ok {
+		dialHeaders = make(map[string][]string)
+
+		if h != nil {
+			for k, v := range h {
+				dialHeaders[k] = make([]string, len(v))
+				copy(dialHeaders[k], v)
+			}
+		}
+
+		if FlipCoin() {
+			dialHeaders.Set("User-Agent", pickUserAgent())
+		} else {
+			dialHeaders.Set("User-Agent", "")
+		}
+
+		return dialHeaders, true
+	}
+
+	return h, false
+}

psiphon/config.go

@@ -20,6 +20,7 @@
 package psiphon
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -71,7 +72,7 @@ const (
 	PSIPHON_API_CLIENT_VERIFICATION_REQUEST_RETRY_PERIOD = 5 * time.Second
 	PSIPHON_API_CLIENT_VERIFICATION_REQUEST_MAX_RETRIES  = 10
 	FETCH_ROUTES_TIMEOUT_SECONDS                         = 60
-	DOWNLOAD_UPGRADE_TIMEOUT                             = 15 * time.Minute
+	DOWNLOAD_UPGRADE_TIMEOUT_SECONDS                     = 60
 	DOWNLOAD_UPGRADE_RETRY_PERIOD_SECONDS                = 30
 	DOWNLOAD_UPGRADE_STALE_PERIOD                        = 6 * time.Hour
 	IMPAIRED_PROTOCOL_CLASSIFICATION_DURATION            = 2 * time.Minute
@@ -120,8 +121,20 @@ type Config struct {
 	// be established to known servers.
 	// This value is supplied by and depends on the Psiphon Network, and is
 	// typically embedded in the client binary.
+	//
+	// Deprecated: Use RemoteServerListURLs. When RemoteServerListURLs is
+	// not nil, this parameter is ignored.
 	RemoteServerListUrl string
 
+	// RemoteServerListURLs is list of URLs which specify locations to fetch
+	// out-of-band server entries. This facility is used when a tunnel cannot
+	// be established to known servers.
+	// This value is supplied by and depends on the Psiphon Network, and is
+	// typically embedded in the client binary.
+	// All URLs must point to the same entity with the same ETag. At least
+	// one DownloadURL must have OnlyAfterAttempts = 0.
+	RemoteServerListURLs []*DownloadURL
+
 	// RemoteServerListDownloadFilename specifies a target filename for
 	// storing the remote server list download. Data is stored in co-located
 	// files (RemoteServerListDownloadFilename.part*) to allow for resumable
@@ -138,8 +151,19 @@ type Config struct {
 	// from which to fetch obfuscated server list files.
 	// This value is supplied by and depends on the Psiphon Network, and is
 	// typically embedded in the client binary.
+	//
+	// Deprecated: Use ObfuscatedServerListRootURLs. When
+	// ObfuscatedServerListRootURLs is not nil, this parameter is ignored.
 	ObfuscatedServerListRootURL string
 
+	// ObfuscatedServerListRootURLs is a list of URLs which specify root
+	// locations from which to fetch obfuscated server list files.
+	// This value is supplied by and depends on the Psiphon Network, and is
+	// typically embedded in the client binary.
+	// All URLs must point to the same entity with the same ETag. At least
+	// one DownloadURL must have OnlyAfterAttempts = 0.
+	ObfuscatedServerListRootURLs []*DownloadURL
+
 	// ObfuscatedServerListDownloadDirectory specifies a target directory for
 	// storing the obfuscated remote server list downloads. Data is stored in
 	// co-located files (<OSL filename>.part*) to allow for resumable
@@ -230,6 +254,11 @@ type Config struct {
 	// deployments.
 	DeviceBinder DeviceBinder
 
+	// IPv6Synthesizer is an interface that allows the core tunnel to call
+	// into the host application to synthesize IPv6 addresses from IPv4 ones. This
+	// is used to correctly lookup IPs on DNS64/NAT64 networks.
+	IPv6Synthesizer IPv6Synthesizer
+
 	// DnsServerGetter is an interface that enables the core tunnel to call
 	// into the host application to discover the native network DNS server settings.
 	// This parameter is only applicable to library deployments.
@@ -284,17 +313,30 @@ type Config struct {
 	// download facility which downloads this resource and emits a notice when complete.
 	// This value is supplied by and depends on the Psiphon Network, and is
 	// typically embedded in the client binary.
+	//
+	// Deprecated: Use UpgradeDownloadURLs. When UpgradeDownloadURLs
+	// is not nil, this parameter is ignored.
 	UpgradeDownloadUrl string
 
+	// UpgradeDownloadURLs is list of URLs which specify locations from which to
+	// download a host client upgrade file, when one is available. The core tunnel
+	// controller provides a resumable download facility which downloads this resource
+	// and emits a notice when complete.
+	// This value is supplied by and depends on the Psiphon Network, and is
+	// typically embedded in the client binary.
+	// All URLs must point to the same entity with the same ETag. At least
+	// one DownloadURL must have OnlyAfterAttempts = 0.
+	UpgradeDownloadURLs []*DownloadURL
+
 	// UpgradeDownloadClientVersionHeader specifies the HTTP header name for the
-	// entity at UpgradeDownloadUrl which specifies the client version (an integer
+	// entity at UpgradeDownloadURLs which specifies the client version (an integer
 	// value). A HEAD request may be made to check the version number available at
-	// UpgradeDownloadUrl. UpgradeDownloadClientVersionHeader is required when
-	// UpgradeDownloadUrl is specified.
+	// UpgradeDownloadURLs. UpgradeDownloadClientVersionHeader is required when
+	// UpgradeDownloadURLs is specified.
 	UpgradeDownloadClientVersionHeader string
 
 	// UpgradeDownloadFilename is the local target filename for an upgrade download.
-	// This parameter is required when UpgradeDownloadUrl is specified.
+	// This parameter is required when UpgradeDownloadURLs is specified.
 	// Data is stored in co-located files (UpgradeDownloadFilename.part*) to allow
 	// for resumable downloading.
 	UpgradeDownloadFilename string
@@ -366,7 +408,7 @@ type Config struct {
 	TunnelSshKeepAlivePeriodicTimeoutSeconds *int
 
 	// FetchRemoteServerListTimeoutSeconds specifies a timeout value for remote server list
-	// HTTP request. Zero value means that request will not time out.
+	// HTTP requests. Zero value means that request will not time out.
 	// If omitted, the default value is FETCH_REMOTE_SERVER_LIST_TIMEOUT_SECONDS.
 	FetchRemoteServerListTimeoutSeconds *int
 
@@ -379,10 +421,15 @@ type Config struct {
 	PsiphonApiServerTimeoutSeconds *int
 
 	// FetchRoutesTimeoutSeconds specifies a timeout value for split tunnel routes
-	// HTTP request. Zero value means that request will not time out.
+	// HTTP requests. Zero value means that request will not time out.
 	// If omitted, the default value is FETCH_ROUTES_TIMEOUT_SECONDS.
 	FetchRoutesTimeoutSeconds *int
 
+	// UpgradeDownloadTimeoutSeconds specifies a timeout value for upgrade download
+	// HTTP requests. Zero value means that request will not time out.
+	// If omitted, the default value is DOWNLOAD_UPGRADE_TIMEOUT_SECONDS.
+	DownloadUpgradeTimeoutSeconds *int
+
 	// HttpProxyOriginServerTimeoutSeconds specifies an HTTP response header timeout
 	// value in various HTTP relays found in httpProxy.
 	// Zero value means that request will not time out.
@@ -414,6 +461,26 @@ type Config struct {
 	EmitSLOKs bool
 }
 
+// DownloadURL specifies a URL for downloading resources along with parameters
+// for the download strategy.
+type DownloadURL struct {
+
+	// URL is the location of the resource. This string is slightly obfuscated
+	// with base64 encoding to mitigate trivial binary executable string scanning.
+	URL string
+
+	// SkipVerify indicates whether to verify HTTPS certificates. It some
+	// circumvention scenarios, verification is not possible. This must
+	// only be set to true when the resource its own verification mechanism.
+	SkipVerify bool
+
+	// OnlyAfterAttempts specifies how to schedule this URL when downloading
+	// the same resource (same entity, same ETag) from multiple different
+	// candidate locations. For a value of N, this URL is only a candidate
+	// after N rounds of attempting the download from other URLs.
+	OnlyAfterAttempts int
+}
+
 // LoadConfig parses and validates a JSON format Psiphon config JSON
 // string and returns a Config struct populated with config values.
 func LoadConfig(configJson []byte) (*Config, error) {
@@ -506,15 +573,37 @@ func LoadConfig(configJson []byte) (*Config, error) {
 			errors.New("invalid TargetApiProtocol"))
 	}
 
-	if config.UpgradeDownloadUrl != "" &&
-		(config.UpgradeDownloadClientVersionHeader == "" || config.UpgradeDownloadFilename == "") {
-		return nil, common.ContextError(errors.New(
-			"UpgradeDownloadUrl requires UpgradeDownloadClientVersionHeader and UpgradeDownloadFilename"))
+	if config.UpgradeDownloadUrl != "" && config.UpgradeDownloadURLs == nil {
+		config.UpgradeDownloadURLs = promoteLegacyDownloadURL(config.UpgradeDownloadUrl)
+	}
+
+	if config.UpgradeDownloadURLs != nil {
+
+		err := decodeAndValidateDownloadURLs("UpgradeDownloadURLs", config.UpgradeDownloadURLs)
+		if err != nil {
+			return nil, common.ContextError(err)
+		}
+
+		if config.UpgradeDownloadClientVersionHeader == "" {
+			return nil, common.ContextError(errors.New("missing UpgradeDownloadClientVersionHeader"))
+		}
+		if config.UpgradeDownloadFilename == "" {
+			return nil, common.ContextError(errors.New("missing UpgradeDownloadFilename"))
+		}
 	}
 
 	if !config.DisableRemoteServerListFetcher {
 
-		if config.RemoteServerListUrl != "" {
+		if config.RemoteServerListUrl != "" && config.RemoteServerListURLs == nil {
+			config.RemoteServerListURLs = promoteLegacyDownloadURL(config.RemoteServerListUrl)
+		}
+
+		if config.RemoteServerListURLs != nil {
+
+			err := decodeAndValidateDownloadURLs("RemoteServerListURLs", config.RemoteServerListURLs)
+			if err != nil {
+				return nil, common.ContextError(err)
+			}
 
 			if config.RemoteServerListSignaturePublicKey == "" {
 				return nil, common.ContextError(errors.New("missing RemoteServerListSignaturePublicKey"))
@@ -525,7 +614,16 @@ func LoadConfig(configJson []byte) (*Config, error) {
 			}
 		}
 
-		if config.ObfuscatedServerListRootURL != "" {
+		if config.ObfuscatedServerListRootURL != "" && config.ObfuscatedServerListRootURLs == nil {
+			config.ObfuscatedServerListRootURLs = promoteLegacyDownloadURL(config.ObfuscatedServerListRootURL)
+		}
+
+		if config.ObfuscatedServerListRootURLs != nil {
+
+			err := decodeAndValidateDownloadURLs("ObfuscatedServerListRootURLs", config.ObfuscatedServerListRootURLs)
+			if err != nil {
+				return nil, common.ContextError(err)
+			}
 
 			if config.RemoteServerListSignaturePublicKey == "" {
 				return nil, common.ContextError(errors.New("missing RemoteServerListSignaturePublicKey"))
@@ -572,6 +670,11 @@ func LoadConfig(configJson []byte) (*Config, error) {
 		config.FetchRoutesTimeoutSeconds = &defaultFetchRoutesTimeoutSeconds
 	}
 
+	if config.DownloadUpgradeTimeoutSeconds == nil {
+		defaultDownloadUpgradeTimeoutSeconds := DOWNLOAD_UPGRADE_TIMEOUT_SECONDS
+		config.DownloadUpgradeTimeoutSeconds = &defaultDownloadUpgradeTimeoutSeconds
+	}
+
 	if config.HttpProxyOriginServerTimeoutSeconds == nil {
 		defaultHttpProxyOriginServerTimeoutSeconds := HTTP_PROXY_ORIGIN_SERVER_TIMEOUT_SECONDS
 		config.HttpProxyOriginServerTimeoutSeconds = &defaultHttpProxyOriginServerTimeoutSeconds
@@ -594,3 +697,73 @@ func LoadConfig(configJson []byte) (*Config, error) {
 
 	return &config, nil
 }
+
+func promoteLegacyDownloadURL(URL string) []*DownloadURL {
+	downloadURLs := make([]*DownloadURL, 1)
+	downloadURLs[0] = &DownloadURL{
+		URL:               base64.StdEncoding.EncodeToString([]byte(URL)),
+		SkipVerify:        false,
+		OnlyAfterAttempts: 0,
+	}
+	return downloadURLs
+}
+
+func decodeAndValidateDownloadURLs(name string, downloadURLs []*DownloadURL) error {
+
+	hasOnlyAfterZero := false
+	for _, downloadURL := range downloadURLs {
+		if downloadURL.OnlyAfterAttempts == 0 {
+			hasOnlyAfterZero = true
+		}
+		decodedURL, err := base64.StdEncoding.DecodeString(downloadURL.URL)
+		if err != nil {
+			return fmt.Errorf("failed to decode URL in %s: %s", name, err)
+		}
+
+		downloadURL.URL = string(decodedURL)
+	}
+
+	var err error
+	if !hasOnlyAfterZero {
+		err = fmt.Errorf("must be at least one DownloadURL with OnlyAfterAttempts = 0 in %s", name)
+	}
+
+	return err
+}
+
+func selectDownloadURL(attempt int, downloadURLs []*DownloadURL) (string, string, bool) {
+
+	// The first OnlyAfterAttempts = 0 URL is the canonical URL. This
+	// is the value used as the key for SetUrlETag when multiple download
+	// URLs can be used to fetch a single entity.
+
+	canonicalURL := ""
+	for _, downloadURL := range downloadURLs {
+		if downloadURL.OnlyAfterAttempts == 0 {
+			canonicalURL = downloadURL.URL
+			break
+		}
+	}
+
+	candidates := make([]int, 0)
+	for index, URL := range downloadURLs {
+		if attempt >= URL.OnlyAfterAttempts {
+			candidates = append(candidates, index)
+		}
+	}
+
+	if len(candidates) < 1 {
+		// This case is not expected, as decodeAndValidateDownloadURLs
+		// should reject configs that would have no candidates for
+		// 0 attempts.
+		return "", "", true
+	}
+
+	selection, err := common.MakeSecureRandomInt(len(candidates))
+	if err != nil {
+		selection = 0
+	}
+	downloadURL := downloadURLs[candidates[selection]]
+
+	return downloadURL.URL, canonicalURL, downloadURL.SkipVerify
+}

psiphon/config_test.go

@@ -20,6 +20,7 @@
 package psiphon
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"io/ioutil"
 	"strings"
@@ -157,3 +158,168 @@ func (suite *ConfigTestSuite) Test_LoadConfig_GoodJson() {
 	_, err = LoadConfig(testObjJSON)
 	suite.Nil(err, "JSON with null for optional values should succeed")
 }
+
+func TestDownloadURLs(t *testing.T) {
+
+	decodedA := "a.example.com"
+	encodedA := base64.StdEncoding.EncodeToString([]byte(decodedA))
+	encodedB := base64.StdEncoding.EncodeToString([]byte("b.example.com"))
+	encodedC := base64.StdEncoding.EncodeToString([]byte("c.example.com"))
+
+	testCases := []struct {
+		description                string
+		downloadURLs               []*DownloadURL
+		attempts                   int
+		expectedValid              bool
+		expectedCanonicalURL       string
+		expectedDistinctSelections int
+	}{
+		{
+			"missing OnlyAfterAttempts = 0",
+			[]*DownloadURL{
+				&DownloadURL{
+					URL:               encodedA,
+					OnlyAfterAttempts: 1,
+				},
+			},
+			1,
+			false,
+			decodedA,
+			0,
+		},
+		{
+			"single URL, multiple attempts",
+			[]*DownloadURL{
+				&DownloadURL{
+					URL:               encodedA,
+					OnlyAfterAttempts: 0,
+				},
+			},
+			2,
+			true,
+			decodedA,
+			1,
+		},
+		{
+			"multiple URLs, single attempt",
+			[]*DownloadURL{
+				&DownloadURL{
+					URL:               encodedA,
+					OnlyAfterAttempts: 0,
+				},
+				&DownloadURL{
+					URL:               encodedB,
+					OnlyAfterAttempts: 1,
+				},
+				&DownloadURL{
+					URL:               encodedC,
+					OnlyAfterAttempts: 1,
+				},
+			},
+			1,
+			true,
+			decodedA,
+			1,
+		},
+		{
+			"multiple URLs, multiple attempts",
+			[]*DownloadURL{
+				&DownloadURL{
+					URL:               encodedA,
+					OnlyAfterAttempts: 0,
+				},
+				&DownloadURL{
+					URL:               encodedB,
+					OnlyAfterAttempts: 1,
+				},
+				&DownloadURL{
+					URL:               encodedC,
+					OnlyAfterAttempts: 1,
+				},
+			},
+			2,
+			true,
+			decodedA,
+			3,
+		},
+		{
+			"multiple URLs, multiple attempts",
+			[]*DownloadURL{
+				&DownloadURL{
+					URL:               encodedA,
+					OnlyAfterAttempts: 0,
+				},
+				&DownloadURL{
+					URL:               encodedB,
+					OnlyAfterAttempts: 1,
+				},
+				&DownloadURL{
+					URL:               encodedC,
+					OnlyAfterAttempts: 3,
+				},
+			},
+			4,
+			true,
+			decodedA,
+			3,
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.description, func(t *testing.T) {
+
+			err := decodeAndValidateDownloadURLs(
+				testCase.description,
+				testCase.downloadURLs)
+
+			if testCase.expectedValid {
+				if err != nil {
+					t.Fatalf("unexpected validation error: %s", err)
+				}
+			} else {
+				if err == nil {
+					t.Fatalf("expected validation error")
+				}
+				return
+			}
+
+			// Track distinct selections for each attempt; the
+			// expected number of distinct should be for at least
+			// one particular attempt.
+			attemptDistinctSelections := make(map[int]map[string]int)
+			for i := 0; i < testCase.attempts; i++ {
+				attemptDistinctSelections[i] = make(map[string]int)
+			}
+
+			// Perform enough runs to account for random selection.
+			runs := 1000
+
+			attempt := 0
+			for i := 0; i < runs; i++ {
+				url, canonicalURL, skipVerify := selectDownloadURL(attempt, testCase.downloadURLs)
+				if canonicalURL != testCase.expectedCanonicalURL {
+					t.Fatalf("unexpected canonical URL: %s", canonicalURL)
+				}
+				if skipVerify {
+					t.Fatalf("expected skipVerify")
+				}
+				attemptDistinctSelections[attempt][url] += 1
+				attempt = (attempt + 1) % testCase.attempts
+			}
+
+			maxDistinctSelections := 0
+			for _, m := range attemptDistinctSelections {
+				if len(m) > maxDistinctSelections {
+					maxDistinctSelections = len(m)
+				}
+			}
+
+			if maxDistinctSelections != testCase.expectedDistinctSelections {
+				t.Fatalf("got %d distinct selections, expected %d",
+					maxDistinctSelections,
+					testCase.expectedDistinctSelections)
+			}
+		})
+	}
+
+}

psiphon/controller.go

@@ -100,6 +100,7 @@ func NewController(config *Config) (controller *Controller, err error) {
 		PendingConns:                  untunneledPendingConns,
 		DeviceBinder:                  config.DeviceBinder,
 		DnsServerGetter:               config.DnsServerGetter,
+		IPv6Synthesizer:               config.IPv6Synthesizer,
 		UseIndistinguishableTLS:       config.UseIndistinguishableTLS,
 		TrustedCACertificatesFilename: config.TrustedCACertificatesFilename,
 		DeviceRegion:                  config.DeviceRegion,
@@ -184,7 +185,7 @@ func (controller *Controller) Run(shutdownBroadcast <-chan struct{}) {
 		retryPeriod := time.Duration(
 			*controller.config.FetchRemoteServerListRetryPeriodSeconds) * time.Second
 
-		if controller.config.RemoteServerListUrl != "" {
+		if controller.config.RemoteServerListURLs != nil {
 			controller.runWaitGroup.Add(1)
 			go controller.remoteServerListFetcher(
 				"common",
@@ -194,7 +195,7 @@ func (controller *Controller) Run(shutdownBroadcast <-chan struct{}) {
 				FETCH_REMOTE_SERVER_LIST_STALE_PERIOD)
 		}
 
-		if controller.config.ObfuscatedServerListRootURL != "" {
+		if controller.config.ObfuscatedServerListRootURLs != nil {
 			controller.runWaitGroup.Add(1)
 			go controller.remoteServerListFetcher(
 				"obfuscated",
@@ -205,9 +206,7 @@ func (controller *Controller) Run(shutdownBroadcast <-chan struct{}) {
 		}
 	}
 
-	if controller.config.UpgradeDownloadUrl != "" &&
-		controller.config.UpgradeDownloadFilename != "" {
-
+	if controller.config.UpgradeDownloadURLs != nil {
 		controller.runWaitGroup.Add(1)
 		go controller.upgradeDownloader()
 	}
@@ -324,7 +323,7 @@ fetcherLoop:
 		}
 
 	retryLoop:
-		for {
+		for attempt := 0; ; attempt++ {
 			// Don't attempt to fetch while there is no network connectivity,
 			// to avoid alert notice noise.
 			if !WaitForNetworkConnectivity(
@@ -339,6 +338,7 @@ fetcherLoop:
 
 			err := fetcher(
 				controller.config,
+				attempt,
 				tunnel,
 				controller.untunneledDialConfig)
 
@@ -491,7 +491,7 @@ downloadLoop:
 		}
 
 	retryLoop:
-		for {
+		for attempt := 0; ; attempt++ {
 			// Don't attempt to download while there is no network connectivity,
 			// to avoid alert notice noise.
 			if !WaitForNetworkConnectivity(
@@ -506,6 +506,7 @@ downloadLoop:
 
 			err := DownloadUpgrade(
 				controller.config,
+				attempt,
 				handshakeVersion,
 				tunnel,
 				controller.untunneledDialConfig)
@@ -584,25 +585,15 @@ loop:
 
 			if controller.isImpairedProtocol(establishedTunnel.protocol) {
 
-				NoticeAlert("established tunnel with impaired protocol: %s", establishedTunnel.protocol)
+				// Protocol was classified as impaired while this tunnel established.
+				// This is most likely to occur with TunnelPoolSize > 0. We log the
+				// event but take no action. Discarding the tunnel would break the
+				// impaired logic unless we did that (a) only if there are other
+				// unimpaired protocols; (b) only during the first interation of the
+				// ESTABLISH_TUNNEL_WORK_TIME loop. By not discarding here, a true
+				// impaired protocol may require an extra reconnect.
 
-				// Take action only when TunnelProtocol is unset, as it limits the
-				// controller to a single protocol. For testing and diagnostics, we
-				// still allow classification when TunnelProtocol is set.
-				if controller.config.TunnelProtocol == "" {
-
-					// Protocol was classified as impaired while this tunnel
-					// established, so discard.
-					controller.discardTunnel(establishedTunnel)
-
-					// Reset establish generator to stop producing tunnels
-					// with impaired protocols.
-					if controller.isEstablishing {
-						controller.stopEstablishing()
-						controller.startEstablishing()
-					}
-					break
-				}
+				NoticeAlert("established tunnel with impaired protocol: %s", establishedTunnel.protocol)
 			}
 
 			tunnelCount, registered := controller.registerTunnel(establishedTunnel)
@@ -693,16 +684,24 @@ loop:
 //
 // Concurrency note: only the runTunnels() goroutine may call classifyImpairedProtocol
 func (controller *Controller) classifyImpairedProtocol(failedTunnel *Tunnel) {
+
 	if failedTunnel.establishedTime.Add(IMPAIRED_PROTOCOL_CLASSIFICATION_DURATION).After(monotime.Now()) {
 		controller.impairedProtocolClassification[failedTunnel.protocol] += 1
 	} else {
 		controller.impairedProtocolClassification[failedTunnel.protocol] = 0
 	}
-	if len(controller.getImpairedProtocols()) == len(protocol.SupportedTunnelProtocols) {
-		// Reset classification if all protocols are classified as impaired as
-		// the network situation (or attack) may not be protocol-specific.
-		// TODO: compare against count of distinct supported protocols for
-		// current known server entries.
+
+	// Reset classification once all known protocols are classified as impaired, as
+	// there is now no way to proceed with only unimpaired protocols. The network
+	// situation (or attack) resulting in classification may not be protocol-specific.
+	//
+	// Note: with controller.config.TunnelProtocol set, this will always reset once
+	// that protocol has reached IMPAIRED_PROTOCOL_CLASSIFICATION_THRESHOLD.
+	if CountNonImpairedProtocols(
+		controller.config.EgressRegion,
+		controller.config.TunnelProtocol,
+		controller.getImpairedProtocols()) == 0 {
+
 		controller.impairedProtocolClassification = make(map[string]int)
 	}
 }
@@ -1090,11 +1089,9 @@ loop:
 			// evade the attack; (b) there's a good chance of false
 			// positives (such as short tunnel durations due to network
 			// hopping on a mobile device).
-			// Impaired protocols logic is not applied when
-			// config.TunnelProtocol is specified.
 			// The edited serverEntry is temporary copy which is not
 			// stored or reused.
-			if i == 0 && controller.config.TunnelProtocol == "" {
+			if i == 0 {
 				serverEntry.DisableImpairedProtocols(impairedProtocols)
 				if len(serverEntry.GetSupportedProtocols()) == 0 {
 					// Skip this server entry, as it has no supported

psiphon/controller_test.go

@@ -37,10 +37,10 @@ import (
 	"time"
 
 	"github.com/Psiphon-Inc/goarista/monotime"
+	"github.com/Psiphon-Inc/goproxy"
 	socks "github.com/Psiphon-Inc/goptlib"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/protocol"
-	"github.com/elazarl/goproxy"
 )
 
 var testDataDirName string
@@ -51,7 +51,7 @@ func TestMain(m *testing.M) {
 	var err error
 	testDataDirName, err = ioutil.TempDir("", "psiphon-controller-test")
 	if err != nil {
-		fmt.Printf("TempDir failed: %s", err)
+		fmt.Printf("TempDir failed: %s\n", err)
 		os.Exit(1)
 	}
 	defer os.RemoveAll(testDataDirName)
@@ -650,8 +650,15 @@ func controllerRun(t *testing.T, runConfig *controllerRunConfig) {
 
 				count, ok := classification[serverProtocol]
 				if ok && count >= IMPAIRED_PROTOCOL_CLASSIFICATION_THRESHOLD {
-					// TODO: wrong goroutine for t.FatalNow()
-					t.Fatalf("unexpected tunnel using impaired protocol: %s, %+v",
+
+					// TODO: Fix this test case. Use of TunnelPoolSize breaks this
+					// case, as many tunnel establishments are occurring in parallel,
+					// and it can happen that a protocol is classified as impaired
+					// while a tunnel with that protocol is established and set
+					// active.
+
+					// *not* t.Fatalf
+					t.Logf("unexpected tunnel using impaired protocol: %s, %+v",
 						serverProtocol, classification)
 				}
 
@@ -827,7 +834,13 @@ func fetchAndVerifyWebsite(t *testing.T, httpProxyPort int) {
 		Timeout: roundTripTimeout,
 	}
 
-	response, err := httpClient.Get(testUrl)
+	request, err := http.NewRequest("GET", testUrl, nil)
+	if err != nil {
+		t.Fatalf("error preparing proxied HTTP request: %s", err)
+	}
+	request.Close = true
+
+	response, err := httpClient.Do(request)
 	if err != nil {
 		t.Fatalf("error sending proxied HTTP request: %s", err)
 	}
@@ -842,6 +855,9 @@ func fetchAndVerifyWebsite(t *testing.T, httpProxyPort int) {
 		t.Fatalf("unexpected proxied HTTP response")
 	}
 
+	// Delay before requesting from external service again
+	time.Sleep(1 * time.Second)
+
 	// Test: use direct URL proxy
 
 	httpClient = &http.Client{
@@ -849,9 +865,17 @@ func fetchAndVerifyWebsite(t *testing.T, httpProxyPort int) {
 		Timeout:   roundTripTimeout,
 	}
 
-	response, err = httpClient.Get(
+	request, err = http.NewRequest(
+		"GET",
 		fmt.Sprintf("http://127.0.0.1:%d/direct/%s",
-			httpProxyPort, url.QueryEscape(testUrl)))
+			httpProxyPort, url.QueryEscape(testUrl)),
+		nil)
+	if err != nil {
+		t.Fatalf("error preparing direct URL request: %s", err)
+	}
+	request.Close = true
+
+	response, err = httpClient.Do(request)
 	if err != nil {
 		t.Fatalf("error sending direct URL request: %s", err)
 	}
@@ -919,26 +943,26 @@ func initDisruptor() {
 	go func() {
 		listener, err := socks.ListenSocks("tcp", disruptorProxyAddress)
 		if err != nil {
-			fmt.Errorf("disruptor proxy listen error: %s", err)
+			fmt.Printf("disruptor proxy listen error: %s\n", err)
 			return
 		}
 		for {
 			localConn, err := listener.AcceptSocks()
 			if err != nil {
-				fmt.Errorf("disruptor proxy accept error: %s", err)
+				fmt.Printf("disruptor proxy accept error: %s\n", err)
 				return
 			}
 			go func() {
 				defer localConn.Close()
 				remoteConn, err := net.Dial("tcp", localConn.Req.Target)
 				if err != nil {
-					fmt.Errorf("disruptor proxy dial error: %s", err)
+					fmt.Printf("disruptor proxy dial error: %s\n", err)
 					return
 				}
 				defer remoteConn.Close()
 				err = localConn.Grant(&net.TCPAddr{IP: net.ParseIP("0.0.0.0"), Port: 0})
 				if err != nil {
-					fmt.Errorf("disruptor proxy grant error: %s", err)
+					fmt.Printf("disruptor proxy grant error: %s\n", err)
 					return
 				}
 
@@ -988,7 +1012,7 @@ func initUpstreamProxy() {
 		proxy.OnRequest().DoFunc(
 			func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
 				if !hasExpectedCustomHeaders(r.Header) {
-					ctx.Logf("missing expected headers: %+v", ctx.Req.Header)
+					fmt.Printf("missing expected headers: %+v\n", ctx.Req.Header)
 					return nil, goproxy.NewResponse(r, goproxy.ContentTypeText, http.StatusUnauthorized, "")
 				}
 				return r, nil
@@ -997,7 +1021,7 @@ func initUpstreamProxy() {
 		proxy.OnRequest().HandleConnectFunc(
 			func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) {
 				if !hasExpectedCustomHeaders(ctx.Req.Header) {
-					ctx.Logf("missing expected headers: %+v", ctx.Req.Header)
+					fmt.Printf("missing expected headers: %+v\n", ctx.Req.Header)
 					return goproxy.RejectConnect, host
 				}
 				return goproxy.OkConnect, host
@@ -1005,7 +1029,7 @@ func initUpstreamProxy() {
 
 		err := http.ListenAndServe("127.0.0.1:2161", proxy)
 		if err != nil {
-			fmt.Printf("upstream proxy failed: %s", err)
+			fmt.Printf("upstream proxy failed: %s\n", err)
 		}
 	}()
 

psiphon/dataStore.go

@@ -365,13 +365,6 @@ func insertRankedServerEntry(tx *bolt.Tx, serverEntryId string, position int) er
 	return nil
 }
 
-func serverEntrySupportsProtocol(serverEntry *protocol.ServerEntry, protocol string) bool {
-	// Note: for meek, the capabilities are FRONTED-MEEK and UNFRONTED-MEEK
-	// and the additonal OSSH service is assumed to be available internally.
-	requiredCapability := strings.TrimSuffix(protocol, "-OSSH")
-	return common.Contains(serverEntry.Capabilities, requiredCapability)
-}
-
 // ServerEntryIterator is used to iterate over
 // stored server entries in rank order.
 type ServerEntryIterator struct {
@@ -573,7 +566,7 @@ func (iterator *ServerEntryIterator) Next() (serverEntry *protocol.ServerEntry,
 
 		// Check filter requirements
 		if (iterator.region == "" || serverEntry.Region == iterator.region) &&
-			(iterator.protocol == "" || serverEntrySupportsProtocol(serverEntry, iterator.protocol)) {
+			(iterator.protocol == "" || serverEntry.SupportsProtocol(iterator.protocol)) {
 
 			break
 		}
@@ -630,7 +623,7 @@ func CountServerEntries(region, tunnelProtocol string) int {
 	count := 0
 	err := scanServerEntries(func(serverEntry *protocol.ServerEntry) {
 		if (region == "" || serverEntry.Region == region) &&
-			(tunnelProtocol == "" || serverEntrySupportsProtocol(serverEntry, tunnelProtocol)) {
+			(tunnelProtocol == "" || serverEntry.SupportsProtocol(tunnelProtocol)) {
 			count += 1
 		}
 	})
@@ -643,6 +636,47 @@ func CountServerEntries(region, tunnelProtocol string) int {
 	return count
 }
 
+// CountNonImpairedProtocols returns the number of distinct tunnel
+// protocols supported by stored server entries, excluding the
+// specified impaired protocols.
+func CountNonImpairedProtocols(
+	region, tunnelProtocol string,
+	impairedProtocols []string) int {
+
+	checkInitDataStore()
+
+	distinctProtocols := make(map[string]bool)
+
+	err := scanServerEntries(func(serverEntry *protocol.ServerEntry) {
+		if region == "" || serverEntry.Region == region {
+			if tunnelProtocol != "" {
+				if serverEntry.SupportsProtocol(tunnelProtocol) {
+					distinctProtocols[tunnelProtocol] = true
+					// Exit early, since only one protocol is enabled
+					return
+				}
+			} else {
+				for _, protocol := range protocol.SupportedTunnelProtocols {
+					if serverEntry.SupportsProtocol(protocol) {
+						distinctProtocols[protocol] = true
+					}
+				}
+			}
+		}
+	})
+
+	for _, protocol := range impairedProtocols {
+		delete(distinctProtocols, protocol)
+	}
+
+	if err != nil {
+		NoticeAlert("CountNonImpairedProtocols failed: %s", err)
+		return 0
+	}
+
+	return len(distinctProtocols)
+}
+
 // ReportAvailableRegions prints a notice with the available egress regions.
 // Note that this report ignores config.TunnelProtocol.
 func ReportAvailableRegions() {

psiphon/feedback.go

@@ -113,6 +113,7 @@ func SendFeedback(configJson, diagnosticsJson, b64EncodedPublicKey, uploadServer
 		UpstreamProxyCustomHeaders:    config.UpstreamProxyCustomHeaders,
 		PendingConns:                  nil,
 		DeviceBinder:                  nil,
+		IPv6Synthesizer:               nil,
 		DnsServerGetter:               nil,
 		UseIndistinguishableTLS:       config.UseIndistinguishableTLS,
 		TrustedCACertificatesFilename: config.TrustedCACertificatesFilename,
@@ -138,20 +139,26 @@ func SendFeedback(configJson, diagnosticsJson, b64EncodedPublicKey, uploadServer
 	}
 
 	for i := 0; i < FEEDBACK_UPLOAD_MAX_RETRIES; i++ {
-		err := uploadFeedback(untunneledDialConfig, secureFeedback, url, headerPieces)
+		err = uploadFeedback(
+			untunneledDialConfig,
+			secureFeedback,
+			url,
+			MakePsiphonUserAgent(config),
+			headerPieces)
 		if err != nil {
-			NoticeAlert("failed to upload feedback: %s", err)
 			time.Sleep(FEEDBACK_UPLOAD_RETRY_DELAY_SECONDS * time.Second)
 		} else {
 			break
 		}
 	}
-	return nil
+
+	return err
 }
 
 // Attempt to upload feedback data to server.
-func uploadFeedback(config *DialConfig, feedbackData []byte, url string, headerPieces []string) error {
-	client, parsedUrl, err := MakeUntunneledHttpsClient(config, nil, url, time.Duration(FEEDBACK_UPLOAD_TIMEOUT_SECONDS*time.Second))
+func uploadFeedback(config *DialConfig, feedbackData []byte, url, userAgent string, headerPieces []string) error {
+	client, parsedUrl, err := MakeUntunneledHttpsClient(
+		config, nil, url, false, time.Duration(FEEDBACK_UPLOAD_TIMEOUT_SECONDS*time.Second))
 	if err != nil {
 		return err
 	}
@@ -160,6 +167,9 @@ func uploadFeedback(config *DialConfig, feedbackData []byte, url string, headerP
 	if err != nil {
 		return common.ContextError(err)
 	}
+
+	req.Header.Set("User-Agent", userAgent)
+
 	req.Header.Set(headerPieces[0], headerPieces[1])
 
 	resp, err := client.Do(req)

psiphon/meekConn.go

@@ -582,10 +582,6 @@ func (meek *MeekConn) roundTrip(sendPayload []byte) (io.ReadCloser, error) {
 			break
 		}
 
-		// Don't use the default user agent ("Go 1.1 package http").
-		// For now, just omit the header (net/http/request.go: "may be blank to not send the header").
-		request.Header.Set("User-Agent", "")
-
 		request.Header.Set("Content-Type", "application/octet-stream")
 
 		// Set additional headers to the HTTP request using the same method we use for adding

psiphon/net.go

@@ -79,6 +79,7 @@ type DialConfig struct {
 	// current active untunneled network DNS server.
 	DeviceBinder    DeviceBinder
 	DnsServerGetter DnsServerGetter
+	IPv6Synthesizer IPv6Synthesizer
 
 	// UseIndistinguishableTLS specifies whether to try to use an
 	// alternative stack for TLS. From a circumvention perspective,
@@ -123,6 +124,11 @@ type DnsServerGetter interface {
 	GetSecondaryDnsServer() string
 }
 
+// IPv6Synthesizer defines the interface to the external IPv6Synthesize provider
+type IPv6Synthesizer interface {
+	IPv6Synthesize(IPv4Addr string) string
+}
+
 // TimeoutError implements the error interface
 type TimeoutError struct{}
 
@@ -239,11 +245,16 @@ func MakeUntunneledHttpsClient(
 	dialConfig *DialConfig,
 	verifyLegacyCertificate *x509.Certificate,
 	requestUrl string,
+	skipVerify bool,
 	requestTimeout time.Duration) (*http.Client, string, error) {
 
 	// Change the scheme to "http"; otherwise http.Transport will try to do
 	// another TLS handshake inside the explicit TLS session. Also need to
 	// force an explicit port, as the default for "http", 80, won't talk TLS.
+	//
+	// TODO: set http.Transport.DialTLS instead of Dial to avoid this hack?
+	// See: https://golang.org/pkg/net/http/#Transport. DialTLS was added in
+	// Go 1.4 but this code may pre-date that.
 
 	urlComponents, err := url.Parse(requestUrl)
 	if err != nil {
@@ -272,7 +283,7 @@ func MakeUntunneledHttpsClient(
 			Dial: NewTCPDialer(dialConfig),
 			VerifyLegacyCertificate:       verifyLegacyCertificate,
 			SNIServerName:                 host,
-			SkipVerify:                    false,
+			SkipVerify:                    skipVerify,
 			UseIndistinguishableTLS:       useIndistinguishableTLS,
 			TrustedCACertificatesFilename: dialConfig.TrustedCACertificatesFilename,
 		})
@@ -297,6 +308,7 @@ func MakeUntunneledHttpsClient(
 func MakeTunneledHttpClient(
 	config *Config,
 	tunnel *Tunnel,
+	skipVerify bool,
 	requestTimeout time.Duration) (*http.Client, error) {
 
 	tunneledDialer := func(_, addr string) (conn net.Conn, err error) {
@@ -307,7 +319,12 @@ func MakeTunneledHttpClient(
 		Dial: tunneledDialer,
 	}
 
-	if config.UseTrustedCACertificatesForStockTLS {
+	if skipVerify {
+
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+
+	} else if config.UseTrustedCACertificatesForStockTLS {
+
 		if config.TrustedCACertificatesFilename == "" {
 			return nil, common.ContextError(errors.New(
 				"UseTrustedCACertificatesForStockTLS requires TrustedCACertificatesFilename"))
@@ -336,6 +353,7 @@ func MakeDownloadHttpClient(
 	tunnel *Tunnel,
 	untunneledDialConfig *DialConfig,
 	requestUrl string,
+	skipVerify bool,
 	requestTimeout time.Duration) (*http.Client, string, error) {
 
 	var httpClient *http.Client
@@ -343,7 +361,8 @@ func MakeDownloadHttpClient(
 
 	if tunnel != nil {
 		// MakeTunneledHttpClient works with both "http" and "https" schemes
-		httpClient, err = MakeTunneledHttpClient(config, tunnel, requestTimeout)
+		httpClient, err = MakeTunneledHttpClient(
+			config, tunnel, skipVerify, requestTimeout)
 		if err != nil {
 			return nil, "", common.ContextError(err)
 		}
@@ -355,7 +374,7 @@ func MakeDownloadHttpClient(
 		// MakeUntunneledHttpsClient works only with "https" schemes
 		if urlComponents.Scheme == "https" {
 			httpClient, requestUrl, err = MakeUntunneledHttpsClient(
-				untunneledDialConfig, nil, requestUrl, requestTimeout)
+				untunneledDialConfig, nil, requestUrl, skipVerify, requestTimeout)
 			if err != nil {
 				return nil, "", common.ContextError(err)
 			}
@@ -389,6 +408,7 @@ func MakeDownloadHttpClient(
 func ResumeDownload(
 	httpClient *http.Client,
 	requestUrl string,
+	userAgent string,
 	downloadFilename string,
 	ifNoneMatchETag string) (int64, string, error) {
 
@@ -432,6 +452,8 @@ func ResumeDownload(
 		return 0, "", common.ContextError(err)
 	}
 
+	request.Header.Set("User-Agent", userAgent)
+
 	request.Header.Add("Range", fmt.Sprintf("bytes=%d-", fileInfo.Size()))
 
 	if partialETag != nil {
@@ -467,6 +489,10 @@ func ResumeDownload(
 	// receive 412 on ETag mismatch.
 	if err == nil &&
 		(response.StatusCode != http.StatusPartialContent &&
+
+			// Certain http servers return 200 OK where we expect 206, so accept that.
+			response.StatusCode != http.StatusOK &&
+
 			response.StatusCode != http.StatusRequestedRangeNotSatisfiable &&
 			response.StatusCode != http.StatusPreconditionFailed &&
 			response.StatusCode != http.StatusNotModified) {

psiphon/notice.go

@@ -351,7 +351,9 @@ func NoticeConnectedTunnelDialStats(ipAddress string, tunnelDialStats *TunnelDia
 		"meekResolvedIPAddress", tunnelDialStats.MeekResolvedIPAddress,
 		"meekSNIServerName", tunnelDialStats.MeekSNIServerName,
 		"meekHostHeader", tunnelDialStats.MeekHostHeader,
-		"meekTransformedHostName", tunnelDialStats.MeekTransformedHostName)
+		"meekTransformedHostName", tunnelDialStats.MeekTransformedHostName,
+		"selectedUserAgent", tunnelDialStats.SelectedUserAgent,
+		"userAgent", tunnelDialStats.UserAgent)
 }
 
 // NoticeBuildInfo reports build version info.

psiphon/remoteServerList.go

@@ -34,7 +34,7 @@ import (
 )
 
 type RemoteServerListFetcher func(
-	config *Config, tunnel *Tunnel, untunneledDialConfig *DialConfig) error
+	config *Config, attempt int, tunnel *Tunnel, untunneledDialConfig *DialConfig) error
 
 // FetchCommonRemoteServerList downloads the common remote server list from
 // config.RemoteServerListUrl. It validates its digital signature using the
@@ -45,16 +45,21 @@ type RemoteServerListFetcher func(
 // be unique and persistent.
 func FetchCommonRemoteServerList(
 	config *Config,
+	attempt int,
 	tunnel *Tunnel,
 	untunneledDialConfig *DialConfig) error {
 
 	NoticeInfo("fetching common remote server list")
 
+	downloadURL, canonicalURL, skipVerify := selectDownloadURL(attempt, config.RemoteServerListURLs)
+
 	newETag, err := downloadRemoteServerListFile(
 		config,
 		tunnel,
 		untunneledDialConfig,
-		config.RemoteServerListUrl,
+		downloadURL,
+		canonicalURL,
+		skipVerify,
 		"",
 		config.RemoteServerListDownloadFilename)
 	if err != nil {
@@ -78,7 +83,7 @@ func FetchCommonRemoteServerList(
 
 	// Now that the server entries are successfully imported, store the response
 	// ETag so we won't re-download this same data again.
-	err = SetUrlETag(config.RemoteServerListUrl, newETag)
+	err = SetUrlETag(canonicalURL, newETag)
 	if err != nil {
 		NoticeAlert("failed to set ETag for common remote server list: %s", common.ContextError(err))
 		// This fetch is still reported as a success, even if we can't store the etag
@@ -100,13 +105,17 @@ func FetchCommonRemoteServerList(
 // must be unique and persistent.
 func FetchObfuscatedServerLists(
 	config *Config,
+	attempt int,
 	tunnel *Tunnel,
 	untunneledDialConfig *DialConfig) error {
 
 	NoticeInfo("fetching obfuscated remote server lists")
 
 	downloadFilename := osl.GetOSLRegistryFilename(config.ObfuscatedServerListDownloadDirectory)
-	downloadURL := osl.GetOSLRegistryURL(config.ObfuscatedServerListRootURL)
+
+	rootURL, canonicalRootURL, skipVerify := selectDownloadURL(attempt, config.ObfuscatedServerListRootURLs)
+	downloadURL := osl.GetOSLRegistryURL(rootURL)
+	canonicalURL := osl.GetOSLRegistryURL(canonicalRootURL)
 
 	// failed is set if any operation fails and should trigger a retry. When the OSL registry
 	// fails to download, any cached registry is used instead; when any single OSL fails
@@ -122,6 +131,8 @@ func FetchObfuscatedServerLists(
 		tunnel,
 		untunneledDialConfig,
 		downloadURL,
+		canonicalURL,
+		skipVerify,
 		"",
 		downloadFilename)
 	if err != nil {
@@ -173,7 +184,7 @@ func FetchObfuscatedServerLists(
 	// When a new registry is downloaded, validated, and parsed, store the
 	// response ETag so we won't re-download this same data again.
 	if !failed && newETag != "" {
-		err = SetUrlETag(downloadURL, newETag)
+		err = SetUrlETag(canonicalURL, newETag)
 		if err != nil {
 			NoticeAlert("failed to set ETag for obfuscated server list registry: %s", common.ContextError(err))
 			// This fetch is still reported as a success, even if we can't store the etag
@@ -199,16 +210,20 @@ func FetchObfuscatedServerLists(
 		})
 
 	for _, oslID := range oslIDs {
+
 		downloadFilename := osl.GetOSLFilename(config.ObfuscatedServerListDownloadDirectory, oslID)
-		downloadURL := osl.GetOSLFileURL(config.ObfuscatedServerListRootURL, oslID)
+
+		downloadURL := osl.GetOSLFileURL(rootURL, oslID)
+		canonicalURL := osl.GetOSLFileURL(canonicalRootURL, oslID)
+
 		hexID := hex.EncodeToString(oslID)
 
 		// Note: the MD5 checksum step assumes the remote server list host's ETag uses MD5
-		// with a hex encoding. If this is not the case, the remoteETag should be left blank.
-		remoteETag := ""
+		// with a hex encoding. If this is not the case, the sourceETag should be left blank.
+		sourceETag := ""
 		md5sum, err := oslRegistry.GetOSLMD5Sum(oslID)
 		if err == nil {
-			remoteETag = hex.EncodeToString(md5sum)
+			sourceETag = fmt.Sprintf("\"%s\"", hex.EncodeToString(md5sum))
 		}
 
 		// TODO: store ETags in OSL registry to enable skipping requests entirely
@@ -218,7 +233,9 @@ func FetchObfuscatedServerLists(
 			tunnel,
 			untunneledDialConfig,
 			downloadURL,
-			remoteETag,
+			canonicalURL,
+			skipVerify,
+			sourceETag,
 			downloadFilename)
 		if err != nil {
 			failed = true
@@ -255,7 +272,7 @@ func FetchObfuscatedServerLists(
 
 		// Now that the server entries are successfully imported, store the response
 		// ETag so we won't re-download this same data again.
-		err = SetUrlETag(downloadURL, newETag)
+		err = SetUrlETag(canonicalURL, newETag)
 		if err != nil {
 			failed = true
 			NoticeAlert("failed to set Etag for obfuscated server list file (%s): %s", hexID, common.ContextError(err))
@@ -280,14 +297,20 @@ func downloadRemoteServerListFile(
 	config *Config,
 	tunnel *Tunnel,
 	untunneledDialConfig *DialConfig,
-	sourceURL, sourceETag, destinationFilename string) (string, error) {
-
-	lastETag, err := GetUrlETag(sourceURL)
+	sourceURL string,
+	canonicalURL string,
+	skipVerify bool,
+	sourceETag string,
+	destinationFilename string) (string, error) {
+
+	// All download URLs with the same canonicalURL
+	// must have the same entity and ETag.
+	lastETag, err := GetUrlETag(canonicalURL)
 	if err != nil {
 		return "", common.ContextError(err)
 	}
 
-	// sourceETag, when specified, is prior knowlegde of the
+	// sourceETag, when specified, is prior knowledge of the
 	// remote ETag that can be used to skip the request entirely.
 	// This will be set in the case of OSL files, from the MD5Sum
 	// values stored in the registry.
@@ -304,13 +327,18 @@ func downloadRemoteServerListFile(
 		tunnel,
 		untunneledDialConfig,
 		sourceURL,
+		skipVerify,
 		time.Duration(*config.FetchRemoteServerListTimeoutSeconds)*time.Second)
 	if err != nil {
 		return "", common.ContextError(err)
 	}
 
 	n, responseETag, err := ResumeDownload(
-		httpClient, requestURL, destinationFilename, lastETag)
+		httpClient,
+		requestURL,
+		MakePsiphonUserAgent(config),
+		destinationFilename,
+		lastETag)
 
 	NoticeRemoteServerListResourceDownloadedBytes(sourceURL, n)
 

psiphon/remoteServerList_test.go

@@ -22,6 +22,7 @@ package psiphon
 import (
 	"bytes"
 	"crypto/md5"
+	"encoding/base64"
 	"encoding/hex"
 	"fmt"
 	"io"
@@ -137,15 +138,30 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 		t.Fatalf("error generating package keys: %s", err)
 	}
 
+	// First Pave() call is to get the OSL ID to pave into
+
+	oslID := ""
+
 	paveFiles, err := oslConfig.Pave(
 		epoch,
 		propagationChannelID,
 		signingPublicKey,
 		signingPrivateKey,
-		[]map[time.Time]string{
-			map[time.Time]string{
-				epoch: string(encodedServerEntry),
-			},
+		map[string][]string{},
+		func(logInfo *osl.PaveLogInfo) {
+			oslID = logInfo.OSLID
+		})
+	if err != nil {
+		t.Fatalf("error paving OSL files: %s", err)
+	}
+
+	paveFiles, err = oslConfig.Pave(
+		epoch,
+		propagationChannelID,
+		signingPublicKey,
+		signingPrivateKey,
+		map[string][]string{
+			oslID: []string{string(encodedServerEntry)},
 		},
 		nil)
 	if err != nil {
@@ -182,38 +198,55 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 	// run mock remote server list host
 	//
 
-	remoteServerListHostAddress := net.JoinHostPort(serverIPaddress, "8081")
+	// Exercise using multiple download URLs
+	remoteServerListHostAddresses := []string{
+		net.JoinHostPort(serverIPaddress, "8081"),
+		net.JoinHostPort(serverIPaddress, "8082"),
+	}
 
 	// The common remote server list fetches will 404
-	remoteServerListURL := fmt.Sprintf("http://%s/server_list_compressed", remoteServerListHostAddress)
+	remoteServerListURL := fmt.Sprintf("http://%s/server_list_compressed", remoteServerListHostAddresses[0])
 	remoteServerListDownloadFilename := filepath.Join(testDataDirName, "server_list_compressed")
 
-	obfuscatedServerListRootURL := fmt.Sprintf("http://%s/", remoteServerListHostAddress)
-	obfuscatedServerListDownloadDirectory := testDataDirName
+	obfuscatedServerListRootURLsJSONConfig := "["
+	obfuscatedServerListRootURLs := make([]string, len(remoteServerListHostAddresses))
+	for i := 0; i < len(remoteServerListHostAddresses); i++ {
 
-	go func() {
-		startTime := time.Now()
-		serveMux := http.NewServeMux()
-		for _, paveFile := range paveFiles {
-			file := paveFile
-			serveMux.HandleFunc("/"+file.Name, func(w http.ResponseWriter, req *http.Request) {
-				md5sum := md5.Sum(file.Contents)
-				w.Header().Add("Content-Type", "application/octet-stream")
-				w.Header().Add("ETag", hex.EncodeToString(md5sum[:]))
-				http.ServeContent(w, req, file.Name, startTime, bytes.NewReader(file.Contents))
-			})
-		}
-		httpServer := &http.Server{
-			Addr:    remoteServerListHostAddress,
-			Handler: serveMux,
-		}
-		err := httpServer.ListenAndServe()
-		if err != nil {
-			// TODO: wrong goroutine for t.FatalNow()
-			t.Fatalf("error running remote server list host: %s", err)
+		obfuscatedServerListRootURLs[i] = fmt.Sprintf("http://%s/", remoteServerListHostAddresses[i])
 
+		obfuscatedServerListRootURLsJSONConfig += fmt.Sprintf(
+			"{\"URL\" : \"%s\"}", base64.StdEncoding.EncodeToString([]byte(obfuscatedServerListRootURLs[i])))
+		if i == len(remoteServerListHostAddresses)-1 {
+			obfuscatedServerListRootURLsJSONConfig += "]"
+		} else {
+			obfuscatedServerListRootURLsJSONConfig += ","
 		}
-	}()
+
+		go func(remoteServerListHostAddress string) {
+			startTime := time.Now()
+			serveMux := http.NewServeMux()
+			for _, paveFile := range paveFiles {
+				file := paveFile
+				serveMux.HandleFunc("/"+file.Name, func(w http.ResponseWriter, req *http.Request) {
+					md5sum := md5.Sum(file.Contents)
+					w.Header().Add("Content-Type", "application/octet-stream")
+					w.Header().Add("ETag", fmt.Sprintf("\"%s\"", hex.EncodeToString(md5sum[:])))
+					http.ServeContent(w, req, file.Name, startTime, bytes.NewReader(file.Contents))
+				})
+			}
+			httpServer := &http.Server{
+				Addr:    remoteServerListHostAddress,
+				Handler: serveMux,
+			}
+			err := httpServer.ListenAndServe()
+			if err != nil {
+				// TODO: wrong goroutine for t.FatalNow()
+				t.Fatalf("error running remote server list host: %s", err)
+			}
+		}(remoteServerListHostAddresses[i])
+	}
+
+	obfuscatedServerListDownloadDirectory := testDataDirName
 
 	//
 	// run Psiphon server
@@ -237,24 +270,24 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 	go func() {
 		listener, err := socks.ListenSocks("tcp", disruptorProxyAddress)
 		if err != nil {
-			fmt.Errorf("disruptor proxy listen error: %s", err)
+			fmt.Printf("disruptor proxy listen error: %s\n", err)
 			return
 		}
 		for {
 			localConn, err := listener.AcceptSocks()
 			if err != nil {
-				fmt.Errorf("disruptor proxy accept error: %s", err)
+				fmt.Printf("disruptor proxy accept error: %s\n", err)
 				return
 			}
 			go func() {
 				remoteConn, err := net.Dial("tcp", localConn.Req.Target)
 				if err != nil {
-					fmt.Errorf("disruptor proxy dial error: %s", err)
+					fmt.Printf("disruptor proxy dial error: %s\n", err)
 					return
 				}
 				err = localConn.Grant(&net.TCPAddr{IP: net.ParseIP("0.0.0.0"), Port: 0})
 				if err != nil {
-					fmt.Errorf("disruptor proxy grant error: %s", err)
+					fmt.Printf("disruptor proxy grant error: %s\n", err)
 					return
 				}
 
@@ -264,7 +297,7 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 					defer waitGroup.Done()
 					io.Copy(remoteConn, localConn)
 				}()
-				if localConn.Req.Target == remoteServerListHostAddress {
+				if common.Contains(remoteServerListHostAddresses, localConn.Req.Target) {
 					io.CopyN(localConn, remoteConn, 500)
 				} else {
 					io.Copy(localConn, remoteConn)
@@ -295,7 +328,7 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 		"RemoteServerListSignaturePublicKey" : "%s",
 		"RemoteServerListUrl" : "%s",
 		"RemoteServerListDownloadFilename" : "%s",
-		"ObfuscatedServerListRootURL" : "%s",
+		"ObfuscatedServerListRootURLs" : %s,
 		"ObfuscatedServerListDownloadDirectory" : "%s",
 		"UpstreamProxyUrl" : "%s"
     }`
@@ -305,7 +338,7 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 		signingPublicKey,
 		remoteServerListURL,
 		remoteServerListDownloadFilename,
-		obfuscatedServerListRootURL,
+		obfuscatedServerListRootURLsJSONConfig,
 		obfuscatedServerListDownloadDirectory,
 		disruptorProxyURL)
 
@@ -360,11 +393,11 @@ func TestObfuscatedRemoteServerLists(t *testing.T) {
 	}
 
 	for _, paveFile := range paveFiles {
-		u, _ := url.Parse(obfuscatedServerListRootURL)
+		u, _ := url.Parse(obfuscatedServerListRootURLs[0])
 		u.Path = path.Join(u.Path, paveFile.Name)
 		etag, _ := GetUrlETag(u.String())
 		md5sum := md5.Sum(paveFile.Contents)
-		if etag != hex.EncodeToString(md5sum[:]) {
+		if etag != fmt.Sprintf("\"%s\"", hex.EncodeToString(md5sum[:])) {
 			t.Fatalf("unexpected ETag for %s", u)
 		}
 	}

psiphon/server/geoip.go

@@ -145,7 +145,7 @@ func (geoIP *GeoIPService) Lookup(ipAddress string) GeoIPData {
 	}
 
 	// Each database will populate geoIPFields with the values it contains. In the
-	// currnt MaxMind deployment, the City database populates Country and City and
+	// current MaxMind deployment, the City database populates Country and City and
 	// the separate ISP database populates ISP.
 	for _, database := range geoIP.databases {
 		database.ReloadableFile.RLock()

psiphon/server/psinet/psinet.go

@@ -353,7 +353,17 @@ func selectServers(servers []Server, discoveryValue int) []Server {
 	bucketCount := calculateBucketCount(len(servers))
 
 	buckets := bucketizeServerList(servers, bucketCount)
+
+	if len(buckets) == 0 {
+		return nil
+	}
+
 	bucket := buckets[discoveryValue%len(buckets)]
+
+	if len(bucket) == 0 {
+		return nil
+	}
+
 	server := bucket[timeStrategyValue%len(bucket)]
 
 	serverList := make([]Server, 1)
@@ -368,24 +378,14 @@ func calculateBucketCount(length int) int {
 	return int(math.Ceil(math.Sqrt(float64(length))))
 }
 
-// Create bucketCount buckets.
-// Each bucket will be of size division or divison-1.
+// Create bucketCount nearly equal sized buckets.
 func bucketizeServerList(servers []Server, bucketCount int) [][]Server {
-	division := float64(len(servers)) / float64(bucketCount)
 
 	buckets := make([][]Server, bucketCount)
 
-	var currentBucketIndex int = 0
-	var serverIndex int = 0
-	for _, server := range servers {
-		bucketEndIndex := int(math.Floor(division * (float64(currentBucketIndex) + 1)))
-
-		buckets[currentBucketIndex] = append(buckets[currentBucketIndex], server)
-
-		serverIndex++
-		if serverIndex > bucketEndIndex {
-			currentBucketIndex++
-		}
+	for index, server := range servers {
+		bucketIndex := index % bucketCount
+		buckets[bucketIndex] = append(buckets[bucketIndex], server)
 	}
 
 	return buckets

psiphon/server/server_test.go

@@ -49,7 +49,7 @@ func TestMain(m *testing.M) {
 	var err error
 	testDataDirName, err = ioutil.TempDir("", "psiphon-server-test")
 	if err != nil {
-		fmt.Printf("TempDir failed: %s", err)
+		fmt.Printf("TempDir failed: %s\n", err)
 		os.Exit(1)
 	}
 	defer os.RemoveAll(testDataDirName)

psiphon/serverApi.go

@@ -525,6 +525,7 @@ func (serverContext *ServerContext) doUntunneledStatusRequest(
 		dialConfig,
 		certificate,
 		url,
+		false,
 		timeout)
 	if err != nil {
 		return common.ContextError(err)
@@ -742,7 +743,14 @@ func (serverContext *ServerContext) DoClientVerificationRequest(
 func (serverContext *ServerContext) doGetRequest(
 	requestUrl string) (responseBody []byte, err error) {
 
-	response, err := serverContext.psiphonHttpsClient.Get(requestUrl)
+	request, err := http.NewRequest("GET", requestUrl, nil)
+	if err != nil {
+		return nil, common.ContextError(err)
+	}
+
+	request.Header.Set("User-Agent", MakePsiphonUserAgent(serverContext.tunnel.config))
+
+	response, err := serverContext.psiphonHttpsClient.Do(request)
 	if err == nil && response.StatusCode != http.StatusOK {
 		response.Body.Close()
 		err = fmt.Errorf("HTTP GET request failed with response code: %d", response.StatusCode)
@@ -763,7 +771,15 @@ func (serverContext *ServerContext) doGetRequest(
 func (serverContext *ServerContext) doPostRequest(
 	requestUrl string, bodyType string, body io.Reader) (responseBody []byte, err error) {
 
-	response, err := serverContext.psiphonHttpsClient.Post(requestUrl, bodyType, body)
+	request, err := http.NewRequest("POST", requestUrl, body)
+	if err != nil {
+		return nil, common.ContextError(err)
+	}
+
+	request.Header.Set("User-Agent", MakePsiphonUserAgent(serverContext.tunnel.config))
+	request.Header.Set("Content-Type", bodyType)
+
+	response, err := serverContext.psiphonHttpsClient.Do(request)
 	if err == nil && response.StatusCode != http.StatusOK {
 		response.Body.Close()
 		err = fmt.Errorf("HTTP POST request failed with response code: %d", response.StatusCode)
@@ -832,6 +848,10 @@ func (serverContext *ServerContext) getBaseParams() requestJSONObject {
 			transformedHostName = "1"
 		}
 		params["meek_transformed_host_name"] = transformedHostName
+
+		if tunnel.dialStats.SelectedUserAgent {
+			params["user_agent"] = tunnel.dialStats.UserAgent
+		}
 	}
 
 	if tunnel.serverEntry.Region != "" {

psiphon/splitTunnel.go

@@ -69,6 +69,7 @@ import (
 type SplitTunnelClassifier struct {
 	mutex                    sync.RWMutex
 	fetchRoutesUrlFormat     string
+	userAgent                string
 	routesSignaturePublicKey string
 	dnsServerAddress         string
 	dnsTunneler              Tunneler
@@ -86,6 +87,7 @@ type classification struct {
 func NewSplitTunnelClassifier(config *Config, tunneler Tunneler) *SplitTunnelClassifier {
 	return &SplitTunnelClassifier{
 		fetchRoutesUrlFormat:     config.SplitTunnelRoutesUrlFormat,
+		userAgent:                MakePsiphonUserAgent(config),
 		routesSignaturePublicKey: config.SplitTunnelRoutesSignaturePublicKey,
 		dnsServerAddress:         config.SplitTunnelDnsServer,
 		dnsTunneler:              tunneler,
@@ -221,6 +223,8 @@ func (classifier *SplitTunnelClassifier) getRoutes(tunnel *Tunnel) (routesData [
 		return nil, common.ContextError(err)
 	}
 
+	request.Header.Set("User-Agent", classifier.userAgent)
+
 	etag, err := GetSplitTunnelRoutesETag(tunnel.serverContext.clientRegion)
 	if err != nil {
 		return nil, common.ContextError(err)

psiphon/tunnel.go

@@ -103,6 +103,8 @@ type TunnelDialStats struct {
 	MeekSNIServerName              string
 	MeekHostHeader                 string
 	MeekTransformedHostName        bool
+	SelectedUserAgent              bool
+	UserAgent                      string
 }
 
 // EstablishTunnel first makes a network transport connection to the
@@ -582,8 +584,10 @@ func dialSsh(
 	// So depending on which protocol is used, multiple layers are initialized.
 
 	useObfuscatedSsh := false
+	dialHeaders := config.UpstreamProxyCustomHeaders
 	var directTCPDialAddress string
 	var meekConfig *MeekConfig
+	var selectedUserAgent bool
 	var err error
 
 	switch selectedProtocol {
@@ -596,6 +600,8 @@ func dialSsh(
 
 	default:
 		useObfuscatedSsh = true
+		dialHeaders, selectedUserAgent = common.UserAgentIfUnset(config.UpstreamProxyCustomHeaders)
+
 		meekConfig, err = initMeekConfig(config, serverEntry, selectedProtocol, sessionId)
 		if err != nil {
 			return nil, common.ContextError(err)
@@ -624,11 +630,12 @@ func dialSsh(
 	// Create the base transport: meek or direct connection
 	dialConfig := &DialConfig{
 		UpstreamProxyUrl:              config.UpstreamProxyUrl,
-		UpstreamProxyCustomHeaders:    config.UpstreamProxyCustomHeaders,
+		UpstreamProxyCustomHeaders:    dialHeaders,
 		ConnectTimeout:                time.Duration(*config.TunnelConnectTimeoutSeconds) * time.Second,
 		PendingConns:                  pendingConns,
 		DeviceBinder:                  config.DeviceBinder,
 		DnsServerGetter:               config.DnsServerGetter,
+		IPv6Synthesizer:               config.IPv6Synthesizer,
 		UseIndistinguishableTLS:       config.UseIndistinguishableTLS,
 		TrustedCACertificatesFilename: config.TrustedCACertificatesFilename,
 		DeviceRegion:                  config.DeviceRegion,
@@ -753,6 +760,11 @@ func dialSsh(
 	if dialConfig.UpstreamProxyUrl != "" || meekConfig != nil {
 		dialStats = &TunnelDialStats{}
 
+		if selectedUserAgent {
+			dialStats.SelectedUserAgent = true
+			dialStats.UserAgent = dialConfig.UpstreamProxyCustomHeaders.Get("User-Agent")
+		}
+
 		if dialConfig.UpstreamProxyUrl != "" {
 
 			// Note: UpstreamProxyUrl should have parsed correctly in the dial
@@ -763,6 +775,9 @@ func dialSsh(
 
 			dialStats.UpstreamProxyCustomHeaderNames = make([]string, 0)
 			for name, _ := range dialConfig.UpstreamProxyCustomHeaders {
+				if selectedUserAgent && name == "User-Agent" {
+					continue
+				}
 				dialStats.UpstreamProxyCustomHeaderNames = append(dialStats.UpstreamProxyCustomHeaderNames, name)
 			}
 		}

psiphon/upgradeDownload.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"os"
 	"strconv"
+	"time"
 
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
 )
@@ -55,10 +56,14 @@ import (
 // upgrade is still pending install by the outer client.
 func DownloadUpgrade(
 	config *Config,
+	attempt int,
 	handshakeVersion string,
 	tunnel *Tunnel,
 	untunneledDialConfig *DialConfig) error {
 
+	// Note: this downloader doesn't use ETags since many client binaries, with
+	// different embedded values, exist for a single version.
+
 	// Check if complete file already downloaded
 
 	if _, err := os.Stat(config.UpgradeDownloadFilename); err == nil {
@@ -68,12 +73,15 @@ func DownloadUpgrade(
 
 	// Select tunneled or untunneled configuration
 
+	downloadURL, _, skipVerify := selectDownloadURL(attempt, config.UpgradeDownloadURLs)
+
 	httpClient, requestUrl, err := MakeDownloadHttpClient(
 		config,
 		tunnel,
 		untunneledDialConfig,
-		config.UpgradeDownloadUrl,
-		DOWNLOAD_UPGRADE_TIMEOUT)
+		downloadURL,
+		skipVerify,
+		time.Duration(*config.DownloadUpgradeTimeoutSeconds)*time.Second)
 
 	// If no handshake version is supplied, make an initial HEAD request
 	// to get the current version from the version header.
@@ -131,7 +139,11 @@ func DownloadUpgrade(
 		"%s.%s", config.UpgradeDownloadFilename, availableClientVersion)
 
 	n, _, err := ResumeDownload(
-		httpClient, requestUrl, downloadFilename, "")
+		httpClient,
+		requestUrl,
+		MakePsiphonUserAgent(config),
+		downloadFilename,
+		"")
 
 	NoticeClientUpgradeDownloadedBytes(n)
 

psiphon/utils.go

@@ -32,6 +32,21 @@ import (
 	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
 )
 
+// MakePsiphonUserAgent constructs a User-Agent value to use for web service
+// requests made by the tunnel-core client. The User-Agent includes useful stats
+// information; it is to be used only for HTTPS requests, where the header
+// cannot be seen by an adversary.
+func MakePsiphonUserAgent(config *Config) string {
+	userAgent := "psiphon-tunnel-core"
+	if config.ClientVersion != "" {
+		userAgent += fmt.Sprintf("/%s", config.ClientVersion)
+	}
+	if config.ClientPlatform != "" {
+		userAgent += fmt.Sprintf(" (%s)", config.ClientPlatform)
+	}
+	return userAgent
+}
+
 func DecodeCertificate(encodedCertificate string) (certificate *x509.Certificate, err error) {
 	derEncodedCertificate, err := base64.StdEncoding.DecodeString(encodedCertificate)
 	if err != nil {