Merge pull request #314 from rod-hynes/master

Build script changes

ConsoleClient/Dockerfile

@@ -30,8 +30,7 @@ RUN curl -L https://storage.googleapis.com/golang/$GOVERSION.linux-amd64.tar.gz
    && echo $GOVERSION > $GOROOT/VERSION
 
 # Get external Go dependencies.
-RUN go get github.com/mitchellh/gox \
-    && go get github.com/pwaller/goupx
+RUN go get github.com/pwaller/goupx
 
 # Setup OpenSSL libray.
 ENV OPENSSL_VERSION=1.0.2h

ConsoleClient/make.bash

@@ -32,7 +32,7 @@ prepare_build () {
   # - pipes to `xargs` again, specifiying `pkg` as the placeholder name for each item being operated on (which is the list of non standard library import paths from the previous step)
   #  - `xargs` runs a bash script (via `-c`) which changes to each import path in sequence, then echoes out `"<import path>":"<subshell output of getting the short git revision>",`
   # - this leaves a trailing `,` at the end, and no close to the JSON object, so simply `sed` replace the comma before the end of the line with `}` and you now have valid JSON
-  DEPENDENCIES=$(echo -n "{" && go list -tags "${BUILD_TAGS}" -f '{{range $dep := .Deps}}{{printf "%s\n" $dep}}{{end}}' | xargs go list -f '{{if not .Standard}}{{.ImportPath}}{{end}}' | xargs -I pkg bash -c 'cd $GOPATH/src/pkg && echo -n "\"pkg\":\"$(git rev-parse --short HEAD)\","' | sed 's/,$/}/')
+  DEPENDENCIES=$(echo -n "{" && go list -tags "$1" -f '{{range $dep := .Deps}}{{printf "%s\n" $dep}}{{end}}' | xargs go list -f '{{if not .Standard}}{{.ImportPath}}{{end}}' | xargs -I pkg bash -c 'cd $GOPATH/src/pkg && echo -n "\"pkg\":\"$(git rev-parse --short HEAD)\","' | sed 's/,$/}/')
 
   LDFLAGS="\
   -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common.buildDate=$BUILDDATE \
@@ -59,7 +59,7 @@ fi
 build_for_windows () {
   echo "...Getting project dependencies (via go get) for Windows. Parameter is: '$1'"
   GOOS=windows go get -d -v -tags "$WINDOWS_BUILD_TAGS" ./...
-  prepare_build
+  prepare_build $WINDOWS_BUILD_TAGS
   if [ $? != 0 ]; then
     echo "....'go get' failed, exiting"
     exit $?
@@ -75,7 +75,7 @@ build_for_windows () {
     CGO_CFLAGS="-I $PKG_CONFIG_PATH/include/" \
     CGO_LDFLAGS="-L $PKG_CONFIG_PATH -L /usr/i686-w64-mingw32/lib/ -lssl -lcrypto -lwsock32 -lcrypt32 -lgdi32" \
     CC=/usr/bin/i686-w64-mingw32-gcc \
-    gox -verbose -ldflags "$LDFLAGS" -osarch windows/386 -tags "$WINDOWS_BUILD_TAGS" -output bin/windows/${EXE_BASENAME}-i686
+    GOOS=windows GOARCH=386 go build -v -x -ldflags "$LDFLAGS" -tags "$WINDOWS_BUILD_TAGS" -o bin/windows/${EXE_BASENAME}-i686
     RETVAL=$?
     echo ".....gox completed, exit code: $?"
     if [ $RETVAL != 0 ]; then
@@ -98,7 +98,7 @@ build_for_windows () {
     CGO_CFLAGS="-I $PKG_CONFIG_PATH/include/" \
     CGO_LDFLAGS="-L $PKG_CONFIG_PATH -L /usr/x86_64-w64-mingw32/lib/ -lssl -lcrypto -lwsock32 -lcrypt32 -lgdi32" \
     CC=/usr/bin/x86_64-w64-mingw32-gcc \
-    gox -verbose -ldflags "$LDFLAGS" -osarch windows/amd64 -tags "$WINDOWS_BUILD_TAGS" -output bin/windows/${EXE_BASENAME}-x86_64
+    GOOS=windows GOARCH=amd64 go build -v -x -ldflags "$LDFLAGS" -tags "$WINDOWS_BUILD_TAGS" -o bin/windows/${EXE_BASENAME}-x86_64
     RETVAL=$?
     if [ $RETVAL != 0 ]; then
       echo ".....gox failed, exiting"
@@ -114,7 +114,7 @@ build_for_windows () {
 build_for_linux () {
   echo "Getting project dependencies (via go get) for Linux. Parameter is: '$1'"
   GOOS=linux go get -d -v -tags "$LINUX_BUILD_TAGS" ./...
-  prepare_build
+  prepare_build $LINUX_BUILD_TAGS
   if [ $? != 0 ]; then
     echo "...'go get' failed, exiting"
     exit $?
@@ -122,7 +122,8 @@ build_for_linux () {
 
   if [ -z $1 ] || [ "$1" == "32" ]; then
     echo "...Building linux-i686"
-    CFLAGS=-m32 gox -verbose -ldflags "$LDFLAGS" -osarch linux/386 -tags "$LINUX_BUILD_TAGS" -output bin/linux/${EXE_BASENAME}-i686
+    # TODO: is "CFLAGS=-m32" required?
+    CFLAGS=-m32 GOOS=linux GOARCH=386 go build -v -x -ldflags "$LDFLAGS" -tags "$LINUX_BUILD_TAGS" -o bin/linux/${EXE_BASENAME}-i686
     RETVAL=$?
     if [ $RETVAL != 0 ]; then
       echo ".....gox failed, exiting"
@@ -142,7 +143,7 @@ build_for_linux () {
 
   if [ -z $1 ] || [ "$1" == "64" ]; then
     echo "...Building linux-x86_64"
-    gox -verbose -ldflags "$LDFLAGS" -osarch linux/amd64 -tags "$LINUX_BUILD_TAGS" -output bin/linux/${EXE_BASENAME}-x86_64
+    GOOS=linux GOARCH=amd64 go build -v -x -ldflags "$LDFLAGS" -tags "$LINUX_BUILD_TAGS" -o bin/linux/${EXE_BASENAME}-x86_64
     RETVAL=$?
     if [ $RETVAL != 0 ]; then
       echo "....gox failed, exiting"
@@ -164,7 +165,7 @@ build_for_linux () {
 build_for_osx () {
   echo "Getting project dependencies (via go get) for OSX"
   GOOS=darwin go get -d -v -tags "$OSX_BUILD_TAGS" ./...
-  prepare_build
+  prepare_build $OSX_BUILD_TAGS
   if [ $? != 0 ]; then
     echo "..'go get' failed, exiting"
     exit $?
@@ -172,7 +173,8 @@ build_for_osx () {
 
   echo "Building darwin-x86_64..."
   echo "..Disabling CGO for this build"
-  CGO_ENABLED=0 gox -verbose -ldflags "$LDFLAGS" -osarch darwin/amd64 -tags "$OSX_BUILD_TAGS" -output bin/darwin/${EXE_BASENAME}-x86_64
+  # TODO: is "CGO_ENABLED=0" required?
+  CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -v -x -ldflags "$LDFLAGS" -tags "$OSX_BUILD_TAGS" -o bin/darwin/${EXE_BASENAME}-x86_64
   # Darwin binaries don't seem to be UPXable when built this way
   echo "..No UPX for this build"
 }

MobileLibrary/Android/make.bash

@@ -55,7 +55,7 @@ echo " Gomobile version: ${GOMOBILEVERSION}"
 echo " Dependencies: ${DEPENDENCIES}"
 echo ""
 
-gomobile bind -v -target=android/arm -tags="${BUILD_TAGS}" -ldflags="$LDFLAGS" github.com/Psiphon-Labs/psiphon-tunnel-core/MobileLibrary/psi
+gomobile bind -v -x -target=android/arm -tags="${BUILD_TAGS}" -ldflags="$LDFLAGS" github.com/Psiphon-Labs/psiphon-tunnel-core/MobileLibrary/psi
 if [ $? != 0 ]; then
   echo "..'gomobile bind' failed, exiting"
   exit $?

Server/make.bash

@@ -57,7 +57,7 @@ build_for_linux () {
     exit $?
   fi
 
-  GOOS=linux GOARCH=amd64 go build -tags "${BUILD_TAGS}" -ldflags "$LDFLAGS" -o psiphond
+  GOOS=linux GOARCH=amd64 go build -v -x -tags "${BUILD_TAGS}" -ldflags "$LDFLAGS" -o psiphond
   if [ $? != 0 ]; then
     echo "...'go build' failed, exiting"
     exit $?

psiphon/common/osl/osl.go

@@ -30,8 +30,6 @@
 package osl
 
 import (
-	"bytes"
-	"compress/zlib"
 	"crypto/hmac"
 	"crypto/md5"
 	"crypto/sha256"
@@ -41,7 +39,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"net/url"
 	"path"
@@ -750,7 +747,7 @@ func (config *Config) Pave(
 						return nil, common.ContextError(err)
 					}
 
-					boxedServerEntries, err := box(fileKey, compress(signedServerEntries))
+					boxedServerEntries, err := box(fileKey, common.Compress(signedServerEntries))
 					if err != nil {
 						return nil, common.ContextError(err)
 					}
@@ -793,7 +790,7 @@ func (config *Config) Pave(
 
 	paveFiles = append(paveFiles, &PaveFile{
 		Name:     REGISTRY_FILENAME,
-		Contents: compress(signedRegistry),
+		Contents: common.Compress(signedRegistry),
 	})
 
 	return paveFiles, nil
@@ -991,7 +988,7 @@ func GetOSLFilename(baseDirectory string, oslID []byte) string {
 func UnpackRegistry(
 	compressedRegistry []byte, signingPublicKey string) (*Registry, []byte, error) {
 
-	packagedRegistry, err := uncompress(compressedRegistry)
+	packagedRegistry, err := common.Decompress(compressedRegistry)
 	if err != nil {
 		return nil, nil, common.ContextError(err)
 	}
@@ -1173,7 +1170,7 @@ func (registry *Registry) UnpackOSL(
 		return "", common.ContextError(err)
 	}
 
-	dataPackage, err := uncompress(decryptedContents)
+	dataPackage, err := common.Decompress(decryptedContents)
 	if err != nil {
 		return "", common.ContextError(err)
 	}
@@ -1283,24 +1280,3 @@ func unbox(key, box []byte) ([]byte, error) {
 	}
 	return plaintext, nil
 }
-
-func compress(data []byte) []byte {
-	var compressedData bytes.Buffer
-	writer := zlib.NewWriter(&compressedData)
-	writer.Write(data)
-	writer.Close()
-	return compressedData.Bytes()
-}
-
-func uncompress(data []byte) ([]byte, error) {
-	reader, err := zlib.NewReader(bytes.NewReader(data))
-	if err != nil {
-		return nil, common.ContextError(err)
-	}
-	uncompressedData, err := ioutil.ReadAll(reader)
-	reader.Close()
-	if err != nil {
-		return nil, common.ContextError(err)
-	}
-	return uncompressedData, nil
-}

psiphon/common/utils.go

@@ -20,11 +20,14 @@
 package common
 
 import (
+	"bytes"
+	"compress/zlib"
 	"crypto/rand"
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"math/big"
 	"runtime"
 	"strings"
@@ -176,3 +179,26 @@ func ContextError(err error) error {
 	pc, _, line, _ := runtime.Caller(1)
 	return fmt.Errorf("%s#%d: %s", getFunctionName(pc), line, err)
 }
+
+// Compress returns zlib compressed data
+func Compress(data []byte) []byte {
+	var compressedData bytes.Buffer
+	writer := zlib.NewWriter(&compressedData)
+	writer.Write(data)
+	writer.Close()
+	return compressedData.Bytes()
+}
+
+// Decompress returns zlib decompressed data
+func Decompress(data []byte) ([]byte, error) {
+	reader, err := zlib.NewReader(bytes.NewReader(data))
+	if err != nil {
+		return nil, ContextError(err)
+	}
+	uncompressedData, err := ioutil.ReadAll(reader)
+	reader.Close()
+	if err != nil {
+		return nil, ContextError(err)
+	}
+	return uncompressedData, nil
+}

psiphon/common/utils_test.go

@@ -20,6 +20,7 @@
 package common
 
 import (
+	"bytes"
 	"testing"
 	"time"
 )
@@ -52,3 +53,19 @@ func TestMakeRandomPeriod(t *testing.T) {
 		t.Error("duration should have randomness difference between calls")
 	}
 }
+
+func TestCompress(t *testing.T) {
+
+	originalData := []byte("test data")
+
+	compressedData := Compress(originalData)
+
+	decompressedData, err := Decompress(compressedData)
+	if err != nil {
+		t.Error("Uncompress failed: %s", err)
+	}
+
+	if bytes.Compare(originalData, decompressedData) != 0 {
+		t.Error("decompressed data doesn't match original data")
+	}
+}