Merge branch 'master' into staging-client

ClientLibrary/Dockerfile

@@ -2,7 +2,7 @@
 #
 # See README.md for usage instructions.
 
-FROM ubuntu:18.04
+FROM --platform=linux/amd64 ubuntu:18.04
 
 # Install system-level dependencies.
 ENV DEBIAN_FRONTEND=noninteractive

ClientLibrary/README.md

@@ -57,9 +57,13 @@ Note that you may need to use `sudo docker` below, depending on your OS.
 
 *Ensure that the command below is run from within the `ClientLibrary` directory*
 
+*The `--user "$(id -u):$(id -g)"` flag must be omitted when building on macOS because manually specifying the UID/GID will cause the build to fail since Docker on macOS handles host to container UID/GID mapping automatically and will be unable to map the manually specified UID/GID to the current user.*
+
 ```bash
 cd .. && \
   docker run \
+  --platform=linux/amd64 \
+  --user "$(id -u):$(id -g)" \
   --rm \
   -v $PWD:/go/src/github.com/Psiphon-Labs/psiphon-tunnel-core \
   psiclientlibrary-builder \

ClientLibrary/build-darwin.sh

@@ -80,9 +80,7 @@ prepare_build () {
   BUILDREV=$(git rev-parse --short HEAD)
   GOVERSION=$(go version | perl -ne '/go version (.*?) / && print $1')
 
-  # see DEPENDENCIES comment in MobileLibrary/Android/make.bash
   cd ${GOPATH}/src/github.com/Psiphon-Labs/psiphon-tunnel-core/ClientLibrary
-  DEPENDENCIES=$(echo -n "{" && GOOS=$1 go list -tags "${BUILD_TAGS}" -f '{{range $dep := .Deps}}{{printf "%s\n" $dep}}{{end}}' | GOOS=$1 xargs go list -tags "${BUILD_TAGS}" -f '{{if not .Standard}}{{.ImportPath}}{{end}}' | xargs -I pkg bash -c 'cd $GOPATH/src/$0 && if echo -n "$0" | grep -vEq "^github.com/Psiphon-Labs/psiphon-tunnel-core/" ; then echo -n "\"$0\":\"$(git rev-parse --short HEAD)\"," ; fi' pkg | sed 's/,$//' | tr -d '\n' && echo -n "}")
 
   LDFLAGS="\
   -s \
@@ -91,15 +89,13 @@ prepare_build () {
   -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.buildRepo=$BUILDREPO \
   -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.buildRev=$BUILDREV \
   -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.goVersion=$GOVERSION \
-  -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.dependencies=$DEPENDENCIES \
   "
 
   echo "Variables for ldflags:"
   echo " Build date: ${BUILDDATE}"
   echo " Build repo: ${BUILDREPO}"
   echo " Build revision: ${BUILDREV}"
   echo " Go version: ${GOVERSION}"
-  echo " Dependencies: ${DEPENDENCIES}"
   echo ""
 
 }

ClientLibrary/make.bash

@@ -10,6 +10,8 @@ fi
 # $2, if specified, is go build tags
 if [ -z ${2+x} ]; then BUILD_TAGS=""; else BUILD_TAGS="$2"; fi
 
+export GOCACHE=/tmp
+
 BUILD_DIR=build
 
 if [ ! -d ${BUILD_DIR} ]; then
@@ -23,25 +25,20 @@ prepare_build () {
   BUILDREV=$(git rev-parse --short HEAD)
   GOVERSION=$(go version | perl -ne '/go version (.*?) / && print $1')
 
-  # see DEPENDENCIES comment in MobileLibrary/Android/make.bash
-  DEPENDENCIES=$(echo -n "{" && GOOS=$1 go list -tags "${BUILD_TAGS}" -f '{{range $dep := .Deps}}{{printf "%s\n" $dep}}{{end}}' | GOOS=$1 xargs go list -tags "${BUILD_TAGS}" -f '{{if not .Standard}}{{.ImportPath}}{{end}}' | xargs -I pkg bash -c 'cd $GOPATH/src/$0 && if echo -n "$0" | grep -vEq "^github.com/Psiphon-Labs/psiphon-tunnel-core/" ; then echo -n "\"$0\":\"$(git rev-parse --short HEAD)\"," ; fi' pkg | sed 's/,$//' | tr -d '\n' && echo -n "}")
-
   LDFLAGS="\
   -s \
   -w \
   -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.buildDate=$BUILDDATE \
   -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.buildRepo=$BUILDREPO \
   -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.buildRev=$BUILDREV \
   -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.goVersion=$GOVERSION \
-  -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.dependencies=$DEPENDENCIES \
   "
 
   echo "Variables for ldflags:"
   echo " Build date: ${BUILDDATE}"
   echo " Build repo: ${BUILDREPO}"
   echo " Build revision: ${BUILDREV}"
   echo " Go version: ${GOVERSION}"
-  echo " Dependencies: ${DEPENDENCIES}"
   echo ""
 
 }

ConsoleClient/Dockerfile

@@ -2,7 +2,7 @@
 #
 # See README.md for usage instructions.
 
-FROM ubuntu:18.04
+FROM --platform=linux/amd64 ubuntu:18.04
 
 # Install system-level dependencies.
 ENV DEBIAN_FRONTEND=noninteractive

ConsoleClient/README.md

@@ -14,9 +14,13 @@ Note that you may need to use `sudo docker` below, depending on your OS.
 
 *Ensure that the command below is run from within the `ConsoleClient` directory*
 
+*The `--user "$(id -u):$(id -g)"` flag must be omitted when building on macOS because manually specifying the UID/GID will cause the build to fail since Docker on macOS handles host to container UID/GID mapping automatically and will be unable to map the manually specified UID/GID to the current user.*
+
 ```bash
 cd .. && \
   docker run \
+  --platform=linux/amd64 \
+  --user "$(id -u):$(id -g)" \
   --rm \
   -v $PWD:/go/src/github.com/Psiphon-Labs/psiphon-tunnel-core \
   psiclient \

ConsoleClient/main.go

@@ -20,16 +20,13 @@
 package main
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/signal"
-	"sort"
 	"strings"
 	"sync"
 	"syscall"
@@ -117,32 +114,9 @@ func main() {
 
 	if versionDetails {
 		b := buildinfo.GetBuildInfo()
-
-		var printableDependencies bytes.Buffer
-		var dependencyMap map[string]string
-		longestRepoUrl := 0
-		json.Unmarshal(b.Dependencies, &dependencyMap)
-
-		sortedRepoUrls := make([]string, 0, len(dependencyMap))
-		for repoUrl := range dependencyMap {
-			repoUrlLength := len(repoUrl)
-			if repoUrlLength > longestRepoUrl {
-				longestRepoUrl = repoUrlLength
-			}
-
-			sortedRepoUrls = append(sortedRepoUrls, repoUrl)
-		}
-		sort.Strings(sortedRepoUrls)
-
-		for repoUrl := range sortedRepoUrls {
-			printableDependencies.WriteString(fmt.Sprintf("    %s  ", sortedRepoUrls[repoUrl]))
-			for i := 0; i < (longestRepoUrl - len(sortedRepoUrls[repoUrl])); i++ {
-				printableDependencies.WriteString(" ")
-			}
-			printableDependencies.WriteString(fmt.Sprintf("%s\n", dependencyMap[sortedRepoUrls[repoUrl]]))
-		}
-
-		fmt.Printf("Psiphon Console Client\n  Build Date: %s\n  Built With: %s\n  Repository: %s\n  Revision: %s\n  Dependencies:\n%s\n", b.BuildDate, b.GoVersion, b.BuildRepo, b.BuildRev, printableDependencies.String())
+		fmt.Printf(
+			"Psiphon Console Client\n  Build Date: %s\n  Built With: %s\n  Repository: %s\n  Revision: %s\n",
+			b.BuildDate, b.GoVersion, b.BuildRepo, b.BuildRev)
 		os.Exit(0)
 	}
 

ConsoleClient/make.bash

@@ -10,6 +10,8 @@ fi
 # $2, if specified, is go build tags
 if [ -z ${2+x} ]; then BUILD_TAGS=""; else BUILD_TAGS="$2"; fi
 
+export GOCACHE=/tmp
+
 EXE_BASENAME="psiphon-tunnel-core"
 
 prepare_build () {
@@ -19,17 +21,13 @@ prepare_build () {
   BUILDREV=$(git rev-parse --short HEAD)
   GOVERSION=$(go version | perl -ne '/go version (.*?) / && print $1')
 
-  # see DEPENDENCIES comment in MobileLibrary/Android/make.bash
-  DEPENDENCIES=$(echo -n "{" && GOOS=$1 go list -tags "${BUILD_TAGS}" -f '{{range $dep := .Deps}}{{printf "%s\n" $dep}}{{end}}' | GOOS=$1 xargs go list -tags "${BUILD_TAGS}" -f '{{if not .Standard}}{{.ImportPath}}{{end}}' | xargs -I pkg bash -c 'cd $GOPATH/src/$0 && if echo -n "$0" | grep -vEq "^github.com/Psiphon-Labs/psiphon-tunnel-core/" ; then echo -n "\"$0\":\"$(git rev-parse --short HEAD)\"," ; fi' pkg | sed 's/,$//' | tr -d '\n' && echo -n "}")
-
   LDFLAGS="\
   -s \
   -w \
   -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.buildDate=$BUILDDATE \
   -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.buildRepo=$BUILDREPO \
   -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.buildRev=$BUILDREV \
   -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.goVersion=$GOVERSION \
-  -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.dependencies=$DEPENDENCIES \
   "
   echo -e "${BUILDDATE}\n${BUILDREPO}\n${BUILDREV}\n" > $BUILDINFOFILE
 
@@ -38,7 +36,6 @@ prepare_build () {
   echo " Build repo: ${BUILDREPO}"
   echo " Build revision: ${BUILDREV}"
   echo " Go version: ${GOVERSION}"
-  echo " Dependencies: ${DEPENDENCIES}"
   echo ""
 }
 

MobileLibrary/Android/Dockerfile

@@ -2,10 +2,13 @@
 #
 # See README.md for usage instructions.
 
-FROM ubuntu:18.04
+FROM --platform=linux/amd64 ubuntu:18.04
 
 # Install system-level dependencies.
 ENV DEBIAN_FRONTEND=noninteractive
+
+RUN mkdir -p /usr/share/man/man1 # fixes https://stackoverflow.com/a/69844974
+
 RUN apt-get update -y && apt-get install -y --no-install-recommends \
     build-essential \
     ca-certificates \
@@ -41,22 +44,14 @@ RUN curl -L https://dl.google.com/android/repository/commandlinetools-linux-8092
   && yes | $ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager --licenses \
   && $ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager --install "platforms;android-${ANDROID_PLATFORM_VERSION}" "ndk;${ANDROID_NDK_VERSION}"
 
-# Install Pinned Gomobile.
+# Install Gomobile from source at MobileLibrary/go-mobile
 # The sed operation patches gomobile, editing a command that assumes modules
-ENV GOMOBILE_PINNED_REV=ce6a79cf6a13dd77095a6f8dbee5f39848fa7da1
-RUN mkdir -p $GOPATH/pkg/gomobile/dl \
-  && cd $GOPATH/pkg/gomobile/dl \
-  && mkdir -p $GOPATH/src/golang.org/x \
-  && cd $GOPATH/src/golang.org/x \
-  && git clone https://github.com/golang/mobile \
-  && cd mobile \
-  && git checkout -b pinned $GOMOBILE_PINNED_REV \
+RUN mkdir -p $GOPATH/src/golang.org/x
+COPY go-mobile $GOPATH/src/golang.org/x/mobile
+RUN cd $GOPATH/src/golang.org/x/mobile \
   && mv ./cmd/gomobile/init.go ./cmd/gomobile/init.go.orig \
   && sed -e 's/golang.org\/x\/mobile\/cmd\/gobind@latest/golang.org\/x\/mobile\/cmd\/gobind/g' ./cmd/gomobile/init.go.orig > ./cmd/gomobile/init.go \
-  && echo "master: $(git rev-parse master)\npinned: $(git rev-parse master)" | tee $GOROOT/MOBILE \
   && export GO111MODULE=off \
-  && go get golang.org/x/mod/modfile \
-  && go get golang.org/x/tools/go/packages \
   && go install golang.org/x/mobile/cmd/gomobile \
   && gomobile init -v
 

MobileLibrary/Android/PsiphonTunnel/PsiphonTunnel.java

@@ -246,26 +246,41 @@ public void routeThroughTunnel() {
         if (!mRoutingThroughTunnel.compareAndSet(false, true)) {
             return;
         }
-        ParcelFileDescriptor tunFd = mTunFd.getAndSet(null);
+        ParcelFileDescriptor tunFd = mTunFd.get();
         if (tunFd == null) {
             return;
         }
 
         String socksServerAddress = "127.0.0.1:" + Integer.toString(mLocalSocksProxyPort.get());
         String udpgwServerAddress = "127.0.0.1:" + Integer.toString(UDPGW_SERVER_PORT);
-        startTun2Socks(
-                tunFd,
-                VPN_INTERFACE_MTU,
-                mPrivateAddress.mRouter,
-                VPN_INTERFACE_NETMASK,
-                socksServerAddress,
-                udpgwServerAddress,
-                true);
-
-        mHostService.onDiagnosticMessage("routing through tunnel");
-
-        // TODO: should double-check tunnel routing; see:
-        // https://bitbucket.org/psiphon/psiphon-circumvention-system/src/1dc5e4257dca99790109f3bf374e8ab3a0ead4d7/Android/PsiphonAndroidLibrary/src/com/psiphon3/psiphonlibrary/TunnelCore.java?at=default#cl-779
+
+        // We may call routeThroughTunnel and stopRouteThroughTunnel more than once within the same
+        // VPN session. Since stopTun2Socks() closes the FD passed to startTun2Socks() we will use a
+        // dup of the original tun FD and close the original only when we call stopVpn().
+        //
+        // Note that ParcelFileDescriptor.dup() may throw an IOException.
+        try {
+            startTun2Socks(
+                    tunFd.dup(),
+                    VPN_INTERFACE_MTU,
+                    mPrivateAddress.mRouter,
+                    VPN_INTERFACE_NETMASK,
+                    socksServerAddress,
+                    udpgwServerAddress,
+                    true);
+            mHostService.onDiagnosticMessage("routing through tunnel");
+
+            // TODO: should double-check tunnel routing; see:
+            // https://bitbucket.org/psiphon/psiphon-circumvention-system/src/1dc5e4257dca99790109f3bf374e8ab3a0ead4d7/Android/PsiphonAndroidLibrary/src/com/psiphon3/psiphonlibrary/TunnelCore.java?at=default#cl-779
+        } catch (IOException e) {
+            mHostService.onDiagnosticMessage("routing through tunnel error: " + e);
+        }
+    }
+
+    public void stopRouteThroughTunnel() {
+        if (mRoutingThroughTunnel.compareAndSet(true, false)) {
+            stopTun2Socks();
+        }
     }
 
     // Throws an exception in error conditions. In the case of an exception, the routing

MobileLibrary/Android/README.md

@@ -40,17 +40,21 @@ Note that you may need to use `sudo docker` below, depending on your OS.
 
 ##### Create the build image:
 
-1. While in the `MobileLibrary/Android` directory, run the command: `docker build --no-cache=true -t psiandroid .`
+1. While in the `MobileLibrary/Android` directory, run the command: `docker build --no-cache=true -t psiandroid -f Dockerfile ..`
 
 2. Once completed, verify that you see an image named `psiandroid` when running: `docker images`
 
 ##### Run the build:
 
 *Ensure that the command below is run from within the `MobileLibrary/Android` directory*
 
+*The `--user "$(id -u):$(id -g)"` flag must be omitted when building on macOS because manually specifying the UID/GID will cause the build to fail since Docker on macOS handles host to container UID/GID mapping automatically and will be unable to map the manually specified UID/GID to the current user.*
+
 ```bash
 cd ../.. && \
   docker run \
+  --platform=linux/amd64 \
+  --user "$(id -u):$(id -g)" \
   --rm \
   -v $(pwd):/go/src/github.com/Psiphon-Labs/psiphon-tunnel-core \
   psiandroid \

MobileLibrary/Android/make.bash

@@ -13,42 +13,13 @@ if [ -z ${1+x} ]; then BUILD_TAGS=""; else BUILD_TAGS="$1"; fi
 # At this time, psiphon-tunnel-core doesn't support modules
 export GO111MODULE=off
 
+export GOCACHE=/tmp
+
 BUILDINFOFILE="psiphon-tunnel-core_buildinfo.txt"
 BUILDDATE=$(date --iso-8601=seconds)
 BUILDREPO=$(git config --get remote.origin.url)
 BUILDREV=$(git rev-parse --short HEAD)
 GOVERSION=$(go version | perl -ne '/go version (.*?) / && print $1')
-GOMOBILEVERSION=$(gomobile version | perl -ne '/gomobile version (.*?) / && print $1')
-
-# DEPENDENCIES
-#
-# - this script produces a JSON object listing all Go package dependencies,
-#   excluding packages under github.com/Psiphon-Labs/psiphon-tunnel-core/
-#   (thus also excluding vendored packages) which will all have the same rev
-#   as BUILDREV
-#
-# - starts the string with a `{` and ends with a `}`
-#
-# - uses the `go list` command and passes it a template string (using the Go
-#   template syntax) saying I want all the dependencies of the package in the
-#   current directory, printing 1/line via printf
-#
-# - pipes to `xargs` to run a command on each line output from the first
-#   command and uses `go list` with a template string to print the "Import
-#   Path" (from just below `$GOPATH/src`) if the package is not part of the
-#   standard library
-#
-# - pipes to `xargs` again, specifiying `pkg` as the placeholder name for each
-#   item being operated on (which is the list of non standard library import
-#   paths from the previous step); `xargs` runs a bash script (via `-c`) which
-#   changes to each import path in sequence, then echoes out, after the
-#   exclusion check, `"<import path>":"<subshell output of getting the short
-#   git revision>",`
-#
-# - for non-empty dependency lists, the last command leaves a trailing `,\n` at
-#   the end, so use `sed` and `tr` to remove the suffix.
-#
-DEPENDENCIES=$(cd ../psi && echo -n "{" && GOOS=android go list -tags "${BUILD_TAGS}" -f '{{range $dep := .Deps}}{{printf "%s\n" $dep}}{{end}}' | GOOS=android xargs go list -tags "${BUILD_TAGS}" -f '{{if not .Standard}}{{.ImportPath}}{{end}}' | xargs -I pkg bash -c 'cd $GOPATH/src/$0 && if echo -n "$0" | grep -vEq "^github.com/Psiphon-Labs/psiphon-tunnel-core/" ; then echo -n "\"$0\":\"$(git rev-parse --short HEAD)\"," ; fi' pkg | sed 's/,$//' | tr -d '\n' && echo -n "}")
 
 LDFLAGS="\
 -s \
@@ -57,8 +28,6 @@ LDFLAGS="\
 -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.buildRepo=$BUILDREPO \
 -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.buildRev=$BUILDREV \
 -X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.goVersion=$GOVERSION \
--X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.gomobileVersion=$GOMOBILEVERSION \
--X github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo.dependencies=$DEPENDENCIES \
 "
 
 echo -e "${BUILDDATE}\n${BUILDREPO}\n${BUILDREV}\n" > $BUILDINFOFILE
@@ -68,8 +37,6 @@ echo " Build date: ${BUILDDATE}"
 echo " Build repo: ${BUILDREPO}"
 echo " Build revision: ${BUILDREV}"
 echo " Go version: ${GOVERSION}"
-echo " Gomobile version: ${GOMOBILEVERSION}"
-echo " Dependencies: ${DEPENDENCIES}"
 echo ""
 
 gomobile bind -v -x -target=android/arm,android/arm64,android/386,android/amd64 -tags="${BUILD_TAGS}" -ldflags="$LDFLAGS" github.com/Psiphon-Labs/psiphon-tunnel-core/MobileLibrary/psi

MobileLibrary/go-mobile/.gitattributes

@@ -0,0 +1,10 @@
+# Treat all files in this repo as binary, with no git magic updating
+# line endings. Windows users contributing to Go will need to use a
+# modern version of git and editors capable of LF line endings.
+#
+# We'll prevent accidental CRLF line endings from entering the repo
+# via the git-review gofmt checks.
+#
+# See golang.org/issue/9281
+
+* -text