Merge commit '448517f4ca361ed5ec8f04fb56c9833050f668d3' into staging-…

…client

.github/workflows/tests.yml

@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ "ubuntu" ]
-        go: [ "1.20.12" ]
+        go: [ "1.20.13" ]
         test-type: [ "detector", "coverage", "memory", "custom-build-tags" ]
 
     runs-on: ${{ matrix.os }}-latest

ClientLibrary/Dockerfile

@@ -21,7 +21,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends \
 
 # Install Go.
 # NOTE: Go 1.10+ is required to build c-shared for windows (https://github.com/golang/go/commit/bb0bfd002ada7e3eb9198d4287b32c2fed6e8da6)
-ENV GOVERSION=go1.20.12 GOROOT=/usr/local/go GOPATH=/go PATH=$PATH:/usr/local/go/bin:/go/bin CGO_ENABLED=1
+ENV GOVERSION=go1.20.13 GOROOT=/usr/local/go GOPATH=/go PATH=$PATH:/usr/local/go/bin:/go/bin CGO_ENABLED=1
 
 RUN curl -L https://storage.googleapis.com/golang/$GOVERSION.linux-amd64.tar.gz -o /tmp/go.tar.gz \
    && tar -C /usr/local -xzf /tmp/go.tar.gz \

ClientLibrary/build-darwin.sh

@@ -9,8 +9,8 @@ if [ -z ${2+x} ]; then BUILD_TAGS=""; else BUILD_TAGS="$2"; fi
 # Note:
 #   clangwrap.sh needs to be updated when the Go version changes.
 #   The last version was:
-#   https://github.com/golang/go/blob/go1.20.12/misc/ios/clangwrap.sh
-GO_VERSION_REQUIRED="1.20.12"
+#   https://github.com/golang/go/blob/go1.20.13/misc/ios/clangwrap.sh
+GO_VERSION_REQUIRED="1.20.13"
 
 BASE_DIR=$(cd "$(dirname "$0")" ; pwd -P)
 cd ${BASE_DIR}

ConsoleClient/Dockerfile

@@ -22,7 +22,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends \
   && rm -rf /var/lib/apt/lists/*
 
 # Install Go.
-ENV GOVERSION=go1.20.12 GOROOT=/usr/local/go GOPATH=/go PATH=$PATH:/usr/local/go/bin:/go/bin CGO_ENABLED=1
+ENV GOVERSION=go1.20.13 GOROOT=/usr/local/go GOPATH=/go PATH=$PATH:/usr/local/go/bin:/go/bin CGO_ENABLED=1
 
 RUN curl -L https://storage.googleapis.com/golang/$GOVERSION.linux-amd64.tar.gz -o /tmp/go.tar.gz \
    && tar -C /usr/local -xzf /tmp/go.tar.gz \

MobileLibrary/Android/Dockerfile

@@ -23,7 +23,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends \
   && rm -rf /var/lib/apt/lists/*
 
 # Install Go.
-ENV GOVERSION=go1.20.12 GOROOT=/usr/local/go GOPATH=/go PATH=$PATH:/usr/local/go/bin:/go/bin CGO_ENABLED=1
+ENV GOVERSION=go1.20.13 GOROOT=/usr/local/go GOPATH=/go PATH=$PATH:/usr/local/go/bin:/go/bin CGO_ENABLED=1
 
 RUN curl -L https://storage.googleapis.com/golang/$GOVERSION.linux-amd64.tar.gz -o /tmp/go.tar.gz \
   && tar -C /usr/local -xzf /tmp/go.tar.gz \

MobileLibrary/iOS/build-psiphon-framework.sh

@@ -17,7 +17,7 @@ set -e -u -x
 if [ -z ${1+x} ]; then BUILD_TAGS=""; else BUILD_TAGS="$1"; fi
 
 # Modify this value as we use newer Go versions.
-GO_VERSION_REQUIRED="1.20.12"
+GO_VERSION_REQUIRED="1.20.13"
 
 # At this time, psiphon-tunnel-core doesn't support modules
 export GO111MODULE=off

Server/Dockerfile-binary-builder

@@ -1,6 +1,6 @@
 FROM alpine:3.18.4
 
-ENV GOLANG_VERSION 1.20.12
+ENV GOLANG_VERSION 1.20.13
 ENV GOLANG_SRC_URL https://golang.org/dl/go$GOLANG_VERSION.src.tar.gz
 
 RUN set -ex \

Server/main.go

@@ -26,6 +26,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"runtime/debug"
 	"strconv"
 	"strings"
 	"syscall"
@@ -248,6 +249,16 @@ func main() {
 		}
 		// Else, this is the child process.
 
+		// As of Go 1.19.10, programs with Linux capabilities or setuid do not
+		// dump panic stacks by default. See:
+		// https://github.com/golang/go/commit/a7b1cd452ddc69a6606c2f35ac5786dc892e62cb.
+		// To restore panic stacks, we call SetTraceback("single"), restoring
+		// the default GOTRACKBACK value. The server program is run as a
+		// non-privileged user and with CAP_NET capabilities; neither the
+		// panic stack traces nor register dumps are expected to expose any
+		// unexpected sensitive information.
+		debug.SetTraceback("single")
+
 		err = server.RunServices(configJSON)
 		if err != nil {
 			fmt.Printf("run failed: %s\n", err)

psiphon/common/parameters/keyStrings.go

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2023, Psiphon Inc.
+ * All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package parameters
+
+// KeyStrings represents a set of key/strings pairs.
+type KeyStrings map[string][]string
+
+// Validates that the keys and values are well formed.
+func (keyStrings KeyStrings) Validate() error {
+	// Always succeeds because KeyStrings is generic and does not impose any
+	// restrictions on keys/values. Consider imposing limits like maximum
+	// map/array/string sizes.
+	return nil
+}
+
+func (p ParametersAccessor) KeyStrings(name, key string) []string {
+	value := KeyStrings{}
+	p.snapshot.getValue(name, &value)
+	return value[key]
+}

psiphon/common/parameters/parameters.go

@@ -311,6 +311,15 @@ const (
 	HoldOffTunnelProtocols                           = "HoldOffTunnelProtocols"
 	HoldOffTunnelFrontingProviderIDs                 = "HoldOffTunnelFrontingProviderIDs"
 	HoldOffTunnelProbability                         = "HoldOffTunnelProbability"
+	HoldOffDirectTunnelMinDuration                   = "HoldOffDirectTunnelMinDuration"
+	HoldOffDirectTunnelMaxDuration                   = "HoldOffDirectTunnelMaxDuration"
+	HoldOffDirectServerEntryRegions                  = "HoldOffDirectServerEntryRegions"
+	HoldOffDirectServerEntryProviderRegions          = "HoldOffDirectServerEntryProviderRegions"
+	HoldOffDirectTunnelProbability                   = "HoldOffDirectTunnelProbability"
+	RestrictDirectProviderIDs                        = "RestrictDirectProviderIDs"
+	RestrictDirectProviderRegions                    = "RestrictDirectProviderRegions"
+	RestrictDirectProviderIDsServerProbability       = "RestrictDirectProviderIDsServerProbability"
+	RestrictDirectProviderIDsClientProbability       = "RestrictDirectProviderIDsClientProbability"
 	RestrictFrontingProviderIDs                      = "RestrictFrontingProviderIDs"
 	RestrictFrontingProviderIDsServerProbability     = "RestrictFrontingProviderIDsServerProbability"
 	RestrictFrontingProviderIDsClientProbability     = "RestrictFrontingProviderIDsClientProbability"
@@ -702,6 +711,17 @@ var defaultParameters = map[string]struct {
 	HoldOffTunnelFrontingProviderIDs: {value: []string{}},
 	HoldOffTunnelProbability:         {value: 0.0, minimum: 0.0},
 
+	HoldOffDirectTunnelMinDuration:          {value: time.Duration(0), minimum: time.Duration(0)},
+	HoldOffDirectTunnelMaxDuration:          {value: time.Duration(0), minimum: time.Duration(0)},
+	HoldOffDirectServerEntryRegions:         {value: []string{}},
+	HoldOffDirectServerEntryProviderRegions: {value: KeyStrings{}},
+	HoldOffDirectTunnelProbability:          {value: 0.0, minimum: 0.0},
+
+	RestrictDirectProviderIDs:                  {value: []string{}},
+	RestrictDirectProviderRegions:              {value: KeyStrings{}},
+	RestrictDirectProviderIDsServerProbability: {value: 0.0, minimum: 0.0, flags: serverSideOnly},
+	RestrictDirectProviderIDsClientProbability: {value: 0.0, minimum: 0.0},
+
 	RestrictFrontingProviderIDs:                  {value: []string{}},
 	RestrictFrontingProviderIDsServerProbability: {value: 0.0, minimum: 0.0, flags: serverSideOnly},
 	RestrictFrontingProviderIDsClientProbability: {value: 0.0, minimum: 0.0},
@@ -1086,6 +1106,14 @@ func (p *Parameters) Set(
 					}
 					return nil, errors.Trace(err)
 				}
+			case KeyStrings:
+				err := v.Validate()
+				if err != nil {
+					if skipOnError {
+						continue
+					}
+					return nil, errors.Trace(err)
+				}
 			case *BPFProgramSpec:
 				if v != nil {
 					err := v.Validate()

psiphon/common/parameters/parameters_test.go

@@ -189,6 +189,13 @@ func TestGetDefaultParameters(t *testing.T) {
 			if !reflect.DeepEqual(v, g) {
 				t.Fatalf("ConjureTransports returned %+v expected %+v", g, v)
 			}
+		case KeyStrings:
+			for key, strings := range v {
+				g := p.Get().KeyStrings(name, key)
+				if !reflect.DeepEqual(strings, g) {
+					t.Fatalf("KeyStrings returned %+v expected %+v", g, strings)
+				}
+			}
 		default:
 			t.Fatalf("Unhandled default type: %s (%T)", name, defaults.value)
 		}

psiphon/common/protocol/protocol.go

@@ -272,6 +272,16 @@ func TunnelProtocolIsCompatibleWithFragmentor(protocol string) bool {
 		protocol == TUNNEL_PROTOCOL_CONJURE_OBFUSCATED_SSH
 }
 
+func TunnelProtocolIsDirect(protocol string) bool {
+	return protocol == TUNNEL_PROTOCOL_SSH ||
+		protocol == TUNNEL_PROTOCOL_OBFUSCATED_SSH ||
+		protocol == TUNNEL_PROTOCOL_TLS_OBFUSCATED_SSH ||
+		protocol == TUNNEL_PROTOCOL_UNFRONTED_MEEK ||
+		protocol == TUNNEL_PROTOCOL_UNFRONTED_MEEK_HTTPS ||
+		protocol == TUNNEL_PROTOCOL_UNFRONTED_MEEK_SESSION_TICKET ||
+		protocol == TUNNEL_PROTOCOL_QUIC_OBFUSCATED_SSH
+}
+
 func TunnelProtocolRequiresTLS12SessionTickets(protocol string) bool {
 	return protocol == TUNNEL_PROTOCOL_UNFRONTED_MEEK_SESSION_TICKET
 }

psiphon/common/protocol/serverEntry.go

@@ -61,6 +61,7 @@ type ServerEntry struct {
 	SshObfuscatedKey                string   `json:"sshObfuscatedKey"`
 	Capabilities                    []string `json:"capabilities"`
 	Region                          string   `json:"region"`
+	ProviderID                      string   `json:"providerID"`
 	FrontingProviderID              string   `json:"frontingProviderID"`
 	TlsOSSHPort                     int      `json:"tlsOSSHPort"`
 	MeekServerPort                  int      `json:"meekServerPort"`
@@ -736,6 +737,10 @@ func (serverEntry *ServerEntry) HasSignature() bool {
 	return serverEntry.Signature != ""
 }
 
+func (serverEntry *ServerEntry) HasProviderID() bool {
+	return serverEntry.ProviderID != ""
+}
+
 func (serverEntry *ServerEntry) GetDiagnosticID() string {
 	return TagToDiagnosticID(serverEntry.Tag)
 }

psiphon/common/quic/obfuscator.go

@@ -253,6 +253,10 @@ func (conn *ObfuscatedPacketConn) Close() error {
 	}
 
 	if conn.isServer {
+
+		// Interrupt any blocked writes.
+		conn.PacketConn.SetWriteDeadline(time.Now())
+
 		close(conn.stopBroadcast)
 		conn.runWaitGroup.Wait()
 	}

psiphon/common/quic/quic.go

@@ -172,6 +172,13 @@ func Listen(
 		return nil, errors.Trace(err)
 	}
 
+	// Note that WriteTimeoutUDPConn is not used here in the server case, as
+	// the server UDP conn will have many concurrent writers, and each
+	// SetWriteDeadline call by WriteTimeoutUDPConn would extend the deadline
+	// for all existing blocked writers. ObfuscatedPacketConn.Close calls
+	// SetWriteDeadline once to interrupt any blocked writers to ensure a
+	// timely shutdown.
+
 	obfuscatedPacketConn, err := NewServerObfuscatedPacketConn(
 		udpConn, true, false, false, obfuscationKey, seed)
 	if err != nil {

psiphon/common/transforms/httpNormalizer.go

@@ -21,7 +21,7 @@ package transforms
 
 import (
 	"bytes"
-	stderrors "errors"
+	std_errors "errors"
 	"io"
 	"net"
 	"net/textproto"
@@ -50,7 +50,7 @@ const (
 	rangeHeader               = "Range"
 )
 
-var ErrPassthroughActive = stderrors.New("passthrough")
+var ErrPassthroughActive = std_errors.New("passthrough")
 
 // HTTPNormalizer wraps a net.Conn, intercepting Read calls, and normalizes any
 // HTTP requests that are read. The HTTP request components preceeding the body

psiphon/common/transforms/httpNormalizer_test.go

@@ -21,7 +21,7 @@ package transforms
 
 import (
 	"bytes"
-	stderrors "errors"
+	std_errors "errors"
 	"io"
 	"net"
 	"strings"
@@ -143,7 +143,7 @@ func runHTTPNormalizerTest(tt *httpNormalizerTest, useNormalizer bool) error {
 		// Subsequent writes should not impact conn or passthroughConn
 
 		_, err = normalizer.Write([]byte("ignored"))
-		if !stderrors.Is(err, ErrPassthroughActive) {
+		if !std_errors.Is(err, ErrPassthroughActive) {
 			return errors.Tracef("expected error io.EOF but got %v", err)
 		}
 
@@ -230,20 +230,20 @@ func TestHTTPNormalizerHTTPRequest(t *testing.T) {
 			headerOrder:  []string{"Host", "Content-Length"},
 			wantOutput:   "POST / HTTP/1.1\r\nHost: example.com\r\nContent-Length: 4\r\n\r\nabcd",
 			chunkSize:    1,
-			connReadErrs: []error{stderrors.New("err1"), stderrors.New("err2")},
+			connReadErrs: []error{std_errors.New("err1"), std_errors.New("err2")},
 		},
 		{
 			name:       "Content-Length missing",
 			input:      "POST / HTTP/1.1\r\n\r\nabcd",
 			wantOutput: "POST / HTTP/1.1\r\n\r\nabcd", // set to ensure all bytes are read
-			wantError:  stderrors.New("Content-Length missing"),
+			wantError:  std_errors.New("Content-Length missing"),
 			chunkSize:  1,
 		},
 		{
 			name:       "invalid Content-Length header value",
 			input:      "POST / HTTP/1.1\r\nContent-Length: X\r\n\r\nabcd",
 			wantOutput: "POST / HTTP/1.1\r\nContent-Length: X\r\nHost: example.com\r\n\r\nabcd", // set to ensure all bytes are read
-			wantError:  stderrors.New("strconv.ParseUint: parsing \"X\": invalid syntax"),
+			wantError:  std_errors.New("strconv.ParseUint: parsing \"X\": invalid syntax"),
 			chunkSize:  1,
 		},
 		{
@@ -330,7 +330,7 @@ func TestHTTPNormalizerHTTPRequest(t *testing.T) {
 			maxHeaderSize: 47, // up to end of Cookie header
 			wantOutput:    "POST / HTTP/1.1\r\nContent-Length: 4\r\nCookie: X\r\nRange: 1234 \r\n\r\nabcd",
 			chunkSize:     1,
-			wantError:     stderrors.New("exceeds maxReqLineAndHeadersSize"),
+			wantError:     std_errors.New("exceeds maxReqLineAndHeadersSize"),
 		},
 	}
 
@@ -424,7 +424,7 @@ func TestHTTPNormalizerHTTPServer(t *testing.T) {
 				if string(cookie) == "valid" {
 					return []byte(validateMeekCookieResult), nil
 				}
-				return nil, stderrors.New("invalid cookie")
+				return nil, std_errors.New("invalid cookie")
 			}
 			normalizer.HeaderWriteOrder = []string{"Host", "Cookie", "Content-Length"}
 
@@ -469,7 +469,7 @@ func TestHTTPNormalizerHTTPServer(t *testing.T) {
 
 				_, err = conn.Write([]byte(listenerType))
 				if err != nil {
-					if stderrors.Is(err, ErrPassthroughActive) {
+					if std_errors.Is(err, ErrPassthroughActive) {
 						return
 					}
 					recv <- &listenerState{