Allow Athens to Propagate Authentication to Mod Download

.drone.yml

@@ -59,6 +59,8 @@ steps:
       from_secret: ATHENS_AZURE_ACCOUNT_NAME
     ATHENS_AZURE_ACCOUNT_KEY: 
       from_secret: ATHENS_AZURE_ACCOUNT_KEY
+    PROPAGATE_AUTH_TEST_TOKEN:
+      from_secret: PROPAGATE_AUTH_TEST_TOKEN
   when:
     branch:
       - main

cmd/proxy/actions/app.go

@@ -51,13 +51,16 @@ func App(conf *config.Config) (http.Handler, error) {
 	lggr := log.New(conf.CloudRuntime, logLvl)
 
 	r := mux.NewRouter()
-	r.Use(mw.LogEntryMiddleware(lggr))
-	r.Use(mw.RequestLogger)
-	r.Use(secure.New(secure.Options{
-		SSLRedirect:     conf.ForceSSL,
-		SSLProxyHeaders: map[string]string{"X-Forwarded-Proto": "https"},
-	}).Handler)
-	r.Use(mw.ContentType)
+	r.Use(
+		mw.LogEntryMiddleware(lggr),
+		mw.RequestLogger,
+		secure.New(secure.Options{
+			SSLRedirect:     conf.ForceSSL,
+			SSLProxyHeaders: map[string]string{"X-Forwarded-Proto": "https"},
+		}).Handler,
+		mw.ContentType,
+		mw.WithAuth,
+	)
 
 	var subRouter *mux.Router
 	if prefix := conf.PathPrefix; prefix != "" {

cmd/proxy/actions/app_proxy.go

@@ -96,12 +96,12 @@ func addProxyRoutes(
 	if err := c.GoBinaryEnvVars.Validate(); err != nil {
 		return err
 	}
-	mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoGetDir, c.GoBinaryEnvVars, fs)
+	mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoGetDir, c.GoBinaryEnvVars, fs, c.PropagateAuth, c.PropagateAuthPatterns)
 	if err != nil {
 		return err
 	}
 
-	lister := module.NewVCSLister(c.GoBinary, c.GoBinaryEnvVars, fs)
+	lister := module.NewVCSLister(c.GoBinary, c.GoBinaryEnvVars, fs, c.PropagateAuth, c.PropagateAuthPatterns)
 	checker := storage.WithChecker(s)
 	withSingleFlight, err := getSingleFlight(c, checker)
 	if err != nil {

cmd/proxy/actions/auth.go

@@ -44,20 +44,20 @@ func netrcFromToken(tok string) {
 	if err != nil {
 		log.Fatalf("netrcFromToken: could not get homedir: %v", err)
 	}
-	rcp := filepath.Join(hdir, getNetrcFileName())
+	rcp := filepath.Join(hdir, getNETRCFilename())
 	if err := ioutil.WriteFile(rcp, []byte(fileContent), 0600); err != nil {
 		log.Fatalf("netrcFromToken: could not write to file: %v", err)
 	}
 }
 
 func transformAuthFileName(authFileName string) string {
 	if root := strings.TrimLeft(authFileName, "._"); root == "netrc" {
-		return getNetrcFileName()
+		return getNETRCFilename()
 	}
 	return authFileName
 }
 
-func getNetrcFileName() string {
+func getNETRCFilename() string {
 	if runtime.GOOS == "windows" {
 		return "_netrc"
 	}

config.dev.toml

@@ -166,6 +166,35 @@ BasicAuthUser = ""
 # Env override: BASIC_AUTH_PASS
 BasicAuthPass = ""
 
+# PropagateAuth, when set to true, will pass the Basic Authentication
+# Headers to the "go mod download" operations. This will allow a user
+# to pass their credentials for a private repository and have Athens be
+# able to download and store it. Note that, once a private repository is stored,
+# Athens will naively serve it to anyone who requests it.
+# 
+# Therefore, it is **important** that you
+# make sure you have a ValidatorHook or put Athens behind an auth proxy that always 
+# ensures access to modules are securely authorized.
+# 
+# Note that "go mod download" uses "git clone" which will look for these credentials 
+# in the $HOME directory of the process. Therefore, turning this feature on means that each
+# "go mod download" will have its own $HOME direcotry with only the .netrc file. If
+# your "go mod download" relies on your global $HOME directory (such as .gitconfig), then
+# you must turn this feature off. If you'd like to specify files to be copied from the global
+# $HOME directory to the temporary one, please open an issue at https://github.com/gomods/athens
+# to gauge demand for such a feature before implementing.
+# 
+# Env override: ATHENS_PROPAGATE_AUTH
+PropagateAuth = false
+
+# PropagateAuthPatterns can be used to limit which modules will have
+# the credentials accompanied when calling "go mod download". Note that
+# this is only when a module is not in storage. See PropagateAuth above
+# for more details.
+# 
+# Env override: ATHENS_PROPAGATE_AUTH_PATTERNS
+PropagateAuthPatterns = ["*"]
+
 # Set to true to force an SSL redirect
 # Env override: PROXY_FORCE_SSL
 ForceSSL = false

pkg/auth/auth.go

@@ -0,0 +1,64 @@
+package auth
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"runtime"
+
+	"github.com/gomods/athens/pkg/errors"
+)
+
+type authkey struct{}
+
+// BasicAuth is the embedded credentials in a context
+type BasicAuth struct {
+	User, Password string
+}
+
+// SetAuthInContext sets the auth value in context
+func SetAuthInContext(ctx context.Context, auth BasicAuth) context.Context {
+	return context.WithValue(ctx, authkey{}, auth)
+}
+
+// FromContext retrieves the auth value
+func FromContext(ctx context.Context) (BasicAuth, bool) {
+	auth, ok := ctx.Value(authkey{}).(BasicAuth)
+	return auth, ok
+}
+
+// WriteNETRC writes the netrc file to the specified directory
+func WriteNETRC(path, host, user, password string) error {
+	const op errors.Op = "auth.WriteNETRC"
+	fileContent := fmt.Sprintf("machine %s login %s password %s\n", host, user, password)
+	if err := ioutil.WriteFile(path, []byte(fileContent), 0600); err != nil {
+		return errors.E(op, fmt.Errorf("netrcFromToken: could not write to file: %v", err))
+	}
+	return nil
+}
+
+// WriteTemporaryNETRC writes a netrc file to a temporary directory, returning
+// the directory it was written to.
+func WriteTemporaryNETRC(host, user, password string) (string, error) {
+	const op errors.Op = "auth.WriteTemporaryNETRC"
+	dir, err := ioutil.TempDir("", "netrcp")
+	if err != nil {
+		return "", errors.E(op, err)
+	}
+	rcp := filepath.Join(dir, GetNETRCFilename())
+	err = WriteNETRC(rcp, host, user, password)
+	if err != nil {
+		return "", errors.E(op, err)
+	}
+	return dir, nil
+}
+
+// GetNETRCFilename returns the name of the netrc file
+// according to the contextual platform
+func GetNETRCFilename() string {
+	if runtime.GOOS == "windows" {
+		return "_netrc"
+	}
+	return ".netrc"
+}

pkg/config/config.go

@@ -21,44 +21,46 @@ const defaultConfigFile = "athens.toml"
 // Config provides configuration values for all components
 type Config struct {
 	TimeoutConf
-	GoEnv            string    `validate:"required" envconfig:"GO_ENV"`
-	GoBinary         string    `validate:"required" envconfig:"GO_BINARY_PATH"`
-	GoProxy          string    `envconfig:"GOPROXY"`
-	GoBinaryEnvVars  EnvList   `envconfig:"ATHENS_GO_BINARY_ENV_VARS"`
-	GoGetWorkers     int       `validate:"required" envconfig:"ATHENS_GOGET_WORKERS"`
-	GoGetDir         string    `envconfig:"ATHENS_GOGOET_DIR"`
-	ProtocolWorkers  int       `validate:"required" envconfig:"ATHENS_PROTOCOL_WORKERS"`
-	LogLevel         string    `validate:"required" envconfig:"ATHENS_LOG_LEVEL"`
-	CloudRuntime     string    `validate:"required" envconfig:"ATHENS_CLOUD_RUNTIME"`
-	EnablePprof      bool      `envconfig:"ATHENS_ENABLE_PPROF"`
-	PprofPort        string    `envconfig:"ATHENS_PPROF_PORT"`
-	FilterFile       string    `envconfig:"ATHENS_FILTER_FILE"`
-	TraceExporterURL string    `envconfig:"ATHENS_TRACE_EXPORTER_URL"`
-	TraceExporter    string    `envconfig:"ATHENS_TRACE_EXPORTER"`
-	StatsExporter    string    `envconfig:"ATHENS_STATS_EXPORTER"`
-	StorageType      string    `validate:"required" envconfig:"ATHENS_STORAGE_TYPE"`
-	GlobalEndpoint   string    `envconfig:"ATHENS_GLOBAL_ENDPOINT"` // This feature is not yet implemented
-	Port             string    `envconfig:"ATHENS_PORT"`
-	BasicAuthUser    string    `envconfig:"BASIC_AUTH_USER"`
-	BasicAuthPass    string    `envconfig:"BASIC_AUTH_PASS"`
-	ForceSSL         bool      `envconfig:"PROXY_FORCE_SSL"`
-	ValidatorHook    string    `envconfig:"ATHENS_PROXY_VALIDATOR"`
-	PathPrefix       string    `envconfig:"ATHENS_PATH_PREFIX"`
-	NETRCPath        string    `envconfig:"ATHENS_NETRC_PATH"`
-	GithubToken      string    `envconfig:"ATHENS_GITHUB_TOKEN"`
-	HGRCPath         string    `envconfig:"ATHENS_HGRC_PATH"`
-	TLSCertFile      string    `envconfig:"ATHENS_TLSCERT_FILE"`
-	TLSKeyFile       string    `envconfig:"ATHENS_TLSKEY_FILE"`
-	SumDBs           []string  `envconfig:"ATHENS_SUM_DBS"`
-	NoSumPatterns    []string  `envconfig:"ATHENS_GONOSUM_PATTERNS"`
-	DownloadMode     mode.Mode `envconfig:"ATHENS_DOWNLOAD_MODE"`
-	DownloadURL      string    `envconfig:"ATHENS_DOWNLOAD_URL"`
-	SingleFlightType string    `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"`
-	RobotsFile       string    `envconfig:"ATHENS_ROBOTS_FILE"`
-	IndexType        string    `envconfig:"ATHENS_INDEX_TYPE"`
-	SingleFlight     *SingleFlight
-	Storage          *Storage
-	Index            *Index
+	GoEnv                 string    `validate:"required" envconfig:"GO_ENV"`
+	GoBinary              string    `validate:"required" envconfig:"GO_BINARY_PATH"`
+	GoProxy               string    `envconfig:"GOPROXY"`
+	GoBinaryEnvVars       EnvList   `envconfig:"ATHENS_GO_BINARY_ENV_VARS"`
+	GoGetWorkers          int       `validate:"required" envconfig:"ATHENS_GOGET_WORKERS"`
+	GoGetDir              string    `envconfig:"ATHENS_GOGOET_DIR"`
+	ProtocolWorkers       int       `validate:"required" envconfig:"ATHENS_PROTOCOL_WORKERS"`
+	LogLevel              string    `validate:"required" envconfig:"ATHENS_LOG_LEVEL"`
+	CloudRuntime          string    `validate:"required" envconfig:"ATHENS_CLOUD_RUNTIME"`
+	EnablePprof           bool      `envconfig:"ATHENS_ENABLE_PPROF"`
+	PprofPort             string    `envconfig:"ATHENS_PPROF_PORT"`
+	FilterFile            string    `envconfig:"ATHENS_FILTER_FILE"`
+	TraceExporterURL      string    `envconfig:"ATHENS_TRACE_EXPORTER_URL"`
+	TraceExporter         string    `envconfig:"ATHENS_TRACE_EXPORTER"`
+	StatsExporter         string    `envconfig:"ATHENS_STATS_EXPORTER"`
+	StorageType           string    `validate:"required" envconfig:"ATHENS_STORAGE_TYPE"`
+	GlobalEndpoint        string    `envconfig:"ATHENS_GLOBAL_ENDPOINT"` // This feature is not yet implemented
+	Port                  string    `envconfig:"ATHENS_PORT"`
+	BasicAuthUser         string    `envconfig:"BASIC_AUTH_USER"`
+	BasicAuthPass         string    `envconfig:"BASIC_AUTH_PASS"`
+	PropagateAuth         bool      `envconfig:"ATHENS_PROPAGATE_AUTH"`
+	PropagateAuthPatterns []string  `envconfig:"ATHENS_PROPAGATE_AUTH_PATTERNS"`
+	ForceSSL              bool      `envconfig:"PROXY_FORCE_SSL"`
+	ValidatorHook         string    `envconfig:"ATHENS_PROXY_VALIDATOR"`
+	PathPrefix            string    `envconfig:"ATHENS_PATH_PREFIX"`
+	NETRCPath             string    `envconfig:"ATHENS_NETRC_PATH"`
+	GithubToken           string    `envconfig:"ATHENS_GITHUB_TOKEN"`
+	HGRCPath              string    `envconfig:"ATHENS_HGRC_PATH"`
+	TLSCertFile           string    `envconfig:"ATHENS_TLSCERT_FILE"`
+	TLSKeyFile            string    `envconfig:"ATHENS_TLSKEY_FILE"`
+	SumDBs                []string  `envconfig:"ATHENS_SUM_DBS"`
+	NoSumPatterns         []string  `envconfig:"ATHENS_GONOSUM_PATTERNS"`
+	DownloadMode          mode.Mode `envconfig:"ATHENS_DOWNLOAD_MODE"`
+	DownloadURL           string    `envconfig:"ATHENS_DOWNLOAD_URL"`
+	SingleFlightType      string    `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"`
+	RobotsFile            string    `envconfig:"ATHENS_ROBOTS_FILE"`
+	IndexType             string    `envconfig:"ATHENS_INDEX_TYPE"`
+	SingleFlight          *SingleFlight
+	Storage               *Storage
+	Index                 *Index
 }
 
 // EnvList is a list of key-value environment
@@ -142,29 +144,30 @@ func Load(configFile string) (*Config, error) {
 
 func defaultConfig() *Config {
 	return &Config{
-		GoBinary:         "go",
-		GoBinaryEnvVars:  EnvList{"GOPROXY=direct"},
-		GoEnv:            "development",
-		GoProxy:          "direct",
-		GoGetWorkers:     10,
-		ProtocolWorkers:  30,
-		LogLevel:         "debug",
-		CloudRuntime:     "none",
-		EnablePprof:      false,
-		PprofPort:        ":3001",
-		StatsExporter:    "prometheus",
-		TimeoutConf:      TimeoutConf{Timeout: 300},
-		StorageType:      "memory",
-		Port:             ":3000",
-		SingleFlightType: "memory",
-		GlobalEndpoint:   "http://localhost:3001",
-		TraceExporterURL: "http://localhost:14268",
-		SumDBs:           []string{"https://sum.golang.org"},
-		NoSumPatterns:    []string{},
-		DownloadMode:     "sync",
-		DownloadURL:      "",
-		RobotsFile:       "robots.txt",
-		IndexType:        "none",
+		GoBinary:              "go",
+		GoBinaryEnvVars:       EnvList{"GOPROXY=direct"},
+		GoEnv:                 "development",
+		GoProxy:               "direct",
+		GoGetWorkers:          10,
+		ProtocolWorkers:       30,
+		LogLevel:              "debug",
+		CloudRuntime:          "none",
+		EnablePprof:           false,
+		PprofPort:             ":3001",
+		StatsExporter:         "prometheus",
+		TimeoutConf:           TimeoutConf{Timeout: 300},
+		StorageType:           "memory",
+		Port:                  ":3000",
+		PropagateAuthPatterns: []string{"*"},
+		SingleFlightType:      "memory",
+		GlobalEndpoint:        "http://localhost:3001",
+		TraceExporterURL:      "http://localhost:14268",
+		SumDBs:                []string{"https://sum.golang.org"},
+		NoSumPatterns:         []string{},
+		DownloadMode:          "sync",
+		DownloadURL:           "",
+		RobotsFile:            "robots.txt",
+		IndexType:             "none",
 		SingleFlight: &SingleFlight{
 			Etcd:  &Etcd{"localhost:2379,localhost:22379,localhost:32379"},
 			Redis: &Redis{"127.0.0.1:6379", ""},

pkg/config/config_test.go

@@ -273,26 +273,27 @@ func TestParseExampleConfig(t *testing.T) {
 		TimeoutConf: TimeoutConf{
 			Timeout: 300,
 		},
-		StorageType:      "memory",
-		GlobalEndpoint:   "http://localhost:3001",
-		Port:             ":3000",
-		EnablePprof:      false,
-		PprofPort:        ":3001",
-		BasicAuthUser:    "",
-		BasicAuthPass:    "",
-		Storage:          expStorage,
-		TraceExporterURL: "http://localhost:14268",
-		TraceExporter:    "",
-		StatsExporter:    "prometheus",
-		SingleFlightType: "memory",
-		GoBinaryEnvVars:  []string{"GOPROXY=direct"},
-		SingleFlight:     &SingleFlight{},
-		SumDBs:           []string{"https://sum.golang.org"},
-		NoSumPatterns:    []string{},
-		DownloadMode:     "sync",
-		RobotsFile:       "robots.txt",
-		IndexType:        "none",
-		Index:            &Index{},
+		StorageType:           "memory",
+		GlobalEndpoint:        "http://localhost:3001",
+		Port:                  ":3000",
+		PropagateAuthPatterns: []string{"*"},
+		EnablePprof:           false,
+		PprofPort:             ":3001",
+		BasicAuthUser:         "",
+		BasicAuthPass:         "",
+		Storage:               expStorage,
+		TraceExporterURL:      "http://localhost:14268",
+		TraceExporter:         "",
+		StatsExporter:         "prometheus",
+		SingleFlightType:      "memory",
+		GoBinaryEnvVars:       []string{"GOPROXY=direct"},
+		SingleFlight:          &SingleFlight{},
+		SumDBs:                []string{"https://sum.golang.org"},
+		NoSumPatterns:         []string{},
+		DownloadMode:          "sync",
+		RobotsFile:            "robots.txt",
+		IndexType:             "none",
+		Index:                 &Index{},
 	}
 
 	absPath, err := filepath.Abs(testConfigFile(t))

pkg/download/protocol_test.go

@@ -37,7 +37,7 @@ func getDP(t *testing.T) Protocol {
 	}
 	goBin := conf.GoBinary
 	fs := afero.NewOsFs()
-	mf, err := module.NewGoGetFetcher(goBin, conf.GoGetDir, conf.GoBinaryEnvVars, fs)
+	mf, err := module.NewGoGetFetcher(goBin, conf.GoGetDir, conf.GoBinaryEnvVars, fs, false, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -46,7 +46,7 @@ func getDP(t *testing.T) Protocol {
 		t.Fatal(err)
 	}
 	st := stash.New(mf, s, nop.New())
-	return New(&Opts{s, st, module.NewVCSLister(goBin, conf.GoBinaryEnvVars, fs), nil})
+	return New(&Opts{s, st, module.NewVCSLister(goBin, conf.GoBinaryEnvVars, fs, false, nil), nil})
 }
 
 type listTest struct {