Allow Athens to Propagate Authentication to Mod Download

cmd/proxy/actions/app.go

@@ -51,13 +51,16 @@ func App(conf *config.Config) (http.Handler, error) {
 	lggr := log.New(conf.CloudRuntime, logLvl)
 
 	r := mux.NewRouter()
-	r.Use(mw.LogEntryMiddleware(lggr))
-	r.Use(mw.RequestLogger)
-	r.Use(secure.New(secure.Options{
-		SSLRedirect:     conf.ForceSSL,
-		SSLProxyHeaders: map[string]string{"X-Forwarded-Proto": "https"},
-	}).Handler)
-	r.Use(mw.ContentType)
+	r.Use(
+		mw.LogEntryMiddleware(lggr),
+		mw.RequestLogger,
+		secure.New(secure.Options{
+			SSLRedirect:     conf.ForceSSL,
+			SSLProxyHeaders: map[string]string{"X-Forwarded-Proto": "https"},
+		}).Handler,
+		mw.ContentType,
+		mw.WithAuth,
+	)
 
 	var subRouter *mux.Router
 	if prefix := conf.PathPrefix; prefix != "" {

cmd/proxy/actions/app_proxy.go

@@ -96,12 +96,12 @@ func addProxyRoutes(
 	if err := c.GoBinaryEnvVars.Validate(); err != nil {
 		return err
 	}
-	mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoGetDir, c.GoBinaryEnvVars, fs)
+	mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoGetDir, c.GoBinaryEnvVars, fs, c.PropagateAuth)
 	if err != nil {
 		return err
 	}
 
-	lister := module.NewVCSLister(c.GoBinary, c.GoBinaryEnvVars, fs)
+	lister := module.NewVCSLister(c.GoBinary, c.GoBinaryEnvVars, fs, c.PropagateAuth)
 	checker := storage.WithChecker(s)
 	withSingleFlight, err := getSingleFlight(c, checker)
 	if err != nil {

cmd/proxy/actions/auth.go

@@ -44,20 +44,20 @@ func netrcFromToken(tok string) {
 	if err != nil {
 		log.Fatalf("netrcFromToken: could not get homedir: %v", err)
 	}
-	rcp := filepath.Join(hdir, getNetrcFileName())
+	rcp := filepath.Join(hdir, getNETRCFilename())
 	if err := ioutil.WriteFile(rcp, []byte(fileContent), 0600); err != nil {
 		log.Fatalf("netrcFromToken: could not write to file: %v", err)
 	}
 }
 
 func transformAuthFileName(authFileName string) string {
 	if root := strings.TrimLeft(authFileName, "._"); root == "netrc" {
-		return getNetrcFileName()
+		return getNETRCFilename()
 	}
 	return authFileName
 }
 
-func getNetrcFileName() string {
+func getNETRCFilename() string {
 	if runtime.GOOS == "windows" {
 		return "_netrc"
 	}

config.dev.toml

@@ -166,6 +166,27 @@ BasicAuthUser = ""
 # Env override: BASIC_AUTH_PASS
 BasicAuthPass = ""
 
+# PropagateAuth, when set to true, will pass the Basic Authentication
+# Headers to the "go mod download" operations. This will allow a user
+# to pass their credentials for a private repository and have Athens be
+# able to download and store it. Note that, once a private repository is stored,
+# Athens will naively serve it to anyone who requests it.
+# 
+# Therefore, it is **important** that you
+# make sure you have a ValidatorHook or put Athens behind an auth proxy that always 
+# ensures access to modules are securely authorized.
+# 
+# Note that "go mod download" uses "git clone" which will look for these credentials 
+# in the $HOME directory of the process. Therefore, turning this feature on means that each
+# "go mod download" will have its own $HOME direcotry with only the .netrc file. If
+# your "go mod download" relies on your global $HOME directory (such as .gitconfig), then
+# you must turn this feature off. If you'd like to specify files to be copied from the global
+# $HOME directory to the temporary one, please open an issue at https://github.com/gomods/athens
+# to gauge demand for such a feature before implementing.
+# 
+# Env override: ATHENS_PROPAGATE_AUTH
+PropagateAuth = false
+
 # Set to true to force an SSL redirect
 # Env override: PROXY_FORCE_SSL
 ForceSSL = false

pkg/auth/auth.go

@@ -0,0 +1,64 @@
+package auth
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"runtime"
+
+	"github.com/gomods/athens/pkg/errors"
+)
+
+type authkey struct{}
+
+// BasicAuth is the embedded credentials in a context
+type BasicAuth struct {
+	User, Password string
+}
+
+// SetAuthInContext sets the auth value in context
+func SetAuthInContext(ctx context.Context, auth BasicAuth) context.Context {
+	return context.WithValue(ctx, authkey{}, auth)
+}
+
+// FromContext retrieves the auth value
+func FromContext(ctx context.Context) (BasicAuth, bool) {
+	auth, ok := ctx.Value(authkey{}).(BasicAuth)
+	return auth, ok
+}
+
+// WriteNETRC writes the netrc file to the specified directory
+func WriteNETRC(path, host, user, password string) error {
+	const op errors.Op = "auth.WriteNETRC"
+	fileContent := fmt.Sprintf("machine %s login %s password %s\n", host, user, password)
+	if err := ioutil.WriteFile(path, []byte(fileContent), 0600); err != nil {
+		return errors.E(op, fmt.Errorf("netrcFromToken: could not write to file: %v", err))
+	}
+	return nil
+}
+
+// WriteTemporaryNETRC writes a netrc file to a temporary directory, returning
+// the directory it was written to.
+func WriteTemporaryNETRC(host, user, password string) (string, error) {
+	const op errors.Op = "auth.WriteTemporaryNETRC"
+	dir, err := ioutil.TempDir("", "netrcp")
+	if err != nil {
+		return "", errors.E(op, err)
+	}
+	rcp := filepath.Join(dir, GetNETRCFilename())
+	err = WriteNETRC(rcp, host, user, password)
+	if err != nil {
+		return "", errors.E(op, err)
+	}
+	return dir, nil
+}
+
+// GetNETRCFilename returns the name of the netrc file
+// according to the contextual platform
+func GetNETRCFilename() string {
+	if runtime.GOOS == "windows" {
+		return "_netrc"
+	}
+	return ".netrc"
+}

pkg/config/config.go

@@ -41,6 +41,7 @@ type Config struct {
 	Port             string    `envconfig:"ATHENS_PORT"`
 	BasicAuthUser    string    `envconfig:"BASIC_AUTH_USER"`
 	BasicAuthPass    string    `envconfig:"BASIC_AUTH_PASS"`
+	PropagateAuth    bool      `envconfig:"ATHENS_PROPAGATE_AUTH"`
 	ForceSSL         bool      `envconfig:"PROXY_FORCE_SSL"`
 	ValidatorHook    string    `envconfig:"ATHENS_PROXY_VALIDATOR"`
 	PathPrefix       string    `envconfig:"ATHENS_PATH_PREFIX"`

pkg/download/protocol_test.go

@@ -37,7 +37,7 @@ func getDP(t *testing.T) Protocol {
 	}
 	goBin := conf.GoBinary
 	fs := afero.NewOsFs()
-	mf, err := module.NewGoGetFetcher(goBin, conf.GoGetDir, conf.GoBinaryEnvVars, fs)
+	mf, err := module.NewGoGetFetcher(goBin, conf.GoGetDir, conf.GoBinaryEnvVars, fs, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -46,7 +46,7 @@ func getDP(t *testing.T) Protocol {
 		t.Fatal(err)
 	}
 	st := stash.New(mf, s, nop.New())
-	return New(&Opts{s, st, module.NewVCSLister(goBin, conf.GoBinaryEnvVars, fs), nil})
+	return New(&Opts{s, st, module.NewVCSLister(goBin, conf.GoBinaryEnvVars, fs, false), nil})
 }
 
 type listTest struct {

pkg/middleware/auth.go

@@ -0,0 +1,22 @@
+package middleware
+
+import (
+	"net/http"
+
+	"github.com/gomods/athens/pkg/auth"
+)
+
+type authkey struct{}
+
+// WithAuth inserts the Authorization header
+// into the request context
+func WithAuth(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		user, password, ok := r.BasicAuth()
+		if ok {
+			ctx := auth.SetAuthInContext(r.Context(), auth.BasicAuth{User: user, Password: password})
+			r = r.WithContext(ctx)
+		}
+		h.ServeHTTP(w, r)
+	})
+}

pkg/middleware/auth_test.go

@@ -0,0 +1,63 @@
+package middleware
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gomods/athens/pkg/auth"
+)
+
+func TestAuthMiddleware(t *testing.T) {
+	var tests = []struct {
+		name     string
+		reqfunc  func(r *http.Request)
+		wantok   bool
+		wantauth auth.BasicAuth
+	}{
+		{
+			name:    "no auth",
+			reqfunc: func(r *http.Request) {},
+		},
+		{
+			name: "with basic auth",
+			reqfunc: func(r *http.Request) {
+				r.SetBasicAuth("user", "pass")
+			},
+			wantok:   true,
+			wantauth: auth.BasicAuth{User: "user", Password: "pass"},
+		},
+		{
+			name: "only user",
+			reqfunc: func(r *http.Request) {
+				r.SetBasicAuth("justuser", "")
+			},
+			wantok:   true,
+			wantauth: auth.BasicAuth{User: "justuser"},
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			var (
+				givenok   bool
+				givenauth auth.BasicAuth
+			)
+			h := WithAuth(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				givenauth.User, givenauth.Password, givenok = r.BasicAuth()
+			}))
+
+			r := httptest.NewRequest("GET", "/", nil)
+			tc.reqfunc(r)
+			w := httptest.NewRecorder()
+
+			h.ServeHTTP(w, r)
+
+			if givenok != tc.wantok {
+				t.Fatalf("expected basic auth existence to be %t but got %t", tc.wantok, givenok)
+			}
+			if givenauth != tc.wantauth {
+				t.Fatalf("expected basic auth to be %+v but got %+v", tc.wantauth, givenauth)
+			}
+		})
+	}
+}

pkg/module/all_test.go

@@ -13,6 +13,9 @@ const (
 	// github.com/NYTimes/gizmo is a example of a path that needs to be encoded so we can cover that case as well
 	repoURI = "github.com/NYTimes/gizmo"
 	version = "v0.1.4"
+
+	privateRepoURI     = "github.com/athens-artifacts/private"
+	privateRepoVersion = "v0.0.1"
 )
 
 type ModuleSuite struct {

pkg/module/go_get_fetcher.go

@@ -10,17 +10,19 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/gomods/athens/pkg/auth"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/observ"
 	"github.com/gomods/athens/pkg/storage"
 	"github.com/spf13/afero"
 )
 
 type goGetFetcher struct {
-	fs           afero.Fs
-	goBinaryName string
-	envVars      []string
-	gogetDir     string
+	fs            afero.Fs
+	goBinaryName  string
+	envVars       []string
+	gogetDir      string
+	propagateAuth bool
 }
 
 type goModule struct {
@@ -36,16 +38,17 @@ type goModule struct {
 }
 
 // NewGoGetFetcher creates fetcher which uses go get tool to fetch modules
-func NewGoGetFetcher(goBinaryName, gogetDir string, envVars []string, fs afero.Fs) (Fetcher, error) {
+func NewGoGetFetcher(goBinaryName, gogetDir string, envVars []string, fs afero.Fs, propagateAuth bool) (Fetcher, error) {
 	const op errors.Op = "module.NewGoGetFetcher"
 	if err := validGoBinary(goBinaryName); err != nil {
 		return nil, errors.E(op, err)
 	}
 	return &goGetFetcher{
-		fs:           fs,
-		goBinaryName: goBinaryName,
-		envVars:      envVars,
-		gogetDir:     gogetDir,
+		fs:            fs,
+		goBinaryName:  goBinaryName,
+		envVars:       envVars,
+		gogetDir:      gogetDir,
+		propagateAuth: propagateAuth,
 	}, nil
 }
 
@@ -68,7 +71,7 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
 		return nil, errors.E(op, err)
 	}
 
-	m, err := downloadModule(g.goBinaryName, g.envVars, g.fs, goPathRoot, modPath, mod, ver)
+	m, err := g.downloadModule(ctx, goPathRoot, modPath, mod, ver)
 	if err != nil {
 		clearFiles(g.fs, goPathRoot)
 		return nil, errors.E(op, err)
@@ -103,19 +106,33 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
 
 // given a filesystem, gopath, repository root, module and version, runs 'go mod download -json'
 // on module@version from the repoRoot with GOPATH=gopath, and returns a non-nil error if anything went wrong.
-func downloadModule(goBinaryName string, envVars []string, fs afero.Fs, gopath, repoRoot, module, version string) (goModule, error) {
+func (g *goGetFetcher) downloadModule(ctx context.Context, gopath, repoRoot, module, version string) (goModule, error) {
 	const op errors.Op = "module.downloadModule"
+	creds, ok := auth.FromContext(ctx)
+	var (
+		netrcDir string
+		err      error
+	)
+	if ok && g.propagateAuth {
+		host := strings.Split(module, "/")[0]
+		netrcDir, err = auth.WriteTemporaryNETRC(host, creds.User, creds.Password)
+		if err != nil {
+			return goModule{}, errors.E(op, err)
+		}
+		defer os.RemoveAll(netrcDir)
+	}
 	uri := strings.TrimSuffix(module, "/")
-	fullURI := fmt.Sprintf("%s@%s", uri, version)
 
-	cmd := exec.Command(goBinaryName, "mod", "download", "-json", fullURI)
-	cmd.Env = prepareEnv(gopath, envVars)
+	fullURI := fmt.Sprintf("%s@%s", uri, version)
+	cmd := exec.CommandContext(ctx, g.goBinaryName, "mod", "download", "-json", fullURI)
+	cmd.Env = prepareEnv(gopath, netrcDir, g.envVars)
 	cmd.Dir = repoRoot
 	stdout := &bytes.Buffer{}
 	stderr := &bytes.Buffer{}
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
-	err := cmd.Run()
+
+	err = cmd.Run()
 	if err != nil {
 		err = fmt.Errorf("%v: %s", err, stderr)
 		var m goModule