From fd91f37ab6b97ec148f53b97ad1b663d296260f1 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Fri, 20 Sep 2024 15:35:13 -0400 Subject: [PATCH 1/6] WIP Signed-off-by: Grant Linville --- pkg/credentials/store.go | 2 +- pkg/credentials/util.go | 32 ++++++++++++++++++++++++---- pkg/repos/get.go | 46 +++++++++++++++++++++++----------------- 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/pkg/credentials/store.go b/pkg/credentials/store.go index 749aba3a..88e70ab5 100644 --- a/pkg/credentials/store.go +++ b/pkg/credentials/store.go @@ -44,7 +44,7 @@ func NewStore(cfg *config.CLIConfig, credentialBuilder CredentialBuilder, credCt return Store{ credCtxs: credCtxs, credBuilder: credentialBuilder, - credHelperDirs: GetCredentialHelperDirs(cacheDir), + credHelperDirs: GetCredentialHelperDirs(cacheDir, cfg.CredentialsStore), cfg: cfg, }, nil } diff --git a/pkg/credentials/util.go b/pkg/credentials/util.go index 39200369..c859a778 100644 --- a/pkg/credentials/util.go +++ b/pkg/credentials/util.go @@ -1,18 +1,42 @@ package credentials import ( + "fmt" "path/filepath" + + runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" ) type CredentialHelperDirs struct { RevisionFile, LastCheckedFile, BinDir string } -func GetCredentialHelperDirs(cacheDir string) CredentialHelperDirs { +func RepoNameForCredentialStore(store string) string { + switch store { + case "sqlite": + return "gptscript-credential-sqlite" + default: + return "gptscript-credential-helpers" + } +} + +func GitURLForRepoName(repoName string) (string, error) { + switch repoName { + case "gptscript-credential-sqlite": + return runtimeEnv.VarOrDefault("GPTSCRIPT_CRED_SQLITE_ROOT", "https://github.com/gptscript-ai/gptscript-credential-sqlite.git"), nil + case "gptscript-credential-helpers": + return runtimeEnv.VarOrDefault("GPTSCRIPT_CRED_HELPERS_ROOT", "https://github.com/gptscript-ai/gptscript-credential-helpers.git"), nil + default: + return "", fmt.Errorf("unknown repo name: %s", repoName) + } +} + +func GetCredentialHelperDirs(cacheDir, store string) CredentialHelperDirs { + repoName := RepoNameForCredentialStore(store) return CredentialHelperDirs{ - RevisionFile: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "revision"), - LastCheckedFile: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "last-checked"), - BinDir: filepath.Join(cacheDir, "repos", "gptscript-credential-helpers", "bin"), + RevisionFile: filepath.Join(cacheDir, "repos", repoName, "revision"), + LastCheckedFile: filepath.Join(cacheDir, "repos", repoName, "last-checked"), + BinDir: filepath.Join(cacheDir, "repos", repoName, "bin"), } } diff --git a/pkg/repos/get.go b/pkg/repos/get.go index 8346b8cf..883b0362 100644 --- a/pkg/repos/get.go +++ b/pkg/repos/get.go @@ -16,7 +16,6 @@ import ( "github.com/BurntSushi/locker" "github.com/gptscript-ai/gptscript/pkg/config" "github.com/gptscript-ai/gptscript/pkg/credentials" - runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/hash" "github.com/gptscript-ai/gptscript/pkg/repos/git" "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/golang" @@ -55,10 +54,10 @@ func (n noopRuntime) Setup(_ context.Context, _ types.Tool, _, _ string, _ []str } type Manager struct { + cacheDir string storageDir string gitDir string runtimeDir string - credHelperDirs credentials.CredentialHelperDirs runtimes []Runtime credHelperConfig *credHelperConfig } @@ -72,11 +71,11 @@ type credHelperConfig struct { func New(cacheDir string, runtimes ...Runtime) *Manager { root := filepath.Join(cacheDir, "repos") return &Manager{ - storageDir: root, - gitDir: filepath.Join(root, "git"), - runtimeDir: filepath.Join(root, "runtimes"), - credHelperDirs: credentials.GetCredentialHelperDirs(cacheDir), - runtimes: runtimes, + cacheDir: cacheDir, + storageDir: root, + gitDir: filepath.Join(root, "git"), + runtimeDir: filepath.Join(root, "runtimes"), + runtimes: runtimes, } } @@ -120,40 +119,49 @@ func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *co distInfo = fmt.Sprintf("-%s-%s", runtime.GOOS, runtime.GOARCH) } - locker.Lock("gptscript-credential-helpers") - defer locker.Unlock("gptscript-credential-helpers") + repoName := credentials.RepoNameForCredentialStore(helperName) + + locker.Lock(repoName) + defer locker.Unlock(repoName) + + credHelperDirs := credentials.GetCredentialHelperDirs(m.cacheDir, helperName) // Load the last-checked file to make sure we haven't checked the repo in the last 24 hours. now := time.Now() - lastChecked, err := os.ReadFile(m.credHelperDirs.LastCheckedFile) + lastChecked, err := os.ReadFile(credHelperDirs.LastCheckedFile) if err == nil { if t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(lastChecked))); err == nil && now.Sub(t) < 24*time.Hour { // Make sure the binary still exists, and if it does, return. - if _, err := os.Stat(filepath.Join(m.credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil { + if _, err := os.Stat(filepath.Join(credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil { log.Debugf("Credential helper %s up-to-date as of %v, checking for updates after %v", helperName, t, t.Add(24*time.Hour)) return nil } } } - if err := os.MkdirAll(filepath.Dir(m.credHelperDirs.LastCheckedFile), 0755); err != nil { + if err := os.MkdirAll(filepath.Dir(credHelperDirs.LastCheckedFile), 0755); err != nil { return err } // Update the last-checked file. - if err := os.WriteFile(m.credHelperDirs.LastCheckedFile, []byte(now.Format(time.RFC3339)), 0644); err != nil { + if err := os.WriteFile(credHelperDirs.LastCheckedFile, []byte(now.Format(time.RFC3339)), 0644); err != nil { + return err + } + + gitURL, err := credentials.GitURLForRepoName(repoName) + if err != nil { return err } tool := types.Tool{ ToolDef: types.ToolDef{ Parameters: types.Parameters{ - Name: "gptscript-credential-helpers", + Name: repoName, }, }, Source: types.ToolSource{ Repo: &types.Repo{ - Root: runtimeEnv.VarOrDefault("GPTSCRIPT_CRED_HELPERS_ROOT", "https://github.com/gptscript-ai/gptscript-credential-helpers.git"), + Root: gitURL, }, }, } @@ -164,12 +172,12 @@ func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *co var needsDownloaded bool // Check the last revision shasum and see if it is different from the current one. - lastRevision, err := os.ReadFile(m.credHelperDirs.RevisionFile) + lastRevision, err := os.ReadFile(credHelperDirs.RevisionFile) if (err == nil && strings.TrimSpace(string(lastRevision)) != tool.Source.Repo.Root+tag) || errors.Is(err, fs.ErrNotExist) { // Need to pull the latest version. needsDownloaded = true // Update the revision file to the new revision. - if err = os.WriteFile(m.credHelperDirs.RevisionFile, []byte(tool.Source.Repo.Root+tag), 0644); err != nil { + if err = os.WriteFile(credHelperDirs.RevisionFile, []byte(tool.Source.Repo.Root+tag), 0644); err != nil { return err } } else if err != nil { @@ -179,7 +187,7 @@ func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *co if !needsDownloaded { // Check for the existence of the credential helper binary. // If it's there, we have no need to download it and can just return. - if _, err = os.Stat(filepath.Join(m.credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil { + if _, err = os.Stat(filepath.Join(credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil { return nil } } @@ -187,7 +195,7 @@ func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *co // Find the Go runtime and use it to build the credential helper. for _, rt := range m.runtimes { if strings.HasPrefix(rt.ID(), "go") { - return rt.(*golang.Runtime).DownloadCredentialHelper(ctx, tool, helperName, distInfo, suffix, m.credHelperDirs.BinDir) + return rt.(*golang.Runtime).DownloadCredentialHelper(ctx, tool, helperName, distInfo, suffix, credHelperDirs.BinDir) } } From 42ced63e1a0010b047cc201cae0135cb6c55c483 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Mon, 23 Sep 2024 11:39:04 -0400 Subject: [PATCH 2/6] WIP Signed-off-by: Grant Linville --- pkg/config/cliconfig.go | 27 ++++++++++++++++++--------- pkg/credentials/store.go | 2 +- pkg/credentials/util.go | 3 ++- pkg/repos/get.go | 4 ++-- pkg/repos/runtimes/golang/golang.go | 3 ++- 5 files changed, 25 insertions(+), 14 deletions(-) diff --git a/pkg/config/cliconfig.go b/pkg/config/cliconfig.go index dd358d52..49d3ffa0 100644 --- a/pkg/config/cliconfig.go +++ b/pkg/config/cliconfig.go @@ -15,13 +15,22 @@ import ( "github.com/docker/cli/cli/config/types" ) -var ( - darwinHelpers = []string{"osxkeychain", "file"} - windowsHelpers = []string{"wincred", "file"} - linuxHelpers = []string{"secretservice", "pass", "file"} +const ( + Wincred = "wincred" + Osxkeychain = "osxkeychain" + Secretservice = "secretservice" + Pass = "pass" + File = "file" + Sqlite = "sqlite" + + GPTScriptHelperPrefix = "gptscript-credential-" ) -const GPTScriptHelperPrefix = "gptscript-credential-" +var ( + darwinHelpers = []string{Osxkeychain, File, Sqlite} + windowsHelpers = []string{Wincred, File, Sqlite} + linuxHelpers = []string{Secretservice, Pass, File, Sqlite} +) type AuthConfig types.AuthConfig @@ -169,11 +178,11 @@ func ReadCLIConfig(gptscriptConfigFile string) (*CLIConfig, error) { func (c *CLIConfig) setDefaultCredentialsStore() error { switch runtime.GOOS { case "darwin": - c.CredentialsStore = "osxkeychain" + c.CredentialsStore = Osxkeychain case "windows": - c.CredentialsStore = "wincred" + c.CredentialsStore = Wincred default: - c.CredentialsStore = "file" + c.CredentialsStore = File } return c.Save() } @@ -187,7 +196,7 @@ func isValidCredentialHelper(helper string) bool { case "linux": return slices.Contains(linuxHelpers, helper) default: - return helper == "file" + return helper == File || helper == Sqlite } } diff --git a/pkg/credentials/store.go b/pkg/credentials/store.go index 88e70ab5..a8351f7e 100644 --- a/pkg/credentials/store.go +++ b/pkg/credentials/store.go @@ -178,7 +178,7 @@ func (s *Store) getStore(ctx context.Context) (credentials.Store, error) { } func (s *Store) getStoreByHelper(ctx context.Context, helper string) (credentials.Store, error) { - if helper == "" || helper == config.GPTScriptHelperPrefix+"file" { + if helper == "" || helper == config.GPTScriptHelperPrefix+config.File { return credentials.NewFileStore(s.cfg), nil } diff --git a/pkg/credentials/util.go b/pkg/credentials/util.go index c859a778..de395526 100644 --- a/pkg/credentials/util.go +++ b/pkg/credentials/util.go @@ -4,6 +4,7 @@ import ( "fmt" "path/filepath" + "github.com/gptscript-ai/gptscript/pkg/config" runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" ) @@ -13,7 +14,7 @@ type CredentialHelperDirs struct { func RepoNameForCredentialStore(store string) string { switch store { - case "sqlite": + case config.Sqlite: return "gptscript-credential-sqlite" default: return "gptscript-credential-helpers" diff --git a/pkg/repos/get.go b/pkg/repos/get.go index 883b0362..c4f051c8 100644 --- a/pkg/repos/get.go +++ b/pkg/repos/get.go @@ -109,11 +109,11 @@ func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *co distInfo, suffix string ) // The file helper is built-in and does not need to be downloaded. - if helperName == "file" { + if helperName == config.File { return nil } switch helperName { - case "wincred": + case config.Wincred: suffix = ".exe" default: distInfo = fmt.Sprintf("-%s-%s", runtime.GOOS, runtime.GOARCH) diff --git a/pkg/repos/runtimes/golang/golang.go b/pkg/repos/runtimes/golang/golang.go index 47e8461f..232fbfe2 100644 --- a/pkg/repos/runtimes/golang/golang.go +++ b/pkg/repos/runtimes/golang/golang.go @@ -18,6 +18,7 @@ import ( "runtime" "strings" + "github.com/gptscript-ai/gptscript/pkg/config" "github.com/gptscript-ai/gptscript/pkg/debugcmd" runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" "github.com/gptscript-ai/gptscript/pkg/hash" @@ -286,7 +287,7 @@ func (r *Runtime) Setup(ctx context.Context, _ types.Tool, dataRoot, toolSource } func (r *Runtime) DownloadCredentialHelper(ctx context.Context, tool types.Tool, helperName, distInfo, suffix string, binDir string) error { - if helperName == "file" { + if helperName == config.File { return nil } From b8d142c5eee9b18f21724e692fa6b26d3b933502 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Mon, 23 Sep 2024 19:43:55 -0400 Subject: [PATCH 3/6] remove sqlite as an option for windows Signed-off-by: Grant Linville --- pkg/config/cliconfig.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/config/cliconfig.go b/pkg/config/cliconfig.go index 49d3ffa0..a7ea2235 100644 --- a/pkg/config/cliconfig.go +++ b/pkg/config/cliconfig.go @@ -28,7 +28,7 @@ const ( var ( darwinHelpers = []string{Osxkeychain, File, Sqlite} - windowsHelpers = []string{Wincred, File, Sqlite} + windowsHelpers = []string{Wincred, File} linuxHelpers = []string{Secretservice, Pass, File, Sqlite} ) @@ -159,11 +159,11 @@ func ReadCLIConfig(gptscriptConfigFile string) (*CLIConfig, error) { errMsg := fmt.Sprintf("invalid credential store '%s'", result.CredentialsStore) switch runtime.GOOS { case "darwin": - errMsg += " (use 'osxkeychain' or 'file')" + errMsg += " (use 'osxkeychain', 'file', or 'sqlite')" case "windows": errMsg += " (use 'wincred' or 'file')" case "linux": - errMsg += " (use 'secretservice', 'pass', or 'file')" + errMsg += " (use 'secretservice', 'pass', 'file', or 'sqlite')" default: errMsg += " (use 'file')" } From a34c7cbdba25d8c2c5d6e16abf0bd221cde842b2 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Mon, 23 Sep 2024 19:46:18 -0400 Subject: [PATCH 4/6] fix Signed-off-by: Grant Linville --- pkg/config/cliconfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/cliconfig.go b/pkg/config/cliconfig.go index a7ea2235..036d8440 100644 --- a/pkg/config/cliconfig.go +++ b/pkg/config/cliconfig.go @@ -196,7 +196,7 @@ func isValidCredentialHelper(helper string) bool { case "linux": return slices.Contains(linuxHelpers, helper) default: - return helper == File || helper == Sqlite + return helper == File } } From 37f0195330e367c50cac3899ca88a62ad755cf6c Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Tue, 24 Sep 2024 09:30:55 -0400 Subject: [PATCH 5/6] PR feedback Signed-off-by: Grant Linville --- pkg/config/cliconfig.go | 44 ++++++++++++++++++----------- pkg/credentials/store.go | 2 +- pkg/credentials/util.go | 2 +- pkg/repos/get.go | 4 +-- pkg/repos/runtimes/golang/golang.go | 2 +- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/pkg/config/cliconfig.go b/pkg/config/cliconfig.go index 036d8440..951da3ae 100644 --- a/pkg/config/cliconfig.go +++ b/pkg/config/cliconfig.go @@ -16,22 +16,32 @@ import ( ) const ( - Wincred = "wincred" - Osxkeychain = "osxkeychain" - Secretservice = "secretservice" - Pass = "pass" - File = "file" - Sqlite = "sqlite" + WincredCredHelper = "wincred" + OsxkeychainCredHelper = "osxkeychain" + SecretserviceCredHelper = "secretservice" + PassCredHelper = "pass" + FileCredHelper = "file" + SqliteCredHelper = "sqlite" GPTScriptHelperPrefix = "gptscript-credential-" ) var ( - darwinHelpers = []string{Osxkeychain, File, Sqlite} - windowsHelpers = []string{Wincred, File} - linuxHelpers = []string{Secretservice, Pass, File, Sqlite} + darwinHelpers = []string{OsxkeychainCredHelper, FileCredHelper, SqliteCredHelper} + windowsHelpers = []string{WincredCredHelper, FileCredHelper} + linuxHelpers = []string{SecretserviceCredHelper, PassCredHelper, FileCredHelper, SqliteCredHelper} ) +func listAsString(helpers []string) string { + if len(helpers) == 0 { + return "" + } else if len(helpers) == 1 { + return helpers[0] + } + + return strings.Join(helpers[:len(helpers)-1], ", ") + " or " + helpers[len(helpers)-1] +} + type AuthConfig types.AuthConfig func (a AuthConfig) MarshalJSON() ([]byte, error) { @@ -159,13 +169,13 @@ func ReadCLIConfig(gptscriptConfigFile string) (*CLIConfig, error) { errMsg := fmt.Sprintf("invalid credential store '%s'", result.CredentialsStore) switch runtime.GOOS { case "darwin": - errMsg += " (use 'osxkeychain', 'file', or 'sqlite')" + errMsg += fmt.Sprintf(" (use %s)", listAsString(darwinHelpers)) case "windows": - errMsg += " (use 'wincred' or 'file')" + errMsg += fmt.Sprintf(" (use %s)", listAsString(windowsHelpers)) case "linux": - errMsg += " (use 'secretservice', 'pass', 'file', or 'sqlite')" + errMsg += fmt.Sprintf(" (use %s)", listAsString(linuxHelpers)) default: - errMsg += " (use 'file')" + errMsg += fmt.Sprintf(" (use file)") } errMsg += fmt.Sprintf("\nPlease edit your config file at %s to fix this.", result.location) @@ -178,11 +188,11 @@ func ReadCLIConfig(gptscriptConfigFile string) (*CLIConfig, error) { func (c *CLIConfig) setDefaultCredentialsStore() error { switch runtime.GOOS { case "darwin": - c.CredentialsStore = Osxkeychain + c.CredentialsStore = OsxkeychainCredHelper case "windows": - c.CredentialsStore = Wincred + c.CredentialsStore = WincredCredHelper default: - c.CredentialsStore = File + c.CredentialsStore = FileCredHelper } return c.Save() } @@ -196,7 +206,7 @@ func isValidCredentialHelper(helper string) bool { case "linux": return slices.Contains(linuxHelpers, helper) default: - return helper == File + return helper == FileCredHelper } } diff --git a/pkg/credentials/store.go b/pkg/credentials/store.go index a8351f7e..f0364851 100644 --- a/pkg/credentials/store.go +++ b/pkg/credentials/store.go @@ -178,7 +178,7 @@ func (s *Store) getStore(ctx context.Context) (credentials.Store, error) { } func (s *Store) getStoreByHelper(ctx context.Context, helper string) (credentials.Store, error) { - if helper == "" || helper == config.GPTScriptHelperPrefix+config.File { + if helper == "" || helper == config.GPTScriptHelperPrefix+config.FileCredHelper { return credentials.NewFileStore(s.cfg), nil } diff --git a/pkg/credentials/util.go b/pkg/credentials/util.go index de395526..72f9eab9 100644 --- a/pkg/credentials/util.go +++ b/pkg/credentials/util.go @@ -14,7 +14,7 @@ type CredentialHelperDirs struct { func RepoNameForCredentialStore(store string) string { switch store { - case config.Sqlite: + case config.SqliteCredHelper: return "gptscript-credential-sqlite" default: return "gptscript-credential-helpers" diff --git a/pkg/repos/get.go b/pkg/repos/get.go index c4f051c8..a36c2fe0 100644 --- a/pkg/repos/get.go +++ b/pkg/repos/get.go @@ -109,11 +109,11 @@ func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *co distInfo, suffix string ) // The file helper is built-in and does not need to be downloaded. - if helperName == config.File { + if helperName == config.FileCredHelper { return nil } switch helperName { - case config.Wincred: + case config.WincredCredHelper: suffix = ".exe" default: distInfo = fmt.Sprintf("-%s-%s", runtime.GOOS, runtime.GOARCH) diff --git a/pkg/repos/runtimes/golang/golang.go b/pkg/repos/runtimes/golang/golang.go index 232fbfe2..f86fa88d 100644 --- a/pkg/repos/runtimes/golang/golang.go +++ b/pkg/repos/runtimes/golang/golang.go @@ -287,7 +287,7 @@ func (r *Runtime) Setup(ctx context.Context, _ types.Tool, dataRoot, toolSource } func (r *Runtime) DownloadCredentialHelper(ctx context.Context, tool types.Tool, helperName, distInfo, suffix string, binDir string) error { - if helperName == config.File { + if helperName == config.FileCredHelper { return nil } From e859ccc853f7a0f87cc0bbc2127a53a2043affca Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Tue, 24 Sep 2024 09:33:51 -0400 Subject: [PATCH 6/6] linter fix Signed-off-by: Grant Linville --- pkg/config/cliconfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/cliconfig.go b/pkg/config/cliconfig.go index 951da3ae..7a82b58a 100644 --- a/pkg/config/cliconfig.go +++ b/pkg/config/cliconfig.go @@ -175,7 +175,7 @@ func ReadCLIConfig(gptscriptConfigFile string) (*CLIConfig, error) { case "linux": errMsg += fmt.Sprintf(" (use %s)", listAsString(linuxHelpers)) default: - errMsg += fmt.Sprintf(" (use file)") + errMsg += " (use file)" } errMsg += fmt.Sprintf("\nPlease edit your config file at %s to fix this.", result.location)