Cache remote config modules

bitrise-io · Jul 8, 2024 · defc1fc · defc1fc
1 parent 1467012
commit defc1fc
Show file tree

Hide file tree

Showing 5 changed files with 124 additions and 14 deletions.
diff --git a/cli/merge.go b/cli/merge.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/bitrise-io/bitrise/configmerge"
+	"github.com/bitrise-io/bitrise/configs"
 	"github.com/bitrise-io/bitrise/models"
 	logV2 "github.com/bitrise-io/go-utils/v2/log"
 	"github.com/urfave/cli"
@@ -28,7 +29,8 @@ func mergeConfig(c *cli.Context) error {
 
 	repoInfoProvider := configmerge.NewRepoInfoProvider()
 	fileReader := configmerge.NewFileReader(logV2.NewLogger())
-	merger := configmerge.NewMerger(repoInfoProvider, fileReader)
+	fileCache := configmerge.NewFileCache(configs.GetBitriseConfigCacheDirPath(), logV2.NewLogger())
+	merger := configmerge.NewMerger(repoInfoProvider, fileReader, fileCache, logV2.NewLogger())
 	mergedConfigContent, configFileTree, err := merger.MergeConfig(configPth)
 	if err != nil {
 		return fmt.Errorf("failed to merge config: %s", err)

diff --git a/configmerge/configmerge.go b/configmerge/configmerge.go
@@ -4,6 +4,7 @@ import (
 	"path/filepath"
 
 	"github.com/bitrise-io/bitrise/models"
+	logV2 "github.com/bitrise-io/go-utils/v2/log"
 	"gopkg.in/yaml.v2"
 )
 
@@ -16,17 +17,26 @@ type FileReader interface {
 	ReadFileFromGitRepository(repository string, branch string, commit string, tag string, path string) ([]byte, error)
 }
 
+type FileCache interface {
+	GetFileContent(key string) ([]byte, error)
+	SetFileContent(key string, content []byte) error
+}
+
 type Merger struct {
 	repoInfoProvider RepoInfoProvider
 	fileReader       FileReader
+	fileCache        FileCache
+	logger           logV2.Logger
 
 	repoInfo RepoInfo
 }
 
-func NewMerger(repoInfoProvider RepoInfoProvider, fileReader FileReader) Merger {
+func NewMerger(repoInfoProvider RepoInfoProvider, fileReader FileReader, fileCache FileCache, logger logV2.Logger) Merger {
 	return Merger{
 		repoInfoProvider: repoInfoProvider,
 		fileReader:       fileReader,
+		fileCache:        fileCache,
+		logger:           logger,
 	}
 }
 
@@ -38,7 +48,7 @@ func (m *Merger) MergeConfig(mainConfigPth string) (string, *models.ConfigFileTr
 	}
 	m.repoInfo = *info
 
-	ref := configReference{
+	ref := ConfigReference{
 		Repository: info.DefaultRemoteURL,
 		Commit:     info.Commit,
 		Tag:        info.Tag,
@@ -64,9 +74,9 @@ func (m *Merger) MergeConfig(mainConfigPth string) (string, *models.ConfigFileTr
 	return mergedConfigContent, configTree, nil
 }
 
-func (m *Merger) buildConfigTree(configContent []byte, reference configReference) (*models.ConfigFileTreeModel, error) {
+func (m *Merger) buildConfigTree(configContent []byte, reference ConfigReference) (*models.ConfigFileTreeModel, error) {
 	var config struct {
-		Include []configReference `yaml:"include" json:"include"`
+		Include []ConfigReference `yaml:"include" json:"include"`
 	}
 	if err := yaml.Unmarshal(configContent, &config); err != nil {
 		return nil, err
@@ -94,7 +104,7 @@ func (m *Merger) buildConfigTree(configContent []byte, reference configReference
 	}, nil
 }
 
-func (m *Merger) readConfigModule(reference configReference, info RepoInfo) ([]byte, error) {
+func (m *Merger) readConfigModule(reference ConfigReference, info RepoInfo) ([]byte, error) {
 	localReference, err := isLocalReference(reference, info)
 	if err != nil {
 		return nil, err
@@ -103,11 +113,31 @@ func (m *Merger) readConfigModule(reference configReference, info RepoInfo) ([]b
 	if localReference {
 		return m.readLocalConfigModule(reference)
 	} else {
-		return m.readRemoteConfigModule(reference)
+		if m.fileCache != nil {
+			b, err := m.fileCache.GetFileContent(reference.Key())
+			if err != nil {
+				m.logger.Warnf("Failed to read file (%s) from cache: %s", err)
+			} else if len(b) > 0 {
+				return b, nil
+			}
+		}
+
+		b, err := m.readRemoteConfigModule(reference)
+		if err != nil {
+			return nil, err
+		}
+
+		if m.fileCache != nil {
+			if err := m.fileCache.SetFileContent(reference.Key(), b); err != nil {
+				m.logger.Warnf("Failed to cache file (%s): %s", err)
+			}
+		}
+
+		return b, nil
 	}
 }
 
-func isLocalReference(reference configReference, info RepoInfo) (bool, error) {
+func isLocalReference(reference ConfigReference, info RepoInfo) (bool, error) {
 	if reference.Repository == "" {
 		return true, nil
 	}
@@ -139,11 +169,11 @@ func isLocalReference(reference configReference, info RepoInfo) (bool, error) {
 	return true, nil
 }
 
-func (m *Merger) readLocalConfigModule(reference configReference) ([]byte, error) {
+func (m *Merger) readLocalConfigModule(reference ConfigReference) ([]byte, error) {
 	return m.fileReader.ReadFileFromFileSystem(reference.Path)
 }
 
-func (m *Merger) readRemoteConfigModule(reference configReference) ([]byte, error) {
+func (m *Merger) readRemoteConfigModule(reference ConfigReference) ([]byte, error) {
 	return m.fileReader.ReadFileFromGitRepository(reference.Repository, reference.Branch, reference.Commit, reference.Tag, reference.Path)
 
 }
diff --git a/configmerge/file_cache.go b/configmerge/file_cache.go
@@ -0,0 +1,60 @@
+package configmerge
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+
+	logV2 "github.com/bitrise-io/go-utils/v2/log"
+)
+
+type fileCache struct {
+	logger   logV2.Logger
+	cacheDir string
+}
+
+func NewFileCache(cacheDir string, logger logV2.Logger) FileCache {
+	return fileCache{
+		logger:   logger,
+		cacheDir: cacheDir,
+	}
+}
+
+func (c fileCache) GetFileContent(key string) ([]byte, error) {
+	pth := filepath.Join(c.cacheDir, key)
+	f, err := os.Open(pth)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+
+		return nil, err
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			c.logger.Warnf("Failed to close cache file: %s", err)
+		}
+	}()
+	return io.ReadAll(f)
+}
+
+func (c fileCache) SetFileContent(key string, content []byte) error {
+	pth := filepath.Join(c.cacheDir, key)
+	f, err := os.Create(pth)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			c.logger.Warnf("Failed to close cache file: %s", err)
+		}
+	}()
+	if _, err := f.Write(content); err != nil {
+		return err
+	}
+	return nil
+}
+
+func cachePathFromReference(reference ConfigReference) string {
+	return filepath.Join(reference.Repository, reference.Commit, reference.Tag, reference.Branch, reference.Path)
+}
diff --git a/configmerge/reference.go b/configmerge/reference.go
@@ -1,19 +1,24 @@
 package configmerge
 
-import "fmt"
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
 
-type configReference struct {
+type ConfigReference struct {
 	Repository string `yaml:"repository" json:"repository"`
 	Branch     string `yaml:"branch" json:"branch"`
 	Commit     string `yaml:"commit" json:"commit"`
 	Tag        string `yaml:"tag" json:"tag"`
 	Path       string `yaml:"path" json:"path"`
 }
 
-func (r configReference) Key() string {
+func (r ConfigReference) Key() string {
 	var key string
 	if r.Repository != "" {
-		key = fmt.Sprintf("repo:%s,%s", r.Repository, r.Path)
+		key = fmt.Sprintf("%s/%s", r.Repository, r.Path)
 	} else {
 		key = r.Path
 	}
@@ -26,5 +31,9 @@ func (r configReference) Key() string {
 		key += fmt.Sprintf("@%s", r.Branch)
 	}
 
+	key = filepath.FromSlash(key)
+	key = strings.ReplaceAll(key, string(os.PathSeparator), "_")
+	key = strings.ReplaceAll(key, ":", "_")
+
 	return key
 }
diff --git a/configs/paths.go b/configs/paths.go
@@ -44,6 +44,10 @@ func GetBitriseToolsDirPath() string {
 	return filepath.Join(GetBitriseHomeDirPath(), "tools")
 }
 
+func GetBitriseConfigCacheDirPath() string {
+	return filepath.Join(GetBitriseHomeDirPath(), "config-cache")
+}
+
 func initBitriseWorkPaths() error {
 	bitriseWorkDirPath, err := pathutil.NormalizedOSTempDirPath("bitrise")
 	if err != nil {
@@ -113,6 +117,11 @@ func InitPaths() error {
 				return fmt.Errorf("Failed to set PATH to include BITRISE_HOME/tools! Error: %s", err)
 			}
 		}
+
+		configCacheDir := GetBitriseConfigCacheDirPath()
+		if err := pathutil.EnsureDirExist(configCacheDir); err != nil {
+			return err
+		}
 	}
 
 	inputEnvstorePath, err := filepath.Abs(filepath.Join(BitriseWorkDirPath, "input_envstore.yml"))