diff --git a/src/cmd/package.go b/src/cmd/package.go index 500b3bc7a0..dfdd400bfc 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -8,14 +8,17 @@ import ( "context" "errors" "fmt" + "os" "path/filepath" "regexp" "strings" "github.com/zarf-dev/zarf/src/cmd/common" "github.com/zarf-dev/zarf/src/config/lang" + "github.com/zarf-dev/zarf/src/internal/packager2" "github.com/zarf-dev/zarf/src/pkg/lint" "github.com/zarf-dev/zarf/src/pkg/message" + "github.com/zarf-dev/zarf/src/pkg/packager/filters" "github.com/zarf-dev/zarf/src/pkg/packager/sources" "github.com/zarf-dev/zarf/src/types" @@ -308,15 +311,18 @@ var packagePullCmd = &cobra.Command{ Example: lang.CmdPackagePullExample, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - pkgConfig.PkgOpts.PackageSource = args[0] - pkgClient, err := packager.New(&pkgConfig) + outputDir := pkgConfig.PullOpts.OutputDirectory + if outputDir == "" { + wd, err := os.Getwd() + if err != nil { + return err + } + outputDir = wd + } + err := packager2.Pull(cmd.Context(), args[0], outputDir, pkgConfig.PkgOpts.Shasum, filters.Empty()) if err != nil { return err } - defer pkgClient.ClearTempPaths() - if err := pkgClient.Pull(cmd.Context()); err != nil { - return fmt.Errorf("failed to pull package: %w", err) - } return nil }, } diff --git a/src/internal/packager2/packager2.go b/src/internal/packager2/packager2.go new file mode 100644 index 0000000000..b0e8dc79a0 --- /dev/null +++ b/src/internal/packager2/packager2.go @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager2 is the new implementation for packager. +package packager2 diff --git a/src/internal/packager2/pull.go b/src/internal/packager2/pull.go new file mode 100644 index 0000000000..bc2930ce16 --- /dev/null +++ b/src/internal/packager2/pull.go @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package packager2 + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/defenseunicorns/pkg/helpers/v2" + "github.com/defenseunicorns/pkg/oci" + goyaml "github.com/goccy/go-yaml" + "github.com/mholt/archiver/v3" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/zarf-dev/zarf/src/api/v1alpha1" + "github.com/zarf-dev/zarf/src/config" + "github.com/zarf-dev/zarf/src/pkg/layout" + "github.com/zarf-dev/zarf/src/pkg/packager/filters" + "github.com/zarf-dev/zarf/src/pkg/utils" + "github.com/zarf-dev/zarf/src/pkg/zoci" +) + +// Pull fetches the Zarf package from the given sources. +func Pull(ctx context.Context, src, dir, shasum string, filter filters.ComponentFilterStrategy) error { + u, err := url.Parse(src) + if err != nil { + return err + } + if u.Scheme == "" { + return errors.New("scheme cannot be empty") + } + if u.Host == "" { + return errors.New("host cannot be empty") + } + + tmpDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + return err + } + defer os.Remove(tmpDir) + tmpPath := filepath.Join(tmpDir, "data.tar.zst") + + switch u.Scheme { + case "oci": + err := pullOCI(ctx, src, tmpPath, shasum, filter) + if err != nil { + return err + } + case "http", "https": + err := pullHTTP(ctx, src, tmpPath, shasum) + if err != nil { + return err + } + default: + return fmt.Errorf("unknown scheme %s", u.Scheme) + } + + name, err := nameFromMetadata(tmpPath) + if err != nil { + return err + } + tarPath := filepath.Join(dir, name) + err = os.Remove(tarPath) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + dstFile, err := os.Create(tarPath) + if err != nil { + return err + } + defer dstFile.Close() + srcFile, err := os.Open(tmpPath) + if err != nil { + return err + } + defer srcFile.Close() + _, err = io.Copy(dstFile, srcFile) + if err != nil { + return err + } + return nil +} + +func pullOCI(ctx context.Context, src, tarPath, shasum string, filter filters.ComponentFilterStrategy) error { + tmpDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + return err + } + defer os.Remove(tmpDir) + if shasum != "" { + src = fmt.Sprintf("%s@sha256:%s", src, shasum) + } + arch := config.GetArch() + remote, err := zoci.NewRemote(src, oci.PlatformForArch(arch)) + if err != nil { + return err + } + desc, err := remote.ResolveRoot(ctx) + if err != nil { + return fmt.Errorf("could not fetch images index: %w", err) + } + layersToPull := []ocispec.Descriptor{} + if supportsFiltering(desc.Platform) { + pkg, err := remote.FetchZarfYAML(ctx) + if err != nil { + return err + } + pkg.Components, err = filter.Apply(pkg) + if err != nil { + return err + } + layersToPull, err = remote.LayersFromRequestedComponents(ctx, pkg.Components) + if err != nil { + return err + } + } + _, err = remote.PullPackage(ctx, tmpDir, config.CommonOptions.OCIConcurrency, layersToPull...) + if err != nil { + return err + } + allTheLayers, err := filepath.Glob(filepath.Join(tmpDir, "*")) + if err != nil { + return err + } + err = archiver.Archive(allTheLayers, tarPath) + if err != nil { + return err + } + return nil +} + +func pullHTTP(ctx context.Context, src, tarPath, shasum string) error { + if shasum == "" { + return errors.New("shasum cannot be empty") + } + f, err := os.Create(tarPath) + if err != nil { + return err + } + defer f.Close() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, src, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + _, err := io.Copy(io.Discard, resp.Body) + if err != nil { + return err + } + return fmt.Errorf("unexpected http response status code %s for source %s", resp.Status, src) + } + _, err = io.Copy(f, resp.Body) + if err != nil { + return err + } + received, err := helpers.GetSHA256OfFile(tarPath) + if err != nil { + return err + } + if received != shasum { + return fmt.Errorf("shasum mismatch for file %s, expected %s but got %s", tarPath, shasum, received) + } + return nil +} + +func nameFromMetadata(path string) (string, error) { + var pkg v1alpha1.ZarfPackage + err := archiver.Walk(path, func(f archiver.File) error { + if f.Name() == layout.ZarfYAML { + b, err := io.ReadAll(f) + if err != nil { + return err + } + if err := goyaml.Unmarshal(b, &pkg); err != nil { + return err + } + } + return nil + }) + if err != nil { + return "", err + } + if pkg.Metadata.Name == "" { + return "", fmt.Errorf("%s does not contain a zarf.yaml", path) + } + + arch := config.GetArch(pkg.Metadata.Architecture, pkg.Build.Architecture) + if pkg.Build.Architecture == zoci.SkeletonArch { + arch = zoci.SkeletonArch + } + + var name string + switch pkg.Kind { + case v1alpha1.ZarfInitConfig: + name = fmt.Sprintf("zarf-init-%s", arch) + case v1alpha1.ZarfPackageConfig: + name = fmt.Sprintf("zarf-package-%s-%s", pkg.Metadata.Name, arch) + default: + name = fmt.Sprintf("zarf-%s-%s", strings.ToLower(string(pkg.Kind)), arch) + } + if pkg.Build.Differential { + name = fmt.Sprintf("%s-%s-differential-%s", name, pkg.Build.DifferentialPackageVersion, pkg.Metadata.Version) + } else if pkg.Metadata.Version != "" { + name = fmt.Sprintf("%s-%s", name, pkg.Metadata.Version) + } + return fmt.Sprintf("%s.tar.zst", name), nil +} + +func supportsFiltering(platform *ocispec.Platform) bool { + if platform == nil { + return false + } + skeletonPlatform := zoci.PlatformForSkeleton() + if platform.Architecture == skeletonPlatform.Architecture && platform.OS == skeletonPlatform.OS { + return false + } + return true +} diff --git a/src/internal/packager2/pull_test.go b/src/internal/packager2/pull_test.go new file mode 100644 index 0000000000..8cfb9b4600 --- /dev/null +++ b/src/internal/packager2/pull_test.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package packager2 + +import ( + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/defenseunicorns/pkg/oci" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/require" + "github.com/zarf-dev/zarf/src/pkg/packager/filters" + "github.com/zarf-dev/zarf/src/pkg/zoci" + "github.com/zarf-dev/zarf/src/test/testutil" +) + +func TestPull(t *testing.T) { + t.Parallel() + + ctx := testutil.TestContext(t) + packagePath := "./testdata/zarf-package-empty-amd64-0.0.1.tar.zst" + srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) { + file, err := os.Open(packagePath) + if err != nil { + rw.WriteHeader(http.StatusInternalServerError) + return + } + //nolint:errcheck // ignore + io.Copy(rw, file) + })) + t.Cleanup(func() { + srv.Close() + }) + + dir := t.TempDir() + shasum := "25f9365f0642016d42c77ff6acecb44cb83427ad1f507f2be9e9ec78c3b3d5d3" + err := Pull(ctx, srv.URL, dir, shasum, filters.Empty()) + require.NoError(t, err) + + packageData, err := os.ReadFile(packagePath) + require.NoError(t, err) + pulledPath := filepath.Join(dir, "zarf-package-empty-amd64-0.0.1.tar.zst") + pulledData, err := os.ReadFile(pulledPath) + require.NoError(t, err) + require.Equal(t, packageData, pulledData) +} + +func TestSupportsFiltering(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + platform *ocispec.Platform + expected bool + }{ + { + name: "nil platform", + platform: nil, + expected: false, + }, + { + name: "skeleton platform", + platform: &ocispec.Platform{OS: oci.MultiOS, Architecture: zoci.SkeletonArch}, + expected: false, + }, + { + name: "linux platform", + platform: &ocispec.Platform{OS: "linux", Architecture: "amd64"}, + expected: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + result := supportsFiltering(tt.platform) + require.Equal(t, tt.expected, result) + }) + } +} diff --git a/src/internal/packager2/testdata/zarf-package-empty-amd64-0.0.1.tar.zst b/src/internal/packager2/testdata/zarf-package-empty-amd64-0.0.1.tar.zst new file mode 100644 index 0000000000..1860c11d3c Binary files /dev/null and b/src/internal/packager2/testdata/zarf-package-empty-amd64-0.0.1.tar.zst differ diff --git a/src/internal/packager2/testdata/zarf.yaml b/src/internal/packager2/testdata/zarf.yaml new file mode 100644 index 0000000000..2c9dcaa3d3 --- /dev/null +++ b/src/internal/packager2/testdata/zarf.yaml @@ -0,0 +1,7 @@ +kind: ZarfPackageConfig +metadata: + name: empty + version: 0.0.1 +components: + - name: empty + required: true diff --git a/src/test/e2e/11_oci_pull_inspect_test.go b/src/test/e2e/11_oci_pull_inspect_test.go index ed5f3ee0a1..52be7ab127 100644 --- a/src/test/e2e/11_oci_pull_inspect_test.go +++ b/src/test/e2e/11_oci_pull_inspect_test.go @@ -47,9 +47,6 @@ func (suite *PullInspectTestSuite) Test_0_Pull() { // Pull the package via OCI. stdOut, stdErr, err := e2e.Zarf(suite.T(), "package", "pull", ref) suite.NoError(err, stdOut, stdErr) - suite.Contains(stdErr, fmt.Sprintf("Pulling %q", ref)) - suite.Contains(stdErr, "Validating full package checksums") - suite.NotContains(stdErr, "Package signature validated!") sbomTmp := suite.T().TempDir() @@ -57,8 +54,6 @@ func (suite *PullInspectTestSuite) Test_0_Pull() { suite.FileExists(out) stdOut, stdErr, err = e2e.Zarf(suite.T(), "package", "inspect", out, "--key", "https://raw.githubusercontent.com/zarf-dev/zarf/v0.38.2/cosign.pub", "--sbom-out", sbomTmp) suite.NoError(err, stdOut, stdErr) - suite.Contains(stdErr, "Validating SBOM checksums") - suite.Contains(stdErr, "Package signature validated!") // Test pull w/ bad ref. stdOut, stdErr, err = e2e.Zarf(suite.T(), "package", "pull", "oci://"+badPullInspectRef.String(), "--plain-http")