dag: Allow multiple pipeline dirs (#1899)

We can pass multiple pipeline dirs for melange configs, so we should
accept multiple in dag.NewPackages.

Signed-off-by: Jon Johnson <jon.johnson@chainguard.dev>

Author: jonjohnsonjr

docs/cmd/wolfictl_dot.md

@@ -34,7 +34,7 @@ Open browser to explore crane's deps recursively, only showing a minimum subgrap
   -d, --dir string                  directory to search for melange configs (default ".")
   -h, --help                        help for dot
   -k, --keyring-append strings      path to extra keys to include in the build environment keyring (default [https://packages.wolfi.dev/os/wolfi-signing.rsa.pub])
-      --pipeline-dir string         directory used to extend defined built-in pipelines
+      --pipeline-dir strings        directory used to extend defined built-in pipelines
   -R, --recursive                   recurse through package dependencies
   -r, --repository-append strings   path to extra repositories to include in the build environment (default [https://packages.wolfi.dev/os])
   -D, --show-dependents             show packages that depend on these packages, instead of these packages' dependencies

docs/man/man1/wolfictl-dot.1

@@ -54,7 +54,7 @@ wolfictl dot \-\-web \-R \-S crane
     path to extra keys to include in the build environment keyring
 
 .PP
-\fB\-\-pipeline\-dir\fP=""
+\fB\-\-pipeline\-dir\fP=[]
     directory used to extend defined built\-in pipelines
 
 .PP

pkg/cli/dot.go

@@ -21,7 +21,8 @@ import (
 )
 
 func cmdSVG() *cobra.Command { //nolint:gocyclo
-	var dir, pipelineDir string
+	var dir string
+	var pipelineDirs []string
 	var showDependents, recursive, span, web bool
 	var extraKeys, extraRepos []string
 	d := &cobra.Command{
@@ -47,11 +48,11 @@ Open browser to explore crane's deps recursively, only showing a minimum subgrap
 `,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
-			if pipelineDir == "" {
-				pipelineDir = filepath.Join(dir, "pipelines")
+			if len(pipelineDirs) == 0 {
+				pipelineDirs = []string{filepath.Join(dir, "pipelines")}
 			}
 
-			pkgs, err := dag.NewPackages(ctx, os.DirFS(dir), dir, pipelineDir)
+			pkgs, err := dag.NewPackages(ctx, os.DirFS(dir), dir, pipelineDirs)
 			if err != nil {
 				return fmt.Errorf("NewPackages: %w", err)
 			}
@@ -298,7 +299,7 @@ Open browser to explore crane's deps recursively, only showing a minimum subgrap
 		},
 	}
 	d.Flags().StringVarP(&dir, "dir", "d", ".", "directory to search for melange configs")
-	d.Flags().StringVar(&pipelineDir, "pipeline-dir", "", "directory used to extend defined built-in pipelines")
+	d.Flags().StringSliceVar(&pipelineDirs, "pipeline-dir", nil, "directory used to extend defined built-in pipelines")
 	d.Flags().BoolVarP(&showDependents, "show-dependents", "D", false, "show packages that depend on these packages, instead of these packages' dependencies")
 	d.Flags().BoolVarP(&recursive, "recursive", "R", false, "recurse through package dependencies")
 	d.Flags().BoolVarP(&span, "spanning-tree", "S", false, "does something like a spanning tree to avoid a huge number of edges")

pkg/cli/text.go

@@ -13,7 +13,8 @@ import (
 )
 
 func cmdText() *cobra.Command {
-	var dir, pipelineDir, arch, t string
+	var dir, arch, t string
+	var pipelineDirs []string
 	var extraKeys, extraRepos []string
 	text := &cobra.Command{
 		Use:   "text",
@@ -27,13 +28,13 @@ If this fails, there may be an unsatisfiable dependency or a cycle in the graph.
 		Hidden: true,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			ctx := cmd.Context()
-			if pipelineDir == "" {
-				pipelineDir = filepath.Join(dir, "pipelines")
+			if len(pipelineDirs) == 0 {
+				pipelineDirs = []string{filepath.Join(dir, "pipelines")}
 			}
 
 			arch := types.ParseArchitecture(arch).ToAPK()
 
-			pkgs, err := dag.NewPackages(ctx, os.DirFS(dir), dir, pipelineDir)
+			pkgs, err := dag.NewPackages(ctx, os.DirFS(dir), dir, pipelineDirs)
 			if err != nil {
 				return fmt.Errorf("constructing new package set from directory %q: %w", dir, err)
 			}
@@ -49,7 +50,7 @@ If this fails, there may be an unsatisfiable dependency or a cycle in the graph.
 		},
 	}
 	text.Flags().StringVarP(&dir, "dir", "d", ".", "directory to search for melange configs")
-	text.Flags().StringVar(&pipelineDir, "pipeline-dir", "", "directory used to extend defined built-in pipelines")
+	text.Flags().StringSliceVar(&pipelineDirs, "pipeline-dir", nil, "directory used to extend defined built-in pipelines")
 	text.Flags().StringVarP(&arch, "arch", "a", "x86_64", "architecture to build for")
 	text.Flags().StringVarP(&t, "type", "t", string(typeTarget), fmt.Sprintf("What type of text to emit; values can be one of: %v", textTypes))
 	text.Flags().StringSliceVarP(&extraKeys, "keyring-append", "k", []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}, "path to extra keys to include in the build environment keyring")

pkg/dag/graph_test.go

@@ -23,7 +23,7 @@ func TestNewGraph(t *testing.T) {
 			testDir = "testdata/basic"
 		)
 		t.Run("allowed dangling", func(t *testing.T) {
-			pkgs, err := NewPackages(ctx, os.DirFS(testDir), testDir, "")
+			pkgs, err := NewPackages(ctx, os.DirFS(testDir), testDir, nil)
 			require.NoError(t, err)
 			graph, err := NewGraph(ctx, pkgs, WithAllowUnresolved())
 			require.NoError(t, err)
@@ -54,7 +54,7 @@ func TestNewGraph(t *testing.T) {
 			}
 		})
 		t.Run("has expected tree", func(t *testing.T) {
-			pkgs, err := NewPackages(ctx, os.DirFS(testDir), testDir, "")
+			pkgs, err := NewPackages(ctx, os.DirFS(testDir), testDir, nil)
 			require.NoError(t, err)
 			graph, err := NewGraph(ctx, pkgs, WithRepos(packageRepo), WithKeys(key))
 			require.NoError(t, err)
@@ -93,13 +93,13 @@ func TestNewGraph(t *testing.T) {
 		ctx := context.Background()
 		var testDir = "testdata/multiple"
 		t.Run("allowed dangling", func(t *testing.T) {
-			pkgs, err := NewPackages(ctx, os.DirFS(testDir), testDir, "")
+			pkgs, err := NewPackages(ctx, os.DirFS(testDir), testDir, nil)
 			require.NoError(t, err)
 			_, err = NewGraph(ctx, pkgs, WithAllowUnresolved())
 			require.NoError(t, err)
 		})
 		t.Run("external dependencies only", func(t *testing.T) {
-			pkgs, err := NewPackages(ctx, os.DirFS(testDir), testDir, "")
+			pkgs, err := NewPackages(ctx, os.DirFS(testDir), testDir, nil)
 			require.NoError(t, err)
 			graph, err := NewGraph(ctx, pkgs, WithRepos(packageRepo), WithKeys(key))
 			require.NoError(t, err)
@@ -134,7 +134,7 @@ func TestNewGraph(t *testing.T) {
 			}
 		})
 		t.Run("internal and external dependencies", func(t *testing.T) {
-			pkgs, err := NewPackages(ctx, os.DirFS(testDir), testDir, "")
+			pkgs, err := NewPackages(ctx, os.DirFS(testDir), testDir, nil)
 			require.NoError(t, err)
 			graph, err := NewGraph(ctx, pkgs, WithRepos(packageRepo), WithKeys(key))
 			require.NoError(t, err)
@@ -180,7 +180,7 @@ func TestNewGraph(t *testing.T) {
 		})
 
 		t.Run("internal dependencies numbered", func(t *testing.T) {
-			pkgs, err := NewPackages(ctx, os.DirFS(testDir), testDir, "")
+			pkgs, err := NewPackages(ctx, os.DirFS(testDir), testDir, nil)
 			require.NoError(t, err)
 			graph, err := NewGraph(ctx, pkgs, WithRepos(packageRepo), WithKeys(key))
 			require.NoError(t, err)
@@ -242,7 +242,7 @@ func TestNewGraph(t *testing.T) {
 				"d": {"a:1.3.5-r1@local"},
 			}
 		)
-		pkgs, err := NewPackages(ctx, os.DirFS(testDir), testDir, "")
+		pkgs, err := NewPackages(ctx, os.DirFS(testDir), testDir, nil)
 		require.NoError(t, err)
 		graph, err := NewGraph(ctx, pkgs, WithRepos(cyclePackageRepo), WithKeys(cycleKey))
 		require.NoError(t, err)
@@ -266,7 +266,7 @@ func TestTargets(t *testing.T) {
 	ctx := context.Background()
 	testDir := "testdata/subpackages"
 
-	pkgs, err := NewPackages(ctx, os.DirFS(testDir), testDir, "")
+	pkgs, err := NewPackages(ctx, os.DirFS(testDir), testDir, nil)
 	require.NoError(t, err)
 	graph, err := NewGraph(ctx, pkgs, WithAllowUnresolved())
 	require.NoError(t, err)

pkg/dag/packages.go

@@ -144,11 +144,11 @@ func (p *Packages) addProvides(c *Configuration, provides []string) error {
 // The input is any fs.FS filesystem implementation. Given a directory path, you
 // can call NewPackages like this:
 //
-// NewPackages(ctx, os.DirFS("/path/to/dir"), "/path/to/dir", "./pipelines")
+// NewPackages(ctx, os.DirFS("/path/to/dir"), "/path/to/dir", []string{"./pipelines"})
 //
 // The repetition of the path is necessary because of how the upstream parser in
 // melange requires the full path to the directory to be passed in.
-func NewPackages(ctx context.Context, fsys fs.FS, dirPath, pipelineDir string) (*Packages, error) {
+func NewPackages(ctx context.Context, fsys fs.FS, dirPath string, pipelineDirs []string) (*Packages, error) {
 	log := clog.FromContext(ctx)
 
 	pkgs := &Packages{
@@ -240,7 +240,7 @@ func NewPackages(ctx context.Context, fsys fs.FS, dirPath, pipelineDir string) (
 			// Resolve all `uses` used by the pipeline. This updates the set of
 			// .environment.contents.packages so the next block can include those as build deps.
 			build := &build.Build{
-				PipelineDirs:  []string{pipelineDir},
+				PipelineDirs:  pipelineDirs,
 				Configuration: c.Configuration,
 			}
 			if err := build.Compile(ctx); err != nil {

pkg/dag/packages_test.go

@@ -12,7 +12,7 @@ func TestNewPackages(t *testing.T) {
 	ctx := context.Background()
 
 	testdir := "testdata/multiple"
-	pkgs, err := NewPackages(ctx, os.DirFS(testdir), testdir, "")
+	pkgs, err := NewPackages(ctx, os.DirFS(testdir), testdir, nil)
 	require.NoError(t, err)
 
 	t.Run("loads data correctly", func(t *testing.T) {
@@ -31,7 +31,7 @@ func TestNewPackages(t *testing.T) {
 
 	t.Run("multiple configurations using the same package name", func(t *testing.T) {
 		testdir = "testdata/duplicate"
-		pkgs, err = NewPackages(ctx, os.DirFS(testdir), testdir, "")
+		pkgs, err = NewPackages(ctx, os.DirFS(testdir), testdir, nil)
 		if err == nil {
 			t.Error("should yield an error but got nil")
 		}