feat: Add modify command and pipeline tag generator (#3213)

- Add new modify command to provide a facility for making arbitrary changes to packages
- Add pipeline-tag modifier to generate missing ingest pipeline processor tags
- Add yamledit package to provide functions for modifying yaml documents
- Add fleetpkg package to provide types and functions for fleet package types and to
support modifying package yaml files via yamledit.

Author: taylor-swanson

README.md

@@ -439,6 +439,17 @@ Use this command to validate the contents of a package using the package specifi
 
 The command ensures that the package is aligned with the package spec and the README file is up-to-date with its template (if present).
 
+### `elastic-package modify`
+
+_Context: package_
+
+Use this command to apply modifications to a package.
+
+These modifications can range from applying best practices, generating ingest pipeline tags, and more. Run this command without any arguments to see a list of modifiers.
+
+Use --modifiers to specify which modifiers to run, separated by commas.
+
+
 ### `elastic-package profiles`
 
 _Context: global_

cmd/modify.go

@@ -0,0 +1,112 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"sort"
+	"text/tabwriter"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	"github.com/elastic/elastic-package/internal/cobraext"
+	"github.com/elastic/elastic-package/internal/fleetpkg"
+	"github.com/elastic/elastic-package/internal/modify"
+	"github.com/elastic/elastic-package/internal/modify/pipelinetag"
+	"github.com/elastic/elastic-package/internal/packages"
+)
+
+const modifyLongDescription = `Use this command to apply modifications to a package.
+
+These modifications can range from applying best practices, generating ingest pipeline tags, and more. Run this command without any arguments to see a list of modifiers.
+
+Use --modifiers to specify which modifiers to run, separated by commas.
+`
+
+func setupModifyCommand() *cobraext.Command {
+	modifiers := []*modify.Modifier{
+		pipelinetag.Modifier,
+	}
+	sort.Slice(modifiers, func(i, j int) bool {
+		return modifiers[i].Name < modifiers[j].Name
+	})
+
+	validModifier := func(name string) bool {
+		for _, modifier := range modifiers {
+			if modifier.Name == name {
+				return true
+			}
+		}
+
+		return false
+	}
+
+	listModifiers := func(w io.Writer) {
+		tw := tabwriter.NewWriter(w, 0, 2, 3, ' ', 0)
+		for _, a := range modifiers {
+			_, _ = fmt.Fprintf(tw, "%s\t%s\n", a.Name, a.Doc)
+		}
+		_ = tw.Flush()
+		_, _ = fmt.Fprintln(w, "")
+	}
+
+	cmd := &cobra.Command{
+		Use:   "modify",
+		Short: "Modify package assets",
+		Long:  modifyLongDescription,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cmd.Println("Modify package assets")
+
+			selectedModifiers, err := cmd.Flags().GetStringSlice("modifiers")
+			if err != nil {
+				return cobraext.FlagParsingError(err, "modifiers")
+			}
+			if len(selectedModifiers) == 0 {
+				_, _ = fmt.Fprint(cmd.OutOrStderr(), "Please provide at least one modifier:\n\n")
+				listModifiers(cmd.OutOrStderr())
+				return nil
+			}
+			for _, selected := range selectedModifiers {
+				if !validModifier(selected) {
+					_, _ = fmt.Fprint(cmd.OutOrStderr(), "Please provide at a valid modifier:\n\n")
+					listModifiers(cmd.OutOrStderr())
+					return cobraext.FlagParsingError(fmt.Errorf("invalid modifier: %q", selected), "modifiers")
+				}
+			}
+
+			pkgRootPath, err := packages.FindPackageRoot()
+			if err != nil {
+				return fmt.Errorf("locating package root failed: %w", err)
+			}
+
+			for _, modifier := range modifiers {
+				pkg, err := fleetpkg.Load(pkgRootPath)
+				if err != nil {
+					return fmt.Errorf("failed to load package from %q: %w", pkgRootPath, err)
+				}
+				if err = modifier.Run(pkg); err != nil {
+					return fmt.Errorf("failed to apply modifier %q: %w", modifier.Name, err)
+				}
+			}
+
+			return nil
+		},
+	}
+
+	cmd.PersistentFlags().StringSliceP("modifiers", "m", nil, "List of modifiers to run, separated by commas")
+
+	for _, m := range modifiers {
+		prefix := m.Name + "."
+
+		m.Flags.VisitAll(func(f *pflag.Flag) {
+			name := prefix + f.Name
+			cmd.Flags().Var(f.Value, name, f.Usage)
+		})
+	}
+
+	return cobraext.NewCommand(cmd, cobraext.ContextPackage)
+}

cmd/root.go

@@ -32,6 +32,7 @@ var commands = []*cobraext.Command{
 	setupInstallCommand(),
 	setupLinksCommand(),
 	setupLintCommand(),
+	setupModifyCommand(),
 	setupProfilesCommand(),
 	setupReportsCommand(),
 	setupServiceCommand(),

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/fatih/color v1.18.0
 	github.com/go-viper/mapstructure/v2 v2.5.0
 	github.com/gobwas/glob v0.2.3
+	github.com/goccy/go-yaml v1.18.0
 	github.com/google/go-cmp v0.7.0
 	github.com/google/go-github/v32 v32.1.0
 	github.com/google/go-querystring v1.2.0
@@ -35,6 +36,7 @@ require (
 	github.com/rogpeppe/go-internal v1.14.1
 	github.com/shirou/gopsutil/v3 v3.24.5
 	github.com/spf13/cobra v1.10.2
+	github.com/spf13/pflag v1.0.10
 	github.com/stretchr/testify v1.11.1
 	go.yaml.in/yaml/v2 v2.4.3
 	go4.org/netipx v0.0.0-20231129151722-fdeea329fbba
@@ -144,7 +146,6 @@ require (
 	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/cast v1.7.0 // indirect
-	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect

go.sum

@@ -144,6 +144,8 @@ github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPE
 github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
+github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=

internal/fleetpkg/fleetpkg.go

@@ -0,0 +1,207 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package fleetpkg
+
+import (
+	"encoding/json"
+
+	"github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/ast"
+
+	"github.com/elastic/elastic-package/internal/yamledit"
+)
+
+// Package is a fleet package.
+type Package struct {
+	Manifest    Manifest
+	Input       *DataStream
+	DataStreams map[string]*DataStream
+
+	sourceDir string
+}
+
+// Path is the path to the root of the package.
+func (i *Package) Path() string {
+	return i.sourceDir
+}
+
+// Manifest is the package manifest.
+type Manifest struct {
+	Name          string `yaml:"name"`
+	Title         string `yaml:"title"`
+	Version       string `yaml:"version"`
+	Description   string `yaml:"description"`
+	Type          string `yaml:"type"`
+	FormatVersion string `yaml:"format_version"`
+	Owner         struct {
+		Github string `yaml:"github"`
+		Type   string `yaml:"type"`
+	} `yaml:"owner"`
+
+	Doc *yamledit.Document `yaml:"-"`
+}
+
+// Path is the path to the manifest file.
+func (m *Manifest) Path() string {
+	return m.Doc.Filename()
+}
+
+// DataStreamManifest is the data stream manifest file.
+type DataStreamManifest struct {
+	Title string `yaml:"title"`
+	Type  string `yaml:"type"`
+
+	Doc *yamledit.Document `yaml:"-"`
+}
+
+// Path is the path to the manifest file.
+func (m *DataStreamManifest) Path() string {
+	return m.Doc.Filename()
+}
+
+// DataStream is a data stream within the package.
+type DataStream struct {
+	Manifest  DataStreamManifest
+	Pipelines map[string]*Pipeline
+
+	sourceDir string
+}
+
+// Path is the path to the data stream.
+func (d *DataStream) Path() string {
+	return d.sourceDir
+}
+
+// Pipeline is an ingest pipeline.
+type Pipeline struct {
+	Description string       `yaml:"description"`
+	Processors  []*Processor `yaml:"processors,omitempty"`
+	OnFailure   []*Processor `yaml:"on_failure,omitempty"`
+
+	Doc *yamledit.Document `yaml:"-"`
+}
+
+// Path is the path to the pipeline.
+func (p *Pipeline) Path() string {
+	return p.Doc.Filename()
+}
+
+// Processor is an ingest pipeline processor.
+type Processor struct {
+	Type       string
+	Attributes map[string]any
+	OnFailure  []*Processor
+
+	Node ast.Node
+}
+
+// GetAttribute gets an attribute of the processor.
+func (p *Processor) GetAttribute(key string) (any, bool) {
+	v, ok := p.Attributes[key]
+	if !ok {
+		return nil, false
+	}
+
+	return v, true
+}
+
+// GetAttributeString gets a string attribute of the processor.
+func (p *Processor) GetAttributeString(key string) (string, bool) {
+	v, ok := p.Attributes[key].(string)
+	if !ok {
+		return "", false
+	}
+
+	return v, true
+}
+
+// GetAttributeFloat gets a float attribute of the processor.
+func (p *Processor) GetAttributeFloat(key string) (float64, bool) {
+	v, ok := p.Attributes[key].(float64)
+	if !ok {
+		return 0, false
+	}
+
+	return v, true
+}
+
+// GetAttributeInt gets an int attribute of the processor.
+func (p *Processor) GetAttributeInt(key string) (int, bool) {
+	v, ok := p.Attributes[key].(int)
+	if !ok {
+		return 0, false
+	}
+
+	return v, true
+}
+
+// GetAttributeBool gets a bool attribute of the processor.
+func (p *Processor) GetAttributeBool(key string) (bool, bool) {
+	v, ok := p.Attributes[key].(bool)
+	if !ok {
+		return false, false
+	}
+
+	return v, true
+}
+
+// UnmarshalYAML implements a YAML unmarshaler.
+func (p *Processor) UnmarshalYAML(node ast.Node) error {
+	var procMap map[string]struct {
+		Attributes map[string]any `yaml:",inline"`
+		OnFailure  []*Processor   `yaml:"on_failure"`
+	}
+	if err := yaml.NodeToValue(node, &procMap); err != nil {
+		return err
+	}
+
+	// The struct representation used here is much more convenient
+	// to work with than the original map of map format.
+	for k, v := range procMap {
+		p.Type = k
+		p.Attributes = v.Attributes
+		p.OnFailure = v.OnFailure
+
+		delete(p.Attributes, "on_failure")
+
+		break
+	}
+
+	p.Node = node
+
+	return nil
+}
+
+// MarshalJSON implements a JSON marshaler.
+func (p *Processor) MarshalJSON() ([]byte, error) {
+	properties := make(map[string]any, len(p.Attributes)+1)
+	for k, v := range p.Attributes {
+		properties[k] = v
+	}
+	if len(p.OnFailure) > 0 {
+		properties["on_failure"] = p.OnFailure
+	}
+	return json.Marshal(map[string]any{
+		p.Type: properties,
+	})
+}
+
+// Validation is the validation.yml file of a package.
+type Validation struct {
+	Errors struct {
+		ExcludeChecks []string `yaml:"exclude_checks,omitempty"`
+	} `yaml:"errors,omitempty"`
+
+	DocsStructureEnforced struct {
+		Enabled bool `yaml:"enabled"`
+		Version int  `yaml:"version"`
+		Skip    []struct {
+			Title  string `yaml:"title"`
+			Reason string `yaml:"reason"`
+		} `yaml:"skip,omitempty"`
+	} `yaml:"docs_structure_enforced"`
+
+	Doc *yamledit.Document `yaml:"-"`
+}