Define new field dynamic_sygnal_types for otel input packages (#1067)

Add a new manifest field dynamic_signal_types under policy_template.

Author: criamico

code/go/internal/validator/semantic/validate_input_dynamic_signal_types.go

@@ -0,0 +1,69 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"io/fs"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+	"github.com/elastic/package-spec/v3/code/go/pkg/specerrors"
+)
+
+const (
+	otelcolInputType string = "otelcol"
+)
+
+type inputPolicyTemplateWithDynamic struct {
+	Name               string `yaml:"name"`
+	Input              string `yaml:"input"`
+	DynamicSignalTypes bool   `yaml:"dynamic_signal_types"` // true or false
+}
+
+type inputPackageManifestDynamic struct {
+	Type            string                           `yaml:"type"`
+	PolicyTemplates []inputPolicyTemplateWithDynamic `yaml:"policy_templates"`
+}
+
+// ValidateInputDynamicSignalTypes validates that dynamic_signal_types field is only used with otelcol input type
+func ValidateInputDynamicSignalTypes(fsys fspath.FS) specerrors.ValidationErrors {
+	var errs specerrors.ValidationErrors
+
+	manifestPath := "manifest.yml"
+	data, err := fs.ReadFile(fsys, manifestPath)
+	if err != nil {
+		return specerrors.ValidationErrors{
+			specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to read manifest: %w", fsys.Path(manifestPath), err)}
+	}
+
+	var manifest inputPackageManifestDynamic
+	err = yaml.Unmarshal(data, &manifest)
+	if err != nil {
+		return specerrors.ValidationErrors{
+			specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to parse manifest: %w", fsys.Path(manifestPath), err)}
+	}
+
+	for _, policyTemplate := range manifest.PolicyTemplates {
+		// Check if dynamic_signal_types is explicitly set
+		if policyTemplate.DynamicSignalTypes {
+			// Must be input package type
+			if manifest.Type != inputPackageType {
+				errs = append(errs, specerrors.NewStructuredErrorf(
+					"file \"%s\" is invalid: policy template \"%s\": dynamic_signal_types is only allowed for input type packages",
+					fsys.Path(manifestPath), policyTemplate.Name))
+				continue
+			}
+			// Must be otelcol input
+			if policyTemplate.Input != otelcolInputType {
+				errs = append(errs, specerrors.NewStructuredErrorf(
+					"file \"%s\" is invalid: policy template \"%s\": dynamic_signal_types is only allowed when input is 'otelcol', got '%s'",
+					fsys.Path(manifestPath), policyTemplate.Name, policyTemplate.Input))
+			}
+		}
+	}
+
+	return errs
+}

code/go/internal/validator/semantic/validate_input_dynamic_signal_types_test.go

@@ -0,0 +1,160 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+)
+
+func TestValidateInputDynamicSignalTypes(t *testing.T) {
+
+	t.Run("valid_otelcol_with_dynamic_signal_types_true", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.WriteFile(d+"/manifest.yml", []byte(`
+type: input
+policy_templates:
+  - name: otel_logs
+    input: otelcol
+    template_path: input.yml.hbs
+    dynamic_signal_types: true
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d))
+		require.Empty(t, errs, "expected no validation errors")
+	})
+
+	t.Run("valid_otelcol_with_dynamic_signal_types_false", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.WriteFile(d+"/manifest.yml", []byte(`
+type: input
+policy_templates:
+  - name: otel_logs
+    input: otelcol
+    template_path: input.yml.hbs
+    dynamic_signal_types: false
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d))
+		require.Empty(t, errs, "expected no validation errors")
+	})
+
+	t.Run("valid_otelcol_without_dynamic_signal_types", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.WriteFile(d+"/manifest.yml", []byte(`
+type: input
+policy_templates:
+  - name: otel_logs
+    input: otelcol
+    template_path: input.yml.hbs
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d))
+		require.Empty(t, errs, "expected no validation errors")
+	})
+
+	t.Run("invalid_non_otelcol_with_dynamic_signal_types_true", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.WriteFile(d+"/manifest.yml", []byte(`
+type: input
+policy_templates:
+  - name: logfile
+    input: logfile
+    template_path: input.yml.hbs
+    dynamic_signal_types: true
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d))
+		require.NotEmpty(t, errs, "expected validation errors")
+		assert.Len(t, errs, 1)
+		assert.Contains(t, errs[0].Error(), "dynamic_signal_types is only allowed when input is 'otelcol'")
+		assert.Contains(t, errs[0].Error(), "got 'logfile'")
+	})
+
+	t.Run("invalid_non_input_package_with_dynamic_signal_types", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.WriteFile(d+"/manifest.yml", []byte(`
+type: integration
+policy_templates:
+  - name: apache
+    dynamic_signal_types: true
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d))
+		require.NotEmpty(t, errs, "expected validation errors for non-input packages")
+		assert.Len(t, errs, 1)
+		assert.Contains(t, errs[0].Error(), "dynamic_signal_types is only allowed for input type packages")
+	})
+
+	t.Run("valid_non_input_package_without_dynamic_signal_types", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.WriteFile(d+"/manifest.yml", []byte(`
+type: integration
+policy_templates:
+  - name: apache
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d))
+		require.Empty(t, errs, "expected no validation errors for non-input packages without field")
+	})
+
+	t.Run("valid_non_otelcol_without_dynamic_signal_types", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.WriteFile(d+"/manifest.yml", []byte(`
+type: input
+policy_templates:
+  - name: logfile
+    input: logfile
+    template_path: input.yml.hbs
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d))
+		require.Empty(t, errs, "expected no validation errors")
+	})
+
+	t.Run("invalid_multiple_templates_one_invalid", func(t *testing.T) {
+		d := t.TempDir()
+
+		err := os.WriteFile(d+"/manifest.yml", []byte(`
+type: input
+policy_templates:
+  - name: otel_logs
+    input: otelcol
+    template_path: input.yml.hbs
+    dynamic_signal_types: true
+  - name: file_logs
+    input: logfile
+    template_path: input2.yml.hbs
+    dynamic_signal_types: true
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateInputDynamicSignalTypes(fspath.DirFS(d))
+		require.NotEmpty(t, errs, "expected validation errors")
+		assert.Len(t, errs, 1)
+		assert.Contains(t, errs[0].Error(), "dynamic_signal_types is only allowed when input is 'otelcol'")
+		assert.Contains(t, errs[0].Error(), "file_logs")
+		assert.Contains(t, errs[0].Error(), "got 'logfile'")
+	})
+}

code/go/internal/validator/spec.go

@@ -223,6 +223,7 @@ func (s Spec) rules(pkgType string, rootSpec spectypes.ItemSpec) validationRules
 		{fn: semantic.ValidateDeploymentModes, types: []string{"integration"}},
 		{fn: semantic.ValidateDurationVariables, since: semver.MustParse("3.5.0")},
 		{fn: semantic.ValidateInputPackagesPolicyTemplates, types: []string{"input"}},
+		{fn: semantic.ValidateInputDynamicSignalTypes},
 		{fn: semantic.ValidateMinimumAgentVersion},
 		{fn: semantic.ValidateIntegrationPolicyTemplates, types: []string{"integration"}},
 		{fn: semantic.ValidatePipelineTags, types: []string{"integration"}, since: semver.MustParse("3.6.0")},

code/go/pkg/validator/validator_test.go

@@ -36,6 +36,7 @@ func TestValidateFile(t *testing.T) {
 		"good_var_groups":                    {},
 		"good_input":                         {},
 		"good_input_otel":                    {},
+		"good_input_dynamic_signal_type":     {},
 		"good_content":                       {},
 		"good_lookup_index":                  {},
 		"good_alert_rule_templates":          {},
@@ -336,6 +337,25 @@ func TestValidateFile(t *testing.T) {
 				"field policy_templates.0.input: Must not be present",
 			},
 		},
+		"bad_input_dynamic_signal_types_non_otel": {
+			"manifest.yml",
+			[]string{
+				"policy template \"sample\": dynamic_signal_types is only allowed when input is 'otelcol', got 'logfile'",
+			},
+		},
+		"bad_integration_dynamic_signal_types": {
+			"manifest.yml",
+			[]string{
+				"field policy_templates.0: Additional property dynamic_signal_types is not allowed",
+				"policy template \"sample\": dynamic_signal_types is only allowed for input type packages",
+			},
+		},
+		"bad_input_dynamic_signal_types_old_version": {
+			"manifest.yml",
+			[]string{
+				"field policy_templates.0: Additional property dynamic_signal_types is not allowed",
+			},
+		},
 		"bad_input_template_path": {
 			"manifest.yml",
 			[]string{

spec/changelog.yml

@@ -30,6 +30,9 @@
     - description: Allow to set time series index mode in input packages.
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/1066
+    - description: Add support for dynamic_signal_types field in input package policy templates for OTel input type.
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/1067
 - version: 3.5.6-next
   changes:
     - description: Add validation for Kibana tag duplicates.

spec/input/manifest.spec.yml

@@ -88,6 +88,10 @@ spec:
           template_path:
             description: "Path to Elasticsearch index template for stream."
             type: string
+          dynamic_signal_types:
+            description: "When enabled, decides the transforms and index templates that need to be created depending on the pipelines specified in the configuration."
+            type: boolean
+            default: false
           deprecated:
             $ref: "../integration/manifest.spec.yml#/definitions/deprecated"
         required:
@@ -131,6 +135,8 @@ spec:
 versions:
   - before: 3.6.0
     patch:
+      - op: remove
+        path: "/properties/policy_templates/items/properties/dynamic_signal_types" # removes dynamic_signal_types field for policy templates
       - op: remove
         path: "/properties/elasticsearch/properties/index_mode" # Fleet needs to check that it is not enabled in non-metrics data streams
       - op: remove

test/packages/bad_input_dynamic_signal_types_non_otel/LICENSE.txt

@@ -0,0 +1 @@
+Apache License 2.0

test/packages/bad_input_dynamic_signal_types_non_otel/_dev/build/docs/README.md

@@ -0,0 +1,3 @@
+# Test package
+
+This is a test package for validating dynamic_signal_types field validation.

test/packages/bad_input_dynamic_signal_types_non_otel/agent/input/input.yml.hbs

@@ -0,0 +1,9 @@
+data_stream:
+  dataset: {{data_stream.dataset}}
+paths:
+{{#each paths as |path i|}}
+  - {{path}}
+{{/each}}
+exclude_files: [".gz$"]
+processors:
+  - add_locale: ~

test/packages/bad_input_dynamic_signal_types_non_otel/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1