fix: omit unset variables without defaults from Fleet API requests (#2980)

Previously, elastic-package would send null values for variables without
defaults, causing Fleet API validation errors for non-required URL-type
variables. This change modifies the variable handling to skip unset
variables that have no default values instead of including them as
nulls.

Fixes #2979

Author: andrewkroh

internal/packages/archetype/data_stream_inputs.go

@@ -84,6 +84,7 @@ func unpackVars(output *[]packages.Variable, input []InputVariable) {
 		newVar.MaxDuration = inputVar.MaxDuration
 		newVar.Description = inputVar.Description
 		if inputVar.Default != nil {
+			newVar.Default = &packages.VarValue{}
 			newVar.Default.Unpack(inputVar.Default)
 		}
 		*output = append(*output, newVar)

internal/packages/packages.go

@@ -104,19 +104,19 @@ func VarValueYamlString(vv VarValue, field string, numSpaces ...int) string {
 
 // Variable is an instance of configuration variable (named, typed).
 type Variable struct {
-	Name                  string   `config:"name" json:"name" yaml:"name"`
-	Type                  string   `config:"type" json:"type" yaml:"type"`
-	Title                 string   `config:"title" json:"title" yaml:"title"`
-	Description           string   `config:"description" json:"description" yaml:"description"`
-	Multi                 bool     `config:"multi" json:"multi" yaml:"multi"`
-	Required              bool     `config:"required" json:"required" yaml:"required"`
-	Secret                bool     `config:"secret" json:"secret" yaml:"secret"`
-	ShowUser              bool     `config:"show_user" json:"show_user" yaml:"show_user"`
-	HideInDeploymentModes []string `config:"hide_in_deployment_modes" json:"hide_in_deployment_modes" yaml:"hide_in_deployment_modes"`
-	UrlAllowedSchemes     []string `config:"url_allowed_schemes" json:"url_allowed_schemes" yaml:"url_allowed_schemes"`
-	MinDuration           string   `config:"min_duration" json:"min_duration" yaml:"min_duration"`
-	MaxDuration           string   `config:"max_duration" json:"max_duration" yaml:"max_duration"`
-	Default               VarValue `config:"default" json:"default" yaml:"default"`
+	Name                  string    `config:"name" json:"name" yaml:"name"`
+	Type                  string    `config:"type" json:"type" yaml:"type"`
+	Title                 string    `config:"title" json:"title" yaml:"title"`
+	Description           string    `config:"description" json:"description" yaml:"description"`
+	Multi                 bool      `config:"multi" json:"multi" yaml:"multi"`
+	Required              bool      `config:"required" json:"required" yaml:"required"`
+	Secret                bool      `config:"secret" json:"secret" yaml:"secret"`
+	ShowUser              bool      `config:"show_user" json:"show_user" yaml:"show_user"`
+	HideInDeploymentModes []string  `config:"hide_in_deployment_modes" json:"hide_in_deployment_modes" yaml:"hide_in_deployment_modes"`
+	UrlAllowedSchemes     []string  `config:"url_allowed_schemes" json:"url_allowed_schemes" yaml:"url_allowed_schemes"`
+	MinDuration           string    `config:"min_duration" json:"min_duration" yaml:"min_duration"`
+	MaxDuration           string    `config:"max_duration" json:"max_duration" yaml:"max_duration"`
+	Default               *VarValue `config:"default" json:"default" yaml:"default"`
 }
 
 // Input is a single input configuration.

internal/resources/fleetpolicy.go

@@ -312,17 +312,24 @@ func createInputPackagePolicy(policy FleetAgentPolicy, manifest packages.Package
 func setKibanaVariables(definitions []packages.Variable, values common.MapStr) kibana.Vars {
 	vars := kibana.Vars{}
 	for _, definition := range definitions {
+		// Elastic Package uses the deprecated 'inputs' array in its /api/fleet/package_policies request.
+		// When using this API parameter, default values are not automatically incorporated into
+		// the policy, whereas with the 'inputs' object, defaults are incorporated by the API service.
+		// This means that our client must include the default values in its request to ensure correct behavior.
 		val := definition.Default
 
 		value, err := values.GetValue(definition.Name)
 		if err == nil {
-			val = packages.VarValue{}
+			val = &packages.VarValue{}
 			val.Unpack(value)
+		} else if errors.Is(err, common.ErrKeyNotFound) && definition.Default == nil {
+			// Do not include nulls for unset variables.
+			continue
 		}
 
 		vars[definition.Name] = kibana.Var{
 			Type:  definition.Type,
-			Value: val,
+			Value: *val,
 		}
 	}
 	return vars

internal/testrunner/runners/system/tester.go

@@ -2049,17 +2049,24 @@ func createInputPackageDatastream(
 func setKibanaVariables(definitions []packages.Variable, values common.MapStr) kibana.Vars {
 	vars := kibana.Vars{}
 	for _, definition := range definitions {
+		// Elastic Package uses the deprecated 'inputs' array in its /api/fleet/package_policies request.
+		// When using this API parameter, default values are not automatically incorporated into
+		// the policy, whereas with the 'inputs' object, defaults are incorporated by the API service.
+		// This means that our client must include the default values in its request to ensure correct behavior.
 		val := definition.Default
 
 		value, err := values.GetValue(definition.Name)
 		if err == nil {
-			val = packages.VarValue{}
+			val = &packages.VarValue{}
 			val.Unpack(value)
+		} else if errors.Is(err, common.ErrKeyNotFound) && definition.Default == nil {
+			// Do not include nulls for unset variables.
+			continue
 		}
 
 		vars[definition.Name] = kibana.Var{
 			Type:  definition.Type,
-			Value: val,
+			Value: *val,
 		}
 	}
 	return vars