WIP: break this up

thockin · thockin · commit a2e702e44594 · 2024-05-10T14:53:15.000-07:00
diff --git a/cmd/kube-controller-manager/app/storageversionmigrator.go b/cmd/kube-controller-manager/app/storageversionmigrator.go
@@ -26,8 +26,9 @@ import (
 	"k8s.io/controller-manager/controller"
 	"k8s.io/kubernetes/cmd/kube-controller-manager/names"
 	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/utils/feature"
 
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	oldutilfeature "k8s.io/apiserver/pkg/util/feature"
 	clientgofeaturegate "k8s.io/client-go/features"
 	svm "k8s.io/kubernetes/pkg/controller/storageversionmigrator"
 )
@@ -45,8 +46,8 @@ func startSVMController(
 	controllerContext ControllerContext,
 	controllerName string,
 ) (controller.Interface, bool, error) {
-	if !utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionMigrator) ||
-		!clientgofeaturegate.FeatureGates().Enabled(clientgofeaturegate.InformerResourceVersion) {
+	if !oldutilfeature.DefaultFeatureGate.Enabled(features.StorageVersionMigrator) &&
+		!feature.Enabled(clientgofeaturegate.InformerResourceVersion3) {
 		return nil, false, nil
 	}
 
diff --git a/cmd/kube-controller-manager/controller-manager.go b/cmd/kube-controller-manager/controller-manager.go
@@ -24,6 +24,8 @@ import (
 	"os"
 	_ "time/tzdata" // for CronJob Time Zone support
 
+	clientfeatures "k8s.io/client-go/features"
+
 	"k8s.io/component-base/cli"
 	_ "k8s.io/component-base/logs/json/register"          // for JSON log format registration
 	_ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugin
@@ -32,6 +34,9 @@ import (
 )
 
 func main() {
+	// FIXME: Example, do not merge
+	clientfeatures.WatchListClient3.Set(true)
+
 	command := app.NewControllerManagerCommand()
 	code := cli.Run(command)
 	os.Exit(code)
diff --git a/pkg/api/pod/util.go b/pkg/api/pod/util.go
@@ -29,6 +29,7 @@ import (
 	"k8s.io/kubernetes/pkg/apis/core/helper"
 	apivalidation "k8s.io/kubernetes/pkg/apis/core/validation"
 	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/utils/feature"
 )
 
 // ContainerType signifies container type
@@ -685,7 +686,8 @@ func dropDisabledFields(
 		// For other types of containers, validateContainers will handle them.
 	}
 
-	if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) && !rroInUse(oldPodSpec) {
+	if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) &&
+		!feature.Enabled(features.RecursiveReadOnlyMounts3) && !rroInUse(oldPodSpec) {
 		for i := range podSpec.Containers {
 			for j := range podSpec.Containers[i].VolumeMounts {
 				podSpec.Containers[i].VolumeMounts[j].RecursiveReadOnly = nil
@@ -809,7 +811,8 @@ func dropDisabledPodStatusFields(podStatus, oldPodStatus *api.PodStatus, podSpec
 		podStatus.HostIPs = nil
 	}
 
-	if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) && !rroInUse(oldPodSpec) {
+	if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) &&
+		!feature.Enabled(features.RecursiveReadOnlyMounts3) && !rroInUse(oldPodSpec) {
 		for i := range podStatus.ContainerStatuses {
 			podStatus.ContainerStatuses[i].VolumeMounts = nil
 		}
diff --git a/pkg/controlplane/apiserver/options/options.go b/pkg/controlplane/apiserver/options/options.go
@@ -24,7 +24,6 @@ import (
 	"strings"
 	"time"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	peerreconcilers "k8s.io/apiserver/pkg/reconcilers"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
@@ -34,6 +33,7 @@ import (
 	logsapi "k8s.io/component-base/logs/api/v1"
 	"k8s.io/component-base/metrics"
 	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/features"
 	netutil "k8s.io/utils/net"
 
 	_ "k8s.io/kubernetes/pkg/features"
@@ -101,7 +101,7 @@ type CompletedOptions struct {
 // NewOptions creates a new ServerRunOptions object with default parameters
 func NewOptions() *Options {
 	s := Options{
-		GenericServerRunOptions: genericoptions.NewServerRunOptions(),
+		GenericServerRunOptions: genericoptions.NewServerRunOptions(features.KubernetesGates()),
 		Etcd:                    genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)),
 		SecureServing:           kubeoptions.NewSecureServingOptions(),
 		Audit:                   genericoptions.NewAuditOptions(),
diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go
@@ -23,6 +23,8 @@ import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	clientfeatures "k8s.io/client-go/features"
 	"k8s.io/component-base/featuregate"
+	kubeaggregatorfeatures "k8s.io/kube-aggregator/pkg/features"
+	"k8s.io/utils/feature"
 )
 
 const (
@@ -964,6 +966,26 @@ const (
 	RecursiveReadOnlyMounts featuregate.Feature = "RecursiveReadOnlyMounts"
 )
 
+var (
+	kubeGates = feature.MergeGateSets(
+		clientfeatures.FeatureGates(),
+		kubeaggregatorfeatures.FeatureGates(),
+		apiextensionsfeatures.FeatureGates())
+
+	// owner: @AkihiroSuda
+	// kep: https://kep.k8s.io/3857
+	// alpha: v1.30
+	//
+	// Allows recursive read-only mounts.
+	RecursiveReadOnlyMounts3 = kubeGates.Add(&feature.Gate{
+		Name: "RecursiveReadOnlyMounts", Default: false, Release: feature.Alpha,
+	})
+)
+
+func KubernetesGates() *feature.GateSet {
+	return kubeGates
+}
+
 func init() {
 	runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates))
 
diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go
@@ -69,6 +69,7 @@ import (
 	"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
 	volumevalidation "k8s.io/kubernetes/pkg/volume/validation"
 	"k8s.io/kubernetes/third_party/forked/golang/expansion"
+	"k8s.io/utils/feature"
 	utilnet "k8s.io/utils/net"
 )
 
@@ -348,7 +349,8 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
 		if err != nil {
 			return nil, cleanupAction, fmt.Errorf("failed to resolve recursive read-only mode: %w", err)
 		}
-		if rro && !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) {
+		if rro && !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) &&
+			!feature.Enabled(features.RecursiveReadOnlyMounts3) {
 			return nil, cleanupAction, fmt.Errorf("recursive read-only mount needs feature gate %q to be enabled", features.RecursiveReadOnlyMounts)
 		}
 
@@ -2136,7 +2138,8 @@ func (kl *Kubelet) convertToAPIContainerStatuses(pod *v1.Pod, podStatus *kubecon
 		}
 		// status.VolumeMounts cannot be propagated from kubecontainer.Status
 		// because the CRI API is unaware of the volume names.
-		if utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) {
+		if utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) ||
+			feature.Enabled(features.RecursiveReadOnlyMounts3) {
 			for _, vol := range container.VolumeMounts {
 				volStatus := v1.VolumeMountStatus{
 					Name:      vol.Name,
@@ -2148,7 +2151,8 @@ func (kl *Kubelet) convertToAPIContainerStatuses(pod *v1.Pod, podStatus *kubecon
 					if b, err := resolveRecursiveReadOnly(vol, supportsRRO); err != nil {
 						klog.ErrorS(err, "failed to resolve recursive read-only mode", "mode", *vol.RecursiveReadOnly)
 					} else if b {
-						if utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) {
+						if utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) ||
+							feature.Enabled(features.RecursiveReadOnlyMounts3) {
 							rroMode = v1.RecursiveReadOnlyEnabled
 						} else {
 							klog.ErrorS(nil, "recursive read-only mount needs feature gate to be enabled",
diff --git a/pkg/kubelet/nodestatus/setters.go b/pkg/kubelet/nodestatus/setters.go
@@ -43,6 +43,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/cm"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/events"
+	"k8s.io/utils/feature"
 	netutils "k8s.io/utils/net"
 
 	"k8s.io/klog/v2"
@@ -485,7 +486,8 @@ func GoRuntime() Setter {
 // RuntimeHandlers returns a Setter that sets RuntimeHandlers on the node.
 func RuntimeHandlers(fn func() []kubecontainer.RuntimeHandler) Setter {
 	return func(ctx context.Context, node *v1.Node) error {
-		if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) {
+		if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) &&
+			!feature.Enabled(features.RecursiveReadOnlyMounts3) {
 			return nil
 		}
 		handlers := fn()
diff --git a/pkg/registry/core/node/strategy.go b/pkg/registry/core/node/strategy.go
@@ -40,6 +40,7 @@ import (
 	"k8s.io/kubernetes/pkg/apis/core/validation"
 	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/client"
+	"k8s.io/utils/feature"
 	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 )
 
@@ -103,7 +104,8 @@ func dropDisabledFields(node *api.Node, oldNode *api.Node) {
 		node.Spec.ConfigSource = nil
 	}
 
-	if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) {
+	if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) &&
+		!feature.Enabled(features.RecursiveReadOnlyMounts3) {
 		node.Status.RuntimeHandlers = nil
 	}
 }
diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go
@@ -60,7 +60,7 @@ type CustomResourceDefinitionsServerOptions struct {
 // NewCustomResourceDefinitionsServerOptions creates default options of an apiextensions-apiserver.
 func NewCustomResourceDefinitionsServerOptions(out, errOut io.Writer) *CustomResourceDefinitionsServerOptions {
 	o := &CustomResourceDefinitionsServerOptions{
-		ServerRunOptions: genericoptions.NewServerRunOptions(),
+		ServerRunOptions: genericoptions.NewServerRunOptions(features.FeatureGates()),
 		RecommendedOptions: genericoptions.NewRecommendedOptions(
 			defaultEtcdPathPrefix,
 			apiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion),
diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go
@@ -19,9 +19,22 @@ package features
 import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/component-base/featuregate"
+	"k8s.io/utils/feature"
 )
 
 const (
+	CRDValidationRatcheting featuregate.Feature = "CRDValidationRatcheting"
+
+	CustomResourceFieldSelectors featuregate.Feature = "CustomResourceFieldSelectors"
+)
+
+func init() {
+	utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates)
+}
+
+var (
+	libGates = &feature.GateSet{}
+
 	// Every feature gate should add method here following this template:
 	//
 	// // owner: @username
@@ -33,18 +46,26 @@ const (
 	//
 	// Ignores errors raised on unchanged fields of Custom Resources
 	// across UPDATE/PATCH requests.
-	CRDValidationRatcheting featuregate.Feature = "CRDValidationRatcheting"
+	CRDValidationRatcheting3 = libGates.Add(&feature.Gate{
+		Name:    "CRDValidationRatcheting",
+		Default: true,
+		Release: feature.Beta,
+	})
 
 	// owner: @jpbetz
 	// alpha: v1.30
 	//
 	// CustomResourceDefinitions may include SelectableFields to declare which fields
 	// may be used as field selectors.
-	CustomResourceFieldSelectors featuregate.Feature = "CustomResourceFieldSelectors"
+	CustomResourceFieldSelectors3 = libGates.Add(&feature.Gate{
+		Name:    "CustomResourceFieldSelectors",
+		Default: false,
+		Release: feature.Beta,
+	})
 )
 
-func init() {
-	utilfeature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates)
+func FeatureGates() *feature.GateSet {
+	return libGates
 }
 
 // defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys.
diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go b/staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go
@@ -23,10 +23,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apiserver/pkg/server"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/utils/feature"
 
 	"github.com/spf13/pflag"
 )
@@ -89,9 +90,11 @@ type ServerRunOptions struct {
 	// This grace period is orthogonal to other grace periods, and
 	// it is not overridden by any other grace period.
 	ShutdownWatchTerminationGracePeriod time.Duration
+
+	FeatureGates *feature.GateSet
 }
 
-func NewServerRunOptions() *ServerRunOptions {
+func NewServerRunOptions(featureGates *feature.GateSet) *ServerRunOptions {
 	defaults := server.NewConfig(serializer.CodecFactory{})
 	return &ServerRunOptions{
 		MaxRequestsInFlight:                 defaults.MaxRequestsInFlight,
@@ -104,6 +107,7 @@ func NewServerRunOptions() *ServerRunOptions {
 		JSONPatchMaxCopyBytes:               defaults.JSONPatchMaxCopyBytes,
 		MaxRequestBodyBytes:                 defaults.MaxRequestBodyBytes,
 		ShutdownSendRetryAfter:              false,
+		FeatureGates:                        featureGates,
 	}
 }
 
@@ -337,5 +341,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"This option, if set, represents the maximum amount of grace period the apiserver will wait "+
 		"for active watch request(s) to drain during the graceful server shutdown window.")
 
-	utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
+	s.FeatureGates.EnableFlagControl("new-feature-gates", fs)
+	s.FeatureGates.EnableEnvControl("KUBE_FEATURE_")
 }
diff --git a/staging/src/k8s.io/client-go/features/features.go b/staging/src/k8s.io/client-go/features/features.go
@@ -19,8 +19,9 @@ package features
 import (
 	"errors"
 
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"sync/atomic"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 )
 
 // NOTE: types Feature, FeatureSpec, prerelease (and its values)
@@ -77,7 +78,7 @@ type Registry interface {
 //
 // Please note that the actual set of the feature gates
 // might be overwritten by calling ReplaceFeatureGates method.
-func FeatureGates() Gates {
+func LegacyFeatureGates() Gates {
 	return featureGates.Load().(*featureGatesWrapper).Gates
 }
 
diff --git a/staging/src/k8s.io/client-go/features/known_features.go b/staging/src/k8s.io/client-go/features/known_features.go
@@ -16,7 +16,19 @@ limitations under the License.
 
 package features
 
+import (
+	"k8s.io/utils/feature"
+)
+
 const (
+	WatchListClient Feature = "WatchListClient"
+
+	InformerResourceVersion Feature = "InformerResourceVersion"
+)
+
+var (
+	libGates = &feature.GateSet{}
+
 	// Every feature gate should add method here following this template:
 	//
 	// // owner: @username
@@ -36,13 +48,23 @@ const (
 	// NOTE:
 	//  The feature is disabled in Beta by default because
 	//  it will only be turned on for selected control plane component(s).
-	WatchListClient Feature = "WatchListClient"
+	WatchListClient3 = libGates.Add(&feature.Gate{
+		Name:    "WatchListClient",
+		Release: feature.Beta,
+	})
 
 	// owner: @nilekhc
 	// alpha: v1.30
-	InformerResourceVersion Feature = "InformerResourceVersion"
+	InformerResourceVersion3 = libGates.Add(&feature.Gate{
+		Name:    "InformerResourceVersion",
+		Release: feature.Alpha,
+	})
 )
 
+func FeatureGates() *feature.GateSet {
+	return libGates
+}
+
 // defaultKubernetesFeatureGates consists of all known Kubernetes-specific feature keys.
 //
 // To add a new feature, define a key for it above and add it here.
diff --git a/staging/src/k8s.io/client-go/tools/cache/shared_informer.go b/staging/src/k8s.io/client-go/tools/cache/shared_informer.go
@@ -29,6 +29,7 @@ import (
 	"k8s.io/client-go/tools/cache/synctrack"
 	"k8s.io/utils/buffer"
 	"k8s.io/utils/clock"
+	"k8s.io/utils/feature"
 
 	"k8s.io/klog/v2"
 
@@ -411,7 +412,8 @@ func (v *dummyController) HasSynced() bool {
 }
 
 func (v *dummyController) LastSyncResourceVersion() string {
-	if clientgofeaturegate.FeatureGates().Enabled(clientgofeaturegate.InformerResourceVersion) {
+	if clientgofeaturegate.FeatureGates().Enabled(clientgofeaturegate.InformerResourceVersion) ||
+		feature.Enabled(clientgofeaturegate.InformerResourceVersion3) {
 		return v.informer.LastSyncResourceVersion()
 	}
 
diff --git a/staging/src/k8s.io/kube-aggregator/pkg/features/known_features.go b/staging/src/k8s.io/kube-aggregator/pkg/features/known_features.go
diff --git a/test/integration/storageversionmigrator/storageversionmigrator_test.go b/test/integration/storageversionmigrator/storageversionmigrator_test.go

Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,7 @@ import (`
`29`	`29`	`"k8s.io/kubernetes/pkg/apis/core/helper"`
`30`	`30`	`apivalidation "k8s.io/kubernetes/pkg/apis/core/validation"`
`31`	`31`	`"k8s.io/kubernetes/pkg/features"`
	`32`	`+ "k8s.io/utils/feature"`
`32`	`33`	`)`
`33`	`34`
`34`	`35`	`// ContainerType signifies container type`
`@@ -685,7 +686,8 @@ func dropDisabledFields(`
`685`	`686`	`// For other types of containers, validateContainers will handle them.`
`686`	`687`	`}`
`687`	`688`
`688`		`- if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) && !rroInUse(oldPodSpec) {`
	`689`	`+ if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) &&`
	`690`	`+ !feature.Enabled(features.RecursiveReadOnlyMounts3) && !rroInUse(oldPodSpec) {`
`689`	`691`	`for i := range podSpec.Containers {`
`690`	`692`	`for j := range podSpec.Containers[i].VolumeMounts {`
`691`	`693`	`podSpec.Containers[i].VolumeMounts[j].RecursiveReadOnly = nil`
`@@ -809,7 +811,8 @@ func dropDisabledPodStatusFields(podStatus, oldPodStatus *api.PodStatus, podSpec`
`809`	`811`	`podStatus.HostIPs = nil`
`810`	`812`	`}`
`811`	`813`
`812`		`- if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) && !rroInUse(oldPodSpec) {`
	`814`	`+ if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) &&`
	`815`	`+ !feature.Enabled(features.RecursiveReadOnlyMounts3) && !rroInUse(oldPodSpec) {`
`813`	`816`	`for i := range podStatus.ContainerStatuses {`
`814`	`817`	`podStatus.ContainerStatuses[i].VolumeMounts = nil`
`815`	`818`	`}`
Original file line number	Diff line number	Diff line change
`@@ -40,6 +40,7 @@ import (`
`40`	`40`	`"k8s.io/kubernetes/pkg/apis/core/validation"`
`41`	`41`	`"k8s.io/kubernetes/pkg/features"`
`42`	`42`	`"k8s.io/kubernetes/pkg/kubelet/client"`
	`43`	`+ "k8s.io/utils/feature"`
`43`	`44`	`"sigs.k8s.io/structured-merge-diff/v4/fieldpath"`
`44`	`45`	`)`
`45`	`46`
`@@ -103,7 +104,8 @@ func dropDisabledFields(node api.Node, oldNode api.Node) {`
`103`	`104`	`node.Spec.ConfigSource = nil`
`104`	`105`	`}`
`105`	`106`
`106`		`- if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) {`
	`107`	`+ if !utilfeature.DefaultFeatureGate.Enabled(features.RecursiveReadOnlyMounts) &&`
	`108`	`+ !feature.Enabled(features.RecursiveReadOnlyMounts3) {`
`107`	`109`	`node.Status.RuntimeHandlers = nil`
`108`	`110`	`}`
`109`	`111`	`}`