WIP: flag plumbing

Notes:

Some of these parse into a map in their config and then call SetFromMap -- need to retain ability to pass in via config.
Option 1: make a `feature.EnableMapControl()`
Option 2: pass map into `feature.EnablePFlagControl()`
Option 3: `feature.GetAllGates(feature.FromFlags(flagName), feature.FromEnv(prefix))`
	This is probablay most correct today and makes validation easy,
	since we do it exactly once.  A little awkward with any dynamic
	source (e.g. API).

Should env enablement be part of "AddFlags" logic in these components?
It's a little muddy how apps are configured.

```
$ for f in _output/bin/*; do X=$($f --help 2>&1 | grep gates); if [[ -z "$X" ]]; then X=$($f init --help 2>&1 | grep gates); fi; if [[ -n "$X" ]]; then echo; echo "$f"; echo "$X"; fi; done

_output/bin/apiextensions-apiserver
      --feature-gates mapStringBool                             4444 A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
      --new-feature-gates mapStringBool                         A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:

_output/bin/e2e_node.test
      --feature-gates mapStringBool                                6666 A set of key=value pairs that describe feature gates for alpha/experimental features.
      --service-feature-gates mapStringBool                        7777 A set of key=value pairs that describe feature gates for alpha/experimental features for API service.

_output/bin/kubeadm
      --feature-gates string                 2222 A set of key=value pairs that describe feature gates for various features. Options are:
      --new-feature-gates mapStringBool      A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:

_output/bin/kube-aggregator
      --feature-gates mapStringBool                             4444 A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
      --new-feature-gates mapStringBool                         A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:

_output/bin/kube-apiserver
      --feature-gates mapStringBool                        4444 A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
      --new-feature-gates mapStringBool                    A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:

_output/bin/kube-controller-manager
      --feature-gates mapStringBool              4444 A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
      --new-feature-gates mapStringBool          A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:

_output/bin/kubelet
      --feature-gates mapStringBool                              3333 A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
      --new-feature-gates mapStringBool                          A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:

_output/bin/kube-proxy
      --feature-gates mapStringBool                  1111 A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
      --new-feature-gates mapStringBool              A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:

_output/bin/kube-scheduler
      --feature-gates mapStringBool   4444 A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
      --new-feature-gates mapStringBool   A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:

```

Author: thockin

cmd/kube-controller-manager/app/options/options.go

@@ -45,6 +45,7 @@ import (
 	kubectrlmgrconfigscheme "k8s.io/kubernetes/pkg/controller/apis/config/scheme"
 	"k8s.io/kubernetes/pkg/controller/garbagecollector"
 	garbagecollectorconfig "k8s.io/kubernetes/pkg/controller/garbagecollector/config"
+	"k8s.io/kubernetes/pkg/features"
 	netutils "k8s.io/utils/net"
 
 	// add the kubernetes feature gates
@@ -274,6 +275,8 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy
 	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
 	fs.StringVar(&s.Generic.ClientConnection.Kubeconfig, "kubeconfig", s.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).")
 	utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))
+	features.KubernetesGates().EnablePFlagControl("new-feature-gates", fss.FlagSet("generic"))
+	features.KubernetesGates().EnableEnvControl("KUBE_FEATURE_", func(err error) { panic(err.Error()) })
 
 	return fss
 }

cmd/kube-proxy/app/server.go

@@ -576,6 +576,10 @@ with the apiserver API to configure the proxy.`,
 	opts.AddFlags(fs)
 	fs.AddGoFlagSet(goflag.CommandLine) // for --boot-id-file and --machine-id-file
 
+	// Enable feature gates
+	features.KubernetesGates().EnablePFlagControl("new-feature-gates", fs)
+	features.KubernetesGates().EnableEnvControl("KUBE_FEATURE_", func(err error) { panic(err.Error()) })
+
 	_ = cmd.MarkFlagFilename("config", "yaml", "yml", "json")
 
 	return cmd

cmd/kube-scheduler/app/options/options.go

@@ -45,6 +45,7 @@ import (
 	"k8s.io/component-base/metrics"
 	"k8s.io/klog/v2"
 	schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/scheduler"
 	kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
 	"k8s.io/kubernetes/pkg/scheduler/apis/config/validation"
@@ -190,6 +191,8 @@ func (o *Options) initFlags() {
 	o.Deprecated.AddFlags(nfs.FlagSet("deprecated"))
 	options.BindLeaderElectionFlags(o.LeaderElection, nfs.FlagSet("leader election"))
 	utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate"))
+	features.KubernetesGates().EnablePFlagControl("new-feature-gates", nfs.FlagSet("generic"))
+	features.KubernetesGates().EnableEnvControl("KUBE_FEATURE_", func(err error) { panic(err.Error()) })
 	o.Metrics.AddFlags(nfs.FlagSet("metrics"))
 	logsapi.AddFlags(o.Logs, nfs.FlagSet("logs"))
 

cmd/kubeadm/app/cmd/init.go

@@ -139,6 +139,10 @@ func newCmdInit(out io.Writer, initOptions *initOptions) *cobra.Command {
 	initOptions.bto.AddTTLFlag(cmd.Flags())
 	options.AddImageMetaFlags(cmd.Flags(), &initOptions.externalClusterCfg.ImageRepository)
 
+	// Enable feature gates
+	features.Gates().EnablePFlagControl("new-feature-gates", cmd.Flags())
+	features.Gates().EnableEnvControl("KUBE_FEATURE_", func(err error) { panic(err.Error()) })
+
 	// defines additional flag that are not used by the init command but that could be eventually used
 	// by the sub-commands automatically generated for phases
 	initRunner.SetAdditionalFlags(func(flags *flag.FlagSet) {

cmd/kubeadm/app/cmd/upgrade/upgrade.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 )
 
 // applyPlanFlags holds the values for the common flags in `kubeadm upgrade apply` and `kubeadm upgrade plan`
@@ -73,4 +74,8 @@ func addApplyPlanFlags(fs *pflag.FlagSet, flags *applyPlanFlags) {
 	fs.BoolVar(&flags.printConfig, "print-config", flags.printConfig, "Specifies whether the configuration file that will be used in the upgrade should be printed or not.")
 	options.AddFeatureGatesStringFlag(fs, &flags.featureGatesString)
 	options.AddIgnorePreflightErrorsFlag(fs, &flags.ignorePreflightErrors)
+
+	// Enable feature gates
+	features.Gates().EnablePFlagControl("new-feature-gates", fs)
+	features.Gates().EnableEnvControl("KUBE_FEATURE_", func(err error) { panic(err.Error()) })
 }

cmd/kubeadm/app/features/features.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/component-base/featuregate"
 	"k8s.io/klog/v2"
+	"k8s.io/utils/feature"
 )
 
 const (
@@ -42,6 +43,16 @@ const (
 	WaitForAllControlPlaneComponents = "WaitForAllControlPlaneComponents"
 )
 
+var (
+	// This is the composed set of gates that kubadm exposes.
+	kubeadmGates = &feature.GateSet{}
+)
+
+// Gates returns the set of feature gates exposed by this library.
+func Gates() *feature.GateSet {
+	return kubeadmGates
+}
+
 // InitFeatureGates are the default feature gates for the init command
 var InitFeatureGates = FeatureList{
 	PublicKeysECDSA: {

cmd/kubelet/app/server.go

@@ -289,6 +289,10 @@ is checked every 20 seconds (also configurable with a flag).`,
 	options.AddGlobalFlags(cleanFlagSet)
 	cleanFlagSet.BoolP("help", "h", false, fmt.Sprintf("help for %s", cmd.Name()))
 
+	// Enable feature gates
+	features.KubernetesGates().EnablePFlagControl("new-feature-gates", cleanFlagSet)
+	features.KubernetesGates().EnableEnvControl("KUBE_FEATURE_", func(err error) { panic(err.Error()) })
+
 	// ugly, but necessary, because Cobra's default UsageFunc and HelpFunc pollute the flagset with global flags
 	const usageFmt = "Usage:\n  %s\n\nFlags:\n%s"
 	cmd.SetUsageFunc(func(cmd *cobra.Command) error {
@@ -373,6 +377,7 @@ func kubeletConfigFlagPrecedence(kc *kubeletconfiginternal.KubeletConfiguration,
 	fs := newFakeFlagSet(newFlagSetWithGlobals())
 	// register throwaway KubeletFlags
 	options.NewKubeletFlags().AddFlags(fs)
+	features.KubernetesGates().EnablePFlagControl("new-feature-gates", fs)
 	// register new KubeletConfiguration
 	options.AddKubeletConfigFlags(fs, kc)
 	// Remember original feature gates, so we can merge with flag gates later

staging/src/k8s.io/cloud-provider/options/options.go

@@ -165,6 +165,7 @@ func (o *CloudControllerManagerOptions) Flags(allControllers []string, disabledB
 	fs.StringVar(&o.Generic.ClientConnection.Kubeconfig, "kubeconfig", o.Generic.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).")
 	fs.DurationVar(&o.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", o.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.")
 	utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))
+	//FIXME: what gateset(s) should this use?
 
 	return fss
 }

staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go

@@ -95,6 +95,7 @@ func NewCommandStartWardleServer(ctx context.Context, defaults *WardleServerOpti
 	flags := cmd.Flags()
 	o.RecommendedOptions.AddFlags(flags)
 	utilfeature.DefaultMutableFeatureGate.AddFlag(flags)
+	//FIXME: what gateset(s) should this use?
 
 	return cmd
 }