add the crd update for DMP reinstall

Author: acekingke

charts/mysql-operator/templates/cluster_rbac.yaml

@@ -9,6 +9,16 @@ metadata:
     release: {{ .Release.Name | quote }}
     heritage: {{ .Release.Service | quote }}
 rules:
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - apps
   resources:

cmd/manager/main.go

@@ -24,6 +24,7 @@ import (
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
 	"k8s.io/apimachinery/pkg/runtime"
@@ -44,6 +45,7 @@ import (
 	"github.com/radondb/radondb-mysql-kubernetes/controllers"
 	"github.com/radondb/radondb-mysql-kubernetes/controllers/backup"
 	"github.com/radondb/radondb-mysql-kubernetes/internal"
+	"github.com/radondb/radondb-mysql-kubernetes/utils"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -58,6 +60,7 @@ func init() {
 	utilruntime.Must(mysqlv1alpha1.AddToScheme(scheme))
 	utilruntime.Must(mysqlv1beta1.AddToScheme(scheme))
 	//+kubebuilder:scaffold:scheme
+	utilruntime.Must(apiextensionsv1.AddToScheme(scheme))
 }
 
 func main() {
@@ -171,10 +174,12 @@ func main() {
 		setupLog.Error(err, "unable to set up ready check")
 		os.Exit(1)
 	}
-
+	// check crds
+	utils.RunUpdeteCRD(mgr.GetClient(), &setupLog)
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}
+
 }

config/rbac/role.yaml

@@ -6,6 +6,16 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - apps
   resources:

config/samples/mysql_v1beta1_mysqlcluster.yaml

@@ -24,7 +24,7 @@ spec:
       #       path: host
 
   image: percona/percona-server:8.0.25
-  imagePullPolicy: Always
+  imagePullPolicy: IfNotPresent
   logOpts:
     resources:
       requests:

controllers/mysqlcluster_controller.go

@@ -55,6 +55,7 @@ type MysqlClusterReconciler struct {
 	internal.XenonExecutor
 }
 
+// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch;update;patch
 // +kubebuilder:rbac:groups=mysql.radondb.com,resources=mysqlclusters,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=mysql.radondb.com,resources=mysqlclusters/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=mysql.radondb.com,resources=mysqlclusters/finalizers,verbs=update

utils/incluster.go

@@ -28,15 +28,20 @@ import (
 	"os/exec"
 	"strconv"
 	"strings"
+	"time"
 
+	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/remotecommand"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -207,3 +212,138 @@ func NewConfig() (*rest.Config, error) {
 	return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
 		loader, &clientcmd.ConfigOverrides{}).ClientConfig()
 }
+
+func UpdateforCRD(crdName string, cli client.Client, log *logr.Logger) error {
+	// TODO: update CRD
+
+	// MYNS=extension-dmp
+	// CRD1=mysqlclusters.mysql.radondb.com
+	// CRD2=backups.mysql.radondb.com
+	// SEC=radondb-mysql-webhook-certs
+	// CERT=$(kubectl -n $MYNS get secrets $SEC -ojsonpath='{.data.tls\.crt}')
+	// kubectl patch CustomResourceDefinition $CRD1 --type=merge -p '{"spec":{"conversion":{"webhook":{"clientConfig":{"caBundle":"'$CERT'","service":{"namespace":"'$MYNS'"}}}}}}'
+	// kubectl patch CustomResourceDefinition $CRD2 --type=merge -p '{"spec":{"conversion":{"webhook":{"clientConfig":{"caBundle":"'$CERT'","service":{"namespace":"'$MYNS'"}}}}}}'
+	// echo $CERT
+	// fetch a secret in dmp-extension namespace which is named radondb-mysql-webhook-certs
+	// 1. first get os environment value MY_NAMESPACE, if not set, use default namespace ,"dmp-extension"
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	ns := os.Getenv("MY_NAMESPACE")
+	if len(ns) == 0 {
+		ns = "extension-dmp"
+	}
+	//2. get os environment value CERT_NAME, if not set, use default namespace "radondb-mysql-webhook-certs"
+	certName := os.Getenv("CERT_NAME")
+	if len(certName) == 0 {
+		certName = "radondb-mysql-webhook-certs"
+	}
+	secret := &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      certName,
+			Namespace: ns,
+		},
+	}
+	err := cli.Get(ctx, types.NamespacedName{Name: secret.Name, Namespace: secret.Namespace}, secret)
+	// if err is not found, return error
+	if errors.IsNotFound(err) {
+		return fmt.Errorf("secret %s not found", certName)
+	}
+	cert := secret.Data["tls.crt"]
+
+	//fetch the  CustomResourceDefinition ,which name is mysqlclusters.mysql.radondb.com
+	// 创建 CRD 实例
+	//apiextensionsv1.AddToScheme(scheme)
+	//CustomResourceDefinition
+	crd := &apiextensionsv1.CustomResourceDefinition{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "CustomResourceDefinition",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: crdName,
+		},
+	}
+	// 使用客户端获取 CRD 资源
+	errCRD := cli.Get(ctx, types.NamespacedName{Name: crdName}, crd)
+	if errCRD != nil {
+		return errCRD
+	}
+	hasBetaVersion := false
+	for _, v := range crd.Spec.Versions {
+		if v.Name == "v1beta1" {
+			hasBetaVersion = true
+		}
+	}
+	if !hasBetaVersion {
+		return fmt.Errorf("has not v1beta1 version")
+	}
+
+	oldCrd := crd.DeepCopy()
+	// if CustomResourceConversion's CABundle of Webhook is not equal to cert, update it
+	// sometime the path and port are missing, I don't know why
+	if oldCrd.Spec.Conversion == nil || oldCrd.Spec.Conversion.Webhook == nil || oldCrd.Spec.Conversion.Webhook.ClientConfig == nil ||
+		oldCrd.Spec.Conversion.Webhook.ClientConfig.CABundle == nil ||
+		oldCrd.Spec.Conversion.Webhook.ClientConfig.Service.Path == nil ||
+		oldCrd.Spec.Conversion.Webhook.ClientConfig.Service.Port == nil ||
+		!bytes.Equal(oldCrd.Spec.Conversion.Webhook.ClientConfig.CABundle, cert) {
+		crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{
+			Strategy: apiextensionsv1.WebhookConverter,
+			Webhook: &apiextensionsv1.WebhookConversion{
+				ClientConfig: &apiextensionsv1.WebhookClientConfig{
+					CABundle: []byte(cert),
+					Service: &apiextensionsv1.ServiceReference{
+						Namespace: ns,
+						Name: func() string {
+							if oldCrd.Spec.Conversion != nil && oldCrd.Spec.Conversion.Webhook != nil && oldCrd.Spec.Conversion.Webhook.ClientConfig != nil {
+								return oldCrd.Spec.Conversion.Webhook.ClientConfig.Service.Name
+							} else {
+								return "radondb-mysql-webhook"
+							}
+						}(),
+						Path: func() *string {
+							var p string = "/convert"
+							return &p
+						}(),
+						Port: func() *int32 {
+							var serverPort int32 = 443
+							return &serverPort
+						}(),
+					},
+				},
+				ConversionReviewVersions: []string{"v1"},
+			},
+		}
+		log.Info("covert crd", "value", crd.Spec.Conversion)
+	} else {
+		return nil
+	}
+	errCRD = cli.Patch(ctx, crd, client.MergeFrom(oldCrd))
+	if errCRD != nil {
+		return errCRD
+	}
+
+	return nil
+}
+
+func RunUpdeteCRD(cli client.Client, log *logr.Logger) {
+	go func() {
+		// Just run in the first 500 seconds,almost eight minutes, because the crd webhook's CABundle is not correct just in the DMP reinstall period
+		// if this process failed, just need to restart the operetor pod
+		for i := 0; i < 100; i++ {
+			time.Sleep(time.Second * 5)
+			err := UpdateforCRD("mysqlclusters.mysql.radondb.com", cli, log)
+			if err != nil {
+				log.Info("update CRD failed", "error", err)
+			}
+			err = UpdateforCRD("backups.mysql.radondb.com", cli, log)
+			if err != nil {
+				log.Info("update CRD failed", "error", err)
+			}
+		}
+		log.Info("check the crd about 8 minutes, now exit.")
+	}()
+}