diff --git a/api/adc/types.go b/api/adc/types.go index b18e6fdbe0..e5966ac2dc 100644 --- a/api/adc/types.go +++ b/api/adc/types.go @@ -779,6 +779,7 @@ type Config struct { ServerAddrs []string Token string TlsVerify bool + BackendType string } // MarshalJSON implements custom JSON marshaling for adcConfig diff --git a/api/v1alpha1/gatewayproxy_types.go b/api/v1alpha1/gatewayproxy_types.go index 620236ca9e..680fa8a956 100644 --- a/api/v1alpha1/gatewayproxy_types.go +++ b/api/v1alpha1/gatewayproxy_types.go @@ -119,7 +119,13 @@ type ControlPlaneAuth struct { // ControlPlaneProvider defines configuration for control plane provider. // +kubebuilder:validation:XValidation:rule="has(self.endpoints) != has(self.service)" +// +kubebuilder:validation:XValidation:rule="oldSelf == null || (!has(self.mode) && !has(oldSelf.mode)) || self.mode == oldSelf.mode",message="mode is immutable" type ControlPlaneProvider struct { + // Mode specifies the mode of control plane provider. + // Can be `apisix` or `apisix-standalone`. + // + // +kubebuilder:validation:Optional + Mode string `json:"mode,omitempty"` // Endpoints specifies the list of control plane endpoints. // +kubebuilder:validation:Optional // +kubebuilder:validation:MinItems=1 diff --git a/config/crd/bases/apisix.apache.org_gatewayproxies.yaml b/config/crd/bases/apisix.apache.org_gatewayproxies.yaml index 313d03051c..23a7ed50e9 100644 --- a/config/crd/bases/apisix.apache.org_gatewayproxies.yaml +++ b/config/crd/bases/apisix.apache.org_gatewayproxies.yaml @@ -127,6 +127,11 @@ spec: type: string minItems: 1 type: array + mode: + description: |- + Mode specifies the mode of control plane provider. + Can be `apisix` or `apisix-standalone`. + type: string service: properties: name: @@ -150,6 +155,9 @@ spec: type: object x-kubernetes-validations: - rule: has(self.endpoints) != has(self.service) + - message: mode is immutable + rule: oldSelf == null || (!has(self.mode) && !has(oldSelf.mode)) + || self.mode == oldSelf.mode type: description: Type specifies the type of provider. Can only be `ControlPlane`. diff --git a/docs/en/latest/reference/api-reference.md b/docs/en/latest/reference/api-reference.md index 8655f8e90e..ec1882a6a9 100644 --- a/docs/en/latest/reference/api-reference.md +++ b/docs/en/latest/reference/api-reference.md @@ -229,6 +229,7 @@ ControlPlaneProvider defines configuration for control plane provider. | Field | Description | | --- | --- | +| `mode` _string_ | Mode specifies the mode of control plane provider. Can be `apisix` or `apisix-standalone`. | | `endpoints` _string array_ | Endpoints specifies the list of control plane endpoints. | | `service` _[ProviderService](#providerservice)_ | | | `tlsVerify` _boolean_ | TlsVerify specifies whether to verify the TLS certificate of the control plane. | diff --git a/internal/adc/client/client.go b/internal/adc/client/client.go index b395199016..8a498b1344 100644 --- a/internal/adc/client/client.go +++ b/internal/adc/client/client.go @@ -41,16 +41,17 @@ type Client struct { mu sync.Mutex *cache.Store - executor ADCExecutor - BackendMode string + executor ADCExecutor ConfigManager *common.ConfigManager[types.NamespacedNameKind, adctypes.Config] ADCDebugProvider *common.ADCDebugProvider + defaultMode string + log logr.Logger } -func New(log logr.Logger, mode string, timeout time.Duration) (*Client, error) { +func New(log logr.Logger, defaultMode string, timeout time.Duration) (*Client, error) { serverURL := os.Getenv("ADC_SERVER_URL") if serverURL == "" { serverURL = defaultHTTPADCExecutorAddr @@ -59,15 +60,15 @@ func New(log logr.Logger, mode string, timeout time.Duration) (*Client, error) { configManager := common.NewConfigManager[types.NamespacedNameKind, adctypes.Config]() logger := log.WithName("client") - logger.Info("ADC client initialized", "mode", mode) + logger.Info("ADC client initialized") return &Client{ Store: store, executor: NewHTTPADCExecutor(log, serverURL, timeout), - BackendMode: mode, ConfigManager: configManager, ADCDebugProvider: common.NewADCDebugProvider(store, configManager), log: logger, + defaultMode: defaultMode, }, nil } @@ -254,8 +255,11 @@ func (c *Client) sync(ctx context.Context, task Task) error { if resourceType == "" { resourceType = "all" } + if config.BackendType == "" { + config.BackendType = c.defaultMode + } - err := c.executor.Execute(ctx, c.BackendMode, config, args) + err := c.executor.Execute(ctx, config, args) duration := time.Since(startTime).Seconds() status := adctypes.StatusSuccess diff --git a/internal/adc/client/executor.go b/internal/adc/client/executor.go index 5d997efc4b..b919dcef88 100644 --- a/internal/adc/client/executor.go +++ b/internal/adc/client/executor.go @@ -44,7 +44,7 @@ const ( ) type ADCExecutor interface { - Execute(ctx context.Context, mode string, config adctypes.Config, args []string) error + Execute(ctx context.Context, config adctypes.Config, args []string) error } type DefaultADCExecutor struct { @@ -52,17 +52,17 @@ type DefaultADCExecutor struct { log logr.Logger } -func (e *DefaultADCExecutor) Execute(ctx context.Context, mode string, config adctypes.Config, args []string) error { - return e.runADC(ctx, mode, config, args) +func (e *DefaultADCExecutor) Execute(ctx context.Context, config adctypes.Config, args []string) error { + return e.runADC(ctx, config, args) } -func (e *DefaultADCExecutor) runADC(ctx context.Context, mode string, config adctypes.Config, args []string) error { +func (e *DefaultADCExecutor) runADC(ctx context.Context, config adctypes.Config, args []string) error { var execErrs = types.ADCExecutionError{ Name: config.Name, } for _, addr := range config.ServerAddrs { - if err := e.runForSingleServerWithTimeout(ctx, addr, mode, config, args); err != nil { + if err := e.runForSingleServerWithTimeout(ctx, addr, config, args); err != nil { e.log.Error(err, "failed to run adc for server", "server", addr) var execErr types.ADCExecutionServerAddrError if errors.As(err, &execErr) { @@ -81,13 +81,13 @@ func (e *DefaultADCExecutor) runADC(ctx context.Context, mode string, config adc return nil } -func (e *DefaultADCExecutor) runForSingleServerWithTimeout(ctx context.Context, serverAddr, mode string, config adctypes.Config, args []string) error { +func (e *DefaultADCExecutor) runForSingleServerWithTimeout(ctx context.Context, serverAddr string, config adctypes.Config, args []string) error { ctx, cancel := context.WithTimeout(ctx, 15*time.Second) defer cancel() - return e.runForSingleServer(ctx, serverAddr, mode, config, args) + return e.runForSingleServer(ctx, serverAddr, config, args) } -func (e *DefaultADCExecutor) runForSingleServer(ctx context.Context, serverAddr, mode string, config adctypes.Config, args []string) error { +func (e *DefaultADCExecutor) runForSingleServer(ctx context.Context, serverAddr string, config adctypes.Config, args []string) error { cmdArgs := append([]string{}, args...) if !config.TlsVerify { cmdArgs = append(cmdArgs, "--tls-skip-verify") @@ -95,7 +95,7 @@ func (e *DefaultADCExecutor) runForSingleServer(ctx context.Context, serverAddr, cmdArgs = append(cmdArgs, "--timeout", "15s") - env := e.prepareEnv(serverAddr, mode, config.Token) + env := e.prepareEnv(serverAddr, config.BackendType, config.Token) var stdout, stderr bytes.Buffer cmd := exec.CommandContext(ctx, "adc", cmdArgs...) @@ -250,26 +250,26 @@ func NewHTTPADCExecutor(log logr.Logger, serverURL string, timeout time.Duration } // Execute implements the ADCExecutor interface using HTTP calls -func (e *HTTPADCExecutor) Execute(ctx context.Context, mode string, config adctypes.Config, args []string) error { - return e.runHTTPSync(ctx, mode, config, args) +func (e *HTTPADCExecutor) Execute(ctx context.Context, config adctypes.Config, args []string) error { + return e.runHTTPSync(ctx, config, args) } // runHTTPSync performs HTTP sync to ADC Server for each server address -func (e *HTTPADCExecutor) runHTTPSync(ctx context.Context, mode string, config adctypes.Config, args []string) error { +func (e *HTTPADCExecutor) runHTTPSync(ctx context.Context, config adctypes.Config, args []string) error { var execErrs = types.ADCExecutionError{ Name: config.Name, } serverAddrs := func() []string { - if mode == "apisix-standalone" { + if config.BackendType == "apisix-standalone" { return []string{strings.Join(config.ServerAddrs, ",")} } return config.ServerAddrs }() - e.log.V(1).Info("running http sync", "serverAddrs", serverAddrs, "mode", mode) + e.log.V(1).Info("running http sync", "serverAddrs", serverAddrs) for _, addr := range serverAddrs { - if err := e.runHTTPSyncForSingleServer(ctx, addr, mode, config, args); err != nil { + if err := e.runHTTPSyncForSingleServer(ctx, addr, config, args); err != nil { e.log.Error(err, "failed to run http sync for server", "server", addr) var execErr types.ADCExecutionServerAddrError if errors.As(err, &execErr) { @@ -289,7 +289,7 @@ func (e *HTTPADCExecutor) runHTTPSync(ctx context.Context, mode string, config a } // runHTTPSyncForSingleServer performs HTTP sync to a single ADC Server -func (e *HTTPADCExecutor) runHTTPSyncForSingleServer(ctx context.Context, serverAddr, mode string, config adctypes.Config, args []string) error { +func (e *HTTPADCExecutor) runHTTPSyncForSingleServer(ctx context.Context, serverAddr string, config adctypes.Config, args []string) error { ctx, cancel := context.WithTimeout(ctx, e.httpClient.Timeout) defer cancel() @@ -306,7 +306,7 @@ func (e *HTTPADCExecutor) runHTTPSyncForSingleServer(ctx context.Context, server } // Build HTTP request - req, err := e.buildHTTPRequest(ctx, serverAddr, mode, config, labels, types, resources) + req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources) if err != nil { return fmt.Errorf("failed to build HTTP request: %w", err) } @@ -379,13 +379,13 @@ func (e *HTTPADCExecutor) loadResourcesFromFile(filePath string) (*adctypes.Reso } // buildHTTPRequest builds the HTTP request for ADC Server -func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr, mode string, config adctypes.Config, labels map[string]string, types []string, resources *adctypes.Resources) (*http.Request, error) { +func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr string, config adctypes.Config, labels map[string]string, types []string, resources *adctypes.Resources) (*http.Request, error) { // Prepare request body tlsVerify := config.TlsVerify reqBody := ADCServerRequest{ Task: ADCServerTask{ Opts: ADCServerOpts{ - Backend: mode, + Backend: config.BackendType, Server: strings.Split(serverAddr, ","), Token: config.Token, LabelSelector: labels, @@ -407,7 +407,7 @@ func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr, mode e.log.V(1).Info("sending HTTP request to ADC Server", "url", e.serverURL+"/sync", "server", serverAddr, - "mode", mode, + "mode", config.BackendType, "cacheKey", config.Name, "labelSelector", labels, "includeResourceType", types, diff --git a/internal/adc/translator/gatewayproxy.go b/internal/adc/translator/gatewayproxy.go index 259c2ac22d..13ace18d61 100644 --- a/internal/adc/translator/gatewayproxy.go +++ b/internal/adc/translator/gatewayproxy.go @@ -31,6 +31,7 @@ import ( types "github.com/apache/apisix-ingress-controller/api/adc" "github.com/apache/apisix-ingress-controller/api/v1alpha1" + "github.com/apache/apisix-ingress-controller/internal/controller/config" "github.com/apache/apisix-ingress-controller/internal/provider" "github.com/apache/apisix-ingress-controller/internal/utils" ) @@ -44,18 +45,20 @@ func (t *Translator) TranslateGatewayProxyToConfig(tctx *provider.TranslateConte if provider.Type != v1alpha1.ProviderTypeControlPlane || provider.ControlPlane == nil { return nil, nil } + cp := provider.ControlPlane - config := types.Config{ - Name: utils.NamespacedNameKind(gatewayProxy).String(), + cfg := types.Config{ + Name: utils.NamespacedNameKind(gatewayProxy).String(), + BackendType: cp.Mode, } - if provider.ControlPlane.TlsVerify != nil { - config.TlsVerify = *provider.ControlPlane.TlsVerify + if cp.TlsVerify != nil { + cfg.TlsVerify = *cp.TlsVerify } - if provider.ControlPlane.Auth.Type == v1alpha1.AuthTypeAdminKey && provider.ControlPlane.Auth.AdminKey != nil { - if provider.ControlPlane.Auth.AdminKey.ValueFrom != nil && provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef != nil { - secretRef := provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef + if cp.Auth.Type == v1alpha1.AuthTypeAdminKey && cp.Auth.AdminKey != nil { + if cp.Auth.AdminKey.ValueFrom != nil && cp.Auth.AdminKey.ValueFrom.SecretKeyRef != nil { + secretRef := cp.Auth.AdminKey.ValueFrom.SecretKeyRef secret, ok := tctx.Secrets[k8stypes.NamespacedName{ // we should use gateway proxy namespace Namespace: gatewayProxy.GetNamespace(), @@ -63,28 +66,34 @@ func (t *Translator) TranslateGatewayProxyToConfig(tctx *provider.TranslateConte }] if ok { if token, ok := secret.Data[secretRef.Key]; ok { - config.Token = string(token) + cfg.Token = string(token) } } - } else if provider.ControlPlane.Auth.AdminKey.Value != "" { - config.Token = provider.ControlPlane.Auth.AdminKey.Value + } else if cp.Auth.AdminKey.Value != "" { + cfg.Token = cp.Auth.AdminKey.Value } } - if config.Token == "" { + if cfg.Token == "" { return nil, errors.New("no token found") } - endpoints := provider.ControlPlane.Endpoints + endpoints := cp.Endpoints if len(endpoints) > 0 { - config.ServerAddrs = endpoints - return &config, nil + cfg.ServerAddrs = endpoints + return &cfg, nil } - if provider.ControlPlane.Service != nil { + // If Mode is empty, use the default static configuration. + // If Mode is set, resolve endpoints only when the ControlPlane is in standalone mode. + if cp.Mode != "" { + resolveEndpoints = cp.Mode == string(config.ProviderTypeStandalone) + } + + if cp.Service != nil { namespacedName := k8stypes.NamespacedName{ Namespace: gatewayProxy.Namespace, - Name: provider.ControlPlane.Service.Name, + Name: cp.Service.Name, } svc, ok := tctx.Services[namespacedName] if !ok { @@ -100,9 +109,9 @@ func (t *Translator) TranslateGatewayProxyToConfig(tctx *provider.TranslateConte } upstreamNodes, _, err := t.TranslateBackendRefWithFilter(tctx, gatewayv1.BackendRef{ BackendObjectReference: gatewayv1.BackendObjectReference{ - Name: gatewayv1.ObjectName(provider.ControlPlane.Service.Name), + Name: gatewayv1.ObjectName(cp.Service.Name), Namespace: (*gatewayv1.Namespace)(&gatewayProxy.Namespace), - Port: ptr.To(gatewayv1.PortNumber(provider.ControlPlane.Service.Port)), + Port: ptr.To(gatewayv1.PortNumber(cp.Service.Port)), }, }, func(endpoint *discoveryv1.Endpoint) bool { if endpoint.Conditions.Terminating != nil && *endpoint.Conditions.Terminating { @@ -115,21 +124,21 @@ func (t *Translator) TranslateGatewayProxyToConfig(tctx *provider.TranslateConte return nil, err } for _, node := range upstreamNodes { - config.ServerAddrs = append(config.ServerAddrs, "http://"+net.JoinHostPort(node.Host, strconv.Itoa(node.Port))) + cfg.ServerAddrs = append(cfg.ServerAddrs, "http://"+net.JoinHostPort(node.Host, strconv.Itoa(node.Port))) } } else { - refPort := provider.ControlPlane.Service.Port + refPort := cp.Service.Port var serverAddr string if svc.Spec.Type == corev1.ServiceTypeExternalName { serverAddr = fmt.Sprintf("http://%s:%d", svc.Spec.ExternalName, refPort) } else { - serverAddr = fmt.Sprintf("http://%s.%s.svc:%d", provider.ControlPlane.Service.Name, gatewayProxy.Namespace, refPort) + serverAddr = fmt.Sprintf("http://%s.%s.svc:%d", cp.Service.Name, gatewayProxy.Namespace, refPort) } - config.ServerAddrs = []string{serverAddr} + cfg.ServerAddrs = []string{serverAddr} } - t.Log.V(1).Info("add server address to config.ServiceAddrs", "config.ServerAddrs", config.ServerAddrs) + t.Log.V(1).Info("add server address to config.ServiceAddrs", "config.ServerAddrs", cfg.ServerAddrs) } - return &config, nil + return &cfg, nil } diff --git a/internal/manager/run.go b/internal/manager/run.go index fe16bafa26..d8f07cf5c9 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -183,7 +183,6 @@ func Run(ctx context.Context, logger logr.Logger) error { SyncTimeout: config.ControllerConfig.ExecADCTimeout.Duration, SyncPeriod: config.ControllerConfig.ProviderConfig.SyncPeriod.Duration, InitSyncDelay: config.ControllerConfig.ProviderConfig.InitSyncDelay.Duration, - BackendMode: string(config.ControllerConfig.ProviderConfig.Type), } provider, err := provider.New(providerType, logger, updater.Writer(), readier, providerOptions) if err != nil { diff --git a/internal/provider/apisix/provider.go b/internal/provider/apisix/provider.go index 0151ad0bfe..d0d8e48a05 100644 --- a/internal/provider/apisix/provider.go +++ b/internal/provider/apisix/provider.go @@ -72,11 +72,11 @@ type apisixProvider struct { func New(log logr.Logger, updater status.Updater, readier readiness.ReadinessManager, opts ...provider.Option) (provider.Provider, error) { o := provider.Options{} o.ApplyOptions(opts) - if o.BackendMode == "" { - o.BackendMode = ProviderTypeAPISIX + if o.DefaultBackendMode == "" { + o.DefaultBackendMode = ProviderTypeAPISIX } - cli, err := adcclient.New(log, o.BackendMode, o.SyncTimeout) + cli, err := adcclient.New(log, o.DefaultBackendMode, o.SyncTimeout) if err != nil { return nil, err } @@ -239,7 +239,7 @@ func (d *apisixProvider) Delete(ctx context.Context, obj client.Object) error { func (d *apisixProvider) buildConfig(tctx *provider.TranslateContext, nnk types.NamespacedNameKind) (map[types.NamespacedNameKind]adctypes.Config, error) { configs := make(map[types.NamespacedNameKind]adctypes.Config, len(tctx.ResourceParentRefs[nnk])) for _, gp := range tctx.GatewayProxies { - config, err := d.translator.TranslateGatewayProxyToConfig(tctx, &gp, d.ResolveEndpoints) + config, err := d.translator.TranslateGatewayProxyToConfig(tctx, &gp, d.DefaultResolveEndpoints) if err != nil { return nil, err } @@ -307,7 +307,7 @@ func (d *apisixProvider) NeedLeaderElection() bool { // updateConfigForGatewayProxy update config for all referrers of the GatewayProxy func (d *apisixProvider) updateConfigForGatewayProxy(tctx *provider.TranslateContext, gp *v1alpha1.GatewayProxy) error { - config, err := d.translator.TranslateGatewayProxyToConfig(tctx, gp, d.ResolveEndpoints) + config, err := d.translator.TranslateGatewayProxyToConfig(tctx, gp, d.DefaultResolveEndpoints) if err != nil { return err } diff --git a/internal/provider/init/init.go b/internal/provider/init/init.go index be21c07dd1..5400cb20ea 100644 --- a/internal/provider/init/init.go +++ b/internal/provider/init/init.go @@ -34,8 +34,8 @@ func init() { readinessManager readiness.ReadinessManager, opts ...provider.Option, ) (provider.Provider, error) { - opts = append(opts, provider.WithBackendMode("apisix-standalone")) - opts = append(opts, provider.WithResolveEndpoints()) + opts = append(opts, provider.WithDefaultBackendMode("apisix-standalone")) + opts = append(opts, provider.WithDefaultResolveEndpoints()) return apisix.New(log, statusUpdater, readinessManager, opts...) }) } diff --git a/internal/provider/options.go b/internal/provider/options.go index 379e8a0bd3..dbb0760bf0 100644 --- a/internal/provider/options.go +++ b/internal/provider/options.go @@ -26,11 +26,11 @@ type Option interface { } type Options struct { - SyncTimeout time.Duration - SyncPeriod time.Duration - InitSyncDelay time.Duration - BackendMode string - ResolveEndpoints bool + SyncTimeout time.Duration + SyncPeriod time.Duration + InitSyncDelay time.Duration + DefaultBackendMode string + DefaultResolveEndpoints bool } func (o *Options) ApplyToList(lo *Options) { @@ -43,11 +43,11 @@ func (o *Options) ApplyToList(lo *Options) { if o.InitSyncDelay > 0 { lo.InitSyncDelay = o.InitSyncDelay } - if o.BackendMode != "" { - lo.BackendMode = o.BackendMode + if o.DefaultBackendMode != "" { + lo.DefaultBackendMode = o.DefaultBackendMode } - if o.ResolveEndpoints { - lo.ResolveEndpoints = o.ResolveEndpoints + if o.DefaultResolveEndpoints { + lo.DefaultResolveEndpoints = o.DefaultResolveEndpoints } } @@ -58,22 +58,22 @@ func (o *Options) ApplyOptions(opts []Option) *Options { return o } -type backendModeOption string +type defaultBackendModeOption string -func (b backendModeOption) ApplyToList(o *Options) { - o.BackendMode = string(b) +func (b defaultBackendModeOption) ApplyToList(o *Options) { + o.DefaultBackendMode = string(b) } -func WithBackendMode(mode string) Option { - return backendModeOption(mode) +func WithDefaultBackendMode(mode string) Option { + return defaultBackendModeOption(mode) } -type resolveEndpointsOption bool +type defaultResolveEndpointsOption bool -func (r resolveEndpointsOption) ApplyToList(o *Options) { - o.ResolveEndpoints = bool(r) +func (r defaultResolveEndpointsOption) ApplyToList(o *Options) { + o.DefaultResolveEndpoints = bool(r) } -func WithResolveEndpoints() Option { - return resolveEndpointsOption(true) +func WithDefaultResolveEndpoints() Option { + return defaultResolveEndpointsOption(true) } diff --git a/test/e2e/apisix/mode.go b/test/e2e/apisix/mode.go new file mode 100644 index 0000000000..17fe62f174 --- /dev/null +++ b/test/e2e/apisix/mode.go @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package apisix + +import ( + "fmt" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/apache/apisix-ingress-controller/test/e2e/framework" + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" +) + +var _ = Describe("Test Multi-Mode Deployment", Label("networking.k8s.io", "ingress"), func() { + s := scaffold.NewDefaultScaffold() + + Context("apisix and apisix-standalone", func() { + var ns1 string + var gatewayProxyYaml = ` +apiVersion: apisix.apache.org/v1alpha1 +kind: GatewayProxy +metadata: + name: apisix-proxy-config +spec: + provider: + type: ControlPlane + controlPlane: + mode: %s + service: + name: %s + port: 9180 + auth: + type: AdminKey + adminKey: + value: "%s" +` + + const ingressClassYaml = ` +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: %s +spec: + controller: %s + parameters: + apiGroup: "apisix.apache.org" + kind: "GatewayProxy" + name: "apisix-proxy-config" + namespace: %s + scope: Namespace +` + var ingressHttpbin = ` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: httpbin +spec: + ingressClassName: %s + rules: + - host: httpbin.example + http: + paths: + - path: /get + pathType: Exact + backend: + service: + name: httpbin-service-e2e-test + port: + number: 80 +` + var ingressHttpbin2 = ` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: httpbin2 +spec: + ingressClassName: %s + rules: + - host: httpbin2.example + http: + paths: + - path: /get + pathType: Exact + backend: + service: + name: httpbin-service-e2e-test + port: + number: 80 +` + + It("apisix and apisix-standalone", func() { + gateway1, svc1, err := s.Deployer.CreateAdditionalGatewayWithOptions("multi-mode-v1", scaffold.DeployDataplaneOptions{ + ProviderType: framework.ProviderTypeAPISIX, + }) + Expect(err).NotTo(HaveOccurred(), "creating Additional Gateway") + + resources1, exists := s.GetAdditionalGateway(gateway1) + Expect(exists).To(BeTrue(), "additional gateway group should exist") + ns1 = resources1.Namespace + + By("create GatewayProxy for Additional Gateway") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayProxyYaml, framework.ProviderTypeAPISIX, svc1.Name, resources1.AdminAPIKey), resources1.Namespace) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy for Additional Gateway") + + By("create IngressClass for Additional Gateway") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingressClassYaml, ns1, s.GetControllerName(), resources1.Namespace), "") + Expect(err).NotTo(HaveOccurred(), "creating IngressClass for Additional Gateway") + + gateway2, svc2, err := s.Deployer.CreateAdditionalGatewayWithOptions("multi-mode-v2", scaffold.DeployDataplaneOptions{ + ProviderType: framework.ProviderTypeAPISIXStandalone, + }) + Expect(err).NotTo(HaveOccurred(), "creating Additional Gateway") + + resources2, exists := s.GetAdditionalGateway(gateway2) + Expect(exists).To(BeTrue(), "additional gateway group should exist") + ns2 := resources2.Namespace + + By("create GatewayProxy for Additional Gateway") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayProxyYaml, framework.ProviderTypeAPISIXStandalone, svc2.Name, resources2.AdminAPIKey), resources2.Namespace) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy for Additional Gateway") + + By("create IngressClass for Additional Gateway") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingressClassYaml, ns2, s.GetControllerName(), resources2.Namespace), "") + Expect(err).NotTo(HaveOccurred(), "creating IngressClass for Additional Gateway") + + Expect(s.CreateResourceFromString(fmt.Sprintf(ingressHttpbin, ns1))).ShouldNot(HaveOccurred(), "creating Ingress in ns1") + Expect(s.CreateResourceFromString(fmt.Sprintf(ingressHttpbin2, ns2))).ShouldNot(HaveOccurred(), "creating Ingress in ns2") + + client1, err := s.NewAPISIXClientForGateway(gateway1) + Expect(err).NotTo(HaveOccurred(), "creating APISIX client for gateway1") + + client2, err := s.NewAPISIXClientForGateway(gateway2) + Expect(err).NotTo(HaveOccurred(), "creating APISIX client for gateway2") + + s.RequestAssert(&scaffold.RequestAssert{ + Client: client1, + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) + s.RequestAssert(&scaffold.RequestAssert{ + Client: client2, + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) + + s.RequestAssert(&scaffold.RequestAssert{ + Client: client1, + Method: "GET", + Path: "/get", + Host: "httpbin2.example", + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) + s.RequestAssert(&scaffold.RequestAssert{ + Client: client2, + Method: "GET", + Path: "/get", + Host: "httpbin2.example", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) + }) + }) +}) diff --git a/test/e2e/framework/apisix_consts.go b/test/e2e/framework/apisix_consts.go index 11451a0066..cbaa91b8ae 100644 --- a/test/e2e/framework/apisix_consts.go +++ b/test/e2e/framework/apisix_consts.go @@ -33,6 +33,9 @@ var ( const ( ProviderTypeAPISIX = "apisix" ProviderTypeAPISIXStandalone = "apisix-standalone" + + ConfigProviderTypeYaml = "yaml" + ConfigProviderTypeEtcd = "etcd" ) var ( diff --git a/test/e2e/scaffold/apisix_deployer.go b/test/e2e/scaffold/apisix_deployer.go index 6e2986d040..51d16d2707 100644 --- a/test/e2e/scaffold/apisix_deployer.go +++ b/test/e2e/scaffold/apisix_deployer.go @@ -204,13 +204,17 @@ func (s *APISIXDeployer) deployDataplane(opts *APISIXDeployOptions) *corev1.Serv if opts.ServiceHTTPSPort == 0 { opts.ServiceHTTPSPort = 443 } - opts.ConfigProvider = "yaml" kubectlOpts := k8s.NewKubectlOptions("", "", opts.Namespace) - if framework.ProviderType == framework.ProviderTypeAPISIX { - opts.ConfigProvider = "etcd" - // deploy etcd + if opts.ConfigProvider == "" { + opts.ConfigProvider = framework.ConfigProviderTypeYaml + if framework.ProviderType == framework.ProviderTypeAPISIX { + opts.ConfigProvider = framework.ConfigProviderTypeEtcd + } + } + + if opts.ConfigProvider == framework.ConfigProviderTypeEtcd { k8s.KubectlApplyFromString(s.GinkgoT, kubectlOpts, framework.EtcdSpec) err := framework.WaitPodsAvailable(s.GinkgoT, kubectlOpts, metav1.ListOptions{ LabelSelector: "app=etcd", @@ -320,6 +324,10 @@ func (s *APISIXDeployer) closeAdminTunnel() { } func (s *APISIXDeployer) CreateAdditionalGateway(namePrefix string) (string, *corev1.Service, error) { + return s.CreateAdditionalGatewayWithOptions(namePrefix, DeployDataplaneOptions{}) +} + +func (s *APISIXDeployer) CreateAdditionalGatewayWithOptions(namePrefix string, opts DeployDataplaneOptions) (string, *corev1.Service, error) { // Create a new namespace for this additional gateway additionalNS := fmt.Sprintf("%s-%d", namePrefix, time.Now().Unix()) @@ -344,13 +352,32 @@ func (s *APISIXDeployer) CreateAdditionalGateway(namePrefix string) (string, *co } // Deploy dataplane for this additional gateway - opts := APISIXDeployOptions{ + o := APISIXDeployOptions{ Namespace: additionalNS, AdminKey: adminKey, ServiceHTTPPort: 9080, ServiceHTTPSPort: 9443, } - svc := s.deployDataplane(&opts) + if opts.Namespace != "" { + o.Namespace = opts.Namespace + } + if opts.AdminKey != "" { + o.AdminKey = opts.AdminKey + } + if opts.ServiceHTTPPort != 0 { + o.ServiceHTTPPort = opts.ServiceHTTPPort + } + if opts.ServiceHTTPSPort != 0 { + o.ServiceHTTPSPort = opts.ServiceHTTPSPort + } + if opts.ProviderType != "" { + if opts.ProviderType == framework.ProviderTypeAPISIX { + o.ConfigProvider = framework.ConfigProviderTypeEtcd + } else { + o.ConfigProvider = framework.ConfigProviderTypeYaml + } + } + svc := s.deployDataplane(&o) resources.DataplaneService = svc diff --git a/test/e2e/scaffold/deployer.go b/test/e2e/scaffold/deployer.go index 41e7d73f53..a1b2e9b030 100644 --- a/test/e2e/scaffold/deployer.go +++ b/test/e2e/scaffold/deployer.go @@ -29,7 +29,8 @@ type Deployer interface { ScaleDataplane(replicas int) BeforeEach() AfterEach() - CreateAdditionalGateway(namePrefix string) (string, *corev1.Service, error) + CreateAdditionalGateway(namePrefix string) (identifier string, svc *corev1.Service, err error) + CreateAdditionalGatewayWithOptions(namePrefix string, opts DeployDataplaneOptions) (identifier string, svc *corev1.Service, err error) CleanupAdditionalGateway(identifier string) error GetAdminEndpoint(...*corev1.Service) string GetAdminServiceName() string @@ -46,4 +47,5 @@ type DeployDataplaneOptions struct { ServiceHTTPSPort int Replicas *int AdminKey string + ProviderType string }