diff --git a/.github/resources/manifests/multiuser/artifact-proxy/seaweed-ha/kustomization.yaml b/.github/resources/manifests/multiuser/artifact-proxy/seaweed-ha/kustomization.yaml new file mode 100644 index 00000000000..bd7248fd607 --- /dev/null +++ b/.github/resources/manifests/multiuser/artifact-proxy/seaweed-ha/kustomization.yaml @@ -0,0 +1,76 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../../../../manifests/kustomize/env/platform-agnostic-multi-user-ha + - ../../../base + +images: + - name: ghcr.io/kubeflow/kfp-api-server + newName: kind-registry:5000/apiserver + newTag: latest + - name: ghcr.io/kubeflow/kfp-persistence-agent + newName: kind-registry:5000/persistenceagent + newTag: latest + - name: ghcr.io/kubeflow/kfp-scheduled-workflow-controller + newName: kind-registry:5000/scheduledworkflow + newTag: latest + - name: ghcr.io/kubeflow/kfp-frontend + newName: kind-registry:5000/frontend + newTag: latest + - name: ghcr.io/kubeflow/kfp-metadata-writer + newName: kind-registry:5000/metadata-writer + newTag: latest + +patches: + - path: ../../../base/apiserver-env.yaml + target: + kind: Deployment + name: ml-pipeline + - path: ../../../base/grpc-specs.yaml + target: + kind: Deployment + name: metadata-grpc-deployment + - path: ../../../base/cache-specs.yaml + target: + kind: Deployment + name: cache-server + - target: + kind: ConfigMap + name: pipeline-install-config + patch: | + - op: add + path: /data/ARTIFACTS_PROXY_ENABLED + value: "true" + - target: + kind: Deployment + name: kubeflow-pipelines-profile-controller + patch: | + - op: add + path: /spec/template/spec/containers/0/env/- + value: + name: DISABLE_ISTIO_SIDECAR + value: "true" # Dangerous and only for CI/CD, not production usage + +replacements: + - source: + kind: ConfigMap + name: dns-config + fieldPath: data.namespaceDns + targets: + - select: + kind: Deployment + name: ml-pipeline + fieldPaths: + - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local] + - select: + kind: Deployment + name: metadata-grpc-deployment + fieldPaths: + - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local] + - select: + kind: Deployment + name: cache-server + fieldPaths: + - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local] + diff --git a/.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml b/.github/resources/manifests/multiuser/artifact-proxy/seaweed-single-pod/kustomization.yaml similarity index 90% rename from .github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml rename to .github/resources/manifests/multiuser/artifact-proxy/seaweed-single-pod/kustomization.yaml index ef02e240179..6a35c764b30 100644 --- a/.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml +++ b/.github/resources/manifests/multiuser/artifact-proxy/seaweed-single-pod/kustomization.yaml @@ -2,8 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../../../../../manifests/kustomize/env/platform-agnostic-multi-user - - ../../base + - ../../../../../../manifests/kustomize/env/platform-agnostic-multi-user + - ../../../base images: - name: ghcr.io/kubeflow/kfp-api-server @@ -23,15 +23,15 @@ images: newTag: latest patches: - - path: ../../base/apiserver-env.yaml + - path: ../../../base/apiserver-env.yaml target: kind: Deployment name: ml-pipeline - - path: ../../base/grpc-specs.yaml + - path: ../../../base/grpc-specs.yaml target: kind: Deployment name: metadata-grpc-deployment - - path: ../../base/cache-specs.yaml + - path: ../../../base/cache-specs.yaml target: kind: Deployment name: cache-server diff --git a/.github/resources/scripts/deploy-kfp.sh b/.github/resources/scripts/deploy-kfp.sh index 146146af010..70884c9bcde 100755 --- a/.github/resources/scripts/deploy-kfp.sh +++ b/.github/resources/scripts/deploy-kfp.sh @@ -167,8 +167,12 @@ elif [ "${MULTI_USER}" == "false" ] && [ "${PIPELINES_STORE}" == "kubernetes" ]; fi elif [ "${MULTI_USER}" == "true" ]; then TEST_MANIFESTS="${TEST_MANIFESTS}/multiuser" - if $ARTIFACT_PROXY_ENABLED && [ "${STORAGE_BACKEND}" == "seaweedfs" ]; then - TEST_MANIFESTS="${TEST_MANIFESTS}/artifact-proxy" + if $ARTIFACT_PROXY_ENABLED; then + if [ "${STORAGE_BACKEND}" == "seaweedfs" ]; then + TEST_MANIFESTS="${TEST_MANIFESTS}/artifact-proxy/seaweed-single-pod" + elif [ "${STORAGE_BACKEND}" == "seaweedfs-ha" ]; then + TEST_MANIFESTS="${TEST_MANIFESTS}/artifact-proxy/seaweed-ha" + fi elif [ "${STORAGE_BACKEND}" == "minio" ]; then TEST_MANIFESTS="${TEST_MANIFESTS}/minio" elif $CACHE_DISABLED && [ "${STORAGE_BACKEND}" == "minio" ]; then diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 5d80a80b33e..535ee08439c 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -181,7 +181,12 @@ jobs: storage: "seaweedfs" k8s_version: "v1.34.0" cache_enabled: "true" - test_label: "E2ECritical" + test_label: "E2ECritical" + - multi_user: "true" + artifact_proxy: "true" + storage: "seaweedfs-ha" + k8s_version: "v1.34.0" + test_label: "E2ECritical" fail-fast: false name: End to End Critical Scenario Multi User Tests - K8s ${{ matrix.k8s_version }} cacheEnabled=${{ matrix.cache_enabled }} multiUser=${{ matrix.multi_user }} storage=${{ matrix.storage }} artifactProxy=${{ matrix.artifact_proxy }} steps: diff --git a/manifests/kustomize/env/platform-agnostic-multi-user-ha/kustomization.yaml b/manifests/kustomize/env/platform-agnostic-multi-user-ha/kustomization.yaml new file mode 100644 index 00000000000..522c0ad2ce1 --- /dev/null +++ b/manifests/kustomize/env/platform-agnostic-multi-user-ha/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../third-party/metacontroller/base +- ../../base/installs/multi-user +- ../../base/metadata/base +- ../../base/metadata/options/istio +- ../../third-party/argo/installs/cluster +- ../../third-party/mysql/base +- ../../third-party/mysql/options/istio +- ../../third-party/seaweedfs/ha/istio + + +# !!! If you want to customize the namespace, +# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml +namespace: kubeflow + +# Identifier for application manager to apply ownerReference. +# The ownerReference ensures the resources get garbage collected +# when application is deleted. +labels: +- includeSelectors: true + pairs: + application-crd-id: kubeflow-pipelines diff --git a/manifests/kustomize/env/platform-agnostic-multi-user/kustomization.yaml b/manifests/kustomize/env/platform-agnostic-multi-user/kustomization.yaml index 014f327cc2a..36fb8a6af32 100644 --- a/manifests/kustomize/env/platform-agnostic-multi-user/kustomization.yaml +++ b/manifests/kustomize/env/platform-agnostic-multi-user/kustomization.yaml @@ -9,7 +9,7 @@ resources: - ../../third-party/argo/installs/cluster - ../../third-party/mysql/base - ../../third-party/mysql/options/istio -- ../../third-party/seaweedfs/istio +- ../../third-party/seaweedfs/single-pod/istio # !!! If you want to customize the namespace, diff --git a/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/kustomization.yaml b/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/kustomization.yaml index f2306acabfe..1fc5adfb004 100644 --- a/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/kustomization.yaml +++ b/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/kustomization.yaml @@ -3,10 +3,6 @@ kind: Kustomization namespace: kubeflow resources: -- seaweedfs-deployment.yaml -- seaweedfs-pvc.yaml -- seaweedfs-networkpolicy.yaml -- seaweedfs-service.yaml - seaweedfs-service-account.yaml - minio-service.yaml - mlpipeline-minio-artifact-secret.yaml diff --git a/manifests/kustomize/third-party/seaweedfs/ha/base/filer-statefulset.yaml b/manifests/kustomize/third-party/seaweedfs/ha/base/filer-statefulset.yaml new file mode 100644 index 00000000000..f3481bdbd1a --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/base/filer-statefulset.yaml @@ -0,0 +1,131 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: seaweedfs + component: filer + name: seaweedfs-filer +spec: + serviceName: seaweedfs-filer + replicas: 2 + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app: seaweedfs + component: filer + template: + metadata: + labels: + app: seaweedfs + component: filer + application-crd-id: kubeflow-pipelines + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app: seaweedfs + component: filer + topologyKey: kubernetes.io/hostname + serviceAccountName: seaweedfs + terminationGracePeriodSeconds: 60 + securityContext: + seccompProfile: + type: RuntimeDefault + fsGroup: 1000 + containers: + - name: seaweedfs-filer + image: 'chrislusf/seaweedfs' + envFrom: + - secretRef: + name: mlpipeline-minio-artifact + args: + - 'filer' + - '-port=8888' + - '-iam' + - '-master=seaweedfs-master-0.seaweedfs-master:9333,seaweedfs-master-1.seaweedfs-master:9333,seaweedfs-master-2.seaweedfs-master:9333' + lifecycle: + postStart: + exec: + command: + - /bin/sh + - -ec + - | + # Only run this on one filer + if [[ $(hostname) == seaweedfs-filer-0 ]]; then + exit 0 + fi + # wait until seaweedfs filer is ready + # Filer will be ready only after masters are ready. + # We run this not directly on one of the masters, because that will not work for some reason. + # This is closer to the single pod setup, where we wait for the s3 endpoint to get ready, which also + # directly depends on the filer. + echo "Wait for cluster ready" + for i in $(seq 1 120); do + if wget -q --spider http://127.0.0.1:8888/ ; then + break + fi + sleep 2 + done + # create bucket if not exists (ignore error if exists) + echo "s3.bucket.create --name mlpipeline" | /usr/bin/weed shell || true + # configure admin user using keys from secret + echo "s3.configure -user kubeflow-admin -access_key $accesskey -secret_key $secretkey -actions Admin -apply" | /usr/bin/weed shell + volumeMounts: + - name: data-filer + mountPath: /data + ports: + - containerPort: 8888 + name: http-filer + - containerPort: 18888 + name: grpc-filer + - containerPort: 8333 + name: http-s3 + - containerPort: 8111 + name: http-iam + env: + # These two env vars are needed to enable weed shell to discover the master server + - name: WEED_CLUSTER_SW_MASTER + value: seaweedfs-master.kubeflow:9333 + - name: WEED_CLUSTER_DEFAULT + value: sw + readinessProbe: + httpGet: + path: / + port: 8888 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 15 + successThreshold: 1 + failureThreshold: 100 + timeoutSeconds: 10 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 2Gi + securityContext: # Using restricted profile + allowPrivilegeEscalation: false + privileged: false + runAsNonRoot: true + # image defaults to root user + runAsUser: 1000 + runAsGroup: 1000 + capabilities: + drop: + - ALL + volumeClaimTemplates: + - metadata: + name: data-filer + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi diff --git a/manifests/kustomize/third-party/seaweedfs/ha/base/filer-svc.yaml b/manifests/kustomize/third-party/seaweedfs/ha/base/filer-svc.yaml new file mode 100644 index 00000000000..e27433f16af --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/base/filer-svc.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + labels: + app: seaweedfs + component: filer + name: seaweedfs-filer-intern +spec: + ports: + - name: grpc-filer + port: 18888 + protocol: TCP + targetPort: 18888 + - name: http-filer + port: 8888 + protocol: TCP + targetPort: 8888 + publishNotReadyAddresses: true + selector: + app: seaweedfs + component: filer +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: seaweedfs + component: filer + name: seaweedfs-filer +spec: + ports: + - name: http-iam + port: 8111 + protocol: TCP + targetPort: 8111 + - name: http-s3 + port: 8333 + protocol: TCP + targetPort: 8333 + - name: grpc-filer + port: 18888 + protocol: TCP + targetPort: 18888 + - name: http-filer + port: 8888 + protocol: TCP + targetPort: 8888 + selector: + app: seaweedfs + component: filer diff --git a/manifests/kustomize/third-party/seaweedfs/ha/base/kustomization.yaml b/manifests/kustomize/third-party/seaweedfs/ha/base/kustomization.yaml new file mode 100644 index 00000000000..2eab6051857 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/base/kustomization.yaml @@ -0,0 +1,38 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow + +resources: +- ../../base +- filer-statefulset.yaml +- filer-svc.yaml +- master-statefulset.yaml +- master-svc.yaml +- s3-gateway-deployment.yaml +- volume-statefulset.yaml +- volume-svc.yaml + +patches: +- target: + version: v1 + kind: Job + name: init-seaweedfs + patch: |- + - op: replace + path: /spec/template/spec/containers/0/env/1/value + value: "seaweedfs-master:9333" + - op: add + path: /spec/template/spec/containers/0/env/- + value: {"name": "WEED_CLUSTER_SW_FILER", "value": "seaweedfs-filer:8888"} +- target: + version: v1 + kind: Service + name: minio-service + patch: |- + - op: add + path: /spec/selector/component + value: s3 + +images: +- name: chrislusf/seaweedfs + newTag: '4.00' diff --git a/manifests/kustomize/third-party/seaweedfs/ha/base/master-statefulset.yaml b/manifests/kustomize/third-party/seaweedfs/ha/base/master-statefulset.yaml new file mode 100644 index 00000000000..3c96fc290e0 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/base/master-statefulset.yaml @@ -0,0 +1,104 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: seaweedfs + component: master + name: seaweedfs-master +spec: + serviceName: seaweedfs-master + replicas: 3 + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app: seaweedfs + component: master + template: + metadata: + labels: + app: seaweedfs + component: master + application-crd-id: kubeflow-pipelines + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app: seaweedfs + component: master + topologyKey: kubernetes.io/hostname + serviceAccountName: seaweedfs + terminationGracePeriodSeconds: 60 + securityContext: + seccompProfile: + type: RuntimeDefault + fsGroup: 1000 + containers: + - name: seaweedfs-master + image: 'chrislusf/seaweedfs' + args: + - 'master' + - '-mdir=/data' + - '-defaultReplication=001' # replicate data to at least another volume (same "rack" and "datacenter") + - '-volumePreallocate=false' + - '-ip=$(POD_NAME).seaweedfs-master.$(NAMESPACE)' + - '-ip.bind=0.0.0.0' + - '-port=9333' + - '-peers=seaweedfs-master-0.seaweedfs-master.$(NAMESPACE):9333,seaweedfs-master-1.seaweedfs-master.$(NAMESPACE):9333,seaweedfs-master-2.seaweedfs-master.$(NAMESPACE):9333' + volumeMounts: + - name : data-master + mountPath: /data + ports: + - containerPort: 9333 + name: http-master + - containerPort: 19333 + name: grpc-master + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + readinessProbe: + httpGet: + path: /cluster/status + port: 9333 + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 15 + successThreshold: 1 + failureThreshold: 100 + timeoutSeconds: 10 + resources: + requests: + cpu: 128m + memory: 256Mi + limits: + memory: 256Mi + securityContext: # Using restricted profile + allowPrivilegeEscalation: false + privileged: false + runAsNonRoot: true + # image defaults to root user + runAsUser: 1000 + runAsGroup: 1000 + capabilities: + drop: + - ALL + volumeClaimTemplates: + - metadata: + name: data-master + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi diff --git a/manifests/kustomize/third-party/seaweedfs/ha/base/master-svc.yaml b/manifests/kustomize/third-party/seaweedfs/ha/base/master-svc.yaml new file mode 100644 index 00000000000..59f895bdccf --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/base/master-svc.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: seaweedfs + component: master + name: seaweedfs-master +spec: + publishNotReadyAddresses: true + ports: + - name: http-master + port: 9333 + protocol: TCP + targetPort: 9333 + - name: grpc-master + port: 19333 + protocol: TCP + targetPort: 19333 + selector: + app: seaweedfs + component: master diff --git a/manifests/kustomize/third-party/seaweedfs/ha/base/s3-gateway-deployment.yaml b/manifests/kustomize/third-party/seaweedfs/ha/base/s3-gateway-deployment.yaml new file mode 100644 index 00000000000..7399f17e1a4 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/base/s3-gateway-deployment.yaml @@ -0,0 +1,83 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: seaweedfs-s3 + labels: + app: seaweedfs + component: s3 +spec: + replicas: 2 + selector: + matchLabels: + app: seaweedfs + component: s3 + template: + metadata: + labels: + app: seaweedfs + component: s3 + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app: seaweedfs + component: s3 + topologyKey: kubernetes.io/hostname + terminationGracePeriodSeconds: 10 + serviceAccountName: seaweedfs + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: seaweedfs + image: chrislusf/seaweedfs + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - 's3' + - '-ip.bind=0.0.0.0' + - '-filer=seaweedfs-filer:8888' + ports: + - containerPort: 8333 + name: swfs-s3 + readinessProbe: + httpGet: + path: /status + port: 8333 + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 50 + timeoutSeconds: 2 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 256Mi + securityContext: # Using restricted profile + allowPrivilegeEscalation: false + privileged: false + runAsNonRoot: true + # image defaults to root user + runAsUser: 1000 + runAsGroup: 1000 + capabilities: + drop: + - ALL diff --git a/manifests/kustomize/third-party/seaweedfs/ha/base/volume-statefulset.yaml b/manifests/kustomize/third-party/seaweedfs/ha/base/volume-statefulset.yaml new file mode 100644 index 00000000000..6292879a3f7 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/base/volume-statefulset.yaml @@ -0,0 +1,113 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: seaweedfs + component: volume + name: seaweedfs-volume +spec: + serviceName: seaweedfs-volume + replicas: 3 + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app: seaweedfs + component: volume + template: + metadata: + labels: + app: seaweedfs + component: volume + application-crd-id: kubeflow-pipelines + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app: seaweedfs + component: volume + topologyKey: kubernetes.io/hostname + serviceAccountName: seaweedfs + terminationGracePeriodSeconds: 150 + securityContext: + seccompProfile: + type: RuntimeDefault + fsGroup: 1000 + containers: + - name: seaweedfs + image: chrislusf/seaweedfs + args: + - volume + - -port=8080 + - -dir /data + - -ip.bind=0.0.0.0 + - -ip=$(POD_NAME).seaweedfs-volume.$(NAMESPACE) + - -mserver=seaweedfs-master-0.seaweedfs-master.$(NAMESPACE):9333,seaweedfs-master-1.seaweedfs-master.$(NAMESPACE):9333,seaweedfs-master-2.seaweedfs-master.$(NAMESPACE):9333 + volumeMounts: + - mountPath: /data/ + name: swfs-vol + ports: + - containerPort: 8080 + name: swfs-vol + protocol: TCP + - containerPort: 18080 + name: swfs-vol-grpc + protocol: TCP + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: WEED_CLUSTER_SW_FILER + value: seaweedfs-filer:8888 + - name: WEED_CLUSTER_SW_MASTER + value: seaweedfs-master:9333 + readinessProbe: + failureThreshold: 100 + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 15 + successThreshold: 1 + timeoutSeconds: 30 + resources: + requests: + cpu: 128m + memory: 256Mi + limits: + memory: 2Gi + securityContext: # Using restricted profile + allowPrivilegeEscalation: false + privileged: false + runAsNonRoot: true + # image defaults to root user + runAsUser: 1000 + runAsGroup: 1000 + capabilities: + drop: + - ALL + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: swfs-vol + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 30Gi + volumeMode: Filesystem diff --git a/manifests/kustomize/third-party/seaweedfs/ha/base/volume-svc.yaml b/manifests/kustomize/third-party/seaweedfs/ha/base/volume-svc.yaml new file mode 100644 index 00000000000..0216ec36589 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/base/volume-svc.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: seaweedfs + component: volume + name: seaweedfs-volume +spec: + ports: + - name: swfs-volume + port: 8080 + protocol: TCP + targetPort: 8080 + - name: swfs-volume-18080 + port: 18080 + protocol: TCP + targetPort: 18080 + selector: + app: seaweedfs + component: volume diff --git a/manifests/kustomize/third-party/seaweedfs/ha/istio/istio-authorization-policy.yaml b/manifests/kustomize/third-party/seaweedfs/ha/istio/istio-authorization-policy.yaml new file mode 100644 index 00000000000..bdc54af3246 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/istio/istio-authorization-policy.yaml @@ -0,0 +1,33 @@ +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: seaweedfs-service +spec: + action: ALLOW + selector: + matchLabels: + app: seaweedfs + component: s3 + rules: + - from: + - source: + principals: + - cluster.local/ns/kubeflow/sa/ml-pipeline + - from: + - source: + principals: + - cluster.local/ns/kubeflow/sa/ml-pipeline-ui + # Allow traffic to s3 endpoint from User Pipeline Pods, which don't have a sidecar. + # Also needed for traffic from seaweedfs init pod. Seaweedfs gives the client an ip to connect to. This can not be + # handled well by istio (AuthPolicy). Instead, access to the sensitive ports will be limited by the NetworkPolicy. + - {} +--- +apiVersion: "networking.istio.io/v1alpha3" +kind: DestinationRule +metadata: + name: ml-pipeline-seaweedfs +spec: + host: seaweedfs.kubeflow.svc.cluster.local + trafficPolicy: + tls: + mode: ISTIO_MUTUAL diff --git a/manifests/kustomize/third-party/seaweedfs/ha/istio/kustomization.yaml b/manifests/kustomize/third-party/seaweedfs/ha/istio/kustomization.yaml new file mode 100644 index 00000000000..449e3cdd912 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/ha/istio/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow + +resources: +- ../base +patches: +- target: + version: security.istio.io/v1beta1 + kind: AuthorizationPolicy + name: seaweedfs-service + patch: |- + - op: add + path: /spec/selector/matchLabels/component + value: s3 diff --git a/manifests/kustomize/third-party/seaweedfs/single-pod/base/kustomization.yaml b/manifests/kustomize/third-party/seaweedfs/single-pod/base/kustomization.yaml new file mode 100644 index 00000000000..719e1505bf3 --- /dev/null +++ b/manifests/kustomize/third-party/seaweedfs/single-pod/base/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../base/ +- seaweedfs-deployment.yaml +- seaweedfs-networkpolicy.yaml +- seaweedfs-pvc.yaml +- seaweedfs-service.yaml \ No newline at end of file diff --git a/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-deployment.yaml b/manifests/kustomize/third-party/seaweedfs/single-pod/base/seaweedfs-deployment.yaml similarity index 100% rename from manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-deployment.yaml rename to manifests/kustomize/third-party/seaweedfs/single-pod/base/seaweedfs-deployment.yaml diff --git a/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-networkpolicy.yaml b/manifests/kustomize/third-party/seaweedfs/single-pod/base/seaweedfs-networkpolicy.yaml similarity index 100% rename from manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-networkpolicy.yaml rename to manifests/kustomize/third-party/seaweedfs/single-pod/base/seaweedfs-networkpolicy.yaml diff --git a/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-pvc.yaml b/manifests/kustomize/third-party/seaweedfs/single-pod/base/seaweedfs-pvc.yaml similarity index 100% rename from manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-pvc.yaml rename to manifests/kustomize/third-party/seaweedfs/single-pod/base/seaweedfs-pvc.yaml diff --git a/manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-service.yaml b/manifests/kustomize/third-party/seaweedfs/single-pod/base/seaweedfs-service.yaml similarity index 100% rename from manifests/kustomize/third-party/seaweedfs/base/seaweedfs/seaweedfs-service.yaml rename to manifests/kustomize/third-party/seaweedfs/single-pod/base/seaweedfs-service.yaml diff --git a/manifests/kustomize/third-party/seaweedfs/istio/istio-authorization-policy.yaml b/manifests/kustomize/third-party/seaweedfs/single-pod/istio/istio-authorization-policy.yaml similarity index 100% rename from manifests/kustomize/third-party/seaweedfs/istio/istio-authorization-policy.yaml rename to manifests/kustomize/third-party/seaweedfs/single-pod/istio/istio-authorization-policy.yaml diff --git a/manifests/kustomize/third-party/seaweedfs/istio/kustomization.yaml b/manifests/kustomize/third-party/seaweedfs/single-pod/istio/kustomization.yaml similarity index 92% rename from manifests/kustomize/third-party/seaweedfs/istio/kustomization.yaml rename to manifests/kustomize/third-party/seaweedfs/single-pod/istio/kustomization.yaml index 2dffd5d150b..f948da42bc4 100644 --- a/manifests/kustomize/third-party/seaweedfs/istio/kustomization.yaml +++ b/manifests/kustomize/third-party/seaweedfs/single-pod/istio/kustomization.yaml @@ -3,5 +3,5 @@ kind: Kustomization namespace: kubeflow resources: -- ../base/ +- ../base - istio-authorization-policy.yaml