Skip to content

Commit 5fdae71

Browse files
williamlardierwilliamlardier
committed
test base 2
1 parent 3c03bff commit 5fdae71

28 files changed

+346
-5932
lines changed

‎.github/scripts/end2end/dump-setup-state.sh‎

Lines changed: 301 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,301 @@
1+
#!/bin/bash
2+
set -eu
3+
4+
NAMESPACE="${1:-default}"
5+
ZENKO_NAME="${2:-end2end}"
6+
7+
echo "=================================================="
8+
echo "ZENKO SETUP STATE DUMP"
9+
echo "Namespace: ${NAMESPACE}"
10+
echo "Zenko Name: ${ZENKO_NAME}"
11+
echo "Timestamp: $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
12+
echo "=================================================="
13+
echo ""
14+
15+
echo "=== QUICK SUMMARY ==="
16+
INSTANCE_ID=$(kubectl get zenko "${ZENKO_NAME}" -n "${NAMESPACE}" -o jsonpath='{.status.instanceID}' 2>/dev/null || echo "NONE")
17+
ZENKO_AVAILABLE=$(kubectl get zenko "${ZENKO_NAME}" -n "${NAMESPACE}" -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>/dev/null || echo "UNKNOWN")
18+
NUM_LOCATIONS=$(kubectl get secret -l "app.kubernetes.io/name=connector-cloudserver-config,app.kubernetes.io/instance=${ZENKO_NAME}" -n "${NAMESPACE}" -o json 2>/dev/null | \
19+
jq '.items[0].data."locationConfig.json"' | sed 's/"//g' | base64 -d 2>/dev/null | jq '. | length' || echo "0")
20+
NUM_SORBET_FWDS=$(kubectl get deployment -n "${NAMESPACE}" -l "app.kubernetes.io/instance=${ZENKO_NAME}" -o json 2>/dev/null | \
21+
jq '[.items[] | select(.metadata.labels."sorbetd-location" != null)] | length' || echo "0")
22+
NUM_ACCOUNTS=$(kubectl get secrets -n "${NAMESPACE}" -l "type=end2end" -o json 2>/dev/null | jq '.items | length' || echo "0")
23+
24+
echo "Instance ID: ${INSTANCE_ID}"
25+
echo "Zenko Available: ${ZENKO_AVAILABLE}"
26+
echo "Locations configured: ${NUM_LOCATIONS}"
27+
echo "Sorbet forwarders: ${NUM_SORBET_FWDS}"
28+
echo "Test accounts: ${NUM_ACCOUNTS}"
29+
echo ""
30+
31+
echo "=== ZENKO CUSTOM RESOURCE ==="
32+
kubectl get zenko "${ZENKO_NAME}" -n "${NAMESPACE}" -o json | jq '.' || echo "ERROR: Failed to get Zenko CR"
33+
echo ""
34+
35+
echo "=== ZENKO STATUS ==="
36+
kubectl get zenko "${ZENKO_NAME}" -n "${NAMESPACE}" -o json | jq '.status' || echo "ERROR: Failed to get status"
37+
echo ""
38+
39+
echo "=== ZENKO ANNOTATIONS ==="
40+
kubectl get zenko "${ZENKO_NAME}" -n "${NAMESPACE}" -o json | jq '.metadata.annotations' || echo "No annotations"
41+
echo ""
42+
43+
echo "=== INSTANCE ID ==="
44+
echo "${INSTANCE_ID}"
45+
echo ""
46+
47+
echo "=== SORBET CONFIGURATION (from Zenko CR) ==="
48+
kubectl get zenko "${ZENKO_NAME}" -n "${NAMESPACE}" -o json | jq '.spec.sorbet' || echo "No sorbet config"
49+
echo ""
50+
51+
echo "=== CLOUDSERVER CONFIG SECRET ==="
52+
kubectl get secret -l "app.kubernetes.io/name=connector-cloudserver-config,app.kubernetes.io/instance=${ZENKO_NAME}" -n "${NAMESPACE}" -o json | \
53+
jq '.items[0].data."config.json"' | sed 's/"//g' | base64 -d | jq '.' || echo "ERROR: Failed to get cloudserver config"
54+
echo ""
55+
56+
echo "=== LOCATION CONFIG (from cloudserver config) ==="
57+
kubectl get secret -l "app.kubernetes.io/name=connector-cloudserver-config,app.kubernetes.io/instance=${ZENKO_NAME}" -n "${NAMESPACE}" -o json | \
58+
jq '.items[0].data."locationConfig.json"' | sed 's/"//g' | base64 -d | jq '.' || echo "ERROR: Failed to get location config"
59+
echo ""
60+
61+
echo "=== BACKBEAT CONFIG ==="
62+
kubectl get secret -l "app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=${ZENKO_NAME}" -n "${NAMESPACE}" -o json | \
63+
jq '.items[0].data."config.json"' | sed 's/"//g' | base64 -d | jq '.' || echo "ERROR: Failed to get backbeat config"
64+
echo ""
65+
66+
echo "=== SORBET CONFIGS (cold location configs) ==="
67+
for secret in $(kubectl get secrets -n "${NAMESPACE}" -l "app.kubernetes.io/instance=${ZENKO_NAME}" -o name | grep sorbet-config); do
68+
echo "--- ${secret} ---"
69+
kubectl get "${secret}" -n "${NAMESPACE}" -o json | \
70+
jq '.data."config.json"' | sed 's/"//g' | base64 -d | jq '.' || echo "ERROR: Failed to decode"
71+
echo ""
72+
done
73+
echo ""
74+
75+
echo "=== SORBET FORWARDER USER CREDENTIALS ==="
76+
kubectl get secret -l "app.kubernetes.io/name=sorbet-fwd-creds,app.kubernetes.io/instance=${ZENKO_NAME}" -n "${NAMESPACE}" -o json | \
77+
jq '.items[0] | {name: .metadata.name, accessKeyId: (.data.accessKey | @base64d)[0:10] + "..."}' || echo "No sorbet-fwd credentials found"
78+
echo ""
79+
80+
echo "=== SERVICE USER CREDENTIALS ==="
81+
for user_type in backbeat-lcbp-user-creds backbeat-lcc-user-creds backbeat-lcop-user-creds backbeat-qp-user-creds; do
82+
echo "--- ${user_type} ---"
83+
kubectl get secret -l "app.kubernetes.io/name=${user_type},app.kubernetes.io/instance=${ZENKO_NAME}" -n "${NAMESPACE}" -o json | \
84+
jq '.items[0].metadata.name' 2>/dev/null || echo "Not found"
85+
done
86+
echo ""
87+
88+
echo "=== ACCOUNT SECRETS ==="
89+
kubectl get secrets -n "${NAMESPACE}" -l "type=end2end" -o json | \
90+
jq '.items[] | {name: .metadata.name, accountId: (.data.AccountId | @base64d), accessKeyId: (.data.AccessKeyId | @base64d)[0:10] + "..."}' || echo "No account secrets found"
91+
echo ""
92+
93+
echo "=== ALL SECRETS FOR THIS INSTANCE ==="
94+
kubectl get secrets -n "${NAMESPACE}" -l "app.kubernetes.io/instance=${ZENKO_NAME}" -o json | \
95+
jq '.items[] | {name: .metadata.name, labels: .metadata.labels, keys: (.data | keys)}' | head -100
96+
echo ""
97+
98+
echo "=== KAFKA TOPICS ==="
99+
if [ -n "${INSTANCE_ID}" ]; then
100+
echo "Instance ID: ${INSTANCE_ID}"
101+
102+
KAFKA_IMAGE=$(yq eval ".kafka.sourceRegistry" ../../../solution/deps.yaml)/$(yq eval ".kafka.image" ../../../solution/deps.yaml):$(yq eval ".kafka.tag" ../../../solution/deps.yaml)
103+
KAFKA_HOST_PORT=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance="${ZENKO_NAME}" -n "${NAMESPACE}" \
104+
-o jsonpath='{.items[0].data.config\.json}' | base64 -d | jq -r '.kafka.hosts')
105+
106+
echo "Listing Kafka topics (grep for ${INSTANCE_ID}):"
107+
kubectl run kafka-list-topics-dump \
108+
--image="${KAFKA_IMAGE}" \
109+
--pod-running-timeout=2m \
110+
--rm \
111+
--restart=Never \
112+
--attach=True \
113+
--namespace="${NAMESPACE}" \
114+
--command -- bash -c "kafka-topics.sh --list --bootstrap-server ${KAFKA_HOST_PORT}" 2>/dev/null | grep "${INSTANCE_ID}" || echo "No topics found for instance"
115+
fi
116+
echo ""
117+
118+
echo "=== ZENKO NOTIFICATION TARGETS ==="
119+
kubectl get zenkonotificationtargets -n "${NAMESPACE}" -o json | jq '.items[] | {name: .metadata.name, spec: .spec}' || echo "No notification targets"
120+
echo ""
121+
122+
echo "=== DEPLOYMENTS STATUS ==="
123+
echo "--- Cloudserver ---"
124+
kubectl get deployment -l "app.kubernetes.io/name=connector-cloudserver,app.kubernetes.io/instance=${ZENKO_NAME}" -n "${NAMESPACE}" -o json | \
125+
jq '.items[0] | {name: .metadata.name, replicas: .status.replicas, ready: .status.readyReplicas, generation: .status.observedGeneration, annotations: .metadata.annotations}' || echo "Not found"
126+
127+
echo "--- Internal Cloudserver ---"
128+
kubectl get deployment -l "app.kubernetes.io/name=internal-s3api,app.kubernetes.io/instance=${ZENKO_NAME}" -n "${NAMESPACE}" -o json | \
129+
jq '.items[0] | {name: .metadata.name, replicas: .status.replicas, ready: .status.readyReplicas}' || echo "Not found"
130+
131+
echo "--- Backbeat Lifecycle Transition Processor ---"
132+
kubectl get deployment -l "app.kubernetes.io/name=backbeat-lifecycle-transition-processor,app.kubernetes.io/instance=${ZENKO_NAME}" -n "${NAMESPACE}" -o json | \
133+
jq '.items[0] | {name: .metadata.name, replicas: .status.replicas, ready: .status.readyReplicas}' || echo "Not found"
134+
135+
echo "--- Backbeat Lifecycle Conductor ---"
136+
kubectl get deployment -l "app.kubernetes.io/name=backbeat-lifecycle-conductor,app.kubernetes.io/instance=${ZENKO_NAME}" -n "${NAMESPACE}" -o json | \
137+
jq '.items[0] | {name: .metadata.name, replicas: .status.replicas, ready: .status.readyReplicas}' || echo "Not found"
138+
139+
echo "--- Backbeat Lifecycle Bucket Processor ---"
140+
kubectl get deployment -l "app.kubernetes.io/name=backbeat-lifecycle-bucket-processor,app.kubernetes.io/instance=${ZENKO_NAME}" -n "${NAMESPACE}" -o json | \
141+
jq '.items[0] | {name: .metadata.name, replicas: .status.replicas, ready: .status.readyReplicas}' || echo "Not found"
142+
143+
echo "--- ALL Sorbet Forwarder Deployments ---"
144+
kubectl get deployment -n "${NAMESPACE}" -l "app.kubernetes.io/instance=${ZENKO_NAME}" -o json | \
145+
jq '.items[] | select(.metadata.labels."sorbetd-location" != null) | {name: .metadata.name, location: .metadata.labels."sorbetd-location", replicas: .status.replicas, ready: .status.readyReplicas}' || echo "No sorbet forwarders found"
146+
147+
echo "--- Mock Sorbet ---"
148+
kubectl get deployment mock-sorbet -n "${NAMESPACE}" -o json | \
149+
jq '{name: .metadata.name, replicas: .status.replicas, ready: .status.readyReplicas}' 2>/dev/null || echo "Not found"
150+
151+
echo "--- Mock Miria ---"
152+
kubectl get deployment mock-miria -n "${NAMESPACE}" -o json | \
153+
jq '{name: .metadata.name, replicas: .status.replicas, ready: .status.readyReplicas}' 2>/dev/null || echo "Not found"
154+
echo ""
155+
156+
echo "=== PODS STATUS ==="
157+
echo "All pods (sorbet and backbeat lifecycle):"
158+
kubectl get pods -n "${NAMESPACE}" --no-headers | grep -E "(sorbet|backbeat-lifecycle)" || echo "No sorbet/backbeat pods found"
159+
echo ""
160+
echo "Failing/Crashing pods in namespace:"
161+
kubectl get pods -n "${NAMESPACE}" --field-selector=status.phase!=Running,status.phase!=Succeeded -o json | \
162+
jq '.items[] | {name: .metadata.name, phase: .status.phase, reason: .status.reason, containers: [.status.containerStatuses[]? | {name: .name, ready: .ready, restartCount: .restartCount, state: .state}]}' || echo "No failing pods"
163+
echo ""
164+
165+
echo "=== DMF VOLUME CHECK (via mock-sorbet pod) ==="
166+
if kubectl get deployment mock-sorbet -n "${NAMESPACE}" >/dev/null 2>&1; then
167+
SORBET_POD=$(kubectl get pods -n "${NAMESPACE}" -l "sorbetd-name=mock-sorbet" -o jsonpath='{.items[0].metadata.name}')
168+
if [ -n "${SORBET_POD}" ]; then
169+
echo "Sorbet pod: ${SORBET_POD}"
170+
echo "Files in /data:"
171+
kubectl exec -n "${NAMESPACE}" "${SORBET_POD}" -- find /data -type f 2>/dev/null | head -20 || echo "ERROR: Cannot access sorbet pod"
172+
echo "File count in /data:"
173+
kubectl exec -n "${NAMESPACE}" "${SORBET_POD}" -- sh -c 'find /data -type f | wc -l' 2>/dev/null || echo "ERROR"
174+
else
175+
echo "ERROR: No mock-sorbet pod found"
176+
fi
177+
else
178+
echo "mock-sorbet deployment not found"
179+
fi
180+
echo ""
181+
182+
echo "=== HOST VOLUME CHECK (via hostPath) ==="
183+
echo "Attempting to check /data/sorbet-data-0 via ephemeral pod..."
184+
kubectl run diagnostic-volume-check \
185+
--image=alpine:3.22 \
186+
--restart=Never \
187+
--rm \
188+
--attach=True \
189+
--pod-running-timeout=1m \
190+
--namespace="${NAMESPACE}" \
191+
--overrides='
192+
{
193+
"spec": {
194+
"containers": [{
195+
"name": "diagnostic",
196+
"image": "alpine:3.22",
197+
"command": ["/bin/sh", "-c"],
198+
"args": ["echo Files in /cold-data: && find /cold-data -type f | wc -l && find /cold-data -type f | head -10"],
199+
"volumeMounts": [{
200+
"name": "volume",
201+
"mountPath": "/cold-data"
202+
}]
203+
}],
204+
"volumes": [{
205+
"name": "volume",
206+
"hostPath": {
207+
"path": "/data/sorbet-data-0",
208+
"type": "DirectoryOrCreate"
209+
}
210+
}]
211+
}
212+
}' 2>&1 || echo "Failed to check host volume"
213+
echo ""
214+
215+
echo "=== PVC STATUS ==="
216+
kubectl get pvc -n "${NAMESPACE}" -o json | jq '.items[] | {name: .metadata.name, status: .status.phase, capacity: .status.capacity.storage, storageClass: .spec.storageClassName}' || echo "No PVCs"
217+
echo ""
218+
219+
echo "=== PV BACKING sorbet-data ==="
220+
kubectl get pv -o json | jq '.items[] | select(.spec.claimRef.name == "sorbet-data") | {name: .metadata.name, hostPath: .spec.hostPath, capacity: .spec.capacity.storage, accessModes: .spec.accessModes}' || echo "No PV for sorbet-data"
221+
echo ""
222+
223+
echo "=== MONGO DB CONFIG ==="
224+
kubectl get zenko "${ZENKO_NAME}" -n "${NAMESPACE}" -o json | jq '.spec.mongodb' || echo "No mongodb config"
225+
echo ""
226+
227+
echo "=== SERVICE ENDPOINTS READY CHECK ==="
228+
for svc in "${ZENKO_NAME}-connector-s3api" "${ZENKO_NAME}-connector-vault-auth-api" "${ZENKO_NAME}-internal-s3api" "mock-sorbet"; do
229+
echo "--- ${svc} ---"
230+
kubectl get endpoints "${svc}" -n "${NAMESPACE}" -o json 2>/dev/null | jq '{addresses: [.subsets[].addresses[].ip]}' || echo "Service not found"
231+
done
232+
echo ""
233+
234+
echo "=== KAFKA CONSUMER GROUPS ==="
235+
if [ -n "${INSTANCE_ID}" ]; then
236+
KAFKA_POD=$(kubectl get pods -n "${NAMESPACE}" -l "brokerId=0,app=kafka" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
237+
if [ -n "${KAFKA_POD}" ]; then
238+
echo "Kafka pod: ${KAFKA_POD}"
239+
echo "Consumer groups for this instance:"
240+
kubectl exec -n "${NAMESPACE}" "${KAFKA_POD}" -- bash -lc "export KAFKA_OPTS='' && kafka-consumer-groups.sh --bootstrap-server localhost:9092 --list" 2>/dev/null | grep "${INSTANCE_ID}" || echo "No consumer groups found"
241+
else
242+
echo "Kafka pod not found"
243+
fi
244+
fi
245+
echo ""
246+
247+
echo "=== BACKBEAT EXTENSIONS (from config) ==="
248+
kubectl get secret -l "app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=${ZENKO_NAME}" -n "${NAMESPACE}" -o json | \
249+
jq '.items[0].data."config.json"' | sed 's/"//g' | base64 -d | jq '.extensions | keys' || echo "ERROR: Failed to get extensions"
250+
echo ""
251+
252+
echo "=== SECRET RESOURCE VERSIONS ==="
253+
echo "connector-cloudserver-config:"
254+
kubectl get secret -l "app.kubernetes.io/name=connector-cloudserver-config,app.kubernetes.io/instance=${ZENKO_NAME}" -n "${NAMESPACE}" -o json | \
255+
jq '.items[0] | {name: .metadata.name, resourceVersion: .metadata.resourceVersion, creationTimestamp: .metadata.creationTimestamp}'
256+
echo "backbeat-config:"
257+
kubectl get secret -l "app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=${ZENKO_NAME}" -n "${NAMESPACE}" -o json | \
258+
jq '.items[0] | {name: .metadata.name, resourceVersion: .metadata.resourceVersion, creationTimestamp: .metadata.creationTimestamp}'
259+
echo ""
260+
261+
echo "=== RECENT POD LOGS (last 30 lines each) ==="
262+
echo "--- Sorbet Forwarder for e2e-cold (if exists) ---"
263+
SORBET_FWD_POD=$(kubectl get pods -n "${NAMESPACE}" -l "sorbetd-location=e2e-cold,app.kubernetes.io/instance=${ZENKO_NAME}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
264+
if [ -n "${SORBET_FWD_POD}" ]; then
265+
kubectl logs -n "${NAMESPACE}" "${SORBET_FWD_POD}" --tail=30 2>/dev/null || echo "Cannot get logs"
266+
else
267+
echo "No sorbet-forwarder pod for e2e-cold"
268+
fi
269+
echo ""
270+
271+
echo "--- Mock Sorbet ---"
272+
MOCK_SORBET_POD=$(kubectl get pods -n "${NAMESPACE}" -l "sorbetd-name=mock-sorbet" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
273+
if [ -n "${MOCK_SORBET_POD}" ]; then
274+
kubectl logs -n "${NAMESPACE}" "${MOCK_SORBET_POD}" --tail=30 2>/dev/null || echo "Cannot get logs"
275+
else
276+
echo "No mock-sorbet pod"
277+
fi
278+
echo ""
279+
280+
echo "--- Backbeat Lifecycle Transition Processor ---"
281+
LTP_POD=$(kubectl get pods -n "${NAMESPACE}" -l "app.kubernetes.io/name=backbeat-lifecycle-transition-processor,app.kubernetes.io/instance=${ZENKO_NAME}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
282+
if [ -n "${LTP_POD}" ]; then
283+
kubectl logs -n "${NAMESPACE}" "${LTP_POD}" --tail=30 2>/dev/null || echo "Cannot get logs"
284+
else
285+
echo "No lifecycle transition processor pod"
286+
fi
287+
echo ""
288+
289+
echo "=== CONFIGMAPS ==="
290+
echo "Sorbet-related ConfigMaps:"
291+
kubectl get configmaps -n "${NAMESPACE}" -l "app.kubernetes.io/instance=${ZENKO_NAME}" -o json | \
292+
jq '.items[] | select(.metadata.name | contains("sorbet")) | {name: .metadata.name, keys: (.data | keys)}' || echo "No sorbet configmaps"
293+
echo ""
294+
295+
echo "=== RECENT EVENTS (last 50) ==="
296+
kubectl get events -n "${NAMESPACE}" --sort-by='.lastTimestamp' | tail -50 || echo "No events"
297+
echo ""
298+
299+
echo "=================================================="
300+
echo "END OF DUMP"
301+
echo "=================================================="

‎.github/workflows/end2end.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -490,6 +490,19 @@ jobs:
490490
- name: Configure E2E CTST test environment
491491
run: bash configure-e2e-ctst.sh
492492
working-directory: ./.github/scripts/end2end
493+
- name: Dump setup state for PRIMARY instance (OLD)
494+
run: bash dump-setup-state.sh default end2end > /tmp/old-primary-dump.txt 2>&1
495+
working-directory: ./.github/scripts/end2end
496+
continue-on-error: true
497+
- name: Dump setup state for PRA instance (OLD)
498+
run: bash dump-setup-state.sh default end2end-pra > /tmp/old-pra-dump.txt 2>&1
499+
working-directory: ./.github/scripts/end2end
500+
continue-on-error: true
501+
- name: Show diagnostic dumps (OLD)
502+
run: |
503+
echo "=== PRIMARY INSTANCE DUMP ===" && cat /tmp/old-primary-dump.txt
504+
echo "=== PRA INSTANCE DUMP ===" && cat /tmp/old-pra-dump.txt
505+
continue-on-error: true
493506
- name: Run CTST end to end tests
494507
run: bash run-e2e-ctst.sh "" "" "" "" --tags @PRA
495508
working-directory: ./.github/scripts/end2end
@@ -662,6 +675,13 @@ jobs:
662675
- name: Configure E2E test environment
663676
run: bash configure-e2e-ctst.sh
664677
working-directory: ./.github/scripts/end2end
678+
- name: Dump setup state before tests (OLD)
679+
run: bash dump-setup-state.sh default end2end > /tmp/old-ctst-sharded-dump.txt 2>&1
680+
working-directory: ./.github/scripts/end2end
681+
continue-on-error: true
682+
- name: Show diagnostic dump (OLD)
683+
run: cat /tmp/old-ctst-sharded-dump.txt
684+
continue-on-error: true
665685
- name: Run CTST end to end tests
666686
run: bash run-e2e-ctst.sh "" "" "" "" --tags 'not @PRA'
667687
working-directory: ./.github/scripts/end2end

‎tests/ctst/features/0-diagnostics.feature‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
@Diagnostics
2+
Feature: Diagnostic dumps for debugging setup issues
3+
4+
@2.6.0
5+
@PreMerge
6+
@DiagnosticDump
7+
Scenario: Dump complete CTST world state
8+
Given DIAGNOSTIC: dump complete world state
9+

‎tests/ctst/features/CountItems/CountItems.feature‎

Lines changed: 0 additions & 13 deletions
This file was deleted.

0 commit comments

Comments
 (0)