Skip to content

Commit bc7733c

Browse files
bigtallcampbellbigtallcampbell
authored
Enhanced testing and remove run as root (#108)
* adding runAsNonRoot and init containers to set permissions * updating to quotes and typo in security context * adding specific user ids * updating to use runAsUserId in service * updating for platformsvcs to run as non-root * adding logging to tests
1 parent dd6a88a commit bc7733c

File tree

9 files changed

+324
-8
lines changed

9 files changed

+324
-8
lines changed

‎chart/templates/_fileserver.tpl‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,20 @@ mountPath: {{ printf "%s/%s/%s" $globalValues.spacefxDirectories.base $volumeNam
140140
{{- end }}
141141
{{- end }}
142142

143+
{{- define "spacefx.fileserver.clientapp.volumemount.mountpath" }}
144+
{{- $serviceValues := .serviceValues }}
145+
{{- $globalValues := .globalValues }}
146+
{{- $volumeName := .volumeName }}
147+
{{- $shareName := printf "%s-%s" $serviceValues.appName $volumeName }}
148+
{{- $mountPath := printf "%s/%s/%s" $globalValues.spacefxDirectories.base $volumeName $serviceValues.appName }}
149+
{{- if and (eq $serviceValues.appName "hostsvc-link") (eq $volumeName "allxfer") }}
150+
{{ printf "%s/%s" $globalValues.spacefxDirectories.base $volumeName }}
151+
{{- else }}
152+
{{ printf "%s/%s/%s" $globalValues.spacefxDirectories.base $volumeName $serviceValues.appName }}
153+
{{- end }}
154+
{{- end }}
155+
156+
143157
{{- define "spacefx.fileserver.clientapp.volume" }}
144158
{{- $serviceValues := .serviceValues }}
145159
{{- $volumeName := .volumeName }}

‎chart/templates/_initcontainers.tpl‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{{- define "spacefx.initcontainers.setperms" }}
2+
{{- $serviceValues := .serviceValues }}
3+
{{- $globalValues := .globalValues }}
4+
{{- $mountPaths := "" }}
5+
initContainers:
6+
- name: init-permissions
7+
image: docker.io/rancher/mirrored-library-busybox:1.36.1
8+
volumeMounts:
9+
{{- range $volumeKey, $volumeName := $globalValues.xferVolumes }}
10+
{{- $fileServerVolumeMount := printf "%s" (include "spacefx.fileserver.clientapp.volumemount" (dict "globalValues" $globalValues "serviceValues" $serviceValues "volumeName" $volumeName) | nindent 2 | trim) }}
11+
{{- $fileServerVolumeMountPath := printf "%s" (include "spacefx.fileserver.clientapp.volumemount.mountpath" (dict "globalValues" $globalValues "serviceValues" $serviceValues "volumeName" $volumeName) | trim) }}
12+
{{- $mountPaths = printf "%s %s" $mountPaths $fileServerVolumeMountPath }}
13+
{{- printf "- %s" $fileServerVolumeMount | nindent 5 }}
14+
{{- end }}
15+
{{- if $serviceValues.xferVolumes }}
16+
{{- range $volumeKey, $volumeName := $serviceValues.xferVolumes }}
17+
{{- $fileServerVolumeMount := printf "%s" (include "spacefx.fileserver.clientapp.volumemount" (dict "globalValues" $globalValues "serviceValues" $serviceValues "volumeName" $volumeName) | nindent 2 | trim) }}
18+
{{- $fileServerVolumeMountPath := printf "%s" (include "spacefx.fileserver.clientapp.volumemount.mountpath" (dict "globalValues" $globalValues "serviceValues" $serviceValues "volumeName" $volumeName) | trim) }}
19+
{{- $mountPaths = printf "%s %s" $mountPaths $fileServerVolumeMountPath }}
20+
{{- printf "- %s" $fileServerVolumeMount | nindent 5 }}
21+
{{- end }}
22+
{{- end }}
23+
command: ["sh", "-c", "chown -R {{ $serviceValues.runAsUserId }}:{{ $serviceValues.runAsUserId }} {{ $mountPaths }}"]
24+
{{- end }}
25+

‎chart/templates/_service.tpl‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,18 @@ spec:
4444
{{- if eq $serviceValues.appName "platform-mts" }}
4545
dnsPolicy: ClusterFirstWithHostNet
4646
hostNetwork: true
47+
{{- end }}
48+
{{- if and ($globalValues.security.forceNonRoot) (ne $serviceValues.appName "hostsvc-link") }}
49+
{{- include "spacefx.initcontainers.setperms" (dict "globalValues" $globalValues "serviceValues" $serviceValues) | indent 6 }}
4750
{{- end }}
4851
containers:
4952
- name: {{ $serviceValues.appName | quote }}
53+
{{- if and ($globalValues.security.forceNonRoot) (ne $serviceValues.appName "hostsvc-link") }}
54+
securityContext:
55+
runAsUser: {{ $serviceValues.runAsUserId }}
56+
runAsGroup: {{ $serviceValues.runAsUserId }}
57+
runAsNonRoot: true
58+
{{- end }}
5059
ports:
5160
- name: app-port
5261
containerPort: 50051
@@ -70,12 +79,6 @@ spec:
7079
timeoutSeconds: {{ $globalValues.probes.startup.timeoutSeconds }}
7180
{{- end }}
7281
{{- end }}
73-
{{- if $serviceValues.securityContext }}
74-
securityContext:
75-
{{- range $index, $securityContext := $serviceValues.securityContext }}
76-
{{ $securityContext.name }}: {{ $securityContext.value }}
77-
{{- end }}
78-
{{- end }}
7982
{{- if $serviceValues.debugShim }}
8083
image: "{{ $serviceValues.repository }}:latest"
8184
imagePullPolicy: "Never"

‎chart/values.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ global:
113113
security:
114114
payloadAppNetworkRestrictionsEnabled: true
115115
topicRestrictionEnabled: true
116+
forceNonRoot: true
116117
subcharts:
117118
dapr:
118119
enabled: false
@@ -134,6 +135,7 @@ services:
134135
workingDirectory: /workspaces/exampleapp
135136
repository: kaniko-project/executor
136137
tag: v1.20.1-slim
138+
runAsUserId: 701
137139
registry:
138140
appName: coresvc-registry
139141
containerCommand:
@@ -163,6 +165,7 @@ services:
163165
cpu:
164166
limit: 1000m
165167
request: 10m
168+
runAsUserId: 702
166169
switchboard:
167170
appName: coresvc-switchboard
168171
serviceNamespace: coresvc
@@ -192,6 +195,7 @@ services:
192195
enabled: true
193196
hasBase: false
194197
enabled: false
198+
runAsUserId: 703
195199
platform:
196200
mts:
197201
appConfig:
@@ -215,6 +219,7 @@ services:
215219
hasBase: false
216220
enabled: false
217221
workingDir: /workspaces/platform-mts
222+
runAsUserId: 704
218223
deployment:
219224
appName: platform-deployment
220225
appHealthChecks: true
@@ -269,6 +274,7 @@ services:
269274
hasBase: false
270275
enabled: false
271276
workingDir: /workspace/platform-deployment
277+
runAsUserId: 705
272278
vth:
273279
appConfig:
274280
- name: enableRoutingToMTS
@@ -291,6 +297,7 @@ services:
291297
hasBase: false
292298
enabled: false
293299
workingDir: /workspaces/vth
300+
runAsUserId: 706
294301
host:
295302
sensor:
296303
appConfig:
@@ -315,6 +322,7 @@ services:
315322
enabled: true
316323
hasBase: false
317324
enabled: false
325+
runAsUserId: 707
318326
workingDir: /workspaces/hostsvc-sensor
319327
link:
320328
appConfig:
@@ -371,6 +379,7 @@ services:
371379
enabled: true
372380
hasBase: false
373381
enabled: false
382+
runAsUserId: 708
374383
workingDir: /workspaces/hostsvc-logging
375384
position:
376385
appName: hostsvc-position
@@ -390,6 +399,7 @@ services:
390399
enabled: true
391400
hasBase: false
392401
enabled: false
402+
runAsUserId: 709
393403
workingDir: /workspaces/hostsvc-position
394404
payloadapp:
395405
payloadappTemplate:

‎scripts/deploy_spacefx.sh‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,11 +86,15 @@ function deploy_spacefx_service_group(){
8686
info_log "Scanning '${service_group}' spacefx services for deploying..."
8787

8888
run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.services.${service_group} | to_entries[] | select(.value.${enabled_filter} == true) | .key' -r" spacefx_services
89+
run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.global.security.forceNonRoot' -r" spacefx_forceNonRoot
90+
8991
run_a_script "kubectl --kubeconfig ${KUBECONFIG} get deployments -A -o json" services_deployed_cache --disable_log
9092

9193
for service in $spacefx_services; do
9294
run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.services.${service_group}.${service}.appName' -r" spacefx_service_appName
9395
run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.services.${service_group}.${service}.serviceNamespace' -r" spacefx_service_serviceNamespace
96+
run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.services.${service_group}.${service}.runAsUserId' -r" spacefx_service_userid
97+
9498

9599
run_a_script "jq -r '.items[] | select(.metadata.name == \"${spacefx_service_appName}\" and (.metadata.namespace == \"${spacefx_service_serviceNamespace}\")) | true' <<< \${services_deployed_cache}" service_deployed
96100

@@ -99,6 +103,78 @@ function deploy_spacefx_service_group(){
99103
continue
100104
fi
101105

106+
# Create users and groups if the service needs one
107+
if [[ "${spacefx_forceNonRoot}" == "true" ]] && [[ "${spacefx_service_userid}" != "null" ]]; then
108+
info_log "...checking if group '${spacefx_service_appName}' (GID: '${spacefx_service_userid}') exists..."
109+
110+
# This will return the group_name for the groupID. i.e. "702"
111+
run_a_script "getent group ${spacefx_service_userid}" preexisting_groupid_by_id --ignore_error
112+
preexisting_groupid_by_id="${preexisting_groupid_by_id%%:*}"
113+
114+
# This will check if a group exists and gets its ID
115+
run_a_script "getent group ${spacefx_service_appName}" preexisting_groupid_by_name --ignore_error
116+
preexisting_groupid_by_name="${preexisting_groupid_by_name%%:*}"
117+
118+
119+
if [[ -n "${preexisting_groupid_by_name}" ]] && [[ "${preexisting_groupid_by_id}" == "${preexisting_groupid_by_name}" ]]; then
120+
info_log "...group '${spacefx_service_appName}' (GID: '${preexisting_groupid_by_id}') already exists. Nothing to do"
121+
else
122+
if [[ -n "${preexisting_groupid_by_id}" ]]; then
123+
info "...GID '${spacefx_service_userid}' already in use, but isn't assigned to '${spacefx_service_appName}'. Attempting to delete..."
124+
run_a_script "getent group ${spacefx_service_userid}" group_to_del
125+
group_to_del="${group_to_del%%:*}"
126+
127+
run_a_script "groupdel -f ${group_to_del}"
128+
info "...successfully deleted previous group '${group_to_del}' (GID: '${username_to_del}')"
129+
fi
130+
131+
if [[ -n "${preexisting_groupid_by_name}" ]]; then
132+
info "...Group '${spacefx_service_appName}' already in use, but isn't assigned to '${spacefx_service_userid}'. Attempting to delete..."
133+
run_a_script "groupdel -f ${spacefx_service_appName}"
134+
info "...successfully deleted previous group '${spacefx_service_appName}'"
135+
fi
136+
137+
info_log "...creating group '${spacefx_service_appName}' with GID '${spacefx_service_userid}'..."
138+
run_a_script "groupadd -r -g ${spacefx_service_userid} ${spacefx_service_appName}" --no_log
139+
info_log "...successfully created group '${spacefx_service_appName}' (GID: '${spacefx_service_userid}')."
140+
fi
141+
142+
143+
info_log "...checking if user '${spacefx_service_appName}' (UID: '${spacefx_service_userid}') exists..."
144+
145+
# This will return a user id if the userid exists. i.e. "701"
146+
run_a_script "id -u ${spacefx_service_userid}" preexisting_userid --ignore_error
147+
148+
# This will return the user id for the username. i.e. "702"
149+
run_a_script "id -u ${spacefx_service_appName}" preexisting_userid_for_username --ignore_error
150+
151+
if [[ -n "${preexisting_userid_for_username}" ]] && [[ "${preexisting_userid}" == "${preexisting_userid_for_username}" ]]; then
152+
info_log "...user '${spacefx_service_appName}' (UID: '${spacefx_service_userid}') already exists. Nothing to do"
153+
else
154+
if [[ -n "${preexisting_userid}" ]]; then
155+
info "...UID '${spacefx_service_userid}' already in use, but isn't assigned to '${spacefx_service_appName}'. Attempting to delete..."
156+
run_a_script "getent passwd ${spacefx_service_userid}" username_to_del
157+
username_to_del="${username_to_del%%:*}"
158+
run_a_script "userdel -f ${username_to_del}"
159+
info "...successfully deleted previous user '${username_to_del}' (UID: '${username_to_del}')"
160+
fi
161+
162+
if [[ -n "${preexisting_userid_for_username}" ]]; then
163+
info "...Username '${spacefx_service_appName}' already in use, but isn't assigned to '${spacefx_service_userid}'. Attempting to delete..."
164+
run_a_script "userdel -f ${spacefx_service_appName}"
165+
info "...successfully deleted previous user '${spacefx_service_appName}' (UID: '${preexisting_userid_for_username}')"
166+
fi
167+
168+
info_log "...creating user '${spacefx_service_appName}' with UID '${spacefx_service_userid}'..."
169+
run_a_script "useradd -r -u ${spacefx_service_userid} -g ${spacefx_service_appName} -d /nonexistent -s /usr/sbin/nologin ${spacefx_service_appName}" --no_log
170+
info_log "...successfully created user '${spacefx_service_appName}' (UID: '${spacefx_service_userid}')."
171+
fi
172+
173+
174+
fi
175+
176+
177+
102178
info_log "...adding '${service}'..."
103179
deploy_group_cmd="${deploy_group_cmd} --set services.${service_group}.${service}.enabled=true \
104180
--set services.${service_group}.${service}.provisionVolumeClaims=true \
@@ -203,6 +279,15 @@ function deploy_apps_to_deployment_service(){
203279
info_log "...successfully copied chart to '${SPACEFX_DIR}/xfer/platform-deployment/tmp/chart/${SPACEFX_VERSION}'"
204280

205281

282+
run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.global.security.forceNonRoot' -r" spacefx_forceNonRoot
283+
284+
if [[ "${spacefx_forceNonRoot}" == "true" ]]; then
285+
info_log "Updating permissions for '${SPACEFX_DIR}/xfer/platform-deployment' to user 'platform-deployment'..."
286+
run_a_script "chown -R platform-deployment:platform-deployment ${SPACEFX_DIR}/xfer/platform-deployment"
287+
run_a_script "chmod -R u+rwx ${SPACEFX_DIR}/xfer/platform-deployment"
288+
info_log "Permissions successfully updated"
289+
fi
290+
206291
info_log "FINISHED: ${FUNCNAME[0]}"
207292
}
208293

‎scripts/stage_spacefx.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,10 @@ function toggle_security_restrictions(){
239239
run_a_script "yq eval '.global.security.topicRestrictionEnabled = false' -i \"${SPACEFX_DIR}/chart/values.yaml\""
240240
info_log "'DEV_ENVIRONMENT' = true. Allowing Links to Platform-Deployment..."
241241
run_a_script "yq eval '(.services.host.link.appConfig[] | select(.name == \"allowLinksToDeploymentSvc\") .value) = true' -i \"${SPACEFX_DIR}/chart/values.yaml\""
242+
243+
info_log "'DEV_ENVIRONMENT' = true. Enabling run as root..."
244+
run_a_script "yq eval '.global.security.forceNonRoot = false' -i \"${SPACEFX_DIR}/chart/values.yaml\""
245+
242246
info_log "...security restrictions configured for development"
243247
else
244248
info_log "'DEV_ENVIRONMENT' = false. Enabling Network Restrictions..."
@@ -251,6 +255,9 @@ function toggle_security_restrictions(){
251255
run_a_script "yq eval '(.services.host.link.appConfig[] | select(.name == \"allowLinksToDeploymentSvc\") .value) = false' -i \"${SPACEFX_DIR}/chart/values.yaml\""
252256
info_log "...Network and Topic restrictions successfully enabled."
253257

258+
info_log "'DEV_ENVIRONMENT' = true. Enabling run as root..."
259+
run_a_script "yq eval '.global.security.forceNonRoot = true' -i \"${SPACEFX_DIR}/chart/values.yaml\""
260+
254261
info_log "...security restrictions configured for production"
255262
fi
256263

‎tests/dev_cluster.sh‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#
88
# "bash ./tests/dev_cluster.sh"
99
set -e
10+
MAX_WAIT_SECS=300
1011
SCRIPT_NAME=$(basename "$0")
1112
WORKING_DIR="$(git rev-parse --show-toplevel)"
1213

@@ -22,6 +23,57 @@ ARTIFACT_PATH=${WORKING_DIR}/output/spacefx-dev/devcontainer-feature-spacefx-dev
2223

2324
echo "Microsoft Azure Orbital Space SDK - Development Cluster Test"
2425

26+
############################################################
27+
# Given a namespace, this function will wait for all pods to enter a running state
28+
############################################################
29+
function wait_for_namespace_to_provision(){
30+
31+
local namespace=""
32+
33+
while [[ "$#" -gt 0 ]]; do
34+
case $1 in
35+
--namespace)
36+
shift
37+
namespace=$1
38+
;;
39+
esac
40+
shift
41+
done
42+
43+
if [[ -z $namespace ]]; then
44+
echo "--namespace not provided to wait_for_namespace_to_provision function"
45+
exit 1
46+
fi
47+
48+
echo "Waiting for namespace '${namespace}' to fully provision (max $MAX_WAIT_SECS seconds)..."
49+
50+
# This returns any pods that are not completed nor succeeded
51+
k3s_deployments_not_ready=$(kubectl get deployments --kubeconfig "${KUBECONFIG}" -n "${namespace}" --output=json | jq '[.items[] | select(.spec.replicas != .status.availableReplicas)] | length')
52+
53+
start_time=$(date +%s)
54+
55+
while [[ $k3s_deployments_not_ready != "0" ]]; do
56+
k3s_deployments_not_ready=$(kubectl get deployments --kubeconfig "${KUBECONFIG}" -n "${namespace}" --output=json | jq '[.items[] | select(.spec.replicas != .status.availableReplicas)] | length')
57+
58+
current_time=$(date +%s)
59+
elapsed_time=$((current_time - start_time))
60+
61+
if [[ $elapsed_time -ge $MAX_WAIT_SECS ]]; then
62+
echo "Timed out waiting for deployment to complete. Check logs for more information"
63+
exit 1
64+
fi
65+
66+
kubectl get pods -A
67+
kubectl get deployments -A
68+
69+
echo "Found incomplete deployments. Rechecking in 5 seconds"
70+
sleep 5
71+
done
72+
73+
echo "Namespace '${namespace}' is provisioned"
74+
75+
}
76+
2577
if [[ -d "/var/spacedev" ]]; then
2678
echo "Preexisting /var/spacedev found. Resetting enviornment with big_red_button.sh"
2779
/var/spacedev/scripts/big_red_button.sh
@@ -94,6 +146,9 @@ if [[ ! -f "${KUBECONFIG}" ]]; then
94146
echo "KUBECONFIG '${KUBECONFIG}' not found. Cluster did not initialize."
95147
exit 1
96148
fi
149+
150+
151+
97152
kubectl --kubeconfig ${KUBECONFIG} get deployment/coresvc-registry -n coresvc
98153
kubectl --kubeconfig ${KUBECONFIG} get deployment/coresvc-switchboard -n coresvc
99154

@@ -107,6 +162,9 @@ kubectl --kubeconfig ${KUBECONFIG} get deployment/platform-deployment -n platfor
107162
kubectl --kubeconfig ${KUBECONFIG} get deployment/platform-mts -n platformsvc
108163
kubectl --kubeconfig ${KUBECONFIG} get deployment/vth -n platformsvc
109164

165+
wait_for_namespace_to_provision --namespace coresvc
166+
wait_for_namespace_to_provision --namespace hostsvc
167+
wait_for_namespace_to_provision --namespace platformsvc
110168

111169
echo ""
112170
echo ""

0 commit comments

Comments
 (0)