feat: sandbox chromium using bubblewrap

Author: nadiamoe

Dockerfile

@@ -11,31 +11,15 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
   --mount=type=cache,target=/root/go/pkg \
   CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /usr/local/bin/crocochrome ./cmd
 
-# For setting caps, use the same image than the final layer is using to avoid pulling two distinct ones.
-FROM ghcr.io/grafana/chromium-swiftshader-alpine:142.0.7444.59-r0-3.22.2@sha256:4bfff84902c23158c54dbcf94ec8267624ee30700e8c642cba9b7ebbdc756785 AS setcapper
-
-RUN apk --no-cache add libcap
-
-COPY --from=buildtools /usr/local/bin/crocochrome /usr/local/bin/crocochrome
-
-# The following capabilities are used by sm-k6-runner to sandbox the k6 binary. More details about what each cap is used
-# for can be found in /sandbox/sandbox.go.
-# WARNING: The container MUST be also granted all of the following capabilities too, or the CRI will refuse to start it.
-RUN setcap cap_setuid,cap_setgid,cap_kill,cap_chown,cap_dac_override,cap_fowner+ep /usr/local/bin/crocochrome
-
 FROM ghcr.io/grafana/chromium-swiftshader-alpine:142.0.7444.59-r0-3.22.2@sha256:4bfff84902c23158c54dbcf94ec8267624ee30700e8c642cba9b7ebbdc756785
 
-RUN adduser --home / --uid 6666 --shell /bin/nologin --disabled-password k6
-
-RUN apk --no-cache add --repository community tini
-
-# As we rely on file capabilities, we cannot set `allowPrivilegeEscalation: false` in k8s. As a workaround, and to lower
-# potential attack surface, we get rid of any file that has the setuid bit set, such as
-# /usr/lib/chromium/chrome-sandbox.
-RUN find / -type f -perm -4000 -delete
+RUN <<EOF
+  adduser --home / --uid 6666 --shell /bin/nologin --disabled-password k6
+  apk --no-cache add --repository community tini bubblewrap
+EOF
 
 # The crocochrome binary has extra capabilities, so we make sure only the k6 user (and not nobody) can run it.
-COPY --from=setcapper --chown=k6:k6 --chmod=0500 /usr/local/bin/crocochrome /usr/local/bin/crocochrome
+COPY --from=buildtools --chown=k6:k6 --chmod=0500 /usr/local/bin/crocochrome /usr/local/bin/crocochrome
 
 USER k6
 

crocochrome.go

@@ -7,11 +7,11 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"io/fs"
 	"log/slog"
 	"net"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -51,6 +51,8 @@ type Options struct {
 	// ChromiumPort is the port where chromium will be instructed to listen.
 	// Defaults to 5222.
 	ChromiumPort string
+	// DisableBubblewrap runs chromium directly, instead of using the bubblewrap sandbox (`bwrap`).
+	DisableBubblewrap bool
 	// Maximum time a browser is allowed to be running, after which it will be killed unconditionally.
 	// Defaults to 5m.
 	SessionTimeout time.Duration
@@ -254,20 +256,29 @@ func (s *Supervisor) launch(ctx context.Context, sessionID string) error {
 	stdout := &bytes.Buffer{}
 	stderr := &bytes.Buffer{}
 
-	tmpDir, err := s.mkdirTemp()
-	if err != nil {
-		return fmt.Errorf("creating temporary directory: %w", err)
+	bwrapArgs := []string{
+		"--die-with-parent", // Ensures child process (COMMAND) dies when bwrap's parent dies.
+		"--unshare-all",
+		"--share-net",
+		"--proc", "/proc",
+		"--dev", "/dev",
+		"--ro-bind", "/etc", "/etc",
+		"--ro-bind", "/usr", "/usr",
+		"--ro-bind", "/lib", "/lib", // TODO: Remove after Alpine's /usr merge.
+		"--ro-bind", "/bin", "/bin", // TODO: Remove after Alpine's /usr merge.
+		"--dir", "/tmp",
+		"--clearenv",
+		"--setenv", "TMPDIR", "/tmp",
 	}
 
-	defer func() {
-		// Clean up files after chromium exits.
-		err := os.RemoveAll(tmpDir)
-		if err != nil {
-			panic(fmt.Errorf("deleting tmpdir, bug or sandbox compromised: %w", err))
-		}
-	}()
+	if s.opts.UserGroup != 0 {
+		bwrapArgs = append(bwrapArgs,
+			"--uid", strconv.Itoa(s.opts.UserGroup),
+			"--gid", strconv.Itoa(s.opts.UserGroup),
+		)
+	}
 
-	args := []string{
+	chromiumArgs := []string{
 		// The following flags have been tested to be required:
 		"--headless",
 		"--remote-debugging-address=0.0.0.0",
@@ -291,42 +302,38 @@ func (s *Supervisor) launch(ctx context.Context, sessionID string) error {
 	}
 
 	if s.userAgent != "" {
-		args = append(
-			args,
+		chromiumArgs = append(
+			chromiumArgs,
 			"--user-agent="+s.userAgent,
 		)
 	}
 
-	cmd := exec.CommandContext(ctx,
-		s.opts.ChromiumPath,
-		args...,
-	)
-	cmd.Env = []string{
-		// Chromium uses this env var to figure where the temporary directory is. We want that to be the directory
-		// we created for this session, because /tmp is read-only in production.
-		// https://github.com/chromium/chromium/blob/7c4f56ca9dba3a884212ef3a71c8db5d3633f0a6/base/files/file_util_posix.cc#L764
-		"TMPDIR=" + tmpDir,
+	var cmd *exec.Cmd
+	if s.opts.DisableBubblewrap {
+		logger.Warn("bubblewrap is disabled, chromium will run with highly elevated permissions")
+
+		cmd = exec.CommandContext(ctx,
+			s.opts.ChromiumPath,
+			chromiumArgs...,
+		)
+	} else {
+		cmd = exec.CommandContext(ctx,
+			"bwrap",
+			append(append(bwrapArgs, s.opts.ChromiumPath), chromiumArgs...)...,
+		)
 	}
+
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
-	if s.opts.UserGroup != 0 {
-		cmd.SysProcAttr = &syscall.SysProcAttr{
-			Credential: &syscall.Credential{
-				Uid: uint32(s.opts.UserGroup),
-				Gid: uint32(s.opts.UserGroup),
-			},
-		}
-	}
 
 	created := time.Now()
 	defer func() {
 		s.metrics.SessionDuration.Observe(time.Since(created).Seconds())
 	}()
 
-	err = cmd.Run()
-
 	attrs := make([]slog.Attr, 0, 9)
 
+	err := cmd.Run()
 	if err != nil {
 		attrs = append(attrs, slog.Attr{Key: "err", Value: slog.AnyValue(err)})
 	}
@@ -382,37 +389,6 @@ func (s *Supervisor) launch(ctx context.Context, sessionID string) error {
 	return nil
 }
 
-func (s *Supervisor) mkdirTemp() (string, error) {
-	_, err := os.Stat(s.opts.TempDir)
-	if errors.Is(err, fs.ErrNotExist) {
-		s.logger.Warn(
-			"Specified TempDir does not exist, is it mounted? Falling back to creating it.",
-			"TempDir", s.opts.TempDir,
-		)
-		err = os.MkdirAll(s.opts.TempDir, 0o755) // 700 would not allow other users to descend into subdirectories.
-		if err != nil {
-			return "", fmt.Errorf("tmpdir does not exist and couldn't be created: %w", err)
-		}
-	}
-
-	tmpDir, err := os.MkdirTemp(s.opts.TempDir, "")
-	if err != nil {
-		return "", err
-	}
-
-	if s.opts.UserGroup == 0 {
-		// No chowning necessary.
-		return tmpDir, nil
-	}
-
-	err = os.Chown(tmpDir, s.opts.UserGroup, s.opts.UserGroup)
-	if err != nil {
-		return "", fmt.Errorf("chowning temporary dir: %w", err)
-	}
-
-	return tmpDir, nil
-}
-
 // ComputeUserAgent runs chromium once, retrieves its default user agent, and stores a patched version so it can be used
 // in all subsequent calls.
 func (s *Supervisor) ComputeUserAgent(ctx context.Context) error {

crocochrome_test.go

@@ -22,7 +22,11 @@ func TestCrocochrome(t *testing.T) {
 
 		hb := testutil.NewHeartbeat(t)
 		port := testutil.HTTPInfo(t, testutil.ChromiumVersionHandler)
-		cc := crocochrome.New(logger, crocochrome.Options{ChromiumPath: hb.Path, ChromiumPort: port})
+		cc := crocochrome.New(logger, crocochrome.Options{
+			ChromiumPath:      hb.Path,
+			ChromiumPort:      port,
+			DisableBubblewrap: true,
+		})
 
 		session, err := cc.Create()
 		if err != nil {
@@ -62,7 +66,11 @@ func TestCrocochrome(t *testing.T) {
 
 			hb := testutil.NewHeartbeat(t)
 			port := testutil.HTTPInfo(t, testutil.InternalServerErrorHandler)
-			cc := crocochrome.New(logger, crocochrome.Options{ChromiumPath: hb.Path, ChromiumPort: port})
+			cc := crocochrome.New(logger, crocochrome.Options{
+				ChromiumPath:      hb.Path,
+				ChromiumPort:      port,
+				DisableBubblewrap: true,
+			})
 
 			_, err := cc.Create()
 			if err == nil {
@@ -76,7 +84,11 @@ func TestCrocochrome(t *testing.T) {
 			t.Parallel()
 
 			hb := testutil.NewHeartbeat(t)
-			cc := crocochrome.New(logger, crocochrome.Options{ChromiumPath: hb.Path, ChromiumPort: "0"})
+			cc := crocochrome.New(logger, crocochrome.Options{
+				ChromiumPath:      hb.Path,
+				ChromiumPort:      "0",
+				DisableBubblewrap: true,
+			})
 
 			_, err := cc.Create()
 			if err == nil {
@@ -92,7 +104,11 @@ func TestCrocochrome(t *testing.T) {
 
 		hb := testutil.NewHeartbeat(t)
 		port := testutil.HTTPInfo(t, testutil.ChromiumVersionHandler)
-		cc := crocochrome.New(logger, crocochrome.Options{ChromiumPath: hb.Path, ChromiumPort: port})
+		cc := crocochrome.New(logger, crocochrome.Options{
+			ChromiumPath:      hb.Path,
+			ChromiumPort:      port,
+			DisableBubblewrap: true,
+		})
 
 		sess, err := cc.Create()
 		if err != nil {
@@ -115,7 +131,11 @@ func TestCrocochrome(t *testing.T) {
 
 		hb := testutil.NewHeartbeat(t)
 		port := testutil.HTTPInfo(t, testutil.ChromiumVersionHandler)
-		cc := crocochrome.New(logger, crocochrome.Options{ChromiumPath: hb.Path, ChromiumPort: port})
+		cc := crocochrome.New(logger, crocochrome.Options{
+			ChromiumPath:      hb.Path,
+			ChromiumPort:      port,
+			DisableBubblewrap: true,
+		})
 
 		sess1, err := cc.Create()
 		if err != nil {
@@ -143,7 +163,12 @@ func TestCrocochrome(t *testing.T) {
 
 		hb := testutil.NewHeartbeat(t)
 		port := testutil.HTTPInfo(t, testutil.ChromiumVersionHandler)
-		cc := crocochrome.New(logger, crocochrome.Options{ChromiumPath: hb.Path, ChromiumPort: port, SessionTimeout: 3 * time.Second})
+		cc := crocochrome.New(logger, crocochrome.Options{
+			ChromiumPath:      hb.Path,
+			ChromiumPort:      port,
+			DisableBubblewrap: true,
+			SessionTimeout:    3 * time.Second,
+		})
 
 		_, err := cc.Create()
 		if err != nil {

integration/integration_test.go

@@ -54,9 +54,7 @@ func TestIntegration(t *testing.T) {
 			ExposedPorts: []string{"8080/tcp"},
 			WaitingFor:   wait.ForExposedPort(),
 			Networks:     []string{network.Name},
-			// Since https://github.com/grafana/crocochrome/pull/12, crocochrome requires /chromium-tmp to exist
-			// and be writable.
-			Mounts: testcontainers.Mounts(testcontainers.VolumeMount("chromium-tmp", "/chromium-tmp")),
+			Privileged:   true, // Required for bubblewrap to work.
 		},
 	})
 	testcontainers.CleanupContainer(t, cc)