scx_utils: package a tarball of vmlinux in the source

scx_utils has grown too large to be uploaded to crates.io. This is
because it includes the vmlinux.hs for each architecture, and they're
huge.

Temporarily handle this issue by compressing the vmlinuxes into a
tar.zst. Zstandard is chosen because it compresses well and there's a
pure Rust library, meaning we don't need to add any build time/runtime
dependencies (other than Rust). The new scx_utils is 1.8MiB, well below
the 10MiB limit for crates.io.

Test plan:
- CI for `cargo build` and Makefiles
- Checked the CI script picks up changes with
  `nix run ".github/include#ci" test`

Author: JakeHillion

.github/include/ci.py

@@ -261,6 +261,24 @@ async def run_clippy():
 
 async def run_tests():
     """Run the test suite."""
+    print("Checking that vmlinux.tar.zst is in sync...", flush=True)
+
+    with tempfile.TemporaryDirectory() as tempdir:
+        await run_command(
+            ["tar", "-xf", "rust/scx_utils/vmlinux.tar.zst", "-C", tempdir],
+            no_capture=True,
+        )
+        await run_command(
+            [
+                "diff",
+                "--no-dereference",
+                "-qr",
+                "scheds/vmlinux/",
+                f"{tempdir}/vmlinux/",
+            ],
+            no_capture=True,
+        )
+
     print("Running tests...", flush=True)
 
     # Make sure the selftest is built in case the build was not already run.

.github/include/flake.nix

@@ -175,11 +175,13 @@
                 bpftools
                 clang
                 coreutils
+                diffutils
                 gcc
                 git
                 gnugrep
                 gnumake
                 gnused
+                gnutar
                 isort
                 jq
                 libbpf-git

Cargo.lock

@@ -60,12 +60,13 @@ export BPF_CFLAGS := -g -O2 -Wall -Wno-compare-distinct-pointer-types \
 # ROOT_SRC_DIR is set by the top-level make call for out-of-source builds
 export ROOT_SRC_DIR ?= $(CURDIR)
 export BPF_INCLUDES := -I$(ROOT_SRC_DIR)/scheds/include \
-	-I$(ROOT_SRC_DIR)/scheds/include/arch/$(TARGET_ARCH) \
 	-I$(ROOT_SRC_DIR)/scheds/include/bpf-compat \
 	-I$(ROOT_SRC_DIR)/scheds/include/lib \
+	-I$(ROOT_SRC_DIR)/scheds/vmlinux \
+	-I$(ROOT_SRC_DIR)/scheds/vmlinux/arch/$(TARGET_ARCH) \
 	$(LIBBPF_CFLAGS)
 
-export CFLAGS := -std=gnu11 -I$(ROOT_SRC_DIR)/scheds/include -I$(SCHED_OBJ_DIR) $(LIBBPF_CFLAGS)
+export CFLAGS := -std=gnu11 -I$(ROOT_SRC_DIR)/scheds/include -I$(ROOT_SRC_DIR)/scheds/vmlinux -I$(SCHED_OBJ_DIR) $(LIBBPF_CFLAGS)
 
 export LIBBPF_DEPS := $(LIBBPF_LIBS) -lelf -lz -lzstd
 export THREAD_DEPS := -lpthread

Makefile

@@ -277,12 +277,13 @@ endif
 # BPF compilation uses the gen_bpf_o generator. The following should be
 # passed in as extra_args.
 bpf_includes = ['-I', join_paths(meson.current_source_dir(), 'scheds/include'),
-                '-I', join_paths(meson.current_source_dir(), 'scheds/include/arch/' + arch_dict[cpu]),
                 '-I', join_paths(meson.current_source_dir(), 'scheds/include/bpf-compat'),
-                '-I', join_paths(meson.current_source_dir(), 'scheds/include/lib'),]
+                '-I', join_paths(meson.current_source_dir(), 'scheds/include/lib'),
+                '-I', join_paths(meson.current_source_dir(), 'scheds/vmlinux'),
+                '-I', join_paths(meson.current_source_dir(), 'scheds/vmlinux/arch/' + arch_dict[cpu]),]
 
 user_c_dep = [declare_dependency(
-  include_directories: include_directories('scheds/include'),
+  include_directories: [include_directories('scheds/include'), include_directories('scheds/vmlinux')],
 )]
 
 lib_objs = []

meson.build

@@ -23,7 +23,8 @@ fn main() {
         root_dir.join("lib/scxtest/"),
         root_dir.join("scheds/include/"),
         root_dir.join("scheds/include/lib"),
-        root_dir.join("scheds/include/arch/x86/"),
+        root_dir.join("scheds/vmlinux/"),
+        root_dir.join("scheds/vmlinux/arch/x86/"),
         root_dir.join("scheds/include/bpf-compat/"),
         env::var("DEP_BPF_INCLUDE")
             .expect("libbpf-sys include must be avaiable")

rust/scx_bpf_unittests/build.rs

@@ -22,6 +22,7 @@ nvml-wrapper = { version = "0.11.0", optional = true }
 nvml-wrapper-sys = { version = "0.9.0", optional = true }
 paste = "1.0"
 regex = "1.11.1"
+ruzstd = "0.8.1"
 scx_stats = { path = "../scx_stats", version = "1.0.15" }
 serde = { version = "1.0.215", features = ["derive"] }
 sscanf = "0.4"
@@ -31,22 +32,22 @@ version-compare = "0.1"
 zbus = { version = "5.3.1", optional = true }
 const_format = "0.2.34"
 num = "0.4.3"
+tempfile = "3.19.1"
 tracing = "0.1"
 tracing-subscriber = "0.3"
 libc = "0.2.175"
 nix = { version = "0.30.1", features = ["resource"] }
 
-[dev-dependencies]
-tempfile = "3.19.1"
-
 [build-dependencies]
 anyhow = "1.0.65"
 bindgen = ">=0.69"
 glob = "0.3.2"
 lazy_static = "1.5.0"
 libbpf-cargo = "=0.26.0-beta.1"
+ruzstd = "0.8.1"
 sscanf = "0.4"
 tar = "0.4"
+tempfile = "3.19.1"
 vergen = { version = "8.0.0", features = ["cargo", "git", "gitcl"] }
 version-compare = "0.1"
 walkdir = "2.5"

rust/scx_utils/Cargo.toml

@@ -5,7 +5,8 @@
 
 use std::{
     fs::File,
-    path::{Path, PathBuf},
+    io::{Read, Seek},
+    path::PathBuf,
 };
 
 include!("clang_info.rs");
@@ -26,6 +27,16 @@ impl Builder {
 
         ar.follow_symlinks(false);
         ar.append_dir_all(".", BPF_H).unwrap();
+
+        let vmlinux_dir = tempfile::tempdir().unwrap();
+        let mut vmlinux_tar_zst = File::open("vmlinux.tar.zst").unwrap();
+        let vmlinux_tar = ruzstd::decoding::StreamingDecoder::new(&mut vmlinux_tar_zst).unwrap();
+        tar::Archive::new(vmlinux_tar)
+            .unpack(vmlinux_dir.path())
+            .unwrap();
+        ar.append_dir_all(".", vmlinux_dir.path().join("vmlinux"))
+            .unwrap();
+
         ar.finish().unwrap();
 
         for ent in walkdir::WalkDir::new(BPF_H) {
@@ -40,16 +51,43 @@ impl Builder {
         let out_dir = env::var("OUT_DIR").unwrap();
         let clang = ClangInfo::new().unwrap();
         let kernel_target = clang.kernel_target().unwrap();
-        let vmlinux_h = Path::new(&BPF_H)
-            .join("arch")
-            .join(kernel_target)
-            .join("vmlinux.h")
-            .to_str()
+
+        let mut vmlinux_tar_zst = File::open("vmlinux.tar.zst").unwrap();
+
+        let mut vmlinux_h = String::new();
+
+        // vmlinux.h is a symlink. dereference it here.
+        let search: PathBuf = format!("vmlinux/arch/{kernel_target}/vmlinux.h").into();
+
+        let mut vmlinux_tar =
+            ruzstd::decoding::StreamingDecoder::new(&mut vmlinux_tar_zst).unwrap();
+        let mut archive = tar::Archive::new(&mut vmlinux_tar);
+        let vmlinux_link_entry = archive
+            .entries()
+            .unwrap()
+            .find(|x| x.as_ref().unwrap().path().unwrap() == search.as_path())
+            .unwrap()
+            .unwrap();
+
+        let vmlinux_path = PathBuf::from(vmlinux_link_entry.path().unwrap())
+            .parent()
+            .unwrap()
+            .join(vmlinux_link_entry.link_name().unwrap().unwrap());
+
+        vmlinux_tar_zst.rewind().unwrap();
+        let vmlinux_tar = ruzstd::decoding::StreamingDecoder::new(&mut vmlinux_tar_zst).unwrap();
+
+        tar::Archive::new(vmlinux_tar)
+            .entries()
+            .unwrap()
+            .find(|x| x.as_ref().unwrap().path().unwrap() == vmlinux_path.as_path())
+            .unwrap()
             .unwrap()
-            .to_string();
+            .read_to_string(&mut vmlinux_h)
+            .unwrap();
 
         let bindings = bindgen::Builder::default()
-            .header(vmlinux_h)
+            .header_contents(&search.to_string_lossy(), &vmlinux_h)
             .allowlist_type("scx_exit_kind")
             .allowlist_type("scx_consts")
             .parse_callbacks(Box::new(bindgen::CargoCallbacks::new()))