Merge branch 'jakehillion/vmlinux-tarball' into version

Author: JakeHillion

.github/include/ci.py

@@ -261,6 +261,24 @@ async def run_clippy():
 
 async def run_tests():
     """Run the test suite."""
+    print("Checking that vmlinux.tar.zst is in sync...", flush=True)
+
+    with tempfile.TemporaryDirectory() as tempdir:
+        await run_command(
+            ["tar", "-xf", "rust/scx_utils/vmlinux.tar.zst", "-C", tempdir],
+            no_capture=True,
+        )
+        await run_command(
+            [
+                "diff",
+                "--no-dereference",
+                "-qr",
+                "scheds/vmlinux/",
+                f"{tempdir}/vmlinux/",
+            ],
+            no_capture=True,
+        )
+
     print("Running tests...", flush=True)
 
     # Make sure the selftest is built in case the build was not already run.

.github/include/flake.nix

@@ -175,11 +175,13 @@
                 bpftools
                 clang
                 coreutils
+                diffutils
                 gcc
                 git
                 gnugrep
                 gnumake
                 gnused
+                gnutar
                 isort
                 jq
                 libbpf-git

Cargo.lock

@@ -60,12 +60,13 @@ export BPF_CFLAGS := -g -O2 -Wall -Wno-compare-distinct-pointer-types \
 # ROOT_SRC_DIR is set by the top-level make call for out-of-source builds
 export ROOT_SRC_DIR ?= $(CURDIR)
 export BPF_INCLUDES := -I$(ROOT_SRC_DIR)/scheds/include \
-	-I$(ROOT_SRC_DIR)/scheds/include/arch/$(TARGET_ARCH) \
 	-I$(ROOT_SRC_DIR)/scheds/include/bpf-compat \
 	-I$(ROOT_SRC_DIR)/scheds/include/lib \
+	-I$(ROOT_SRC_DIR)/scheds/vmlinux \
+	-I$(ROOT_SRC_DIR)/scheds/vmlinux/arch/$(TARGET_ARCH) \
 	$(LIBBPF_CFLAGS)
 
-export CFLAGS := -std=gnu11 -I$(ROOT_SRC_DIR)/scheds/include -I$(SCHED_OBJ_DIR) $(LIBBPF_CFLAGS)
+export CFLAGS := -std=gnu11 -I$(ROOT_SRC_DIR)/scheds/include -I$(ROOT_SRC_DIR)/scheds/vmlinux -I$(SCHED_OBJ_DIR) $(LIBBPF_CFLAGS)
 
 export LIBBPF_DEPS := $(LIBBPF_LIBS) -lelf -lz -lzstd
 export THREAD_DEPS := -lpthread

Makefile

@@ -277,12 +277,13 @@ endif
 # BPF compilation uses the gen_bpf_o generator. The following should be
 # passed in as extra_args.
 bpf_includes = ['-I', join_paths(meson.current_source_dir(), 'scheds/include'),
-                '-I', join_paths(meson.current_source_dir(), 'scheds/include/arch/' + arch_dict[cpu]),
                 '-I', join_paths(meson.current_source_dir(), 'scheds/include/bpf-compat'),
-                '-I', join_paths(meson.current_source_dir(), 'scheds/include/lib'),]
+                '-I', join_paths(meson.current_source_dir(), 'scheds/include/lib'),
+                '-I', join_paths(meson.current_source_dir(), 'scheds/vmlinux'),
+                '-I', join_paths(meson.current_source_dir(), 'scheds/vmlinux/arch/' + arch_dict[cpu]),]
 
 user_c_dep = [declare_dependency(
-  include_directories: include_directories('scheds/include'),
+  include_directories: [include_directories('scheds/include'), include_directories('scheds/vmlinux')],
 )]
 
 lib_objs = []

meson.build

@@ -23,7 +23,8 @@ fn main() {
         root_dir.join("lib/scxtest/"),
         root_dir.join("scheds/include/"),
         root_dir.join("scheds/include/lib"),
-        root_dir.join("scheds/include/arch/x86/"),
+        root_dir.join("scheds/vmlinux/"),
+        root_dir.join("scheds/vmlinux/arch/x86/"),
         root_dir.join("scheds/include/bpf-compat/"),
         env::var("DEP_BPF_INCLUDE")
             .expect("libbpf-sys include must be avaiable")

rust/scx_bpf_unittests/build.rs

@@ -22,6 +22,7 @@ nvml-wrapper = { version = "0.11.0", optional = true }
 nvml-wrapper-sys = { version = "0.9.0", optional = true }
 paste = "1.0"
 regex = "1.11.2"
+ruzstd = "0.8.1"
 scx_stats = { path = "../scx_stats", version = "1.0.16" }
 serde = { version = "1.0.215", features = ["derive"] }
 sscanf = "0.4"
@@ -31,22 +32,22 @@ version-compare = "0.1"
 zbus = { version = "5.3.1", optional = true }
 const_format = "0.2.34"
 num = "0.4.3"
+tempfile = "3.19.1"
 tracing = "0.1"
 tracing-subscriber = "0.3"
 libc = "0.2.175"
 nix = { version = "0.30.1", features = ["resource"] }
 
-[dev-dependencies]
-tempfile = "3.19.1"
-
 [build-dependencies]
 anyhow = "1.0.65"
 bindgen = ">=0.69"
 glob = "0.3.2"
 lazy_static = "1.5.0"
 libbpf-cargo = "=0.26.0-beta.1"
+ruzstd = "0.8.1"
 sscanf = "0.4"
 tar = "0.4"
+tempfile = "3.19.1"
 vergen = { version = "8.0.0", features = ["cargo", "git", "gitcl"] }
 version-compare = "0.1"
 walkdir = "2.5"

rust/scx_utils/Cargo.toml

@@ -5,7 +5,8 @@
 
 use std::{
     fs::File,
-    path::{Path, PathBuf},
+    io::{Read, Seek},
+    path::PathBuf,
 };
 
 include!("clang_info.rs");
@@ -26,6 +27,16 @@ impl Builder {
 
         ar.follow_symlinks(false);
         ar.append_dir_all(".", BPF_H).unwrap();
+
+        let vmlinux_dir = tempfile::tempdir().unwrap();
+        let mut vmlinux_tar_zst = File::open("vmlinux.tar.zst").unwrap();
+        let vmlinux_tar = ruzstd::decoding::StreamingDecoder::new(&mut vmlinux_tar_zst).unwrap();
+        tar::Archive::new(vmlinux_tar)
+            .unpack(vmlinux_dir.path())
+            .unwrap();
+        ar.append_dir_all(".", vmlinux_dir.path().join("vmlinux"))
+            .unwrap();
+
         ar.finish().unwrap();
 
         for ent in walkdir::WalkDir::new(BPF_H) {
@@ -40,16 +51,43 @@ impl Builder {
         let out_dir = env::var("OUT_DIR").unwrap();
         let clang = ClangInfo::new().unwrap();
         let kernel_target = clang.kernel_target().unwrap();
-        let vmlinux_h = Path::new(&BPF_H)
-            .join("arch")
-            .join(kernel_target)
-            .join("vmlinux.h")
-            .to_str()
+
+        let mut vmlinux_tar_zst = File::open("vmlinux.tar.zst").unwrap();
+
+        let mut vmlinux_h = String::new();
+
+        // vmlinux.h is a symlink. dereference it here.
+        let search: PathBuf = format!("vmlinux/arch/{kernel_target}/vmlinux.h").into();
+
+        let mut vmlinux_tar =
+            ruzstd::decoding::StreamingDecoder::new(&mut vmlinux_tar_zst).unwrap();
+        let mut archive = tar::Archive::new(&mut vmlinux_tar);
+        let vmlinux_link_entry = archive
+            .entries()
+            .unwrap()
+            .find(|x| x.as_ref().unwrap().path().unwrap() == search.as_path())
+            .unwrap()
+            .unwrap();
+
+        let vmlinux_path = PathBuf::from(vmlinux_link_entry.path().unwrap())
+            .parent()
+            .unwrap()
+            .join(vmlinux_link_entry.link_name().unwrap().unwrap());
+
+        vmlinux_tar_zst.rewind().unwrap();
+        let vmlinux_tar = ruzstd::decoding::StreamingDecoder::new(&mut vmlinux_tar_zst).unwrap();
+
+        tar::Archive::new(vmlinux_tar)
+            .entries()
+            .unwrap()
+            .find(|x| x.as_ref().unwrap().path().unwrap() == vmlinux_path.as_path())
+            .unwrap()
             .unwrap()
-            .to_string();
+            .read_to_string(&mut vmlinux_h)
+            .unwrap();
 
         let bindings = bindgen::Builder::default()
-            .header(vmlinux_h)
+            .header_contents(&search.to_string_lossy(), &vmlinux_h)
             .allowlist_type("scx_exit_kind")
             .allowlist_type("scx_consts")
             .parse_callbacks(Box::new(bindgen::CargoCallbacks::new()))