Add support for rustdoc mergeable cross-crate info parts

This is an unstable feature that we designed to fix several
performance problems with the old system:

1. You couldn't easily build crate docs in hermetic environments.
   This doesn't matter for Cargo, but it was one of the original
   reasons to implement the feature.
2. We have to build all the doc resources in their final form at
   every step, instead of delaying slow parts (mostly the
   search index) until the end and only doing them once.
3. It requires rustdoc to take a lock at the end. This reduces
   available concurrency for generating docs.

Author: notriddle

src/cargo/core/compiler/build_context/target_info.rs

@@ -1184,6 +1184,15 @@ impl RustDocFingerprint {
             .map(|kind| build_runner.files().layout(*kind).artifact_dir().doc())
             .filter(|path| path.exists())
             .try_for_each(|path| clean_doc(path))?;
+        if build_runner.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+            build_runner
+                .bcx
+                .all_kinds
+                .iter()
+                .map(|kind| build_runner.files().layout(*kind).build_dir().doc_parts())
+                .filter(|path| path.exists())
+                .try_for_each(|path| clean_doc(&path))?;
+        }
         write_fingerprint()?;
         return Ok(());
 

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -324,6 +324,13 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
             .build_script(&dir)
     }
 
+    /// Returns the directory where mergeable cross crate info for docs is stored.
+    pub fn doc_parts_dir(&self, unit: &Unit) -> PathBuf {
+        assert!(unit.mode.is_doc());
+        assert!(self.metas.contains_key(unit));
+        self.layout(unit.kind).build_dir().doc_parts().to_path_buf()
+    }
+
     /// Returns the directory for compiled artifacts files.
     /// `/path/to/target/{debug,release}/deps/artifact/KIND/PKG-HASH`
     fn artifact_dir(&self, unit: &Unit) -> PathBuf {

src/cargo/core/compiler/build_runner/mod.rs

@@ -1,6 +1,7 @@
 //! [`BuildRunner`] is the mutable state used during the build process.
 
 use std::collections::{BTreeSet, HashMap, HashSet};
+use std::ffi::OsString;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
 
@@ -310,6 +311,49 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
                     .insert(dir.clone().into_path_buf());
             }
         }
+
+        if self.bcx.build_config.intent.is_doc()
+            && self.bcx.gctx.cli_unstable().rustdoc_mergeable_info
+            && let Some(unit) = self
+                .bcx
+                .roots
+                .iter()
+                .filter(|unit| unit.mode.is_doc())
+                .next()
+        {
+            let mut rustdoc = self.compilation.rustdoc_process(unit, None)?;
+            let doc_dir = self.files().out_dir(unit);
+            let mut include_arg = OsString::from("--include-parts-dir=");
+            include_arg.push(self.files().doc_parts_dir(&unit));
+            rustdoc
+                .arg("-o")
+                .arg(&doc_dir)
+                .arg("--emit=toolchain-shared-resources")
+                .arg("-Zunstable-options")
+                .arg("--merge=finalize")
+                .arg(include_arg);
+            exec.exec(
+                &rustdoc,
+                unit.pkg.package_id(),
+                &unit.target,
+                CompileMode::Doc,
+                // This is always single-threaded, and always gets run,
+                // so thread delinterleaving isn't needed and neither is
+                // the output cache.
+                &mut |line| {
+                    let mut shell = self.bcx.gctx.shell();
+                    shell.print_ansi_stdout(line.as_bytes())?;
+                    shell.err().write_all(b"\n")?;
+                    Ok(())
+                },
+                &mut |line| {
+                    let mut shell = self.bcx.gctx.shell();
+                    shell.print_ansi_stderr(line.as_bytes())?;
+                    shell.err().write_all(b"\n")?;
+                    Ok(())
+                },
+            )?;
+        }
         Ok(self.compilation)
     }
 

src/cargo/core/compiler/layout.rs

@@ -337,6 +337,10 @@ impl BuildDirLayout {
             self.build().join(pkg_dir)
         }
     }
+    /// Fetch the doc parts path.
+    pub fn doc_parts(&self) -> PathBuf {
+        self.build().join("doc.parts")
+    }
     /// Fetch the build script execution path.
     pub fn build_script_execution(&self, pkg_dir: &str) -> PathBuf {
         if self.is_new_layout {

src/cargo/core/compiler/mod.rs

@@ -831,8 +831,13 @@ fn prepare_rustdoc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoResu
     if build_runner.bcx.gctx.cli_unstable().rustdoc_depinfo {
         // toolchain-shared-resources is required for keeping the shared styling resources
         // invocation-specific is required for keeping the original rustdoc emission
-        let mut arg =
-            OsString::from("--emit=toolchain-shared-resources,invocation-specific,dep-info=");
+        let mut arg = if build_runner.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+            // toolchain resources are written at the end, at the same time as merging
+            OsString::from("--emit=invocation-specific,dep-info=")
+        } else {
+            // if not using mergeable CCI, everything is written every time
+            OsString::from("--emit=toolchain-shared-resources,invocation-specific,dep-info=")
+        };
         arg.push(rustdoc_dep_info_loc(build_runner, unit));
         rustdoc.arg(arg);
 
@@ -841,6 +846,18 @@ fn prepare_rustdoc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoResu
         }
 
         rustdoc.arg("-Zunstable-options");
+    } else if build_runner.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+        // toolchain resources are written at the end, at the same time as merging
+        rustdoc.arg("--emit=invocation-specific");
+        rustdoc.arg("-Zunstable-options");
+    }
+
+    if build_runner.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+        // write out mergeable data to be imported
+        rustdoc.arg("--merge=none");
+        let mut arg = OsString::from("--parts-out-dir=");
+        arg.push(build_runner.files().doc_parts_dir(&unit));
+        rustdoc.arg(arg);
     }
 
     if let Some(trim_paths) = unit.profile.trim_paths.as_ref() {

src/cargo/core/features.rs

@@ -883,6 +883,7 @@ unstable_cli_options!(
     root_dir: Option<PathBuf> = ("Set the root directory relative to which paths are printed (defaults to workspace root)"),
     rustdoc_depinfo: bool = ("Use dep-info files in rustdoc rebuild detection"),
     rustdoc_map: bool = ("Allow passing external documentation mappings to rustdoc"),
+    rustdoc_mergeable_info: bool = ("Use rustdoc mergeable cross-crate-info files"),
     rustdoc_scrape_examples: bool = ("Allows Rustdoc to scrape code examples from reverse-dependencies"),
     sbom: bool = ("Enable the `sbom` option in build config in .cargo/config.toml file"),
     script: bool = ("Enable support for single-file, `.rs` packages"),
@@ -1413,6 +1414,7 @@ impl CliUnstable {
             "root-dir" => self.root_dir = v.map(|v| v.into()),
             "rustdoc-depinfo" => self.rustdoc_depinfo = parse_empty(k, v)?,
             "rustdoc-map" => self.rustdoc_map = parse_empty(k, v)?,
+            "rustdoc-mergeable-info" => self.rustdoc_mergeable_info = parse_empty(k, v)?,
             "rustdoc-scrape-examples" => self.rustdoc_scrape_examples = parse_empty(k, v)?,
             "sbom" => self.sbom = parse_empty(k, v)?,
             "section-timings" => self.section_timings = parse_empty(k, v)?,

tests/testsuite/cargo/z_help/stdout.term.svg

@@ -182,6 +182,126 @@ fn doc_no_deps() {
     assert!(!p.root().join("target/doc/bar/index.html").is_file());
 }
 
+#[cargo_test(nightly, reason = "rustdoc mergeable crate info is unstable")]
+fn doc_deps_rustdoc_mergeable_info() {
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.0.1"
+                edition = "2015"
+                authors = []
+
+                [dependencies.bar]
+                path = "bar"
+            "#,
+        )
+        .file("src/lib.rs", "extern crate bar; pub fn foo() {}")
+        .file("bar/Cargo.toml", &basic_manifest("bar", "0.0.1"))
+        .file("bar/src/lib.rs", "pub fn bar() {}")
+        .build();
+
+    p.cargo("doc -Zunstable-options -Zrustdoc-mergeable-info")
+        .masquerade_as_nightly_cargo(&["rustdoc-mergeable-info"])
+        .with_stderr_data(
+            str![[r#"
+[LOCKING] 1 package to latest compatible version
+[DOCUMENTING] bar v0.0.1 ([ROOT]/foo/bar)
+[CHECKING] bar v0.0.1 ([ROOT]/foo/bar)
+[DOCUMENTING] foo v0.0.1 ([ROOT]/foo)
+[FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
+[GENERATED] [ROOT]/foo/target/doc/foo/index.html
+
+"#]]
+            .unordered(),
+        )
+        .run();
+
+    assert!(p.root().join("target/doc").is_dir());
+    assert!(p.root().join("target/doc/foo/index.html").is_file());
+    assert!(p.root().join("target/doc/bar/index.html").is_file());
+
+    // Verify that it only emits rmeta for the dependency.
+    assert_eq!(p.glob("target/debug/**/*.rlib").count(), 0);
+    assert_eq!(p.glob("target/debug/deps/libbar-*.rmeta").count(), 1);
+
+    // Make sure it doesn't recompile.
+    p.cargo("doc -Zunstable-options -Zrustdoc-mergeable-info")
+        .masquerade_as_nightly_cargo(&["rustdoc-mergeable-info"])
+        .with_stderr_data(str![[r#"
+[FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
+[GENERATED] [ROOT]/foo/target/doc/foo/index.html
+
+"#]])
+        .run();
+
+    assert!(p.root().join("target/doc").is_dir());
+    assert!(p.root().join("target/doc/foo/index.html").is_file());
+    assert!(p.root().join("target/doc/bar/index.html").is_file());
+}
+
+#[cargo_test(nightly, reason = "rustdoc mergeable crate info is unstable")]
+ = "rustdoc scrape examples flags are unstable")]
+fn doc_no_deps_rustdoc_mergeable_info() {
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.0.1"
+                edition = "2015"
+                authors = []
+
+                [dependencies.bar]
+                path = "bar"
+            "#,
+        )
+        .file("src/lib.rs", "extern crate bar; pub fn foo() {}")
+        .file("bar/Cargo.toml", &basic_manifest("bar", "0.0.1"))
+        .file("bar/src/lib.rs", "pub fn bar() {}")
+        .build();
+
+    p.cargo("doc --no-deps -Zunstable-options -Zrustdoc-mergeable-info")
+        .masquerade_as_nightly_cargo(&["rustdoc-mergeable-info"])
+        .with_stderr_data(
+            str![[r#"
+[LOCKING] 1 package to latest compatible version
+[CHECKING] bar v0.0.1 ([ROOT]/foo/bar)
+[DOCUMENTING] foo v0.0.1 ([ROOT]/foo)
+[FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
+[GENERATED] [ROOT]/foo/target/doc/foo/index.html
+
+"#]]
+            .unordered(),
+        )
+        .run();
+
+    assert!(p.root().join("target/doc").is_dir());
+    assert!(p.root().join("target/doc/foo/index.html").is_file());
+    assert!(!p.root().join("target/doc/bar/index.html").is_file());
+
+    // Verify that it only emits rmeta for the dependency.
+    assert_eq!(p.glob("target/debug/**/*.rlib").count(), 0);
+    assert_eq!(p.glob("target/debug/deps/libbar-*.rmeta").count(), 1);
+
+    // Make sure it doesn't recompile.
+    p.cargo("doc --no-deps -Zunstable-options -Zrustdoc-mergeable-info")
+        .masquerade_as_nightly_cargo(&["rustdoc-mergeable-info"])
+        .with_stderr_data(str![[r#"
+[FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
+[GENERATED] [ROOT]/foo/target/doc/foo/index.html
+
+"#]])
+        .run();
+
+    assert!(p.root().join("target/doc").is_dir());
+    assert!(p.root().join("target/doc/foo/index.html").is_file());
+    assert!(!p.root().join("target/doc/bar/index.html").is_file());
+}
+
 #[cargo_test]
 fn doc_only_bin() {
     let p = project()