feat: run all package ops and doc builds in separate processes

Author: pfitzseb

.github/workflows/CI.yml

@@ -16,9 +16,9 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.8'
           - '1.9'
           - '1.10'
+          - '1.11'
         os:
           - ubuntu-latest
         arch:

Project.toml

@@ -23,7 +23,7 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]
 AbstractTrees = "0.4"
-Documenter = "0.24, 0.25, 0.26, 0.27"
+Documenter = "0.24, 0.25, 0.26, 0.27, 1"
 GitHub = "5.1, 5.2, 5.3, 5.4"
 GitForge = "0.4"
 GithubMarkdown = "0.2"

src/DocumentationGenerator.jl

@@ -29,17 +29,36 @@ function try_install_package(packagespec, envdir)
     return success
 end
 
-function try_use_package(packagespec)
+function julia()
+    return ```
+        $(first(Base.julia_cmd()))
+            --startup-file=no
+            --compiled-modules=$(VERSION >= v"1.11" ? "existing" : "no")
+            $(VERSION >= v"1.9" ? "--pkgimages=$(VERSION >= v"1.11" ? "existing" : "no")" : [])
+    ```
+end
+
+function try_use_package(packagespec, envdir)
     pkg_sym = Symbol(packagespec.name)
 
-    pkg_module = try
-        @eval(Main, (using $pkg_sym; $pkg_sym))
+    cmd = ```
+    $(julia())
+        --project="$(envdir)"
+        -e "using $(pkg_sym)"
+    ```
+
+    succeeded = try
+        success(pipeline(addenv(cmd, "JULIA_LOAD_PATH" => nothing); stdout=stdout, stderr=stderr))
     catch err
-        @error("`using $(pkg_sym) did not succeed.`", exception=err)
-        nothing
+        @error "using $pkgname issue" exception=(err, catch_backtrace())
+        false
+    end
+
+    if !succeeded
+        @error("`using $(pkg_sym) did not succeed.`")
     end
 
-    return pkg_module
+    return succeeded
 end
 
 function build_package_docs(packagespec::Pkg.Types.PackageSpec, buildpath, registry; src_prefix="", href_prefix="")
@@ -73,7 +92,7 @@ function build_documentation(
         packages;
         processes::Int = 8,
         sleeptime = 0.5,
-        juliacmd = first(Base.julia_cmd()),
+        juliacmd = julia(),
         basepath = joinpath(@__DIR__, ".."),
         envpath = normpath(joinpath(@__DIR__, "..")),
         filter_versions = last,
@@ -293,14 +312,10 @@ function start_builder(package, version;
         Pkg.develop(path=docgen_path)
         Pkg.add("Pkg")
         ## api_url needs to be set to "-", since empty string should not be passed as CLI argument
-        pkgimagesopt = VERSION >= v"1.9" ? "--pkgimages=no" : ""
         cmd = ```
             $(juliacmd)
                 --project=$(Base.active_project())
                 --color=no
-                --compiled-modules=no
-                $(isempty(pkgimagesopt) ? [] : pkgimagesopt)
-                -O0
                 $workerfile
                 $uuid
                 $name

src/builders.jl

@@ -103,7 +103,7 @@ function build_local_docs(packagespec, buildpath, uri, pkgroot = nothing; gitdir
             pkgfile = Base.find_package(pkgname)
             pkgroot = normpath(joinpath(pkgfile, "..", ".."))
         end
-        mod = try_use_package(packagespec)
+        could_use_pkg = try_use_package(packagespec, envdir)
 
         # actual Documenter docs
         try
@@ -119,7 +119,7 @@ function build_local_docs(packagespec, buildpath, uri, pkgroot = nothing; gitdir
                             "doctype" => gitdirdocs ? "git-repo" : "documenter",
                             "documenter_errored" => documenter_errored,
                             "installable" => true,
-                            "using_failed" => mod == nothing,
+                            "using_failed" => !could_use_pkg,
                             "success" => true
                         )
                     else
@@ -139,22 +139,22 @@ function build_local_docs(packagespec, buildpath, uri, pkgroot = nothing; gitdir
 
         # fallback docs (readme & docstrings)
         return mktempdir() do docsdir
-            output = build_readme_docs(pkgname, pkgroot, docsdir, mod, src_prefix, href_prefix)
+            output = build_readme_docs(pkgname, pkgroot, docsdir, src_prefix, href_prefix, could_use_pkg)
             if output !== nothing
                 cp(output, buildpath, force = true)
                 return Dict(
                     "doctype" => "fallback_autodocs",
                     "documenter_errored" => documenter_errored,
                     "installable" => true,
-                    "using_failed" => mod == nothing,
+                    "using_failed" => !could_use_pkg,
                     "success" => true
                 )
             end
             return Dict(
                 "doctype" => "fallback_autodocs",
                 "documenter_errored" => documenter_errored,
                 "installable" => true,
-                "using_failed" => mod == nothing,
+                "using_failed" => !could_use_pkg,
                 "success" => false
             )
         end
@@ -206,11 +206,9 @@ function build_documenter(packagespec, docdir)
         _, builddir = fix_makefile(makefile)
         pkgimagesopt = VERSION >= v"1.9" ? "--pkgimages=no" : ""
         cmd = ```
-            $(first(Base.julia_cmd()))
+            $(julia())
                 --project="$(docdir)"
-                --compiled-modules=no
                 $(isempty(pkgimagesopt) ? [] : pkgimagesopt)
-                -O0
                 $(rundcocumenter)
                 $(pkgdir)
                 $(makefile)
@@ -231,51 +229,84 @@ function build_documenter(packagespec, docdir)
     end
 end
 
-function build_readme_docs(pkgname, pkgroot, docsdir, mod, src_prefix, href_prefix)
+function build_readme_docs(pkgname, pkgroot, docsdir, src_prefix, href_prefix, could_use_pkg)
     @info("Generating readme-only fallback docs.")
 
     if pkgroot === nothing || !ispath(pkgroot)
         @error("Julia could not find the package directory. Aborting.")
         return
     end
 
-    pkgloads = mod !== nothing
-
     readme = find_readme(pkgroot)
     doc_src = joinpath(docsdir, "src")
     mkpath(doc_src)
     index = joinpath(doc_src, "index.md")
 
     render_html(readme, index, src_prefix, href_prefix; documenter = true)
 
+    pages = ["Readme" => "index.md"]
+
+    if could_use_pkg
+        @info("Deploying `autodocs`.")
+        add_autodocs(doc_src, Symbol(pkgname))
+        push!(pages, "Docstrings" => "autodocs.md")
+    end
+
     if !isfile(index)
         open(index, "w") do io
             println(io, """
             # $pkgname
             """)
+
+            if !could_use_pkg
+                println(io, """
+                > No documentation or readme found for this package.
+                """)
+            end
         end
     end
 
-    pages = ["Readme" => "index.md"]
-    modules = :(Module[Module()])
+    makejl_str = """
+    using Pkg
+    Pkg.add(name="Documenter", version="1")
+    Pkg.develop(path="$(pkgroot)")
 
-    if pkgloads
-        @info("Deploying `autodocs`.")
-        add_autodocs(doc_src, mod)
-        push!(pages, "Docstrings" => "autodocs.md")
-        modules = :(Module[$mod])
-    end
-
-    @eval Module() begin
-        const Documenter = $Documenter
-        using .Documenter
-        makedocs(
-            format = Documenter.HTML(),
-            sitename = "$($pkgname).jl",
-            modules = $(modules),
-            root = $(docsdir),
-            pages = $(pages)
-        )
+    using Documenter
+    using $(pkgname)
+
+    makedocs(
+        format = Documenter.HTML(),
+        sitename = "$(pkgname).jl",
+        modules = [$(pkgname)],
+        root = "$(docsdir)",
+        pages = $(pages),
+        remotes = nothing,
+        repo = "",
+        doctest = false
+    )
+    """
+
+    makejl_path = joinpath(docsdir, "make.jl")
+
+    open(makejl_path, "w") do io
+        println(io, makejl_str)
+    end
+
+    cmd = ```
+    $(julia())
+        --project="$(docsdir)"
+        $(makejl_path)
+    ```
+
+    succeeded = try
+        success(pipeline(addenv(cmd, "JULIA_LOAD_PATH" => nothing); stdout=stdout, stderr=stderr))
+    catch err
+        @error "autodocs $pkgname issue" exception=(err, catch_backtrace())
+        false
+    end
+
+    if !succeeded
+        @error("Failed to generate readme autodocs for $pkgname")
     end
 
     build_dir = joinpath(docsdir, "build")
@@ -308,11 +339,10 @@ function find_readme(pkgroot)
 end
 
 function add_autodocs(docsdir, mod)
-    module_list = join((string(m) for m in submodules(mod)), ", ")
     open(joinpath(docsdir, "autodocs.md"), "w") do io
         println(io, """
         ```@autodocs
-        Modules = [$module_list]
+        Modules = [$mod]
         ```
         """)
     end

src/rundocumenter.jl

@@ -27,28 +27,32 @@ let anonymous_module = Module()
             end
             Pkg.status()
 
-            documenter_version = v"0.24.10"
+            documenter_version = v"1.8.1"
             try
                 manifest = joinpath(docsdir, "Manifest.toml")
                 if isfile(manifest)
                     pm = TOML.parsefile(manifest)
-                    global documenter_version = VersionNumber(first(pm["Documenter"])["version"])
+                    if haskey(pm, "Documenter")
+                        documenter_version = VersionNumber(first(pm["Documenter"])["version"])
+                    else
+                        @warn("Documenter not found in `$(manifest). Defaulting to $(documenter_version).")
+                    end
                 else
                     @warn("No Mainfest.toml found at `$(manifest)`. Defaulting to $(documenter_version).")
                 end
             catch err
                 @error(exception = err)
             end
 
-            @info("Detected Documenter version $(documenter_version).")
+            @info("Using Documenter version $(documenter_version).")
 
             expr, bpath = fix_makefile(makefile, documenter_version)
 
             @info("`cd`ing to `$(docsdir)` and setting `tls[:SOURCE_PATH]` to `$(makefile)`.")
             task_local_storage()[:SOURCE_PATH] = makefile
             cd(docsdir) do
                 @info("Evaluating the following `make` expr:")
-                @info(expr)
+                println(stderr, "\n------------------\n", expr, "\n------------------\n")
                 Base.eval(Main, expr)
             end
         end

src/utils/rewrite.jl

@@ -24,7 +24,7 @@ Takes in the path to a Documenter.jl-compatible `make.jl` file and
 
 Return a tuple of `new_make_expr, buildpath`.
 """
-function fix_makefile(makefile, documenter_version = v"0.24")
+function fix_makefile(makefile, documenter_version = v"1.8.1")
     # default output path:
     buildpath = joinpath(dirname(makefile), "build")
     should_break = false
@@ -56,7 +56,7 @@ function fix_makefile(makefile, documenter_version = v"0.24")
             fixkwarg = argument -> begin
                 if Meta.isexpr(argument, :kw)
                     name, arg = argument.args
-                    # assure that we generate HTML
+                    # ensure that we generate HTML
                     if name == :format
                         has_fmt = true
                         if Meta.isexpr(arg, :call) && arg.args[1] == :(Documenter.HTML) && !(html isa QuoteNode)
@@ -123,25 +123,22 @@ function fix_makefile(makefile, documenter_version = v"0.24")
             if !has_fmt
                 push!(new_args, Expr(:kw, :format, html))
             end
-
             if !has_linkcheck
                 push!(new_args, Expr(:kw, :linkcheck, false))
             end
-
             if !has_doctest
                 push!(new_args, Expr(:kw, :doctest, false))
             end
-
             if !has_root
                 push!(new_args,Expr(:kw, :root, dirname(makefile)))
             end
-            if !has_remotes
+            if !has_remotes && documenter_version >= v"1"
                 push!(new_args, Expr(:kw, :remotes, nothing))
             end
             if !has_repo
                 push!(new_args, Expr(:kw, :repo, ""))
             end
-            if !has_warnonly
+            if !has_warnonly && documenter_version >= v"1"
                 push!(new_args, Expr(:kw, :warnonly, true))
             end