Add CLI flag to enable DocC adding minimal per-page HTML content to each index.html file (#1396)

d-ronnqvist · anferbui · web-flow · commit b21d94f8cdfe · 2025-12-18T14:51:40.000+01:00
* Add CLI flag to insert minimal HTML content in each "index.html" file

rdar://163326857

* Support index.html files that are missing the expected HTML elements

* Test that index.html files with content includes the hosting base path

* Support custom header/footer templates alongside the per-page content

* Remove properties that are never read

* Avoid HTML encoding difference between platforms in new test

* Add code comments about confusing test behaviors

* Apply suggestions from code review

Co-authored-by: Andrea Fernandez Buitrago &lt;15234535+anferbui@users.noreply.github.com&gt;

* Fix typo in test data

---------

Co-authored-by: Andrea Fernandez Buitrago &lt;15234535+anferbui@users.noreply.github.com&gt;
diff --git a/Sources/DocCCommandLine/Action/Actions/Convert/ConvertAction.swift b/Sources/DocCCommandLine/Action/Actions/Convert/ConvertAction.swift
@@ -30,6 +30,7 @@ public struct ConvertAction: AsyncAction {
     let diagnosticEngine: DiagnosticEngine
 
     private let transformForStaticHosting: Bool
+    private let includeContentInEachHTMLFile: Bool
     private let hostingBasePath: String?
     
     let sourceRepository: SourceRepository?
@@ -64,6 +65,7 @@ public struct ConvertAction: AsyncAction {
     ///   - experimentalEnableCustomTemplates: `true` if the convert action should enable support for custom "header.html" and "footer.html" template files, otherwise `false`.
     ///   - experimentalModifyCatalogWithGeneratedCuration: `true` if the convert action should write documentation extension files containing markdown representations of DocC's automatic curation into the `documentationBundleURL`, otherwise `false`.
     ///   - transformForStaticHosting: `true` if the convert action should process the build documentation archive so that it supports a static hosting environment, otherwise `false`.
+    ///   - includeContentInEachHTMLFile: `true` if the convert action should process each static hosting HTML file so that it includes documentation content for environments without JavaScript enabled, otherwise `false`.
     ///   - allowArbitraryCatalogDirectories: `true` if the convert action should consider the root location as a documentation bundle if it doesn't discover another bundle, otherwise `false`.
     ///   - hostingBasePath: The base path where the built documentation archive will be hosted at.
     ///   - sourceRepository: The source repository where the documentation's sources are hosted.
@@ -91,6 +93,7 @@ public struct ConvertAction: AsyncAction {
         experimentalEnableCustomTemplates: Bool = false,
         experimentalModifyCatalogWithGeneratedCuration: Bool = false,
         transformForStaticHosting: Bool = false,
+        includeContentInEachHTMLFile: Bool = false,
         allowArbitraryCatalogDirectories: Bool = false,
         hostingBasePath: String? = nil,
         sourceRepository: SourceRepository? = nil,
@@ -105,6 +108,7 @@ public struct ConvertAction: AsyncAction {
         self.temporaryDirectory = temporaryDirectory
         self.documentationCoverageOptions = documentationCoverageOptions
         self.transformForStaticHosting = transformForStaticHosting
+        self.includeContentInEachHTMLFile = includeContentInEachHTMLFile
         self.hostingBasePath = hostingBasePath
         self.sourceRepository = sourceRepository
         
@@ -189,6 +193,11 @@ public struct ConvertAction: AsyncAction {
     /// A block of extra work that tests perform to affect the time it takes to convert documentation
     var _extraTestWork: (() async -> Void)?
 
+    /// The `Indexer` type doesn't work with virtual file systems.
+    ///
+    /// Tests that don't verify the contents of the navigator index can set this to `true` so that they can use a virtual, in-memory, file system.
+    var _completelySkipBuildingIndex: Bool = false
+    
     /// Converts each eligible file from the source documentation bundle,
     /// saves the results in the given output alongside the template files.
     public func perform(logHandle: inout LogHandle) async throws -> ActionResult {
@@ -286,7 +295,7 @@ public struct ConvertAction: AsyncAction {
             workingDirectory: temporaryFolder,
             fileManager: fileManager)
 
-        let indexer = try Indexer(outputURL: temporaryFolder, bundleID: inputs.id)
+        let indexer = _completelySkipBuildingIndex ? nil : try Indexer(outputURL: temporaryFolder, bundleID: inputs.id)
 
         let registerInterval = signposter.beginInterval("Register", id: signposter.makeSignpostID())
         let context = try await DocumentationContext(bundle: inputs, dataProvider: dataProvider, diagnosticEngine: diagnosticEngine, configuration: configuration)
@@ -299,9 +308,23 @@ public struct ConvertAction: AsyncAction {
             context: context,
             indexer: indexer,
             enableCustomTemplates: experimentalEnableCustomTemplates,
-            transformForStaticHostingIndexHTML: transformForStaticHosting ? indexHTML : nil,
+            // Don't transform for static hosting if the `FileWritingHTMLContentConsumer` will create per-page index.html files
+            transformForStaticHostingIndexHTML: transformForStaticHosting && !includeContentInEachHTMLFile ? indexHTML : nil,
             bundleID: inputs.id
         )
+        
+        let htmlConsumer: FileWritingHTMLContentConsumer?
+        if includeContentInEachHTMLFile, let indexHTML {
+            htmlConsumer = try FileWritingHTMLContentConsumer(
+                targetFolder: temporaryFolder,
+                fileManager: fileManager,
+                htmlTemplate: indexHTML,
+                customHeader: experimentalEnableCustomTemplates ? inputs.customHeader : nil,
+                customFooter: experimentalEnableCustomTemplates ? inputs.customFooter : nil
+            )
+        } else {
+            htmlConsumer = nil
+        }
 
         if experimentalModifyCatalogWithGeneratedCuration, let catalogURL = rootURL {
             let writer = GeneratedCurationWriter(context: context, catalogURL: catalogURL, outputURL: catalogURL)
@@ -320,7 +343,7 @@ public struct ConvertAction: AsyncAction {
                 try ConvertActionConverter.convert(
                     context: context,
                     outputConsumer: outputConsumer,
-                    htmlContentConsumer: nil,
+                    htmlContentConsumer: htmlConsumer,
                     sourceRepository: sourceRepository,
                     emitDigest: emitDigest,
                     documentationCoverageOptions: documentationCoverageOptions
@@ -375,7 +398,7 @@ public struct ConvertAction: AsyncAction {
         }
         
         // If we're building a navigation index, finalize the process and collect encountered problems.
-        do {
+        if let indexer {
             let finalizeNavigationIndexMetric = benchmark(begin: Benchmark.Duration(id: "finalize-navigation-index"))
             
             // Always emit a JSON representation of the index but only emit the LMDB
diff --git a/Sources/DocCCommandLine/Action/Actions/Convert/ConvertFileWritingConsumer.swift b/Sources/DocCCommandLine/Action/Actions/Convert/ConvertFileWritingConsumer.swift
@@ -237,7 +237,7 @@ struct ConvertFileWritingConsumer: ConvertOutputConsumer, ExternalNodeConsumer {
         let template = "<template id=\"\(id.rawValue)\">\(templateContents)</template>"
         var newIndexContents = indexContents
         newIndexContents.replaceSubrange(bodyTagRange, with: indexContents[bodyTagRange] + template)
-        try newIndexContents.write(to: index, atomically: true, encoding: .utf8)
+        try fileManager.createFile(at: index, contents: Data(newIndexContents.utf8))
     }
     
     /// File name for the documentation coverage file emitted during conversion.
diff --git a/Sources/DocCCommandLine/Action/Actions/Convert/FileWritingHTMLContentConsumer.swift b/Sources/DocCCommandLine/Action/Actions/Convert/FileWritingHTMLContentConsumer.swift
@@ -20,8 +20,6 @@ import SwiftDocC
 import DocCHTML
 
 struct FileWritingHTMLContentConsumer: HTMLContentConsumer {
-    var targetFolder: URL
-    var fileManager: any FileManagerProtocol
     var prettyPrintOutput: Bool
     
     private struct HTMLTemplate {
@@ -30,24 +28,51 @@ struct FileWritingHTMLContentConsumer: HTMLContentConsumer {
         var titleReplacementRange:       Range<String.Index>
         var descriptionReplacementRange: Range<String.Index>
         
-        init(data: Data) throws {
-            let content = String(decoding: data, as: UTF8.self)
+        struct CustomTemplate {
+            var id, content: String
+        }
+        
+        init(data: Data, customTemplates: [CustomTemplate]) throws {
+            var content = String(decoding: data, as: UTF8.self)
             
-            // ???: Should we parse the content with XMLParser instead? If so, what do we do if it's not valid XHTML?
-            let noScriptStart = content.utf8.firstRange(of:  "<noscript>".utf8)!.upperBound
-            let noScriptEnd   = content.utf8.firstRange(of: "</noscript>".utf8)!.lowerBound
+            // Ensure that the index.html file has at least a `<head>` and a `<body>`.
+            guard var beforeEndOfHead  = content.utf8.firstRange(of: "</head>".utf8)?.lowerBound,
+                  var afterStartOfBody = content.range(of: "<body[^>]*>", options: .regularExpression)?.upperBound
+            else {
+                struct MissingRequiredTagsError: DescribedError {
+                    let errorDescription = "Missing required `<head>` and `<body>` elements in \"index.html\" file."
+                }
+                throw MissingRequiredTagsError()
+            }
             
-            let titleStart = content.utf8.firstRange(of:  "<title>".utf8)!.upperBound
-            let titleEnd   = content.utf8.firstRange(of: "</title>".utf8)!.lowerBound
+            for template in customTemplates { // Use the order as `ConvertFileWritingConsumer`
+                content.insert(contentsOf: "<template id=\"\(template.id)\">\(template.content)</template>", at: afterStartOfBody)
+            }
             
-            let beforeHeadEnd = content.utf8.firstRange(of: "</head>".utf8)!.lowerBound
+            if let titleStart = content.utf8.firstRange(of:  "<title>".utf8)?.upperBound,
+               let titleEnd   = content.utf8.firstRange(of: "</title>".utf8)?.lowerBound
+            {
+                titleReplacementRange = titleStart ..< titleEnd
+            } else {
+                content.insert(contentsOf: "<title></title>", at: beforeEndOfHead)
+                content.utf8.formIndex(&beforeEndOfHead,  offsetBy: "<title></title>".utf8.count)
+                content.utf8.formIndex(&afterStartOfBody, offsetBy: "<title></title>".utf8.count)
+                let titleInside = content.utf8.index(beforeEndOfHead, offsetBy: -"</title>".utf8.count)
+                titleReplacementRange = titleInside ..< titleInside
+            }
             
+            if let noScriptStart = content.utf8.firstRange(of:  "<noscript>".utf8)?.upperBound,
+               let noScriptEnd   = content.utf8.firstRange(of: "</noscript>".utf8)?.lowerBound
+            {
+                contentReplacementRange = noScriptStart ..< noScriptEnd
+            } else {
+                content.insert(contentsOf: "<noscript></noscript>", at: afterStartOfBody)
+                let noScriptInside = content.utf8.index(afterStartOfBody, offsetBy: "<noscript>".utf8.count)
+                contentReplacementRange = noScriptInside ..< noScriptInside
+            }
+                        
             original = content
-            // TODO: If the template doesn't already contain a <noscript> element, add one to the start of the <body> element
-            // TODO: If the template doesn't already contain a <title> element, add one to the end of the <head> element
-            contentReplacementRange     = noScriptStart ..< noScriptEnd
-            titleReplacementRange       = titleStart    ..< titleEnd
-            descriptionReplacementRange = beforeHeadEnd ..< beforeHeadEnd
+            descriptionReplacementRange = beforeEndOfHead ..< beforeEndOfHead
             
             assert(titleReplacementRange.upperBound       < descriptionReplacementRange.lowerBound, "The title replacement range should be before the description replacement range")
             assert(descriptionReplacementRange.upperBound < contentReplacementRange.lowerBound,     "The description replacement range should be before the content replacement range")
@@ -78,11 +103,27 @@ struct FileWritingHTMLContentConsumer: HTMLContentConsumer {
         targetFolder: URL,
         fileManager: some FileManagerProtocol,
         htmlTemplate: URL,
+        customHeader: URL?,
+        customFooter: URL?,
         prettyPrintOutput: Bool = shouldPrettyPrintOutputJSON
     ) throws {
-        self.targetFolder = targetFolder
-        self.fileManager = fileManager
-        self.htmlTemplate = try HTMLTemplate(data: fileManager.contents(of: htmlTemplate))
+        var customTemplates: [HTMLTemplate.CustomTemplate] = []
+        if let customHeader {
+            customTemplates.append(.init(
+                id: "custom-header",
+                content: String(decoding: try fileManager.contents(of: customHeader), as: UTF8.self)
+            ))
+        }
+        if let customFooter {
+            customTemplates.append(.init(
+                id: "custom-footer",
+                content: String(decoding: try fileManager.contents(of: customFooter), as: UTF8.self)
+            ))
+        }
+        self.htmlTemplate = try HTMLTemplate(
+            data: fileManager.contents(of: htmlTemplate),
+            customTemplates: customTemplates
+        )
         self.prettyPrintOutput = prettyPrintOutput
         self.fileWriter = JSONEncodingRenderNodeWriter(
             targetFolder: targetFolder,
diff --git a/Sources/DocCCommandLine/ArgumentParsing/ActionExtensions/ConvertAction+CommandInitialization.swift b/Sources/DocCCommandLine/ArgumentParsing/ActionExtensions/ConvertAction+CommandInitialization.swift
@@ -78,6 +78,7 @@ extension ConvertAction {
             experimentalEnableCustomTemplates: convert.experimentalEnableCustomTemplates,
             experimentalModifyCatalogWithGeneratedCuration: convert.experimentalModifyCatalogWithGeneratedCuration,
             transformForStaticHosting: convert.transformForStaticHosting,
+            includeContentInEachHTMLFile: convert.experimentalTransformForStaticHostingWithContent,
             allowArbitraryCatalogDirectories: convert.allowArbitraryCatalogDirectories,
             hostingBasePath: convert.hostingBasePath,
             sourceRepository: SourceRepository(from: convert.sourceRepositoryArguments),
diff --git a/Sources/DocCCommandLine/ArgumentParsing/Subcommands/Convert.swift b/Sources/DocCCommandLine/ArgumentParsing/Subcommands/Convert.swift
@@ -184,6 +184,20 @@ extension Docc {
                 help: "Produce a DocC archive that supports static hosting environments."
             )
             var transformForStaticHosting = true
+            
+            @Flag(help: "Include documentation content in each HTML file for static hosting environments.")
+            var experimentalTransformForStaticHostingWithContent = false
+            
+            mutating func validate() throws {
+                if experimentalTransformForStaticHostingWithContent, !transformForStaticHosting {
+                    warnAboutDiagnostic(.init(
+                        severity: .warning,
+                        identifier: "org.swift.docc.IgnoredNoTransformForStaticHosting",
+                        summary: "Passing '--experimental-transform-for-static-hosting-with-content' also implies '--transform-for-static-hosting'. Passing '--no-transform-for-static-hosting' has no effect."
+                    ))
+                    transformForStaticHosting = true
+                }
+            }
         }
         
         /// A Boolean value that is true if the DocC archive produced by this conversion will support static hosting environments.
@@ -194,6 +208,12 @@ extension Docc {
             set { hostingOptions.transformForStaticHosting = newValue }
         }
         
+        /// A Boolean value that is true if the DocC archive produced by this conversion will support browsing without JavaScript enabled.
+        public var experimentalTransformForStaticHostingWithContent: Bool {
+            get { hostingOptions.experimentalTransformForStaticHostingWithContent }
+            set { hostingOptions.experimentalTransformForStaticHostingWithContent = newValue }
+        }
+        
         /// A user-provided relative path to be used in the archived output
         var hostingBasePath: String? {
             hostingOptions.hostingBasePath
diff --git a/Tests/DocCCommandLineTests/ArgumentParsing/ConvertSubcommandTests.swift b/Tests/DocCCommandLineTests/ArgumentParsing/ConvertSubcommandTests.swift
@@ -596,6 +596,37 @@ class ConvertSubcommandTests: XCTestCase {
         let disabledFlagConvert = try Docc.Convert.parse(["--disable-mentioned-in"])
         XCTAssertEqual(disabledFlagConvert.enableMentionedIn, false)
     }
+    
+    func testStaticHostingWithContentFlag() throws {
+        // The feature is disabled when no flag is passed.
+        let noFlagConvert = try Docc.Convert.parse([])
+        XCTAssertEqual(noFlagConvert.experimentalTransformForStaticHostingWithContent, false)
+        
+        let enabledFlagConvert = try Docc.Convert.parse(["--experimental-transform-for-static-hosting-with-content"])
+        XCTAssertEqual(enabledFlagConvert.experimentalTransformForStaticHostingWithContent, true)
+        
+        // The '...-transform...-with-content' flag also implies the base '--transform-...' flag.
+        do {
+            let originalErrorLogHandle = Docc.Convert._errorLogHandle
+            let originalDiagnosticFormattingOptions = Docc.Convert._diagnosticFormattingOptions
+            defer {
+                Docc.Convert._errorLogHandle = originalErrorLogHandle
+                Docc.Convert._diagnosticFormattingOptions = originalDiagnosticFormattingOptions
+            }
+            
+            let logStorage = LogHandle.LogStorage()
+            Docc.Convert._errorLogHandle = .memory(logStorage)
+            Docc.Convert._diagnosticFormattingOptions = .formatConsoleOutputForTools
+            
+            let conflictingFlagsConvert = try Docc.Convert.parse(["--experimental-transform-for-static-hosting-with-content", "--no-transform-for-static-hosting"])
+            XCTAssertEqual(conflictingFlagsConvert.experimentalTransformForStaticHostingWithContent, true)
+            XCTAssertEqual(conflictingFlagsConvert.transformForStaticHosting, true)
+            
+            XCTAssertEqual(logStorage.text.trimmingCharacters(in: .whitespacesAndNewlines), """
+            warning: Passing '--experimental-transform-for-static-hosting-with-content' also implies '--transform-for-static-hosting'. Passing '--no-transform-for-static-hosting' has no effect.
+            """)
+        }
+    }
 
     // This test calls ``ConvertOptions.infoPlistFallbacks._unusedVersionForBackwardsCompatibility`` which is deprecated.
     // Deprecating the test silences the deprecation warning when running the tests. It doesn't skip the test.
diff --git a/Tests/DocCCommandLineTests/FileWritingHTMLContentConsumerTests.swift b/Tests/DocCCommandLineTests/FileWritingHTMLContentConsumerTests.swift
diff --git a/Tests/DocCCommandLineTests/StaticHostingWithContentTests.swift b/Tests/DocCCommandLineTests/StaticHostingWithContentTests.swift
diff --git a/features.json b/features.json

Original file line number	Diff line number	Diff line change
`@@ -237,7 +237,7 @@ struct ConvertFileWritingConsumer: ConvertOutputConsumer, ExternalNodeConsumer {`
`237`	`237`	`let template = "<template id=\"\(id.rawValue)\">\(templateContents)</template>"`
`238`	`238`	`var newIndexContents = indexContents`
`239`	`239`	`newIndexContents.replaceSubrange(bodyTagRange, with: indexContents[bodyTagRange] + template)`
`240`		`- try newIndexContents.write(to: index, atomically: true, encoding: .utf8)`
	`240`	`+ try fileManager.createFile(at: index, contents: Data(newIndexContents.utf8))`
`241`	`241`	`}`
`242`	`242`
`243`	`243`	`/// File name for the documentation coverage file emitted during conversion.`