Merge pull request #8072 from QwikDev/fix-ssg-windows

fix: ssg on windows

Author: wmertens

.github/workflows/ci.yml

@@ -682,10 +682,10 @@ jobs:
           # too slow and flaky. Perhaps better in v2?
           # - host: ubuntu-latest
           #   browser: firefox
+          - host: windows-latest
+            browser: chromium
           - host: macos-latest
             browser: webkit
-          # - host: windows-latest
-          #   browser: chromium
 
     runs-on: ${{ matrix.settings.host }}
 

e2e/adapters-e2e/playwright.config.ts

@@ -44,5 +44,6 @@ export default defineConfig({
     port: 3000,
     stdout: 'pipe',
     reuseExistingServer: !process.env.CI,
+    timeout: 120000,
   },
 });

packages/qwik-city/src/adapters/shared/vite/index.ts

@@ -202,34 +202,6 @@ export function viteAdapter(opts: ViteAdapterPluginOptions) {
                 `\n==============================================`
             );
           }
-          if (opts.ssg !== null) {
-            /**
-             * HACK: for some reason the build hangs after SSG. `why-is-node-running` shows 4
-             * culprits:
-             *
-             * ```
-             * There are 4 handle(s) keeping the process running.
-             *
-             * # CustomGC
-             * ./node_modules/.pnpm/[email protected]/node_modules/lightningcss/node/index.js:20 - module.exports = require(`lightningcss-${parts.join('-')}`);
-             *
-             * # CustomGC
-             * ./node_modules/.pnpm/@[email protected]/node_modules/@tailwindcss/oxide/index.js:229 - return require('@tailwindcss/oxide-linux-x64-gnu')
-             *
-             * # Timeout
-             * node_modules/.vite-temp/vite.config.timestamp-1755270314169-a2a97ad5233f9.mjs:357
-             * ./node_modules/.pnpm/[email protected]_@[email protected][email protected][email protected][email protected][email protected][email protected]/node_modules/vite/dist/node/chunks/dep-CMEinpL-.js:36657 - return (await import(pathToFileURL(tempFileName).href)).default;
-             *
-             * # CustomGC
-             * ./packages/qwik/dist/optimizer.mjs:1328 - const mod2 = module.default.createRequire(import.meta.url)(`../bindings/${triple.platformArchABI}`);
-             * ```
-             *
-             * For now, we'll force exit the process after SSG with some delay.
-             */
-            setTimeout(() => {
-              process.exit(0);
-            }, 5000).unref();
-          }
         }
       },
     },

packages/qwik-city/src/static/main-thread.ts

@@ -80,13 +80,17 @@ export async function mainThread(sys: System) {
         while (!isCompleted && main.hasAvailableWorker() && queue.length > 0) {
           const staticRoute = queue.shift();
           if (staticRoute) {
-            render(staticRoute);
+            render(staticRoute).catch((e) => {
+              console.error(`render failed for ${staticRoute.pathname}`, e);
+            });
           }
         }
 
         if (!isCompleted && isRoutesLoaded && queue.length === 0 && active.size === 0) {
           isCompleted = true;
-          completed();
+          completed().catch((e) => {
+            console.error('SSG completion failed', e);
+          });
         }
       };
 
@@ -134,6 +138,7 @@ export async function mainThread(sys: System) {
 
           flushQueue();
         } catch (e) {
+          console.error(`render failed for ${staticRoute.pathname}`, e);
           isCompleted = true;
           reject(e);
         }
@@ -216,8 +221,12 @@ export async function mainThread(sys: System) {
         flushQueue();
       };
 
-      loadStaticRoutes();
+      loadStaticRoutes().catch((e) => {
+        console.error('SSG route loading failed', e);
+        reject(e);
+      });
     } catch (e) {
+      console.error('SSG main thread failed', e);
       reject(e);
     }
   });
@@ -244,6 +253,6 @@ function validateOptions(opts: StaticGenerateOptions) {
   try {
     new URL(siteOrigin);
   } catch (e) {
-    throw new Error(`Invalid "origin": ${e}`);
+    throw new Error(`Invalid "origin"`, { cause: e as Error });
   }
 }

packages/qwik-city/src/static/node/index.ts

@@ -1,6 +1,6 @@
 import type { StaticGenerateOptions } from '../types';
 import { createSystem } from './node-system';
-import { isMainThread, workerData } from 'node:worker_threads';
+import { isMainThread, workerData, threadId } from 'node:worker_threads';
 import { mainThread } from '../main-thread';
 import { workerThread } from '../worker-thread';
 
@@ -15,9 +15,20 @@ export async function generate(opts: StaticGenerateOptions) {
 }
 
 if (!isMainThread && workerData) {
+  const opts = workerData as StaticGenerateOptions;
   (async () => {
-    // self initializing worker thread with workerData
-    const sys = await createSystem(workerData);
-    await workerThread(sys);
-  })();
+    try {
+      if (opts.log === 'debug') {
+        // eslint-disable-next-line no-console
+        console.debug(`Worker thread starting (ID: ${threadId})`);
+      }
+      // self initializing worker thread with workerData
+      const sys = await createSystem(opts, threadId);
+      await workerThread(sys);
+    } catch (error) {
+      console.error(`Error occurred in worker thread (ID: ${threadId}): ${error}`);
+    }
+  })().catch((e) => {
+    console.error(e);
+  });
 }

packages/qwik-city/src/static/node/node-main.ts

@@ -10,10 +10,9 @@ import type {
 import fs from 'node:fs';
 import { cpus as nodeCpus } from 'node:os';
 import { Worker } from 'node:worker_threads';
-import { isAbsolute, resolve } from 'node:path';
+import { dirname, extname, isAbsolute, join, resolve } from 'node:path';
 import { ensureDir } from './node-system';
 import { normalizePath } from '../../utils/fs';
-import { createSingleThreadWorker } from '../worker-thread';
 
 export async function createNodeMainProcess(sys: System, opts: StaticGenerateOptions) {
   const ssgWorkers: StaticGeneratorWorker[] = [];
@@ -51,45 +50,34 @@ export async function createNodeMainProcess(sys: System, opts: StaticGenerateOpt
     }
   }
 
-  const singleThreadWorker = await createSingleThreadWorker(sys);
-
-  const createWorker = (workerIndex: number) => {
-    if (workerIndex === 0) {
-      // same thread worker, don't start a new process
-      const ssgSameThreadWorker: StaticGeneratorWorker = {
-        activeTasks: 0,
-        totalTasks: 0,
-
-        render: async (staticRoute) => {
-          ssgSameThreadWorker.activeTasks++;
-          ssgSameThreadWorker.totalTasks++;
-          const result = await singleThreadWorker(staticRoute);
-          ssgSameThreadWorker.activeTasks--;
-          return result;
-        },
-
-        terminate: async () => {},
-      };
-      return ssgSameThreadWorker;
-    }
-
+  const createWorker = () => {
     let terminateResolve: (() => void) | null = null;
     const mainTasks = new Map<string, WorkerMainTask>();
 
     let workerFilePath: string | URL;
+    let terminateTimeout: number | null = null;
 
+    // Launch the worker using the package's index module, which bootstraps the worker thread.
     if (typeof __filename === 'string') {
-      workerFilePath = __filename;
+      // CommonJS path
+      const ext = extname(__filename) || '.js';
+      workerFilePath = join(dirname(__filename), `index${ext}`);
     } else {
-      workerFilePath = import.meta.url;
-    }
+      // ESM path (import.meta.url)
+      const thisUrl = new URL(import.meta.url);
+      const pathname = thisUrl.pathname || '';
+      let ext = '.js';
+      if (pathname.endsWith('.ts')) {
+        ext = '.ts';
+      } else if (pathname.endsWith('.mjs')) {
+        ext = '.mjs';
+      }
 
-    if (typeof workerFilePath === 'string' && workerFilePath.startsWith('file://')) {
-      workerFilePath = new URL(workerFilePath);
+      workerFilePath = new URL(`./index${ext}`, thisUrl);
     }
 
     const nodeWorker = new Worker(workerFilePath, { workerData: opts });
-
+    nodeWorker.unref();
     const ssgWorker: StaticGeneratorWorker = {
       activeTasks: 0,
       totalTasks: 0,
@@ -116,7 +104,9 @@ export async function createNodeMainProcess(sys: System, opts: StaticGenerateOpt
           terminateResolve = resolve;
           nodeWorker.postMessage(msg);
         });
-        await nodeWorker.terminate();
+        terminateTimeout = setTimeout(async () => {
+          await nodeWorker.terminate();
+        }, 1000) as unknown as number;
       },
     };
 
@@ -146,7 +136,11 @@ export async function createNodeMainProcess(sys: System, opts: StaticGenerateOpt
     });
 
     nodeWorker.on('exit', (code) => {
-      if (code !== 1) {
+      if (terminateTimeout) {
+        clearTimeout(terminateTimeout);
+        terminateTimeout = null;
+      }
+      if (code !== 0) {
         console.error(`worker exit ${code}`);
       }
     });
@@ -200,9 +194,15 @@ export async function createNodeMainProcess(sys: System, opts: StaticGenerateOpt
         console.error(e);
       }
     }
-    ssgWorkers.length = 0;
 
     await Promise.all(promises);
+    ssgWorkers.length = 0;
+
+    // On Windows, give extra time for all workers to fully exit
+    // This prevents resource conflicts in back-to-back builds
+    if (process.platform === 'win32') {
+      await new Promise((resolve) => setTimeout(resolve, 300));
+    }
   };
 
   if (sitemapOutFile) {
@@ -214,7 +214,11 @@ export async function createNodeMainProcess(sys: System, opts: StaticGenerateOpt
   }
 
   for (let i = 0; i < maxWorkers; i++) {
-    ssgWorkers.push(createWorker(i));
+    ssgWorkers.push(createWorker());
+    // On Windows, add delay between worker creation to avoid resource contention
+    if (process.platform === 'win32' && i < maxWorkers - 1) {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    }
   }
 
   const mainCtx: MainContext = {

packages/qwik-city/src/static/node/node-system.ts

@@ -2,15 +2,15 @@
 import type { StaticGenerateOptions, System } from '../types';
 import fs from 'node:fs';
 import { dirname, join } from 'node:path';
-import { patchGlobalThis } from '../../middleware/node/node-fetch';
 import { createNodeMainProcess } from './node-main';
 import { createNodeWorkerProcess } from './node-worker';
 import { normalizePath } from '../../utils/fs';
 
 /** @public */
-export async function createSystem(opts: StaticGenerateOptions) {
-  patchGlobalThis();
-
+export async function createSystem(
+  opts: StaticGenerateOptions,
+  threadId?: number
+): Promise<System> {
   const createWriteStream = (filePath: string) => {
     return fs.createWriteStream(filePath, {
       flags: 'w',
@@ -29,6 +29,13 @@ export async function createSystem(opts: StaticGenerateOptions) {
   };
 
   const createLogger = async () => {
+    if (threadId !== undefined) {
+      return {
+        debug: opts.log === 'debug' ? console.debug.bind(console, `[${threadId}]`) : () => {},
+        error: console.error.bind(console, `[${threadId}]`),
+        info: console.info.bind(console, `[${threadId}]`),
+      };
+    }
     return {
       debug: opts.log === 'debug' ? console.debug.bind(console) : () => {},
       error: console.error.bind(console),

packages/qwik-city/src/static/node/node-worker.ts

@@ -6,5 +6,8 @@ export async function createNodeWorkerProcess(
 ) {
   parentPort?.on('message', async (msg: WorkerInputMessage) => {
     parentPort?.postMessage(await onMessage(msg));
+    if (msg.type === 'close') {
+      parentPort?.close();
+    }
   });
 }

packages/qwik-city/src/static/types.ts

@@ -6,7 +6,7 @@ export interface System {
   createMainProcess: (() => Promise<MainContext>) | null;
   createWorkerProcess: (
     onMessage: (msg: WorkerInputMessage) => Promise<WorkerOutputMessage>
-  ) => void;
+  ) => void | Promise<void>;
   createLogger: () => Promise<Logger>;
   getOptions: () => StaticGenerateOptions;
   ensureDir: (filePath: string) => Promise<void>;