[DRAFT] libpglite

.github/workflows/build_and_test.yml

@@ -164,7 +164,7 @@ jobs:
         with:
           issue-number: ${{ github.event.pull_request.number }}
           comment-author: 'github-actions[bot]'
-          body-includes: '- PGlite:'
+          body-includes: '- PGlite with node:'
 
       - name: Create or update build outputs comment
         uses: peter-evans/create-or-update-comment@v4
@@ -174,7 +174,7 @@ jobs:
           comment-id: ${{ steps.fc.outputs.comment-id }}
           issue-number: ${{ github.event.pull_request.number }}
           body: |
-            - PGlite: ${{ steps.upload-pglite-package.outputs.artifact-url }}
+            - PGlite with node v${{ matrix.node }}: ${{ steps.upload-pglite-package.outputs.artifact-url }}
           edit-mode: append
 
   build-and-test-pglite-dependents:

packages/pglite-tools/src/pgDumpModFactory.ts

@@ -22,16 +22,16 @@ export interface PgDumpMod
   FS: FS
   WASM_PREFIX: string
   INITIAL_MEMORY: number
-  _set_read_write_cbs: (read_cb: number, write_cb: number) => void
+  _pgl_set_rw_cbs: (read_cb: number, write_cb: number) => void
   addFunction: (
     cb: (ptr: any, length: number) => void,
     signature: string,
   ) => number
   removeFunction: (f: number) => void
-  _main: (args: string[]) => number
   onExit: (status: number) => void
   print: (test: string) => void
   printErr: (text: string) => void
+  callMain: (args?: string[]) => number
 }
 
 type PgDumpFactory<T extends PgDumpMod = PgDumpMod> = (

packages/pglite-tools/src/pg_dump.ts

@@ -36,11 +36,10 @@ async function execPgDump({
   args: string[]
 }): Promise<ExecResult> {
   let pgdump_write, pgdump_read
-  let exitStatus = 0
+  let exitCode = 0
   let stderrOutput: string = ''
   let stdoutOutput: string = ''
   const emscriptenOpts: Partial<PgDumpMod> = {
-    arguments: args,
     noExitRuntime: false,
     print: (text) => {
       stdoutOutput += text
@@ -49,7 +48,7 @@ async function execPgDump({
       stderrOutput += text
     },
     onExit: (status: number) => {
-      exitStatus = status
+      exitCode = status
     },
     preRun: [
       (mod: PgDumpMod) => {
@@ -83,7 +82,7 @@ async function execPgDump({
             return length
           }, 'iii')
 
-          mod._set_read_write_cbs(pgdump_read, pgdump_write)
+          mod._pgl_set_rw_cbs(pgdump_read, pgdump_write)
           // default $HOME in emscripten is /home/web_user
           mod.FS.chmod('/home/web_user/.pgpass', 0o0600) // https://www.postgresql.org/docs/current/libpq-pgpass.html
         }
@@ -92,13 +91,14 @@ async function execPgDump({
   }
 
   const mod = await PgDumpModFactory(emscriptenOpts)
+  mod.callMain(args)
   let fileContents = ''
-  if (!exitStatus) {
+  if (!exitCode) {
     fileContents = mod.FS.readFile(dumpFilePath, { encoding: 'utf8' })
   }
 
   return {
-    exitCode: exitStatus,
+    exitCode,
     fileContents,
     stderr: stderrOutput,
     stdout: stdoutOutput,

packages/pglite/src/pglite.ts

@@ -76,14 +76,14 @@ export class PGlite
   #globalNotifyListeners = new Set<(channel: string, payload: string) => void>()
 
   // receive data from wasm
-  #pglite_write: number = -1
+  #pglite_socket_write: number = -1
 
   #currentResults: BackendMessage[] = []
   #currentThrowOnError: boolean = false
   #currentOnNotice: ((notice: NoticeMessage) => void) | undefined
 
   // send data to wasm
-  #pglite_read: number = -1
+  #pglite_socket_read: number = -1
   // buffer that holds the data to be sent to wasm
   #outputData: any = []
   // read index in the buffer
@@ -214,12 +214,12 @@ export class PGlite
     const extensionInitFns: Array<() => Promise<void>> = []
 
     const args = [
-      `PGDATA=${PGDATA}`,
-      `PREFIX=${WASM_PREFIX}`,
-      `PGUSER=${options.username ?? 'postgres'}`,
-      `PGDATABASE=${options.database ?? 'template1'}`,
-      'MODE=REACT',
-      'REPL=N',
+      // `PGDATA=${PGDATA}`,
+      // `PREFIX=${WASM_PREFIX}`,
+      // `PGUSER=${options.username ?? 'postgres'}`,
+      // `PGDATABASE=${options.database ?? 'template1'}`,
+      // 'MODE=REACT',
+      // 'REPL=N',
       // "-F", // Disable fsync (TODO: Only for in-memory mode?)
       ...(this.debug ? ['-d', this.debug.toString()] : []),
     ]
@@ -332,6 +332,20 @@ export class PGlite
           }
           mod.FS.registerDevice(devId, devOpt)
           mod.FS.mkdev('/dev/blob', devId)
+          // mod.FS.mkdir('/tmp') && mod.FS.chmod('/tmp', 0o700)
+        },
+        (mod: any) => {
+          mod.ENV.MODE = 'REACT'
+          mod.ENV.PGDATA = PGDATA
+          mod.ENV.PREFIX = WASM_PREFIX
+          mod.ENV.PGUSER = options.username ?? 'postgres'
+          mod.ENV.PGDATABASE = options.database ?? 'template1'
+          mod.ENV.LC_CTYPE = 'en_US.UTF-8'
+          mod.ENV.TZ = 'UTC'
+          mod.ENV.PGTZ = 'UTC'
+          mod.ENV.PGCLIENTENCODING = 'UTF8'
+          //   mod.ENV.PGDATABASE = 'template1'
+          // mod.ENV.PG_COLOR = 'always'
         },
       ],
     }
@@ -387,66 +401,73 @@ export class PGlite
     this.mod = await PostgresModFactory(emscriptenOpts)
 
     // set the write callback
-    this.#pglite_write = this.mod.addFunction((ptr: any, length: number) => {
-      let bytes
-      try {
-        bytes = this.mod!.HEAPU8.subarray(ptr, ptr + length)
-      } catch (e: any) {
-        console.error('error', e)
-        throw e
-      }
-      this.#protocolParser.parse(bytes, (msg) => {
-        this.#parse(msg)
-      })
-      if (this.#keepRawResponse) {
-        const copied = bytes.slice()
-
-        let requiredSize = this.#writeOffset + copied.length
-
-        if (requiredSize > this.#inputData.length) {
-          const newSize =
-            this.#inputData.length +
-            (this.#inputData.length >> 1) +
-            requiredSize
-          if (requiredSize > PGlite.MAX_BUFFER_SIZE) {
-            requiredSize = PGlite.MAX_BUFFER_SIZE
+    this.#pglite_socket_write = this.mod.addFunction(
+      (ptr: any, length: number) => {
+        let bytes
+        try {
+          bytes = this.mod!.HEAPU8.subarray(ptr, ptr + length)
+        } catch (e: any) {
+          console.error('error', e)
+          throw e
+        }
+        this.#protocolParser.parse(bytes, (msg) => {
+          this.#parse(msg)
+        })
+        if (this.#keepRawResponse) {
+          const copied = bytes.slice()
+          let requiredSize = this.#writeOffset + copied.length
+          if (requiredSize > this.#inputData.length) {
+            const newSize =
+              this.#inputData.length +
+              (this.#inputData.length >> 1) +
+              requiredSize
+            if (requiredSize > PGlite.MAX_BUFFER_SIZE) {
+              requiredSize = PGlite.MAX_BUFFER_SIZE
+            }
+            const newBuffer = new Uint8Array(newSize)
+            newBuffer.set(this.#inputData.subarray(0, this.#writeOffset))
+            this.#inputData = newBuffer
           }
-          const newBuffer = new Uint8Array(newSize)
-          newBuffer.set(this.#inputData.subarray(0, this.#writeOffset))
-          this.#inputData = newBuffer
+          this.#inputData.set(copied, this.#writeOffset)
+          this.#writeOffset += copied.length
+          return this.#inputData.length
         }
-
-        this.#inputData.set(copied, this.#writeOffset)
-        this.#writeOffset += copied.length
-
-        return this.#inputData.length
-      }
-      return length
-    }, 'iii')
+        return length
+      },
+      'iii',
+    )
 
     // set the read callback
-    this.#pglite_read = this.mod.addFunction((ptr: any, max_length: number) => {
-      // copy current data to wasm buffer
-      let length = this.#outputData.length - this.#readOffset
-      if (length > max_length) {
-        length = max_length
-      }
-      try {
-        this.mod!.HEAP8.set(
-          (this.#outputData as Uint8Array).subarray(
-            this.#readOffset,
-            this.#readOffset + length,
-          ),
-          ptr,
-        )
-        this.#readOffset += length
-      } catch (e) {
-        console.log(e)
-      }
-      return length
-    }, 'iii')
+    this.#pglite_socket_read = this.mod.addFunction(
+      (ptr: any, max_length: number) => {
+        // copy current data to wasm buffer
+        let length = this.#outputData.length - this.#readOffset
+        if (length > max_length) {
+          length = max_length
+        }
+        try {
+          this.mod!.HEAP8.set(
+            (this.#outputData as Uint8Array).subarray(
+              this.#readOffset,
+              this.#readOffset + length,
+            ),
+            ptr,
+          )
+          this.#readOffset += length
+        } catch (e) {
+          console.error(e)
+        }
+        return length
+      },
+      'iii',
+    )
+
+    this.mod._pgl_set_rw_cbs(
+      this.#pglite_socket_read,
+      this.#pglite_socket_write,
+    )
 
-    this.mod._set_read_write_cbs(this.#pglite_read, this.#pglite_write)
+    this.mod._pgl_startup(args)
 
     // Sync the filesystem from any previous store
     await this.fs!.initialSyncFs()
@@ -576,8 +597,8 @@ export class PGlite
     try {
       await this.execProtocol(serialize.end())
       this.mod!._pgl_shutdown()
-      this.mod!.removeFunction(this.#pglite_read)
-      this.mod!.removeFunction(this.#pglite_write)
+      this.mod!.removeFunction(this.#pglite_socket_read)
+      this.mod!.removeFunction(this.#pglite_socket_write)
     } catch (e) {
       const err = e as { name: string; status: number }
       if (err.name === 'ExitStatus' && err.status === 0) {
@@ -671,7 +692,7 @@ export class PGlite
     }
 
     // execute the message
-    mod._interactive_one(message.length, message[0])
+    mod._pgl_interactive_one(message.length, message[0])
 
     this.#outputData = []
 

packages/pglite/src/postgresMod.ts

@@ -20,21 +20,21 @@ export interface PostgresMod
   preRun: Array<{ (mod: PostgresMod): void }>
   postRun: Array<{ (mod: PostgresMod): void }>
   FS: FS
-  FD_BUFFER_MAX: number
   WASM_PREFIX: string
   INITIAL_MEMORY: number
   pg_extensions: Record<string, Promise<Blob | null>>
   _pgl_initdb: () => number
   _pgl_backend: () => void
   _pgl_shutdown: () => void
-  _interactive_write: (msgLength: number) => void
-  _interactive_one: (length: number, peek: number) => void
-  _set_read_write_cbs: (read_cb: number, write_cb: number) => void
+  _pgl_interactive_one: (length: number, peek: number) => void
+  _pgl_set_rw_cbs: (read_cb: number, write_cb: number) => void
+  _pgl_startup: (args?: string[]) => number
   addFunction: (
     cb: (ptr: any, length: number) => void,
     signature: string,
   ) => number
   removeFunction: (f: number) => void
+  callMain: (args?: string[]) => number
 }
 
 type PostgresFactory<T extends PostgresMod = PostgresMod> = (