Loadtest E2Es and monitor memory for leak detection (#611)

Co-authored-by: theguild-bot <[email protected]>

Author: enisdenjo

.github/workflows/bench.yml

@@ -11,7 +11,6 @@ concurrency:
 
 env:
   NODE_NO_WARNINGS: 1
-  CI: true
 
 jobs:
   bench:
@@ -22,7 +21,7 @@ jobs:
           - 10
           - 100
           - 1000
-    name: Benchmark / ${{matrix.e2e_runner}} / ${{matrix.products_size}} items
+    name: ${{matrix.e2e_runner}} / ${{matrix.products_size}} items
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -36,4 +35,3 @@ jobs:
         env:
           PRODUCTS_SIZE: ${{matrix.products_size}}
           E2E_GATEWAY_RUNNER: ${{matrix.e2e_runner}}
-          CI: true

.github/workflows/memtest.yml

@@ -0,0 +1,44 @@
+name: Memtest
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  NODE_NO_WARNINGS: 1
+  K6_VERSION: v0.56.0
+
+jobs:
+  memtest:
+    strategy:
+      matrix:
+        e2e_runner:
+          - node
+          # - bun TODO: get memory snaps and heap sampling for bun. is it even necessary?
+    name: ${{matrix.e2e_runner}}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install k6
+        run: |
+          mkdir -p "$HOME/.local/bin"
+          cd "$HOME/.local/bin"
+          curl https://github.com/grafana/k6/releases/download/${{ env.K6_VERSION }}/k6-${{ env.K6_VERSION }}-linux-amd64.tar.gz -L | tar xvz --strip-components 1
+          echo "$PWD" >> $GITHUB_PATH
+      - name: Set up env
+        uses: the-guild-org/shared-config/setup@v1
+        with:
+          # TODO: should we test more node versions? we usually always recommend upgrading to
+          #       latest when people suspect leaks - latest is always the most stable
+          node-version-file: .node-version
+      - name: Test
+        run: yarn test:mem
+        # TODO: publish heap allocation sampling profile to artifact
+        env:
+          E2E_GATEWAY_RUNNER: ${{matrix.e2e_runner}}

.github/workflows/release.yml

@@ -175,6 +175,8 @@ jobs:
         uses: the-guild-org/shared-config/setup@v1
         with:
           node-version-file: .node-version
+          # we skip-build on ubuntu arm because the "canvas" package wont build and there are no prebuilts
+          install-command: ${{ matrix.os == 'ubuntu-24.04-arm' && 'yarn install --immutable --mode=skip-build' || '' }}
       - name: Build
         run: yarn build
       - name: Bundle

.gitignore

@@ -14,3 +14,7 @@ hive-gateway
 /examples/**/*/supergraph.graphql
 .helix/config.toml
 .helix/languages.toml
+/e2e/**/*/memtest-memory-usage_*.svg
+/e2e/**/*/*.heapsnapshot
+/e2e/**/*/*.heapprofile
+/e2e/**/*/loadtest.out

DEPS_RESOLUTIONS_NOTES.md

@@ -14,3 +14,8 @@ Here we collect reasons and write explanations about why some resolutions or pat
 ### vitest-tsconfig-paths
 
 1. Resolve tsconfig paths in modules that have been [inlined](https://vitest.dev/config/#server-deps-inline).
+
+### @memlab/core
+
+1. Define package.json#export for `@memlab/core/Types`
+1. Define package.json#export for `@memlab/core/Utils`

e2e/auto-type-merging/auto-type-merging.memtest.ts

@@ -0,0 +1,39 @@
+import { createTenv, type Container } from '@internal/e2e';
+import { memtest } from '@internal/perf/memtest';
+import { beforeAll } from 'vitest';
+
+const cwd = __dirname;
+const { service, gateway, container } = createTenv(cwd);
+
+let petstore!: Container;
+beforeAll(async () => {
+  petstore = await container({
+    name: 'petstore',
+    image: 'swaggerapi/petstore3:1.0.7',
+    containerPort: 8080,
+    healthcheck: ['CMD-SHELL', 'wget --spider http://localhost:8080'],
+  });
+});
+
+memtest(
+  {
+    cwd,
+    query: /* GraphQL */ `
+      query GetPet {
+        getPetById(petId: 1) {
+          __typename
+          id
+          name
+          vaccinated
+        }
+      }
+    `,
+  },
+  async () =>
+    gateway({
+      supergraph: {
+        with: 'mesh',
+        services: [petstore, await service('vaccination')],
+      },
+    }),
+);

e2e/cloudflare-workers/cloudflare-workers.e2e.ts

@@ -78,18 +78,18 @@ describe.skipIf(gatewayRunner !== 'node')('Cloudflare Workers', () => {
     OTLP_SERVICE_NAME: string;
   }) {
     const port = await getAvailablePort();
-    await spawn('yarn wrangler', {
-      args: [
-        'dev',
-        '--port',
-        port.toString(),
-        '--var',
-        'OTLP_EXPORTER_URL:' + env.OTLP_EXPORTER_URL,
-        '--var',
-        'OTLP_SERVICE_NAME:' + env.OTLP_SERVICE_NAME,
-        ...(isDebug() ? ['--var', 'DEBUG:1'] : []),
-      ],
-    });
+    await spawn([
+      'yarn',
+      'wrangler',
+      'dev',
+      '--port',
+      port.toString(),
+      '--var',
+      'OTLP_EXPORTER_URL:' + env.OTLP_EXPORTER_URL,
+      '--var',
+      'OTLP_SERVICE_NAME:' + env.OTLP_SERVICE_NAME,
+      ...(isDebug() ? ['--var', 'DEBUG:1'] : []),
+    ]);
     const hostname = await getLocalhost(port);
     return {
       url: `${hostname}:${port}`,

e2e/federation-example/federation-example.memtest.ts

@@ -0,0 +1,18 @@
+import { createExampleSetup, createTenv } from '@internal/e2e';
+import { memtest } from '@internal/perf/memtest';
+
+const cwd = __dirname;
+
+const { gateway } = createTenv(cwd);
+const { supergraph, query } = createExampleSetup(cwd);
+
+memtest(
+  {
+    cwd,
+    query,
+  },
+  async () =>
+    gateway({
+      supergraph: await supergraph(),
+    }),
+);

e2e/federation-mixed/federation-mixed.memtest.ts

@@ -0,0 +1,26 @@
+import { createExampleSetup, createTenv } from '@internal/e2e';
+import { memtest } from '@internal/perf/memtest';
+
+const cwd = __dirname;
+
+const { service, gateway } = createTenv(cwd);
+const exampleSetup = createExampleSetup(cwd);
+
+memtest(
+  {
+    cwd,
+    query: exampleSetup.query,
+  },
+  async () =>
+    gateway({
+      supergraph: {
+        with: 'mesh',
+        services: [
+          await service('accounts'),
+          await exampleSetup.service('inventory'),
+          await exampleSetup.service('products'),
+          await exampleSetup.service('reviews'),
+        ],
+      },
+    }),
+);

e2e/federation-subscriptions-passthrough/federation-subscriptions-passthrough.memtest.ts

@@ -0,0 +1,79 @@
+import { createTenv } from '@internal/e2e';
+import { memtest } from '@internal/perf/memtest';
+import { getAvailablePort } from '@internal/testing';
+import { describe } from 'vitest';
+
+const cwd = __dirname;
+
+const { service, gateway } = createTenv(cwd);
+
+describe('upstream subscriptions via websockets', () => {
+  memtest(
+    {
+      cwd,
+      query: /* GraphQL */ `
+        subscription {
+          newProduct {
+            id
+            name
+            price
+            reviews {
+              score
+            }
+          }
+        }
+      `,
+    },
+    async () =>
+      gateway({
+        supergraph: {
+          with: 'apollo',
+          services: [
+            await service('products', { env: { MEMTEST: 1 } }),
+            await service('reviews'),
+          ],
+        },
+      }),
+  );
+});
+
+describe('upstream subscriptions via http callbacks', () => {
+  memtest(
+    {
+      cwd,
+      query: /* GraphQL */ `
+        subscription {
+          newReview {
+            score
+          }
+        }
+      `,
+      expectedHeavyFrame: (frame) =>
+        // these frames are not leaks and have been confirmed to be stable analysing the heap snapshots (they do allocate a lot, but they all of their memory gets freed)
+        [
+          'delete',
+          'get pathname',
+          'onRequest',
+          'Repeater.next',
+          'Set',
+        ].includes(frame.name),
+    },
+    async () => {
+      const availablePort = await getAvailablePort();
+      const publicUrl = `http://0.0.0.0:${availablePort}`;
+      return gateway({
+        supergraph: {
+          with: 'apollo',
+          services: [
+            await service('products', { env: { MEMTEST: 1 } }),
+            await service('reviews'),
+          ],
+        },
+        port: availablePort,
+        env: {
+          PUBLIC_URL: publicUrl,
+        },
+      });
+    },
+  );
+});