Loadtest E2Es and monitor memory for leak detection

.github/workflows/bench.yml

@@ -11,7 +11,6 @@ concurrency:
 
 env:
   NODE_NO_WARNINGS: 1
-  CI: true
 
 jobs:
   bench:
@@ -22,7 +21,7 @@ jobs:
           - 10
           - 100
           - 1000
-    name: Benchmark / ${{matrix.e2e_runner}} / ${{matrix.products_size}} items
+    name: ${{matrix.e2e_runner}} / ${{matrix.products_size}} items
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -36,4 +35,3 @@ jobs:
         env:
           PRODUCTS_SIZE: ${{matrix.products_size}}
           E2E_GATEWAY_RUNNER: ${{matrix.e2e_runner}}
-          CI: true

.github/workflows/memtest.yml

@@ -0,0 +1,44 @@
+name: Memtest
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  NODE_NO_WARNINGS: 1
+  K6_VERSION: v0.56.0
+
+jobs:
+  memtest:
+    strategy:
+      matrix:
+        e2e_runner:
+          - node
+          # - bun TODO: get memory snaps and heap sampling for bun. is it even necessary?
+    name: ${{matrix.e2e_runner}}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install k6
+        run: |
+          mkdir -p "$HOME/.local/bin"
+          cd "$HOME/.local/bin"
+          curl https://github.com/grafana/k6/releases/download/${{ env.K6_VERSION }}/k6-${{ env.K6_VERSION }}-linux-amd64.tar.gz -L | tar xvz --strip-components 1
+          echo "$PWD" >> $GITHUB_PATH
+      - name: Set up env
+        uses: the-guild-org/shared-config/setup@v1
+        with:
+          # TODO: should we test more node versions? we usually always recommend upgrading to
+          #       latest when people suspect leaks - latest is always the most stable
+          node-version-file: .node-version
+      - name: Test
+        run: yarn test:mem
+        # TODO: publish heap allocation sampling profile to artifact
+        env:
+          E2E_GATEWAY_RUNNER: ${{matrix.e2e_runner}}

.github/workflows/release.yml

@@ -175,6 +175,8 @@ jobs:
         uses: the-guild-org/shared-config/setup@v1
         with:
           node-version-file: .node-version
+          # we skip-build on ubuntu arm because the "canvas" package wont build and there are no prebuilts
+          install-command: ${{ matrix.os == 'ubuntu-24.04-arm' && 'yarn install --immutable --mode=skip-build' || '' }}
       - name: Build
         run: yarn build
       - name: Bundle

.gitignore

@@ -14,3 +14,7 @@ hive-gateway
 /examples/**/*/supergraph.graphql
 .helix/config.toml
 .helix/languages.toml
+/e2e/**/*/memtest-memory-usage_*.svg
+/e2e/**/*/*.heapsnapshot
+/e2e/**/*/*.heapprofile
+/e2e/**/*/loadtest.out

DEPS_RESOLUTIONS_NOTES.md

@@ -14,3 +14,8 @@ Here we collect reasons and write explanations about why some resolutions or pat
 ### vitest-tsconfig-paths
 
 1. Resolve tsconfig paths in modules that have been [inlined](https://vitest.dev/config/#server-deps-inline).
+
+### @memlab/core
+
+1. Define package.json#export for `@memlab/core/Types`
+1. Define package.json#export for `@memlab/core/Utils`

e2e/auto-type-merging/auto-type-merging.memtest.ts

@@ -0,0 +1,39 @@
+import { createTenv, type Container } from '@internal/e2e';
+import { memtest } from '@internal/perf/memtest';
+import { beforeAll } from 'vitest';
+
+const cwd = __dirname;
+const { service, gateway, container } = createTenv(cwd);
+
+let petstore!: Container;
+beforeAll(async () => {
+  petstore = await container({
+    name: 'petstore',
+    image: 'swaggerapi/petstore3:1.0.7',
+    containerPort: 8080,
+    healthcheck: ['CMD-SHELL', 'wget --spider http://localhost:8080'],
+  });
+});
+
+memtest(
+  {
+    cwd,
+    query: /* GraphQL */ `
+      query GetPet {
+        getPetById(petId: 1) {
+          __typename
+          id
+          name
+          vaccinated
+        }
+      }
+    `,
+  },
+  async () =>
+    gateway({
+      supergraph: {
+        with: 'mesh',
+        services: [petstore, await service('vaccination')],
+      },
+    }),
+);

e2e/cloudflare-workers/cloudflare-workers.e2e.ts

@@ -78,18 +78,18 @@ describe.skipIf(gatewayRunner !== 'node')('Cloudflare Workers', () => {
     OTLP_SERVICE_NAME: string;
   }) {
     const port = await getAvailablePort();
-    await spawn('yarn wrangler', {
-      args: [
-        'dev',
-        '--port',
-        port.toString(),
-        '--var',
-        'OTLP_EXPORTER_URL:' + env.OTLP_EXPORTER_URL,
-        '--var',
-        'OTLP_SERVICE_NAME:' + env.OTLP_SERVICE_NAME,
-        ...(isDebug() ? ['--var', 'DEBUG:1'] : []),
-      ],
-    });
+    await spawn([
+      'yarn',
+      'wrangler',
+      'dev',
+      '--port',
+      port.toString(),
+      '--var',
+      'OTLP_EXPORTER_URL:' + env.OTLP_EXPORTER_URL,
+      '--var',
+      'OTLP_SERVICE_NAME:' + env.OTLP_SERVICE_NAME,
+      ...(isDebug() ? ['--var', 'DEBUG:1'] : []),
+    ]);
     const hostname = await getLocalhost(port);
     return {
       url: `${hostname}:${port}`,

e2e/federation-example/federation-example.memtest.ts

@@ -0,0 +1,18 @@
+import { createExampleSetup, createTenv } from '@internal/e2e';
+import { memtest } from '@internal/perf/memtest';
+
+const cwd = __dirname;
+
+const { gateway } = createTenv(cwd);
+const { supergraph, query } = createExampleSetup(cwd);
+
+memtest(
+  {
+    cwd,
+    query,
+  },
+  async () =>
+    gateway({
+      supergraph: await supergraph(),
+    }),
+);

e2e/federation-mixed/federation-mixed.memtest.ts

@@ -0,0 +1,26 @@
+import { createExampleSetup, createTenv } from '@internal/e2e';
+import { memtest } from '@internal/perf/memtest';
+
+const cwd = __dirname;
+
+const { service, gateway } = createTenv(cwd);
+const exampleSetup = createExampleSetup(cwd);
+
+memtest(
+  {
+    cwd,
+    query: exampleSetup.query,
+  },
+  async () =>
+    gateway({
+      supergraph: {
+        with: 'mesh',
+        services: [
+          await service('accounts'),
+          await exampleSetup.service('inventory'),
+          await exampleSetup.service('products'),
+          await exampleSetup.service('reviews'),
+        ],
+      },
+    }),
+);

e2e/federation-subscriptions-passthrough/federation-subscriptions-passthrough.memtest.ts

@@ -0,0 +1,79 @@
+import { createTenv } from '@internal/e2e';
+import { memtest } from '@internal/perf/memtest';
+import { getAvailablePort } from '@internal/testing';
+import { describe } from 'vitest';
+
+const cwd = __dirname;
+
+const { service, gateway } = createTenv(cwd);
+
+describe('upstream subscriptions via websockets', () => {
+  memtest(
+    {
+      cwd,
+      query: /* GraphQL */ `
+        subscription {
+          newProduct {
+            id
+            name
+            price
+            reviews {
+              score
+            }
+          }
+        }
+      `,
+    },
+    async () =>
+      gateway({
+        supergraph: {
+          with: 'apollo',
+          services: [
+            await service('products', { env: { MEMTEST: 1 } }),
+            await service('reviews'),
+          ],
+        },
+      }),
+  );
+});
+
+describe('upstream subscriptions via http callbacks', () => {
+  memtest(
+    {
+      cwd,
+      query: /* GraphQL */ `
+        subscription {
+          newReview {
+            score
+          }
+        }
+      `,
+      expectedHeavyFrame: (frame) =>
+        // these frames are not leaks and have been confirmed to be stable analysing the heap snapshots (they do allocate a lot, but they all of their memory gets freed)
+        [
+          'delete',
+          'get pathname',
+          'onRequest',
+          'Repeater.next',
+          'Set',
+        ].includes(frame.name),
+    },
+    async () => {
+      const availablePort = await getAvailablePort();
+      const publicUrl = `http://0.0.0.0:${availablePort}`;
+      return gateway({
+        supergraph: {
+          with: 'apollo',
+          services: [
+            await service('products', { env: { MEMTEST: 1 } }),
+            await service('reviews'),
+          ],
+        },
+        port: availablePort,
+        env: {
+          PUBLIC_URL: publicUrl,
+        },
+      });
+    },
+  );
+});

e2e/federation-subscriptions-passthrough/services/products/server.ts

@@ -53,6 +53,13 @@ const schema = buildSubgraphSchema([
               clearInterval(interval);
             }),
         },
+        newProduct: {
+          async *subscribe() {
+            for (const product of products) {
+              yield { newProduct: product };
+            }
+          },
+        },
       },
     },
   },
@@ -76,6 +83,10 @@ const graphqlWsServer = useServer(
         console.error('Multiple WebSocket connections attempted');
         process.exit(1);
       }
+      if (process.env['MEMTEST']) {
+        // no need to authenticate in memtests
+        return true;
+      }
       // make sure the authorization header is propagated by the gateway
       if (connectionParams?.['token'] !== TOKEN) {
         return false;

e2e/federation-subscriptions-passthrough/services/products/typeDefs.graphql

@@ -4,6 +4,7 @@ extend type Query {
 
 type Subscription {
   productPriceChanged: Product!
+  newProduct: Product!
 }
 
 type Product @key(fields: "id") {

e2e/federation-subscriptions-passthrough/services/reviews/server.ts

@@ -26,6 +26,13 @@ const schema = buildSubgraphSchema([
           },
           resolve: (count: number) => count,
         },
+        newReview: {
+          async *subscribe() {
+            for (const review of reviews) {
+              yield { newReview: review };
+            }
+          },
+        },
       },
     },
   },

e2e/federation-subscriptions-passthrough/services/reviews/typeDefs.graphql

@@ -9,4 +9,5 @@ type Review {
 
 type Subscription {
   countdown(from: Int!): Int
+  newReview: Review!
 }

e2e/interface-additional-resolvers/interface-additional-resolvers.memtest.ts

@@ -0,0 +1,35 @@
+import { createTenv } from '@internal/e2e';
+import { memtest } from '@internal/perf/memtest';
+
+const cwd = __dirname;
+
+const { gateway, service } = createTenv(cwd);
+
+memtest(
+  {
+    cwd,
+    query: /* GraphQL */ `
+      query {
+        node(id: "1") {
+          id
+          ... on User {
+            name
+          }
+          self {
+            id
+            ... on User {
+              name
+            }
+          }
+        }
+      }
+    `,
+  },
+  async () =>
+    await gateway({
+      supergraph: {
+        with: 'mesh',
+        services: [await service('Test')],
+      },
+    }),
+);

e2e/nestjs/nestjs.e2e.ts

@@ -6,6 +6,7 @@ import { expect, it } from 'vitest';
 const { service } = createTenv(__dirname);
 const { supergraph, query, result } = createExampleSetup(__dirname);
 
+// TODO: run tests without needing to build the project
 it.todo('executes the query', async () => {
   const supergraphPath = await supergraph();
   const { port } = await service('nestjs', {

e2e/opentelemetry/gateway.config.ts

@@ -51,5 +51,10 @@ export const gatewayConfig = defineConfig({
     ],
     serviceName: process.env['OTLP_SERVICE_NAME'],
   },
-  plugins: () => [useOnFetchTracer()],
+  plugins: () =>
+    process.env['MEMTEST']
+      ? [
+          // disable the plugin in memtests because the upstreamCallHeaders will grew forever reporting a false positive leak
+        ]
+      : [useOnFetchTracer()],
 });