test(core/protocols): add shape serde perf baselines

Author: kuhe

packages/core/src/submodules/protocols/cbor/CborCodec.spec.ts

@@ -0,0 +1,72 @@
+import { CborCodec } from "@smithy/core/cbor";
+import { describe, test as it } from "vitest";
+
+import { createNestingWidget, nestingWidget } from "../test-schema.spec";
+
+describe("performance baseline indicator", () => {
+  const codec = new CborCodec();
+  const serializer = codec.createSerializer();
+  const deserializer = codec.createDeserializer();
+
+  it("should serialize objects", () => {
+    const timings: string[] = [];
+    const objects = [];
+
+    // warmup
+    for (let i = 0; i < 13; ++i) {
+      const o = createNestingWidget(2 ** i);
+      objects.push(o);
+      serializer.write(nestingWidget, o);
+      serializer.flush();
+    }
+
+    for (let i = 0; i < objects.length; ++i) {
+      const o = objects[i];
+
+      const A = performance.now();
+      serializer.write(nestingWidget, o);
+      const serialization = serializer.flush();
+      const B = performance.now();
+
+      timings.push(
+        `${B - A} (byte length = ${serialization.byteLength}, ${serialization.byteLength / 1024 / (B - A)} kb/ms)`
+      );
+    }
+
+    /**
+     * No assertion here.
+     * In the initial dual-pass implementation,
+     * par time is 0 to 10ms for up to 233,000 bytes of CBOR. Up to 30 kb/ms. (kuhe's computer)
+     */
+    console.log("CborShapeSerializer performance timings", timings);
+  });
+
+  it("should deserialize bytes", async () => {
+    const timings: string[] = [];
+    const strings = [];
+
+    // warmup
+    for (let i = 0; i < 12; ++i) {
+      const o = createNestingWidget(2 ** i);
+      serializer.write(nestingWidget, o);
+      const json = serializer.flush();
+      strings.push(json);
+      await deserializer.read(nestingWidget, json);
+    }
+
+    for (const s of strings) {
+      const A = performance.now();
+      await deserializer.read(nestingWidget, s);
+      const B = performance.now();
+
+      timings.push(`${B - A} (byte length = ${s.byteLength}, ${s.byteLength / 1024 / (B - A)} kb/ms)`);
+    }
+
+    /**
+     * No assertion here.
+     * In the initial dual-pass implementation,
+     * par time is 0 to 3ms for up to 116000 bytes of CBOR. Up to 45 kb/ms. (kuhe's computer)
+     */
+    console.log("CborShapeDeserializer performance timings", timings);
+  });
+});

packages/core/src/submodules/protocols/json/JsonShapeDeserializer.spec.ts

@@ -2,8 +2,9 @@ import { NumericValue } from "@smithy/core/serde";
 import type { TimestampEpochSecondsSchema } from "@smithy/types";
 import { describe, expect, test as it } from "vitest";
 
-import { widget } from "../test-schema.spec";
+import { createNestingWidget, nestingWidget, widget } from "../test-schema.spec";
 import { JsonShapeDeserializer } from "./JsonShapeDeserializer";
+import { JsonShapeSerializer } from "./JsonShapeSerializer";
 
 describe(JsonShapeDeserializer.name, () => {
   let contextSourceAvailable = false;
@@ -153,4 +154,45 @@ describe(JsonShapeDeserializer.name, () => {
     expect(await deserializer.read(widget, JSON.stringify({ scalar: "-Infinity" }))).toEqual({ scalar: -Infinity });
     expect(await deserializer.read(widget, JSON.stringify({ scalar: "NaN" }))).toEqual({ scalar: NaN });
   });
+
+  describe("performance baseline indicator", () => {
+    const serializer = new JsonShapeSerializer({
+      jsonName: true,
+      timestampFormat: { default: 7 satisfies TimestampEpochSecondsSchema, useTrait: true },
+    });
+    serializer.setSerdeContext({
+      base64Encoder: (input: Uint8Array) => {
+        return Buffer.from(input).toString("base64");
+      },
+    } as any);
+
+    it("should deserialize JSON strings", async () => {
+      const timings: string[] = [];
+      const strings = [];
+
+      // warmup
+      for (let i = 0; i < 12; ++i) {
+        const o = createNestingWidget(2 ** i);
+        serializer.write(nestingWidget, o);
+        const json = serializer.flush();
+        strings.push(json);
+        await deserializer.read(nestingWidget, json);
+      }
+
+      for (const s of strings) {
+        const A = performance.now();
+        await deserializer.read(nestingWidget, s);
+        const B = performance.now();
+
+        timings.push(`${B - A} (JSON length = ${s.length}, ${s.length / 1024 / (B - A)} kb/ms)`);
+      }
+
+      /**
+       * No assertion here.
+       * In the initial dual-pass implementation,
+       * par time is 0 to 10ms for up to 161,000 chars of JSON. Up to 20 kb/ms. (kuhe's computer)
+       */
+      console.log("JsonShapeDeserializer performance timings", timings);
+    });
+  });
 });

packages/core/src/submodules/protocols/json/JsonShapeSerializer.spec.ts

@@ -2,21 +2,21 @@ import { NumericValue } from "@smithy/core/serde";
 import type { TimestampEpochSecondsSchema } from "@smithy/types";
 import { describe, expect, test as it } from "vitest";
 
-import { widget } from "../test-schema.spec";
+import { createNestingWidget, nestingWidget, widget } from "../test-schema.spec";
 import { JsonShapeSerializer } from "./JsonShapeSerializer";
 
 describe(JsonShapeSerializer.name, () => {
-  it("serializes data to JSON", async () => {
-    const serializer = new JsonShapeSerializer({
-      jsonName: true,
-      timestampFormat: { default: 7 satisfies TimestampEpochSecondsSchema, useTrait: true },
-    });
-    serializer.setSerdeContext({
-      base64Encoder: (input: Uint8Array) => {
-        return Buffer.from(input).toString("base64");
-      },
-    } as any);
+  const serializer = new JsonShapeSerializer({
+    jsonName: true,
+    timestampFormat: { default: 7 satisfies TimestampEpochSecondsSchema, useTrait: true },
+  });
+  serializer.setSerdeContext({
+    base64Encoder: (input: Uint8Array) => {
+      return Buffer.from(input).toString("base64");
+    },
+  } as any);
 
+  it("serializes data to JSON", async () => {
     const data = {
       timestamp: new Date(0),
       bigint: 10000000000000000000000054321n,
@@ -29,4 +29,39 @@ describe(JsonShapeSerializer.name, () => {
       `{"blob":"AAAAAQ==","timestamp":0,"bigint":10000000000000000000000054321,"bigdecimal":0.10000000000000000000000054321}`
     );
   });
+
+  describe("performance baseline indicator", () => {
+    it("should serialize objects", () => {
+      const timings: string[] = [];
+      const objects = [];
+
+      // warmup
+      for (let i = 0; i < 13; ++i) {
+        const o = createNestingWidget(2 ** i);
+        objects.push(o);
+        serializer.write(nestingWidget, o);
+        serializer.flush();
+      }
+
+      for (let i = 0; i < objects.length; ++i) {
+        const o = objects[i];
+
+        const A = performance.now();
+        serializer.write(nestingWidget, o);
+        const serialization = serializer.flush();
+        const B = performance.now();
+
+        timings.push(
+          `${B - A} (JSON length = ${serialization.length}, ${serialization.length / 1024 / (B - A)} kb/ms)`
+        );
+      }
+
+      /**
+       * No assertion here.
+       * In the initial dual-pass implementation,
+       * par time is 0 to 30ms for up to 323,000 chars of JSON. Up to 20 kb/ms. (kuhe's computer)
+       */
+      console.log("JsonShapeSerializer performance timings", timings);
+    });
+  });
 });

packages/core/src/submodules/protocols/test-schema.spec.ts

@@ -6,6 +6,8 @@ import type {
   StaticListSchema,
   StaticOperationSchema,
   StaticStructureSchema,
+  StringSchema,
+  TimestampDefaultSchema,
   TimestampEpochSecondsSchema,
 } from "@smithy/types";
 import { describe, test as it } from "vitest";
@@ -34,6 +36,28 @@ export const widget = [
   ],
 ] satisfies StaticStructureSchema;
 
+export const nestingWidget: StaticStructureSchema = [
+  3,
+  "ns",
+  "Struct",
+  0,
+  ["string", "date", "blob", "nested"],
+  [0 satisfies StringSchema, 4 satisfies TimestampDefaultSchema, 21 satisfies BlobSchema, () => nestingWidget],
+];
+
+export function createNestingWidget(nesting = 0) {
+  const object = {
+    string: "hello, world",
+    date: new Date(),
+    blob: new Uint8Array([0, 1, 2, 3, 4, 5, 6, 7]),
+    nested: undefined,
+  } as any;
+  if (nesting > 0) {
+    object.nested = createNestingWidget(nesting - 1);
+  }
+  return object;
+}
+
 export const deleteObjects: StaticOperationSchema = [
   9,
   "ns",

packages/core/src/submodules/protocols/xml/XmlShapeDeserializer.spec.ts

@@ -2,24 +2,35 @@ import { NumericValue } from "@smithy/core/serde";
 import type { TimestampDateTimeSchema } from "@smithy/types";
 import { describe, expect, test as it } from "vitest";
 
-import { widget } from "../test-schema.spec";
+import { createNestingWidget, nestingWidget, widget } from "../test-schema.spec";
 import { XmlShapeDeserializer } from "./XmlShapeDeserializer";
+import { XmlShapeSerializer } from "./XmlShapeSerializer";
+
+describe(XmlShapeDeserializer.name, () => {
+  const deserializer = new XmlShapeDeserializer({
+    httpBindings: true,
+    serviceNamespace: "namespace",
+    timestampFormat: { default: 5 satisfies TimestampDateTimeSchema, useTrait: true },
+    xmlNamespace: "namespace",
+  });
+  const serializer = new XmlShapeSerializer({
+    xmlNamespace: "namespace",
+    serviceNamespace: "namespace",
+    timestampFormat: { default: 5 satisfies TimestampDateTimeSchema, useTrait: true },
+  });
+  serializer.setSerdeContext({
+    base64Encoder: (input: Uint8Array) => {
+      return Buffer.from(input).toString("base64");
+    },
+  } as any);
 
-describe("", () => {
   it("placeholder", async () => {
     const xml = `<Struct xmlns="namespace">
   <blob>QUFBQQ==</blob>
   <timestamp>0</timestamp>
   <bigint>10000000000000000000000054321</bigint>
   <bigdecimal>0.10000000000000000000000054321</bigdecimal>
 </Struct>`;
-    const deserializer = new XmlShapeDeserializer({
-      httpBindings: true,
-      serviceNamespace: "namespace",
-      timestampFormat: { default: 5 satisfies TimestampDateTimeSchema, useTrait: true },
-      xmlNamespace: "namespace",
-    });
-
     const result = await deserializer.read(widget, xml);
     expect(result).toEqual({
       blob: new Uint8Array([65, 65, 65, 65]),
@@ -28,4 +39,35 @@ describe("", () => {
       bigdecimal: new NumericValue("0.10000000000000000000000054321", "bigDecimal"),
     });
   });
+
+  describe("performance baseline indicator", () => {
+    it("should deserialize XML strings", async () => {
+      const timings: string[] = [];
+      const strings = [];
+
+      // warmup
+      for (let i = 0; i < 13; ++i) {
+        const o = createNestingWidget(2 ** i);
+        serializer.write(nestingWidget, o);
+        const json = serializer.flush();
+        strings.push(json);
+        await deserializer.read(nestingWidget, json);
+      }
+
+      for (const s of strings) {
+        const A = performance.now();
+        await deserializer.read(nestingWidget, s);
+        const B = performance.now();
+
+        timings.push(`${B - A} (XML length = ${s.length}, ${s.length / 1024 / (B - A)} kb/ms)`);
+      }
+
+      /**
+       * No assertion here.
+       * In the initial dual-pass implementation,
+       * par time is 0 to 45ms for up to 442000 chars of XML. Up to 10 kb/ms. (kuhe's computer)
+       */
+      console.log("XmlShapeDeserializer performance timings", timings);
+    });
+  });
 });

packages/core/src/submodules/protocols/xml/XmlShapeSerializer.spec.ts

@@ -2,23 +2,23 @@ import { NumericValue } from "@smithy/core/serde";
 import type { TimestampDateTimeSchema } from "@smithy/types";
 import { describe, expect, test as it } from "vitest";
 
-import { widget } from "../test-schema.spec";
+import { createNestingWidget, nestingWidget, widget } from "../test-schema.spec";
 import { simpleFormatXml } from "./simpleFormatXml";
 import { XmlShapeSerializer } from "./XmlShapeSerializer";
 
 describe(XmlShapeSerializer.name, () => {
-  it("serializes data to Query", async () => {
-    const serializer = new XmlShapeSerializer({
-      xmlNamespace: "namespace",
-      serviceNamespace: "namespace",
-      timestampFormat: { default: 5 satisfies TimestampDateTimeSchema, useTrait: true },
-    });
-    serializer.setSerdeContext({
-      base64Encoder: (input: Uint8Array) => {
-        return Buffer.from(input).toString("base64");
-      },
-    } as any);
+  const serializer = new XmlShapeSerializer({
+    xmlNamespace: "namespace",
+    serviceNamespace: "namespace",
+    timestampFormat: { default: 5 satisfies TimestampDateTimeSchema, useTrait: true },
+  });
+  serializer.setSerdeContext({
+    base64Encoder: (input: Uint8Array) => {
+      return Buffer.from(input).toString("base64");
+    },
+  } as any);
 
+  it("serializes data to Query", async () => {
     const data = {
       timestamp: new Date(0),
       bigint: 10000000000000000000000054321n,
@@ -42,4 +42,37 @@ describe(XmlShapeSerializer.name, () => {
   </bigdecimal>
 </Struct>`);
   });
+
+  describe("performance baseline indicator", () => {
+    it("should serialize objects", () => {
+      const timings: string[] = [];
+      const objects = [];
+
+      // warmup
+      for (let i = 0; i < 13; ++i) {
+        const o = createNestingWidget(2 ** i);
+        objects.push(o);
+        serializer.write(nestingWidget, o);
+        serializer.flush();
+      }
+
+      for (let i = 0; i < objects.length; ++i) {
+        const o = objects[i];
+
+        const A = performance.now();
+        serializer.write(nestingWidget, o);
+        const serialization = serializer.flush();
+        const B = performance.now();
+
+        timings.push(`${B - A} (XML length = ${serialization.length}, ${serialization.length / 1024 / (B - A)} kb/ms)`);
+      }
+
+      /**
+       * No assertion here.
+       * In the initial dual-pass implementation,
+       * par time is 0 to 30ms for up to 442,000 chars of XML. Up to 25 kb/ms. (kuhe's computer)
+       */
+      console.log("XmlShapeSerializer performance timings", timings);
+    });
+  });
 });