Skip to content

Commit b6268d9

Browse files
scottn12Copilot
andauthored
test(ordered-collection): Add Fuzz testing for ConsensusOrderedCollection (#25829)
## Description This PR enables fuzz and local server stress testing for the `ConsensusOrderedCollection` DDS. ### `applyStashedOp()` This PR also changes the `applyStashedOp` implementation from throwing "not implemented" to an empty implementation. This allows fuzz testing in rehydration scenarios as well as local server stress tests scenarios. This is also in-line with `ConsensusRegisterCollection`. Alternatively, we can leave the original `applyStashedOp` implementation and disable rehydration scenarios in fuzz testing. However, this will prevent us from integrating `ConsensusOrderedCollection` into local server stress tests. ## Misc [AB#52361](https://dev.azure.com/fluidframework/235294da-091d-4c29-84fc-cdfc3d90890b/_workitems/edit/52361) --------- Co-authored-by: Copilot <[email protected]>
1 parent 7451000 commit b6268d9

File tree

11 files changed

+291
-5
lines changed

11 files changed

+291
-5
lines changed

packages/dds/ordered-collection/package.json

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,18 @@
4242
"types": "./dist/index.d.ts",
4343
"default": "./dist/index.js"
4444
}
45+
},
46+
"./internal/test": {
47+
"allow-ff-test-exports": {
48+
"import": {
49+
"types": "./lib/test/index.d.ts",
50+
"default": "./lib/test/index.js"
51+
},
52+
"require": {
53+
"types": "./dist/test/index.d.ts",
54+
"default": "./dist/test/index.js"
55+
}
56+
}
4557
}
4658
},
4759
"main": "lib/index.js",
@@ -62,7 +74,7 @@
6274
"build:test": "npm run build:test:esm && npm run build:test:cjs",
6375
"build:test:cjs": "fluid-tsc commonjs --project ./src/test/tsconfig.cjs.json",
6476
"build:test:esm": "tsc --project ./src/test/tsconfig.json",
65-
"check:are-the-types-wrong": "attw --pack .",
77+
"check:are-the-types-wrong": "attw --pack . --exclude-entrypoints ./internal/test",
6678
"check:biome": "biome check .",
6779
"check:exports": "concurrently \"npm:check:exports:*\"",
6880
"check:exports:bundle-release-tags": "api-extractor run --config api-extractor/api-extractor-lint-bundle.json",
@@ -130,6 +142,7 @@
130142
"@arethetypeswrong/cli": "^0.17.1",
131143
"@biomejs/biome": "~1.9.3",
132144
"@fluid-internal/mocha-test-setup": "workspace:~",
145+
"@fluid-private/stochastic-test-utils": "workspace:~",
133146
"@fluid-private/test-dds-utils": "workspace:~",
134147
"@fluid-tools/build-cli": "^0.60.0",
135148
"@fluidframework/build-common": "^2.0.3",

packages/dds/ordered-collection/src/consensusOrderedCollection.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -434,7 +434,7 @@ export class ConsensusOrderedCollection<T = any>
434434
}
435435

436436
protected applyStashedOp(): void {
437-
throw new Error("not implemented");
437+
// empty implementation
438438
}
439439

440440
/**
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*!
2+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3+
* Licensed under the MIT License.
4+
*/
5+
6+
import { createDDSFuzzSuite } from "@fluid-private/test-dds-utils";
7+
import { FlushMode } from "@fluidframework/runtime-definitions/internal";
8+
9+
import { baseConsensusOrderedCollectionModel, defaultOptions } from "./fuzzUtils.js";
10+
11+
describe("ConsensusOrderedCollection fuzz testing", () => {
12+
createDDSFuzzSuite(baseConsensusOrderedCollectionModel, {
13+
...defaultOptions,
14+
// Uncomment this line to replay a specific seed:
15+
// replay: 0,
16+
// This can be useful for quickly minimizing failure json while attempting to root-cause a failure.
17+
});
18+
});
19+
20+
describe("ConsensusOrderedCollection fuzz testing with rebasing", () => {
21+
createDDSFuzzSuite(baseConsensusOrderedCollectionModel, {
22+
...defaultOptions,
23+
containerRuntimeOptions: {
24+
flushMode: FlushMode.TurnBased,
25+
enableGroupedBatching: true,
26+
},
27+
rebaseProbability: 0.15,
28+
// Uncomment this line to replay a specific seed:
29+
// replay: 0,
30+
// This can be useful for quickly minimizing failure json while attempting to root-cause a failure.
31+
});
32+
});
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
/*!
2+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3+
* Licensed under the MIT License.
4+
*/
5+
6+
// Problem:
7+
// - `__dirname` is not defined in ESM
8+
// - `import.meta.url` is not defined in CJS
9+
// Solution:
10+
// - Export '__dirname' from a .cjs file in the same directory.
11+
//
12+
// Note that *.cjs files are always CommonJS, but can be imported from ESM.
13+
export const _dirname = __dirname;
Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
/*!
2+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3+
* Licensed under the MIT License.
4+
*/
5+
6+
import { strict as assert } from "node:assert";
7+
import * as path from "node:path";
8+
9+
import type {
10+
AsyncGenerator as Generator,
11+
Reducer,
12+
} from "@fluid-private/stochastic-test-utils";
13+
import {
14+
combineReducers,
15+
createWeightedAsyncGenerator as createWeightedGenerator,
16+
makeRandom,
17+
takeAsync,
18+
} from "@fluid-private/stochastic-test-utils";
19+
import type {
20+
DDSFuzzModel,
21+
DDSFuzzSuiteOptions,
22+
DDSFuzzTestState,
23+
} from "@fluid-private/test-dds-utils";
24+
25+
import { ConsensusQueueFactory } from "../consensusOrderedCollectionFactory.js";
26+
import type { IConsensusOrderedCollection, IOrderedCollection } from "../interfaces.js";
27+
import { ConsensusResult } from "../interfaces.js";
28+
29+
import { _dirname } from "./dirname.cjs";
30+
31+
/**
32+
* Config options for generating ConsensusOrderedCollection operations
33+
*/
34+
interface ConsensusOrderedCollectionValueConfig {
35+
/**
36+
* Number of values to be generated for the pool
37+
*/
38+
valuePoolSize?: number;
39+
/**
40+
* Length of value strings
41+
*/
42+
valueStringLength?: number;
43+
}
44+
45+
const valueConfigs: Required<ConsensusOrderedCollectionValueConfig> = {
46+
valuePoolSize: 3,
47+
valueStringLength: 5,
48+
};
49+
50+
/**
51+
* Default options for ConsensusOrderedCollection fuzz testing
52+
*/
53+
export const defaultOptions: Partial<DDSFuzzSuiteOptions> = {
54+
validationStrategy: { type: "fixedInterval", interval: 10 },
55+
clientJoinOptions: {
56+
maxNumberOfClients: 6,
57+
clientAddProbability: 0.05,
58+
},
59+
defaultTestCount: 100,
60+
saveFailures: { directory: path.join(_dirname, "../../src/test/results") },
61+
};
62+
63+
type FuzzTestState = DDSFuzzTestState<ConsensusQueueFactory>;
64+
65+
export interface AddOperation {
66+
type: "add";
67+
value: string;
68+
}
69+
70+
export interface AcquireOperation {
71+
type: "acquire";
72+
result: ConsensusResult;
73+
}
74+
75+
/**
76+
* Represents ConsensusOrderedCollection operation types for fuzz testing
77+
*/
78+
export type ConsensusOrderedCollectionOperation = AddOperation | AcquireOperation;
79+
80+
function makeOperationGenerator(): Generator<
81+
ConsensusOrderedCollectionOperation,
82+
FuzzTestState
83+
> {
84+
type OpSelectionState = FuzzTestState & {
85+
itemValue: string;
86+
pendingCallbacks: Map<string, string>;
87+
};
88+
89+
const valuePoolRandom = makeRandom(0);
90+
const dedupe = <T>(arr: T[]): T[] => [...new Set(arr)];
91+
const valuePool = dedupe(
92+
Array.from({ length: valueConfigs.valuePoolSize }, () =>
93+
valuePoolRandom.string(valueConfigs.valueStringLength),
94+
),
95+
);
96+
97+
async function add(state: OpSelectionState): Promise<AddOperation> {
98+
return {
99+
type: "add",
100+
value: state.itemValue,
101+
};
102+
}
103+
104+
async function acquire(state: OpSelectionState): Promise<AcquireOperation> {
105+
return {
106+
type: "acquire",
107+
result: state.random.pick([ConsensusResult.Complete, ConsensusResult.Release]),
108+
};
109+
}
110+
111+
const clientBaseOperationGenerator = createWeightedGenerator<
112+
ConsensusOrderedCollectionOperation,
113+
OpSelectionState
114+
>([
115+
[add, 1],
116+
[acquire, 1],
117+
]);
118+
119+
return async (state: FuzzTestState) =>
120+
clientBaseOperationGenerator({
121+
...state,
122+
itemValue: state.random.pick(valuePool),
123+
pendingCallbacks: new Map<string, string>(),
124+
});
125+
}
126+
127+
// Track async errors that occur during fire-and-forget operations
128+
let pendingAsyncError: Error | undefined;
129+
130+
function makeReducer(): Reducer<ConsensusOrderedCollectionOperation, FuzzTestState> {
131+
const reducer = combineReducers<ConsensusOrderedCollectionOperation, FuzzTestState>({
132+
add: ({ client }, { value }) => {
133+
client.channel.add(value).catch((error: Error) => {
134+
pendingAsyncError = error;
135+
});
136+
},
137+
acquire: ({ client }, { result }) => {
138+
client.channel
139+
.acquire(async (_value: string): Promise<ConsensusResult> => {
140+
return result;
141+
})
142+
.catch((error: Error) => {
143+
pendingAsyncError = error;
144+
});
145+
},
146+
});
147+
return reducer;
148+
}
149+
150+
function assertEqualConsensusOrderedCollections(
151+
a: IConsensusOrderedCollection,
152+
b: IConsensusOrderedCollection,
153+
): void {
154+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access
155+
const aData = (a as any).data as IOrderedCollection<string>;
156+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access
157+
const bData = (b as any).data as IOrderedCollection<string>;
158+
assert.equal(aData.size, bData.size, "Data sizes should be equal");
159+
assert.deepEqual(aData.asArray(), bData.asArray(), "Data contents should be equal");
160+
161+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access
162+
const aJobTracking = (a as any).jobTracking as Map<
163+
string,
164+
{ value: string; clientId: string | undefined }
165+
>;
166+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access
167+
const bJobTracking = (b as any).jobTracking as Map<
168+
string,
169+
{ value: string; clientId: string | undefined }
170+
>;
171+
172+
assert.equal(aJobTracking.size, bJobTracking.size, "Job tracking sizes should be equal");
173+
for (const [key, aJob] of aJobTracking.entries()) {
174+
const bJob = bJobTracking.get(key);
175+
assert.deepEqual(aJob, bJob, `Job tracking entry for key ${key} should be equal`);
176+
}
177+
}
178+
179+
/**
180+
* Base fuzz model for ConsensusOrderedCollection
181+
*/
182+
export const baseConsensusOrderedCollectionModel: DDSFuzzModel<
183+
ConsensusQueueFactory,
184+
ConsensusOrderedCollectionOperation,
185+
FuzzTestState
186+
> = {
187+
workloadName: "default configuration",
188+
generatorFactory: () => takeAsync(100, makeOperationGenerator()),
189+
reducer: makeReducer(),
190+
validateConsistency: (a, b) => {
191+
// Check if any async errors occurred during fire-and-forget operations
192+
if (pendingAsyncError !== undefined) {
193+
throw pendingAsyncError;
194+
}
195+
assertEqualConsensusOrderedCollections(a.channel, b.channel);
196+
},
197+
factory: new ConsensusQueueFactory(),
198+
};
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
/*!
2+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3+
* Licensed under the MIT License.
4+
*/
5+
6+
export { baseConsensusOrderedCollectionModel } from "./fuzzUtils.js";

packages/dds/ordered-collection/src/test/tsconfig.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
"rootDir": "./",
55
"outDir": "../../lib/test",
66
"types": ["mocha", "node"],
7+
"declaration": true,
8+
"declarationMap": true,
79
},
810
"include": ["./**/*"],
911
"references": [

packages/test/local-server-stress-tests/package.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@
7979
"@fluidframework/local-driver": "workspace:~",
8080
"@fluidframework/map": "workspace:~",
8181
"@fluidframework/matrix": "workspace:~",
82+
"@fluidframework/ordered-collection": "workspace:~",
8283
"@fluidframework/runtime-definitions": "workspace:~",
8384
"@fluidframework/runtime-utils": "workspace:~",
8485
"@fluidframework/sequence": "workspace:~",
@@ -113,7 +114,8 @@
113114
"@fluidframework/matrix#build:test",
114115
"@fluidframework/tree#build:test",
115116
"@fluidframework/task-manager#build:test",
116-
"@fluidframework/legacy-dds#build:test"
117+
"@fluidframework/legacy-dds#build:test",
118+
"@fluidframework/ordered-collection#build:test"
117119
]
118120
}
119121
},

packages/test/local-server-stress-tests/src/ddsModels.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import type { IChannelFactory } from "@fluidframework/datastore-definitions/inte
1111
import { baseSharedArrayModel } from "@fluidframework/legacy-dds/internal/test";
1212
import { baseMapModel, baseDirModel } from "@fluidframework/map/internal/test";
1313
import { baseSharedMatrixModel } from "@fluidframework/matrix/internal/test";
14+
import { baseConsensusOrderedCollectionModel } from "@fluidframework/ordered-collection/internal/test";
1415
import {
1516
baseSharedStringModel,
1617
baseIntervalModel,
@@ -74,4 +75,5 @@ export const ddsModelMap = generateSubModelMap(
7475
baseSharedArrayModel,
7576
baseTaskManagerModel,
7677
baseCounterModel,
78+
baseConsensusOrderedCollectionModel,
7779
);

packages/test/local-server-stress-tests/src/localServerStressHarness.ts

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -913,8 +913,20 @@ async function synchronizeClients(connectedClients: Client[]) {
913913
connectedClients[0].container.deltaManager.lastSequenceNumber,
914914
)
915915
) {
916-
resolve();
917-
off();
916+
// There are some cases where more ops are generated as a result of processing ops.
917+
// For example, for ConsensusOrderedCollection, a `complete`/`release` op will be generated
918+
// when processing an `acquire` op.
919+
// If that type of op is the last op before a final sync, then we will miss it by waiting
920+
// for the initial saved event. To ensure we process all ops we wait for two JS turns.
921+
// The first allows any ops generated during the processing of ops to be submitted.
922+
// The second allows for the processing of those ops.
923+
// TODO: AB#53704: Support async reducers in DDS Fuzz models to avoid this workaround.
924+
setTimeout(() => {
925+
setTimeout(() => {
926+
resolve();
927+
off();
928+
}, 0);
929+
}, 0);
918930
}
919931
};
920932
// if you hit timeout issues in the

0 commit comments

Comments
 (0)