Skip to content

Commit 6ea7ad0

Browse files
k-ylek-yle
committed
automatically split large changesets into chunks before uploading
1 parent 6fcc9c3 commit 6ea7ad0

File tree

5 files changed

+368
-20
lines changed

5 files changed

+368
-20
lines changed

‎src/api/changesets/__tests__/chunkOsmChange.test.ts‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
import { describe, expect, it } from "vitest";
2+
import type { OsmChange, OsmFeature, OsmFeatureType } from "../../../types";
3+
import { chunkOsmChange } from "../chunkOsmChange";
4+
import type { ApiCapabilities } from "../../getCapabilities";
5+
6+
/** use with {@link Array.sort} to randomise the order */
7+
const shuffle = () => 0.5 - Math.random();
8+
9+
const createMockFeatures = (
10+
type: OsmFeatureType,
11+
count: number,
12+
_label: string
13+
) => Array.from<OsmFeature>({ length: count }).fill(<never>{ type, _label });
14+
15+
const capabilities = <ApiCapabilities>{
16+
api: { changesets: { maximum_elements: 6 } },
17+
};
18+
19+
describe("chunkOsmChange", () => {
20+
it("returns the input if the changeset is already small enough", () => {
21+
const input: OsmChange = {
22+
create: createMockFeatures("node", 3, "create"),
23+
modify: createMockFeatures("node", 1, "modify"),
24+
delete: createMockFeatures("node", 1, "delete"),
25+
};
26+
expect(chunkOsmChange(input, capabilities)).toStrictEqual([input]);
27+
});
28+
29+
it("returns the input if the changeset has exactly the max number of items", () => {
30+
const input: OsmChange = {
31+
create: createMockFeatures("node", 3, "create"),
32+
modify: createMockFeatures("node", 1, "modify"),
33+
delete: createMockFeatures("node", 2, "delete"),
34+
};
35+
expect(chunkOsmChange(input, capabilities)).toStrictEqual([input]);
36+
});
37+
38+
it("chunks features in a schematically valid order", () => {
39+
const input: OsmChange = {
40+
create: [
41+
...createMockFeatures("node", 4, "create"),
42+
...createMockFeatures("way", 3, "create"),
43+
...createMockFeatures("relation", 4, "create"),
44+
].sort(shuffle),
45+
modify: [
46+
...createMockFeatures("node", 1, "modify"),
47+
...createMockFeatures("way", 1, "modify"),
48+
...createMockFeatures("relation", 1, "modify"),
49+
].sort(shuffle),
50+
delete: [
51+
...createMockFeatures("node", 1, "delete"),
52+
...createMockFeatures("way", 2, "delete"),
53+
...createMockFeatures("relation", 3, "delete"),
54+
].sort(shuffle),
55+
};
56+
expect(chunkOsmChange(input, capabilities)).toStrictEqual([
57+
// chunk 1:
58+
{
59+
create: [
60+
...createMockFeatures("node", 4, "create"),
61+
...createMockFeatures("way", 2, "create"),
62+
],
63+
modify: [],
64+
delete: [],
65+
},
66+
// chunk 2:
67+
{
68+
create: [
69+
...createMockFeatures("way", 1, "create"),
70+
...createMockFeatures("relation", 4, "create"),
71+
],
72+
modify: [],
73+
delete: createMockFeatures("relation", 1, "delete"),
74+
},
75+
// chunk 3:
76+
{
77+
create: [],
78+
modify: createMockFeatures("node", 1, "modify"),
79+
delete: [
80+
...createMockFeatures("relation", 2, "delete"),
81+
...createMockFeatures("way", 2, "delete"),
82+
...createMockFeatures("node", 1, "delete"),
83+
],
84+
},
85+
// chunk 4:
86+
{
87+
create: [],
88+
modify: [
89+
...createMockFeatures("way", 1, "modify"),
90+
...createMockFeatures("relation", 1, "modify"),
91+
],
92+
delete: [],
93+
},
94+
]);
95+
});
96+
97+
it("exposes the default limit to consumers", () => {
98+
expect(chunkOsmChange.DEFAULT_LIMIT).toBeTypeOf("number");
99+
});
100+
});

‎src/api/changesets/__tests__/uploadChangeset.test.ts‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
import { beforeEach, describe, expect, it, vi } from "vitest";
2+
import type { OsmChange, OsmFeature, OsmFeatureType } from "../../../types";
3+
import { uploadChangeset } from "../uploadChangeset";
4+
import { chunkOsmChange } from "../chunkOsmChange";
5+
import { osmFetch } from "../../_osmFetch";
6+
7+
let nextId = 0;
8+
vi.mock("../../_osmFetch", () => ({ osmFetch: vi.fn(() => ++nextId) }));
9+
10+
/** use with {@link Array.sort} to randomise the order */
11+
const shuffle = () => 0.5 - Math.random();
12+
13+
const createMockFeatures = (
14+
type: OsmFeatureType,
15+
count: number,
16+
_label: string
17+
) =>
18+
Array.from<OsmFeature>({ length: count }).fill(<never>{
19+
type,
20+
_label,
21+
nodes: [],
22+
members: [],
23+
});
24+
25+
describe("uploadChangeset", () => {
26+
beforeEach(() => {
27+
nextId = 0;
28+
vi.clearAllMocks();
29+
chunkOsmChange.DEFAULT_LIMIT = 6; // don't do this in production
30+
});
31+
32+
const huge: OsmChange = {
33+
create: [
34+
...createMockFeatures("node", 4, "create"),
35+
...createMockFeatures("way", 3, "create"),
36+
...createMockFeatures("relation", 4, "create"),
37+
].sort(shuffle),
38+
modify: [
39+
...createMockFeatures("node", 1, "modify"),
40+
...createMockFeatures("way", 1, "modify"),
41+
...createMockFeatures("relation", 1, "modify"),
42+
].sort(shuffle),
43+
delete: [
44+
...createMockFeatures("node", 1, "delete"),
45+
...createMockFeatures("way", 2, "delete"),
46+
...createMockFeatures("relation", 3, "delete"),
47+
].sort(shuffle),
48+
};
49+
50+
it("splits changesets into chunks and uploads them in a schematically valid order", async () => {
51+
const output = await uploadChangeset({ created_by: "me" }, huge);
52+
53+
expect(osmFetch).toHaveBeenCalledTimes(12);
54+
55+
for (const index of [0, 1, 2, 3]) {
56+
expect(osmFetch).toHaveBeenNthCalledWith(
57+
1 + 3 * index, // 3 API requests per changeset
58+
"/0.6/changeset/create",
59+
undefined,
60+
expect.objectContaining({
61+
body: `<osm>
62+
<changeset>
63+
<tag k="created_by" v="me"/>
64+
<tag k="chunk" v="${index + 1}/4"/>
65+
</changeset>
66+
</osm>
67+
`,
68+
})
69+
);
70+
}
71+
72+
expect(output).toBe(1);
73+
});
74+
75+
it("splits changesets into chunks and suports a custom tag function", async () => {
76+
const output = await uploadChangeset({ created_by: "me" }, huge, {
77+
onChunk: ({ changesetIndex, changesetTotal, featureCount }) => ({
78+
comment: "hiiii",
79+
part: `${changesetIndex + 1} out of ${changesetTotal}`,
80+
totalSize: featureCount.toLocaleString("en"),
81+
}),
82+
});
83+
84+
expect(osmFetch).toHaveBeenCalledTimes(12);
85+
86+
for (const index of [0, 1, 2, 3]) {
87+
expect(osmFetch).toHaveBeenNthCalledWith(
88+
1 + 3 * index, // 3 API requests per changeset
89+
"/0.6/changeset/create",
90+
undefined,
91+
expect.objectContaining({
92+
body: `<osm>
93+
<changeset>
94+
<tag k="comment" v="hiiii"/>
95+
<tag k="part" v="${index + 1} out of 4"/>
96+
<tag k="totalSize" v="20"/>
97+
</changeset>
98+
</osm>
99+
`,
100+
})
101+
);
102+
}
103+
104+
expect(output).toBe(1);
105+
});
106+
});

‎src/api/changesets/chunkOsmChange.ts‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
import type { OsmChange, OsmFeature, OsmFeatureType } from "../../types";
2+
import type { ApiCapabilities } from "../getCapabilities";
3+
4+
const ACTIONS = (<const>[
5+
"create",
6+
"modify",
7+
"delete",
8+
]) satisfies (keyof OsmChange)[];
9+
10+
type Action = (typeof ACTIONS)[number];
11+
type Group = `${Action}-${OsmFeatureType}`;
12+
13+
/** to ensure the uploads are valid, we must follow this order */
14+
const UPLOAD_ORDER: Group[] = [
15+
"create-node",
16+
"create-way",
17+
"create-relation",
18+
"delete-relation",
19+
"delete-way",
20+
"delete-node",
21+
"modify-node",
22+
"modify-way",
23+
"modify-relation",
24+
];
25+
26+
const EMPTY_CHANGESET = (): OsmChange => ({
27+
create: [],
28+
modify: [],
29+
delete: [],
30+
});
31+
32+
/** @internal */
33+
export function getOsmChangeSize(osmChange: OsmChange) {
34+
return (
35+
osmChange.create.length + osmChange.modify.length + osmChange.delete.length
36+
);
37+
}
38+
39+
/**
40+
* If a changeset is too big to upload at once, this function can split
41+
* the changeset into smaller chunks, which can be uploaded separately.
42+
*
43+
* @param capabilities - optional, this data can be fetched from `getApiCapabilities`.
44+
* if not supplied, {@link chunkOsmChange.DEFAULT_LIMIT} is used.
45+
*/
46+
export function chunkOsmChange(
47+
osmChange: OsmChange,
48+
capabilities?: ApiCapabilities
49+
): OsmChange[] {
50+
const max =
51+
capabilities?.api.changesets.maximum_elements ??
52+
chunkOsmChange.DEFAULT_LIMIT;
53+
54+
// abort early if there's nothing to do.
55+
if (getOsmChangeSize(osmChange) <= max) return [osmChange];
56+
57+
const grouped: Partial<Record<Group, OsmFeature[]>> = {};
58+
59+
for (const action of ACTIONS) {
60+
for (const feature of osmChange[action]) {
61+
const group: Group = `${action}-${feature.type}`;
62+
63+
grouped[group] ||= [];
64+
grouped[group].push(feature);
65+
}
66+
}
67+
68+
const chunks: OsmChange[] = [EMPTY_CHANGESET()];
69+
70+
function getNext() {
71+
for (const group of UPLOAD_ORDER) {
72+
const action = <Action>group.split("-")[0];
73+
const feature = grouped[group]?.pop();
74+
if (feature) return { action, feature };
75+
}
76+
return undefined;
77+
}
78+
79+
let next: ReturnType<typeof getNext>;
80+
while ((next = getNext())) {
81+
const head = chunks[0];
82+
83+
head[next.action].push(next.feature);
84+
85+
// if the changeset is now too big, create a new chunk
86+
if (getOsmChangeSize(head) >= max) {
87+
chunks.unshift(EMPTY_CHANGESET());
88+
}
89+
}
90+
91+
return chunks.reverse();
92+
}
93+
94+
chunkOsmChange.DEFAULT_LIMIT = 10_000;

‎src/api/changesets/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
export * from "./chunkOsmChange";
12
export * from "./createChangesetComment";
23
export * from "./getChangesetDiff";
34
export * from "./getChangesets";

0 commit comments

Comments
 (0)