Skip to content

Commit e83ddbc

Browse files
t3chguyt3chguy
authored
Improve handling of animated images, add support for AVIF animations (#30932)
* Only set MSC4230 is_animated flag if we are able to tell if the media is animated Signed-off-by: Michael Telatynski <[email protected]> * Set blob type correctly to not need to weave the mimetype around Signed-off-by: Michael Telatynski <[email protected]> * Use ImageDecoder to determine whether media is animated or not, adding support for AVIF and other formats Signed-off-by: Michael Telatynski <[email protected]> * Fix test Signed-off-by: Michael Telatynski <[email protected]> * Iterate Signed-off-by: Michael Telatynski <[email protected]> * Add test Signed-off-by: Michael Telatynski <[email protected]> * Fix test Signed-off-by: Michael Telatynski <[email protected]> --------- Signed-off-by: Michael Telatynski <[email protected]>
1 parent 5f084c2 commit e83ddbc

File tree

6 files changed

+115
-33
lines changed

6 files changed

+115
-33
lines changed

‎src/ContentMessages.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -158,14 +158,17 @@ async function infoForImageFile(matrixClient: MatrixClient, roomId: string, imag
158158
}
159159

160160
// We don't await this immediately so it can happen in the background
161-
const isAnimatedPromise = blobIsAnimated(imageFile.type, imageFile);
161+
const isAnimatedPromise = blobIsAnimated(imageFile);
162162

163163
const imageElement = await loadImageElement(imageFile);
164164

165165
const result = await createThumbnail(imageElement.img, imageElement.width, imageElement.height, thumbnailType);
166166
const imageInfo = result.info;
167167

168-
imageInfo["org.matrix.msc4230.is_animated"] = await isAnimatedPromise;
168+
const isAnimated = await isAnimatedPromise;
169+
if (isAnimated !== undefined) {
170+
imageInfo["org.matrix.msc4230.is_animated"] = await isAnimatedPromise;
171+
}
169172

170173
// For lesser supported image types, always include the thumbnail even if it is larger
171174
if (!ALWAYS_INCLUDE_THUMBNAIL.includes(imageFile.type)) {

‎src/components/views/messages/MImageBody.tsx‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -311,10 +311,7 @@ export class MImageBodyInner extends React.Component<IProps, IState> {
311311
// then we need to check if the image is animated by downloading it.
312312
if (
313313
content.info?.["org.matrix.msc4230.is_animated"] === false ||
314-
!(await blobIsAnimated(
315-
content.info?.mimetype,
316-
await this.props.mediaEventHelper!.sourceBlob.value,
317-
))
314+
(await blobIsAnimated(await this.props.mediaEventHelper!.sourceBlob.value)) === false
318315
) {
319316
isAnimated = false;
320317
}

‎src/utils/Image.ts‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
import { arrayHasDiff } from "./arrays";
1010

1111
export function mayBeAnimated(mimeType?: string): boolean {
12-
// AVIF animation support at the time of writing is only available in Chrome hence not having `blobIsAnimated` check
1312
return ["image/gif", "image/webp", "image/png", "image/apng", "image/avif"].includes(mimeType!);
1413
}
1514

@@ -26,8 +25,28 @@ function arrayBufferReadStr(arr: ArrayBuffer, start: number, len: number): strin
2625
return String.fromCharCode.apply(null, Array.from(arrayBufferRead(arr, start, len)));
2726
}
2827

29-
export async function blobIsAnimated(mimeType: string | undefined, blob: Blob): Promise<boolean> {
30-
switch (mimeType) {
28+
/**
29+
* Check if a Blob contains an animated image.
30+
* @param blob The Blob to check.
31+
* @returns True if the image is animated, false if not, or undefined if it could not be determined.
32+
*/
33+
export async function blobIsAnimated(blob: Blob): Promise<boolean | undefined> {
34+
try {
35+
// Try parse the image using ImageDecoder as this is the most coherent way of asserting whether a piece of media
36+
// is or is not animated. Limited availability at time of writing, notably Safari lacks support.
37+
// https://developer.mozilla.org/en-US/docs/Web/API/ImageDecoder
38+
const data = await blob.arrayBuffer();
39+
const decoder = new ImageDecoder({ data, type: blob.type });
40+
await decoder.tracks.ready;
41+
if ([...decoder.tracks].some((track) => track.animated)) {
42+
return true;
43+
}
44+
} catch (e) {
45+
console.warn("ImageDecoder not supported or failed to decode image", e);
46+
// Not supported by this browser, fall through to manual checks
47+
}
48+
49+
switch (blob.type) {
3150
case "image/webp": {
3251
// Only extended file format WEBP images support animation, so grab the expected data range and verify header.
3352
// Based on https://developers.google.com/speed/webp/docs/riff_container#extended_file_format
@@ -42,7 +61,7 @@ export async function blobIsAnimated(mimeType: string | undefined, blob: Blob):
4261
const animationFlagMask = 1 << 1;
4362
return (flags & animationFlagMask) != 0;
4463
}
45-
break;
64+
return false;
4665
}
4766

4867
case "image/gif": {
@@ -100,9 +119,7 @@ export async function blobIsAnimated(mimeType: string | undefined, blob: Blob):
100119
}
101120
i += length + 4;
102121
}
103-
break;
122+
return false;
104123
}
105124
}
106-
107-
return false;
108125
}

‎src/utils/MediaEventHelper.ts‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -72,18 +72,25 @@ export class MediaEventHelper implements IDestroyable {
7272
};
7373

7474
private fetchSource = (): Promise<Blob> => {
75+
const content = this.event.getContent<MediaEventContent>();
7576
if (this.media.isEncrypted) {
76-
const content = this.event.getContent<MediaEventContent>();
7777
return decryptFile(content.file!, content.info);
7878
}
79-
return this.media.downloadSource().then((r) => r.blob());
79+
80+
return (
81+
this.media
82+
.downloadSource()
83+
.then((r) => r.blob())
84+
// Set the mime type from the event info on the blob
85+
.then((blob) => blob.slice(0, blob.size, content.info?.mimetype ?? blob.type))
86+
);
8087
};
8188

8289
private fetchThumbnail = (): Promise<Blob | null> => {
8390
if (!this.media.hasThumbnail) return Promise.resolve(null);
8491

92+
const content = this.event.getContent<ImageContent>();
8593
if (this.media.isEncrypted) {
86-
const content = this.event.getContent<ImageContent>();
8794
if (content.info?.thumbnail_file) {
8895
return decryptFile(content.info.thumbnail_file, content.info.thumbnail_info);
8996
} else {
@@ -96,7 +103,12 @@ export class MediaEventHelper implements IDestroyable {
96103
const thumbnailHttp = this.media.thumbnailHttp;
97104
if (!thumbnailHttp) return Promise.resolve(null);
98105

99-
return fetch(thumbnailHttp).then((r) => r.blob());
106+
return (
107+
fetch(thumbnailHttp)
108+
.then((r) => r.blob())
109+
// Set the mime type from the event info on the blob
110+
.then((blob) => blob.slice(0, blob.size, content.info?.thumbnail_info?.mimetype ?? blob.type))
111+
);
100112
};
101113

102114
public static isEligible(event: MatrixEvent): boolean {

‎test/unit-tests/Image-test.ts‎

Lines changed: 29 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -32,42 +32,55 @@ describe("Image", () => {
3232

3333
describe("blobIsAnimated", () => {
3434
it("Animated GIF", async () => {
35-
const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.gif"))]);
36-
expect(await blobIsAnimated("image/gif", img)).toBeTruthy();
35+
const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.gif"))], {
36+
type: "image/gif",
37+
});
38+
expect(await blobIsAnimated(img)).toBeTruthy();
3739
});
3840

3941
it("Static GIF", async () => {
40-
const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.gif"))]);
41-
expect(await blobIsAnimated("image/gif", img)).toBeFalsy();
42+
const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.gif"))], {
43+
type: "image/gif",
44+
});
45+
expect(await blobIsAnimated(img)).toBeFalsy();
4246
});
4347

4448
it("Animated WEBP", async () => {
45-
const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.webp"))]);
46-
expect(await blobIsAnimated("image/webp", img)).toBeTruthy();
49+
const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.webp"))], {
50+
type: "image/webp",
51+
});
52+
expect(await blobIsAnimated(img)).toBeTruthy();
4753
});
4854

4955
it("Static WEBP", async () => {
50-
const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.webp"))]);
51-
expect(await blobIsAnimated("image/webp", img)).toBeFalsy();
56+
const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.webp"))], {
57+
type: "image/webp",
58+
});
59+
expect(await blobIsAnimated(img)).toBeFalsy();
5260
});
5361

5462
it("Static WEBP in extended file format", async () => {
55-
const img = new Blob([
56-
fs.readFileSync(path.resolve(__dirname, "images", "static-logo-extended-file-format.webp")),
57-
]);
58-
expect(await blobIsAnimated("image/webp", img)).toBeFalsy();
63+
const img = new Blob(
64+
[fs.readFileSync(path.resolve(__dirname, "images", "static-logo-extended-file-format.webp"))],
65+
{ type: "image/webp" },
66+
);
67+
expect(await blobIsAnimated(img)).toBeFalsy();
5968
});
6069

6170
it("Animated PNG", async () => {
6271
const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.apng"))]);
63-
expect(await blobIsAnimated("image/png", img)).toBeTruthy();
64-
expect(await blobIsAnimated("image/apng", img)).toBeTruthy();
72+
const pngBlob = img.slice(0, img.size, "image/png");
73+
const apngBlob = img.slice(0, img.size, "image/apng");
74+
expect(await blobIsAnimated(pngBlob)).toBeTruthy();
75+
expect(await blobIsAnimated(apngBlob)).toBeTruthy();
6576
});
6677

6778
it("Static PNG", async () => {
6879
const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.png"))]);
69-
expect(await blobIsAnimated("image/png", img)).toBeFalsy();
70-
expect(await blobIsAnimated("image/apng", img)).toBeFalsy();
80+
const pngBlob = img.slice(0, img.size, "image/png");
81+
const apngBlob = img.slice(0, img.size, "image/apng");
82+
expect(await blobIsAnimated(pngBlob)).toBeFalsy();
83+
expect(await blobIsAnimated(apngBlob)).toBeFalsy();
7184
});
7285
});
7386
});

‎test/unit-tests/utils/MediaEventHelper-test.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
Copyright 2025 New Vector Ltd.
3+
4+
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
5+
Please see LICENSE files in the repository root for full details.
6+
*/
7+
8+
import { MatrixEvent } from "matrix-js-sdk/src/matrix";
9+
10+
import { MediaEventHelper } from "../../../src/utils/MediaEventHelper.ts";
11+
import { stubClient } from "../../test-utils";
12+
13+
describe("MediaEventHelper", () => {
14+
it("should set the mime type on the blob based on the event metadata", async () => {
15+
stubClient();
16+
17+
const event = new MatrixEvent({
18+
type: "m.room.message",
19+
content: {
20+
msgtype: "m.image",
21+
body: "image.png",
22+
info: {
23+
mimetype: "image/png",
24+
size: 1234,
25+
w: 100,
26+
h: 100,
27+
thumbnail_info: {
28+
mimetype: "image/png",
29+
},
30+
thumbnail_url: "mxc://matrix.org/thumbnail",
31+
},
32+
url: "mxc://matrix.org/abcdef",
33+
},
34+
});
35+
const helper = new MediaEventHelper(event);
36+
37+
const blob = await helper.thumbnailBlob.value;
38+
expect(blob?.type).toBe(event.getContent().info.thumbnail_info?.mimetype);
39+
});
40+
});

0 commit comments

Comments
 (0)