fix(react): Prevent frame exhaustion in fallback MediaStreamTrackProcessor (#2019)

### 💡 Overview
Stop `pull()` from exhausting frames in fallback
MediaStreamTrackProcessor by performing timestamp change check in
`transform()`

### 📝 Implementation notes

🎫 Ticket: https://linear.app/stream/issue/XYZ-123

📑 Docs: https://github.com/GetStream/docs-content/pull/<id>

Author: jdimovska

packages/video-filters-web/src/FallbackProcessor.ts

@@ -49,7 +49,6 @@ class FallbackProcessor implements MediaStreamTrackProcessor<VideoFrame> {
     let timestamp = 0;
     const frameRate = track.getSettings().frameRate || 30;
     let frameDuration = 1000 / frameRate;
-    let lastVideoTime = -1;
 
     this.workerTimer = new WorkerTimer({ useWorker: true });
     this.readable = new ReadableStream({
@@ -77,18 +76,6 @@ class FallbackProcessor implements MediaStreamTrackProcessor<VideoFrame> {
         }
         timestamp = performance.now();
 
-        const currentTime = this.video.currentTime;
-        const hasNewFrame = currentTime !== lastVideoTime;
-
-        if (!hasNewFrame) {
-          await new Promise((r: (value?: unknown) => void) =>
-            this.workerTimer.setTimeout(r, frameDuration),
-          );
-          return;
-        }
-
-        lastVideoTime = currentTime;
-
         if (
           canvas.width !== this.video.videoWidth ||
           canvas.height !== this.video.videoHeight
@@ -100,7 +87,9 @@ class FallbackProcessor implements MediaStreamTrackProcessor<VideoFrame> {
         ctx.drawImage(this.video, 0, 0);
 
         try {
-          const frame = new VideoFrame(canvas, { timestamp });
+          const frame = new VideoFrame(canvas, {
+            timestamp: Math.round(this.video.currentTime * 1000000),
+          });
           controller.enqueue(frame);
         } catch (err) {
           running = false;

packages/video-filters-web/src/VirtualBackground.ts

@@ -21,14 +21,16 @@ export class VirtualBackground {
 
   private canvas!: OffscreenCanvas;
   private segmenter: ImageSegmenter | null = null;
-  private isSegmenterReady = false;
   private webGlRenderer!: WebGLRenderer;
   private abortController: AbortController;
 
   private segmenterDelayTotal = 0;
   private frames = 0;
   private lastStatsTime = 0;
 
+  private latestCategoryMask: WebGLTexture | undefined = undefined;
+  private latestConfidenceMask: WebGLTexture | undefined = undefined;
+
   constructor(
     private readonly track: MediaStreamVideoTrack,
     private readonly options: BackgroundOptions = {},
@@ -59,22 +61,43 @@ export class VirtualBackground {
 
     const opts = await this.initializeSegmenterOptions();
 
+    let lastFrameTime = -1;
     const transformStream = new TransformStream<VideoFrame, VideoFrame>({
-      transform: async (frame, controller) => {
+      transform: (frame, controller) => {
         try {
           if (this.abortController.signal.aborted) {
-            return frame.close();
+            frame.close();
+            return;
           }
 
-          const processed = await this.transform(frame, opts);
-          controller.enqueue(processed);
-        } catch (e) {
-          console.error('[virtual-background] error processing frame:', e);
-          this.hooks.onError?.(e);
+          if (
+            this.canvas.width !== frame.displayWidth ||
+            this.canvas.height !== frame.displayHeight
+          ) {
+            this.canvas.width = frame.displayWidth;
+            this.canvas.height = frame.displayHeight;
+          }
 
-          if (!this.abortController.signal.aborted) {
-            controller.enqueue(frame);
+          const currentTime = frame.timestamp;
+          const hasNewFrame = currentTime !== lastFrameTime;
+
+          lastFrameTime = currentTime;
+          if (hasNewFrame) {
+            this.runSegmentation(frame);
           }
+          this.webGlRenderer.render(
+            frame,
+            opts,
+            this.latestCategoryMask,
+            this.latestConfidenceMask,
+          );
+
+          controller.enqueue(
+            new VideoFrame(this.canvas, { timestamp: frame.timestamp }),
+          );
+        } catch (e) {
+          console.error('[virtual-background] error processing frame:', e);
+          onError?.(e);
         } finally {
           frame.close();
         }
@@ -84,7 +107,6 @@ export class VirtualBackground {
           this.segmenter.close();
           this.segmenter = null;
         }
-        this.isSegmenterReady = false;
       },
     });
 
@@ -103,6 +125,47 @@ export class VirtualBackground {
     return this.generator;
   }
 
+  private runSegmentation(frame: VideoFrame) {
+    if (!this.segmenter) return;
+
+    const start = performance.now();
+
+    this.segmenter.segmentForVideo(frame, frame.timestamp, (result) => {
+      try {
+        this.latestCategoryMask = result.categoryMask?.getAsWebGLTexture();
+
+        this.latestConfidenceMask =
+          result.confidenceMasks?.[0]?.getAsWebGLTexture();
+
+        const now = performance.now();
+        this.segmenterDelayTotal += now - start;
+        this.frames++;
+
+        if (this.lastStatsTime === 0) {
+          this.lastStatsTime = now;
+        }
+
+        if (now - this.lastStatsTime > 1000) {
+          const delay =
+            Math.round((this.segmenterDelayTotal / this.frames) * 100) / 100;
+          const fps = Math.round(
+            (1000 * this.frames) / (now - this.lastStatsTime),
+          );
+
+          this.hooks.onStats?.({ delay, fps, timestamp: now });
+
+          this.lastStatsTime = now;
+          this.segmenterDelayTotal = 0;
+          this.frames = 0;
+        }
+      } catch (err) {
+        console.error('[virtual-background] segmentation error:', err);
+      } finally {
+        result.close();
+      }
+    });
+  }
+
   /**
    * Loads and initializes the MediaPipe `ImageSegmenter`.
    */
@@ -130,82 +193,14 @@ export class VirtualBackground {
         outputConfidenceMasks: true,
         canvas: this.canvas,
       });
-
-      this.isSegmenterReady = true;
     } catch (error) {
       console.error(
         '[virtual-background] Failed to initialize MediaPipe segmenter:',
         error,
       );
-      this.isSegmenterReady = false;
     }
   }
 
-  /**
-   * Processes a single video frame.
-   *
-   * Performs segmentation via MediaPipe and then composites the frame
-   * through the WebGL renderer to apply background effects.
-   *
-   * @param frame - The incoming frame from the processor.
-   * @param opts - The segmentation options to use.
-   *
-   * @returns A new `VideoFrame` containing the processed image.
-   */
-  private async transform(
-    frame: VideoFrame,
-    opts: SegmenterOptions,
-  ): Promise<VideoFrame> {
-    if (this.isSegmenterReady && this.segmenter) {
-      try {
-        const start = performance.now();
-        await new Promise<void>((resolve) => {
-          this.segmenter!.segmentForVideo(frame, frame.timestamp, (result) => {
-            const categoryMask = result.categoryMask!.getAsWebGLTexture();
-            const confidenceMask =
-              result.confidenceMasks![0].getAsWebGLTexture();
-
-            this.webGlRenderer.render(
-              frame,
-              opts,
-              categoryMask,
-              confidenceMask,
-            );
-
-            const now = performance.now();
-            this.segmenterDelayTotal += now - start;
-            this.frames++;
-
-            if (this.lastStatsTime === 0) {
-              this.lastStatsTime = now;
-            }
-
-            if (now - this.lastStatsTime > 1000) {
-              const delay =
-                Math.round((this.segmenterDelayTotal / this.frames) * 100) /
-                100;
-              const fps = Math.round(
-                (1000 * this.frames) / (now - this.lastStatsTime),
-              );
-
-              this.hooks.onStats?.({ delay, fps, timestamp: now });
-
-              this.lastStatsTime = now;
-              this.segmenterDelayTotal = 0;
-              this.frames = 0;
-            }
-
-            resolve();
-          });
-        });
-      } catch (error) {
-        console.error('[virtual-background] Error during segmentation:', error);
-      }
-    }
-
-    return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
-  }
-
   private async loadBackground(url: string | undefined) {
     if (!url) {
       return;
@@ -271,6 +266,5 @@ export class VirtualBackground {
       this.segmenter.close();
       this.segmenter = null;
     }
-    this.isSegmenterReady = false;
   }
 }