feat(web): show detected faces in spherical photos (#23974)

Author: meesfrensel

pnpm-lock.yaml

@@ -31,11 +31,12 @@
     "@immich/ui": "^0.43.0",
     "@mapbox/mapbox-gl-rtl-text": "0.2.3",
     "@mdi/js": "^7.4.47",
-    "@photo-sphere-viewer/core": "^5.11.5",
-    "@photo-sphere-viewer/equirectangular-video-adapter": "^5.11.5",
-    "@photo-sphere-viewer/resolution-plugin": "^5.11.5",
-    "@photo-sphere-viewer/settings-plugin": "^5.11.5",
-    "@photo-sphere-viewer/video-plugin": "^5.11.5",
+    "@photo-sphere-viewer/core": "^5.14.0",
+    "@photo-sphere-viewer/equirectangular-video-adapter": "^5.14.0",
+    "@photo-sphere-viewer/markers-plugin": "^5.14.0",
+    "@photo-sphere-viewer/resolution-plugin": "^5.14.0",
+    "@photo-sphere-viewer/settings-plugin": "^5.14.0",
+    "@photo-sphere-viewer/video-plugin": "^5.14.0",
     "@types/geojson": "^7946.0.16",
     "@zoom-image/core": "^0.41.0",
     "@zoom-image/svelte": "^0.3.0",

web/package.json

@@ -1,4 +1,5 @@
 <script lang="ts">
+  import { boundingBoxesArray, type Faces } from '$lib/stores/people.store';
   import { alwaysLoadOriginalFile } from '$lib/stores/preferences.store';
   import {
     EquirectangularAdapter,
@@ -8,11 +9,21 @@
     type PluginConstructor,
   } from '@photo-sphere-viewer/core';
   import '@photo-sphere-viewer/core/index.css';
+  import { MarkersPlugin } from '@photo-sphere-viewer/markers-plugin';
+  import '@photo-sphere-viewer/markers-plugin/index.css';
   import { ResolutionPlugin } from '@photo-sphere-viewer/resolution-plugin';
   import { SettingsPlugin } from '@photo-sphere-viewer/settings-plugin';
   import '@photo-sphere-viewer/settings-plugin/index.css';
   import { onDestroy, onMount } from 'svelte';
 
+  // Adapted as well as possible from classlist 'border-solid border-white border-3 rounded-lg'
+  const FACE_BOX_SVG_STYLE = {
+    fill: 'rgba(0, 0, 0, 0)',
+    stroke: '#ffffff',
+    strokeWidth: '3px',
+    strokeLinejoin: 'round',
+  };
+
   interface Props {
     panorama: string | { source: string };
     originalPanorama?: string | { source: string };
@@ -26,6 +37,62 @@
   let container: HTMLDivElement | undefined = $state();
   let viewer: Viewer;
 
+  let animationInProgress: { cancel: () => void } | undefined;
+  let previousFaces: Faces[] = [];
+
+  const boundingBoxesUnsubscribe = boundingBoxesArray.subscribe((faces: Faces[]) => {
+    // Debounce; don't do anything when the data didn't actually change.
+    if (faces === previousFaces) {
+      return;
+    }
+    previousFaces = faces;
+
+    if (animationInProgress) {
+      animationInProgress.cancel();
+      animationInProgress = undefined;
+    }
+    if (!viewer || !viewer.state.textureData || !viewer.getPlugin(MarkersPlugin)) {
+      return;
+    }
+    const markersPlugin = viewer.getPlugin<MarkersPlugin>(MarkersPlugin);
+
+    // croppedWidth is the size of the texture, which might be cropped to be less than 360/180 degrees.
+    // This is what we want because the facial recognition is done on the image, not the sphere.
+    const currentTextureWidth = viewer.state.textureData.panoData.croppedWidth;
+
+    markersPlugin.clearMarkers();
+    for (const [index, face] of faces.entries()) {
+      const { boundingBoxX1: x1, boundingBoxY1: y1, boundingBoxX2: x2, boundingBoxY2: y2 } = face;
+      const ratio = currentTextureWidth / face.imageWidth;
+      // Pixel values are translated to spherical coordinates and only then added to the panorama;
+      // no need to recalculate when the texture image changes to the original size.
+      markersPlugin.addMarker({
+        id: `face_${index}`,
+        polygonPixels: [
+          [x1 * ratio, y1 * ratio],
+          [x2 * ratio, y1 * ratio],
+          [x2 * ratio, y2 * ratio],
+          [x1 * ratio, y2 * ratio],
+        ],
+        svgStyle: FACE_BOX_SVG_STYLE,
+      });
+    }
+
+    // Smoothly pan to the highlighted (hovered-over) face.
+    if (faces.length === 1) {
+      const { boundingBoxX1: x1, boundingBoxY1: y1, boundingBoxX2: x2, boundingBoxY2: y2, imageWidth: w } = faces[0];
+      const ratio = currentTextureWidth / w;
+      const x = ((x1 + x2) * ratio) / 2;
+      const y = ((y1 + y2) * ratio) / 2;
+      animationInProgress = viewer.animate({
+        textureX: x,
+        textureY: y,
+        zoom: Math.min(viewer.getZoomLevel(), 75),
+        speed: 500, // duration in ms
+      });
+    }
+  });
+
   onMount(() => {
     if (!container) {
       return;
@@ -34,6 +101,7 @@
     viewer = new Viewer({
       adapter,
       plugins: [
+        MarkersPlugin,
         SettingsPlugin,
         [
           ResolutionPlugin,
@@ -68,7 +136,7 @@
       zoomSpeed: 0.5,
       fisheye: false,
     });
-    const resolutionPlugin = viewer.getPlugin(ResolutionPlugin) as ResolutionPlugin;
+    const resolutionPlugin = viewer.getPlugin<ResolutionPlugin>(ResolutionPlugin);
     const zoomHandler = ({ zoomLevel }: events.ZoomUpdatedEvent) => {
       // zoomLevel range: [0, 100]
       if (Math.round(zoomLevel) >= 75) {
@@ -89,6 +157,7 @@
     if (viewer) {
       viewer.destroy();
     }
+    boundingBoxesUnsubscribe();
   });
 </script>