refactor: create DisplayMetrics so overlays work with object-fit or explicit positions

web/src/lib/components/asset-viewer/face-editor/face-editor.svelte

@@ -3,6 +3,7 @@
   import { assetViewingStore } from '$lib/stores/asset-viewing.store';
   import { isFaceEditMode } from '$lib/stores/face-edit.svelte';
   import { getPeopleThumbnailUrl } from '$lib/utils';
+  import { getDisplayMetrics, getNaturalSize } from '$lib/utils/container-utils';
   import { handleError } from '$lib/utils/handle-error';
   import { createFace, getAllPeople, type PersonResponseDto } from '@immich/sdk';
   import { Button, Input, modalManager, toastManager } from '@immich/ui';
@@ -78,17 +79,13 @@
   });
 
   $effect(() => {
-    const { actualWidth, actualHeight } = getContainedSize(htmlElement);
-    const offsetArea = {
-      width: (containerWidth - actualWidth) / 2,
-      height: (containerHeight - actualHeight) / 2,
-    };
+    const metrics = getDisplayMetrics(htmlElement);
 
     const imageBoundingBox = {
-      top: offsetArea.height,
-      left: offsetArea.width,
-      width: containerWidth - offsetArea.width * 2,
-      height: containerHeight - offsetArea.height * 2,
+      top: metrics.offsetY,
+      left: metrics.offsetX,
+      width: metrics.displayWidth,
+      height: metrics.displayHeight,
     };
 
     if (!canvas) {
@@ -113,32 +110,6 @@
     positionFaceSelector();
   });
 
-  const getContainedSize = (
-    img: HTMLImageElement | HTMLVideoElement,
-  ): { actualWidth: number; actualHeight: number } => {
-    if (img instanceof HTMLImageElement) {
-      const ratio = img.naturalWidth / img.naturalHeight;
-      let actualWidth = img.height * ratio;
-      let actualHeight = img.height;
-      if (actualWidth > img.width) {
-        actualWidth = img.width;
-        actualHeight = img.width / ratio;
-      }
-      return { actualWidth, actualHeight };
-    } else if (img instanceof HTMLVideoElement) {
-      const ratio = img.videoWidth / img.videoHeight;
-      let actualWidth = img.clientHeight * ratio;
-      let actualHeight = img.clientHeight;
-      if (actualWidth > img.clientWidth) {
-        actualWidth = img.clientWidth;
-        actualHeight = img.clientWidth / ratio;
-      }
-      return { actualWidth, actualHeight };
-    }
-
-    return { actualWidth: 0, actualHeight: 0 };
-  };
-
   const cancel = () => {
     isFaceEditMode.value = false;
   };
@@ -229,48 +200,22 @@
     }
 
     const { left, top, width, height } = faceRect.getBoundingRect();
-    const { actualWidth, actualHeight } = getContainedSize(htmlElement);
+    const metrics = getDisplayMetrics(htmlElement);
+    const natural = getNaturalSize(htmlElement);
 
-    const offsetArea = {
-      width: (containerWidth - actualWidth) / 2,
-      height: (containerHeight - actualHeight) / 2,
+    const x1Coeff = (left - metrics.offsetX) / metrics.displayWidth;
+    const y1Coeff = (top - metrics.offsetY) / metrics.displayHeight;
+    const x2Coeff = (left + width - metrics.offsetX) / metrics.displayWidth;
+    const y2Coeff = (top + height - metrics.offsetY) / metrics.displayHeight;
+
+    return {
+      imageWidth: natural.width,
+      imageHeight: natural.height,
+      x: Math.floor(x1Coeff * natural.width),
+      y: Math.floor(y1Coeff * natural.height),
+      width: Math.floor((x2Coeff - x1Coeff) * natural.width),
+      height: Math.floor((y2Coeff - y1Coeff) * natural.height),
     };
-
-    const x1Coeff = (left - offsetArea.width) / actualWidth;
-    const y1Coeff = (top - offsetArea.height) / actualHeight;
-    const x2Coeff = (left + width - offsetArea.width) / actualWidth;
-    const y2Coeff = (top + height - offsetArea.height) / actualHeight;
-
-    // transpose to the natural image location
-    if (htmlElement instanceof HTMLImageElement) {
-      const x1 = x1Coeff * htmlElement.naturalWidth;
-      const y1 = y1Coeff * htmlElement.naturalHeight;
-      const x2 = x2Coeff * htmlElement.naturalWidth;
-      const y2 = y2Coeff * htmlElement.naturalHeight;
-
-      return {
-        imageWidth: htmlElement.naturalWidth,
-        imageHeight: htmlElement.naturalHeight,
-        x: Math.floor(x1),
-        y: Math.floor(y1),
-        width: Math.floor(x2 - x1),
-        height: Math.floor(y2 - y1),
-      };
-    } else if (htmlElement instanceof HTMLVideoElement) {
-      const x1 = x1Coeff * htmlElement.videoWidth;
-      const y1 = y1Coeff * htmlElement.videoHeight;
-      const x2 = x2Coeff * htmlElement.videoWidth;
-      const y2 = y2Coeff * htmlElement.videoHeight;
-
-      return {
-        imageWidth: htmlElement.videoWidth,
-        imageHeight: htmlElement.videoHeight,
-        x: Math.floor(x1),
-        y: Math.floor(y1),
-        width: Math.floor(x2 - x1),
-        height: Math.floor(y2 - y1),
-      };
-    }
   };
 
   const tagFace = async (person: PersonResponseDto) => {

web/src/lib/components/asset-viewer/photo-viewer.svelte

@@ -15,6 +15,7 @@
   import { SlideshowLook, SlideshowState, slideshowLookCssMapping, slideshowStore } from '$lib/stores/slideshow.store';
   import { getAssetUrl, targetImageSize as getTargetImageSize, handlePromiseError } from '$lib/utils';
   import { canCopyImageToClipboard, copyImageToClipboard } from '$lib/utils/asset-utils';
+  import { type DisplayMetrics, getDisplayMetrics } from '$lib/utils/container-utils';
   import { handleError } from '$lib/utils/handle-error';
   import { getOcrBoundingBoxes } from '$lib/utils/ocr-utils';
   import { getBoundingBox } from '$lib/utils/people-utils';
@@ -52,6 +53,7 @@
   let imageLoaded: boolean = $state(false);
   let originalImageLoaded: boolean = $state(false);
   let imageError: boolean = $state(false);
+  let visibleImageReady: boolean = $state(false);
 
   let loader = $state<HTMLImageElement>();
 
@@ -67,11 +69,23 @@
     $boundingBoxesArray = [];
   });
 
-  let ocrBoxes = $derived(
-    ocrManager.showOverlay && assetViewerManager.imgRef
-      ? getOcrBoundingBoxes(ocrManager.data, assetViewerManager.zoomState, assetViewerManager.imgRef)
-      : [],
-  );
+  const overlayMetrics = $derived.by((): DisplayMetrics => {
+    if (!assetViewerManager.imgRef || !visibleImageReady) {
+      return { displayWidth: 0, displayHeight: 0, offsetX: 0, offsetY: 0 };
+    }
+
+    const baseMetrics = getDisplayMetrics(assetViewerManager.imgRef);
+    const zoom = assetViewerManager.zoomState;
+
+    return {
+      displayWidth: baseMetrics.displayWidth * zoom.currentZoom,
+      displayHeight: baseMetrics.displayHeight * zoom.currentZoom,
+      offsetX: baseMetrics.offsetX * zoom.currentZoom + zoom.currentPositionX,
+      offsetY: baseMetrics.offsetY * zoom.currentZoom + zoom.currentPositionY,
+    };
+  });
+
+  let ocrBoxes = $derived(ocrManager.showOverlay ? getOcrBoundingBoxes(ocrManager.data, overlayMetrics) : []);
 
   let isOcrActive = $derived(ocrManager.showOverlay);
 
@@ -176,6 +190,7 @@
         imageLoaded = false;
         originalImageLoaded = false;
         imageError = false;
+        visibleImageReady = false;
       });
     }
     lastUrl = imageLoaderUrl;
@@ -226,6 +241,7 @@
       <img
         bind:this={assetViewerManager.imgRef}
         src={imageLoaderUrl}
+        onload={() => (visibleImageReady = true)}
         alt={$getAltText(toTimelineAsset(asset))}
         class="h-full w-full {$slideshowState === SlideshowState.None
           ? 'object-contain'
@@ -233,7 +249,7 @@
         draggable="false"
       />
       <!-- eslint-disable-next-line svelte/require-each-key -->
-      {#each getBoundingBox($boundingBoxesArray, assetViewerManager.zoomState, assetViewerManager.imgRef) as boundingbox}
+      {#each getBoundingBox($boundingBoxesArray, overlayMetrics) as boundingbox}
         <div
           class="absolute border-solid border-white border-3 rounded-lg"
           style="top: {boundingbox.top}px; left: {boundingbox.left}px; height: {boundingbox.height}px; width: {boundingbox.width}px;"

web/src/lib/utils/container-utils.ts

@@ -0,0 +1,47 @@
+export interface DisplayMetrics {
+  displayWidth: number;
+  displayHeight: number;
+  offsetX: number;
+  offsetY: number;
+}
+
+export const getContainedSize = (element: HTMLImageElement | HTMLVideoElement): { width: number; height: number } => {
+  if (element instanceof HTMLVideoElement) {
+    const ratio = element.videoWidth / element.videoHeight;
+    let width = element.clientHeight * ratio;
+    let height = element.clientHeight;
+    if (width > element.clientWidth) {
+      width = element.clientWidth;
+      height = element.clientWidth / ratio;
+    }
+    return { width, height };
+  }
+
+  const ratio = element.naturalWidth / element.naturalHeight;
+  let width = element.height * ratio;
+  let height = element.height;
+  if (width > element.width) {
+    width = element.width;
+    height = element.width / ratio;
+  }
+  return { width, height };
+};
+
+export const getDisplayMetrics = (element: HTMLImageElement | HTMLVideoElement): DisplayMetrics => {
+  const { width: displayWidth, height: displayHeight } = getContainedSize(element);
+  const clientWidth = element instanceof HTMLVideoElement ? element.clientWidth : element.width;
+  const clientHeight = element instanceof HTMLVideoElement ? element.clientHeight : element.height;
+  return {
+    displayWidth,
+    displayHeight,
+    offsetX: (clientWidth - displayWidth) / 2,
+    offsetY: (clientHeight - displayHeight) / 2,
+  };
+};
+
+export const getNaturalSize = (element: HTMLImageElement | HTMLVideoElement): { width: number; height: number } => {
+  if (element instanceof HTMLVideoElement) {
+    return { width: element.videoWidth, height: element.videoHeight };
+  }
+  return { width: element.naturalWidth, height: element.naturalHeight };
+};

web/src/lib/utils/ocr-utils.ts

@@ -1,16 +1,5 @@
 import type { OcrBoundingBox } from '$lib/stores/ocr.svelte';
-import type { ZoomImageWheelState } from '@zoom-image/core';
-
-const getContainedSize = (img: HTMLImageElement): { width: number; height: number } => {
-  const ratio = img.naturalWidth / img.naturalHeight;
-  let width = img.height * ratio;
-  let height = img.height;
-  if (width > img.width) {
-    width = img.width;
-    height = img.width / ratio;
-  }
-  return { width, height };
-};
+import type { DisplayMetrics } from '$lib/utils/container-utils';
 
 export type Point = {
   x: number;
@@ -66,53 +55,17 @@ export const calculateBoundingBoxMatrix = (points: Point[]): { matrix: number[];
   return { matrix, width, height };
 };
 
-/**
- * Convert normalized OCR coordinates to screen coordinates
- * OCR coordinates are normalized (0-1) and represent the 4 corners of a rotated rectangle
- */
-export const getOcrBoundingBoxes = (
-  ocrData: OcrBoundingBox[],
-  zoom: ZoomImageWheelState,
-  photoViewer: HTMLImageElement | null,
-): OcrBox[] => {
-  if (photoViewer === null || !photoViewer.naturalWidth || !photoViewer.naturalHeight) {
-    return [];
-  }
-
-  const clientHeight = photoViewer.clientHeight;
-  const clientWidth = photoViewer.clientWidth;
-  const { width, height } = getContainedSize(photoViewer);
-
-  const offset = {
-    x: ((clientWidth - width) / 2) * zoom.currentZoom + zoom.currentPositionX,
-    y: ((clientHeight - height) / 2) * zoom.currentZoom + zoom.currentPositionY,
-  };
-
-  return getOcrBoundingBoxesAtSize(
-    ocrData,
-    { width: width * zoom.currentZoom, height: height * zoom.currentZoom },
-    offset,
-  );
-};
-
-export const getOcrBoundingBoxesAtSize = (
-  ocrData: OcrBoundingBox[],
-  targetSize: { width: number; height: number },
-  offset?: Point,
-) => {
+export const getOcrBoundingBoxes = (ocrData: OcrBoundingBox[], metrics: DisplayMetrics): OcrBox[] => {
   const boxes: OcrBox[] = [];
-
   for (const ocr of ocrData) {
-    // Convert normalized coordinates (0-1) to actual pixel positions
-    // OCR provides 4 corners of a potentially rotated rectangle
     const points = [
       { x: ocr.x1, y: ocr.y1 },
       { x: ocr.x2, y: ocr.y2 },
       { x: ocr.x3, y: ocr.y3 },
       { x: ocr.x4, y: ocr.y4 },
     ].map((point) => ({
-      x: targetSize.width * point.x + (offset?.x ?? 0),
-      y: targetSize.height * point.y + (offset?.y ?? 0),
+      x: point.x * metrics.displayWidth + metrics.offsetX,
+      y: point.y * metrics.displayHeight + metrics.offsetY,
     }));
 
     boxes.push({

web/src/lib/utils/people-utils.ts

@@ -1,65 +1,27 @@
 import type { Faces } from '$lib/stores/people.store';
 import { getAssetMediaUrl } from '$lib/utils';
+import type { DisplayMetrics } from '$lib/utils/container-utils';
 import { AssetTypeEnum, type AssetFaceResponseDto } from '@immich/sdk';
-import type { ZoomImageWheelState } from '@zoom-image/core';
 
-const getContainedSize = (img: HTMLImageElement): { width: number; height: number } => {
-  const ratio = img.naturalWidth / img.naturalHeight;
-  let width = img.height * ratio;
-  let height = img.height;
-  if (width > img.width) {
-    width = img.width;
-    height = img.width / ratio;
-  }
-  return { width, height };
-};
-
-export interface boundingBox {
+export interface BoundingBox {
   top: number;
   left: number;
   width: number;
   height: number;
 }
 
-export const getBoundingBox = (
-  faces: Faces[],
-  zoom: ZoomImageWheelState,
-  photoViewer: HTMLImageElement | undefined,
-): boundingBox[] => {
-  const boxes: boundingBox[] = [];
-
-  if (!photoViewer) {
-    return boxes;
-  }
-  const clientHeight = photoViewer.clientHeight;
-  const clientWidth = photoViewer.clientWidth;
-
-  const { width, height } = getContainedSize(photoViewer);
+export const getBoundingBox = (faces: Faces[], metrics: DisplayMetrics): BoundingBox[] => {
+  const boxes: BoundingBox[] = [];
 
   for (const face of faces) {
-    /*
-     *
-     * Create the coordinates of the box based on the displayed image.
-     * The coordinates must take into account margins due to the 'object-fit: contain;' css property of the photo-viewer.
-     *
-     */
+    const scaleX = metrics.displayWidth / face.imageWidth;
+    const scaleY = metrics.displayHeight / face.imageHeight;
+
     const coordinates = {
-      x1:
-        (width / face.imageWidth) * zoom.currentZoom * face.boundingBoxX1 +
-        ((clientWidth - width) / 2) * zoom.currentZoom +
-        zoom.currentPositionX,
-      x2:
-        (width / face.imageWidth) * zoom.currentZoom * face.boundingBoxX2 +
-        ((clientWidth - width) / 2) * zoom.currentZoom +
-        zoom.currentPositionX,
-      y1:
-        (height / face.imageHeight) * zoom.currentZoom * face.boundingBoxY1 +
-        ((clientHeight - height) / 2) * zoom.currentZoom +
-        zoom.currentPositionY,
-      y2:
-        (height / face.imageHeight) * zoom.currentZoom * face.boundingBoxY2 +
-        ((clientHeight - height) / 2) * zoom.currentZoom +
-        zoom.currentPositionY,
+      x1: scaleX * face.boundingBoxX1 + metrics.offsetX,
+      x2: scaleX * face.boundingBoxX2 + metrics.offsetX,
+      y1: scaleY * face.boundingBoxY1 + metrics.offsetY,
+      y2: scaleY * face.boundingBoxY2 + metrics.offsetY,
     };
 
     boxes.push({
@@ -69,6 +31,7 @@ export const getBoundingBox = (
       height: Math.round(coordinates.y2 - coordinates.y1),
     });
   }
+
   return boxes;
 };