Merge branch 'main' of https://github.com/immich-app/immich into feat/crawl-wrapper

pnpm-lock.yaml

@@ -343,6 +343,9 @@ importers:
       '@extism/extism':
         specifier: 2.0.0-rc13
         version: 2.0.0-rc13
+      '@immich/walkrs':
+        specifier: ^0.0.13
+        version: 0.0.13
       '@nestjs/bullmq':
         specifier: ^11.0.1
         version: 11.0.4(@nestjs/common@11.1.13(class-transformer@0.5.1)(class-validator@0.14.3)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.13)(bullmq@5.68.0)
@@ -454,9 +457,6 @@ importers:
       express:
         specifier: ^5.1.0
         version: 5.2.1
-      fast-glob:
-        specifier: ^3.3.2
-        version: 3.3.3
       fluent-ffmpeg:
         specifier: ^2.1.2
         version: 2.1.3
@@ -3023,6 +3023,10 @@ packages:
     peerDependencies:
       svelte: ^5.0.0
 
+  '@immich/walkrs@0.0.13':
+    resolution: {integrity: sha512-qKDoXFgy3d2Z7SIJBn25BcNyQnPLAp2zZEBcewpWxG5+qAXDPi3M3sweJ9qJ11Eha+YlmpUO3c8yd5CCBeq96A==}
+    engines: {pnpm: '>=10.0.0'}
+
   '@inquirer/ansi@1.0.2':
     resolution: {integrity: sha512-S8qNSZiYzFd0wAcyG5AXCvUHC5Sr7xpZ9wZ2py9XR88jUz8wooStVx5M6dRzczbBWjic9NP7+rY0Xi7qqK/aMQ==}
     engines: {node: '>=18'}
@@ -14835,6 +14839,8 @@ snapshots:
     transitivePeerDependencies:
       - '@sveltejs/kit'
 
+  '@immich/walkrs@0.0.13': {}
+
   '@inquirer/ansi@1.0.2': {}
 
   '@inquirer/checkbox@4.3.2(@types/node@24.10.13)':

pnpm-workspace.yaml

@@ -9,6 +9,9 @@ packages:
   - plugins
   - web
   - .github
+
+dedupePeerDependents: false
+
 ignoredBuiltDependencies:
   - '@nestjs/core'
   - '@parcel/watcher'
@@ -25,42 +28,48 @@ ignoredBuiltDependencies:
   - protobufjs
   - ssh2
   - utimes
+
+injectWorkspacePackages: true
+
 onlyBuiltDependencies:
   - sharp
   - '@tailwindcss/oxide'
   - bcrypt
+
 overrides:
   canvas: 2.11.2
   sharp: ^0.34.5
+
 packageExtensions:
-  nestjs-kysely:
+  '@immich/ui':
     dependencies:
-      tslib: '*'
-  nestjs-otel:
-    dependencies:
-      tslib: '*'
+      tailwindcss: '>=4.1'
   '@photo-sphere-viewer/equirectangular-video-adapter':
     dependencies:
       three: '*'
   '@photo-sphere-viewer/video-plugin':
     dependencies:
       three: '*'
-  sharp:
-    dependencies:
-      node-addon-api: '*'
-      node-gyp: '*'
-  '@immich/ui':
-    dependencies:
-      tailwindcss: '>=4.1'
-  tailwind-variants:
-    dependencies:
-      tailwindcss: '>=4.1'
   bcrypt:
     dependencies:
       node-addon-api: '*'
       node-gyp: '*'
-dedupePeerDependents: false
+  nestjs-kysely:
+    dependencies:
+      tslib: '*'
+  nestjs-otel:
+    dependencies:
+      tslib: '*'
+  sharp:
+    dependencies:
+      node-addon-api: '*'
+      node-gyp: '*'
+  tailwind-variants:
+    dependencies:
+      tailwindcss: '>=4.1'
+
 preferWorkspacePackages: true
-injectWorkspacePackages: true
+
 shamefullyHoist: false
+
 verifyDepsBeforeRun: install

server/package.json

@@ -35,6 +35,7 @@
   },
   "dependencies": {
     "@extism/extism": "2.0.0-rc13",
+    "@immich/walkrs": "^0.0.13",
     "@nestjs/bullmq": "^11.0.1",
     "@nestjs/common": "^11.0.4",
     "@nestjs/core": "^11.0.4",
@@ -72,7 +73,6 @@
     "cron": "4.4.0",
     "exiftool-vendored": "^34.3.0",
     "express": "^5.1.0",
-    "fast-glob": "^3.3.2",
     "fluent-ffmpeg": "^2.1.2",
     "geo-tz": "^8.0.0",
     "handlebars": "^4.7.8",

server/src/dtos/library.dto.ts

@@ -54,16 +54,12 @@ export class UpdateLibraryDto {
   exclusionPatterns?: string[];
 }
 
-export interface CrawlOptionsDto {
-  pathsToCrawl: string[];
+export interface WalkOptionsDto {
+  pathsToWalk: string[];
   includeHidden?: boolean;
   exclusionPatterns?: string[];
 }
 
-export interface WalkOptionsDto extends CrawlOptionsDto {
-  take: number;
-}
-
 export class ValidateLibraryDto {
   @ApiPropertyOptional({ description: 'Import paths to validate (max 128)' })
   @Optional()

server/src/repositories/asset.repository.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common';
 import { ExpressionBuilder, Insertable, Kysely, NotNull, Selectable, sql, Updateable, UpdateResult } from 'kysely';
 import { jsonArrayFrom } from 'kysely/helpers/postgres';
 import { isEmpty, isUndefined, omitBy } from 'lodash';
+
 import { InjectKysely } from 'nestjs-kysely';
 import { LockableProperty, Stack } from 'src/database';
 import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';

server/src/repositories/storage.repository.spec.ts

@@ -1,208 +0,0 @@
-import mockfs from 'mock-fs';
-import { CrawlOptionsDto } from 'src/dtos/library.dto';
-import { LoggingRepository } from 'src/repositories/logging.repository';
-import { StorageRepository } from 'src/repositories/storage.repository';
-import { automock } from 'test/utils';
-
-interface Test {
-  test: string;
-  options: CrawlOptionsDto;
-  files: Record<string, boolean>;
-}
-
-const cwd = process.cwd();
-
-const tests: Test[] = [
-  {
-    test: 'should return empty when crawling an empty path list',
-    options: {
-      pathsToCrawl: [],
-    },
-    files: {},
-  },
-  {
-    test: 'should crawl a single path',
-    options: {
-      pathsToCrawl: ['/photos/'],
-    },
-    files: {
-      '/photos/image.jpg': true,
-    },
-  },
-  {
-    test: 'should exclude by file extension',
-    options: {
-      pathsToCrawl: ['/photos/'],
-      exclusionPatterns: ['**/*.tif'],
-    },
-    files: {
-      '/photos/image.jpg': true,
-      '/photos/image.tif': false,
-    },
-  },
-  {
-    test: 'should exclude by file extension without case sensitivity',
-    options: {
-      pathsToCrawl: ['/photos/'],
-      exclusionPatterns: ['**/*.TIF'],
-    },
-    files: {
-      '/photos/image.jpg': true,
-      '/photos/image.tif': false,
-    },
-  },
-  {
-    test: 'should exclude by folder',
-    options: {
-      pathsToCrawl: ['/photos/'],
-      exclusionPatterns: ['**/raw/**'],
-    },
-    files: {
-      '/photos/image.jpg': true,
-      '/photos/raw/image.jpg': false,
-      '/photos/raw2/image.jpg': true,
-      '/photos/folder/raw/image.jpg': false,
-      '/photos/crawl/image.jpg': true,
-    },
-  },
-  {
-    test: 'should crawl multiple paths',
-    options: {
-      pathsToCrawl: ['/photos/', '/images/', '/albums/'],
-    },
-    files: {
-      '/photos/image1.jpg': true,
-      '/images/image2.jpg': true,
-      '/albums/image3.jpg': true,
-    },
-  },
-  {
-    test: 'should crawl a single path without trailing slash',
-    options: {
-      pathsToCrawl: ['/photos'],
-    },
-    files: {
-      '/photos/image.jpg': true,
-    },
-  },
-  {
-    test: 'should crawl a single path',
-    options: {
-      pathsToCrawl: ['/photos/'],
-    },
-    files: {
-      '/photos/image.jpg': true,
-      '/photos/subfolder/image1.jpg': true,
-      '/photos/subfolder/image2.jpg': true,
-      '/image1.jpg': false,
-    },
-  },
-  {
-    test: 'should filter file extensions',
-    options: {
-      pathsToCrawl: ['/photos/'],
-    },
-    files: {
-      '/photos/image.jpg': true,
-      '/photos/image.txt': false,
-      '/photos/1': false,
-    },
-  },
-  {
-    test: 'should include photo and video extensions',
-    options: {
-      pathsToCrawl: ['/photos/', '/videos/'],
-    },
-    files: {
-      '/photos/image.jpg': true,
-      '/photos/image.jpeg': true,
-      '/photos/image.heic': true,
-      '/photos/image.heif': true,
-      '/photos/image.png': true,
-      '/photos/image.gif': true,
-      '/photos/image.tif': true,
-      '/photos/image.tiff': true,
-      '/photos/image.webp': true,
-      '/photos/image.dng': true,
-      '/photos/image.nef': true,
-      '/videos/video.mp4': true,
-      '/videos/video.mov': true,
-      '/videos/video.webm': true,
-    },
-  },
-  {
-    test: 'should check file extensions without case sensitivity',
-    options: {
-      pathsToCrawl: ['/photos/'],
-    },
-    files: {
-      '/photos/image.jpg': true,
-      '/photos/image.Jpg': true,
-      '/photos/image.jpG': true,
-      '/photos/image.JPG': true,
-      '/photos/image.jpEg': true,
-      '/photos/image.TIFF': true,
-      '/photos/image.tif': true,
-      '/photos/image.dng': true,
-      '/photos/image.NEF': true,
-    },
-  },
-  {
-    test: 'should normalize the path',
-    options: {
-      pathsToCrawl: ['/photos/1/../2'],
-    },
-    files: {
-      '/photos/1/image.jpg': false,
-      '/photos/2/image.jpg': true,
-    },
-  },
-  {
-    test: 'should return absolute paths',
-    options: {
-      pathsToCrawl: ['photos'],
-    },
-    files: {
-      [`${cwd}/photos/1.jpg`]: true,
-      [`${cwd}/photos/2.jpg`]: true,
-      [`/photos/3.jpg`]: false,
-    },
-  },
-  {
-    test: 'should support special characters in paths',
-    options: {
-      pathsToCrawl: ['/photos (new)'],
-    },
-    files: {
-      ['/photos (new)/1.jpg']: true,
-    },
-  },
-];
-
-describe(StorageRepository.name, () => {
-  let sut: StorageRepository;
-
-  beforeEach(() => {
-    // eslint-disable-next-line no-sparse-arrays
-    sut = new StorageRepository(automock(LoggingRepository, { args: [, { getEnv: () => ({}) }], strict: false }));
-  });
-
-  afterEach(() => {
-    mockfs.restore();
-  });
-
-  describe('crawl', () => {
-    for (const { test, options, files } of tests) {
-      it(test, async () => {
-        mockfs(Object.fromEntries(Object.keys(files).map((file) => [file, ''])));
-
-        const actual = await sut.crawl(options);
-        const expected = Object.entries(files)
-          .filter((entry) => entry[1])
-          .map(([file]) => file);
-
-        expect(actual.toSorted()).toEqual(expected.toSorted());
-      });
-    }
-  });
-});

server/src/repositories/storage.repository.ts

@@ -1,13 +1,13 @@
+import type { WalkItem } from '@immich/walkrs' with { 'resolution-mode': 'import' };
 import { Injectable } from '@nestjs/common';
 import archiver from 'archiver';
 import chokidar, { ChokidarOptions } from 'chokidar';
-import { escapePath, glob, globStream } from 'fast-glob';
 import { constants, createReadStream, createWriteStream, existsSync, mkdirSync, ReadOptionsWithBuffer } from 'node:fs';
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import { PassThrough, Readable, Writable } from 'node:stream';
 import { createGunzip, createGzip } from 'node:zlib';
-import { CrawlOptionsDto, WalkOptionsDto } from 'src/dtos/library.dto';
+import { WalkOptionsDto } from 'src/dtos/library.dto';
 import { LoggingRepository } from 'src/repositories/logging.repository';
 import { mimeTypes } from 'src/utils/mime-types';
 
@@ -198,54 +198,22 @@ export class StorageRepository {
     };
   }
 
-  crawl(crawlOptions: CrawlOptionsDto): Promise<string[]> {
-    const { pathsToCrawl, exclusionPatterns, includeHidden } = crawlOptions;
-    if (pathsToCrawl.length === 0) {
-      return Promise.resolve([]);
+  async *walk(walkOptions: WalkOptionsDto): AsyncGenerator<WalkItem[], void, unknown> {
+    const { pathsToWalk, exclusionPatterns, includeHidden } = walkOptions;
+    if (pathsToWalk.length === 0) {
+      return;
     }
 
-    const globbedPaths = pathsToCrawl.map((path) => this.asGlob(path));
+    const { walk } = await import('@immich/walkrs');
 
-    return glob(globbedPaths, {
-      absolute: true,
-      caseSensitiveMatch: false,
-      onlyFiles: true,
-      dot: includeHidden,
-      ignore: exclusionPatterns,
+    yield* walk({
+      paths: pathsToWalk.map((p) => path.resolve(p)),
+      includeHidden: includeHidden ?? false,
+      exclusionPatterns,
+      extensions: mimeTypes.getSupportedFileExtensions(),
     });
   }
 
-  async *walk(walkOptions: WalkOptionsDto): AsyncGenerator<string[]> {
-    const { pathsToCrawl, exclusionPatterns, includeHidden } = walkOptions;
-    if (pathsToCrawl.length === 0) {
-      async function* emptyGenerator() {}
-      return emptyGenerator();
-    }
-
-    const globbedPaths = pathsToCrawl.map((path) => this.asGlob(path));
-
-    const stream = globStream(globbedPaths, {
-      absolute: true,
-      caseSensitiveMatch: false,
-      onlyFiles: true,
-      dot: includeHidden,
-      ignore: exclusionPatterns,
-    });
-
-    let batch: string[] = [];
-    for await (const value of stream) {
-      batch.push(value.toString());
-      if (batch.length === walkOptions.take) {
-        yield batch;
-        batch = [];
-      }
-    }
-
-    if (batch.length > 0) {
-      yield batch;
-    }
-  }
-
   watch(paths: string[], options: ChokidarOptions, events: Partial<WatchEvents>) {
     const watcher = chokidar.watch(paths, options);
 
@@ -257,10 +225,4 @@ export class StorageRepository {
 
     return () => watcher.close();
   }
-
-  private asGlob(pathToCrawl: string): string {
-    const escapedPath = escapePath(pathToCrawl).replaceAll('"', '["]').replaceAll("'", "[']").replaceAll('`', '[`]');
-    const extensions = `*{${mimeTypes.getSupportedFileExtensions().join(',')}}`;
-    return `${escapedPath}/**/${extensions}`;
-  }
 }

server/src/services/library.service.spec.ts

@@ -1,7 +1,6 @@
 import { BadRequestException } from '@nestjs/common';
 import { Stats } from 'node:fs';
 import { defaults, SystemConfig } from 'src/config';
-import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants';
 import { mapLibrary } from 'src/dtos/library.dto';
 import { AssetType, CronJob, ImmichWorker, JobName, JobStatus } from 'src/enum';
 import { LibraryService } from 'src/services/library.service';
@@ -14,10 +13,6 @@ import { factory, newDate, newUuid } from 'test/small.factory';
 import { makeStream, newTestService, ServiceMocks } from 'test/utils';
 import { vitest } from 'vitest';
 
-async function* mockWalk() {
-  yield await Promise.resolve(['/data/user1/photo.jpg']);
-}
-
 describe(LibraryService.name, () => {
   let sut: LibraryService;
 
@@ -165,7 +160,11 @@ describe(LibraryService.name, () => {
       const library = factory.library({ importPaths: ['/foo', '/bar'] });
 
       mocks.library.get.mockResolvedValue(library);
-      mocks.storage.walk.mockImplementation(mockWalk);
+      mocks.storage.walk.mockReturnValue(
+        (async function* () {
+          yield await Promise.resolve([{ type: 'entry', path: '/data/user1/photo.jpg' }]);
+        })(),
+      );
       mocks.storage.stat.mockResolvedValue({ isDirectory: () => true } as Stats);
       mocks.storage.checkFileExists.mockResolvedValue(true);
       mocks.asset.filterNewExternalAssetPaths.mockResolvedValue(['/data/user1/photo.jpg']);
@@ -201,16 +200,20 @@ describe(LibraryService.name, () => {
       });
 
       mocks.storage.checkFileExists.mockResolvedValue(true);
-
+      mocks.storage.walk.mockReturnValue(
+        (async function* () {
+          yield await Promise.resolve([{ type: 'entry', path: '/data/user1/photo.jpg' }]);
+        })(),
+      );
       mocks.library.get.mockResolvedValue(library);
+      mocks.asset.filterNewExternalAssetPaths.mockResolvedValue(['/data/user1/photo.jpg']);
 
       await sut.handleQueueSyncFiles({ id: library.id });
 
       expect(mocks.storage.walk).toHaveBeenCalledWith({
-        pathsToCrawl: [library.importPaths[1]],
+        pathsToWalk: [library.importPaths[1]],
         exclusionPatterns: [],
         includeHidden: false,
-        take: JOBS_LIBRARY_PAGINATION_SIZE,
       });
     });
   });
@@ -220,7 +223,11 @@ describe(LibraryService.name, () => {
       const library = factory.library({ importPaths: ['/foo', '/bar'] });
 
       mocks.library.get.mockResolvedValue(library);
-      mocks.storage.walk.mockImplementation(mockWalk);
+      mocks.storage.walk.mockReturnValue(
+        (async function* () {
+          yield await Promise.resolve([{ type: 'entry', path: '/data/user1/photo.jpg' }]);
+        })(),
+      );
       mocks.storage.stat.mockResolvedValue({ isDirectory: () => true } as Stats);
       mocks.storage.checkFileExists.mockResolvedValue(true);
       mocks.asset.filterNewExternalAssetPaths.mockResolvedValue(['/data/user1/photo.jpg']);
@@ -242,33 +249,6 @@ describe(LibraryService.name, () => {
 
       await expect(sut.handleQueueSyncFiles({ id: library.id })).resolves.toBe(JobStatus.Skipped);
     });
-
-    it('should ignore import paths that do not exist', async () => {
-      const library = factory.library({ importPaths: ['/foo', '/bar'] });
-
-      mocks.storage.stat.mockImplementation((path): Promise<Stats> => {
-        if (path === library.importPaths[0]) {
-          const error = { code: 'ENOENT' } as any;
-          throw error;
-        }
-        return Promise.resolve({
-          isDirectory: () => true,
-        } as Stats);
-      });
-
-      mocks.storage.checkFileExists.mockResolvedValue(true);
-
-      mocks.library.get.mockResolvedValue(library);
-
-      await sut.handleQueueSyncFiles({ id: library.id });
-
-      expect(mocks.storage.walk).toHaveBeenCalledWith({
-        pathsToCrawl: [library.importPaths[1]],
-        exclusionPatterns: [],
-        includeHidden: false,
-        take: JOBS_LIBRARY_PAGINATION_SIZE,
-      });
-    });
   });
 
   describe('handleQueueSyncAssets', () => {
@@ -276,7 +256,11 @@ describe(LibraryService.name, () => {
       const library = factory.library();
 
       mocks.library.get.mockResolvedValue(library);
-      mocks.storage.walk.mockImplementation(async function* generator() {});
+      mocks.storage.walk.mockReturnValue(
+        (async function* () {
+          yield await Promise.resolve([]);
+        })(),
+      );
       mocks.asset.getLibraryAssetCount.mockResolvedValue(1);
       mocks.asset.detectOfflineExternalAssets.mockResolvedValue({ numUpdatedRows: 1n });
 
@@ -294,7 +278,11 @@ describe(LibraryService.name, () => {
       const library = factory.library();
 
       mocks.library.get.mockResolvedValue(library);
-      mocks.storage.walk.mockImplementation(async function* generator() {});
+      mocks.storage.walk.mockReturnValue(
+        (async function* () {
+          yield await Promise.resolve([]);
+        })(),
+      );
       mocks.asset.getLibraryAssetCount.mockResolvedValue(0);
       mocks.asset.detectOfflineExternalAssets.mockResolvedValue({ numUpdatedRows: 1n });
 
@@ -309,7 +297,11 @@ describe(LibraryService.name, () => {
       const asset = AssetFactory.create({ libraryId: library.id, isExternal: true });
 
       mocks.library.get.mockResolvedValue(library);
-      mocks.storage.walk.mockImplementation(async function* generator() {});
+      mocks.storage.walk.mockReturnValue(
+        (async function* () {
+          yield await Promise.resolve([]);
+        })(),
+      );
       mocks.library.streamAssetIds.mockReturnValue(makeStream([asset]));
       mocks.asset.getLibraryAssetCount.mockResolvedValue(1);
       mocks.asset.detectOfflineExternalAssets.mockResolvedValue({ numUpdatedRows: 0n });

server/src/services/library.service.ts

@@ -4,6 +4,7 @@ import { R_OK } from 'node:constants';
 import { Stats } from 'node:fs';
 import path, { basename, isAbsolute, parse } from 'node:path';
 import picomatch from 'picomatch';
+
 import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants';
 import { StorageCore } from 'src/cores/storage.core';
 import { OnEvent, OnJob } from 'src/decorators';
@@ -247,9 +248,11 @@ export class LibraryService extends BaseService {
       return JobStatus.Failed;
     }
 
+    const newPaths = await this.assetRepository.filterNewExternalAssetPaths(library.id, job.paths);
+
     const assetImports: Insertable<AssetTable>[] = [];
     await Promise.all(
-      job.paths.map((path) =>
+      newPaths.map((path) =>
         this.processEntity(path, library.ownerId, job.libraryId)
           .then((asset) => assetImports.push(asset))
           .catch((error: any) => this.logger.error(`Error processing ${path} for library ${job.libraryId}: ${error}`)),
@@ -394,6 +397,7 @@ export class LibraryService extends BaseService {
 
   private async processEntity(filePath: string, ownerId: string, libraryId: string) {
     const assetPath = path.normalize(filePath);
+
     const stat = await this.storageRepository.stat(assetPath);
 
     return {
@@ -636,42 +640,56 @@ export class LibraryService extends BaseService {
       return JobStatus.Skipped;
     }
 
-    const pathsOnDisk = this.storageRepository.walk({
-      pathsToCrawl: validImportPaths,
-      includeHidden: false,
-      exclusionPatterns: library.exclusionPatterns,
-      take: JOBS_LIBRARY_PAGINATION_SIZE,
-    });
-
-    let importCount = 0;
-    let crawlCount = 0;
-
     this.logger.log(`Starting disk crawl of ${validImportPaths.length} import path(s) for library ${library.id}...`);
 
-    for await (const pathBatch of pathsOnDisk) {
-      crawlCount += pathBatch.length;
-      const paths = await this.assetRepository.filterNewExternalAssetPaths(library.id, pathBatch);
+    const fileWalker = this.storageRepository.walk({
+      pathsToWalk: validImportPaths,
+      includeHidden: false, // TODO: make this configurable?
+      exclusionPatterns: library.exclusionPatterns,
+    });
 
-      if (paths.length > 0) {
-        importCount += paths.length;
+    const walkStart = Date.now();
+    let progressCounter = 0;
+    let lastLoggedMilestone = 0;
 
-        await this.jobRepository.queue({
-          name: JobName.LibrarySyncFiles,
-          data: {
-            libraryId: library.id,
-            paths,
-            progressCounter: crawlCount,
-          },
-        });
+    for await (const walkItems of fileWalker) {
+      const paths: string[] = [];
+      for (const item of walkItems) {
+        if (item.type === 'error') {
+          this.logger.warn(`Error walking ${item.path ?? 'unknown path'}: ${item.message} for library ${library.id}`);
+        } else {
+          paths.push(item.path);
+        }
       }
 
-      this.logger.log(
-        `Crawled ${crawlCount} file(s) so far: ${paths.length} of current batch of ${pathBatch.length} will be imported to library ${library.id}...`,
-      );
+      if (paths.length === 0) {
+        continue;
+      }
+
+      progressCounter += paths.length;
+
+      await this.jobRepository.queue({
+        name: JobName.LibrarySyncFiles,
+        data: {
+          libraryId: library.id,
+          paths,
+          progressCounter,
+        },
+      });
+
+      const currentMilestone = Math.floor(progressCounter / 100_000);
+      // Log every 100k files found to give some feedback on progress for large libraries
+      if (currentMilestone > lastLoggedMilestone) {
+        const roundedCount = currentMilestone * 100_000;
+        this.logger.log(
+          `Disk walk found ${roundedCount} file(s) so far (${((Date.now() - walkStart) / 1000).toFixed(2)}s elapsed) for library ${library.id}...`,
+        );
+        lastLoggedMilestone = currentMilestone;
+      }
     }
 
     this.logger.log(
-      `Finished disk crawl, ${crawlCount} file(s) found on disk and queued ${importCount} file(s) for import into ${library.id}`,
+      `Finished disk walk, ${progressCounter} file(s) found on disk in ${((Date.now() - walkStart) / 1000).toFixed(2)}s for library ${library.id}`,
     );
 
     await this.libraryRepository.update(job.id, { refreshedAt: new Date() });

server/test/medium/specs/repositories/storage.repository.spec.ts

@@ -0,0 +1,333 @@
+import type { WalkError, WalkItem } from '@immich/walkrs' with { 'resolution-mode': 'import' };
+import { Kysely } from 'kysely';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path, { join } from 'node:path';
+import { WalkOptionsDto } from 'src/dtos/library.dto';
+import { LoggingRepository } from 'src/repositories/logging.repository';
+import { StorageRepository } from 'src/repositories/storage.repository';
+import { DB } from 'src/schema';
+import { BaseService } from 'src/services/base.service';
+import { newMediumService } from 'test/medium.factory';
+import { getKyselyDB } from 'test/utils';
+
+let defaultDatabase: Kysely<DB>;
+
+interface Test {
+  test: string;
+  options: WalkOptionsDto;
+  files: Record<string, boolean>;
+}
+
+const createTestFiles = async (basePath: string, files: string[]) => {
+  await Promise.all(
+    files.map(async (file) => {
+      const fullPath = path.join(basePath, file.replace(/^\//, ''));
+      await fs.mkdir(path.dirname(fullPath), { recursive: true });
+      await fs.writeFile(fullPath, '');
+    }),
+  );
+};
+
+const tests: Test[] = [
+  {
+    test: 'should return empty when walking an empty path list',
+    options: {
+      pathsToWalk: [],
+    },
+    files: {},
+  },
+  {
+    test: 'should walk a single path',
+    options: {
+      pathsToWalk: ['/photos/'],
+    },
+    files: {
+      '/photos/image.jpg': true,
+    },
+  },
+  {
+    test: 'should exclude by file extension',
+    options: {
+      pathsToWalk: ['/photos/'],
+      exclusionPatterns: ['**/*.tif'],
+    },
+    files: {
+      '/photos/image.jpg': true,
+      '/photos/image.tif': false,
+    },
+  },
+  {
+    test: 'should exclude by file extension without case sensitivity',
+    options: {
+      pathsToWalk: ['/photos/'],
+      exclusionPatterns: ['**/*.TIF'],
+    },
+    files: {
+      '/photos/image.jpg': true,
+      '/photos/image.tif': false,
+      '/photos/image.tIf': false,
+      '/photos/image.TIF': false,
+    },
+  },
+  {
+    test: 'should exclude by folder',
+    options: {
+      pathsToWalk: ['/photos/'],
+      exclusionPatterns: ['**/raw/**'],
+    },
+    files: {
+      '/photos/image.jpg': true,
+      '/photos/raw/image.jpg': false,
+      '/photos/raw2/image.jpg': true,
+      '/photos/folder/raw/image.jpg': false,
+      '/photos/walk/image.jpg': true,
+    },
+  },
+  {
+    test: 'should walk multiple paths',
+    options: {
+      pathsToWalk: ['/photos/', '/images/', '/albums/'],
+    },
+    files: {
+      '/photos/image1.jpg': true,
+      '/images/image2.jpg': true,
+      '/albums/image3.jpg': true,
+    },
+  },
+  {
+    test: 'should walk a single path without trailing slash',
+    options: {
+      pathsToWalk: ['/photos'],
+    },
+    files: {
+      '/photos/image.jpg': true,
+    },
+  },
+  {
+    test: 'should walk a single path',
+    options: {
+      pathsToWalk: ['/photos/'],
+    },
+    files: {
+      '/photos/image.jpg': true,
+      '/photos/subfolder/image1.jpg': true,
+      '/photos/subfolder/image2.jpg': true,
+      '/image1.jpg': false,
+    },
+  },
+  {
+    test: 'should filter file extensions',
+    options: {
+      pathsToWalk: ['/photos/'],
+    },
+    files: {
+      '/photos/image.jpg': true,
+      '/photos/image.txt': false,
+      '/photos/1': false,
+    },
+  },
+  {
+    test: 'should include photo and video extensions',
+    options: {
+      pathsToWalk: ['/photos/', '/videos/'],
+    },
+    files: {
+      '/photos/image.jpg': true,
+      '/photos/image.jpeg': true,
+      '/photos/image.heic': true,
+      '/photos/image.heif': true,
+      '/photos/image.png': true,
+      '/photos/image.gif': true,
+      '/photos/image.tif': true,
+      '/photos/image.tiff': true,
+      '/photos/image.webp': true,
+      '/photos/image.dng': true,
+      '/photos/image.nef': true,
+      '/videos/video.mp4': true,
+      '/videos/video.mov': true,
+      '/videos/video.webm': true,
+    },
+  },
+  {
+    test: 'should check file extensions without case sensitivity',
+    options: {
+      pathsToWalk: ['/photos/'],
+    },
+    files: {
+      '/photos/image.jpg': true,
+      '/photos/image.Jpg': true,
+      '/photos/image.jpG': true,
+      '/photos/image.JPG': true,
+      '/photos/image.jpEg': true,
+      '/photos/image.TIFF': true,
+      '/photos/image.tif': true,
+      '/photos/image.dng': true,
+      '/photos/image.NEF': true,
+    },
+  },
+  {
+    test: 'should normalize the path',
+    options: {
+      pathsToWalk: ['/photos/1/../2'],
+    },
+    files: {
+      '/photos/1/image.jpg': false,
+      '/photos/2/image.jpg': true,
+    },
+  },
+  {
+    test: 'should support special characters in paths',
+    options: {
+      pathsToWalk: ['/photos (new)'],
+    },
+    files: {
+      ['/photos (new)/1.jpg']: true,
+    },
+  },
+];
+
+const setup = (db?: Kysely<DB>) => {
+  const { ctx } = newMediumService(BaseService, {
+    database: db || defaultDatabase,
+    real: [],
+    mock: [LoggingRepository],
+  });
+  return { sut: ctx.get(StorageRepository) };
+};
+
+beforeAll(async () => {
+  defaultDatabase = await getKyselyDB();
+});
+
+describe(StorageRepository.name, () => {
+  let sut: StorageRepository;
+
+  beforeEach(() => {
+    ({ sut } = setup());
+  });
+
+  describe('walk', () => {
+    for (const { test, options, files } of tests) {
+      describe(test, () => {
+        const fileList = Object.keys(files);
+        let tempDir: string;
+
+        beforeEach(async () => {
+          tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'immich-storage-test-'));
+          await createTestFiles(tempDir, fileList);
+        });
+
+        afterEach(async () => {
+          await fs.rm(tempDir, { recursive: true, force: true });
+        });
+
+        it('returns expected files', async () => {
+          const adjustedOptions = {
+            ...options,
+            pathsToWalk: options.pathsToWalk.map((p) => path.join(tempDir, p.replace(/^\//, ''))),
+          };
+
+          const actual: string[] = [];
+          for await (const batch of sut.walk(adjustedOptions)) {
+            for (const item of batch) {
+              if (item.type === 'entry') {
+                actual.push(item.path);
+              }
+            }
+          }
+          const expected = Object.entries(files)
+            .filter((entry) => entry[1])
+            .map(([file]) => path.join(tempDir, file.replace(/^\//, '')));
+
+          expect(actual.toSorted()).toEqual(expected.toSorted());
+        });
+      });
+    }
+
+    it('should handle access denied errors gracefully', async () => {
+      const testDir = await fs.mkdtemp(join(os.tmpdir(), 'immich-test-access-denied-'));
+      const restrictedDir = join(testDir, 'restricted');
+      const restrictedFile = join(restrictedDir, 'file.jpg');
+      const accessibleFile = join(testDir, 'accessible.jpg');
+
+      try {
+        // Create test directory structure
+        await fs.mkdir(restrictedDir, { recursive: true });
+        await fs.writeFile(accessibleFile, 'accessible content');
+        await fs.writeFile(restrictedFile, 'restricted content');
+
+        // Remove all permissions from restricted directory to simulate access denied
+        await fs.chmod(restrictedDir, 0o000);
+
+        const actual: string[] = [];
+        const errors: WalkItem[] = [];
+        for await (const batch of sut.walk({ pathsToWalk: [testDir] })) {
+          for (const item of batch) {
+            if (item.type === 'entry') {
+              actual.push(item.path);
+            } else {
+              errors.push(item);
+            }
+          }
+        }
+
+        // Should successfully walk accessible file but skip restricted directory
+        expect(actual).toContain(accessibleFile);
+        expect(actual).not.toContain(restrictedFile);
+        // Should have encountered an error for the restricted directory
+        expect(errors.length).toBe(1);
+        expect(errors.some((e) => e.type === 'error' && e.message?.includes('restricted'))).toBe(true);
+      } finally {
+        // Cleanup: restore permissions before deletion
+        try {
+          await fs.chmod(restrictedDir, 0o755);
+        } catch {
+          // Ignore errors if directory was already deleted or permissions cannot be restored
+        }
+        await fs.rm(testDir, { recursive: true, force: true });
+      }
+    });
+
+    it('should return error details for access denied paths', async () => {
+      const testDir = await fs.mkdtemp(join(os.tmpdir(), 'immich-test-access-denied-'));
+      const restrictedDir = join(testDir, 'restricted');
+      const restrictedFile = join(restrictedDir, 'file.jpg');
+      const accessibleFile = join(testDir, 'accessible.jpg');
+
+      try {
+        // Create test directory structure
+        await fs.mkdir(restrictedDir, { recursive: true });
+        await fs.writeFile(accessibleFile, 'accessible content');
+        await fs.writeFile(restrictedFile, 'restricted content');
+
+        // Remove all permissions from restricted directory to simulate access denied
+        await fs.chmod(restrictedDir, 0o000);
+
+        const errors: WalkError[] = [];
+        for await (const batch of sut.walk({ pathsToWalk: [testDir] })) {
+          for (const item of batch) {
+            if (item.type === 'error') {
+              errors.push(item);
+            }
+          }
+        }
+
+        // Should have error details including path and message
+        expect(errors.length).toBe(1);
+        const restrictedError = errors.find((e) => e.type === 'error' && e.message?.includes('restricted'));
+        expect(restrictedError).toBeDefined();
+        expect(restrictedError?.type).toBe('error');
+        expect(restrictedError?.message).toBeDefined();
+      } finally {
+        // Cleanup: restore permissions before deletion
+        try {
+          await fs.chmod(restrictedDir, 0o755);
+        } catch {
+          // Ignore errors if directory was already deleted or permissions cannot be restored
+        }
+        await fs.rm(testDir, { recursive: true, force: true });
+      }
+    });
+  });
+});

server/test/repositories/storage.repository.mock.ts

@@ -68,8 +68,7 @@ export const newStorageRepositoryMock = (): Mocked<RepositoryInterface<StorageRe
     readdir: vitest.fn(),
     realpath: vitest.fn().mockImplementation((filepath: string) => Promise.resolve(filepath)),
     stat: vitest.fn(),
-    crawl: vitest.fn(),
-    walk: vitest.fn().mockImplementation(async function* () {}),
+    walk: vitest.fn(),
     rename: vitest.fn(),
     copyFile: vitest.fn(),
     utimes: vitest.fn(),