diff --git a/mobile/android/app/CMakeLists.txt b/mobile/android/app/CMakeLists.txt index 133bde4fc0..9cad7b915a 100644 --- a/mobile/android/app/CMakeLists.txt +++ b/mobile/android/app/CMakeLists.txt @@ -7,6 +7,7 @@ project(native_buffer LANGUAGES C) add_library(native_buffer SHARED src/main/cpp/native_buffer.c + src/main/cpp/native_image.c ) target_link_libraries(native_buffer jnigraphics) diff --git a/mobile/android/app/src/main/cpp/native_image.c b/mobile/android/app/src/main/cpp/native_image.c new file mode 100644 index 0000000000..07bb7e9a2c --- /dev/null +++ b/mobile/android/app/src/main/cpp/native_image.c @@ -0,0 +1,109 @@ +#include +#include +#include +#include + +// Cache-friendly block size for the tiled rotation (in pixels). 32x32 uint32 = 4KB, fits L1. +#define TILE 32 + +// EXIF orientation values (androidx.exifinterface.media.ExifInterface.ORIENTATION_*). +enum { + ORIENTATION_FLIP_HORIZONTAL = 2, + ORIENTATION_ROTATE_180 = 3, + ORIENTATION_FLIP_VERTICAL = 4, + ORIENTATION_TRANSPOSE = 5, + ORIENTATION_ROTATE_90 = 6, + ORIENTATION_TRANSVERSE = 7, + ORIENTATION_ROTATE_270 = 8, +}; + +// The orientations that swap width and height. Must stay in sync with affine_for's dim usage. +static int swaps_dims(int o) { + return o == ORIENTATION_ROTATE_90 || o == ORIENTATION_ROTATE_270 || + o == ORIENTATION_TRANSPOSE || o == ORIENTATION_TRANSVERSE; +} + +// A source pixel (sx, sy) maps to destination index base + sx*stepX + sy*stepY, where dw is the +// destination width. This affine form covers all 8 EXIF orientations and matches the pixel layout +// of Bitmap.createBitmap(src, matrixForExifOrientation(o)). int64_t so it stays correct on +// armeabi-v7a (32-bit long) regardless of how large MAX_RAW_DECODE_PIXELS grows. +static void affine_for(int o, int sw, int sh, int dw, int64_t *base, int64_t *stepX, int64_t *stepY) { + switch (o) { + case ORIENTATION_ROTATE_90: *base = sh - 1; *stepX = dw; *stepY = -1; break; + case ORIENTATION_ROTATE_270: *base = (int64_t) (sw - 1) * dw; *stepX = -dw; *stepY = 1; break; + case ORIENTATION_ROTATE_180: *base = (int64_t) (sh - 1) * dw + (sw - 1); *stepX = -1; *stepY = -dw; break; + case ORIENTATION_FLIP_HORIZONTAL: *base = sw - 1; *stepX = -1; *stepY = dw; break; + case ORIENTATION_FLIP_VERTICAL: *base = (int64_t) (sh - 1) * dw; *stepX = 1; *stepY = -dw; break; + case ORIENTATION_TRANSPOSE: *base = 0; *stepX = dw; *stepY = 1; break; + case ORIENTATION_TRANSVERSE: *base = (int64_t) (sw - 1) * dw + (sh - 1); *stepX = -dw; *stepY = -1; break; + default: *base = 0; *stepX = 1; *stepY = dw; break; + } +} + +// Copy each source pixel (whole uint32, so channel order/premult is irrelevant) to its rotated +// destination, walking TILE x TILE blocks so the scattered writes of a 90/270 transpose stay +// cache-resident. dst is densely packed (rowBytes == dw*4, no padding), which the affine math relies on. +static void rotate_tiled(const uint8_t *src, int srcStride, uint32_t *dst, + int sw, int sh, int64_t base, int64_t stepX, int64_t stepY) { + for (int ty = 0; ty < sh; ty += TILE) { + int yEnd = ty + TILE < sh ? ty + TILE : sh; + for (int tx = 0; tx < sw; tx += TILE) { + int xEnd = tx + TILE < sw ? tx + TILE : sw; + for (int sy = ty; sy < yEnd; sy++) { + const uint32_t *srcRow = (const uint32_t *) (src + (size_t) sy * srcStride); + int64_t idx = base + (int64_t) sy * stepY + (int64_t) tx * stepX; + for (int sx = tx; sx < xEnd; sx++) { + dst[idx] = srcRow[sx]; + idx += stepX; + } + } + } + } +} + +// Rotates an RGBA_8888 bitmap to the given EXIF orientation into a freshly malloc'd buffer (free it +// via NativeBuffer.free). Fills outInfo with {width, height, rowBytes} and returns the buffer +// address, or 0 if the bitmap can't be handled (e.g. a non-8888 format) so the caller can fall back. +JNIEXPORT jlong JNICALL +Java_app_alextran_immich_NativeImage_rotate( + JNIEnv *env, jclass clazz, jobject bitmap, jint orientation, jintArray outInfo) { + AndroidBitmapInfo info; + if (AndroidBitmap_getInfo(env, bitmap, &info) != ANDROID_BITMAP_RESULT_SUCCESS) { + return 0; + } + if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888) { + return 0; + } + + int sw = (int) info.width; + int sh = (int) info.height; + int dw = swaps_dims(orientation) ? sh : sw; + int dh = swaps_dims(orientation) ? sw : sh; + + uint32_t *dst = (uint32_t *) malloc((size_t) dw * dh * 4); + if (dst == NULL) { + return 0; + } + + void *srcPixels = NULL; + if (AndroidBitmap_lockPixels(env, bitmap, &srcPixels) != ANDROID_BITMAP_RESULT_SUCCESS) { + free(dst); + return 0; + } + + int64_t base, stepX, stepY; + affine_for(orientation, sw, sh, dw, &base, &stepX, &stepY); + rotate_tiled((const uint8_t *) srcPixels, (int) info.stride, dst, sw, sh, base, stepX, stepY); + + AndroidBitmap_unlockPixels(env, bitmap); + + jint dims[3] = {dw, dh, dw * 4}; + (*env)->SetIntArrayRegion(env, outInfo, 0, 3, dims); + // Keep ownership in C until the buffer is safely handed back: if outInfo was somehow too small, + // SetIntArrayRegion left a pending exception and Kotlin will never receive (or free) dst. + if ((*env)->ExceptionCheck(env)) { + free(dst); + return 0; + } + return (jlong) dst; +} diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/NativeImage.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/NativeImage.kt new file mode 100644 index 0000000000..b1398d84b8 --- /dev/null +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/NativeImage.kt @@ -0,0 +1,19 @@ +package app.alextran.immich + +import android.graphics.Bitmap + +object NativeImage { + init { + // rotate() is compiled into the native_buffer shared lib (which already links jnigraphics). + System.loadLibrary("native_buffer") + } + + /** + * Rotates an RGBA_8888 [bitmap] to the given EXIF [orientation], writing the result into a freshly + * malloc'd native buffer. Returns the buffer address (free it with [NativeBuffer.free]) and fills + * [outInfo] with {width, height, rowBytes}. Returns 0 when the bitmap can't be handled (e.g. a + * non-8888 config) so the caller can fall back. + */ + @JvmStatic + external fun rotate(bitmap: Bitmap, orientation: Int, outInfo: IntArray): Long +} diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt index 13e045307e..a98ccb9271 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt @@ -14,6 +14,7 @@ import android.util.Size import androidx.annotation.RequiresApi import androidx.exifinterface.media.ExifInterface import app.alextran.immich.NativeBuffer +import app.alextran.immich.NativeImage import kotlin.math.* import java.io.IOException import java.util.concurrent.Executors @@ -76,9 +77,9 @@ class LocalImagesImpl(context: Context) : LocalImageApi { val CANCELLED = Result.success?>(null) val OPTIONS = BitmapFactory.Options().apply { inPreferredConfig = Bitmap.Config.ARGB_8888 } - // "Load original" decodes a raw at full res, and rotating it (below) needs a second bitmap, so a - // huge DNG would briefly hold two large copies. Cap the decode resolution to bound that. This - // only trims pixels on very large raws - they still come out upright, just downsampled. + // "Load original" decodes a raw at full res, and the orientation pass then walks every pixel, so + // cap the decode resolution to keep that bounded on huge DNGs. This only trims pixels on very + // large raws - they still come out upright, just downsampled. const val MAX_RAW_DECODE_PIXELS = 24_000_000L } @@ -187,38 +188,44 @@ class LocalImagesImpl(context: Context) : LocalImageApi { val id = assetId.toLong() signal.throwIfCanceled() - val bitmap = if (isVideo) { - decodeVideoThumbnail(id, size, signal) - } else { - decodeImage(id, size, signal) - } - try { - signal.throwIfCanceled() - val res = bitmap.toNativeBuffer() - signal.throwIfCanceled() + val res = if (isVideo) { + decodeVideoThumbnail(id, size, signal).toNativeBuffer() + } else { + val (bitmap, orientation) = decodeImage(id, size, signal) + signal.throwIfCanceled() + if (orientation == ExifInterface.ORIENTATION_NORMAL || orientation == ExifInterface.ORIENTATION_UNDEFINED) { + bitmap.toNativeBuffer() + } else { + rotateToNativeBuffer(bitmap, orientation, signal) + } + } + // Don't re-check cancellation here: res owns a malloc'd buffer, and bailing to CANCELLED would + // orphan it. Deliver it; Dart frees the buffer itself if the request was cancelled meanwhile. callback(Result.success(res)) } catch (e: Exception) { callback(if (e is OperationCanceledException) CANCELLED else Result.failure(e)) } } - private fun decodeImage(id: Long, size: Size, signal: CancellationSignal): Bitmap { + // Returns the decoded bitmap plus the EXIF orientation that still needs applying. Only Q+ raw + // decodes come back unrotated (ImageDecoder / loadThumbnail skip EXIF for raw like DNG); every + // other path already orients itself, so it reports ORIENTATION_NORMAL. + private fun decodeImage(id: Long, size: Size, signal: CancellationSignal): Pair { signal.throwIfCanceled() val uri = ContentUris.withAppendedId(Images.Media.EXTERNAL_CONTENT_URI, id) - // Only the Q+ ImageDecoder / loadThumbnail decoders skip EXIF orientation for raw (e.g. DNG). - // The pre-Q Glide / MediaStore-thumbnail paths already orient raw, so don't rotate those again. val handleRaw = Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && isRawMime(uri) + val orientation = if (handleRaw) rawOrientation(uri) else ExifInterface.ORIENTATION_NORMAL if (size.width <= 0 || size.height <= 0 || size.width > 768 || size.height > 768) { - // A "load original" request is unsized -> a full-res decode. For raw, that plus the rotation - // below would briefly hold two large bitmaps, so cap the raw decode to a safe pixel budget. + // A "load original" request is unsized -> a full-res decode. For raw, cap it so the later + // orientation pass stays within a safe pixel budget. val bitmap = if (handleRaw && (size.width <= 0 || size.height <= 0)) { decodeRawCapped(uri, signal) } else { decodeSource(uri, size, signal) } - return if (handleRaw) applyExifRotation(uri, bitmap, signal) else bitmap + return bitmap to orientation } val bitmap = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { @@ -227,7 +234,7 @@ class LocalImagesImpl(context: Context) : LocalImageApi { signal.setOnCancelListener { Images.Thumbnails.cancelThumbnailRequest(resolver, id) } Images.Thumbnails.getThumbnail(resolver, id, Images.Thumbnails.MINI_KIND, OPTIONS) } - return if (handleRaw) applyExifRotation(uri, bitmap, signal) else bitmap + return bitmap to orientation } private fun isRawMime(uri: Uri): Boolean { @@ -235,6 +242,12 @@ class LocalImagesImpl(context: Context) : LocalImageApi { return mime.startsWith("image/x-") || mime == "image/dng" } + private fun rawOrientation(uri: Uri): Int { + return resolver.openInputStream(uri)?.use { + ExifInterface(it).getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL) + } ?: ExifInterface.ORIENTATION_NORMAL + } + // Full-res raw decode for "load original", sampled down to MAX_RAW_DECODE_PIXELS (power of two). // Caps resolution only; the caller still rotates the result, so even huge raws end up upright. @RequiresApi(Build.VERSION_CODES.Q) @@ -255,25 +268,37 @@ class LocalImagesImpl(context: Context) : LocalImageApi { } // ImageDecoder / loadThumbnail skip EXIF orientation for raw (e.g. DNG) on Q+, so the decoded - // bitmap comes back unrotated. Rotate it ourselves to match the file. Runs on the decode pool. - private fun applyExifRotation(uri: Uri, bitmap: Bitmap, signal: CancellationSignal): Bitmap { + // bitmap comes back unrotated. Rotate it into the output buffer in native code (one pass, no + // intermediate rotated bitmap), falling back to Skia for any config the native path can't take. + private fun rotateToNativeBuffer(bitmap: Bitmap, orientation: Int, signal: CancellationSignal): Map { signal.throwIfCanceled() - val orientation = resolver.openInputStream(uri)?.use { - ExifInterface(it).getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL) - } ?: ExifInterface.ORIENTATION_NORMAL - val matrix = matrixForExifOrientation(orientation) ?: return bitmap - signal.throwIfCanceled() - // createBitmap cannot read a hardware-backed source; copy to a software bitmap first if needed. - val src = if (bitmap.config == Bitmap.Config.HARDWARE) { - bitmap.copy(Bitmap.Config.ARGB_8888, false).also { bitmap.recycle() } + // Force ARGB_8888 so both the native pass and the Skia fallback are 4 bytes/pixel: the native + // rotate needs a lockable 8888 buffer, and toNativeBuffer() below allocates width*height*4 (an + // F16/HDR decode would otherwise under-allocate). No-op for the common already-8888 case. + val src = if (bitmap.config != Bitmap.Config.ARGB_8888) { + val converted = bitmap.copy(Bitmap.Config.ARGB_8888, false) + bitmap.recycle() + converted ?: throw IOException("could not convert bitmap to ARGB_8888") } else { bitmap } - val rotated = Bitmap.createBitmap(src, 0, 0, src.width, src.height, matrix, true) - if (rotated != src) { - src.recycle() + try { + val info = IntArray(3) + val pointer = NativeImage.rotate(src, orientation, info) + if (pointer != 0L) { + return mapOf( + "pointer" to pointer, + "width" to info[0].toLong(), + "height" to info[1].toLong(), + "rowBytes" to info[2].toLong() + ) + } + // Native path declined (unsupported config) -> rotate via Skia, then copy out. + val matrix = matrixForExifOrientation(orientation) ?: return src.toNativeBuffer() + return Bitmap.createBitmap(src, 0, 0, src.width, src.height, matrix, true).toNativeBuffer() + } finally { + if (!src.isRecycled) src.recycle() } - return rotated } // EXIF orientation (1-8) -> transform matrix, or null when no rotation/flip is needed. @@ -285,8 +310,8 @@ class LocalImagesImpl(context: Context) : LocalImageApi { ExifInterface.ORIENTATION_ROTATE_270 -> matrix.postRotate(270f) ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.postScale(-1f, 1f) ExifInterface.ORIENTATION_FLIP_VERTICAL -> matrix.postScale(1f, -1f) - ExifInterface.ORIENTATION_TRANSPOSE -> matrix.apply { postRotate(90f); postScale(-1f, 1f) } - ExifInterface.ORIENTATION_TRANSVERSE -> matrix.apply { postRotate(270f); postScale(-1f, 1f) } + ExifInterface.ORIENTATION_TRANSPOSE -> matrix.apply { postRotate(270f); postScale(-1f, 1f) } + ExifInterface.ORIENTATION_TRANSVERSE -> matrix.apply { postRotate(90f); postScale(-1f, 1f) } else -> return null } return matrix