diff --git a/mobile/lib/utils/face/face_box_crop.dart b/mobile/lib/utils/face/face_box_crop.dart index 2f63ca7e20..94228f8cd3 100644 --- a/mobile/lib/utils/face/face_box_crop.dart +++ b/mobile/lib/utils/face/face_box_crop.dart @@ -16,15 +16,19 @@ Future?> getFaceCrops( EnteFile file, Map faceBoxeMap, ) async { - late Uint8List? ioFileBytes; + late String? imagePath; if (file.fileType != FileType.video) { final File? ioFile = await getFile(file); if (ioFile == null) { return null; } - ioFileBytes = await ioFile.readAsBytes(); + imagePath = ioFile.path; } else { - ioFileBytes = await getThumbnail(file); + final thumbnail = await getThumbnailForUploadedFile(file); + if (thumbnail == null) { + return null; + } + imagePath = thumbnail.path; } final List faceIds = []; final List faceBoxes = []; @@ -34,7 +38,7 @@ Future?> getFaceCrops( } final List faceCrop = await ImageMlIsolate.instance.generateFaceThumbnailsForImage( - ioFileBytes!, + imagePath, faceBoxes, ); final Map result = {}; diff --git a/mobile/lib/utils/image_ml_isolate.dart b/mobile/lib/utils/image_ml_isolate.dart index cc89c32afd..f55e28d77d 100644 --- a/mobile/lib/utils/image_ml_isolate.dart +++ b/mobile/lib/utils/image_ml_isolate.dart @@ -1,8 +1,10 @@ import 'dart:async'; +import "dart:io" show File; import 'dart:isolate'; import 'dart:typed_data' show Float32List, Uint8List; import 'dart:ui'; +import "package:flutter/rendering.dart"; import 'package:flutter_isolate/flutter_isolate.dart'; import "package:logging/logging.dart"; import "package:photos/face/model/box.dart"; @@ -13,13 +15,13 @@ import "package:photos/utils/image_ml_util.dart"; import "package:synchronized/synchronized.dart"; enum ImageOperation { + @Deprecated("No longer using BlazeFace`") preprocessBlazeFace, preprocessYoloOnnx, preprocessFaceAlign, preprocessMobileFaceNet, preprocessMobileFaceNetOnnx, - generateFaceThumbnail, - generateFaceThumbnailsForImage, + generateFaceThumbnails, cropAndPadFace, } @@ -205,23 +207,14 @@ class ImageMlIsolate { 'originalWidth': originalSize.width, 'originalHeight': originalSize.height, }); - case ImageOperation.generateFaceThumbnail: - final imageData = args['imageData'] as Uint8List; - final faceDetectionJson = - args['faceDetection'] as Map; - final faceDetection = - FaceDetectionRelative.fromJson(faceDetectionJson); - final Uint8List result = - await generateFaceThumbnailFromData(imageData, faceDetection); - sendPort.send([result]); - case ImageOperation.generateFaceThumbnailsForImage: - final imageData = args['imageData'] as Uint8List; + case ImageOperation.generateFaceThumbnails: + final imagePath = args['imagePath'] as String; + final Uint8List imageData = await File(imagePath).readAsBytes(); final faceBoxesJson = args['faceBoxesList'] as List>; final List faceBoxes = faceBoxesJson.map((json) => FaceBox.fromJson(json)).toList(); - final List results = - await generateFaceThumbnailsFromDataAndDetections( + final List results = await generateFaceThumbnails( imageData, faceBoxes, ); @@ -471,44 +464,28 @@ class ImageMlIsolate { return (inputs, alignmentResults, isBlurs, blurValues, originalSize); } - /// Generates a face thumbnail from [imageData] and a [faceDetection]. - /// - /// Uses [generateFaceThumbnailFromData] inside the isolate. - Future generateFaceThumbnail( - Uint8List imageData, - FaceDetectionRelative faceDetection, - ) async { - return await _runInIsolate( - ( - ImageOperation.generateFaceThumbnail, - { - 'imageData': imageData, - 'faceDetection': faceDetection.toJson(), - }, - ), - ).then((value) => value[0] as Uint8List); - } - /// Generates face thumbnails for all [faceBoxes] in [imageData]. /// - /// Uses [generateFaceThumbnailsFromDataAndDetections] inside the isolate. + /// Uses [generateFaceThumbnails] inside the isolate. Future> generateFaceThumbnailsForImage( - Uint8List imageData, + String imagePath, List faceBoxes, ) async { final List> faceBoxesJson = faceBoxes.map((box) => box.toJson()).toList(); return await _runInIsolate( ( - ImageOperation.generateFaceThumbnailsForImage, + ImageOperation.generateFaceThumbnails, { - 'imageData': imageData, + 'imagePath': imagePath, 'faceBoxesList': faceBoxesJson, }, ), ).then((value) => value.cast()); } + @Deprecated('For second pass of BlazeFace, no longer used') + /// Generates cropped and padded image data from [imageData] and a [faceBox]. /// /// The steps are: diff --git a/mobile/lib/utils/image_ml_util.dart b/mobile/lib/utils/image_ml_util.dart index 3896b8e09c..6ff6972232 100644 --- a/mobile/lib/utils/image_ml_util.dart +++ b/mobile/lib/utils/image_ml_util.dart @@ -15,6 +15,7 @@ import "dart:ui"; // ImageConfiguration; // import 'package:flutter/material.dart' as material show Image; import 'package:flutter/painting.dart' as paint show decodeImageFromList; +import 'package:image/image.dart' as img_lib; import 'package:ml_linalg/linalg.dart'; import "package:photos/face/model/box.dart"; import 'package:photos/models/ml/ml_typedefs.dart'; @@ -36,7 +37,7 @@ Color readPixelColor( if (x < 0 || x >= image.width || y < 0 || y >= image.height) { // throw ArgumentError('Invalid pixel coordinates.'); log('[WARNING] `readPixelColor`: Invalid pixel coordinates, out of bounds'); - return const Color(0x00000000); + return const Color.fromARGB(255, 208, 16, 208); } assert(byteData.lengthInBytes == 4 * image.width * image.height); @@ -44,12 +45,39 @@ Color readPixelColor( return Color(_rgbaToArgb(byteData.getUint32(byteOffset))); } +void setPixelColor( + Size imageSize, + ByteData byteData, + int x, + int y, + Color color, +) { + if (x < 0 || x >= imageSize.width || y < 0 || y >= imageSize.height) { + log('[WARNING] `setPixelColor`: Invalid pixel coordinates, out of bounds'); + return; + } + assert(byteData.lengthInBytes == 4 * imageSize.width * imageSize.height); + + final int byteOffset = 4 * (imageSize.width.toInt() * y + x); + byteData.setUint32(byteOffset, _argbToRgba(color.value)); +} + int _rgbaToArgb(int rgbaColor) { final int a = rgbaColor & 0xFF; final int rgb = rgbaColor >> 8; return rgb + (a << 24); } +int _argbToRgba(int argbColor) { + final int r = (argbColor >> 16) & 0xFF; + final int g = (argbColor >> 8) & 0xFF; + final int b = argbColor & 0xFF; + final int a = (argbColor >> 24) & 0xFF; + return (r << 24) + (g << 16) + (b << 8) + a; +} + +@Deprecated('Used in TensorFlow Lite only, no longer needed') + /// Creates an empty matrix with the specified shape. /// /// The `shape` argument must be a list of length 2 or 3, where the first @@ -464,11 +492,53 @@ Future resizeAndCenterCropImage( return resizedImage; } +/// Crops an [image] based on the specified [x], [y], [width] and [height]. +Future cropImage( + Image image, + ByteData imgByteData, { + required int x, + required int y, + required int width, + required int height, +}) async { + // final newByteData = ByteData(width * height * 4); + // for (var h = y; h < y + height; h++) { + // for (var w = x; w < x + width; w++) { + // final pixel = readPixelColor(image, imgByteData, w, y); + // setPixelColor( + // Size(width.toDouble(), height.toDouble()), + // newByteData, + // w, + // h, + // pixel, + // ); + // } + // } + // final newImage = + // decodeImageFromRgbaBytes(newByteData.buffer.asUint8List(), width, height); + + final newImage = img_lib.Image(width: width, height: height); + + for (var h = y; h < y + height; h++) { + for (var w = x; w < x + width; w++) { + final pixel = readPixelColor(image, imgByteData, w, h); + newImage.setPixel( + w - x, + h - y, + img_lib.ColorRgb8(pixel.red, pixel.green, pixel.blue), + ); + } + } + final newImageDataPng = img_lib.encodePng(newImage); + + return newImageDataPng; +} + /// Crops an [image] based on the specified [x], [y], [width] and [height]. /// Optionally, the cropped image can be resized to comply with a [maxSize] and/or [minSize]. /// Optionally, the cropped image can be rotated from the center by [rotation] radians. /// Optionally, the [quality] of the resizing interpolation can be specified. -Future cropImage( +Future cropImageWithCanvas( Image image, { required double x, required double y, @@ -798,7 +868,7 @@ Future> preprocessFaceAlignToUint8List( continue; } final alignmentBox = getAlignedFaceBox(alignmentResult); - final Image alignedFace = await cropImage( + final Image alignedFace = await cropImageWithCanvas( image, x: alignmentBox[0], y: alignmentBox[1], @@ -886,7 +956,7 @@ Future< continue; } final alignmentBox = getAlignedFaceBox(alignmentResult); - final Image alignedFace = await cropImage( + final Image alignedFace = await cropImageWithCanvas( image, x: alignmentBox[0], y: alignmentBox[1], @@ -970,7 +1040,7 @@ Future<(Float32List, List, List, List, Size)> continue; } final alignmentBox = getAlignedFaceBox(alignmentResult); - final Image alignedFace = await cropImage( + final Image alignedFace = await cropImageWithCanvas( image, x: alignmentBox[0], y: alignmentBox[1], @@ -1168,29 +1238,52 @@ void warpAffineFloat32List( } } -/// Generates a face thumbnail from [imageData] and a [faceDetection]. -/// -/// Returns a [Uint8List] image, in png format. -Future generateFaceThumbnailFromData( +Future> generateFaceThumbnails( Uint8List imageData, - FaceDetectionRelative faceDetection, + List faceBoxes, ) async { + final stopwatch = Stopwatch()..start(); + final Image image = await decodeImageFromData(imageData); + final ByteData imgByteData = await getByteDataFromImage(image); - final Image faceThumbnail = await cropImage( - image, - x: (faceDetection.xMinBox * image.width).round() - 20, - y: (faceDetection.yMinBox * image.height).round() - 30, - width: (faceDetection.width * image.width).round() + 40, - height: (faceDetection.height * image.height).round() + 60, - ); + // int i = 0; + try { + final List faceThumbnails = []; - return await encodeImageToUint8List( - faceThumbnail, - format: ImageByteFormat.png, - ); + for (final faceBox in faceBoxes) { + final xCrop = (faceBox.x - faceBox.width / 2).round(); + final yCrop = (faceBox.y - faceBox.height / 2).round(); + final widthCrop = (faceBox.width * 2).round(); + final heightCrop = (faceBox.height * 2).round(); + final Uint8List faceThumbnail = await cropImage( + image, + imgByteData, + x: xCrop, + y: yCrop, + width: widthCrop, + height: heightCrop, + ); + // final Uint8List faceThumbnailPng = await encodeImageToUint8List( + // faceThumbnail, + // format: ImageByteFormat.png, + // ); + faceThumbnails.add(faceThumbnail); + // i++; + } + stopwatch.stop(); + log('Face thumbnail generation took: ${stopwatch.elapsedMilliseconds} ms'); + + return faceThumbnails; + } catch (e, s) { + log('[ImageMlUtils] Error generating face thumbnails: $e, \n stackTrace: $s'); + // log('[ImageMlUtils] cropImage problematic input argument: ${faceBoxes[i]}'); + rethrow; + } } +@Deprecated("Old method using canvas, replaced by `generateFaceThumbnails`") + /// Generates a face thumbnail from [imageData] and a [faceDetection]. /// /// Returns a [Uint8List] image, in png format. @@ -1205,7 +1298,7 @@ Future> generateFaceThumbnailsFromDataAndDetections( final List faceThumbnails = []; for (final faceBox in faceBoxes) { - final Image faceThumbnail = await cropImage( + final Image faceThumbnail = await cropImageWithCanvas( image, x: faceBox.x - faceBox.width / 2, y: faceBox.y - faceBox.height / 2, @@ -1227,6 +1320,8 @@ Future> generateFaceThumbnailsFromDataAndDetections( } } +@Deprecated('For second pass of BlazeFace, no longer used') + /// Generates cropped and padded image data from [imageData] and a [faceBox]. /// /// The steps are: @@ -1241,7 +1336,7 @@ Future cropAndPadFaceData( ) async { final Image image = await decodeImageFromData(imageData); - final Image faceCrop = await cropImage( + final Image faceCrop = await cropImageWithCanvas( image, x: (faceBox[0] * image.width), y: (faceBox[1] * image.height), @@ -1390,6 +1485,7 @@ Color getPixelBicubic(num fx, num fy, Image image, ByteData byteDataRgba) { return Color.fromRGBO(c0, c1, c2, 1.0); } +@Deprecated('Old method only used in other deprecated methods') List getAlignedFaceBox(AlignmentResult alignment) { final List box = [ // [xMinBox, yMinBox, xMaxBox, yMaxBox]