import 'dart:async'; import "dart:io" show File; import 'dart:isolate'; import 'dart:typed_data' show Float32List, Uint8List; import 'dart:ui'; import "package:flutter/rendering.dart"; import 'package:flutter_isolate/flutter_isolate.dart'; import "package:logging/logging.dart"; import "package:photos/face/model/box.dart"; import "package:photos/face/model/dimension.dart"; import 'package:photos/models/ml/ml_typedefs.dart'; import 'package:photos/services/machine_learning/face_ml/face_alignment/alignment_result.dart'; import 'package:photos/services/machine_learning/face_ml/face_detection/detection.dart'; import "package:photos/utils/image_ml_util.dart"; import "package:synchronized/synchronized.dart"; enum ImageOperation { @Deprecated("No longer using BlazeFace`") preprocessBlazeFace, preprocessYoloOnnx, preprocessFaceAlign, preprocessMobileFaceNet, preprocessMobileFaceNetOnnx, generateFaceThumbnails, cropAndPadFace, } /// The isolate below uses functions from ["package:photos/utils/image_ml_util.dart"] to preprocess images for ML models. /// This class is responsible for all image operations needed for ML models. It runs in a separate isolate to avoid jank. /// /// It can be accessed through the singleton `ImageConversionIsolate.instance`. e.g. `ImageConversionIsolate.instance.convert(imageData)` /// /// IMPORTANT: Make sure to dispose of the isolate when you're done with it with `dispose()`, e.g. `ImageConversionIsolate.instance.dispose();` class ImageMlIsolate { // static const String debugName = 'ImageMlIsolate'; final _logger = Logger('ImageMlIsolate'); Timer? _inactivityTimer; final Duration _inactivityDuration = const Duration(seconds: 60); int _activeTasks = 0; final _initLock = Lock(); final _functionLock = Lock(); late FlutterIsolate _isolate; late ReceivePort _receivePort = ReceivePort(); late SendPort _mainSendPort; bool isSpawned = false; // singleton pattern ImageMlIsolate._privateConstructor(); /// Use this instance to access the ImageConversionIsolate service. Make sure to call `init()` before using it. /// e.g. `await ImageConversionIsolate.instance.init();` /// And kill the isolate when you're done with it with `dispose()`, e.g. `ImageConversionIsolate.instance.dispose();` /// /// Then you can use `convert()` to get the image, so `ImageConversionIsolate.instance.convert(imageData, imagePath: imagePath)` static final ImageMlIsolate instance = ImageMlIsolate._privateConstructor(); factory ImageMlIsolate() => instance; Future init() async { return _initLock.synchronized(() async { if (isSpawned) return; _receivePort = ReceivePort(); try { _isolate = await FlutterIsolate.spawn( _isolateMain, _receivePort.sendPort, ); _mainSendPort = await _receivePort.first as SendPort; isSpawned = true; _resetInactivityTimer(); } catch (e) { _logger.severe('Could not spawn isolate', e); isSpawned = false; } }); } Future ensureSpawned() async { if (!isSpawned) { await init(); } } @pragma('vm:entry-point') static void _isolateMain(SendPort mainSendPort) async { final receivePort = ReceivePort(); mainSendPort.send(receivePort.sendPort); receivePort.listen((message) async { final functionIndex = message[0] as int; final function = ImageOperation.values[functionIndex]; final args = message[1] as Map; final sendPort = message[2] as SendPort; try { switch (function) { case ImageOperation.preprocessBlazeFace: final imageData = args['imageData'] as Uint8List; final normalize = args['normalize'] as bool; final int normalization = normalize ? 2 : -1; final requiredWidth = args['requiredWidth'] as int; final requiredHeight = args['requiredHeight'] as int; final qualityIndex = args['quality'] as int; final maintainAspectRatio = args['maintainAspectRatio'] as bool; final quality = FilterQuality.values[qualityIndex]; final (result, originalSize, newSize) = await preprocessImageToMatrix( imageData, normalization: normalization, requiredWidth: requiredWidth, requiredHeight: requiredHeight, quality: quality, maintainAspectRatio: maintainAspectRatio, ); sendPort.send({ 'inputs': result, 'originalWidth': originalSize.width, 'originalHeight': originalSize.height, 'newWidth': newSize.width, 'newHeight': newSize.height, }); case ImageOperation.preprocessYoloOnnx: final imageData = args['imageData'] as Uint8List; final normalize = args['normalize'] as bool; final int normalization = normalize ? 1 : -1; final requiredWidth = args['requiredWidth'] as int; final requiredHeight = args['requiredHeight'] as int; final maintainAspectRatio = args['maintainAspectRatio'] as bool; final Image image = await decodeImageFromData(imageData); final imageByteData = await getByteDataFromImage(image); final (result, originalSize, newSize) = await preprocessImageToFloat32ChannelsFirst( image, imageByteData, normalization: normalization, requiredWidth: requiredWidth, requiredHeight: requiredHeight, maintainAspectRatio: maintainAspectRatio, ); sendPort.send({ 'inputs': result, 'originalWidth': originalSize.width, 'originalHeight': originalSize.height, 'newWidth': newSize.width, 'newHeight': newSize.height, }); case ImageOperation.preprocessFaceAlign: final imageData = args['imageData'] as Uint8List; final faceLandmarks = args['faceLandmarks'] as List>>; final List result = await preprocessFaceAlignToUint8List( imageData, faceLandmarks, ); sendPort.send(List.from(result)); case ImageOperation.preprocessMobileFaceNet: final imageData = args['imageData'] as Uint8List; final facesJson = args['facesJson'] as List>; final ( inputs, alignmentResults, isBlurs, blurValues, originalSize ) = await preprocessToMobileFaceNetInput( imageData, facesJson, ); final List> alignmentResultsJson = alignmentResults.map((result) => result.toJson()).toList(); sendPort.send({ 'inputs': inputs, 'alignmentResultsJson': alignmentResultsJson, 'isBlurs': isBlurs, 'blurValues': blurValues, 'originalWidth': originalSize.width, 'originalHeight': originalSize.height, }); case ImageOperation.preprocessMobileFaceNetOnnx: final imagePath = args['imagePath'] as String; final facesJson = args['facesJson'] as List>; final List relativeFaces = facesJson .map((face) => FaceDetectionRelative.fromJson(face)) .toList(); final imageData = await File(imagePath).readAsBytes(); final Image image = await decodeImageFromData(imageData); final imageByteData = await getByteDataFromImage(image); final ( inputs, alignmentResults, isBlurs, blurValues, originalSize ) = await preprocessToMobileFaceNetFloat32List( image, imageByteData, relativeFaces, ); final List> alignmentResultsJson = alignmentResults.map((result) => result.toJson()).toList(); sendPort.send({ 'inputs': inputs, 'alignmentResultsJson': alignmentResultsJson, 'isBlurs': isBlurs, 'blurValues': blurValues, 'originalWidth': originalSize.width, 'originalHeight': originalSize.height, }); case ImageOperation.generateFaceThumbnails: final imagePath = args['imagePath'] as String; final Uint8List imageData = await File(imagePath).readAsBytes(); final faceBoxesJson = args['faceBoxesList'] as List>; final List faceBoxes = faceBoxesJson.map((json) => FaceBox.fromJson(json)).toList(); final List results = await generateFaceThumbnails( imageData, faceBoxes, ); sendPort.send(List.from(results)); case ImageOperation.cropAndPadFace: final imageData = args['imageData'] as Uint8List; final faceBox = args['faceBox'] as List; final Uint8List result = await cropAndPadFaceData(imageData, faceBox); sendPort.send([result]); } } catch (e, stackTrace) { sendPort .send({'error': e.toString(), 'stackTrace': stackTrace.toString()}); } }); } /// The common method to run any operation in the isolate. It sends the [message] to [_isolateMain] and waits for the result. Future _runInIsolate( (ImageOperation, Map) message, ) async { await ensureSpawned(); return _functionLock.synchronized(() async { _resetInactivityTimer(); final completer = Completer(); final answerPort = ReceivePort(); _activeTasks++; _mainSendPort.send([message.$1.index, message.$2, answerPort.sendPort]); answerPort.listen((receivedMessage) { if (receivedMessage is Map && receivedMessage.containsKey('error')) { // Handle the error final errorMessage = receivedMessage['error']; final errorStackTrace = receivedMessage['stackTrace']; final exception = Exception(errorMessage); final stackTrace = StackTrace.fromString(errorStackTrace); completer.completeError(exception, stackTrace); } else { completer.complete(receivedMessage); } }); _activeTasks--; return completer.future; }); } /// Resets a timer that kills the isolate after a certain amount of inactivity. /// /// Should be called after initialization (e.g. inside `init()`) and after every call to isolate (e.g. inside `_runInIsolate()`) void _resetInactivityTimer() { _inactivityTimer?.cancel(); _inactivityTimer = Timer(_inactivityDuration, () { if (_activeTasks > 0) { _logger.info('Tasks are still running. Delaying isolate disposal.'); // Optionally, reschedule the timer to check again later. _resetInactivityTimer(); } else { _logger.info( 'Clustering Isolate has been inactive for ${_inactivityDuration.inSeconds} seconds with no tasks running. Killing isolate.', ); dispose(); } }); } /// Disposes the isolate worker. void dispose() { if (!isSpawned) return; isSpawned = false; _isolate.kill(); _receivePort.close(); _inactivityTimer?.cancel(); } /// Preprocesses [imageData] for standard ML models inside a separate isolate. /// /// Returns a [Num3DInputMatrix] image usable for ML inference with BlazeFace. /// /// Uses [preprocessImageToMatrix] inside the isolate. @Deprecated("No longer using BlazeFace") Future<(Num3DInputMatrix, Size, Size)> preprocessImageBlazeFace( Uint8List imageData, { required bool normalize, required int requiredWidth, required int requiredHeight, FilterQuality quality = FilterQuality.medium, bool maintainAspectRatio = true, }) async { final Map results = await _runInIsolate( ( ImageOperation.preprocessBlazeFace, { 'imageData': imageData, 'normalize': normalize, 'requiredWidth': requiredWidth, 'requiredHeight': requiredHeight, 'quality': quality.index, 'maintainAspectRatio': maintainAspectRatio, }, ), ); final inputs = results['inputs'] as Num3DInputMatrix; final originalSize = Size( results['originalWidth'] as double, results['originalHeight'] as double, ); final newSize = Size( results['newWidth'] as double, results['newHeight'] as double, ); return (inputs, originalSize, newSize); } /// Uses [preprocessImageToFloat32ChannelsFirst] inside the isolate. @Deprecated( "Old method, not needed since we now run the whole ML pipeline for faces in a single isolate", ) Future<(Float32List, Dimensions, Dimensions)> preprocessImageYoloOnnx( Uint8List imageData, { required bool normalize, required int requiredWidth, required int requiredHeight, FilterQuality quality = FilterQuality.medium, bool maintainAspectRatio = true, }) async { final Map results = await _runInIsolate( ( ImageOperation.preprocessYoloOnnx, { 'imageData': imageData, 'normalize': normalize, 'requiredWidth': requiredWidth, 'requiredHeight': requiredHeight, 'quality': quality.index, 'maintainAspectRatio': maintainAspectRatio, }, ), ); final inputs = results['inputs'] as Float32List; final originalSize = Dimensions( width:results['originalWidth'] as int, height: results['originalHeight'] as int, ); final newSize = Dimensions( width: results['newWidth'] as int, height: results['newHeight'] as int, ); return (inputs, originalSize, newSize); } /// Preprocesses [imageData] for face alignment inside a separate isolate, to display the aligned faces. Mostly used for debugging. /// /// Returns a list of [Uint8List] images, one for each face, in png format. /// /// Uses [preprocessFaceAlignToUint8List] inside the isolate. /// /// WARNING: For preprocessing for MobileFaceNet, use [preprocessMobileFaceNet] instead! @Deprecated( "Old method, not needed since we now run the whole ML pipeline for faces in a single isolate", ) Future> preprocessFaceAlign( Uint8List imageData, List faces, ) async { final faceLandmarks = faces.map((face) => face.allKeypoints).toList(); return await _runInIsolate( ( ImageOperation.preprocessFaceAlign, { 'imageData': imageData, 'faceLandmarks': faceLandmarks, }, ), ).then((value) => value.cast()); } /// Preprocesses [imageData] for MobileFaceNet input inside a separate isolate. /// /// Returns a list of [Num3DInputMatrix] images, one for each face. /// /// Uses [preprocessToMobileFaceNetInput] inside the isolate. @Deprecated("Old method used in TensorFlow Lite") Future< ( List, List, List, List, Size, )> preprocessMobileFaceNet( Uint8List imageData, List faces, ) async { final List> facesJson = faces.map((face) => face.toJson()).toList(); final Map results = await _runInIsolate( ( ImageOperation.preprocessMobileFaceNet, { 'imageData': imageData, 'facesJson': facesJson, }, ), ); final inputs = results['inputs'] as List; final alignmentResultsJson = results['alignmentResultsJson'] as List>; final alignmentResults = alignmentResultsJson.map((json) { return AlignmentResult.fromJson(json); }).toList(); final isBlurs = results['isBlurs'] as List; final blurValues = results['blurValues'] as List; final originalSize = Size( results['originalWidth'] as double, results['originalHeight'] as double, ); return (inputs, alignmentResults, isBlurs, blurValues, originalSize); } /// Uses [preprocessToMobileFaceNetFloat32List] inside the isolate. @Deprecated( "Old method, not needed since we now run the whole ML pipeline for faces in a single isolate", ) Future<(Float32List, List, List, List, Size)> preprocessMobileFaceNetOnnx( String imagePath, List faces, ) async { final List> facesJson = faces.map((face) => face.toJson()).toList(); final Map results = await _runInIsolate( ( ImageOperation.preprocessMobileFaceNetOnnx, { 'imagePath': imagePath, 'facesJson': facesJson, }, ), ); final inputs = results['inputs'] as Float32List; final alignmentResultsJson = results['alignmentResultsJson'] as List>; final alignmentResults = alignmentResultsJson.map((json) { return AlignmentResult.fromJson(json); }).toList(); final isBlurs = results['isBlurs'] as List; final blurValues = results['blurValues'] as List; final originalSize = Size( results['originalWidth'] as double, results['originalHeight'] as double, ); return (inputs, alignmentResults, isBlurs, blurValues, originalSize); } /// Generates face thumbnails for all [faceBoxes] in [imageData]. /// /// Uses [generateFaceThumbnails] inside the isolate. Future> generateFaceThumbnailsForImage( String imagePath, List faceBoxes, ) async { final List> faceBoxesJson = faceBoxes.map((box) => box.toJson()).toList(); return await _runInIsolate( ( ImageOperation.generateFaceThumbnails, { 'imagePath': imagePath, 'faceBoxesList': faceBoxesJson, }, ), ).then((value) => value.cast()); } @Deprecated('For second pass of BlazeFace, no longer used') /// Generates cropped and padded image data from [imageData] and a [faceBox]. /// /// The steps are: /// 1. Crop the image to the face bounding box /// 2. Resize this cropped image to a square that is half the BlazeFace input size /// 3. Pad the image to the BlazeFace input size /// /// Uses [cropAndPadFaceData] inside the isolate. Future cropAndPadFace( Uint8List imageData, List faceBox, ) async { return await _runInIsolate( ( ImageOperation.cropAndPadFace, { 'imageData': imageData, 'faceBox': List.from(faceBox), }, ), ).then((value) => value[0] as Uint8List); } }