[mob] Switch to new APIs

mobile/ios/Podfile.lock

@@ -178,6 +178,8 @@ PODS:
   - photo_manager (2.0.0):
     - Flutter
     - FlutterMacOS
+  - privacy_screen (0.0.1):
+    - Flutter
   - PromisesObjC (2.4.0)
   - receive_sharing_intent (1.6.8):
     - Flutter
@@ -275,6 +277,7 @@ DEPENDENCIES:
   - path_provider_foundation (from `.symlinks/plugins/path_provider_foundation/darwin`)
   - permission_handler_apple (from `.symlinks/plugins/permission_handler_apple/ios`)
   - photo_manager (from `.symlinks/plugins/photo_manager/ios`)
+  - privacy_screen (from `.symlinks/plugins/privacy_screen/ios`)
   - receive_sharing_intent (from `.symlinks/plugins/receive_sharing_intent/ios`)
   - screen_brightness_ios (from `.symlinks/plugins/screen_brightness_ios/ios`)
   - sentry_flutter (from `.symlinks/plugins/sentry_flutter/ios`)
@@ -392,6 +395,8 @@ EXTERNAL SOURCES:
     :path: ".symlinks/plugins/permission_handler_apple/ios"
   photo_manager:
     :path: ".symlinks/plugins/photo_manager/ios"
+  privacy_screen:
+    :path: ".symlinks/plugins/privacy_screen/ios"
   receive_sharing_intent:
     :path: ".symlinks/plugins/receive_sharing_intent/ios"
   screen_brightness_ios:
@@ -473,6 +478,7 @@ SPEC CHECKSUMS:
   path_provider_foundation: 3784922295ac71e43754bd15e0653ccfd36a147c
   permission_handler_apple: 9878588469a2b0d0fc1e048d9f43605f92e6cec2
   photo_manager: ff695c7a1dd5bc379974953a2b5c0a293f7c4c8a
+  privacy_screen: 1a131c052ceb3c3659934b003b0d397c2381a24e
   PromisesObjC: f5707f49cb48b9636751c5b2e7d227e43fba9f47
   receive_sharing_intent: 6837b01768e567fe8562182397bf43d63d8c6437
   screen_brightness_ios: 715ca807df953bf676d339f11464e438143ee625

mobile/ios/Runner.xcodeproj/project.pbxproj

@@ -325,6 +325,7 @@
 				"${BUILT_PRODUCTS_DIR}/package_info_plus/package_info_plus.framework",
 				"${BUILT_PRODUCTS_DIR}/path_provider_foundation/path_provider_foundation.framework",
 				"${BUILT_PRODUCTS_DIR}/photo_manager/photo_manager.framework",
+				"${BUILT_PRODUCTS_DIR}/privacy_screen/privacy_screen.framework",
 				"${BUILT_PRODUCTS_DIR}/receive_sharing_intent/receive_sharing_intent.framework",
 				"${BUILT_PRODUCTS_DIR}/screen_brightness_ios/screen_brightness_ios.framework",
 				"${BUILT_PRODUCTS_DIR}/sentry_flutter/sentry_flutter.framework",
@@ -417,6 +418,7 @@
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/package_info_plus.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/path_provider_foundation.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/photo_manager.framework",
+				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/privacy_screen.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/receive_sharing_intent.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/screen_brightness_ios.framework",
 				"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/sentry_flutter.framework",

mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart

@@ -106,7 +106,7 @@ class FaceRecognitionService {
       _logger.info("embeddingResponse ${res.debugLog()}");
       final List<Face> faces = [];
       final List<ClipEmbedding> clipEmbeddings = [];
-      for (RemoteFileML fileMl in res.mlData.values) {
+      for (RemoteFileDerivedData fileMl in res.mlData.values) {
         final existingInstruction = pendingIndex[fileMl.fileID]!;
         final facesFromRemoteEmbedding = _getFacesFromRemoteEmbedding(fileMl);
         //Note: Always do null check, empty value means no face was found.
@@ -143,11 +143,6 @@ class FaceRecognitionService {
           }
         }
       }
-      if (res.noEmbeddingFileIDs.isNotEmpty) {
-        for (final fileID in res.noEmbeddingFileIDs) {
-          faces.add(Face.empty(fileID, error: false));
-        }
-      }
       await FaceMLDataDB.instance.bulkInsertFaces(faces);
       await EmbeddingsDB.instance.putMany(clipEmbeddings);
     }
@@ -160,7 +155,7 @@ class FaceRecognitionService {
 
   // Returns a list of faces from the given remote fileML. null if the version is less than the current version
   // or if the remote faceEmbedding is null.
-  List<Face>? _getFacesFromRemoteEmbedding(RemoteFileML fileMl) {
+  List<Face>? _getFacesFromRemoteEmbedding(RemoteFileDerivedData fileMl) {
     final RemoteFaceEmbedding? remoteFaceEmbedding = fileMl.faceEmbedding;
     if (shouldDiscardRemoteEmbedding(fileMl)) {
       return null;

mobile/lib/services/machine_learning/file_ml/file_ml.dart

@@ -3,25 +3,37 @@ import "package:photos/face/model/face.dart";
 const _faceKey = 'face';
 const _clipKey = 'clip';
 
-class RemoteFileML {
+class RemoteFileDerivedData {
   final int fileID;
   final Map<String, dynamic> remoteRawData;
 
-  RemoteFileML(
+  RemoteFileDerivedData(
     this.fileID,
     this.remoteRawData,
   );
 
-  factory RemoteFileML.fromRemote(int fileID, Map<String, dynamic> json) {
-    return RemoteFileML(
+  void putSanityCheck() {
+    if (remoteRawData[_faceKey] == null) {
+      throw Exception('Face embedding is null');
+    }
+    if (remoteRawData[_clipKey] == null) {
+      throw Exception('Clip embedding is null');
+    }
+  }
+
+  factory RemoteFileDerivedData.fromRemote(
+    int fileID,
+    Map<String, dynamic> json,
+  ) {
+    return RemoteFileDerivedData(
       fileID,
       json,
     );
   }
 
-  static RemoteFileML empty(int i) {
+  static RemoteFileDerivedData empty(int i) {
     final Map<String, dynamic> json = {};
-    return RemoteFileML(i, json);
+    return RemoteFileDerivedData(i, json);
   }
 
   void putFaceIfNotNull(RemoteFaceEmbedding? faceEmbedding) {

mobile/lib/services/machine_learning/file_ml/files_ml_data_response.dart

@@ -1,10 +1,7 @@
 import 'package:photos/services/machine_learning/file_ml/file_ml.dart';
 
 class FilesMLDataResponse {
-  final Map<int, RemoteFileML> mlData;
-  // fileIDs that were indexed but they don't contain any meaningful embeddings
-  // and hence should be discarded for re-indexing
-  final Set<int> noEmbeddingFileIDs;
+  final Map<int, RemoteFileDerivedData> mlData;
   // fetchErrorFileIDs are the fileIDs for whom we failed failed to fetch embeddings
   // from the storage
   final Set<int> fetchErrorFileIDs;
@@ -12,21 +9,23 @@ class FilesMLDataResponse {
   final Set<int> pendingIndexFileIDs;
   FilesMLDataResponse(
     this.mlData, {
-    required this.noEmbeddingFileIDs,
     required this.fetchErrorFileIDs,
     required this.pendingIndexFileIDs,
   });
 
+  FilesMLDataResponse.empty({
+    this.mlData = const {},
+    this.fetchErrorFileIDs = const {},
+    this.pendingIndexFileIDs = const {},
+  });
+
   String debugLog() {
-    final nonZeroNoEmbeddingFileIDs = noEmbeddingFileIDs.isNotEmpty
-        ? ', smallEmbeddings: ${noEmbeddingFileIDs.length}'
-        : '';
     final nonZeroFetchErrorFileIDs = fetchErrorFileIDs.isNotEmpty
         ? ', errorForFileIDs: ${fetchErrorFileIDs.length}'
         : '';
     final nonZeroPendingIndexFileIDs = pendingIndexFileIDs.isNotEmpty
         ? ', pendingIndexFileIDs: ${pendingIndexFileIDs.length}'
         : '';
-    return 'MLRemote(mlData: ${mlData.length}$nonZeroNoEmbeddingFileIDs$nonZeroFetchErrorFileIDs$nonZeroPendingIndexFileIDs)';
+    return 'MLRemote(mlData: ${mlData.length}$nonZeroFetchErrorFileIDs$nonZeroPendingIndexFileIDs)';
   }
 }

mobile/lib/services/machine_learning/file_ml/remote_embedding.dart

@@ -1,37 +1,30 @@
 import "dart:convert";
 
-class RemoteEmbedding {
+class FileDataEntity {
   final int fileID;
-  final String model;
-  final String encryptedEmbedding;
+  final String type;
+  final String encryptedData;
   final String decryptionHeader;
   final int updatedAt;
 
-  RemoteEmbedding({
+  FileDataEntity({
     required this.fileID,
-    required this.model,
-    required this.encryptedEmbedding,
+    required this.type,
+    required this.encryptedData,
     required this.decryptionHeader,
     required this.updatedAt,
   });
 
-  factory RemoteEmbedding.fromMap(Map<String, dynamic> map) {
-    return RemoteEmbedding(
+  factory FileDataEntity.fromMap(Map<String, dynamic> map) {
+    return FileDataEntity(
       fileID: map['fileID']?.toInt() ?? 0,
-      model: map['model'] ?? '',
-      encryptedEmbedding: map['encryptedEmbedding'] ?? '',
+      type: map['type'] ?? '',
+      encryptedData: map['encryptedData'] ?? '',
       decryptionHeader: map['decryptionHeader'] ?? '',
       updatedAt: map['updatedAt']?.toInt() ?? 0,
     );
   }
 
-  factory RemoteEmbedding.fromJson(String source) =>
-      RemoteEmbedding.fromMap(json.decode(source));
-}
-
-class RemoteEmbeddings {
-  final List<RemoteEmbedding> embeddings;
-  final bool hasMore;
-
-  RemoteEmbeddings(this.embeddings, this.hasMore);
+  factory FileDataEntity.fromJson(String source) =>
+      FileDataEntity.fromMap(json.decode(source));
 }

mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart

@@ -29,16 +29,18 @@ class RemoteFileMLService {
 
   Future<void> putFileEmbedding(
     EnteFile file,
-    RemoteFileML fileML, {
+    RemoteFileDerivedData fileML, {
     RemoteClipEmbedding? clipEmbedding,
     RemoteFaceEmbedding? faceEmbedding,
   }) async {
     fileML.putClipIfNotNull(clipEmbedding);
     fileML.putFaceIfNotNull(faceEmbedding);
+    fileML.putSanityCheck();
     final ChaChaEncryptionResult encryptionResult = await gzipAndEncryptJson(
       fileML.remoteRawData,
       getFileKey(file),
     );
+
     try {
       final _ = await _dio.put(
         "/files/data/",
@@ -66,22 +68,18 @@ class RemoteFileMLService {
           "type": _derivedDataType,
         },
       );
-      final remoteEmb = res.data['embeddings'] as List;
+      final remoteEntries = res.data['data'] as List;
       final pendingIndexFiles = res.data['pendingIndexFileIDs'] as List;
-      final noEmbeddingFiles = res.data['noEmbeddingFileIDs'] as List;
       final errFileIds = res.data['errFileIDs'] as List;
 
-      final List<RemoteEmbedding> remoteEmbeddings = <RemoteEmbedding>[];
-      for (var entry in remoteEmb) {
-        final embedding = RemoteEmbedding.fromMap(entry);
-        remoteEmbeddings.add(embedding);
+      final List<FileDataEntity> encFileData = <FileDataEntity>[];
+      for (var entry in remoteEntries) {
+        encFileData.add(FileDataEntity.fromMap(entry));
       }
 
-      final fileIDToFileMl = await decryptFileMLData(remoteEmbeddings);
+      final fileIDToFileMl = await decryptFileMLData(encFileData);
       return FilesMLDataResponse(
         fileIDToFileMl,
-        noEmbeddingFileIDs:
-            Set<int>.from(noEmbeddingFiles.map((x) => x as int)),
         fetchErrorFileIDs: Set<int>.from(errFileIds.map((x) => x as int)),
         pendingIndexFileIDs:
             Set<int>.from(pendingIndexFiles.map((x) => x as int)),
@@ -92,10 +90,10 @@ class RemoteFileMLService {
     }
   }
 
-  Future<Map<int, RemoteFileML>> decryptFileMLData(
-    List<RemoteEmbedding> remoteEmbeddings,
+  Future<Map<int, RemoteFileDerivedData>> decryptFileMLData(
+    List<FileDataEntity> remoteEmbeddings,
   ) async {
-    final result = <int, RemoteFileML>{};
+    final result = <int, RemoteFileDerivedData>{};
     if (remoteEmbeddings.isEmpty) {
       return result;
     }
@@ -111,7 +109,8 @@ class RemoteFileMLService {
       final input = EmbeddingsDecoderInput(embedding, fileKey);
       inputs.add(input);
     }
-    return _computer.compute<Map<String, dynamic>, Map<int, RemoteFileML>>(
+    return _computer
+        .compute<Map<String, dynamic>, Map<int, RemoteFileDerivedData>>(
       _decryptFileMLComputer,
       param: {
         "inputs": inputs,
@@ -120,19 +119,19 @@ class RemoteFileMLService {
   }
 }
 
-Future<Map<int, RemoteFileML>> _decryptFileMLComputer(
+Future<Map<int, RemoteFileDerivedData>> _decryptFileMLComputer(
   Map<String, dynamic> args,
 ) async {
-  final result = <int, RemoteFileML>{};
+  final result = <int, RemoteFileDerivedData>{};
   final inputs = args["inputs"] as List<EmbeddingsDecoderInput>;
   for (final input in inputs) {
     final decodedJson = decryptAndUnzipJsonSync(
       input.decryptionKey,
-      encryptedData: input.embedding.encryptedEmbedding,
-      header: input.embedding.decryptionHeader,
+      encryptedData: input.data.encryptedData,
+      header: input.data.decryptionHeader,
     );
-    result[input.embedding.fileID] = RemoteFileML.fromRemote(
-      input.embedding.fileID,
+    result[input.data.fileID] = RemoteFileDerivedData.fromRemote(
+      input.data.fileID,
       decodedJson,
     );
   }
@@ -140,8 +139,8 @@ Future<Map<int, RemoteFileML>> _decryptFileMLComputer(
 }
 
 class EmbeddingsDecoderInput {
-  final RemoteEmbedding embedding;
+  final FileDataEntity data;
   final Uint8List decryptionKey;
 
-  EmbeddingsDecoderInput(this.embedding, this.decryptionKey);
+  EmbeddingsDecoderInput(this.data, this.decryptionKey);
 }

mobile/lib/services/machine_learning/ml_service.dart

@@ -453,7 +453,7 @@ class MLService {
           await RemoteFileMLService.instance.putFileEmbedding(
             instruction.file,
             instruction.existingRemoteFileML ??
-                RemoteFileML.empty(
+                RemoteFileDerivedData.empty(
                   instruction.file.uploadedFileID!,
                 ),
             faceEmbedding: result.facesRan

mobile/lib/utils/ml_util.dart

@@ -37,7 +37,7 @@ class FileMLInstruction {
   final EnteFile file;
   bool shouldRunFaces;
   bool shouldRunClip;
-  RemoteFileML? existingRemoteFileML;
+  RemoteFileDerivedData? existingRemoteFileML;
 
   FileMLInstruction({
     required this.file,
@@ -131,11 +131,10 @@ Future<List<FileMLInstruction>> getFilesForMlIndexing() async {
   _logger.info(
     "Getting list of files to index for ML took ${DateTime.now().difference(time).inMilliseconds} ms",
   );
-
   return sortedBylocalID;
 }
 
-bool shouldDiscardRemoteEmbedding(RemoteFileML fileML) {
+bool shouldDiscardRemoteEmbedding(RemoteFileDerivedData fileML) {
   final fileID = fileML.fileID;
   final RemoteFaceEmbedding? faceEmbedding = fileML.faceEmbedding;
   if (faceEmbedding == null || faceEmbedding.version < faceMlVersion) {