kenmirrors/ente
1
0
Fork 0
mirror of https://github.com/ente-io/ente.git synced 2025-08-10 16:32:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'mobile_face' of https://github.com/ente-io/auth into mobile_face

Browse Source
This commit is contained in:
Neeraj Gupta 2024-04-29 17:18:32 +05:30
commit 7d2633190f
21 changed files with 1248 additions and 592 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

626
mobile/lib/face/db.dart
View File

@ -1,5 +1,4 @@
import 'dart:async'; import 'dart:async';
import "dart:io" show Directory;
import "dart:math"; import "dart:math";
import "package:collection/collection.dart"; import "package:collection/collection.dart";
@ -14,14 +13,16 @@ import "package:photos/face/model/face.dart";
import "package:photos/models/file/file.dart"; import "package:photos/models/file/file.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_info_for_clustering.dart"; import "package:photos/services/machine_learning/face_ml/face_clustering/face_info_for_clustering.dart";
import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart';
import 'package:sqflite/sqflite.dart'; import 'package:sqlite_async/sqlite_async.dart';
import 'package:sqlite_async/sqlite_async.dart' as sqlite_async;
/// Stores all data for the ML-related features. The database can be accessed by `MlDataDB.instance.database`. /// Stores all data for the FacesML-related features. The database can be accessed by `FaceMLDataDB.instance.database`.
/// ///
/// This includes: /// This includes:
/// [facesTable] - Stores all the detected faces and its embeddings in the images. /// [facesTable] - Stores all the detected faces and its embeddings in the images.
/// [personTable] - Stores all the clusters of faces which are considered to be the same person. /// [createFaceClustersTable] - Stores all the mappings from the faces (faceID) to the clusters (clusterID).
/// [clusterPersonTable] - Stores all the clusters that are mapped to a certain person.
/// [clusterSummaryTable] - Stores a summary of each cluster, containg the mean embedding and the number of faces in the cluster.
/// [notPersonFeedback] - Stores the clusters that are confirmed not to belong to a certain person by the user
class FaceMLDataDB { class FaceMLDataDB {
static final Logger _logger = Logger("FaceMLDataDB"); static final Logger _logger = Logger("FaceMLDataDB");
@ -33,75 +34,81 @@ class FaceMLDataDB {
static final FaceMLDataDB instance = FaceMLDataDB._privateConstructor(); static final FaceMLDataDB instance = FaceMLDataDB._privateConstructor();
// only have a single app-wide reference to the database // only have a single app-wide reference to the database
static Future<Database>? _dbFuture; static Future<SqliteDatabase>? _sqliteAsyncDBFuture;
static Future<sqlite_async.SqliteDatabase>? _sqliteAsyncDBFuture;
Future<Database> get database async { Future<SqliteDatabase> get asyncDB async {
_dbFuture ??= _initDatabase();
return _dbFuture!;
}
Future<sqlite_async.SqliteDatabase> get sqliteAsyncDB async {
_sqliteAsyncDBFuture ??= _initSqliteAsyncDatabase(); _sqliteAsyncDBFuture ??= _initSqliteAsyncDatabase();
return _sqliteAsyncDBFuture!; return _sqliteAsyncDBFuture!;
} }
Future<Database> _initDatabase() async { Future<SqliteDatabase> _initSqliteAsyncDatabase() async {
final documentsDirectory = await getApplicationDocumentsDirectory(); final documentsDirectory = await getApplicationDocumentsDirectory();
final String databaseDirectory =
join(documentsDirectory.path, _databaseName);
return await openDatabase(
databaseDirectory,
version: _databaseVersion,
onCreate: _onCreate,
);
}
Future<sqlite_async.SqliteDatabase> _initSqliteAsyncDatabase() async {
final Directory documentsDirectory =
await getApplicationDocumentsDirectory();
final String databaseDirectory = final String databaseDirectory =
join(documentsDirectory.path, _databaseName); join(documentsDirectory.path, _databaseName);
_logger.info("Opening sqlite_async access: DB path " + databaseDirectory); _logger.info("Opening sqlite_async access: DB path " + databaseDirectory);
return sqlite_async.SqliteDatabase(path: databaseDirectory, maxReaders: 1); final asyncDBConnection =
SqliteDatabase(path: databaseDirectory, maxReaders: 2);
await _onCreate(asyncDBConnection);
return asyncDBConnection;
} }
Future _onCreate(Database db, int version) async { Future<void> _onCreate(SqliteDatabase asyncDBConnection) async {
await db.execute(createFacesTable); final migrations = SqliteMigrations()
await db.execute(createFaceClustersTable); ..add(
await db.execute(createClusterPersonTable); SqliteMigration(_databaseVersion, (tx) async {
await db.execute(createClusterSummaryTable); await tx.execute(createFacesTable);
await db.execute(createNotPersonFeedbackTable); await tx.execute(createFaceClustersTable);
await db.execute(fcClusterIDIndex); await tx.execute(createClusterPersonTable);
await tx.execute(createClusterSummaryTable);
await tx.execute(createNotPersonFeedbackTable);
await tx.execute(fcClusterIDIndex);
}),
);
await migrations.migrate(asyncDBConnection);
} }
// bulkInsertFaces inserts the faces in the database in batches of 1000. // bulkInsertFaces inserts the faces in the database in batches of 1000.
// This is done to avoid the error "too many SQL variables" when inserting // This is done to avoid the error "too many SQL variables" when inserting
// a large number of faces. // a large number of faces.
Future<void> bulkInsertFaces(List<Face> faces) async { Future<void> bulkInsertFaces(List<Face> faces) async {
final db = await instance.database; final db = await instance.asyncDB;
const batchSize = 500; const batchSize = 500;
final numBatches = (faces.length / batchSize).ceil(); final numBatches = (faces.length / batchSize).ceil();
for (int i = 0; i < numBatches; i++) { for (int i = 0; i < numBatches; i++) {
final start = i * batchSize; final start = i * batchSize;
final end = min((i + 1) * batchSize, faces.length); final end = min((i + 1) * batchSize, faces.length);
final batch = faces.sublist(start, end); final batch = faces.sublist(start, end);
final batchInsert = db.batch();
for (final face in batch) { const String sql = '''
batchInsert.insert( INSERT INTO $facesTable (
facesTable, $fileIDColumn, $faceIDColumn, $faceDetectionColumn, $faceEmbeddingBlob, $faceScore, $faceBlur, $isSideways, $imageHeight, $imageWidth, $mlVersionColumn
mapRemoteToFaceDB(face), ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
conflictAlgorithm: ConflictAlgorithm.ignore, ON CONFLICT($fileIDColumn, $faceIDColumn) DO UPDATE SET $faceIDColumn = excluded.$faceIDColumn, $faceDetectionColumn = excluded.$faceDetectionColumn, $faceEmbeddingBlob = excluded.$faceEmbeddingBlob, $faceScore = excluded.$faceScore, $faceBlur = excluded.$faceBlur, $isSideways = excluded.$isSideways, $imageHeight = excluded.$imageHeight, $imageWidth = excluded.$imageWidth, $mlVersionColumn = excluded.$mlVersionColumn
); ''';
} final parameterSets = batch.map((face) {
await batchInsert.commit(noResult: true); final map = mapRemoteToFaceDB(face);
return [
map[fileIDColumn],
map[faceIDColumn],
map[faceDetectionColumn],
map[faceEmbeddingBlob],
map[faceScore],
map[faceBlur],
map[isSideways],
map[imageHeight],
map[imageWidth],
map[mlVersionColumn],
];
}).toList();
await db.executeBatch(sql, parameterSets);
} }
} }
Future<void> updateClusterIdToFaceId( Future<void> updateFaceIdToClusterId(
Map<String, int> faceIDToClusterID, Map<String, int> faceIDToClusterID,
) async { ) async {
final db = await instance.database; final db = await instance.asyncDB;
const batchSize = 500; const batchSize = 500;
final numBatches = (faceIDToClusterID.length / batchSize).ceil(); final numBatches = (faceIDToClusterID.length / batchSize).ceil();
for (int i = 0; i < numBatches; i++) { for (int i = 0; i < numBatches; i++) {
@ -109,24 +116,20 @@ class FaceMLDataDB {
final end = min((i + 1) * batchSize, faceIDToClusterID.length); final end = min((i + 1) * batchSize, faceIDToClusterID.length);
final batch = faceIDToClusterID.entries.toList().sublist(start, end); final batch = faceIDToClusterID.entries.toList().sublist(start, end);
final batchUpdate = db.batch(); const String sql = '''
INSERT INTO $faceClustersTable ($fcFaceId, $fcClusterID)
VALUES (?, ?)
ON CONFLICT($fcFaceId) DO UPDATE SET $fcClusterID = excluded.$fcClusterID
''';
final parameterSets = batch.map((e) => [e.key, e.value]).toList();
for (final entry in batch) { await db.executeBatch(sql, parameterSets);
final faceID = entry.key;
final clusterID = entry.value;
batchUpdate.insert(
faceClustersTable,
{fcClusterID: clusterID, fcFaceId: faceID},
conflictAlgorithm: ConflictAlgorithm.replace,
);
}
await batchUpdate.commit(noResult: true);
} }
} }
/// Returns a map of fileID to the indexed ML version /// Returns a map of fileID to the indexed ML version
Future<Map<int, int>> getIndexedFileIds() async { Future<Map<int, int>> getIndexedFileIds() async {
final db = await instance.sqliteAsyncDB; final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll( final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fileIDColumn, $mlVersionColumn FROM $facesTable', 'SELECT $fileIDColumn, $mlVersionColumn FROM $facesTable',
); );
@ -138,7 +141,7 @@ class FaceMLDataDB {
} }
Future<int> getIndexedFileCount() async { Future<int> getIndexedFileCount() async {
final db = await instance.sqliteAsyncDB; final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll( final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT COUNT(DISTINCT $fileIDColumn) as count FROM $facesTable', 'SELECT COUNT(DISTINCT $fileIDColumn) as count FROM $facesTable',
); );
@ -146,8 +149,8 @@ class FaceMLDataDB {
} }
Future<Map<int, int>> clusterIdToFaceCount() async { Future<Map<int, int>> clusterIdToFaceCount() async {
final db = await instance.database; final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.rawQuery( final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID, COUNT(*) as count FROM $faceClustersTable where $fcClusterID IS NOT NULL GROUP BY $fcClusterID ', 'SELECT $fcClusterID, COUNT(*) as count FROM $faceClustersTable where $fcClusterID IS NOT NULL GROUP BY $fcClusterID ',
); );
final Map<int, int> result = {}; final Map<int, int> result = {};
@ -158,15 +161,15 @@ class FaceMLDataDB {
} }
Future<Set<int>> getPersonIgnoredClusters(String personID) async { Future<Set<int>> getPersonIgnoredClusters(String personID) async {
final db = await instance.database; final db = await instance.asyncDB;
// find out clusterIds that are assigned to other persons using the clusters table // find out clusterIds that are assigned to other persons using the clusters table
final List<Map<String, dynamic>> maps = await db.rawQuery( final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $clusterIDColumn FROM $clusterPersonTable WHERE $personIdColumn != ? AND $personIdColumn IS NOT NULL', 'SELECT $clusterIDColumn FROM $clusterPersonTable WHERE $personIdColumn != ? AND $personIdColumn IS NOT NULL',
[personID], [personID],
); );
final Set<int> ignoredClusterIDs = final Set<int> ignoredClusterIDs =
maps.map((e) => e[clusterIDColumn] as int).toSet(); maps.map((e) => e[clusterIDColumn] as int).toSet();
final List<Map<String, dynamic>> rejectMaps = await db.rawQuery( final List<Map<String, dynamic>> rejectMaps = await db.getAll(
'SELECT $clusterIDColumn FROM $notPersonFeedback WHERE $personIdColumn = ?', 'SELECT $clusterIDColumn FROM $notPersonFeedback WHERE $personIdColumn = ?',
[personID], [personID],
); );
@ -176,8 +179,8 @@ class FaceMLDataDB {
} }
Future<Set<int>> getPersonClusterIDs(String personID) async { Future<Set<int>> getPersonClusterIDs(String personID) async {
final db = await instance.database; final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.rawQuery( final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $clusterIDColumn FROM $clusterPersonTable WHERE $personIdColumn = ?', 'SELECT $clusterIDColumn FROM $clusterPersonTable WHERE $personIdColumn = ?',
[personID], [personID],
); );
@ -185,20 +188,21 @@ class FaceMLDataDB {
} }
Future<void> clearTable() async { Future<void> clearTable() async {
final db = await instance.database; final db = await instance.asyncDB;
await db.delete(facesTable);
await db.delete(clusterPersonTable); await db.execute(deleteFacesTable);
await db.delete(clusterSummaryTable); await db.execute(dropClusterPersonTable);
await db.delete(personTable); await db.execute(dropClusterSummaryTable);
await db.delete(notPersonFeedback); await db.execute(deletePersonTable);
await db.execute(dropNotPersonFeedbackTable);
} }
Future<Iterable<Uint8List>> getFaceEmbeddingsForCluster( Future<Iterable<Uint8List>> getFaceEmbeddingsForCluster(
int clusterID, { int clusterID, {
int? limit, int? limit,
}) async { }) async {
final db = await instance.database; final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.rawQuery( final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $faceEmbeddingBlob FROM $facesTable WHERE $faceIDColumn in (SELECT $fcFaceId from $faceClustersTable where $fcClusterID = ?) ${limit != null ? 'LIMIT $limit' : ''}', 'SELECT $faceEmbeddingBlob FROM $facesTable WHERE $faceIDColumn in (SELECT $fcFaceId from $faceClustersTable where $fcClusterID = ?) ${limit != null ? 'LIMIT $limit' : ''}',
[clusterID], [clusterID],
); );
@ -209,7 +213,7 @@ class FaceMLDataDB {
Iterable<int> clusterIDs, { Iterable<int> clusterIDs, {
int? limit, int? limit,
}) async { }) async {
final db = await instance.database; final db = await instance.asyncDB;
final Map<int, List<Uint8List>> result = {}; final Map<int, List<Uint8List>> result = {};
final selectQuery = ''' final selectQuery = '''
@ -220,7 +224,7 @@ class FaceMLDataDB {
${limit != null ? 'LIMIT $limit' : ''} ${limit != null ? 'LIMIT $limit' : ''}
'''; ''';
final List<Map<String, dynamic>> maps = await db.rawQuery(selectQuery); final List<Map<String, dynamic>> maps = await db.getAll(selectQuery);
for (final map in maps) { for (final map in maps) {
final clusterID = map[fcClusterID] as int; final clusterID = map[fcClusterID] as int;
@ -238,7 +242,7 @@ class FaceMLDataDB {
int? clusterID, int? clusterID,
}) async { }) async {
// read person from db // read person from db
final db = await instance.database; final db = await instance.asyncDB;
if (personID != null) { if (personID != null) {
final List<int> fileId = [recentFileID]; final List<int> fileId = [recentFileID];
int? avatarFileId; int? avatarFileId;
@ -248,15 +252,18 @@ class FaceMLDataDB {
fileId.add(avatarFileId); fileId.add(avatarFileId);
} }
} }
final cluterRows = await db.query( const String queryClusterID = '''
clusterPersonTable, SELECT $clusterIDColumn
columns: [clusterIDColumn], FROM $clusterPersonTable
where: '$personIdColumn = ?', WHERE $personIdColumn = ?
whereArgs: [personID], ''';
final clusterRows = await db.getAll(
queryClusterID,
[personID],
); );
final clusterIDs = final clusterIDs =
cluterRows.map((e) => e[clusterIDColumn] as int).toList(); clusterRows.map((e) => e[clusterIDColumn] as int).toList();
final List<Map<String, dynamic>> faceMaps = await db.rawQuery( final List<Map<String, dynamic>> faceMaps = await db.getAll(
'SELECT * FROM $facesTable where ' 'SELECT * FROM $facesTable where '
'$faceIDColumn in (SELECT $fcFaceId from $faceClustersTable where $fcClusterID IN (${clusterIDs.join(",")}))' '$faceIDColumn in (SELECT $fcFaceId from $faceClustersTable where $fcClusterID IN (${clusterIDs.join(",")}))'
'AND $fileIDColumn in (${fileId.join(",")}) AND $faceScore > $kMinimumQualityFaceScore ORDER BY $faceScore DESC', 'AND $fileIDColumn in (${fileId.join(",")}) AND $faceScore > $kMinimumQualityFaceScore ORDER BY $faceScore DESC',
@ -274,11 +281,14 @@ class FaceMLDataDB {
} }
} }
if (clusterID != null) { if (clusterID != null) {
final List<Map<String, dynamic>> faceMaps = await db.query( const String queryFaceID = '''
faceClustersTable, SELECT $fcFaceId
columns: [fcFaceId], FROM $faceClustersTable
where: '$fcClusterID = ?', WHERE $fcClusterID = ?
whereArgs: [clusterID], ''';
final List<Map<String, dynamic>> faceMaps = await db.getAll(
queryFaceID,
[clusterID],
); );
final List<Face>? faces = await getFacesForGivenFileID(recentFileID); final List<Face>? faces = await getFacesForGivenFileID(recentFileID);
if (faces != null) { if (faces != null) {
@ -297,22 +307,14 @@ class FaceMLDataDB {
} }
Future<List<Face>?> getFacesForGivenFileID(int fileUploadID) async { Future<List<Face>?> getFacesForGivenFileID(int fileUploadID) async {
final db = await instance.database; final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.query( const String query = '''
facesTable, SELECT * FROM $facesTable
columns: [ WHERE $fileIDColumn = ?
faceIDColumn, ''';
fileIDColumn, final List<Map<String, dynamic>> maps = await db.getAll(
faceEmbeddingBlob, query,
faceScore, [fileUploadID],
faceDetectionColumn,
faceBlur,
imageHeight,
imageWidth,
mlVersionColumn,
],
where: '$fileIDColumn = ?',
whereArgs: [fileUploadID],
); );
if (maps.isEmpty) { if (maps.isEmpty) {
return null; return null;
@ -321,8 +323,8 @@ class FaceMLDataDB {
} }
Future<Face?> getFaceForFaceID(String faceID) async { Future<Face?> getFaceForFaceID(String faceID) async {
final db = await instance.database; final db = await instance.asyncDB;
final result = await db.rawQuery( final result = await db.getAll(
'SELECT * FROM $facesTable where $faceIDColumn = ?', 'SELECT * FROM $facesTable where $faceIDColumn = ?',
[faceID], [faceID],
); );
@ -332,8 +334,50 @@ class FaceMLDataDB {
return mapRowToFace(result.first); return mapRowToFace(result.first);
} }
Future<Map<int, Iterable<String>>> getClusterToFaceIDs(
Set<int> clusterIDs,
) async {
final db = await instance.asyncDB;
final Map<int, List<String>> result = {};
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable WHERE $fcClusterID IN (${clusterIDs.join(",")})',
);
for (final map in maps) {
final clusterID = map[fcClusterID] as int;
final faceID = map[fcFaceId] as String;
result.putIfAbsent(clusterID, () => <String>[]).add(faceID);
}
return result;
}
Future<int?> getClusterIDForFaceID(String faceID) async {
final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID FROM $faceClustersTable WHERE $fcFaceId = ?',
[faceID],
);
if (maps.isEmpty) {
return null;
}
return maps.first[fcClusterID] as int;
}
Future<Map<int, Iterable<String>>> getAllClusterIdToFaceIDs() async {
final db = await instance.asyncDB;
final Map<int, List<String>> result = {};
final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable',
);
for (final map in maps) {
final clusterID = map[fcClusterID] as int;
final faceID = map[fcFaceId] as String;
result.putIfAbsent(clusterID, () => <String>[]).add(faceID);
}
return result;
}
Future<Iterable<String>> getFaceIDsForCluster(int clusterID) async { Future<Iterable<String>> getFaceIDsForCluster(int clusterID) async {
final db = await instance.sqliteAsyncDB; final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll( final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcFaceId FROM $faceClustersTable ' 'SELECT $fcFaceId FROM $faceClustersTable '
'WHERE $faceClustersTable.$fcClusterID = ?', 'WHERE $faceClustersTable.$fcClusterID = ?',
@ -343,7 +387,7 @@ class FaceMLDataDB {
} }
Future<Iterable<String>> getFaceIDsForPerson(String personID) async { Future<Iterable<String>> getFaceIDsForPerson(String personID) async {
final db = await instance.sqliteAsyncDB; final db = await instance.asyncDB;
final faceIdsResult = await db.getAll( final faceIdsResult = await db.getAll(
'SELECT $fcFaceId FROM $faceClustersTable LEFT JOIN $clusterPersonTable ' 'SELECT $fcFaceId FROM $faceClustersTable LEFT JOIN $clusterPersonTable '
'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$clusterIDColumn ' 'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$clusterIDColumn '
@ -354,7 +398,7 @@ class FaceMLDataDB {
} }
Future<Iterable<double>> getBlurValuesForCluster(int clusterID) async { Future<Iterable<double>> getBlurValuesForCluster(int clusterID) async {
final db = await instance.sqliteAsyncDB; final db = await instance.asyncDB;
const String query = ''' const String query = '''
SELECT $facesTable.$faceBlur SELECT $facesTable.$faceBlur
FROM $facesTable FROM $facesTable
@ -376,7 +420,7 @@ class FaceMLDataDB {
Future<Map<String, double>> getFaceIDsToBlurValues( Future<Map<String, double>> getFaceIDsToBlurValues(
int maxBlurValue, int maxBlurValue,
) async { ) async {
final db = await instance.sqliteAsyncDB; final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll( final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $faceIDColumn, $faceBlur FROM $facesTable WHERE $faceBlur < $maxBlurValue AND $faceBlur > 1 ORDER BY $faceBlur ASC', 'SELECT $faceIDColumn, $faceBlur FROM $facesTable WHERE $faceBlur < $maxBlurValue AND $faceBlur > 1 ORDER BY $faceBlur ASC',
); );
@ -390,8 +434,8 @@ class FaceMLDataDB {
Future<Map<String, int?>> getFaceIdsToClusterIds( Future<Map<String, int?>> getFaceIdsToClusterIds(
Iterable<String> faceIds, Iterable<String> faceIds,
) async { ) async {
final db = await instance.database; final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.rawQuery( final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcFaceId, $fcClusterID FROM $faceClustersTable where $fcFaceId IN (${faceIds.map((id) => "'$id'").join(",")})', 'SELECT $fcFaceId, $fcClusterID FROM $faceClustersTable where $fcFaceId IN (${faceIds.map((id) => "'$id'").join(",")})',
); );
final Map<String, int?> result = {}; final Map<String, int?> result = {};
@ -403,8 +447,8 @@ class FaceMLDataDB {
Future<Map<int, Set<int>>> getFileIdToClusterIds() async { Future<Map<int, Set<int>>> getFileIdToClusterIds() async {
final Map<int, Set<int>> result = {}; final Map<int, Set<int>> result = {};
final db = await instance.database; final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.rawQuery( final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable', 'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable',
); );
@ -421,36 +465,31 @@ class FaceMLDataDB {
Future<void> forceUpdateClusterIds( Future<void> forceUpdateClusterIds(
Map<String, int> faceIDToClusterID, Map<String, int> faceIDToClusterID,
) async { ) async {
final db = await instance.database; final db = await instance.asyncDB;
// Start a batch const String sql = '''
final batch = db.batch(); INSERT INTO $faceClustersTable ($fcFaceId, $fcClusterID)
VALUES (?, ?)
for (final map in faceIDToClusterID.entries) { ON CONFLICT($fcFaceId) DO UPDATE SET $fcClusterID = excluded.$fcClusterID
final faceID = map.key; ''';
final clusterID = map.value; final parameterSets =
batch.insert( faceIDToClusterID.entries.map((e) => [e.key, e.value]).toList();
faceClustersTable, await db.executeBatch(sql, parameterSets);
{fcFaceId: faceID, fcClusterID: clusterID},
conflictAlgorithm: ConflictAlgorithm.replace,
);
}
// Commit the batch
await batch.commit(noResult: true);
} }
Future<void> removePerson(String personID) async { Future<void> removePerson(String personID) async {
final db = await instance.database; final db = await instance.asyncDB;
await db.delete(
clusterPersonTable, await db.writeTransaction((tx) async {
where: '$personIdColumn = ?', await tx.execute(
whereArgs: [personID], 'DELETE FROM $clusterPersonTable WHERE $personIdColumn = ?',
); [personID],
await db.delete( );
notPersonFeedback, await tx.execute(
where: '$personIdColumn = ?', 'DELETE FROM $notPersonFeedback WHERE $personIdColumn = ?',
whereArgs: [personID], [personID],
); );
});
} }
Future<Set<FaceInfoForClustering>> getFaceInfoForClustering({ Future<Set<FaceInfoForClustering>> getFaceInfoForClustering({
@ -464,7 +503,7 @@ class FaceMLDataDB {
w.logAndReset( w.logAndReset(
'reading as float offset: $offset, maxFaces: $maxFaces, batchSize: $batchSize', 'reading as float offset: $offset, maxFaces: $maxFaces, batchSize: $batchSize',
); );
final db = await instance.sqliteAsyncDB; final db = await instance.asyncDB;
final Set<FaceInfoForClustering> result = {}; final Set<FaceInfoForClustering> result = {};
while (true) { while (true) {
@ -519,7 +558,7 @@ class FaceMLDataDB {
w.logAndReset( w.logAndReset(
'reading as float offset: $offset, maxFaces: $maxFaces, batchSize: $batchSize', 'reading as float offset: $offset, maxFaces: $maxFaces, batchSize: $batchSize',
); );
final db = await instance.sqliteAsyncDB; final db = await instance.asyncDB;
final Map<String, (int?, Uint8List)> result = {}; final Map<String, (int?, Uint8List)> result = {};
while (true) { while (true) {
@ -563,7 +602,7 @@ class FaceMLDataDB {
List<int> fileIDs, List<int> fileIDs,
) async { ) async {
_logger.info('reading face embeddings for ${fileIDs.length} files'); _logger.info('reading face embeddings for ${fileIDs.length} files');
final db = await instance.database; final db = await instance.asyncDB;
// Define the batch size // Define the batch size
const batchSize = 10000; const batchSize = 10000;
@ -572,15 +611,23 @@ class FaceMLDataDB {
final Map<String, Uint8List> result = {}; final Map<String, Uint8List> result = {};
while (true) { while (true) {
// Query a batch of rows // Query a batch of rows
final List<Map<String, dynamic>> maps = await db.query(
facesTable, final List<Map<String, dynamic>> maps = await db.getAll('''
columns: [faceIDColumn, faceEmbeddingBlob], SELECT $faceIDColumn, $faceEmbeddingBlob
where: FROM $facesTable
'$faceScore > $kMinimumQualityFaceScore AND $faceBlur > $kLaplacianHardThreshold AND $fileIDColumn IN (${fileIDs.join(",")})', WHERE $faceScore > $kMinimumQualityFaceScore AND $faceBlur > $kLaplacianHardThreshold AND $fileIDColumn IN (${fileIDs.join(",")})
limit: batchSize, ORDER BY $faceIDColumn DESC
offset: offset, LIMIT $batchSize OFFSET $offset
orderBy: '$faceIDColumn DESC', ''');
); // final List<Map<String, dynamic>> maps = await db.query(
// facesTable,
// columns: [faceIDColumn, faceEmbeddingBlob],
// where:
// '$faceScore > $kMinimumQualityFaceScore AND $faceBlur > $kLaplacianHardThreshold AND $fileIDColumn IN (${fileIDs.join(",")})',
// limit: batchSize,
// offset: offset,
// orderBy: '$faceIDColumn DESC',
// );
// Break the loop if no more rows // Break the loop if no more rows
if (maps.isEmpty) { if (maps.isEmpty) {
break; break;
@ -602,7 +649,7 @@ class FaceMLDataDB {
Iterable<String> faceIDs, Iterable<String> faceIDs,
) async { ) async {
_logger.info('reading face embeddings for ${faceIDs.length} faces'); _logger.info('reading face embeddings for ${faceIDs.length} faces');
final db = await instance.sqliteAsyncDB; final db = await instance.asyncDB;
// Define the batch size // Define the batch size
const batchSize = 10000; const batchSize = 10000;
@ -639,7 +686,7 @@ class FaceMLDataDB {
Future<int> getTotalFaceCount({ Future<int> getTotalFaceCount({
double minFaceScore = kMinimumQualityFaceScore, double minFaceScore = kMinimumQualityFaceScore,
}) async { }) async {
final db = await instance.sqliteAsyncDB; final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.getAll( final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore > $minFaceScore AND $faceBlur > $kLaplacianHardThreshold', 'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore > $minFaceScore AND $faceBlur > $kLaplacianHardThreshold',
); );
@ -647,7 +694,7 @@ class FaceMLDataDB {
} }
Future<double> getClusteredToTotalFacesRatio() async { Future<double> getClusteredToTotalFacesRatio() async {
final db = await instance.sqliteAsyncDB; final db = await instance.asyncDB;
final List<Map<String, dynamic>> totalFacesMaps = await db.getAll( final List<Map<String, dynamic>> totalFacesMaps = await db.getAll(
'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore > $kMinimumQualityFaceScore AND $faceBlur > $kLaplacianHardThreshold', 'SELECT COUNT(*) as count FROM $facesTable WHERE $faceScore > $kMinimumQualityFaceScore AND $faceBlur > $kLaplacianHardThreshold',
@ -665,105 +712,107 @@ class FaceMLDataDB {
Future<int> getBlurryFaceCount([ Future<int> getBlurryFaceCount([
int blurThreshold = kLaplacianHardThreshold, int blurThreshold = kLaplacianHardThreshold,
]) async { ]) async {
final db = await instance.database; final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.rawQuery( final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT COUNT(*) as count FROM $facesTable WHERE $faceBlur <= $blurThreshold AND $faceScore > $kMinimumQualityFaceScore', 'SELECT COUNT(*) as count FROM $facesTable WHERE $faceBlur <= $blurThreshold AND $faceScore > $kMinimumQualityFaceScore',
); );
return maps.first['count'] as int; return maps.first['count'] as int;
} }
Future<void> resetClusterIDs() async { Future<void> resetClusterIDs() async {
final db = await instance.database; try {
await db.execute(dropFaceClustersTable); final db = await instance.asyncDB;
await db.execute(createFaceClustersTable);
await db.execute(fcClusterIDIndex); await db.execute(dropFaceClustersTable);
await db.execute(createFaceClustersTable);
await db.execute(fcClusterIDIndex);
} catch (e, s) {
_logger.severe('Error resetting clusterIDs', e, s);
}
} }
Future<void> assignClusterToPerson({ Future<void> assignClusterToPerson({
required String personID, required String personID,
required int clusterID, required int clusterID,
}) async { }) async {
final db = await instance.database; final db = await instance.asyncDB;
await db.insert(
clusterPersonTable, const String sql = '''
{ INSERT INTO $clusterPersonTable ($personIdColumn, $clusterIDColumn) VALUES (?, ?) ON CONFLICT($personIdColumn, $clusterIDColumn) DO NOTHING
personIdColumn: personID, ''';
clusterIDColumn: clusterID, await db.execute(sql, [personID, clusterID]);
},
);
} }
Future<void> bulkAssignClusterToPersonID( Future<void> bulkAssignClusterToPersonID(
Map<int, String> clusterToPersonID, Map<int, String> clusterToPersonID,
) async { ) async {
final db = await instance.database; final db = await instance.asyncDB;
final batch = db.batch();
for (final entry in clusterToPersonID.entries) { const String sql = '''
final clusterID = entry.key; INSERT INTO $clusterPersonTable ($personIdColumn, $clusterIDColumn) VALUES (?, ?) ON CONFLICT($personIdColumn, $clusterIDColumn) DO NOTHING
final personID = entry.value; ''';
batch.insert( final parameterSets =
clusterPersonTable, clusterToPersonID.entries.map((e) => [e.value, e.key]).toList();
{ await db.executeBatch(sql, parameterSets);
personIdColumn: personID, // final batch = db.batch();
clusterIDColumn: clusterID, // for (final entry in clusterToPersonID.entries) {
}, // final clusterID = entry.key;
conflictAlgorithm: ConflictAlgorithm.replace, // final personID = entry.value;
); // batch.insert(
} // clusterPersonTable,
await batch.commit(noResult: true); // {
// personIdColumn: personID,
// clusterIDColumn: clusterID,
// },
// conflictAlgorithm: ConflictAlgorithm.replace,
// );
// }
// await batch.commit(noResult: true);
} }
Future<void> captureNotPersonFeedback({ Future<void> captureNotPersonFeedback({
required String personID, required String personID,
required int clusterID, required int clusterID,
}) async { }) async {
final db = await instance.database; final db = await instance.asyncDB;
await db.insert(
notPersonFeedback, const String sql = '''
{ INSERT INTO $notPersonFeedback ($personIdColumn, $clusterIDColumn) VALUES (?, ?) ON CONFLICT($personIdColumn, $clusterIDColumn) DO NOTHING
personIdColumn: personID, ''';
clusterIDColumn: clusterID, await db.execute(sql, [personID, clusterID]);
},
);
} }
Future<void> bulkCaptureNotPersonFeedback( Future<void> bulkCaptureNotPersonFeedback(
Map<int, String> clusterToPersonID, Map<int, String> clusterToPersonID,
) async { ) async {
final db = await instance.database; final db = await instance.asyncDB;
final batch = db.batch();
for (final entry in clusterToPersonID.entries) { const String sql = '''
final clusterID = entry.key; INSERT INTO $notPersonFeedback ($personIdColumn, $clusterIDColumn) VALUES (?, ?) ON CONFLICT($personIdColumn, $clusterIDColumn) DO NOTHING
final personID = entry.value; ''';
batch.insert( final parameterSets =
notPersonFeedback, clusterToPersonID.entries.map((e) => [e.value, e.key]).toList();
{
personIdColumn: personID, await db.executeBatch(sql, parameterSets);
clusterIDColumn: clusterID,
},
conflictAlgorithm: ConflictAlgorithm.replace,
);
}
await batch.commit(noResult: true);
} }
Future<int> removeClusterToPerson({ Future<void> removeClusterToPerson({
required String personID, required String personID,
required int clusterID, required int clusterID,
}) async { }) async {
final db = await instance.database; final db = await instance.asyncDB;
return db.delete(
clusterPersonTable, const String sql = '''
where: '$personIdColumn = ? AND $clusterIDColumn = ?', DELETE FROM $clusterPersonTable WHERE $personIdColumn = ? AND $clusterIDColumn = ?
whereArgs: [personID, clusterID], ''';
); await db.execute(sql, [personID, clusterID]);
} }
// for a given personID, return a map of clusterID to fileIDs using join query // for a given personID, return a map of clusterID to fileIDs using join query
Future<Map<int, Set<int>>> getFileIdToClusterIDSet(String personID) { Future<Map<int, Set<int>>> getFileIdToClusterIDSet(String personID) {
final db = instance.database; final db = instance.asyncDB;
return db.then((db) async { return db.then((db) async {
final List<Map<String, dynamic>> maps = await db.rawQuery( final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $faceClustersTable.$fcClusterID, $fcFaceId FROM $faceClustersTable ' 'SELECT $faceClustersTable.$fcClusterID, $fcFaceId FROM $faceClustersTable '
'INNER JOIN $clusterPersonTable ' 'INNER JOIN $clusterPersonTable '
'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$clusterIDColumn ' 'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$clusterIDColumn '
@ -784,9 +833,9 @@ class FaceMLDataDB {
Future<Map<int, Set<int>>> getFileIdToClusterIDSetForCluster( Future<Map<int, Set<int>>> getFileIdToClusterIDSetForCluster(
Set<int> clusterIDs, Set<int> clusterIDs,
) { ) {
final db = instance.database; final db = instance.asyncDB;
return db.then((db) async { return db.then((db) async {
final List<Map<String, dynamic>> maps = await db.rawQuery( final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable ' 'SELECT $fcClusterID, $fcFaceId FROM $faceClustersTable '
'WHERE $fcClusterID IN (${clusterIDs.join(",")})', 'WHERE $fcClusterID IN (${clusterIDs.join(",")})',
); );
@ -802,37 +851,57 @@ class FaceMLDataDB {
} }
Future<void> clusterSummaryUpdate(Map<int, (Uint8List, int)> summary) async { Future<void> clusterSummaryUpdate(Map<int, (Uint8List, int)> summary) async {
final db = await instance.database; final db = await instance.asyncDB;
var batch = db.batch();
const String sql = '''
INSERT INTO $clusterSummaryTable ($clusterIDColumn, $avgColumn, $countColumn) VALUES (?, ?, ?) ON CONFLICT($clusterIDColumn) DO UPDATE SET $avgColumn = excluded.$avgColumn, $countColumn = excluded.$countColumn
''';
final List<List<Object?>> parameterSets = [];
int batchCounter = 0; int batchCounter = 0;
for (final entry in summary.entries) { for (final entry in summary.entries) {
if (batchCounter == 400) { if (batchCounter == 400) {
await batch.commit(noResult: true); await db.executeBatch(sql, parameterSets);
batch = db.batch();
batchCounter = 0; batchCounter = 0;
parameterSets.clear();
} }
final int cluserID = entry.key; final int clusterID = entry.key;
final int count = entry.value.$2; final int count = entry.value.$2;
final Uint8List avg = entry.value.$1; final Uint8List avg = entry.value.$1;
batch.insert( parameterSets.add([clusterID, avg, count]);
clusterSummaryTable,
{
clusterIDColumn: cluserID,
avgColumn: avg,
countColumn: count,
},
conflictAlgorithm: ConflictAlgorithm.replace,
);
batchCounter++; batchCounter++;
} }
await batch.commit(noResult: true); await db.executeBatch(sql, parameterSets);
// var batch = db.batch();
// int batchCounter = 0;
// for (final entry in summary.entries) {
// if (batchCounter == 400) {
// await batch.commit(noResult: true);
// batch = db.batch();
// batchCounter = 0;
// }
// final int cluserID = entry.key;
// final int count = entry.value.$2;
// final Uint8List avg = entry.value.$1;
// batch.insert(
// clusterSummaryTable,
// {
// clusterIDColumn: cluserID,
// avgColumn: avg,
// countColumn: count,
// },
// conflictAlgorithm: ConflictAlgorithm.replace,
// );
// batchCounter++;
// }
// await batch.commit(noResult: true);
} }
/// Returns a map of clusterID to (avg embedding, count) /// Returns a map of clusterID to (avg embedding, count)
Future<Map<int, (Uint8List, int)>> getAllClusterSummary([ Future<Map<int, (Uint8List, int)>> getAllClusterSummary([
int? minClusterSize, int? minClusterSize,
]) async { ]) async {
final db = await instance.sqliteAsyncDB; final db = await instance.asyncDB;
final Map<int, (Uint8List, int)> result = {}; final Map<int, (Uint8List, int)> result = {};
final rows = await db.getAll( final rows = await db.getAll(
'SELECT * FROM $clusterSummaryTable${minClusterSize != null ? ' WHERE $countColumn >= $minClusterSize' : ''}', 'SELECT * FROM $clusterSummaryTable${minClusterSize != null ? ' WHERE $countColumn >= $minClusterSize' : ''}',
@ -846,9 +915,26 @@ class FaceMLDataDB {
return result; return result;
} }
Future<Map<int, (Uint8List, int)>> getClusterToClusterSummary(
Iterable<int> clusterIDs,
) async {
final db = await instance.asyncDB;
final Map<int, (Uint8List, int)> result = {};
final rows = await db.getAll(
'SELECT * FROM $clusterSummaryTable WHERE $clusterIDColumn IN (${clusterIDs.join(",")})',
);
for (final r in rows) {
final id = r[clusterIDColumn] as int;
final avg = r[avgColumn] as Uint8List;
final count = r[countColumn] as int;
result[id] = (avg, count);
}
return result;
}
Future<Map<int, String>> getClusterIDToPersonID() async { Future<Map<int, String>> getClusterIDToPersonID() async {
final db = await instance.database; final db = await instance.asyncDB;
final List<Map<String, dynamic>> maps = await db.rawQuery( final List<Map<String, dynamic>> maps = await db.getAll(
'SELECT $personIdColumn, $clusterIDColumn FROM $clusterPersonTable', 'SELECT $personIdColumn, $clusterIDColumn FROM $clusterPersonTable',
); );
final Map<int, String> result = {}; final Map<int, String> result = {};
@ -860,43 +946,55 @@ class FaceMLDataDB {
/// WARNING: This will delete ALL data in the database! Only use this for debug/testing purposes! /// WARNING: This will delete ALL data in the database! Only use this for debug/testing purposes!
Future<void> dropClustersAndPersonTable({bool faces = false}) async { Future<void> dropClustersAndPersonTable({bool faces = false}) async {
final db = await instance.database; try {
if (faces) { final db = await instance.asyncDB;
await db.execute(deleteFacesTable); if (faces) {
await db.execute(createFacesTable); await db.execute(deleteFacesTable);
await db.execute(dropFaceClustersTable); await db.execute(createFacesTable);
await db.execute(createFaceClustersTable); await db.execute(dropFaceClustersTable);
await db.execute(fcClusterIDIndex); await db.execute(createFaceClustersTable);
} await db.execute(fcClusterIDIndex);
await db.execute(deletePersonTable); }
await db.execute(dropClusterPersonTable);
await db.execute(dropClusterSummaryTable);
await db.execute(dropNotPersonFeedbackTable);
await db.execute(createClusterPersonTable); await db.execute(deletePersonTable);
await db.execute(createNotPersonFeedbackTable); await db.execute(dropClusterPersonTable);
await db.execute(createClusterSummaryTable); await db.execute(dropClusterSummaryTable);
await db.execute(dropNotPersonFeedbackTable);
await db.execute(createClusterPersonTable);
await db.execute(createNotPersonFeedbackTable);
await db.execute(createClusterSummaryTable);
} catch (e, s) {
_logger.severe('Error dropping clusters and person table', e, s);
}
} }
/// WARNING: This will delete ALL data in the database! Only use this for debug/testing purposes! /// WARNING: This will delete ALL data in the database! Only use this for debug/testing purposes!
Future<void> dropFeedbackTables() async { Future<void> dropFeedbackTables() async {
final db = await instance.database; try {
final db = await instance.asyncDB;
await db.execute(deletePersonTable); // Drop the tables
await db.execute(dropClusterPersonTable); await db.execute(deletePersonTable);
await db.execute(dropNotPersonFeedbackTable); await db.execute(dropClusterPersonTable);
await db.execute(dropClusterSummaryTable); await db.execute(dropNotPersonFeedbackTable);
await db.execute(createClusterPersonTable); await db.execute(dropClusterSummaryTable);
await db.execute(createNotPersonFeedbackTable);
await db.execute(createClusterSummaryTable); // Recreate the tables
await db.execute(createClusterPersonTable);
await db.execute(createNotPersonFeedbackTable);
await db.execute(createClusterSummaryTable);
} catch (e) {
_logger.severe('Error dropping feedback tables', e);
}
} }
Future<void> removeFilesFromPerson( Future<void> removeFilesFromPerson(
List<EnteFile> files, List<EnteFile> files,
String personID, String personID,
) async { ) async {
final db = await instance.database; final db = await instance.asyncDB;
final faceIdsResult = await db.rawQuery( final faceIdsResult = await db.getAll(
'SELECT $fcFaceId FROM $faceClustersTable LEFT JOIN $clusterPersonTable ' 'SELECT $fcFaceId FROM $faceClustersTable LEFT JOIN $clusterPersonTable '
'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$clusterIDColumn ' 'ON $faceClustersTable.$fcClusterID = $clusterPersonTable.$clusterIDColumn '
'WHERE $clusterPersonTable.$personIdColumn = ?', 'WHERE $clusterPersonTable.$personIdColumn = ?',
@ -922,8 +1020,8 @@ class FaceMLDataDB {
List<EnteFile> files, List<EnteFile> files,
int clusterID, int clusterID,
) async { ) async {
final db = await instance.database; final db = await instance.asyncDB;
final faceIdsResult = await db.rawQuery( final faceIdsResult = await db.getAll(
'SELECT $fcFaceId FROM $faceClustersTable ' 'SELECT $fcFaceId FROM $faceClustersTable '
'WHERE $faceClustersTable.$fcClusterID = ?', 'WHERE $faceClustersTable.$fcClusterID = ?',
[clusterID], [clusterID],

5
mobile/lib/face/db_fields.dart
View File

@ -16,7 +16,7 @@ const mlVersionColumn = 'ml_version';
const createFacesTable = '''CREATE TABLE IF NOT EXISTS $facesTable ( const createFacesTable = '''CREATE TABLE IF NOT EXISTS $facesTable (
$fileIDColumn INTEGER NOT NULL, $fileIDColumn INTEGER NOT NULL,
$faceIDColumn TEXT NOT NULL, $faceIDColumn TEXT NOT NULL UNIQUE,
$faceDetectionColumn TEXT NOT NULL, $faceDetectionColumn TEXT NOT NULL,
$faceEmbeddingBlob BLOB NOT NULL, $faceEmbeddingBlob BLOB NOT NULL,
$faceScore REAL NOT NULL, $faceScore REAL NOT NULL,
@ -95,7 +95,8 @@ const notPersonFeedback = 'not_person_feedback';
const createNotPersonFeedbackTable = ''' const createNotPersonFeedbackTable = '''
CREATE TABLE IF NOT EXISTS $notPersonFeedback ( CREATE TABLE IF NOT EXISTS $notPersonFeedback (
$personIdColumn TEXT NOT NULL, $personIdColumn TEXT NOT NULL,
$clusterIDColumn INTEGER NOT NULL $clusterIDColumn INTEGER NOT NULL,
PRIMARY KEY($personIdColumn, $clusterIDColumn)
); );
'''; ''';
const dropNotPersonFeedbackTable = 'DROP TABLE IF EXISTS $notPersonFeedback'; const dropNotPersonFeedbackTable = 'DROP TABLE IF EXISTS $notPersonFeedback';

2
mobile/lib/face/model/detection.dart
View File

@ -155,7 +155,7 @@ class Detection {
(nose[0] < min(leftEye[0], rightEye[0]) - 0.5 * eyeDistanceX) && (nose[0] < min(leftEye[0], rightEye[0]) - 0.5 * eyeDistanceX) &&
(nose[0] < min(leftMouth[0], rightMouth[0])); (nose[0] < min(leftMouth[0], rightMouth[0]));
final bool noseStickingOutRight = final bool noseStickingOutRight =
(nose[0] > max(leftEye[0], rightEye[0]) - 0.5 * eyeDistanceX) && (nose[0] > max(leftEye[0], rightEye[0]) + 0.5 * eyeDistanceX) &&
(nose[0] > max(leftMouth[0], rightMouth[0])); (nose[0] > max(leftMouth[0], rightMouth[0]));
return faceIsUpright && (noseStickingOutLeft || noseStickingOutRight); return faceIsUpright && (noseStickingOutLeft || noseStickingOutRight);

2
mobile/lib/services/entity_service.dart
View File

@ -61,7 +61,7 @@ class EntityService {
}) async { }) async {
final key = await getOrCreateEntityKey(type); final key = await getOrCreateEntityKey(type);
final encryptedKeyData = await CryptoUtil.encryptChaCha( final encryptedKeyData = await CryptoUtil.encryptChaCha(
utf8.encode(plainText) as Uint8List, utf8.encode(plainText),
key, key,
); );
final String encryptedData = final String encryptedData =

13
mobile/lib/services/machine_learning/face_ml/face_clustering/cosine_distance.dart
View File

@ -1,5 +1,18 @@
import 'dart:math' show sqrt; import 'dart:math' show sqrt;
import "package:ml_linalg/vector.dart";
/// Calculates the cosine distance between two embeddings/vectors using SIMD from ml_linalg
///
/// WARNING: This assumes both vectors are already normalized!
double cosineDistanceSIMD(Vector vector1, Vector vector2) {
if (vector1.length != vector2.length) {
throw ArgumentError('Vectors must be the same length');
}
return 1 - vector1.dot(vector2);
}
/// Calculates the cosine distance between two embeddings/vectors. /// Calculates the cosine distance between two embeddings/vectors.
/// ///
/// Throws an ArgumentError if the vectors are of different lengths or /// Throws an ArgumentError if the vectors are of different lengths or

32
mobile/lib/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart
View File

@ -69,7 +69,7 @@ class FaceClusteringService {
bool isRunning = false; bool isRunning = false;
static const kRecommendedDistanceThreshold = 0.24; static const kRecommendedDistanceThreshold = 0.24;
static const kConservativeDistanceThreshold = 0.06; static const kConservativeDistanceThreshold = 0.16;
// singleton pattern // singleton pattern
FaceClusteringService._privateConstructor(); FaceClusteringService._privateConstructor();
@ -560,10 +560,10 @@ class FaceClusteringService {
for (int j = i - 1; j >= 0; j--) { for (int j = i - 1; j >= 0; j--) {
late double distance; late double distance;
if (sortedFaceInfos[i].vEmbedding != null) { if (sortedFaceInfos[i].vEmbedding != null) {
distance = 1.0 - distance = cosineDistanceSIMD(
sortedFaceInfos[i] sortedFaceInfos[i].vEmbedding!,
.vEmbedding! sortedFaceInfos[j].vEmbedding!,
.dot(sortedFaceInfos[j].vEmbedding!); );
} else { } else {
distance = cosineDistForNormVectors( distance = cosineDistForNormVectors(
sortedFaceInfos[i].embedding!, sortedFaceInfos[i].embedding!,
@ -624,7 +624,7 @@ class FaceClusteringService {
} }
// analyze the results // analyze the results
FaceClusteringService._analyzeClusterResults(sortedFaceInfos); // FaceClusteringService._analyzeClusterResults(sortedFaceInfos);
return ClusteringResult( return ClusteringResult(
newFaceIdToCluster: newFaceIdToCluster, newFaceIdToCluster: newFaceIdToCluster,
@ -804,8 +804,10 @@ class FaceClusteringService {
double closestDistance = double.infinity; double closestDistance = double.infinity;
for (int j = 0; j < totalFaces; j++) { for (int j = 0; j < totalFaces; j++) {
if (i == j) continue; if (i == j) continue;
final double distance = final double distance = cosineDistanceSIMD(
1.0 - faceInfos[i].vEmbedding!.dot(faceInfos[j].vEmbedding!); faceInfos[i].vEmbedding!,
faceInfos[j].vEmbedding!,
);
if (distance < closestDistance) { if (distance < closestDistance) {
closestDistance = distance; closestDistance = distance;
closestIdx = j; closestIdx = j;
@ -855,10 +857,10 @@ class FaceClusteringService {
for (int i = 0; i < clusterIds.length; i++) { for (int i = 0; i < clusterIds.length; i++) {
for (int j = 0; j < clusterIds.length; j++) { for (int j = 0; j < clusterIds.length; j++) {
if (i == j) continue; if (i == j) continue;
final double newDistance = 1.0 - final double newDistance = cosineDistanceSIMD(
clusterIdToMeanEmbeddingAndWeight[clusterIds[i]]!.$1.dot( clusterIdToMeanEmbeddingAndWeight[clusterIds[i]]!.$1,
clusterIdToMeanEmbeddingAndWeight[clusterIds[j]]!.$1, clusterIdToMeanEmbeddingAndWeight[clusterIds[j]]!.$1,
); );
if (newDistance < distance) { if (newDistance < distance) {
distance = newDistance; distance = newDistance;
clusterIDsToMerge = (clusterIds[i], clusterIds[j]); clusterIDsToMerge = (clusterIds[i], clusterIds[j]);
@ -959,9 +961,9 @@ class FaceClusteringService {
// Run the DBSCAN clustering // Run the DBSCAN clustering
final List<List<int>> clusterOutput = dbscan.run(embeddings); final List<List<int>> clusterOutput = dbscan.run(embeddings);
final List<List<FaceInfo>> clusteredFaceInfos = clusterOutput // final List<List<FaceInfo>> clusteredFaceInfos = clusterOutput
.map((cluster) => cluster.map((idx) => faceInfos[idx]).toList()) // .map((cluster) => cluster.map((idx) => faceInfos[idx]).toList())
.toList(); // .toList();
final List<List<String>> clusteredFaceIDs = clusterOutput final List<List<String>> clusteredFaceIDs = clusterOutput
.map((cluster) => cluster.map((idx) => faceInfos[idx].faceID).toList()) .map((cluster) => cluster.map((idx) => faceInfos[idx].faceID).toList())
.toList(); .toList();

7
mobile/lib/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart
View File

@ -1,8 +1,8 @@
import 'package:photos/services/machine_learning/face_ml/face_detection/face_detection_service.dart'; import 'package:photos/services/machine_learning/face_ml/face_detection/face_detection_service.dart';
/// Blur detection threshold /// Blur detection threshold
const kLaplacianHardThreshold = 15; const kLaplacianHardThreshold = 10;
const kLaplacianSoftThreshold = 100; const kLaplacianSoftThreshold = 50;
const kLaplacianVerySoftThreshold = 200; const kLaplacianVerySoftThreshold = 200;
/// Default blur value /// Default blur value
@ -15,3 +15,6 @@ const kHighQualityFaceScore = 0.90;
/// The minimum score for a face to be detected, regardless of quality. Use [kMinimumQualityFaceScore] for high quality faces. /// The minimum score for a face to be detected, regardless of quality. Use [kMinimumQualityFaceScore] for high quality faces.
const kMinFaceDetectionScore = FaceDetectionService.kMinScoreSigmoidThreshold; const kMinFaceDetectionScore = FaceDetectionService.kMinScoreSigmoidThreshold;
/// The minimum cluster size for displaying a cluster in the UI
const kMinimumClusterSizeSearchResult = 20;

21
mobile/lib/services/machine_learning/face_ml/face_ml_service.dart
View File

@ -295,6 +295,7 @@ class FaceMlService {
bool clusterInBuckets = true, bool clusterInBuckets = true,
}) async { }) async {
_logger.info("`clusterAllImages()` called"); _logger.info("`clusterAllImages()` called");
final clusterAllImagesTime = DateTime.now();
try { try {
// Get a sense of the total number of faces in the database // Get a sense of the total number of faces in the database
@ -349,7 +350,7 @@ class FaceMlService {
} }
await FaceMLDataDB.instance await FaceMLDataDB.instance
.updateClusterIdToFaceId(clusteringResult.newFaceIdToCluster); .updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster);
await FaceMLDataDB.instance await FaceMLDataDB.instance
.clusterSummaryUpdate(clusteringResult.newClusterSummaries!); .clusterSummaryUpdate(clusteringResult.newClusterSummaries!);
_logger.info( _logger.info(
@ -402,13 +403,14 @@ class FaceMlService {
'Updating ${clusteringResult.newFaceIdToCluster.length} FaceIDs with clusterIDs in the DB', 'Updating ${clusteringResult.newFaceIdToCluster.length} FaceIDs with clusterIDs in the DB',
); );
await FaceMLDataDB.instance await FaceMLDataDB.instance
.updateClusterIdToFaceId(clusteringResult.newFaceIdToCluster); .updateFaceIdToClusterId(clusteringResult.newFaceIdToCluster);
await FaceMLDataDB.instance await FaceMLDataDB.instance
.clusterSummaryUpdate(clusteringResult.newClusterSummaries!); .clusterSummaryUpdate(clusteringResult.newClusterSummaries!);
_logger.info('Done updating FaceIDs with clusterIDs in the DB, in ' _logger.info('Done updating FaceIDs with clusterIDs in the DB, in '
'${DateTime.now().difference(clusterDoneTime).inSeconds} seconds'); '${DateTime.now().difference(clusterDoneTime).inSeconds} seconds');
} }
_logger.info('clusterAllImages() finished'); _logger.info('clusterAllImages() finished, in '
'${DateTime.now().difference(clusterAllImagesTime).inSeconds} seconds');
} catch (e, s) { } catch (e, s) {
_logger.severe("`clusterAllImages` failed", e, s); _logger.severe("`clusterAllImages` failed", e, s);
} }
@ -868,7 +870,7 @@ class FaceMlService {
stopwatch.stop(); stopwatch.stop();
_logger.info( _logger.info(
"Finished Analyze image (${result.faces.length} faces) with uploadedFileID ${enteFile.uploadedFileID}, in " "Finished Analyze image (${result.faces.length} faces) with uploadedFileID ${enteFile.uploadedFileID}, in "
"${stopwatch.elapsedMilliseconds} ms", "${stopwatch.elapsedMilliseconds} ms (including time waiting for inference engine availability)",
); );
return result; return result;
@ -964,7 +966,12 @@ class FaceMlService {
switch (typeOfData) { switch (typeOfData) {
case FileDataForML.fileData: case FileDataForML.fileData:
final stopwatch = Stopwatch()..start(); final stopwatch = Stopwatch()..start();
final File? file = await getFile(enteFile, isOrigin: true); File? file;
if (enteFile.fileType == FileType.video) {
file = await getThumbnailForUploadedFile(enteFile);
} else {
file = await getFile(enteFile, isOrigin: true);
}
if (file == null) { if (file == null) {
_logger.warning("Could not get file for $enteFile"); _logger.warning("Could not get file for $enteFile");
imagePath = null; imagePath = null;
@ -1292,10 +1299,6 @@ class FaceMlService {
if (!enteFile.isUploaded || enteFile.isOwner == false) { if (!enteFile.isUploaded || enteFile.isOwner == false) {
return true; return true;
} }
// Skip if the file is a video
if (enteFile.fileType == FileType.video) {
return true;
}
// I don't know how motionPhotos and livePhotos work, so I'm also just skipping them for now // I don't know how motionPhotos and livePhotos work, so I'm also just skipping them for now
if (enteFile.fileType == FileType.other) { if (enteFile.fileType == FileType.other) {
return true; return true;

451
mobile/lib/services/machine_learning/face_ml/feedback/cluster_feedback.dart
View File

@ -1,20 +1,20 @@
import 'dart:developer' as dev; import 'dart:developer' as dev;
import "dart:math" show Random; import "dart:math" show Random, min;
import "package:flutter/foundation.dart"; import "package:flutter/foundation.dart";
import "package:logging/logging.dart"; import "package:logging/logging.dart";
import "package:ml_linalg/linalg.dart";
import "package:photos/core/event_bus.dart"; import "package:photos/core/event_bus.dart";
import "package:photos/db/files_db.dart"; import "package:photos/db/files_db.dart";
// import "package:photos/events/files_updated_event.dart";
// import "package:photos/events/local_photos_updated_event.dart";
import "package:photos/events/people_changed_event.dart"; import "package:photos/events/people_changed_event.dart";
import "package:photos/extensions/stop_watch.dart"; import "package:photos/extensions/stop_watch.dart";
import "package:photos/face/db.dart"; import "package:photos/face/db.dart";
import "package:photos/face/model/person.dart"; import "package:photos/face/model/person.dart";
import "package:photos/generated/protos/ente/common/vector.pb.dart"; import "package:photos/generated/protos/ente/common/vector.pb.dart";
import "package:photos/models/file/file.dart"; import "package:photos/models/file/file.dart";
import 'package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart'; import "package:photos/services/machine_learning/face_ml/face_clustering/cosine_distance.dart";
import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart"; import "package:photos/services/machine_learning/face_ml/face_clustering/face_clustering_service.dart";
import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart";
import "package:photos/services/machine_learning/face_ml/face_ml_result.dart"; import "package:photos/services/machine_learning/face_ml/face_ml_result.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import "package:photos/services/search_service.dart"; import "package:photos/services/search_service.dart";
@ -24,12 +24,14 @@ class ClusterSuggestion {
final double distancePersonToCluster; final double distancePersonToCluster;
final bool usedOnlyMeanForSuggestion; final bool usedOnlyMeanForSuggestion;
final List<EnteFile> filesInCluster; final List<EnteFile> filesInCluster;
final List<String> faceIDsInCluster;
ClusterSuggestion( ClusterSuggestion(
this.clusterIDToMerge, this.clusterIDToMerge,
this.distancePersonToCluster, this.distancePersonToCluster,
this.usedOnlyMeanForSuggestion, this.usedOnlyMeanForSuggestion,
this.filesInCluster, this.filesInCluster,
this.faceIDsInCluster,
); );
} }
@ -59,19 +61,27 @@ class ClusterFeedbackService {
bool extremeFilesFirst = true, bool extremeFilesFirst = true,
}) async { }) async {
_logger.info( _logger.info(
'getClusterFilesForPersonID ${kDebugMode ? person.data.name : person.remoteID}', 'getSuggestionForPerson ${kDebugMode ? person.data.name : person.remoteID}',
); );
try { try {
// Get the suggestions for the person using centroids and median // Get the suggestions for the person using centroids and median
final List<(int, double, bool)> suggestClusterIds = final startTime = DateTime.now();
await _getSuggestionsUsingMedian(person); final List<(int, double, bool)> foundSuggestions =
await _getSuggestions(person);
final findSuggestionsTime = DateTime.now();
_logger.info(
'getSuggestionForPerson `_getSuggestions`: Found ${foundSuggestions.length} suggestions in ${findSuggestionsTime.difference(startTime).inMilliseconds} ms',
);
// Get the files for the suggestions // Get the files for the suggestions
final suggestionClusterIDs = foundSuggestions.map((e) => e.$1).toSet();
final Map<int, Set<int>> fileIdToClusterID = final Map<int, Set<int>> fileIdToClusterID =
await FaceMLDataDB.instance.getFileIdToClusterIDSetForCluster( await FaceMLDataDB.instance.getFileIdToClusterIDSetForCluster(
suggestClusterIds.map((e) => e.$1).toSet(), suggestionClusterIDs,
); );
final clusterIdToFaceIDs =
await FaceMLDataDB.instance.getClusterToFaceIDs(suggestionClusterIDs);
final Map<int, List<EnteFile>> clusterIDToFiles = {}; final Map<int, List<EnteFile>> clusterIDToFiles = {};
final allFiles = await SearchService.instance.getAllFiles(); final allFiles = await SearchService.instance.getAllFiles();
for (final f in allFiles) { for (final f in allFiles) {
@ -88,25 +98,31 @@ class ClusterFeedbackService {
} }
} }
final List<ClusterSuggestion> clusterIdAndFiles = []; final List<ClusterSuggestion> finalSuggestions = [];
for (final clusterSuggestion in suggestClusterIds) { for (final clusterSuggestion in foundSuggestions) {
if (clusterIDToFiles.containsKey(clusterSuggestion.$1)) { if (clusterIDToFiles.containsKey(clusterSuggestion.$1)) {
clusterIdAndFiles.add( finalSuggestions.add(
ClusterSuggestion( ClusterSuggestion(
clusterSuggestion.$1, clusterSuggestion.$1,
clusterSuggestion.$2, clusterSuggestion.$2,
clusterSuggestion.$3, clusterSuggestion.$3,
clusterIDToFiles[clusterSuggestion.$1]!, clusterIDToFiles[clusterSuggestion.$1]!,
clusterIdToFaceIDs[clusterSuggestion.$1]!.toList(),
), ),
); );
} }
} }
final getFilesTime = DateTime.now();
final sortingStartTime = DateTime.now();
if (extremeFilesFirst) { if (extremeFilesFirst) {
await _sortSuggestionsOnDistanceToPerson(person, clusterIdAndFiles); await _sortSuggestionsOnDistanceToPerson(person, finalSuggestions);
} }
_logger.info(
'getSuggestionForPerson post-processing suggestions took ${DateTime.now().difference(findSuggestionsTime).inMilliseconds} ms, of which sorting took ${DateTime.now().difference(sortingStartTime).inMilliseconds} ms and getting files took ${getFilesTime.difference(findSuggestionsTime).inMilliseconds} ms',
);
return clusterIdAndFiles; return finalSuggestions;
} catch (e, s) { } catch (e, s) {
_logger.severe("Error in getClusterFilesForPersonID", e, s); _logger.severe("Error in getClusterFilesForPersonID", e, s);
rethrow; rethrow;
@ -228,20 +244,20 @@ class ClusterFeedbackService {
final ignoredClusters = await faceMlDb.getPersonIgnoredClusters(p.remoteID); final ignoredClusters = await faceMlDb.getPersonIgnoredClusters(p.remoteID);
final personClusters = await faceMlDb.getPersonClusterIDs(p.remoteID); final personClusters = await faceMlDb.getPersonClusterIDs(p.remoteID);
dev.log( dev.log(
'existing clusters for ${p.data.name} are $personClusters', '${p.data.name} has ${personClusters.length} existing clusters',
name: "ClusterFeedbackService", name: "ClusterFeedbackService",
); );
// Get and update the cluster summary to get the avg (centroid) and count // Get and update the cluster summary to get the avg (centroid) and count
final EnteWatch watch = EnteWatch("ClusterFeedbackService")..start(); final EnteWatch watch = EnteWatch("ClusterFeedbackService")..start();
final Map<int, List<double>> clusterAvg = await _getUpdateClusterAvg( final Map<int, Vector> clusterAvg = await _getUpdateClusterAvg(
allClusterIdsToCountMap, allClusterIdsToCountMap,
ignoredClusters, ignoredClusters,
); );
watch.log('computed avg for ${clusterAvg.length} clusters'); watch.log('computed avg for ${clusterAvg.length} clusters');
// Find the actual closest clusters for the person // Find the actual closest clusters for the person
final Map<int, List<(int, double)>> suggestions = _calcSuggestionsMean( final List<(int, double)> suggestions = _calcSuggestionsMean(
clusterAvg, clusterAvg,
personClusters, personClusters,
ignoredClusters, ignoredClusters,
@ -257,21 +273,17 @@ class ClusterFeedbackService {
} }
// log suggestions // log suggestions
for (final entry in suggestions.entries) { dev.log(
dev.log( 'suggestions for ${p.data.name} for cluster ID ${p.remoteID} are suggestions $suggestions}',
' ${entry.value.length} suggestion for ${p.data.name} for cluster ID ${entry.key} are suggestions ${entry.value}}', name: "ClusterFeedbackService",
name: "ClusterFeedbackService", );
);
}
for (final suggestionsPerCluster in suggestions.values) { for (final suggestion in suggestions) {
for (final suggestion in suggestionsPerCluster) { final clusterID = suggestion.$1;
final clusterID = suggestion.$1; await PersonService.instance.assignClusterToPerson(
await PersonService.instance.assignClusterToPerson( personID: p.remoteID,
personID: p.remoteID, clusterID: clusterID,
clusterID: clusterID, );
);
}
} }
Bus.instance.fire(PeopleChangedEvent()); Bus.instance.fire(PeopleChangedEvent());
@ -400,7 +412,7 @@ class ClusterFeedbackService {
final newClusterID = startClusterID + blurValue ~/ 10; final newClusterID = startClusterID + blurValue ~/ 10;
faceIdToCluster[faceID] = newClusterID; faceIdToCluster[faceID] = newClusterID;
} }
await FaceMLDataDB.instance.updateClusterIdToFaceId(faceIdToCluster); await FaceMLDataDB.instance.updateFaceIdToClusterId(faceIdToCluster);
Bus.instance.fire(PeopleChangedEvent()); Bus.instance.fire(PeopleChangedEvent());
} catch (e, s) { } catch (e, s) {
@ -433,111 +445,89 @@ class ClusterFeedbackService {
return; return;
} }
/// Returns a map of person's clusterID to map of closest clusterID to with disstance
Future<Map<int, List<(int, double)>>> getSuggestionsUsingMean(
PersonEntity p, {
double maxClusterDistance = 0.4,
}) async {
// Get all the cluster data
final faceMlDb = FaceMLDataDB.instance;
final allClusterIdsToCountMap = (await faceMlDb.clusterIdToFaceCount());
final ignoredClusters = await faceMlDb.getPersonIgnoredClusters(p.remoteID);
final personClusters = await faceMlDb.getPersonClusterIDs(p.remoteID);
dev.log(
'existing clusters for ${p.data.name} are $personClusters',
name: "ClusterFeedbackService",
);
// Get and update the cluster summary to get the avg (centroid) and count
final EnteWatch watch = EnteWatch("ClusterFeedbackService")..start();
final Map<int, List<double>> clusterAvg = await _getUpdateClusterAvg(
allClusterIdsToCountMap,
ignoredClusters,
);
watch.log('computed avg for ${clusterAvg.length} clusters');
// Find the actual closest clusters for the person
final Map<int, List<(int, double)>> suggestions = _calcSuggestionsMean(
clusterAvg,
personClusters,
ignoredClusters,
maxClusterDistance,
);
// log suggestions
for (final entry in suggestions.entries) {
dev.log(
' ${entry.value.length} suggestion for ${p.data.name} for cluster ID ${entry.key} are suggestions ${entry.value}}',
name: "ClusterFeedbackService",
);
}
return suggestions;
}
/// Returns a list of suggestions. For each suggestion we return a record consisting of the following elements: /// Returns a list of suggestions. For each suggestion we return a record consisting of the following elements:
/// 1. clusterID: the ID of the cluster /// 1. clusterID: the ID of the cluster
/// 2. distance: the distance between the person's cluster and the suggestion /// 2. distance: the distance between the person's cluster and the suggestion
/// 3. usedMean: whether the suggestion was found using the mean (true) or the median (false) /// 3. usedMean: whether the suggestion was found using the mean (true) or the median (false)
Future<List<(int, double, bool)>> _getSuggestionsUsingMedian( Future<List<(int, double, bool)>> _getSuggestions(
PersonEntity p, { PersonEntity p, {
int sampleSize = 50, int sampleSize = 50,
double maxMedianDistance = 0.65, double maxMedianDistance = 0.62,
double goodMedianDistance = 0.55, double goodMedianDistance = 0.55,
double maxMeanDistance = 0.65, double maxMeanDistance = 0.65,
double goodMeanDistance = 0.4, double goodMeanDistance = 0.50,
}) async { }) async {
final w = (kDebugMode ? EnteWatch('getSuggestions') : null)?..start();
// Get all the cluster data // Get all the cluster data
final faceMlDb = FaceMLDataDB.instance; final faceMlDb = FaceMLDataDB.instance;
// final Map<int, List<(int, double)>> suggestions = {}; final allClusterIdsToCountMap = await faceMlDb.clusterIdToFaceCount();
final allClusterIdsToCountMap = (await faceMlDb.clusterIdToFaceCount());
final ignoredClusters = await faceMlDb.getPersonIgnoredClusters(p.remoteID); final ignoredClusters = await faceMlDb.getPersonIgnoredClusters(p.remoteID);
final personClusters = await faceMlDb.getPersonClusterIDs(p.remoteID); final personClusters = await faceMlDb.getPersonClusterIDs(p.remoteID);
dev.log( final personFaceIDs =
'existing clusters for ${p.data.name} are $personClusters', await FaceMLDataDB.instance.getFaceIDsForPerson(p.remoteID);
name: "getSuggestionsUsingMedian", final personFileIDs = personFaceIDs.map(getFileIdFromFaceId).toSet();
w?.log(
'${p.data.name} has ${personClusters.length} existing clusters, getting all database data done',
); );
final allClusterIdToFaceIDs =
await FaceMLDataDB.instance.getAllClusterIdToFaceIDs();
w?.log('getAllClusterIdToFaceIDs done');
// Get and update the cluster summary to get the avg (centroid) and count // First only do a simple check on the big clusters, if the person does not have small clusters yet
final EnteWatch watch = EnteWatch("ClusterFeedbackService")..start(); final smallestPersonClusterSize = personClusters
final Map<int, List<double>> clusterAvg = await _getUpdateClusterAvg( .map((clusterID) => allClusterIdsToCountMap[clusterID] ?? 0)
allClusterIdsToCountMap, .reduce((value, element) => min(value, element));
ignoredClusters, final checkSizes = [20, kMinimumClusterSizeSearchResult, 10, 5, 1];
); late Map<int, Vector> clusterAvgBigClusters;
watch.log('computed avg for ${clusterAvg.length} clusters'); final List<(int, double)> suggestionsMean = [];
for (final minimumSize in checkSizes.toSet()) {
// Find the other cluster candidates based on the mean // if (smallestPersonClusterSize >= minimumSize) {
final Map<int, List<(int, double)>> suggestionsMean = _calcSuggestionsMean( clusterAvgBigClusters = await _getUpdateClusterAvg(
clusterAvg, allClusterIdsToCountMap,
personClusters, ignoredClusters,
ignoredClusters, minClusterSize: minimumSize,
goodMeanDistance, );
); w?.log(
if (suggestionsMean.isNotEmpty) { 'Calculate avg for ${clusterAvgBigClusters.length} clusters of min size $minimumSize',
final List<(int, double)> suggestClusterIds = []; );
for (final List<(int, double)> suggestion in suggestionsMean.values) { final List<(int, double)> suggestionsMeanBigClusters =
suggestClusterIds.addAll(suggestion); _calcSuggestionsMean(
clusterAvgBigClusters,
personClusters,
ignoredClusters,
goodMeanDistance,
);
w?.log(
'Calculate suggestions using mean for ${clusterAvgBigClusters.length} clusters of min size $minimumSize',
);
for (final suggestion in suggestionsMeanBigClusters) {
// Skip suggestions that have a high overlap with the person's files
final suggestionSet = allClusterIdToFaceIDs[suggestion.$1]!
.map((faceID) => getFileIdFromFaceId(faceID))
.toSet();
final overlap = personFileIDs.intersection(suggestionSet);
if (overlap.isNotEmpty &&
((overlap.length / suggestionSet.length) > 0.5)) {
await FaceMLDataDB.instance.captureNotPersonFeedback(
personID: p.remoteID,
clusterID: suggestion.$1,
);
continue;
}
suggestionsMean.add(suggestion);
}
if (suggestionsMean.isNotEmpty) {
return suggestionsMean
.map((e) => (e.$1, e.$2, true))
.toList(growable: false);
// }
} }
suggestClusterIds.sort(
(a, b) => allClusterIdsToCountMap[b.$1]!
.compareTo(allClusterIdsToCountMap[a.$1]!),
);
final suggestClusterIdsSizes = suggestClusterIds
.map((e) => allClusterIdsToCountMap[e.$1]!)
.toList(growable: false);
final suggestClusterIdsDistances =
suggestClusterIds.map((e) => e.$2).toList(growable: false);
_logger.info(
"Already found good suggestions using mean: $suggestClusterIds, with sizes $suggestClusterIdsSizes and distances $suggestClusterIdsDistances",
);
return suggestClusterIds
.map((e) => (e.$1, e.$2, true))
.toList(growable: false);
} }
w?.reset();
// Find the other cluster candidates based on the median // Find the other cluster candidates based on the median
final Map<int, List<(int, double)>> moreSuggestionsMean = final clusterAvg = clusterAvgBigClusters;
_calcSuggestionsMean( final List<(int, double)> moreSuggestionsMean = _calcSuggestionsMean(
clusterAvg, clusterAvg,
personClusters, personClusters,
ignoredClusters, ignoredClusters,
@ -549,12 +539,8 @@ class ClusterFeedbackService {
return []; return [];
} }
final List<(int, double)> temp = []; moreSuggestionsMean.sort((a, b) => a.$2.compareTo(b.$2));
for (final List<(int, double)> suggestion in moreSuggestionsMean.values) { final otherClusterIdsCandidates = moreSuggestionsMean
temp.addAll(suggestion);
}
temp.sort((a, b) => a.$2.compareTo(b.$2));
final otherClusterIdsCandidates = temp
.map( .map(
(e) => e.$1, (e) => e.$1,
) )
@ -563,21 +549,26 @@ class ClusterFeedbackService {
"Found potential suggestions from loose mean for median test: $otherClusterIdsCandidates", "Found potential suggestions from loose mean for median test: $otherClusterIdsCandidates",
); );
watch.logAndReset("Starting median test"); w?.logAndReset("Starting median test");
// Take the embeddings from the person's clusters in one big list and sample from it // Take the embeddings from the person's clusters in one big list and sample from it
final List<Uint8List> personEmbeddingsProto = []; final List<Uint8List> personEmbeddingsProto = [];
for (final clusterID in personClusters) { for (final clusterID in personClusters) {
final Iterable<Uint8List> embedings = final Iterable<Uint8List> embeddings =
await FaceMLDataDB.instance.getFaceEmbeddingsForCluster(clusterID); await FaceMLDataDB.instance.getFaceEmbeddingsForCluster(clusterID);
personEmbeddingsProto.addAll(embedings); personEmbeddingsProto.addAll(embeddings);
} }
final List<Uint8List> sampledEmbeddingsProto = final List<Uint8List> sampledEmbeddingsProto =
_randomSampleWithoutReplacement( _randomSampleWithoutReplacement(
personEmbeddingsProto, personEmbeddingsProto,
sampleSize, sampleSize,
); );
final List<List<double>> sampledEmbeddings = sampledEmbeddingsProto final List<Vector> sampledEmbeddings = sampledEmbeddingsProto
.map((embedding) => EVector.fromBuffer(embedding).values) .map(
(embedding) => Vector.fromList(
EVector.fromBuffer(embedding).values,
dtype: DType.float32,
),
)
.toList(growable: false); .toList(growable: false);
// Find the actual closest clusters for the person using median // Find the actual closest clusters for the person using median
@ -593,16 +584,20 @@ class ClusterFeedbackService {
otherEmbeddingsProto, otherEmbeddingsProto,
sampleSize, sampleSize,
); );
final List<List<double>> sampledOtherEmbeddings = final List<Vector> sampledOtherEmbeddings = sampledOtherEmbeddingsProto
sampledOtherEmbeddingsProto .map(
.map((embedding) => EVector.fromBuffer(embedding).values) (embedding) => Vector.fromList(
.toList(growable: false); EVector.fromBuffer(embedding).values,
dtype: DType.float32,
),
)
.toList(growable: false);
// Calculate distances and find the median // Calculate distances and find the median
final List<double> distances = []; final List<double> distances = [];
for (final otherEmbedding in sampledOtherEmbeddings) { for (final otherEmbedding in sampledOtherEmbeddings) {
for (final embedding in sampledEmbeddings) { for (final embedding in sampledEmbeddings) {
distances.add(cosineDistForNormVectors(embedding, otherEmbedding)); distances.add(cosineDistanceSIMD(embedding, otherEmbedding));
} }
} }
distances.sort(); distances.sort();
@ -616,7 +611,7 @@ class ClusterFeedbackService {
} }
} }
} }
watch.log("Finished median test"); w?.log("Finished median test");
if (suggestionsMedian.isEmpty) { if (suggestionsMedian.isEmpty) {
_logger.info("No suggestions found using median"); _logger.info("No suggestions found using median");
return []; return [];
@ -648,23 +643,29 @@ class ClusterFeedbackService {
return finalSuggestionsMedian; return finalSuggestionsMedian;
} }
Future<Map<int, List<double>>> _getUpdateClusterAvg( Future<Map<int, Vector>> _getUpdateClusterAvg(
Map<int, int> allClusterIdsToCountMap, Map<int, int> allClusterIdsToCountMap,
Set<int> ignoredClusters, { Set<int> ignoredClusters, {
int minClusterSize = 1, int minClusterSize = 1,
int maxClusterInCurrentRun = 500, int maxClusterInCurrentRun = 500,
int maxEmbeddingToRead = 10000, int maxEmbeddingToRead = 10000,
}) async { }) async {
final w = (kDebugMode ? EnteWatch('_getUpdateClusterAvg') : null)?..start();
final startTime = DateTime.now();
final faceMlDb = FaceMLDataDB.instance; final faceMlDb = FaceMLDataDB.instance;
_logger.info( _logger.info(
'start getUpdateClusterAvg for ${allClusterIdsToCountMap.length} clusters, minClusterSize $minClusterSize, maxClusterInCurrentRun $maxClusterInCurrentRun', 'start getUpdateClusterAvg for ${allClusterIdsToCountMap.length} clusters, minClusterSize $minClusterSize, maxClusterInCurrentRun $maxClusterInCurrentRun',
); );
final Map<int, (Uint8List, int)> clusterToSummary = final Map<int, (Uint8List, int)> clusterToSummary =
await faceMlDb.getAllClusterSummary(); await faceMlDb.getAllClusterSummary(minClusterSize);
final Map<int, (Uint8List, int)> updatesForClusterSummary = {}; final Map<int, (Uint8List, int)> updatesForClusterSummary = {};
final Map<int, List<double>> clusterAvg = {}; final Map<int, Vector> clusterAvg = {};
w?.log(
'getUpdateClusterAvg database call for getAllClusterSummary',
);
final allClusterIds = allClusterIdsToCountMap.keys.toSet(); final allClusterIds = allClusterIdsToCountMap.keys.toSet();
int ignoredClustersCnt = 0, alreadyUpdatedClustersCnt = 0; int ignoredClustersCnt = 0, alreadyUpdatedClustersCnt = 0;
@ -676,7 +677,10 @@ class ClusterFeedbackService {
} }
if (clusterToSummary[id]?.$2 == allClusterIdsToCountMap[id]) { if (clusterToSummary[id]?.$2 == allClusterIdsToCountMap[id]) {
allClusterIds.remove(id); allClusterIds.remove(id);
clusterAvg[id] = EVector.fromBuffer(clusterToSummary[id]!.$1).values; clusterAvg[id] = Vector.fromList(
EVector.fromBuffer(clusterToSummary[id]!.$1).values,
dtype: DType.float32,
);
alreadyUpdatedClustersCnt++; alreadyUpdatedClustersCnt++;
} }
if (allClusterIdsToCountMap[id]! < minClusterSize) { if (allClusterIdsToCountMap[id]! < minClusterSize) {
@ -684,9 +688,20 @@ class ClusterFeedbackService {
smallerClustersCnt++; smallerClustersCnt++;
} }
} }
w?.log(
'serialization of embeddings',
);
_logger.info( _logger.info(
'Ignored $ignoredClustersCnt clusters, already updated $alreadyUpdatedClustersCnt clusters, $smallerClustersCnt clusters are smaller than $minClusterSize', 'Ignored $ignoredClustersCnt clusters, already updated $alreadyUpdatedClustersCnt clusters, $smallerClustersCnt clusters are smaller than $minClusterSize',
); );
if (allClusterIds.isEmpty) {
_logger.info(
'No clusters to update, getUpdateClusterAvg done in ${DateTime.now().difference(startTime).inMilliseconds} ms',
);
return clusterAvg;
}
// get clusterIDs sorted by count in descending order // get clusterIDs sorted by count in descending order
final sortedClusterIDs = allClusterIds.toList(); final sortedClusterIDs = allClusterIds.toList();
sortedClusterIDs.sort( sortedClusterIDs.sort(
@ -694,12 +709,7 @@ class ClusterFeedbackService {
allClusterIdsToCountMap[b]!.compareTo(allClusterIdsToCountMap[a]!), allClusterIdsToCountMap[b]!.compareTo(allClusterIdsToCountMap[a]!),
); );
int indexedInCurrentRun = 0; int indexedInCurrentRun = 0;
final EnteWatch? w = kDebugMode ? EnteWatch("computeAvg") : null; w?.reset();
w?.start();
w?.log(
'reading embeddings for $maxClusterInCurrentRun or ${sortedClusterIDs.length} clusters',
);
int currentPendingRead = 0; int currentPendingRead = 0;
final List<int> clusterIdsToRead = []; final List<int> clusterIdsToRead = [];
@ -730,19 +740,17 @@ class ClusterFeedbackService {
); );
for (final clusterID in clusterEmbeddings.keys) { for (final clusterID in clusterEmbeddings.keys) {
late List<double> avg; final Iterable<Uint8List> embeddings = clusterEmbeddings[clusterID]!;
final Iterable<Uint8List> embedings = clusterEmbeddings[clusterID]!; final Iterable<Vector> vectors = embeddings.map(
final List<double> sum = List.filled(192, 0); (e) => Vector.fromList(
for (final embedding in embedings) { EVector.fromBuffer(e).values,
final data = EVector.fromBuffer(embedding).values; dtype: DType.float32,
for (int i = 0; i < sum.length; i++) { ),
sum[i] += data[i]; );
} final avg = vectors.reduce((a, b) => a + b) / vectors.length;
} final avgEmbeddingBuffer = EVector(values: avg).writeToBuffer();
avg = sum.map((e) => e / embedings.length).toList();
final avgEmbeedingBuffer = EVector(values: avg).writeToBuffer();
updatesForClusterSummary[clusterID] = updatesForClusterSummary[clusterID] =
(avgEmbeedingBuffer, embedings.length); (avgEmbeddingBuffer, embeddings.length);
// store the intermediate updates // store the intermediate updates
indexedInCurrentRun++; indexedInCurrentRun++;
if (updatesForClusterSummary.length > 100) { if (updatesForClusterSummary.length > 100) {
@ -760,26 +768,31 @@ class ClusterFeedbackService {
await faceMlDb.clusterSummaryUpdate(updatesForClusterSummary); await faceMlDb.clusterSummaryUpdate(updatesForClusterSummary);
} }
w?.logAndReset('done computing avg '); w?.logAndReset('done computing avg ');
_logger.info('end getUpdateClusterAvg for ${clusterAvg.length} clusters'); _logger.info(
'end getUpdateClusterAvg for ${clusterAvg.length} clusters, done in ${DateTime.now().difference(startTime).inMilliseconds} ms',
);
return clusterAvg; return clusterAvg;
} }
/// Returns a map of person's clusterID to map of closest clusterID to with disstance /// Returns a map of person's clusterID to map of closest clusterID to with disstance
Map<int, List<(int, double)>> _calcSuggestionsMean( List<(int, double)> _calcSuggestionsMean(
Map<int, List<double>> clusterAvg, Map<int, Vector> clusterAvg,
Set<int> personClusters, Set<int> personClusters,
Set<int> ignoredClusters, Set<int> ignoredClusters,
double maxClusterDistance, double maxClusterDistance, {
) { Map<int, int>? allClusterIdsToCountMap,
}) {
final Map<int, List<(int, double)>> suggestions = {}; final Map<int, List<(int, double)>> suggestions = {};
int suggestionCount = 0;
final w = (kDebugMode ? EnteWatch('getSuggestions') : null)?..start();
for (final otherClusterID in clusterAvg.keys) { for (final otherClusterID in clusterAvg.keys) {
// ignore the cluster that belong to the person or is ignored // ignore the cluster that belong to the person or is ignored
if (personClusters.contains(otherClusterID) || if (personClusters.contains(otherClusterID) ||
ignoredClusters.contains(otherClusterID)) { ignoredClusters.contains(otherClusterID)) {
continue; continue;
} }
final otherAvg = clusterAvg[otherClusterID]!; final Vector otherAvg = clusterAvg[otherClusterID]!;
int? nearestPersonCluster; int? nearestPersonCluster;
double? minDistance; double? minDistance;
for (final personCluster in personClusters) { for (final personCluster in personClusters) {
@ -787,8 +800,8 @@ class ClusterFeedbackService {
_logger.info('no avg for cluster $personCluster'); _logger.info('no avg for cluster $personCluster');
continue; continue;
} }
final avg = clusterAvg[personCluster]!; final Vector avg = clusterAvg[personCluster]!;
final distance = cosineDistForNormVectors(avg, otherAvg); final distance = cosineDistanceSIMD(avg, otherAvg);
if (distance < maxClusterDistance) { if (distance < maxClusterDistance) {
if (minDistance == null || distance < minDistance) { if (minDistance == null || distance < minDistance) {
minDistance = distance; minDistance = distance;
@ -800,13 +813,39 @@ class ClusterFeedbackService {
suggestions suggestions
.putIfAbsent(nearestPersonCluster, () => []) .putIfAbsent(nearestPersonCluster, () => [])
.add((otherClusterID, minDistance)); .add((otherClusterID, minDistance));
suggestionCount++;
}
if (suggestionCount >= 2000) {
break;
} }
} }
for (final entry in suggestions.entries) { w?.log('calculation inside calcSuggestionsMean');
entry.value.sort((a, b) => a.$1.compareTo(b.$1));
}
return suggestions; if (suggestions.isNotEmpty) {
final List<(int, double)> suggestClusterIds = [];
for (final List<(int, double)> suggestion in suggestions.values) {
suggestClusterIds.addAll(suggestion);
}
suggestClusterIds.sort(
(a, b) => a.$2.compareTo(b.$2),
); // sort by distance
// List<int>? suggestClusterIdsSizes;
// if (allClusterIdsToCountMap != null) {
// suggestClusterIdsSizes = suggestClusterIds
// .map((e) => allClusterIdsToCountMap[e.$1]!)
// .toList(growable: false);
// }
// final suggestClusterIdsDistances =
// suggestClusterIds.map((e) => e.$2).toList(growable: false);
_logger.info(
"Already found ${suggestClusterIds.length} good suggestions using mean",
);
return suggestClusterIds.sublist(0, min(suggestClusterIds.length, 20));
} else {
_logger.info("No suggestions found using mean");
return <(int, double)>[];
}
} }
List<T> _randomSampleWithoutReplacement<T>( List<T> _randomSampleWithoutReplacement<T>(
@ -841,56 +880,88 @@ class ClusterFeedbackService {
Future<void> _sortSuggestionsOnDistanceToPerson( Future<void> _sortSuggestionsOnDistanceToPerson(
PersonEntity person, PersonEntity person,
List<ClusterSuggestion> suggestions, List<ClusterSuggestion> suggestions, {
) async { bool onlySortBigSuggestions = true,
}) async {
if (suggestions.isEmpty) { if (suggestions.isEmpty) {
debugPrint('No suggestions to sort'); debugPrint('No suggestions to sort');
return; return;
} }
if (onlySortBigSuggestions) {
final bigSuggestions = suggestions
.where(
(s) => s.filesInCluster.length > kMinimumClusterSizeSearchResult,
)
.toList();
if (bigSuggestions.isEmpty) {
debugPrint('No big suggestions to sort');
return;
}
}
final startTime = DateTime.now(); final startTime = DateTime.now();
final faceMlDb = FaceMLDataDB.instance; final faceMlDb = FaceMLDataDB.instance;
// Get the cluster averages for the person's clusters and the suggestions' clusters // Get the cluster averages for the person's clusters and the suggestions' clusters
final Map<int, (Uint8List, int)> clusterToSummary = final personClusters = await faceMlDb.getPersonClusterIDs(person.remoteID);
await faceMlDb.getAllClusterSummary(); final Map<int, (Uint8List, int)> personClusterToSummary =
await faceMlDb.getClusterToClusterSummary(personClusters);
final clusterSummaryCallTime = DateTime.now();
// Calculate the avg embedding of the person // Calculate the avg embedding of the person
final personClusters = await faceMlDb.getPersonClusterIDs(person.remoteID); final w = (kDebugMode ? EnteWatch('sortSuggestions') : null)?..start();
final personEmbeddingsCount = personClusters final personEmbeddingsCount = personClusters
.map((e) => clusterToSummary[e]!.$2) .map((e) => personClusterToSummary[e]!.$2)
.reduce((a, b) => a + b); .reduce((a, b) => a + b);
final List<double> personAvg = List.filled(192, 0); Vector personAvg = Vector.filled(192, 0);
for (final personClusterID in personClusters) { for (final personClusterID in personClusters) {
final personClusterBlob = clusterToSummary[personClusterID]!.$1; final personClusterBlob = personClusterToSummary[personClusterID]!.$1;
final personClusterAvg = EVector.fromBuffer(personClusterBlob).values; final personClusterAvg = Vector.fromList(
EVector.fromBuffer(personClusterBlob).values,
dtype: DType.float32,
);
final clusterWeight = final clusterWeight =
clusterToSummary[personClusterID]!.$2 / personEmbeddingsCount; personClusterToSummary[personClusterID]!.$2 / personEmbeddingsCount;
for (int i = 0; i < personClusterAvg.length; i++) { personAvg += personClusterAvg * clusterWeight;
personAvg[i] += personClusterAvg[i] *
clusterWeight; // Weighted sum of the cluster averages
}
} }
w?.log('calculated person avg');
// Sort the suggestions based on the distance to the person // Sort the suggestions based on the distance to the person
for (final suggestion in suggestions) { for (final suggestion in suggestions) {
if (onlySortBigSuggestions) {
if (suggestion.filesInCluster.length <= 8) {
continue;
}
}
final clusterID = suggestion.clusterIDToMerge; final clusterID = suggestion.clusterIDToMerge;
final faceIdToEmbeddingMap = await faceMlDb.getFaceEmbeddingMapForFile( final faceIDs = suggestion.faceIDsInCluster;
suggestion.filesInCluster.map((e) => e.uploadedFileID!).toList(), final faceIdToEmbeddingMap = await faceMlDb.getFaceEmbeddingMapForFaces(
faceIDs,
);
final faceIdToVectorMap = faceIdToEmbeddingMap.map(
(key, value) => MapEntry(
key,
Vector.fromList(
EVector.fromBuffer(value).values,
dtype: DType.float32,
),
),
);
w?.log(
'got ${faceIdToEmbeddingMap.values.length} embeddings for ${suggestion.filesInCluster.length} files for cluster $clusterID',
); );
final fileIdToDistanceMap = {}; final fileIdToDistanceMap = {};
for (final entry in faceIdToEmbeddingMap.entries) { for (final entry in faceIdToVectorMap.entries) {
fileIdToDistanceMap[getFileIdFromFaceId(entry.key)] = fileIdToDistanceMap[getFileIdFromFaceId(entry.key)] =
cosineDistForNormVectors( cosineDistanceSIMD(personAvg, entry.value);
personAvg,
EVector.fromBuffer(entry.value).values,
);
} }
w?.log('calculated distances for cluster $clusterID');
suggestion.filesInCluster.sort((b, a) { suggestion.filesInCluster.sort((b, a) {
//todo: review with @laurens, added this to avoid null safety issue //todo: review with @laurens, added this to avoid null safety issue
final double distanceA = fileIdToDistanceMap[a.uploadedFileID!] ?? -1; final double distanceA = fileIdToDistanceMap[a.uploadedFileID!] ?? -1;
final double distanceB = fileIdToDistanceMap[b.uploadedFileID!] ?? -1; final double distanceB = fileIdToDistanceMap[b.uploadedFileID!] ?? -1;
return distanceA.compareTo(distanceB); return distanceA.compareTo(distanceB);
}); });
w?.log('sorted files for cluster $clusterID');
debugPrint( debugPrint(
"[${_logger.name}] Sorted suggestions for cluster $clusterID based on distance to person: ${suggestion.filesInCluster.map((e) => fileIdToDistanceMap[e.uploadedFileID]).toList()}", "[${_logger.name}] Sorted suggestions for cluster $clusterID based on distance to person: ${suggestion.filesInCluster.map((e) => fileIdToDistanceMap[e.uploadedFileID]).toList()}",
@ -899,7 +970,7 @@ class ClusterFeedbackService {
final endTime = DateTime.now(); final endTime = DateTime.now();
_logger.info( _logger.info(
"Sorting suggestions based on distance to person took ${endTime.difference(startTime).inMilliseconds} ms for ${suggestions.length} suggestions", "Sorting suggestions based on distance to person took ${endTime.difference(startTime).inMilliseconds} ms for ${suggestions.length} suggestions, of which ${clusterSummaryCallTime.difference(startTime).inMilliseconds} ms was spent on the cluster summary call",
); );
} }
} }

13
mobile/lib/services/machine_learning/face_ml/person/person_service.dart
View File

@ -1,3 +1,4 @@
import "dart:async" show unawaited;
import "dart:convert"; import "dart:convert";
import "package:flutter/foundation.dart"; import "package:flutter/foundation.dart";
@ -102,10 +103,12 @@ class PersonService {
faces: faceIds.toSet(), faces: faceIds.toSet(),
); );
personData.assigned!.add(clusterInfo); personData.assigned!.add(clusterInfo);
await entityService.addOrUpdate( unawaited(
EntityType.person, entityService.addOrUpdate(
json.encode(personData.toJson()), EntityType.person,
id: personID, json.encode(personData.toJson()),
id: personID,
),
); );
await faceMLDataDB.assignClusterToPerson( await faceMLDataDB.assignClusterToPerson(
personID: personID, personID: personID,
@ -190,7 +193,7 @@ class PersonService {
} }
logger.info("Storing feedback for ${faceIdToClusterID.length} faces"); logger.info("Storing feedback for ${faceIdToClusterID.length} faces");
await faceMLDataDB.updateClusterIdToFaceId(faceIdToClusterID); await faceMLDataDB.updateFaceIdToClusterId(faceIdToClusterID);
await faceMLDataDB.bulkAssignClusterToPersonID(clusterToPersonID); await faceMLDataDB.bulkAssignClusterToPersonID(clusterToPersonID);
} }

3
mobile/lib/services/search_service.dart
View File

@ -28,6 +28,7 @@ import "package:photos/models/search/search_constants.dart";
import "package:photos/models/search/search_types.dart"; import "package:photos/models/search/search_types.dart";
import 'package:photos/services/collections_service.dart'; import 'package:photos/services/collections_service.dart';
import "package:photos/services/location_service.dart"; import "package:photos/services/location_service.dart";
import "package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import 'package:photos/services/machine_learning/semantic_search/semantic_search_service.dart'; import 'package:photos/services/machine_learning/semantic_search/semantic_search_service.dart';
import "package:photos/states/location_screen_state.dart"; import "package:photos/states/location_screen_state.dart";
@ -824,7 +825,7 @@ class SearchService {
"Cluster $clusterId should not have person id ${clusterIDToPersonID[clusterId]}", "Cluster $clusterId should not have person id ${clusterIDToPersonID[clusterId]}",
); );
} }
if (files.length < 20 && sortedClusterIds.length > 3) { if (files.length < kMinimumClusterSizeSearchResult && sortedClusterIds.length > 3) {
continue; continue;
} }
facesResult.add( facesResult.add(

68
mobile/lib/ui/settings/debug/face_debug_section_widget.dart
View File

@ -8,7 +8,6 @@ import "package:photos/events/people_changed_event.dart";
import "package:photos/face/db.dart"; import "package:photos/face/db.dart";
import "package:photos/face/model/person.dart"; import "package:photos/face/model/person.dart";
import 'package:photos/services/machine_learning/face_ml/face_ml_service.dart'; import 'package:photos/services/machine_learning/face_ml/face_ml_service.dart';
import "package:photos/services/machine_learning/face_ml/feedback/cluster_feedback.dart";
import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import 'package:photos/theme/ente_theme.dart'; import 'package:photos/theme/ente_theme.dart';
import 'package:photos/ui/components/captioned_text_widget.dart'; import 'package:photos/ui/components/captioned_text_widget.dart';
@ -217,9 +216,14 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
trailingIcon: Icons.chevron_right_outlined, trailingIcon: Icons.chevron_right_outlined,
trailingIconIsMuted: true, trailingIconIsMuted: true,
onTap: () async { onTap: () async {
await FaceMLDataDB.instance.dropFeedbackTables(); try {
Bus.instance.fire(PeopleChangedEvent()); await FaceMLDataDB.instance.dropFeedbackTables();
showShortToast(context, "Done"); Bus.instance.fire(PeopleChangedEvent());
showShortToast(context, "Done");
} catch (e, s) {
_logger.warning('reset feedback failed ', e, s);
await showGenericErrorDialog(context: context, error: e);
}
}, },
), ),
sectionOptionSpacing, sectionOptionSpacing,
@ -284,34 +288,34 @@ class _FaceDebugSectionWidgetState extends State<FaceDebugSectionWidget> {
); );
}, },
), ),
sectionOptionSpacing, // sectionOptionSpacing,
MenuItemWidget( // MenuItemWidget(
captionedTextWidget: const CaptionedTextWidget( // captionedTextWidget: const CaptionedTextWidget(
title: "Rank blurs", // title: "Rank blurs",
), // ),
pressedColor: getEnteColorScheme(context).fillFaint, // pressedColor: getEnteColorScheme(context).fillFaint,
trailingIcon: Icons.chevron_right_outlined, // trailingIcon: Icons.chevron_right_outlined,
trailingIconIsMuted: true, // trailingIconIsMuted: true,
onTap: () async { // onTap: () async {
await showChoiceDialog( // await showChoiceDialog(
context, // context,
title: "Are you sure?", // title: "Are you sure?",
body: // body:
"This will delete all clusters and put blurry faces in separate clusters per ten points.", // "This will delete all clusters and put blurry faces in separate clusters per ten points.",
firstButtonLabel: "Yes, confirm", // firstButtonLabel: "Yes, confirm",
firstButtonOnTap: () async { // firstButtonOnTap: () async {
try { // try {
await ClusterFeedbackService.instance // await ClusterFeedbackService.instance
.createFakeClustersByBlurValue(); // .createFakeClustersByBlurValue();
showShortToast(context, "Done"); // showShortToast(context, "Done");
} catch (e, s) { // } catch (e, s) {
_logger.warning('Failed to rank faces on blur values ', e, s); // _logger.warning('Failed to rank faces on blur values ', e, s);
await showGenericErrorDialog(context: context, error: e); // await showGenericErrorDialog(context: context, error: e);
} // }
}, // },
); // );
}, // },
), // ),
sectionOptionSpacing, sectionOptionSpacing,
MenuItemWidget( MenuItemWidget(
captionedTextWidget: const CaptionedTextWidget( captionedTextWidget: const CaptionedTextWidget(

167
mobile/lib/ui/viewer/file_details/face_widget.dart
View File

@ -1,10 +1,10 @@
import "dart:developer" show log; import "dart:developer" show log;
import "dart:io" show Platform;
import "dart:typed_data"; import "dart:typed_data";
import "package:flutter/cupertino.dart"; import "package:flutter/cupertino.dart";
import "package:flutter/foundation.dart" show kDebugMode; import "package:flutter/foundation.dart" show kDebugMode;
import "package:flutter/material.dart"; import "package:flutter/material.dart";
import "package:photos/extensions/stop_watch.dart";
import "package:photos/face/db.dart"; import "package:photos/face/db.dart";
import "package:photos/face/model/face.dart"; import "package:photos/face/model/face.dart";
import "package:photos/face/model/person.dart"; import "package:photos/face/model/person.dart";
@ -21,6 +21,8 @@ import "package:photos/utils/face/face_box_crop.dart";
import "package:photos/utils/thumbnail_util.dart"; import "package:photos/utils/thumbnail_util.dart";
// import "package:photos/utils/toast_util.dart"; // import "package:photos/utils/toast_util.dart";
const useGeneratedFaceCrops = true;
class FaceWidget extends StatefulWidget { class FaceWidget extends StatefulWidget {
final EnteFile file; final EnteFile file;
final Face face; final Face face;
@ -48,12 +50,13 @@ class _FaceWidgetState extends State<FaceWidget> {
@override @override
Widget build(BuildContext context) { Widget build(BuildContext context) {
if (Platform.isIOS) { if (useGeneratedFaceCrops) {
return FutureBuilder<Uint8List?>( return FutureBuilder<Uint8List?>(
future: getFaceCrop(), future: getFaceCrop(),
builder: (context, snapshot) { builder: (context, snapshot) {
if (snapshot.hasData) { if (snapshot.hasData) {
final ImageProvider imageProvider = MemoryImage(snapshot.data!); final ImageProvider imageProvider = MemoryImage(snapshot.data!);
return GestureDetector( return GestureDetector(
onTap: () async { onTap: () async {
if (widget.editMode) return; if (widget.editMode) return;
@ -63,7 +66,50 @@ class _FaceWidgetState extends State<FaceWidget> {
name: "FaceWidget", name: "FaceWidget",
); );
if (widget.person == null && widget.clusterID == null) { if (widget.person == null && widget.clusterID == null) {
return; // Get faceID and double check that it doesn't belong to an existing clusterID. If it does, push that cluster page
final w = (kDebugMode ? EnteWatch('FaceWidget') : null)
?..start();
final existingClusterID = await FaceMLDataDB.instance
.getClusterIDForFaceID(widget.face.faceID);
w?.log('getting existing clusterID for faceID');
if (existingClusterID != null) {
final fileIdsToClusterIds =
await FaceMLDataDB.instance.getFileIdToClusterIds();
final files = await SearchService.instance.getAllFiles();
final clusterFiles = files
.where(
(file) =>
fileIdsToClusterIds[file.uploadedFileID]
?.contains(existingClusterID) ??
false,
)
.toList();
await Navigator.of(context).push(
MaterialPageRoute(
builder: (context) => ClusterPage(
clusterFiles,
clusterID: existingClusterID,
),
),
);
}
// Create new clusterID for the faceID and update DB to assign the faceID to the new clusterID
final int newClusterID =
DateTime.now().microsecondsSinceEpoch;
await FaceMLDataDB.instance.updateFaceIdToClusterId(
{widget.face.faceID: newClusterID},
);
// Push page for the new cluster
await Navigator.of(context).push(
MaterialPageRoute(
builder: (context) => ClusterPage(
[widget.file],
clusterID: newClusterID,
),
),
);
} }
if (widget.person != null) { if (widget.person != null) {
await Navigator.of(context).push( await Navigator.of(context).push(
@ -228,7 +274,49 @@ class _FaceWidgetState extends State<FaceWidget> {
name: "FaceWidget", name: "FaceWidget",
); );
if (widget.person == null && widget.clusterID == null) { if (widget.person == null && widget.clusterID == null) {
return; // Get faceID and double check that it doesn't belong to an existing clusterID. If it does, push that cluster page
final w = (kDebugMode ? EnteWatch('FaceWidget') : null)
?..start();
final existingClusterID = await FaceMLDataDB.instance
.getClusterIDForFaceID(widget.face.faceID);
w?.log('getting existing clusterID for faceID');
if (existingClusterID != null) {
final fileIdsToClusterIds =
await FaceMLDataDB.instance.getFileIdToClusterIds();
final files = await SearchService.instance.getAllFiles();
final clusterFiles = files
.where(
(file) =>
fileIdsToClusterIds[file.uploadedFileID]
?.contains(existingClusterID) ??
false,
)
.toList();
await Navigator.of(context).push(
MaterialPageRoute(
builder: (context) => ClusterPage(
clusterFiles,
clusterID: existingClusterID,
),
),
);
}
// Create new clusterID for the faceID and update DB to assign the faceID to the new clusterID
final int newClusterID = DateTime.now().microsecondsSinceEpoch;
await FaceMLDataDB.instance.updateFaceIdToClusterId(
{widget.face.faceID: newClusterID},
);
// Push page for the new cluster
await Navigator.of(context).push(
MaterialPageRoute(
builder: (context) => ClusterPage(
[widget.file],
clusterID: newClusterID,
),
),
);
} }
if (widget.person != null) { if (widget.person != null) {
await Navigator.of(context).push( await Navigator.of(context).push(
@ -262,33 +350,56 @@ class _FaceWidgetState extends State<FaceWidget> {
}, },
child: Column( child: Column(
children: [ children: [
Container( Stack(
height: 60, children: [
width: 60, Container(
decoration: ShapeDecoration(
shape: RoundedRectangleBorder(
borderRadius:
const BorderRadius.all(Radius.elliptical(16, 12)),
side: widget.highlight
? BorderSide(
color: getEnteColorScheme(context).primary700,
width: 2.0,
)
: BorderSide.none,
),
),
child: ClipRRect(
borderRadius:
const BorderRadius.all(Radius.elliptical(16, 12)),
child: SizedBox(
width: 60,
height: 60, height: 60,
child: CroppedFaceImageView( width: 60,
enteFile: widget.file, decoration: ShapeDecoration(
face: widget.face, shape: RoundedRectangleBorder(
borderRadius: const BorderRadius.all(
Radius.elliptical(16, 12),
),
side: widget.highlight
? BorderSide(
color: getEnteColorScheme(context).primary700,
width: 1.0,
)
: BorderSide.none,
),
),
child: ClipRRect(
borderRadius:
const BorderRadius.all(Radius.elliptical(16, 12)),
child: SizedBox(
width: 60,
height: 60,
child: CroppedFaceImageView(
enteFile: widget.file,
face: widget.face,
),
),
), ),
), ),
), // TODO: the edges of the green line are still not properly rounded around ClipRRect
if (widget.editMode)
Positioned(
right: 0,
top: 0,
child: GestureDetector(
onTap: _cornerIconPressed,
child: isJustRemoved
? const Icon(
CupertinoIcons.add_circled_solid,
color: Colors.green,
)
: const Icon(
Icons.cancel,
color: Colors.red,
),
),
),
],
), ),
const SizedBox(height: 8), const SizedBox(height: 8),
if (widget.person != null) if (widget.person != null)

30
mobile/lib/ui/viewer/file_details/faces_item_widget.dart
View File

@ -71,9 +71,9 @@ class _FacesItemWidgetState extends State<FacesItemWidget> {
]; ];
} }
// Remove faces with low scores and blurry faces // Remove faces with low scores
if (!kDebugMode) { if (!kDebugMode) {
faces.removeWhere((face) => (face.isBlurry || face.score < 0.75)); faces.removeWhere((face) => (face.score < 0.75));
} }
if (faces.isEmpty) { if (faces.isEmpty) {
@ -85,9 +85,6 @@ class _FacesItemWidgetState extends State<FacesItemWidget> {
]; ];
} }
// Sort the faces by score in descending order, so that the highest scoring face is first.
faces.sort((Face a, Face b) => b.score.compareTo(a.score));
// TODO: add deduplication of faces of same person // TODO: add deduplication of faces of same person
final faceIdsToClusterIds = await FaceMLDataDB.instance final faceIdsToClusterIds = await FaceMLDataDB.instance
.getFaceIdsToClusterIds(faces.map((face) => face.faceID)); .getFaceIdsToClusterIds(faces.map((face) => face.faceID));
@ -96,6 +93,29 @@ class _FacesItemWidgetState extends State<FacesItemWidget> {
final clusterIDToPerson = final clusterIDToPerson =
await FaceMLDataDB.instance.getClusterIDToPersonID(); await FaceMLDataDB.instance.getClusterIDToPersonID();
// Sort faces by name and score
final faceIdToPersonID = <String, String>{};
for (final face in faces) {
final clusterID = faceIdsToClusterIds[face.faceID];
if (clusterID != null) {
final personID = clusterIDToPerson[clusterID];
if (personID != null) {
faceIdToPersonID[face.faceID] = personID;
}
}
}
faces.sort((Face a, Face b) {
final aPersonID = faceIdToPersonID[a.faceID];
final bPersonID = faceIdToPersonID[b.faceID];
if (aPersonID != null && bPersonID == null) {
return -1;
} else if (aPersonID == null && bPersonID != null) {
return 1;
} else {
return b.score.compareTo(a.score);
}
});
final lastViewedClusterID = ClusterFeedbackService.lastViewedClusterID; final lastViewedClusterID = ClusterFeedbackService.lastViewedClusterID;
final faceWidgets = <FaceWidget>[]; final faceWidgets = <FaceWidget>[];

16
mobile/lib/ui/viewer/people/cluster_app_bar.dart
View File

@ -207,14 +207,14 @@ class _AppBarWidgetState extends State<ClusterAppBar> {
if (embedding.key == otherEmbedding.key) { if (embedding.key == otherEmbedding.key) {
continue; continue;
} }
final distance64 = 1.0 - final distance64 = cosineDistanceSIMD(
Vector.fromList(embedding.value, dtype: DType.float64).dot( Vector.fromList(embedding.value, dtype: DType.float64),
Vector.fromList(otherEmbedding.value, dtype: DType.float64), Vector.fromList(otherEmbedding.value, dtype: DType.float64),
); );
final distance32 = 1.0 - final distance32 = cosineDistanceSIMD(
Vector.fromList(embedding.value, dtype: DType.float32).dot( Vector.fromList(embedding.value, dtype: DType.float32),
Vector.fromList(otherEmbedding.value, dtype: DType.float32), Vector.fromList(otherEmbedding.value, dtype: DType.float32),
); );
final distance = cosineDistForNormVectors( final distance = cosineDistForNormVectors(
embedding.value, embedding.value,
otherEmbedding.value, otherEmbedding.value,

26
mobile/lib/ui/viewer/people/cropped_face_image_view.dart
View File

@ -4,8 +4,10 @@ import "dart:io" show File;
import 'package:flutter/material.dart'; import 'package:flutter/material.dart';
import "package:photos/face/model/face.dart"; import "package:photos/face/model/face.dart";
import "package:photos/models/file/file.dart"; import "package:photos/models/file/file.dart";
import "package:photos/models/file/file_type.dart";
import "package:photos/ui/viewer/file/thumbnail_widget.dart"; import "package:photos/ui/viewer/file/thumbnail_widget.dart";
import "package:photos/utils/file_util.dart"; import "package:photos/utils/file_util.dart";
import "package:photos/utils/thumbnail_util.dart";
class CroppedFaceInfo { class CroppedFaceInfo {
final Image image; final Image image;
@ -38,7 +40,8 @@ class CroppedFaceImageView extends StatelessWidget {
builder: (context, snapshot) { builder: (context, snapshot) {
if (snapshot.hasData) { if (snapshot.hasData) {
return LayoutBuilder( return LayoutBuilder(
builder: (BuildContext context, BoxConstraints constraints) { builder: ((context, constraints) {
final double imageAspectRatio = enteFile.width / enteFile.height;
final Image image = snapshot.data!; final Image image = snapshot.data!;
final double viewWidth = constraints.maxWidth; final double viewWidth = constraints.maxWidth;
@ -51,14 +54,13 @@ class CroppedFaceImageView extends StatelessWidget {
final double relativeFaceCenterY = final double relativeFaceCenterY =
faceBox.yMin + faceBox.height / 2; faceBox.yMin + faceBox.height / 2;
const double desiredFaceHeightRelativeToWidget = 1 / 2; const double desiredFaceHeightRelativeToWidget = 8 / 10;
final double scale = final double scale =
(1 / faceBox.height) * desiredFaceHeightRelativeToWidget; (1 / faceBox.height) * desiredFaceHeightRelativeToWidget;
final double widgetCenterX = viewWidth / 2; final double widgetCenterX = viewWidth / 2;
final double widgetCenterY = viewHeight / 2; final double widgetCenterY = viewHeight / 2;
final double imageAspectRatio = enteFile.width / enteFile.height;
final double widgetAspectRatio = viewWidth / viewHeight; final double widgetAspectRatio = viewWidth / viewHeight;
final double imageToWidgetRatio = final double imageToWidgetRatio =
imageAspectRatio / widgetAspectRatio; imageAspectRatio / widgetAspectRatio;
@ -68,16 +70,15 @@ class CroppedFaceImageView extends StatelessWidget {
double offsetY = double offsetY =
(widgetCenterY - relativeFaceCenterY * viewHeight) * scale; (widgetCenterY - relativeFaceCenterY * viewHeight) * scale;
if (imageAspectRatio > widgetAspectRatio) { if (imageAspectRatio < widgetAspectRatio) {
// Landscape Image: Adjust offsetX more conservatively // Landscape Image: Adjust offsetX more conservatively
offsetX = offsetX * imageToWidgetRatio; offsetX = offsetX * imageToWidgetRatio;
} else { } else {
// Portrait Image: Adjust offsetY more conservatively // Portrait Image: Adjust offsetY more conservatively
offsetY = offsetY / imageToWidgetRatio; offsetY = offsetY / imageToWidgetRatio;
} }
return ClipRRect(
return ClipRect( borderRadius: const BorderRadius.all(Radius.elliptical(16, 12)),
clipBehavior: Clip.antiAlias,
child: Transform.translate( child: Transform.translate(
offset: Offset( offset: Offset(
offsetX, offsetX,
@ -89,7 +90,7 @@ class CroppedFaceImageView extends StatelessWidget {
), ),
), ),
); );
}, }),
); );
} else { } else {
if (snapshot.hasError) { if (snapshot.hasError) {
@ -104,13 +105,18 @@ class CroppedFaceImageView extends StatelessWidget {
} }
Future<Image?> getImage() async { Future<Image?> getImage() async {
final File? ioFile = await getFile(enteFile); final File? ioFile;
if (enteFile.fileType == FileType.video) {
ioFile = await getThumbnailForUploadedFile(enteFile);
} else {
ioFile = await getFile(enteFile);
}
if (ioFile == null) { if (ioFile == null) {
return null; return null;
} }
final imageData = await ioFile.readAsBytes(); final imageData = await ioFile.readAsBytes();
final image = Image.memory(imageData, fit: BoxFit.cover); final image = Image.memory(imageData, fit: BoxFit.contain);
return image; return image;
} }

82
mobile/lib/ui/viewer/people/person_cluster_suggestion.dart
View File

@ -1,3 +1,4 @@
import "dart:async" show StreamSubscription, unawaited;
import "dart:math"; import "dart:math";
import "package:flutter/foundation.dart" show kDebugMode; import "package:flutter/foundation.dart" show kDebugMode;
@ -29,16 +30,25 @@ class PersonReviewClusterSuggestion extends StatefulWidget {
class _PersonClustersState extends State<PersonReviewClusterSuggestion> { class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
int currentSuggestionIndex = 0; int currentSuggestionIndex = 0;
bool fetch = true;
Key futureBuilderKey = UniqueKey(); Key futureBuilderKey = UniqueKey();
// Declare a variable for the future // Declare a variable for the future
late Future<List<ClusterSuggestion>> futureClusterSuggestions; late Future<List<ClusterSuggestion>> futureClusterSuggestions;
late StreamSubscription<PeopleChangedEvent> _peopleChangedEvent;
@override @override
void initState() { void initState() {
super.initState(); super.initState();
// Initialize the future in initState // Initialize the future in initState
_fetchClusterSuggestions(); if (fetch) _fetchClusterSuggestions();
fetch = true;
}
@override
void dispose() {
_peopleChangedEvent.cancel();
super.dispose();
} }
@override @override
@ -61,12 +71,27 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
), ),
); );
} }
final numberOfDifferentSuggestions = snapshot.data!.length;
final currentSuggestion = snapshot.data![currentSuggestionIndex]; final allSuggestions = snapshot.data!;
final numberOfDifferentSuggestions = allSuggestions.length;
final currentSuggestion = allSuggestions[currentSuggestionIndex];
final int clusterID = currentSuggestion.clusterIDToMerge; final int clusterID = currentSuggestion.clusterIDToMerge;
final double distance = currentSuggestion.distancePersonToCluster; final double distance = currentSuggestion.distancePersonToCluster;
final bool usingMean = currentSuggestion.usedOnlyMeanForSuggestion; final bool usingMean = currentSuggestion.usedOnlyMeanForSuggestion;
final List<EnteFile> files = currentSuggestion.filesInCluster; final List<EnteFile> files = currentSuggestion.filesInCluster;
_peopleChangedEvent =
Bus.instance.on<PeopleChangedEvent>().listen((event) {
if (event.type == PeopleEventType.removedFilesFromCluster &&
(event.source == clusterID.toString())) {
for (var updatedFile in event.relevantFiles!) {
files.remove(updatedFile);
}
fetch = false;
setState(() {});
}
});
return InkWell( return InkWell(
onTap: () { onTap: () {
Navigator.of(context).push( Navigator.of(context).push(
@ -90,6 +115,7 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
usingMean, usingMean,
files, files,
numberOfDifferentSuggestions, numberOfDifferentSuggestions,
allSuggestions,
), ),
), ),
); );
@ -116,20 +142,25 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
clusterID: clusterID, clusterID: clusterID,
); );
Bus.instance.fire(PeopleChangedEvent()); Bus.instance.fire(PeopleChangedEvent());
// Increment the suggestion index
if (mounted) {
setState(() => currentSuggestionIndex++);
}
// Check if we need to fetch new data
if (currentSuggestionIndex >= (numberOfSuggestions)) {
setState(() {
currentSuggestionIndex = 0;
futureBuilderKey = UniqueKey(); // Reset to trigger FutureBuilder
_fetchClusterSuggestions();
});
}
} else { } else {
await FaceMLDataDB.instance.captureNotPersonFeedback( await FaceMLDataDB.instance.captureNotPersonFeedback(
personID: widget.person.remoteID, personID: widget.person.remoteID,
clusterID: clusterID, clusterID: clusterID,
); );
} // Recalculate the suggestions when a suggestion is rejected
// Increment the suggestion index
if (mounted) {
setState(() => currentSuggestionIndex++);
}
// Check if we need to fetch new data
if (currentSuggestionIndex >= (numberOfSuggestions)) {
setState(() { setState(() {
currentSuggestionIndex = 0; currentSuggestionIndex = 0;
futureBuilderKey = UniqueKey(); // Reset to trigger FutureBuilder futureBuilderKey = UniqueKey(); // Reset to trigger FutureBuilder
@ -150,9 +181,10 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
bool usingMean, bool usingMean,
List<EnteFile> files, List<EnteFile> files,
int numberOfSuggestions, int numberOfSuggestions,
List<ClusterSuggestion> allSuggestions,
) { ) {
return Column( final widgetToReturn = Column(
key: ValueKey("cluster_id-$clusterID"), key: ValueKey("cluster_id-$clusterID-files-${files.length}"),
children: <Widget>[ children: <Widget>[
if (kDebugMode) if (kDebugMode)
Text( Text(
@ -228,6 +260,28 @@ class _PersonClustersState extends State<PersonReviewClusterSuggestion> {
), ),
], ],
); );
// Precompute face thumbnails for next suggestions, in case there are
const precompute = 6;
const maxComputations = 10;
int compCount = 0;
if (allSuggestions.length > currentSuggestionIndex + 1) {
for (final suggestion in allSuggestions.sublist(
currentSuggestionIndex + 1,
min(allSuggestions.length, currentSuggestionIndex + precompute),
)) {
final files = suggestion.filesInCluster;
final clusterID = suggestion.clusterIDToMerge;
for (final file in files.sublist(0, min(files.length, 8))) {
unawaited(PersonFaceWidget.precomputeFaceCrops(file, clusterID));
compCount++;
if (compCount >= maxComputations) {
break;
}
}
}
}
return widgetToReturn;
} }
List<Widget> _buildThumbnailWidgets( List<Widget> _buildThumbnailWidgets(

60
mobile/lib/ui/viewer/search/result/person_face_widget.dart
View File

@ -1,5 +1,5 @@
import "dart:developer"; import "dart:developer";
import "dart:io"; // import "dart:io";
import "dart:typed_data"; import "dart:typed_data";
import 'package:flutter/widgets.dart'; import 'package:flutter/widgets.dart';
@ -10,6 +10,7 @@ import "package:photos/face/model/person.dart";
import 'package:photos/models/file/file.dart'; import 'package:photos/models/file/file.dart';
import "package:photos/services/machine_learning/face_ml/person/person_service.dart"; import "package:photos/services/machine_learning/face_ml/person/person_service.dart";
import 'package:photos/ui/viewer/file/thumbnail_widget.dart'; import 'package:photos/ui/viewer/file/thumbnail_widget.dart';
import "package:photos/ui/viewer/file_details/face_widget.dart";
import "package:photos/ui/viewer/people/cropped_face_image_view.dart"; import "package:photos/ui/viewer/people/cropped_face_image_view.dart";
import "package:photos/utils/face/face_box_crop.dart"; import "package:photos/utils/face/face_box_crop.dart";
import "package:photos/utils/thumbnail_util.dart"; import "package:photos/utils/thumbnail_util.dart";
@ -32,9 +33,64 @@ class PersonFaceWidget extends StatelessWidget {
), ),
super(key: key); super(key: key);
static Future<void> precomputeFaceCrops(file, clusterID) async {
try {
final Face? face = await FaceMLDataDB.instance.getCoverFaceForPerson(
recentFileID: file.uploadedFileID!,
clusterID: clusterID,
);
if (face == null) {
debugPrint(
"No cover face for cluster $clusterID and recentFile ${file.uploadedFileID}",
);
return;
}
final Uint8List? cachedFace = faceCropCache.get(face.faceID);
if (cachedFace != null) {
return;
}
final faceCropCacheFile = cachedFaceCropPath(face.faceID);
if ((await faceCropCacheFile.exists())) {
final data = await faceCropCacheFile.readAsBytes();
faceCropCache.put(face.faceID, data);
return;
}
EnteFile? fileForFaceCrop = file;
if (face.fileID != file.uploadedFileID!) {
fileForFaceCrop =
await FilesDB.instance.getAnyUploadedFile(face.fileID);
}
if (fileForFaceCrop == null) {
return;
}
final result = await pool.withResource(
() async => await getFaceCrops(
fileForFaceCrop!,
{
face.faceID: face.detection.box,
},
),
);
final Uint8List? computedCrop = result?[face.faceID];
if (computedCrop != null) {
faceCropCache.put(face.faceID, computedCrop);
faceCropCacheFile.writeAsBytes(computedCrop).ignore();
}
return;
} catch (e, s) {
log(
"Error getting cover face for cluster $clusterID",
error: e,
stackTrace: s,
);
return;
}
}
@override @override
Widget build(BuildContext context) { Widget build(BuildContext context) {
if (Platform.isIOS || Platform.isAndroid) { if (useGeneratedFaceCrops) {
return FutureBuilder<Uint8List?>( return FutureBuilder<Uint8List?>(
future: getFaceCrop(), future: getFaceCrop(),
builder: (context, snapshot) { builder: (context, snapshot) {

7
mobile/lib/utils/face/face_box_crop.dart
View File

@ -5,13 +5,13 @@ import "package:photos/core/cache/lru_map.dart";
import "package:photos/face/model/box.dart"; import "package:photos/face/model/box.dart";
import "package:photos/models/file/file.dart"; import "package:photos/models/file/file.dart";
import "package:photos/models/file/file_type.dart"; import "package:photos/models/file/file_type.dart";
import "package:photos/utils/face/face_util.dart";
import "package:photos/utils/file_util.dart"; import "package:photos/utils/file_util.dart";
import "package:photos/utils/image_ml_isolate.dart";
import "package:photos/utils/thumbnail_util.dart"; import "package:photos/utils/thumbnail_util.dart";
import "package:pool/pool.dart"; import "package:pool/pool.dart";
final LRUMap<String, Uint8List?> faceCropCache = LRUMap(1000); final LRUMap<String, Uint8List?> faceCropCache = LRUMap(1000);
final pool = Pool(5, timeout: const Duration(seconds: 15)); final pool = Pool(10, timeout: const Duration(seconds: 15));
Future<Map<String, Uint8List>?> getFaceCrops( Future<Map<String, Uint8List>?> getFaceCrops(
EnteFile file, EnteFile file,
Map<String, FaceBox> faceBoxeMap, Map<String, FaceBox> faceBoxeMap,
@ -37,7 +37,8 @@ Future<Map<String, Uint8List>?> getFaceCrops(
faceBoxes.add(e.value); faceBoxes.add(e.value);
} }
final List<Uint8List> faceCrop = final List<Uint8List> faceCrop =
await ImageMlIsolate.instance.generateFaceThumbnailsForImage( // await ImageMlIsolate.instance.generateFaceThumbnailsForImage(
await generateJpgFaceThumbnails(
imagePath, imagePath,
faceBoxes, faceBoxes,
); );

175
mobile/lib/utils/face/face_util.dart Normal file
View File

@ -0,0 +1,175 @@
import "dart:math";
import "dart:typed_data";
import "package:computer/computer.dart";
import "package:flutter_image_compress/flutter_image_compress.dart";
import "package:image/image.dart" as img;
import "package:logging/logging.dart";
import "package:photos/face/model/box.dart";
/// Bounding box of a face.
///
/// [xMin] and [yMin] are the coordinates of the top left corner of the box, and
/// [width] and [height] are the width and height of the box.
///
/// One unit is equal to one pixel in the original image.
class FaceBoxImage {
final int xMin;
final int yMin;
final int width;
final int height;
FaceBoxImage({
required this.xMin,
required this.yMin,
required this.width,
required this.height,
});
}
final _logger = Logger("FaceUtil");
final _computer = Computer.shared();
const _faceImageBufferFactor = 0.2;
///Convert img.Image to ui.Image and use RawImage to display.
Future<List<img.Image>> generateImgFaceThumbnails(
String imagePath,
List<FaceBox> faceBoxes,
) async {
final faceThumbnails = <img.Image>[];
final image = await decodeToImgImage(imagePath);
for (FaceBox faceBox in faceBoxes) {
final croppedImage = cropFaceBoxFromImage(image, faceBox);
faceThumbnails.add(croppedImage);
}
return faceThumbnails;
}
Future<List<Uint8List>> generateJpgFaceThumbnails(
String imagePath,
List<FaceBox> faceBoxes,
) async {
final image = await decodeToImgImage(imagePath);
final croppedImages = <img.Image>[];
for (FaceBox faceBox in faceBoxes) {
final croppedImage = cropFaceBoxFromImage(image, faceBox);
croppedImages.add(croppedImage);
}
return await _computer
.compute(_encodeImagesToJpg, param: {"images": croppedImages});
}
Future<img.Image> decodeToImgImage(String imagePath) async {
img.Image? image =
await _computer.compute(_decodeImageFile, param: {"filePath": imagePath});
if (image == null) {
_logger.info(
"Failed to decode image. Compressing to jpg and decoding",
);
final compressedJPGImage =
await FlutterImageCompress.compressWithFile(imagePath);
image = await _computer.compute(
_decodeJpg,
param: {"image": compressedJPGImage},
);
if (image == null) {
throw Exception("Failed to decode image");
} else {
return image;
}
} else {
return image;
}
}
/// Returns an Image from 'package:image/image.dart'
img.Image cropFaceBoxFromImage(img.Image image, FaceBox faceBox) {
final squareFaceBox = _getSquareFaceBoxImage(image, faceBox);
final squareFaceBoxWithBuffer =
_addBufferAroundFaceBox(squareFaceBox, _faceImageBufferFactor);
return img.copyCrop(
image,
x: squareFaceBoxWithBuffer.xMin,
y: squareFaceBoxWithBuffer.yMin,
width: squareFaceBoxWithBuffer.width,
height: squareFaceBoxWithBuffer.height,
antialias: false,
);
}
/// Returns a square face box image from the original image with
/// side length equal to the maximum of the width and height of the face box in
/// the OG image.
FaceBoxImage _getSquareFaceBoxImage(img.Image image, FaceBox faceBox) {
final width = (image.width * faceBox.width).round();
final height = (image.height * faceBox.height).round();
final side = max(width, height);
final xImage = (image.width * faceBox.xMin).round();
final yImage = (image.height * faceBox.yMin).round();
if (height >= width) {
final xImageAdj = (xImage - (height - width) / 2).round();
return FaceBoxImage(
xMin: xImageAdj,
yMin: yImage,
width: side,
height: side,
);
} else {
final yImageAdj = (yImage - (width - height) / 2).round();
return FaceBoxImage(
xMin: xImage,
yMin: yImageAdj,
width: side,
height: side,
);
}
}
///To add some buffer around the face box so that the face isn't cropped
///too close to the face.
FaceBoxImage _addBufferAroundFaceBox(
FaceBoxImage faceBoxImage,
double bufferFactor,
) {
final heightBuffer = faceBoxImage.height * bufferFactor;
final widthBuffer = faceBoxImage.width * bufferFactor;
final xMinWithBuffer = faceBoxImage.xMin - widthBuffer;
final yMinWithBuffer = faceBoxImage.yMin - heightBuffer;
final widthWithBuffer = faceBoxImage.width + 2 * widthBuffer;
final heightWithBuffer = faceBoxImage.height + 2 * heightBuffer;
//Do not add buffer if the top left edge of the image is out of bounds
//after adding the buffer.
if (xMinWithBuffer < 0 || yMinWithBuffer < 0) {
return faceBoxImage;
}
//Another similar case that can be handled is when the bottom right edge
//of the image is out of bounds after adding the buffer. But the
//the visual difference is not as significant as when the top left edge
//is out of bounds, so we are not handling that case.
return FaceBoxImage(
xMin: xMinWithBuffer.round(),
yMin: yMinWithBuffer.round(),
width: widthWithBuffer.round(),
height: heightWithBuffer.round(),
);
}
List<Uint8List> _encodeImagesToJpg(Map args) {
final images = args["images"] as List<img.Image>;
return images.map((img.Image image) => img.encodeJpg(image)).toList();
}
Future<img.Image?> _decodeImageFile(Map args) async {
return await img.decodeImageFile(args["filePath"]);
}
img.Image? _decodeJpg(Map args) {
return img.decodeJpg(args["image"])!;
}

34
mobile/lib/utils/image_util.dart
View File

@ -1,6 +1,8 @@
import 'dart:async'; import 'dart:async';
import 'dart:ui' as ui;
import 'package:flutter/widgets.dart'; import 'package:flutter/widgets.dart';
import 'package:image/image.dart' as img;
Future<ImageInfo> getImageInfo(ImageProvider imageProvider) { Future<ImageInfo> getImageInfo(ImageProvider imageProvider) {
final completer = Completer<ImageInfo>(); final completer = Completer<ImageInfo>();
@ -14,3 +16,35 @@ Future<ImageInfo> getImageInfo(ImageProvider imageProvider) {
completer.future.whenComplete(() => imageStream.removeListener(listener)); completer.future.whenComplete(() => imageStream.removeListener(listener));
return completer.future; return completer.future;
} }
Future<ui.Image> convertImageToFlutterUi(img.Image image) async {
if (image.format != img.Format.uint8 || image.numChannels != 4) {
final cmd = img.Command()
..image(image)
..convert(format: img.Format.uint8, numChannels: 4);
final rgba8 = await cmd.getImageThread();
if (rgba8 != null) {
image = rgba8;
}
}
final ui.ImmutableBuffer buffer =
await ui.ImmutableBuffer.fromUint8List(image.toUint8List());
final ui.ImageDescriptor id = ui.ImageDescriptor.raw(
buffer,
height: image.height,
width: image.width,
pixelFormat: ui.PixelFormat.rgba8888,
);
final ui.Codec codec = await id.instantiateCodec(
targetHeight: image.height,
targetWidth: image.width,
);
final ui.FrameInfo fi = await codec.getNextFrame();
final ui.Image uiImage = fi.image;
return uiImage;
}