kenmirrors/ente
1
0
Fork 0
mirror of https://github.com/ente-io/ente.git synced 2025-08-10 00:12:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

[mob] Better face clustering logging

Browse Source
This commit is contained in:
laurenspriem 2024-04-13 13:08:34 +05:30
parent c67a1fa52a
commit c58a8dc773
2 changed files with 13 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
mobile/lib/services/machine_learning/face_ml/face_clustering/linear_clustering_service.dart
View File

@ -104,10 +104,12 @@ class FaceClustering {
final fileIDToCreationTime = final fileIDToCreationTime =
args['fileIDToCreationTime'] as Map<int, int>?; args['fileIDToCreationTime'] as Map<int, int>?;
final distanceThreshold = args['distanceThreshold'] as double; final distanceThreshold = args['distanceThreshold'] as double;
final offset = args['offset'] as int?;
final result = FaceClustering._runLinearClustering( final result = FaceClustering._runLinearClustering(
input, input,
fileIDToCreationTime: fileIDToCreationTime, fileIDToCreationTime: fileIDToCreationTime,
distanceThreshold: distanceThreshold, distanceThreshold: distanceThreshold,
offset: offset,
); );
sendPort.send(result); sendPort.send(result);
break; break;
@ -201,6 +203,7 @@ class FaceClustering {
Map<String, (int?, Uint8List)> input, { Map<String, (int?, Uint8List)> input, {
Map<int, int>? fileIDToCreationTime, Map<int, int>? fileIDToCreationTime,
double distanceThreshold = kRecommendedDistanceThreshold, double distanceThreshold = kRecommendedDistanceThreshold,
int? offset,
}) async { }) async {
if (input.isEmpty) { if (input.isEmpty) {
_logger.warning( _logger.warning(
@ -229,6 +232,7 @@ class FaceClustering {
'input': input, 'input': input,
'fileIDToCreationTime': fileIDToCreationTime, 'fileIDToCreationTime': fileIDToCreationTime,
'distanceThreshold': distanceThreshold, 'distanceThreshold': distanceThreshold,
'offset': offset,
} }
), ),
); );
@ -299,6 +303,7 @@ class FaceClustering {
Map<String, (int?, Uint8List)> x, { Map<String, (int?, Uint8List)> x, {
Map<int, int>? fileIDToCreationTime, Map<int, int>? fileIDToCreationTime,
double distanceThreshold = kRecommendedDistanceThreshold, double distanceThreshold = kRecommendedDistanceThreshold,
int? offset,
}) { }) {
log( log(
"[ClusterIsolate] ${DateTime.now()} Copied to isolate ${x.length} faces", "[ClusterIsolate] ${DateTime.now()} Copied to isolate ${x.length} faces",
@ -363,7 +368,7 @@ class FaceClustering {
// Start actual clustering // Start actual clustering
log( log(
"[ClusterIsolate] ${DateTime.now()} Processing $totalFaces faces", "[ClusterIsolate] ${DateTime.now()} Processing $totalFaces faces in total in this round ${offset != null ? "on top of ${offset + facesWithClusterID.length} earlier processed faces" : ""}",
); );
// set current epoch time as clusterID // set current epoch time as clusterID
int clusterID = DateTime.now().microsecondsSinceEpoch; int clusterID = DateTime.now().microsecondsSinceEpoch;
@ -384,7 +389,7 @@ class FaceClustering {
int closestIdx = -1; int closestIdx = -1;
double closestDistance = double.infinity; double closestDistance = double.infinity;
if (i % 250 == 0) { if (i % 250 == 0) {
log("[ClusterIsolate] ${DateTime.now()} Processing $i faces"); log("[ClusterIsolate] ${DateTime.now()} Processed ${offset != null ? i + offset : i} faces");
} }
for (int j = i - 1; j >= 0; j--) { for (int j = i - 1; j >= 0; j--) {
late double distance; late double distance;

6
mobile/lib/services/machine_learning/face_ml/face_ml_service.dart
View File

@ -380,6 +380,7 @@ class FaceMlService {
const int batchSize = 20000; const int batchSize = 20000;
const int offsetIncrement = 7500; const int offsetIncrement = 7500;
int offset = 0; int offset = 0;
int bucket = 1;
while (true) { while (true) {
final faceIdToEmbeddingBucket = final faceIdToEmbeddingBucket =
@ -402,6 +403,7 @@ class FaceMlService {
final faceIdToCluster = await FaceClustering.instance.predictLinear( final faceIdToCluster = await FaceClustering.instance.predictLinear(
faceIdToEmbeddingBucket, faceIdToEmbeddingBucket,
fileIDToCreationTime: fileIDToCreationTime, fileIDToCreationTime: fileIDToCreationTime,
offset: offset,
); );
if (faceIdToCluster == null) { if (faceIdToCluster == null) {
_logger.warning("faceIdToCluster is null"); _logger.warning("faceIdToCluster is null");
@ -409,7 +411,11 @@ class FaceMlService {
} }
await FaceMLDataDB.instance.updateClusterIdToFaceId(faceIdToCluster); await FaceMLDataDB.instance.updateClusterIdToFaceId(faceIdToCluster);
_logger.info(
'Done with clustering ${offset + faceIdToEmbeddingBucket.length} embeddings (${(100 * (offset + faceIdToEmbeddingBucket.length) / totalFaces).toStringAsFixed(0)}%) in bucket $bucket, offset: $offset',
);
offset += offsetIncrement; offset += offsetIncrement;
bucket++;
} }
} else { } else {
final int totalFaces = await FaceMLDataDB.instance final int totalFaces = await FaceMLDataDB.instance