kenmirrors/ente
1
0
Fork 0
mirror of https://github.com/ente-io/ente.git synced 2025-08-08 23:39:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove unused ML debug code

Browse Source
This commit is contained in:
Manav Rathi 2024-03-31 18:25:24 +05:30
parent f3a0240f1d
commit 707e14702e
No known key found for this signature in database
6 changed files with 9 additions and 469 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

154
web/apps/photos/src/services/machineLearning/machineLearningService.ts
View File

@ -1,63 +1,34 @@
import { getLocalFiles } from "services/fileService";
import { EnteFile } from "types/file";
import { APPS } from "@ente/shared/apps/constants";
import { CustomError, parseUploadErrorCodes } from "@ente/shared/error";
import { addLogLine } from "@ente/shared/logging";
import { logError } from "@ente/shared/sentry";
import "@tensorflow/tfjs-backend-cpu";
import "@tensorflow/tfjs-backend-webgl";
import * as tf from "@tensorflow/tfjs-core";
// import '@tensorflow/tfjs-backend-wasm';
// import { setWasmPaths } from '@tensorflow/tfjs-backend-wasm';
// import '@tensorflow/tfjs-backend-cpu';
import { MAX_ML_SYNC_ERROR_COUNT } from "constants/mlConfig";
import downloadManager from "services/download";
import { getLocalFiles } from "services/fileService";
import { EnteFile } from "types/file";
import {
MlFileData,
MLSyncContext,
MLSyncFileContext,
MLSyncResult,
MlFileData,
} from "types/machineLearning";
// import { toTSNE } from 'utils/machineLearning/visualization';
// import {
// incrementIndexVersion,
// mlFilesStore
// } from 'utils/storage/mlStorage';
// import { getAllFacesFromMap } from 'utils/machineLearning';
import { CustomError, parseUploadErrorCodes } from "@ente/shared/error";
import { MAX_ML_SYNC_ERROR_COUNT } from "constants/mlConfig";
import { getMLSyncConfig } from "utils/machineLearning/config";
import mlIDbStorage from "utils/storage/mlIDbStorage";
import FaceService from "./faceService";
import { MLFactory } from "./machineLearningFactory";
import ObjectService from "./objectService";
import PeopleService from "./peopleService";
// import TextService from './textService';
import { APPS } from "@ente/shared/apps/constants";
import { addLogLine } from "@ente/shared/logging";
import { logError } from "@ente/shared/sentry";
import downloadManager from "services/download";
import ReaderService from "./readerService";
class MachineLearningService {
private initialized = false;
// private faceDetectionService: FaceDetectionService;
// private faceLandmarkService: FAPIFaceLandmarksService;
// private faceAlignmentService: FaceAlignmentService;
// private faceEmbeddingService: FaceEmbeddingService;
// private faceEmbeddingService: FAPIFaceEmbeddingService;
// private clusteringService: ClusteringService;
private localSyncContext: Promise<MLSyncContext>;
private syncContext: Promise<MLSyncContext>;
public constructor() {
// setWasmPaths('/js/tfjs/');
// this.faceDetectionService = new TFJSFaceDetectionService();
// this.faceLandmarkService = new FAPIFaceLandmarksService();
// this.faceAlignmentService = new ArcfaceAlignmentService();
// this.faceEmbeddingService = new TFJSFaceEmbeddingService();
// this.faceEmbeddingService = new FAPIFaceEmbeddingService();
// this.clusteringService = new ClusteringService();
}
public async sync(token: string, userID: number): Promise<MLSyncResult> {
if (!token) {
throw Error("Token needed by ml service to sync file");
@ -198,30 +169,6 @@ class MachineLearningService {
addLogLine("syncLocalFiles", Date.now() - startTime, "ms");
}
// TODO: not required if ml data is stored as field inside ente file object
// remove, not required now
// it removes ml data for files in trash, they will be resynced if restored
// private async syncRemovedFiles(syncContext: MLSyncContext) {
// const db = await mlIDbStorage.db;
// const localFileIdMap = await this.getLocalFilesMap(syncContext);
// const removedFileIds: Array<string> = [];
// await mlFilesStore.iterate((file, idStr) => {
// if (!localFileIdMap.has(parseInt(idStr))) {
// removedFileIds.push(idStr);
// }
// });
// if (removedFileIds.length < 1) {
// return;
// }
// removedFileIds.forEach((fileId) => mlFilesStore.removeItem(fileId));
// addLogLine('Removed local file ids: ', removedFileIds);
// await incrementIndexVersion('files');
// }
private async getOutOfSyncFiles(syncContext: MLSyncContext) {
const startTime = Date.now();
const fileIds = await mlIDbStorage.getFileIds(
@ -239,50 +186,6 @@ class MachineLearningService {
addLogLine("getOutOfSyncFiles", Date.now() - startTime, "ms");
}
// TODO: optimize, use indexdb indexes, move facecrops to cache to reduce io
// remove, already done
private async getUniqueOutOfSyncFilesNoIdx(
syncContext: MLSyncContext,
files: EnteFile[],
) {
const limit = syncContext.config.batchSize;
const mlVersion = syncContext.config.mlVersion;
const uniqueFiles: Map<number, EnteFile> = new Map<number, EnteFile>();
for (let i = 0; uniqueFiles.size < limit && i < files.length; i++) {
const mlFileData = await this.getMLFileData(files[i].id);
const mlFileVersion = mlFileData?.mlVersion || 0;
if (
!uniqueFiles.has(files[i].id) &&
(!mlFileData?.errorCount || mlFileData.errorCount < 2) &&
(mlFileVersion < mlVersion ||
syncContext.config.imageSource !== mlFileData.imageSource)
) {
uniqueFiles.set(files[i].id, files[i]);
}
}
return [...uniqueFiles.values()];
}
// private async getOutOfSyncFilesNoIdx(syncContext: MLSyncContext) {
// const existingFilesMap = await this.getLocalFilesMap(syncContext);
// // existingFiles.sort(
// // (a, b) => b.metadata.creationTime - a.metadata.creationTime
// // );
// console.time('getUniqueOutOfSyncFiles');
// syncContext.outOfSyncFiles = await this.getUniqueOutOfSyncFilesNoIdx(
// syncContext,
// [...existingFilesMap.values()]
// );
// addLogLine('getUniqueOutOfSyncFiles');
// addLogLine(
// 'Got unique outOfSyncFiles: ',
// syncContext.outOfSyncFiles.length,
// 'for batchSize: ',
// syncContext.config.batchSize
// );
// }
private async syncFiles(syncContext: MLSyncContext) {
try {
const functions = syncContext.outOfSyncFiles.map(
@ -448,22 +351,12 @@ class MachineLearningService {
try {
await ReaderService.getImageBitmap(syncContext, fileContext);
// await this.syncFaceDetections(syncContext, fileContext);
// await ObjectService.syncFileObjectDetections(
// syncContext,
// fileContext
// );
await Promise.all([
this.syncFaceDetections(syncContext, fileContext),
ObjectService.syncFileObjectDetections(
syncContext,
fileContext,
),
// TextService.syncFileTextDetections(
// syncContext,
// fileContext,
// textDetectionTimeoutIndex
// ),
]);
newMlFile.errorCount = 0;
newMlFile.lastErrorMessage = undefined;
@ -495,30 +388,15 @@ class MachineLearningService {
await tf.ready();
addLogLine("01 TF Memory stats: ", JSON.stringify(tf.memory()));
// await tfjsFaceDetectionService.init();
// // addLogLine('02 TF Memory stats: ',JSON.stringify(tf.memory()));
// await this.faceLandmarkService.init();
// await faceapi.nets.faceLandmark68Net.loadFromUri('/models/face-api/');
// // addLogLine('03 TF Memory stats: ',JSON.stringify(tf.memory()));
// await tfjsFaceEmbeddingService.init();
// await faceapi.nets.faceRecognitionNet.loadFromUri('/models/face-api/');
// addLogLine('04 TF Memory stats: ',JSON.stringify(tf.memory()));
this.initialized = true;
}
public async dispose() {
this.initialized = false;
// await this.faceDetectionService.dispose();
// addLogLine('11 TF Memory stats: ',JSON.stringify(tf.memory()));
// await this.faceLandmarkService.dispose();
// addLogLine('12 TF Memory stats: ',JSON.stringify(tf.memory()));
// await this.faceEmbeddingService.dispose();
// addLogLine('13 TF Memory stats: ',JSON.stringify(tf.memory()));
}
private async getMLFileData(fileId: number) {
// return mlFilesStore.getItem<MlFileData>(fileId);
return mlIDbStorage.getFile(fileId);
}
@ -526,7 +404,6 @@ class MachineLearningService {
syncContext: MLSyncContext,
mlFileData: MlFileData,
) {
// return mlFilesStore.setItem(mlFileData.fileId.toString(), mlFileData);
mlIDbStorage.putFile(mlFileData);
}
@ -559,14 +436,12 @@ class MachineLearningService {
}
private async persistMLLibraryData(syncContext: MLSyncContext) {
// return mlLibraryStore.setItem('data', syncContext.mlLibraryData);
return mlIDbStorage.putLibraryData(syncContext.mlLibraryData);
}
public async syncIndex(syncContext: MLSyncContext) {
await this.getMLLibraryData(syncContext);
// await this.init();
await PeopleService.syncPeopleIndex(syncContext);
await ObjectService.syncThingsIndex(syncContext);
@ -574,17 +449,6 @@ class MachineLearningService {
await this.persistMLLibraryData(syncContext);
}
// private async runTSNE(syncContext: MLSyncContext) {
// const allFacesMap = await FaceService.getAllSyncedFacesMap(syncContext);
// const allFaces = getAllFacesFromMap(allFacesMap);
// const input = allFaces
// .slice(0, syncContext.config.tsne.samples)
// .map((f) => Array.from(f.embedding));
// syncContext.tsne = toTSNE(input, syncContext.config.tsne);
// addLogLine('tsne: ', syncContext.tsne);
// }
private async syncFaceDetections(
syncContext: MLSyncContext,
fileContext: MLSyncFileContext,

39
web/apps/photos/src/types/machineLearning/index.ts
View File

@ -1,15 +1,6 @@
import * as tf from "@tensorflow/tfjs-core";
// import {
// FaceDetection,
// FaceLandmarks68,
// WithFaceDescriptor,
// WithFaceLandmarks,
// } from 'face-api.js';
import { DebugInfo } from "hdbscan";
import PQueue from "p-queue";
// import { Point as D3Point, RawNodeDatum } from 'react-d3-tree/lib/types/common';
import { EnteFile } from "types/file";
import { Dimensions } from "types/image";
import { Box, Point } from "../../../thirdparty/face-api/classes";
@ -24,33 +15,9 @@ export interface MLSyncResult {
error?: Error;
}
export interface DebugFace {
fileId: string;
// face: FaceApiResult;
face: AlignedFace;
embedding: FaceEmbedding;
faceImage: FaceImage;
}
// export interface MLDebugResult {
// allFaces: DebugFace[];
// clustersWithNoise: FacesClustersWithNoise;
// tree: RawNodeDatum;
// tsne: TSNEData;
// }
export declare type FaceImage = Array<Array<Array<number>>>;
export declare type FaceImageBlob = Blob;
// export declare type FaceApiResult = WithFaceDescriptor<
// WithFaceLandmarks<
// {
// detection: FaceDetection;
// },
// FaceLandmarks68
// >
// >;
export declare type FaceDescriptor = Float32Array;
export declare type Cluster = Array<number>;
@ -79,12 +46,6 @@ export interface NearestCluster {
distance: number;
}
// export interface TSNEData {
// width: number;
// height: number;
// dataset: D3Point[];
// }
export declare type Landmark = Point;
export declare type ImageType = "Original" | "Preview";

12
web/apps/photos/src/utils/machineLearning/clustering.ts
View File

@ -4,22 +4,10 @@ import { euclidean } from "hdbscan";
import { addLogLine } from "@ente/shared/logging";
import {
FacesCluster,
// Cluster,
// FaceDescriptor,
FaceWithEmbedding,
MLSyncContext,
NearestCluster,
} from "types/machineLearning";
// import { getAllFacesMap } from 'utils/storage/mlStorage';
// export function getClusterSummary(cluster: Cluster): FaceDescriptor {
// const faceScore = (f) => f.detection.score; // f.alignedRect.box.width *
// return cluster
// .map((f) => this.allFaces[f].face)
// .sort((f1, f2) => faceScore(f2) - faceScore(f1))[0].descriptor;
// const descriptors = cluster.map((f) => this.allFaces[f].embedding);
// return f32Average(descriptors);
// }
export function updateClusterSummaries(syncContext: MLSyncContext) {
if (

136
web/apps/photos/src/utils/machineLearning/migrations.ts
View File

@ -1,136 +0,0 @@
import { addLogLine } from "@ente/shared/logging";
import { Face, MlFileData } from "types/machineLearning";
import mlIDbStorage from "utils/storage/mlIDbStorage";
import { mlFilesStore } from "utils/storage/mlStorage";
import { getFaceId } from ".";
import { storeFaceCropForBlob } from "./faceCrop";
// TODO: for migrating existing data, to be removed
export async function migrateExistingFiles() {
const existingFiles: Array<MlFileData> = [];
await mlFilesStore.iterate((mlFileData: MlFileData) => {
if (!mlFileData.errorCount) {
mlFileData.errorCount = 0;
existingFiles.push(mlFileData);
}
});
addLogLine("existing files: ", existingFiles.length);
try {
for (const file of existingFiles) {
await mlIDbStorage.putFile(file);
}
await mlIDbStorage.setIndexVersion("files", 1);
addLogLine("migrateExistingFiles done");
} catch (e) {
console.error(e);
}
}
export async function migrateFaceCropsToCache() {
const startTime = Date.now();
addLogLine("migrateFaceCropsToCache started");
const allFiles = await mlIDbStorage.getAllFiles();
const allFilesWithFaces = allFiles.filter(
(f) => f.faces && f.faces.length > 0,
);
const updatedFacesMap = new Map<number, Array<Face>>();
for (const file of allFilesWithFaces) {
let updated = false;
for (const face of file.faces) {
if (!face["id"]) {
const faceCropBlob = face.crop["image"];
const faceId = getFaceId(face, file.imageDimensions);
face.crop = await storeFaceCropForBlob(
faceId,
face.crop.imageBox,
faceCropBlob,
);
face["id"] = faceId;
updated = true;
}
}
if (updated) {
updatedFacesMap.set(file.fileId, file.faces);
}
}
if (updatedFacesMap.size > 0) {
addLogLine("updating face crops: ", updatedFacesMap.size);
await mlIDbStorage.updateFaces(updatedFacesMap);
} else {
addLogLine("not updating face crops: ", updatedFacesMap.size);
}
addLogLine("migrateFaceCropsToCache", Date.now() - startTime, "ms");
}
export async function migrateFaceInterfaceUpdate() {
const startTime = Date.now();
addLogLine("migrateFaceInterfaceUpdate started");
const faceSchemaVersion = await mlIDbStorage.getIndexVersion("faceSchema");
if (faceSchemaVersion) {
addLogLine("not running migrateFaceInterfaceUpdate");
return;
}
const allFiles = await mlIDbStorage.getAllFiles();
const updatedFiles = allFiles.map((file) => {
const updatedFaces = file.faces?.map((f) => {
const updatedFace = {
id: f["faceId"],
fileId: f.fileId,
detection: {
box: f["box"],
landmarks: f["landmarks"],
probability: f["probability"],
},
crop: f["faceCrop"],
alignment: {
affineMatrix: f["affineMatrix"],
center: f["center"],
rotation: f["rotation"],
size: f["size"],
},
embedding: Float32Array.from(f.embedding),
personId: f.personId,
} as Face;
if (!updatedFace.id) {
updatedFace.id = getFaceId(updatedFace, file.imageDimensions);
}
return updatedFace;
});
const updated: MlFileData = {
fileId: file.fileId,
faceDetectionMethod: file["detectionMethod"],
faceCropMethod: {
value: "ArcFace",
version: 1,
},
faceAlignmentMethod: file["alignmentMethod"],
faceEmbeddingMethod: file["embeddingMethod"],
faces: updatedFaces,
imageDimensions: file.imageDimensions,
imageSource: file.imageSource,
errorCount: file.errorCount,
lastErrorMessage: file.lastErrorMessage,
mlVersion: file.mlVersion,
};
return updated;
});
addLogLine("migrateFaceInterfaceUpdate updating: ", updatedFiles.length);
await mlIDbStorage.putAllFilesInTx(updatedFiles);
await mlIDbStorage.setIndexVersion("faceSchema", 1);
addLogLine("migrateFaceInterfaceUpdate done");
addLogLine("migrateFaceInterfaceUpdate", Date.now() - startTime, "ms");
}

40
web/apps/photos/src/utils/machineLearning/visualization.ts
View File

@ -1,40 +0,0 @@
// // import TSNE from 'tsne-js';
// import { TSNEConfig, TSNEData } from 'types/machineLearning';
// export function toD3Tsne(tsne) {
// const data: TSNEData = {
// width: 800,
// height: 800,
// dataset: [],
// };
// data.dataset = tsne.map((t) => {
// return {
// x: (data.width * (t[0] + 1.0)) / 2,
// y: (data.height * (t[1] + 1.0)) / 2,
// };
// });
// return data;
// }
// export function toTSNE(denseInput: Array<Array<number>>, config: TSNEConfig) {
// if (!denseInput || denseInput.length < 1) {
// return null;
// }
// const model = new TSNE(config);
// model.init({
// data: denseInput,
// type: 'dense',
// });
// // `error`, `iter`: final error and iteration number
// // note: computation-heavy action happens here
// model.run();
// // `outputScaled` is `output` scaled to a range of [-1, 1]
// return model.getOutputScaled();
// }
export {};

97
web/apps/photos/src/utils/storage/mlStorage.ts
View File

@ -1,97 +0,0 @@
import localForage from "localforage";
import { EnteFile } from "types/file";
import {
Face,
MlFileData,
MLIndex,
MLSyncContext,
} from "types/machineLearning";
export const mlFilesStore = localForage.createInstance({
driver: localForage.INDEXEDDB,
name: "ml-data",
version: 1.0,
storeName: "files",
});
export const mlPeopleStore = localForage.createInstance({
driver: localForage.INDEXEDDB,
name: "ml-data",
version: 1.0,
storeName: "people",
});
export const mlLibraryStore = localForage.createInstance({
driver: localForage.INDEXEDDB,
name: "ml-data",
version: 1.0,
storeName: "library",
});
export const mlVersionStore = localForage.createInstance({
driver: localForage.INDEXEDDB,
name: "ml-data",
version: 1.0,
storeName: "versions",
});
export async function clearMLStorage() {
await mlFilesStore.clear();
await mlPeopleStore.clear();
await mlLibraryStore.clear();
await mlVersionStore.clear();
}
export async function getIndexVersion(index: MLIndex): Promise<number> {
return ((await mlVersionStore.getItem(`${index}`)) as number) || 0;
}
export async function setIndexVersion(
index: MLIndex,
version: number,
): Promise<number> {
await mlVersionStore.setItem(`${index}`, version);
return version;
}
export async function incrementIndexVersion(index: MLIndex): Promise<number> {
let currentVersion = await getIndexVersion(index);
currentVersion = currentVersion + 1;
await setIndexVersion(index, currentVersion);
return currentVersion;
}
export async function isVersionOutdated(index: MLIndex, thanIndex: MLIndex) {
const indexVersion = await getIndexVersion(index);
const thanIndexVersion = await getIndexVersion(thanIndex);
return indexVersion < thanIndexVersion;
}
export function newMlData(
syncContext: MLSyncContext,
enteFile: EnteFile,
): MlFileData {
return {
fileId: enteFile.id,
imageSource: syncContext.config.imageSource,
faceDetectionMethod: syncContext.faceDetectionService.method,
faceCropMethod: syncContext.faceCropService.method,
faceAlignmentMethod: syncContext.faceAlignmentService.method,
faceEmbeddingMethod: syncContext.faceEmbeddingService.method,
errorCount: 0,
mlVersion: 0,
};
}
export async function getAllFacesMap() {
const allSyncedFacesMap = new Map<number, Array<Face>>();
await mlFilesStore.iterate((mlFileData: MlFileData) => {
mlFileData.faces &&
allSyncedFacesMap.set(mlFileData.fileId, mlFileData.faces);
});
return allSyncedFacesMap;
}