From 707e14702e8045f245c1d6cf36a06c02b7ca9060 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sun, 31 Mar 2024 18:25:24 +0530 Subject: [PATCH] Remove unused ML debug code --- .../machineLearning/machineLearningService.ts | 154 +----------------- .../photos/src/types/machineLearning/index.ts | 39 ----- .../src/utils/machineLearning/clustering.ts | 12 -- .../src/utils/machineLearning/migrations.ts | 136 ---------------- .../utils/machineLearning/visualization.ts | 40 ----- .../photos/src/utils/storage/mlStorage.ts | 97 ----------- 6 files changed, 9 insertions(+), 469 deletions(-) delete mode 100644 web/apps/photos/src/utils/machineLearning/migrations.ts delete mode 100644 web/apps/photos/src/utils/machineLearning/visualization.ts delete mode 100644 web/apps/photos/src/utils/storage/mlStorage.ts diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index c95effc763..a76c740e19 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -1,63 +1,34 @@ -import { getLocalFiles } from "services/fileService"; -import { EnteFile } from "types/file"; - +import { APPS } from "@ente/shared/apps/constants"; +import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; +import { addLogLine } from "@ente/shared/logging"; +import { logError } from "@ente/shared/sentry"; import "@tensorflow/tfjs-backend-cpu"; import "@tensorflow/tfjs-backend-webgl"; import * as tf from "@tensorflow/tfjs-core"; -// import '@tensorflow/tfjs-backend-wasm'; -// import { setWasmPaths } from '@tensorflow/tfjs-backend-wasm'; -// import '@tensorflow/tfjs-backend-cpu'; - +import { MAX_ML_SYNC_ERROR_COUNT } from "constants/mlConfig"; +import downloadManager from "services/download"; +import { getLocalFiles } from "services/fileService"; +import { EnteFile } from "types/file"; import { - MlFileData, MLSyncContext, MLSyncFileContext, MLSyncResult, + MlFileData, } from "types/machineLearning"; - -// import { toTSNE } from 'utils/machineLearning/visualization'; -// import { -// incrementIndexVersion, -// mlFilesStore -// } from 'utils/storage/mlStorage'; -// import { getAllFacesFromMap } from 'utils/machineLearning'; -import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; -import { MAX_ML_SYNC_ERROR_COUNT } from "constants/mlConfig"; import { getMLSyncConfig } from "utils/machineLearning/config"; import mlIDbStorage from "utils/storage/mlIDbStorage"; import FaceService from "./faceService"; import { MLFactory } from "./machineLearningFactory"; import ObjectService from "./objectService"; import PeopleService from "./peopleService"; -// import TextService from './textService'; -import { APPS } from "@ente/shared/apps/constants"; -import { addLogLine } from "@ente/shared/logging"; -import { logError } from "@ente/shared/sentry"; -import downloadManager from "services/download"; import ReaderService from "./readerService"; class MachineLearningService { private initialized = false; - // private faceDetectionService: FaceDetectionService; - // private faceLandmarkService: FAPIFaceLandmarksService; - // private faceAlignmentService: FaceAlignmentService; - // private faceEmbeddingService: FaceEmbeddingService; - // private faceEmbeddingService: FAPIFaceEmbeddingService; - // private clusteringService: ClusteringService; private localSyncContext: Promise; private syncContext: Promise; - public constructor() { - // setWasmPaths('/js/tfjs/'); - // this.faceDetectionService = new TFJSFaceDetectionService(); - // this.faceLandmarkService = new FAPIFaceLandmarksService(); - // this.faceAlignmentService = new ArcfaceAlignmentService(); - // this.faceEmbeddingService = new TFJSFaceEmbeddingService(); - // this.faceEmbeddingService = new FAPIFaceEmbeddingService(); - // this.clusteringService = new ClusteringService(); - } - public async sync(token: string, userID: number): Promise { if (!token) { throw Error("Token needed by ml service to sync file"); @@ -198,30 +169,6 @@ class MachineLearningService { addLogLine("syncLocalFiles", Date.now() - startTime, "ms"); } - // TODO: not required if ml data is stored as field inside ente file object - // remove, not required now - // it removes ml data for files in trash, they will be resynced if restored - // private async syncRemovedFiles(syncContext: MLSyncContext) { - // const db = await mlIDbStorage.db; - // const localFileIdMap = await this.getLocalFilesMap(syncContext); - - // const removedFileIds: Array = []; - // await mlFilesStore.iterate((file, idStr) => { - // if (!localFileIdMap.has(parseInt(idStr))) { - // removedFileIds.push(idStr); - // } - // }); - - // if (removedFileIds.length < 1) { - // return; - // } - - // removedFileIds.forEach((fileId) => mlFilesStore.removeItem(fileId)); - // addLogLine('Removed local file ids: ', removedFileIds); - - // await incrementIndexVersion('files'); - // } - private async getOutOfSyncFiles(syncContext: MLSyncContext) { const startTime = Date.now(); const fileIds = await mlIDbStorage.getFileIds( @@ -239,50 +186,6 @@ class MachineLearningService { addLogLine("getOutOfSyncFiles", Date.now() - startTime, "ms"); } - // TODO: optimize, use indexdb indexes, move facecrops to cache to reduce io - // remove, already done - private async getUniqueOutOfSyncFilesNoIdx( - syncContext: MLSyncContext, - files: EnteFile[], - ) { - const limit = syncContext.config.batchSize; - const mlVersion = syncContext.config.mlVersion; - const uniqueFiles: Map = new Map(); - for (let i = 0; uniqueFiles.size < limit && i < files.length; i++) { - const mlFileData = await this.getMLFileData(files[i].id); - const mlFileVersion = mlFileData?.mlVersion || 0; - if ( - !uniqueFiles.has(files[i].id) && - (!mlFileData?.errorCount || mlFileData.errorCount < 2) && - (mlFileVersion < mlVersion || - syncContext.config.imageSource !== mlFileData.imageSource) - ) { - uniqueFiles.set(files[i].id, files[i]); - } - } - - return [...uniqueFiles.values()]; - } - - // private async getOutOfSyncFilesNoIdx(syncContext: MLSyncContext) { - // const existingFilesMap = await this.getLocalFilesMap(syncContext); - // // existingFiles.sort( - // // (a, b) => b.metadata.creationTime - a.metadata.creationTime - // // ); - // console.time('getUniqueOutOfSyncFiles'); - // syncContext.outOfSyncFiles = await this.getUniqueOutOfSyncFilesNoIdx( - // syncContext, - // [...existingFilesMap.values()] - // ); - // addLogLine('getUniqueOutOfSyncFiles'); - // addLogLine( - // 'Got unique outOfSyncFiles: ', - // syncContext.outOfSyncFiles.length, - // 'for batchSize: ', - // syncContext.config.batchSize - // ); - // } - private async syncFiles(syncContext: MLSyncContext) { try { const functions = syncContext.outOfSyncFiles.map( @@ -448,22 +351,12 @@ class MachineLearningService { try { await ReaderService.getImageBitmap(syncContext, fileContext); - // await this.syncFaceDetections(syncContext, fileContext); - // await ObjectService.syncFileObjectDetections( - // syncContext, - // fileContext - // ); await Promise.all([ this.syncFaceDetections(syncContext, fileContext), ObjectService.syncFileObjectDetections( syncContext, fileContext, ), - // TextService.syncFileTextDetections( - // syncContext, - // fileContext, - // textDetectionTimeoutIndex - // ), ]); newMlFile.errorCount = 0; newMlFile.lastErrorMessage = undefined; @@ -495,30 +388,15 @@ class MachineLearningService { await tf.ready(); addLogLine("01 TF Memory stats: ", JSON.stringify(tf.memory())); - // await tfjsFaceDetectionService.init(); - // // addLogLine('02 TF Memory stats: ',JSON.stringify(tf.memory())); - // await this.faceLandmarkService.init(); - // await faceapi.nets.faceLandmark68Net.loadFromUri('/models/face-api/'); - // // addLogLine('03 TF Memory stats: ',JSON.stringify(tf.memory())); - // await tfjsFaceEmbeddingService.init(); - // await faceapi.nets.faceRecognitionNet.loadFromUri('/models/face-api/'); - // addLogLine('04 TF Memory stats: ',JSON.stringify(tf.memory())); this.initialized = true; } public async dispose() { this.initialized = false; - // await this.faceDetectionService.dispose(); - // addLogLine('11 TF Memory stats: ',JSON.stringify(tf.memory())); - // await this.faceLandmarkService.dispose(); - // addLogLine('12 TF Memory stats: ',JSON.stringify(tf.memory())); - // await this.faceEmbeddingService.dispose(); - // addLogLine('13 TF Memory stats: ',JSON.stringify(tf.memory())); } private async getMLFileData(fileId: number) { - // return mlFilesStore.getItem(fileId); return mlIDbStorage.getFile(fileId); } @@ -526,7 +404,6 @@ class MachineLearningService { syncContext: MLSyncContext, mlFileData: MlFileData, ) { - // return mlFilesStore.setItem(mlFileData.fileId.toString(), mlFileData); mlIDbStorage.putFile(mlFileData); } @@ -559,14 +436,12 @@ class MachineLearningService { } private async persistMLLibraryData(syncContext: MLSyncContext) { - // return mlLibraryStore.setItem('data', syncContext.mlLibraryData); return mlIDbStorage.putLibraryData(syncContext.mlLibraryData); } public async syncIndex(syncContext: MLSyncContext) { await this.getMLLibraryData(syncContext); - // await this.init(); await PeopleService.syncPeopleIndex(syncContext); await ObjectService.syncThingsIndex(syncContext); @@ -574,17 +449,6 @@ class MachineLearningService { await this.persistMLLibraryData(syncContext); } - // private async runTSNE(syncContext: MLSyncContext) { - // const allFacesMap = await FaceService.getAllSyncedFacesMap(syncContext); - // const allFaces = getAllFacesFromMap(allFacesMap); - - // const input = allFaces - // .slice(0, syncContext.config.tsne.samples) - // .map((f) => Array.from(f.embedding)); - // syncContext.tsne = toTSNE(input, syncContext.config.tsne); - // addLogLine('tsne: ', syncContext.tsne); - // } - private async syncFaceDetections( syncContext: MLSyncContext, fileContext: MLSyncFileContext, diff --git a/web/apps/photos/src/types/machineLearning/index.ts b/web/apps/photos/src/types/machineLearning/index.ts index 64cf8f2f41..2634a0e0f7 100644 --- a/web/apps/photos/src/types/machineLearning/index.ts +++ b/web/apps/photos/src/types/machineLearning/index.ts @@ -1,15 +1,6 @@ import * as tf from "@tensorflow/tfjs-core"; - -// import { -// FaceDetection, -// FaceLandmarks68, -// WithFaceDescriptor, -// WithFaceLandmarks, -// } from 'face-api.js'; import { DebugInfo } from "hdbscan"; import PQueue from "p-queue"; - -// import { Point as D3Point, RawNodeDatum } from 'react-d3-tree/lib/types/common'; import { EnteFile } from "types/file"; import { Dimensions } from "types/image"; import { Box, Point } from "../../../thirdparty/face-api/classes"; @@ -24,33 +15,9 @@ export interface MLSyncResult { error?: Error; } -export interface DebugFace { - fileId: string; - // face: FaceApiResult; - face: AlignedFace; - embedding: FaceEmbedding; - faceImage: FaceImage; -} - -// export interface MLDebugResult { -// allFaces: DebugFace[]; -// clustersWithNoise: FacesClustersWithNoise; -// tree: RawNodeDatum; -// tsne: TSNEData; -// } - export declare type FaceImage = Array>>; export declare type FaceImageBlob = Blob; -// export declare type FaceApiResult = WithFaceDescriptor< -// WithFaceLandmarks< -// { -// detection: FaceDetection; -// }, -// FaceLandmarks68 -// > -// >; - export declare type FaceDescriptor = Float32Array; export declare type Cluster = Array; @@ -79,12 +46,6 @@ export interface NearestCluster { distance: number; } -// export interface TSNEData { -// width: number; -// height: number; -// dataset: D3Point[]; -// } - export declare type Landmark = Point; export declare type ImageType = "Original" | "Preview"; diff --git a/web/apps/photos/src/utils/machineLearning/clustering.ts b/web/apps/photos/src/utils/machineLearning/clustering.ts index 26d8f803d1..c630715b18 100644 --- a/web/apps/photos/src/utils/machineLearning/clustering.ts +++ b/web/apps/photos/src/utils/machineLearning/clustering.ts @@ -4,22 +4,10 @@ import { euclidean } from "hdbscan"; import { addLogLine } from "@ente/shared/logging"; import { FacesCluster, - // Cluster, - // FaceDescriptor, FaceWithEmbedding, MLSyncContext, NearestCluster, } from "types/machineLearning"; -// import { getAllFacesMap } from 'utils/storage/mlStorage'; - -// export function getClusterSummary(cluster: Cluster): FaceDescriptor { -// const faceScore = (f) => f.detection.score; // f.alignedRect.box.width * -// return cluster -// .map((f) => this.allFaces[f].face) -// .sort((f1, f2) => faceScore(f2) - faceScore(f1))[0].descriptor; -// const descriptors = cluster.map((f) => this.allFaces[f].embedding); -// return f32Average(descriptors); -// } export function updateClusterSummaries(syncContext: MLSyncContext) { if ( diff --git a/web/apps/photos/src/utils/machineLearning/migrations.ts b/web/apps/photos/src/utils/machineLearning/migrations.ts deleted file mode 100644 index 111fabfc97..0000000000 --- a/web/apps/photos/src/utils/machineLearning/migrations.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { addLogLine } from "@ente/shared/logging"; -import { Face, MlFileData } from "types/machineLearning"; -import mlIDbStorage from "utils/storage/mlIDbStorage"; -import { mlFilesStore } from "utils/storage/mlStorage"; -import { getFaceId } from "."; -import { storeFaceCropForBlob } from "./faceCrop"; - -// TODO: for migrating existing data, to be removed -export async function migrateExistingFiles() { - const existingFiles: Array = []; - await mlFilesStore.iterate((mlFileData: MlFileData) => { - if (!mlFileData.errorCount) { - mlFileData.errorCount = 0; - existingFiles.push(mlFileData); - } - }); - addLogLine("existing files: ", existingFiles.length); - - try { - for (const file of existingFiles) { - await mlIDbStorage.putFile(file); - } - await mlIDbStorage.setIndexVersion("files", 1); - addLogLine("migrateExistingFiles done"); - } catch (e) { - console.error(e); - } -} - -export async function migrateFaceCropsToCache() { - const startTime = Date.now(); - addLogLine("migrateFaceCropsToCache started"); - const allFiles = await mlIDbStorage.getAllFiles(); - const allFilesWithFaces = allFiles.filter( - (f) => f.faces && f.faces.length > 0, - ); - const updatedFacesMap = new Map>(); - - for (const file of allFilesWithFaces) { - let updated = false; - for (const face of file.faces) { - if (!face["id"]) { - const faceCropBlob = face.crop["image"]; - const faceId = getFaceId(face, file.imageDimensions); - face.crop = await storeFaceCropForBlob( - faceId, - face.crop.imageBox, - faceCropBlob, - ); - face["id"] = faceId; - updated = true; - } - } - if (updated) { - updatedFacesMap.set(file.fileId, file.faces); - } - } - - if (updatedFacesMap.size > 0) { - addLogLine("updating face crops: ", updatedFacesMap.size); - await mlIDbStorage.updateFaces(updatedFacesMap); - } else { - addLogLine("not updating face crops: ", updatedFacesMap.size); - } - addLogLine("migrateFaceCropsToCache", Date.now() - startTime, "ms"); -} - -export async function migrateFaceInterfaceUpdate() { - const startTime = Date.now(); - addLogLine("migrateFaceInterfaceUpdate started"); - - const faceSchemaVersion = await mlIDbStorage.getIndexVersion("faceSchema"); - if (faceSchemaVersion) { - addLogLine("not running migrateFaceInterfaceUpdate"); - return; - } - - const allFiles = await mlIDbStorage.getAllFiles(); - - const updatedFiles = allFiles.map((file) => { - const updatedFaces = file.faces?.map((f) => { - const updatedFace = { - id: f["faceId"], - fileId: f.fileId, - - detection: { - box: f["box"], - landmarks: f["landmarks"], - probability: f["probability"], - }, - crop: f["faceCrop"], - alignment: { - affineMatrix: f["affineMatrix"], - center: f["center"], - rotation: f["rotation"], - size: f["size"], - }, - embedding: Float32Array.from(f.embedding), - - personId: f.personId, - } as Face; - if (!updatedFace.id) { - updatedFace.id = getFaceId(updatedFace, file.imageDimensions); - } - return updatedFace; - }); - const updated: MlFileData = { - fileId: file.fileId, - - faceDetectionMethod: file["detectionMethod"], - faceCropMethod: { - value: "ArcFace", - version: 1, - }, - faceAlignmentMethod: file["alignmentMethod"], - faceEmbeddingMethod: file["embeddingMethod"], - - faces: updatedFaces, - - imageDimensions: file.imageDimensions, - imageSource: file.imageSource, - errorCount: file.errorCount, - lastErrorMessage: file.lastErrorMessage, - mlVersion: file.mlVersion, - }; - - return updated; - }); - - addLogLine("migrateFaceInterfaceUpdate updating: ", updatedFiles.length); - await mlIDbStorage.putAllFilesInTx(updatedFiles); - - await mlIDbStorage.setIndexVersion("faceSchema", 1); - addLogLine("migrateFaceInterfaceUpdate done"); - addLogLine("migrateFaceInterfaceUpdate", Date.now() - startTime, "ms"); -} diff --git a/web/apps/photos/src/utils/machineLearning/visualization.ts b/web/apps/photos/src/utils/machineLearning/visualization.ts deleted file mode 100644 index 949dde6f26..0000000000 --- a/web/apps/photos/src/utils/machineLearning/visualization.ts +++ /dev/null @@ -1,40 +0,0 @@ -// // import TSNE from 'tsne-js'; -// import { TSNEConfig, TSNEData } from 'types/machineLearning'; - -// export function toD3Tsne(tsne) { -// const data: TSNEData = { -// width: 800, -// height: 800, -// dataset: [], -// }; -// data.dataset = tsne.map((t) => { -// return { -// x: (data.width * (t[0] + 1.0)) / 2, -// y: (data.height * (t[1] + 1.0)) / 2, -// }; -// }); - -// return data; -// } - -// export function toTSNE(denseInput: Array>, config: TSNEConfig) { -// if (!denseInput || denseInput.length < 1) { -// return null; -// } - -// const model = new TSNE(config); - -// model.init({ -// data: denseInput, -// type: 'dense', -// }); - -// // `error`, `iter`: final error and iteration number -// // note: computation-heavy action happens here -// model.run(); - -// // `outputScaled` is `output` scaled to a range of [-1, 1] -// return model.getOutputScaled(); -// } - -export {}; diff --git a/web/apps/photos/src/utils/storage/mlStorage.ts b/web/apps/photos/src/utils/storage/mlStorage.ts deleted file mode 100644 index 31fc38fab5..0000000000 --- a/web/apps/photos/src/utils/storage/mlStorage.ts +++ /dev/null @@ -1,97 +0,0 @@ -import localForage from "localforage"; -import { EnteFile } from "types/file"; -import { - Face, - MlFileData, - MLIndex, - MLSyncContext, -} from "types/machineLearning"; - -export const mlFilesStore = localForage.createInstance({ - driver: localForage.INDEXEDDB, - name: "ml-data", - version: 1.0, - storeName: "files", -}); - -export const mlPeopleStore = localForage.createInstance({ - driver: localForage.INDEXEDDB, - name: "ml-data", - version: 1.0, - storeName: "people", -}); - -export const mlLibraryStore = localForage.createInstance({ - driver: localForage.INDEXEDDB, - name: "ml-data", - version: 1.0, - storeName: "library", -}); - -export const mlVersionStore = localForage.createInstance({ - driver: localForage.INDEXEDDB, - name: "ml-data", - version: 1.0, - storeName: "versions", -}); - -export async function clearMLStorage() { - await mlFilesStore.clear(); - await mlPeopleStore.clear(); - await mlLibraryStore.clear(); - await mlVersionStore.clear(); -} - -export async function getIndexVersion(index: MLIndex): Promise { - return ((await mlVersionStore.getItem(`${index}`)) as number) || 0; -} - -export async function setIndexVersion( - index: MLIndex, - version: number, -): Promise { - await mlVersionStore.setItem(`${index}`, version); - - return version; -} - -export async function incrementIndexVersion(index: MLIndex): Promise { - let currentVersion = await getIndexVersion(index); - currentVersion = currentVersion + 1; - await setIndexVersion(index, currentVersion); - - return currentVersion; -} - -export async function isVersionOutdated(index: MLIndex, thanIndex: MLIndex) { - const indexVersion = await getIndexVersion(index); - const thanIndexVersion = await getIndexVersion(thanIndex); - - return indexVersion < thanIndexVersion; -} - -export function newMlData( - syncContext: MLSyncContext, - enteFile: EnteFile, -): MlFileData { - return { - fileId: enteFile.id, - imageSource: syncContext.config.imageSource, - faceDetectionMethod: syncContext.faceDetectionService.method, - faceCropMethod: syncContext.faceCropService.method, - faceAlignmentMethod: syncContext.faceAlignmentService.method, - faceEmbeddingMethod: syncContext.faceEmbeddingService.method, - errorCount: 0, - mlVersion: 0, - }; -} - -export async function getAllFacesMap() { - const allSyncedFacesMap = new Map>(); - await mlFilesStore.iterate((mlFileData: MlFileData) => { - mlFileData.faces && - allSyncedFacesMap.set(mlFileData.fileId, mlFileData.faces); - }); - - return allSyncedFacesMap; -}