import 'package:logging/logging.dart'; import "package:photos/core/configuration.dart"; import 'package:photos/core/network/network.dart'; import 'package:photos/db/files_db.dart'; import 'package:photos/models/duplicate_files.dart'; import 'package:photos/models/file/file.dart'; import "package:photos/models/file/file_type.dart"; import "package:photos/services/collections_service.dart"; import "package:photos/services/files_service.dart"; class DeduplicationService { final _logger = Logger("DeduplicationService"); final _enteDio = NetworkClient.instance.enteDio; DeduplicationService._privateConstructor(); static final DeduplicationService instance = DeduplicationService._privateConstructor(); Future> getDuplicateFiles() async { try { final List result = await _getDuplicateFiles(); return result; } catch (e, s) { _logger.severe("failed to get dedupeFile", e, s); rethrow; } } // Returns a list of DuplicateFiles, where each DuplicateFiles object contains // a list of files that have the same hash Future> _getDuplicateFiles() async { Map uploadIDToSize = {}; final bool hasFileSizes = await FilesService.instance.hasMigratedSizes(); if (!hasFileSizes) { final DuplicateFilesResponse dupes = await _fetchDuplicateFileIDs(); uploadIDToSize = dupes.toUploadIDToSize(); } final Set allowedCollectionIDs = CollectionsService.instance.nonHiddenOwnedCollections(); final List allFiles = await FilesDB.instance.getAllFilesFromDB( CollectionsService.instance.getHiddenCollectionIds(), dedupeByUploadId: false, ); final int ownerID = Configuration.instance.getUserID()!; final List filteredFiles = []; for (final file in allFiles) { if (!file.isUploaded || (file.hash ?? '').isEmpty || (file.ownerID ?? 0) != ownerID || (!allowedCollectionIDs.contains(file.collectionID!))) { continue; } if ((file.fileSize ?? 0) <= 0) { file.fileSize = uploadIDToSize[file.uploadedFileID!] ?? 0; } if ((file.fileSize ?? 0) <= 0) { continue; } filteredFiles.add(file); } final Map> sizeHashToFilesMap = {}; final Map> sizeHashToCollectionsSet = {}; final Map> livePhotoHashToFilesMap = {}; final Map> livePhotoHashToCollectionsSet = {}; final Set processedFileIds = {}; for (final file in filteredFiles) { // Note: For live photos, the zipped file size could be different if // the files were uploaded from different devices. So, we dedupe live // photos based on hash only. if (file.fileType == FileType.livePhoto) { final key = '${file.hash}'; if (!livePhotoHashToFilesMap.containsKey(key)) { livePhotoHashToFilesMap[key] = []; livePhotoHashToCollectionsSet[key] = {}; } livePhotoHashToCollectionsSet[key]!.add(file.collectionID!); if (!processedFileIds.contains(file.uploadedFileID)) { livePhotoHashToFilesMap[key]!.add(file); processedFileIds.add(file.uploadedFileID!); } } else { final key = '${file.fileSize}-${file.hash}'; if (!sizeHashToFilesMap.containsKey(key)) { sizeHashToFilesMap[key] = []; sizeHashToCollectionsSet[key] = {}; } sizeHashToCollectionsSet[key]!.add(file.collectionID!); if (!processedFileIds.contains(file.uploadedFileID)) { sizeHashToFilesMap[key]!.add(file); processedFileIds.add(file.uploadedFileID!); } } } final List dupesByHash = []; for (final key in sizeHashToFilesMap.keys) { final List files = sizeHashToFilesMap[key]!; final Set collectionIds = sizeHashToCollectionsSet[key]!; if (files.length > 1) { final size = files[0].fileSize!; dupesByHash.add(DuplicateFiles(files, size, collectionIds)); } } for (final key in livePhotoHashToFilesMap.keys) { final List files = livePhotoHashToFilesMap[key]!; final Set collectionIds = livePhotoHashToCollectionsSet[key]!; if (files.length > 1 && (files.first.fileSize ?? 0) > 0) { dupesByHash .add(DuplicateFiles(files, files.first.fileSize!, collectionIds)); } } return dupesByHash; } Future _fetchDuplicateFileIDs() async { final response = await _enteDio.get("/files/duplicates"); return DuplicateFilesResponse.fromMap(response.data); } }