import log from "@/base/log"; import type { FileInfoExif } from "@/gallery/components/FileInfo"; import { downloadManager, type LivePhotoSourceURL, } from "@/gallery/services/download"; import { extractRawExif, parseExif } from "@/gallery/services/exif"; import type { EnteFile } from "@/media/file"; import { FileType } from "@/media/file-type"; import { ensureString } from "@/utils/ensure"; /** * This is a subset of the fields expected by PhotoSwipe itself (see the * {@link SlideData} type exported by PhotoSwipe). */ interface PhotoSwipeSlideData { /** * The image URL expected by PhotoSwipe. * * This is set to the URL of the image that should be shown in the image * viewer component provided by PhotoSwipe. * * It will be a renderable (i.e., possibly converted) object URL obtained * from the current "best" image we have. * * For example, if all we have is the thumbnail, then this'll be an * renderable object URL obtained from the thumbnail data. Then later when * we fetch the original image, this'll be the renderable object URL derived * from the original. But if it is a video, this will just be cleared. */ src?: string | undefined; /** * The width (in pixels) of the {@link src} image. */ width?: number | undefined; /** * The height (in pixels) of the {@link src} image. */ height?: number | undefined; } /** * The data returned by the flagship {@link itemDataForFile} function provided * by the file viewer data source module. * * This is the minimal data expected by PhotoSwipe, plus some fields we use * ourselves in the custom scaffolding we have around PhotoSwipe. */ export type ItemData = PhotoSwipeSlideData & { /** * The ID of the {@link EnteFile} whose data we are. */ fileID: number; /** * The {@link EnteFile} type of the file whose data we are. */ fileType: FileType; /** * The renderable object URL of the image associated with the file. * * - For images, this will be the object URL of a renderable image. * - For videos, this will not be defined. * - For live photos, this will be a renderable object URL of the image * portion of the live photo. */ imageURL?: string; /** * The original image associated with the file, as a Blob. * * - For images, this will be the original image itself. * - For live photos, this will be the image component of the live photo. * - For videos, this will be not be present. */ originalImageBlob?: Blob; /** * The renderable object URL of the video associated with the file. * * - For images, this will not be defined. * - For videos, this will be the object URL of a renderable video. * - For live photos, this will be a renderable object URL of the video * portion of the live photo. */ videoURL?: string; /** * `true` if we should indicate to the user that we're still fetching data * for this file. * * Note that this doesn't imply that the data is final. e.g. for a live * photo, this will be not be set after we get the original image component, * but the fetch for the video component might still be ongoing. */ isContentLoading?: boolean; /** * This will be explicitly set to `false` when we want to disable * PhotoSwipe's built in image zoom. * * It is set while the thumbnail is loaded. */ isContentZoomable?: boolean; /** * This will be `true` if the fetch for the file's data has failed. * * It is possible for this to be set in tandem with the content URLs also * being set (e.g. if we were able to use the cached thumbnail, but the * original file could not be fetched because of an network error). */ fetchFailed?: boolean; }; /** * This module stores and serves data required by our custom PhotoSwipe * instance, effectively acting as an in-memory cache. * * By keeping this independent of the lifetime of the PhotoSwipe instance, we * can reuse the same cache for multiple displays of our file viewer. * * This will be cleared on logout. */ class FileViewerDataSourceState { /** * The best data we have for a particular file (ID). */ itemDataByFileID = new Map(); /** * The latest callback registered for notifications of better data being * available for a particular file (ID). */ needsRefreshByFileID = new Map void>(); /** * The exif data we have for a particular file (ID). */ fileInfoExifByFileID = new Map(); /** * The latest callback registered for notifications of exif data being * available for a particular file (ID). */ exifObserverByFileID = new Map void>(); } /** * State shared by functions in this module. * * See {@link FileViewerDataSourceState}. */ let _state = new FileViewerDataSourceState(); /** * Clear any internal state maintained by the file viewer data source. */ // TODO(PS): Call me during logout sequence once this is integrated. export const logoutFileViewerDataSource = () => { _state = new FileViewerDataSourceState(); }; /** * Return the best available ItemData for rendering the given {@link file}. * * If an entry does not exist for a particular file, then it is lazily added on * demand, and updated as we keep getting better data (thumbnail, original) for * the file. * * At each step, we call the provided callback so that file viewer can call us * again to get the updated data. * * --- * * Detailed flow: * * If we already have the final data about the file, then this function will * return it and do nothing subsequently. * * Otherwise, it will: * * 1. Return empty slide data; PhotoSwipe will not show anything in the image * area but will otherwise render UI controls properly (in most cases a * cached renderable thumbnail URL will be available shortly). * * 2. Insert this empty data in its cache so that we don't enqueue multiple * updates. * * Then it we start fetching data for the file. * * First it'll fetch the thumbnail. Once that is done, it'll update the data it * has cached, and notify the caller (using the provided callback) so it can * refresh the slide. * * Then it'll continue fetching the original. * * - For images and videos, this will be the single original. * * - For live photos, this will also be a two step process, first fetching the * original image, then again the video component. * * At this point, the data for this file will be considered final, and * subsequent calls for the same file will return this same value unless it is * invalidated. * * If at any point an error occurs, we reset our cache for this file so that the * next time the data is requested we repeat the process instead of continuing * to serve the incomplete result. */ export const itemDataForFile = (file: EnteFile, needsRefresh: () => void) => { const fileID = file.id; const fileType = file.metadata.fileType; let itemData = _state.itemDataByFileID.get(fileID); // We assume that there is only one file viewer that is using us at a given // point of time. This assumption is currently valid. _state.needsRefreshByFileID.set(file.id, needsRefresh); if (!itemData) { itemData = { fileID, fileType, isContentLoading: true }; _state.itemDataByFileID.set(file.id, itemData); void enqueueUpdates(file); } return itemData; }; /** * Forget item data for the given {@link file} if its fetch had failed. * * This is called when the user moves away from a slide so that we attempt a * full retry when they come back the next time. */ export const forgetFailedItemDataForFileID = (fileID: number) => { if (_state.itemDataByFileID.get(fileID)?.fetchFailed) { _state.itemDataByFileID.delete(fileID); } }; /** * Forget item data for the all files whose fetch had failed. * * This is called when the user closes the file viewer so that we attempt a full * retry when they reopen the viewer the next time. */ export const forgetFailedItems = () => [..._state.itemDataByFileID.keys()].forEach(forgetFailedItemDataForFileID); const enqueueUpdates = async (file: EnteFile) => { const fileID = file.id; const fileType = file.metadata.fileType; const update = (itemData: Partial) => { _state.itemDataByFileID.set(file.id, { ...itemData, fileType, fileID }); _state.needsRefreshByFileID.get(file.id)?.(); }; // Use the last best available data, but stop showing the loading indicator // and instead show the error indicator. const markFailed = () => { const lastData: Partial = _state.itemDataByFileID.get(file.id) ?? {}; delete lastData.isContentLoading; update({ ...lastData, fetchFailed: true }); }; try { const thumbnailURL = await downloadManager.renderableThumbnailURL(file); // While the types don't reflect it, it is safe to use the ! (null // assertion) here since renderableThumbnailURL can throw but will not // return undefined by default. const thumbnailData = await withDimensions(ensureString(thumbnailURL)); update({ ...thumbnailData, isContentLoading: true, isContentZoomable: false, }); } catch (e) { // If we can't even get the thumbnail, then a network error is likely // (download manager already has retries); in particular, it cannot be a // format error since thumbnails are already standard JPEGs. // // Notify the user of the error. The entire process will be retried when // they reopen the slide later. // // See: [Note: File viewer error handling] log.error("Failed to fetch thumbnail", e); markFailed(); return; } try { switch (fileType) { case FileType.image: { const sourceURLs = await downloadManager.renderableSourceURLs(file); const imageURL = ensureString(sourceURLs.url); const originalImageBlob = sourceURLs.originalImageBlob!; const itemData = await withDimensions(imageURL); update({ ...itemData, imageURL, originalImageBlob }); break; } case FileType.video: { const sourceURLs = await downloadManager.renderableSourceURLs(file); // TODO(PS): update({ videoURL: sourceURLs.url as string }); break; } case FileType.livePhoto: { const sourceURLs = await downloadManager.renderableSourceURLs(file); const livePhotoSourceURLs = sourceURLs.url as LivePhotoSourceURL; const imageURL = ensureString( await livePhotoSourceURLs.image(), ); const originalImageBlob = livePhotoSourceURLs.originalImageBlob()!; const imageData = { ...(await withDimensions(imageURL)), imageURL, originalImageBlob, }; update(imageData); const videoURL = await livePhotoSourceURLs.video(); update({ ...imageData, videoURL }); break; } } } catch (e) { // [Note: File viewer error handling] // // Generally, file downloads will fail because of two reasons: Network // errors, or format errors. // // In the first case (network error), the `renderableSourceURLs` method // above will throw. We will show an error indicator icon in the UI, but // will keep showing the thumbnail. // // In the second case (format error), we'll get back a URL, but if a // file conversion was needed but not possible (say, it is an // unsupported format), we might have at our hands the original file's // untouched URL, that the browser might not know how to render. // // In this case we won't get into an error state here, but PhotoSwipe // will encounter an error when trying to render it, and it will show // our customized error message ("This file could not be previewed"), // but the user will still be able to download it. log.error("Failed to fetch file", e); markFailed(); } }; /** * Take a image URL, determine its dimensions using browser APIs, and return the URL * and its dimensions in a form that can directly be passed to PhotoSwipe as * {@link ItemData}. */ const withDimensions = (imageURL: string): Promise> => new Promise((resolve, reject) => { const image = new Image(); image.onload = () => resolve({ src: imageURL, width: image.naturalWidth, height: image.naturalHeight, }); image.onerror = reject; image.src = imageURL; }); /** * Return the cached Exif data for the given {@link file}. * * The shape of the returned data is such that it can directly be used by the * {@link FileInfo} sidebar. * * Exif extraction is not too expensive, and takes around 10-200 ms usually, so * this can be done preemptively. As soon as we get data for a particular item * as the user swipes through the file viewer, we extract its exif data using * {@link updateFileInfoExifIfNeeded}. * * Then if the user were to open the file info sidebar for that particular file, * the associated exif data will be returned by this function. Since the happy * path is for synchronous use in a React component, this function synchronously * returns the cached value (and the callback is never invoked). * * The user can open the file info sidebar before the original has been fetched, * so it is possible that this function gets called before * {@link updateFileInfoExifIfNeeded} has completed. In such cases, this * function will synchronously return `undefined`, and then later call the * provided {@link observer} once the extraction results are available. */ export const fileInfoExifForFile = ( file: EnteFile, observer: (exifData: FileInfoExif) => void, ) => { const fileID = file.id; const exifData = _state.fileInfoExifByFileID.get(fileID); if (exifData) return exifData; _state.exifObserverByFileID.set(fileID, observer); return undefined; }; /** * Update, if needed, the cached Exif data for with the given {@link itemData}. * * This function is expected to be called when an item is loaded as PhotoSwipe * content. It can be safely called multiple times - it will ignore calls until * the item has an associated {@link originalImageBlob}, and it will also ignore calls * that are made after exif data has already been extracted. * * If required, it will extract the exif data from the file, massage it to a * form suitable for use by {@link FileInfo}, and stash it in its caches, and * notify the most recent observer for that file attached via * {@link fileInfoExifForFile}. * * See also {@link forgetExifForItemData}. */ export const updateFileInfoExifIfNeeded = async (itemData: ItemData) => { const { fileID, fileType, originalImageBlob } = itemData; // We already have it available. if (_state.fileInfoExifByFileID.has(fileID)) return; const updateNotifyAndReturn = (exifData: FileInfoExif) => { _state.fileInfoExifByFileID.set(fileID, exifData); _state.exifObserverByFileID.get(fileID)?.(exifData); return exifData; }; // For videos, insert a placeholder. if (fileType === FileType.video) { return updateNotifyAndReturn(createPlaceholderFileInfoExif()); } // This is not a video, but the original image is not available yet. if (!originalImageBlob) return; try { const file = new File([originalImageBlob], ""); const tags = await extractRawExif(file); const parsed = parseExif(tags); return updateNotifyAndReturn({ tags, parsed }); } catch (e) { log.error("Failed to extract exif", e); // Save the empty placeholder exif corresponding to the file, no point // in unnecessarily retrying this, it will deterministically fail again. return updateNotifyAndReturn(createPlaceholderFileInfoExif()); } }; const createPlaceholderFileInfoExif = (): FileInfoExif => ({ tags: undefined, parsed: undefined, }); /** * Clear any cached {@link FileInfoExif} for the given {@link ItemData}. */ export const forgetExifForItemData = ({ fileID }: ItemData) => { _state.fileInfoExifByFileID.delete(fileID); _state.exifObserverByFileID.delete(fileID); }; /** * Clear all cached {@link FileInfoExif}. */ export const forgetExif = () => { _state.fileInfoExifByFileID.clear(); _state.exifObserverByFileID.clear(); };