use context to track video ref so that toolbar can also save current frame

This commit is contained in:
Mary Hipp
2025-08-25 15:51:54 -04:00
committed by psychedelicious
parent 01563eab3b
commit d6a3e7b7a4
9 changed files with 368 additions and 133 deletions

View File

@@ -0,0 +1,153 @@
import { Button, Divider, Flex, IconButton, Menu, MenuButton, MenuList } from '@invoke-ai/ui-library';
import { useStore } from '@nanostores/react';
import { logger } from 'app/logging/logger';
import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
import { DeleteImageButton } from 'features/deleteImageModal/components/DeleteImageButton';
import SingleSelectionVideoMenuItems from 'features/gallery/components/ContextMenu/SingleSelectionVideoMenuItems';
import { boardIdSelected } from 'features/gallery/store/gallerySlice';
import { toast } from 'features/toast/toast';
import { navigationApi } from 'features/ui/layouts/navigation-api';
import { useGalleryPanel } from 'features/ui/layouts/use-gallery-panel';
import { selectActiveTab } from 'features/ui/store/uiSelectors';
import { useVideoViewerContext } from 'features/video/context/VideoViewerContext';
import { useCaptureVideoFrame } from 'features/video/hooks/useCaptureVideoFrame';
import { memo, useCallback, useState } from 'react';
import { flushSync } from 'react-dom';
import { useTranslation } from 'react-i18next';
import { PiCameraBold, PiCrosshairBold, PiDotsThreeOutlineFill, PiSpinnerBold } from 'react-icons/pi';
import { serializeError } from 'serialize-error';
import { uploadImage } from 'services/api/endpoints/images';
import { useDeleteVideosMutation } from 'services/api/endpoints/videos';
import type { VideoDTO } from 'services/api/types';
const log = logger('video');
export const CurrentVideoButtons = memo(({ videoDTO }: { videoDTO: VideoDTO }) => {
const { t } = useTranslation();
const tab = useAppSelector(selectActiveTab);
const dispatch = useAppDispatch();
const activeTab = useAppSelector(selectActiveTab);
const galleryPanel = useGalleryPanel(activeTab);
const [deleteVideos] = useDeleteVideosMutation();
// Video frame capture functionality
const { $videoRef } = useVideoViewerContext();
const videoRef = useStore($videoRef);
const { captureFrame } = useCaptureVideoFrame();
const [capturing, setCapturing] = useState(false);
const locateInGallery = useCallback(() => {
navigationApi.expandRightPanel();
galleryPanel.expand();
flushSync(() => {
dispatch(
boardIdSelected({
boardId: videoDTO.board_id ?? 'none',
select: {
selection: [{ type: 'video', id: videoDTO.video_id }],
galleryView: 'videos',
},
})
);
});
}, [dispatch, galleryPanel, videoDTO]);
const handleDelete = useCallback(() => {
deleteVideos({ video_ids: [videoDTO.video_id] });
}, [deleteVideos, videoDTO]);
const onClickSaveFrame = useCallback(async () => {
setCapturing(true);
let file: File;
try {
if (!videoRef) {
toast({
status: 'error',
title: 'Video not ready',
description: 'Please wait for the video to load before capturing a frame.',
});
return;
}
file = captureFrame(videoRef);
await uploadImage({ file, image_category: 'user', is_intermediate: false, silent: true });
toast({
status: 'success',
title: 'Frame saved to assets tab',
});
} catch (error) {
log.error({ error: serializeError(error as Error) }, 'Failed to capture frame');
toast({
status: 'error',
title: 'Failed to capture frame',
description: 'There was an error capturing the current video frame.',
});
} finally {
setCapturing(false);
}
}, [captureFrame, videoRef]);
const doesTabHaveGallery = tab === 'canvas' || tab === 'generate' || tab === 'workflows' || tab === 'upscaling';
// const recallAll = useRecallAll(imageDTO);
// const recallRemix = useRecallRemix(imageDTO);
// const recallPrompts = useRecallPrompts(imageDTO);
// const recallSeed = useRecallSeed(imageDTO);
// const recallDimensions = useRecallDimensions(imageDTO);
// const loadWorkflow = useLoadWorkflow(imageDTO);
// const editImage = useEditImage(imageDTO);
// const deleteImage = useDeleteImage(imageDTO);
return (
<>
<Menu isLazy>
<MenuButton
as={IconButton}
aria-label={t('parameters.imageActions')}
tooltip={t('parameters.imageActions')}
isDisabled={!videoDTO}
variant="link"
alignSelf="stretch"
icon={<PiDotsThreeOutlineFill />}
/>
<MenuList>{videoDTO && <SingleSelectionVideoMenuItems videoDTO={videoDTO} />}</MenuList>
</Menu>
<Divider orientation="vertical" h={8} mx={2} />
<Button
leftIcon={capturing ? <PiSpinnerBold /> : <PiCameraBold />}
onClick={onClickSaveFrame}
isDisabled={capturing || !videoRef}
variant="link"
size="sm"
alignSelf="stretch"
px={2}
isLoading={capturing}
loadingText="Capturing..."
>
{capturing ? 'Capturing...' : 'Save Current Frame'}
</Button>
<Divider orientation="vertical" h={8} mx={2} />
{doesTabHaveGallery && (
<IconButton
icon={<PiCrosshairBold />}
aria-label={t('boards.locateInGalery')}
tooltip={t('boards.locateInGalery')}
onClick={locateInGallery}
variant="link"
size="sm"
alignSelf="stretch"
/>
)}
<Divider orientation="vertical" h={8} mx={2} />
<DeleteImageButton onClick={handleDelete} />
</>
);
});
CurrentVideoButtons.displayName = 'CurrentVideoButtons';

View File

@@ -3,7 +3,7 @@ import { useAppSelector } from 'app/store/storeHooks';
import VideoMetadataViewer from 'features/gallery/components/ImageMetadataViewer/VideoMetadataViewer';
import NextPrevItemButtons from 'features/gallery/components/NextPrevItemButtons';
import { selectShouldShowItemDetails } from 'features/ui/store/uiSelectors';
import { VideoViewer } from 'features/video/components/VideoViewer';
import { VideoView } from 'features/video/components/VideoView';
import type { AnimationProps } from 'framer-motion';
import { AnimatePresence, motion } from 'framer-motion';
import { memo, useCallback, useRef, useState } from 'react';
@@ -39,7 +39,7 @@ export const CurrentVideoPreview = memo(({ videoDTO }: { videoDTO: VideoDTO | nu
>
{videoDTO && videoDTO.video_url && (
<Flex w="full" h="full" position="absolute" alignItems="center" justifyContent="center">
<VideoViewer />
<VideoView />
</Flex>
)}
{!videoDTO && <NoContentForViewer />}

View File

@@ -1,6 +1,7 @@
import { Divider, Flex } from '@invoke-ai/ui-library';
import { useAppSelector } from 'app/store/storeHooks';
import { selectLastSelectedItem } from 'features/gallery/store/gallerySelectors';
import { VideoViewerContextProvider } from 'features/video/context/VideoViewerContext';
import { memo } from 'react';
import { useVideoDTO } from 'services/api/endpoints/videos';
@@ -10,14 +11,17 @@ import { VideoViewerToolbar } from './VideoViewerToolbar';
export const VideoViewer = memo(() => {
const lastSelectedItem = useAppSelector(selectLastSelectedItem);
const videoDTO = useVideoDTO(lastSelectedItem?.type === 'video' ? lastSelectedItem.id : null);
return (
<Flex flexDir="column" w="full" h="full" overflow="hidden" gap={2} position="relative">
<VideoViewerToolbar />
<Divider />
<Flex w="full" h="full" position="relative">
<CurrentVideoPreview videoDTO={videoDTO ?? null} />
<VideoViewerContextProvider>
<Flex flexDir="column" w="full" h="full" overflow="hidden" gap={2} position="relative">
<VideoViewerToolbar />
<Divider />
<Flex w="full" h="full" position="relative">
<CurrentVideoPreview videoDTO={videoDTO ?? null} />
</Flex>
</Flex>
</Flex>
</VideoViewerContextProvider>
);
});

View File

@@ -1,16 +1,19 @@
import { Flex } from '@invoke-ai/ui-library';
import { Flex, Spacer } from '@invoke-ai/ui-library';
import { useAppSelector } from 'app/store/storeHooks';
import { ToggleMetadataViewerButton } from 'features/gallery/components/ImageViewer/ToggleMetadataViewerButton';
import { selectLastSelectedItem } from 'features/gallery/store/gallerySelectors';
import { memo } from 'react';
import { useVideoDTO } from 'services/api/endpoints/videos';
import { CurrentVideoButtons } from './CurrentVideoButtons';
export const VideoViewerToolbar = memo(() => {
const lastSelectedItem = useAppSelector(selectLastSelectedItem);
const videoDTO = useVideoDTO(lastSelectedItem?.type === 'video' ? lastSelectedItem.id : null);
return (
<Flex w="full" justifyContent="flex-end" h={8}>
<Flex w="full" justifyContent="center" h={8}>
{videoDTO && <CurrentVideoButtons videoDTO={videoDTO} />}
{videoDTO && <ToggleMetadataViewerButton />}
</Flex>
);

View File

@@ -1,68 +1,32 @@
import { Flex, Icon, IconButton } from '@invoke-ai/ui-library';
import { logger } from 'app/logging/logger';
import { Flex } from '@invoke-ai/ui-library';
import { $authToken } from 'app/store/nanostores/authToken';
import { useVideoContextMenu } from 'features/gallery/components/ContextMenu/VideoContextMenu';
import { toast } from 'features/toast/toast';
import { useCaptureVideoFrame } from 'features/video/hooks/useCaptureVideoFrame';
import {
MediaControlBar,
MediaController,
MediaFullscreenButton,
MediaPlayButton,
MediaTimeDisplay,
MediaTimeRange,
} from 'media-chrome/react';
import type { CSSProperties } from 'react';
import { memo, useCallback, useRef, useState } from 'react';
import { PiArrowsOutBold, PiCameraBold, PiPauseFill, PiPlayFill, PiSpinnerBold } from 'react-icons/pi';
import { useVideoViewerContext } from 'features/video/context/VideoViewerContext';
import type { MediaStateOwner } from 'media-chrome/dist/media-store/state-mediator.js';
import { MediaController } from 'media-chrome/react';
import { memo, useCallback, useRef } from 'react';
import ReactPlayer from 'react-player';
import { serializeError } from 'serialize-error';
import { uploadImage } from 'services/api/endpoints/images';
import type { VideoDTO } from 'services/api/types';
const NoHoverBackground = {
'--media-control-hover-background': 'transparent',
'--media-text-color': 'base.200',
'--media-font-size': '12px',
} as CSSProperties;
import { VideoPlayerControls } from './VideoPlayerControls';
const log = logger('video');
interface VideoPlayerProps {
videoDTO: VideoDTO;
}
export const VideoPlayer = memo(({ videoDTO }: { videoDTO: VideoDTO }) => {
export const VideoPlayer = memo(({ videoDTO }: VideoPlayerProps) => {
const ref = useRef<HTMLDivElement>(null);
const reactPlayerRef = useRef<HTMLVideoElement>(null);
const [capturing, setCapturing] = useState(false);
const { captureFrame } = useCaptureVideoFrame();
const onClick = useCallback(async () => {
setCapturing(true);
let file: File;
try {
const reactPlayer = reactPlayerRef.current;
if (!reactPlayer) {
return;
}
file = captureFrame(reactPlayer);
await uploadImage({ file, image_category: 'user', is_intermediate: false, silent: true });
toast({
status: 'success',
title: 'Frame saved to assets tab',
});
} catch (error) {
log.error({ error: serializeError(error as Error) }, 'Failed to capture frame');
toast({
status: 'error',
title: 'Failed to capture frame',
});
} finally {
setCapturing(false);
}
}, [captureFrame]);
const reactPlayerRef = useRef<MediaStateOwner | null>(null);
useVideoContextMenu(videoDTO, ref);
const { setVideoRef } = useVideoViewerContext();
const handleMediaRef = useCallback(
(mediaEl: MediaStateOwner | null | undefined) => {
reactPlayerRef.current = mediaEl || null;
setVideoRef(mediaEl || null);
},
[setVideoRef]
);
return (
<Flex ref={ref} w="full" h="full" flexDirection="column" gap={4} alignItems="center" justifyContent="center">
@@ -74,7 +38,7 @@ export const VideoPlayer = memo(({ videoDTO }: { videoDTO: VideoDTO }) => {
>
<ReactPlayer
slot="media"
ref={reactPlayerRef}
ref={handleMediaRef}
src={videoDTO.video_url}
controls={false}
width={videoDTO.width}
@@ -82,32 +46,8 @@ export const VideoPlayer = memo(({ videoDTO }: { videoDTO: VideoDTO }) => {
pip={false}
crossOrigin={$authToken.get() ? 'use-credentials' : 'anonymous'}
/>
<MediaControlBar>
<MediaPlayButton noTooltip={true} style={NoHoverBackground}>
<span slot="play">
<Icon as={PiPlayFill} boxSize={6} color="base.200" />
</span>
<span slot="pause">
<Icon as={PiPauseFill} boxSize={6} color="base.200" />
</span>
</MediaPlayButton>
<MediaTimeRange style={NoHoverBackground} />
<MediaTimeDisplay showDuration style={NoHoverBackground} noToggle={true} />
<MediaFullscreenButton noTooltip={true} style={NoHoverBackground}>
<span slot="icon">
<Icon as={PiArrowsOutBold} boxSize={6} color="base.200" />
</span>
</MediaFullscreenButton>
<IconButton
tooltip={capturing ? 'Capturing...' : 'Save Current Frame as Asset'}
icon={<Icon as={capturing ? PiSpinnerBold : PiCameraBold} boxSize={6} color="base.200" />}
size="lg"
variant="unstyled"
onClick={onClick}
aria-label="Save Current Frame"
/>
</MediaControlBar>
<VideoPlayerControls />
</MediaController>
</Flex>
);

View File

@@ -0,0 +1,87 @@
import { Icon, IconButton } from '@invoke-ai/ui-library';
import { logger } from 'app/logging/logger';
import { toast } from 'features/toast/toast';
import { useVideoViewerContext } from 'features/video/context/VideoViewerContext';
import { useCaptureVideoFrame } from 'features/video/hooks/useCaptureVideoFrame';
import {
MediaControlBar,
MediaFullscreenButton,
MediaPlayButton,
MediaTimeDisplay,
MediaTimeRange,
} from 'media-chrome/react';
import type { CSSProperties } from 'react';
import { useCallback, useState } from 'react';
import { PiArrowsOutBold, PiCameraBold, PiPauseFill, PiPlayFill, PiSpinnerBold } from 'react-icons/pi';
import { serializeError } from 'serialize-error';
import { uploadImage } from 'services/api/endpoints/images';
const log = logger('video');
const NoHoverBackground = {
'--media-control-hover-background': 'transparent',
'--media-text-color': 'base.200',
'--media-font-size': '12px',
} as CSSProperties;
export const VideoPlayerControls = () => {
const { captureFrame } = useCaptureVideoFrame();
const [capturing, setCapturing] = useState(false);
const { $videoRef } = useVideoViewerContext();
const onClickSaveFrame = useCallback(async () => {
setCapturing(true);
let file: File;
try {
const videoRef = $videoRef.get();
if (!videoRef) {
return;
}
file = captureFrame(videoRef);
await uploadImage({ file, image_category: 'user', is_intermediate: false, silent: true });
toast({
status: 'success',
title: 'Frame saved to assets tab',
});
} catch (error) {
log.error({ error: serializeError(error as Error) }, 'Failed to capture frame');
toast({
status: 'error',
title: 'Failed to capture frame',
});
} finally {
setCapturing(false);
}
}, [captureFrame, $videoRef]);
return (
<MediaControlBar>
<MediaPlayButton noTooltip={true} style={NoHoverBackground}>
<span slot="play">
<Icon as={PiPlayFill} boxSize={6} color="base.200" />
</span>
<span slot="pause">
<Icon as={PiPauseFill} boxSize={6} color="base.200" />
</span>
</MediaPlayButton>
<MediaTimeRange style={NoHoverBackground} />
<MediaTimeDisplay showDuration style={NoHoverBackground} noToggle={true} />
<MediaFullscreenButton noTooltip={true} style={NoHoverBackground}>
<span slot="icon">
<Icon as={PiArrowsOutBold} boxSize={6} color="base.200" />
</span>
</MediaFullscreenButton>
<IconButton
tooltip={capturing ? 'Capturing...' : 'Save Current Frame as Asset'}
icon={<Icon as={capturing ? PiSpinnerBold : PiCameraBold} boxSize={6} color="base.200" />}
size="lg"
variant="unstyled"
onClick={onClickSaveFrame}
aria-label="Save Current Frame"
isDisabled={capturing}
/>
</MediaControlBar>
);
};

View File

@@ -7,7 +7,7 @@ import { useVideoDTO } from 'services/api/endpoints/videos';
import { VideoPlayer } from './VideoPlayer';
export const VideoViewer = () => {
export const VideoView = () => {
const ref = useRef<HTMLDivElement>(null);
const lastSelectedItem = useAppSelector(selectLastSelectedItem);
const videoDTO = useVideoDTO(lastSelectedItem?.id);

View File

@@ -0,0 +1,40 @@
import type { MediaStateOwner } from 'media-chrome/dist/media-store/state-mediator.js';
import { type Atom, atom } from 'nanostores';
import type { PropsWithChildren } from 'react';
import { createContext, memo, useCallback, useContext, useMemo, useState } from 'react';
import { assert } from 'tsafe';
type VideoViewerContextValue = {
$videoRef: Atom<MediaStateOwner | null>;
setVideoRef: (ref: MediaStateOwner | null) => void;
onLoadVideo: () => void;
};
const VideoViewerContext = createContext<VideoViewerContextValue | null>(null);
export const VideoViewerContextProvider = memo((props: PropsWithChildren) => {
const $videoRef = useState(() => atom<MediaStateOwner | null>(null))[0];
const setVideoRef = useCallback(
(ref: MediaStateOwner | null) => {
$videoRef.set(ref);
},
[$videoRef]
);
const onLoadVideo = useCallback(() => {
// Reset video ref when loading new video
$videoRef.set(null);
}, [$videoRef]);
const value = useMemo(() => ({ $videoRef, setVideoRef, onLoadVideo }), [$videoRef, setVideoRef, onLoadVideo]);
return <VideoViewerContext.Provider value={value}>{props.children}</VideoViewerContext.Provider>;
});
VideoViewerContextProvider.displayName = 'VideoViewerContextProvider';
export const useVideoViewerContext = () => {
const value = useContext(VideoViewerContext);
assert(value !== null, 'useVideoViewerContext must be used within a VideoViewerContextProvider');
return value;
};

View File

@@ -1,57 +1,65 @@
import { useCallback, useRef } from 'react';
import { useStore } from '@nanostores/react';
import { useVideoViewerContext } from 'features/video/context/VideoViewerContext';
import type { MediaStateOwner } from 'media-chrome/dist/media-store/state-mediator.js';
import { useCallback } from 'react';
export const useCaptureVideoFrame = () => {
const videoRef = useRef<HTMLVideoElement>(null);
const { $videoRef } = useVideoViewerContext();
const videoRef = useStore($videoRef);
const captureFrame = useCallback((video: HTMLVideoElement): File => {
// Validate video element
if (!video || video.videoWidth === 0 || video.videoHeight === 0) {
throw new Error('Invalid video element or video not loaded');
}
const captureFrame = useCallback(
(video?: MediaStateOwner): File => {
// Use provided video or fall back to context video ref
const targetVideo = video || videoRef;
// Validate video element
if (!targetVideo || targetVideo.videoWidth === 0 || targetVideo.videoHeight === 0) {
throw new Error('Invalid video element or video not loaded');
}
// Check if video is ready for capture
if (video.readyState < 2) {
// HAVE_CURRENT_DATA = 2
throw new Error('Video is not ready for frame capture');
}
// Check if video is ready for capture
if (targetVideo.readyState && targetVideo.readyState < 2) {
// HAVE_CURRENT_DATA = 2
throw new Error('Video is not ready for frame capture');
}
const canvas = document.createElement('canvas');
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
const canvas = document.createElement('canvas');
canvas.width = targetVideo.videoWidth || 0;
canvas.height = targetVideo.videoHeight || 0;
const context = canvas.getContext('2d');
if (!context) {
throw new Error('Failed to get canvas 2D context');
}
const context = canvas.getContext('2d');
if (!context) {
throw new Error('Failed to get canvas 2D context');
}
// Draw the current video frame to canvas
context.drawImage(video, 0, 0);
// Draw the current video frame to canvas
context.drawImage(targetVideo as HTMLVideoElement, 0, 0);
// Convert to data URL with proper format
const dataUri = canvas.toDataURL('image/png', 0.92);
const data = dataUri.split(',')[1];
const mimeType = dataUri.split(';')[0]?.slice(5);
// Convert to data URL with proper format
const dataUri = canvas.toDataURL('image/png', 0.92);
const data = dataUri.split(',')[1];
const mimeType = dataUri.split(';')[0]?.slice(5);
if (!data || !mimeType) {
throw new Error('Failed to extract image data from canvas');
}
if (!data || !mimeType) {
throw new Error('Failed to extract image data from canvas');
}
// Convert to blob
const bytes = window.atob(data);
const buf = new ArrayBuffer(bytes.length);
const arr = new Uint8Array(buf);
// Convert to blob
const bytes = window.atob(data);
const buf = new ArrayBuffer(bytes.length);
const arr = new Uint8Array(buf);
for (let i = 0; i < bytes.length; i++) {
arr[i] = bytes.charCodeAt(i);
}
for (let i = 0; i < bytes.length; i++) {
arr[i] = bytes.charCodeAt(i);
}
const blob = new Blob([arr], { type: mimeType });
const file = new File([blob], 'frame.png', { type: mimeType });
return file;
}, []);
const blob = new Blob([arr], { type: mimeType });
const file = new File([blob], 'frame.png', { type: mimeType });
return file;
},
[videoRef]
);
return {
captureFrame,
videoRef,
};
};