feat(results): add live annotation overlay on raw video playback

2026-07-08 17:24:20 +05:30
parent 23186ffd29
commit 7a82e17bbb
13 changed files with 540 additions and 57 deletions
--- a/src/components/annotation/boundingBoxOverlay.tsx
+++ b/src/components/annotation/boundingBoxOverlay.tsx
@@ -1,8 +1,8 @@
 import { getDefectVisual } from '@/constants/defectVisualConfig';
-import type { DetectionResultItem } from '@/types';
+import type { BoundingBoxOverlayItem } from '@/types';

 interface BoundingBoxOverlayProps {
-  detections: DetectionResultItem[];
+  detections: BoundingBoxOverlayItem[];
  videoWidth: number;
  videoHeight: number;
 }
@@ -23,7 +23,7 @@ export default function BoundingBoxOverlay({
      viewBox={`0 0 ${videoWidth} ${videoHeight}`}
      preserveAspectRatio="xMidYMid meet"
    >
-      {detections.map(({ id, detection }) => {
+      {detections.map((detection) => {
        const { bounding_box: box } = detection;
        const color = getDefectVisual(detection.class_name).boundingBoxColor;
        const label = `${detection.display_name} ${(detection.confidence * 100).toFixed(0)}%`;
@@ -31,7 +31,7 @@ export default function BoundingBoxOverlay({
        const labelWidth = Math.max(150, label.length * 17);

        return (
-          <g key={id}>
+          <g key={detection.id}>
            <rect
              x={box.x1}
              y={box.y1}
--- a/src/components/video/annotatedDetectionImage.tsx
+++ b/src/components/video/annotatedDetectionImage.tsx
@@ -4,7 +4,7 @@ import { AlertTriangle, ImageIcon, Loader2 } from 'lucide-react';
 import Image from 'next/image';

 import { useProtectedMediaObjectUrl } from '@/hooks/useProtectedMedia';
-import type { DetectionResultItem } from '@/types';
+import type { BoundingBoxOverlayItem, DetectionResultItem } from '@/types';

 import BoundingBoxOverlay from '@/components/annotation/boundingBoxOverlay';

@@ -22,6 +22,17 @@ export default function AnnotatedDetectionImage({
  const { objectUrl, isLoading, isError } = useProtectedMediaObjectUrl(
    detection?.detection.context_image_url,
  );
+  const overlayDetections: BoundingBoxOverlayItem[] = detection
+    ? [
+        {
+          id: detection.id,
+          class_name: detection.detection.class_name,
+          display_name: detection.detection.display_name,
+          confidence: detection.detection.confidence,
+          bounding_box: detection.detection.bounding_box,
+        },
+      ]
+    : [];

  const aspectRatio =
    videoWidth > 0 && videoHeight > 0
@@ -58,7 +69,7 @@ export default function AnnotatedDetectionImage({
            className="object-contain"
          />
          <BoundingBoxOverlay
-            detections={[detection]}
+            detections={overlayDetections}
            videoWidth={videoWidth}
            videoHeight={videoHeight}
          />
--- a/src/components/video/annotatedVideoPlayer.tsx
+++ b/src/components/video/annotatedVideoPlayer.tsx
@@ -2,7 +2,10 @@

 import { RefObject, useEffect } from 'react';
 import { AlertTriangle, Loader2 } from 'lucide-react';
-import type { DetectionResultLog } from '@/types';
+import type {
+  DetectionResultLog,
+  VideoAnnotationFrameDetection,
+} from '@/types';
 import { useDetectionThumbnails } from './useDetectionThumbnails';
 import BoundingBoxOverlay from '@/components/annotation/boundingBoxOverlay';
 import {
@@ -21,7 +24,7 @@ import {
 interface AnnotatedVideoPlayerProps {
  videoRef: RefObject<MediaPlayerInstance | null>;
  logs: DetectionResultLog[];
-  visibleLogs: DetectionResultLog[];
+  visibleDetections: VideoAnnotationFrameDetection[];
  videoWidth: number;
  videoHeight: number;
  videoUrl: string | null;
@@ -60,7 +63,7 @@ function VideoFrameSync({
 export default function AnnotatedVideoPlayer({
  videoRef,
  logs,
-  visibleLogs,
+  visibleDetections,
  videoWidth,
  videoHeight,
  videoUrl,
@@ -92,7 +95,7 @@ export default function AnnotatedVideoPlayer({
          </MediaProvider>
          <VideoFrameSync onVideoFrame={onVideoFrame} />
          <BoundingBoxOverlay
-            detections={visibleLogs}
+            detections={visibleDetections}
            videoWidth={videoWidth}
            videoHeight={videoHeight}
          />
--- a/src/components/video/useVideoAnnotationPlayback.ts
+++ b/src/components/video/useVideoAnnotationPlayback.ts
@@ -0,0 +1,242 @@
+'use client';
+
+import { useQueryClient } from '@tanstack/react-query';
+import {
+  RefObject,
+  useCallback,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from 'react';
+import type { MediaPlayerInstance } from '@vidstack/react';
+
+import { useVideoAnnotationFramesQuery } from '@/app/(modules)/results/hooks/useVideoResults';
+import { videoKeys } from '@/app/(modules)/results/queries/videoKeys';
+import { videoService } from '@/services/api';
+import type {
+  DetectionResultLog,
+  VideoAnnotationFrameDetection,
+  VideoAnnotationFramesParams,
+  VideoAnnotationFramesResponse,
+} from '@/types';
+
+const ANNOTATION_WINDOW_MS = 30000;
+const ANNOTATION_PREFETCH_THRESHOLD_MS = 5000;
+
+const buildWindowParams = (
+  startTimeMs: number,
+): VideoAnnotationFramesParams => ({
+  start_time_ms: startTimeMs,
+  end_time_ms: startTimeMs + ANNOTATION_WINDOW_MS,
+});
+
+const getWindowStartMs = (currentTimeMs: number) =>
+  Math.floor(Math.max(currentTimeMs, 0) / ANNOTATION_WINDOW_MS) *
+  ANNOTATION_WINDOW_MS;
+
+interface UseVideoAnnotationPlaybackParams {
+  videoId: string | undefined;
+  logs: DetectionResultLog[];
+  fps: number;
+  videoRef: RefObject<MediaPlayerInstance | null>;
+}
+
+export function useVideoAnnotationPlayback({
+  videoId,
+  logs,
+  fps,
+  videoRef,
+}: UseVideoAnnotationPlaybackParams) {
+  const queryClient = useQueryClient();
+  const shouldPauseAfterSeekRef = useRef(false);
+  const visibleDetectionIdsRef = useRef('');
+  const [currentWindowStartMs, setCurrentWindowStartMs] = useState(0);
+  const [activeLog, setActiveLog] = useState<DetectionResultLog | undefined>();
+  const [visibleDetections, setVisibleDetections] = useState<
+    VideoAnnotationFrameDetection[]
+  >([]);
+
+  const sortedLogs = useMemo(
+    () =>
+      [...logs].sort(
+        (a, b) => a.frame.timestamp_seconds - b.frame.timestamp_seconds,
+      ),
+    [logs],
+  );
+  const currentWindowParams = useMemo(
+    () => buildWindowParams(currentWindowStartMs),
+    [currentWindowStartMs],
+  );
+  const annotationFramesQuery = useVideoAnnotationFramesQuery(
+    videoId,
+    currentWindowParams,
+  );
+
+  useEffect(() => {
+    setActiveLog((current) => {
+      if (current && sortedLogs.some((log) => log.id === current.id)) {
+        return current;
+      }
+
+      return sortedLogs[0];
+    });
+  }, [sortedLogs]);
+
+  const prefetchWindow = useCallback(
+    (startTimeMs: number) => {
+      if (!videoId || startTimeMs < 0) return;
+
+      const params = buildWindowParams(startTimeMs);
+
+      void queryClient.prefetchQuery({
+        queryKey: videoKeys.annotationFrames(videoId, params),
+        queryFn: () => videoService.getVideoAnnotationFrames(videoId, params),
+        staleTime: 1000 * 60 * 5,
+        gcTime: 1000 * 60 * 30,
+      });
+    },
+    [queryClient, videoId],
+  );
+
+  const getWindowData = useCallback(
+    (startTimeMs: number) => {
+      if (!videoId) return undefined;
+
+      const params = buildWindowParams(startTimeMs);
+
+      return (
+        queryClient.getQueryData<VideoAnnotationFramesResponse>(
+          videoKeys.annotationFrames(videoId, params),
+        ) ??
+        (startTimeMs === currentWindowStartMs
+          ? annotationFramesQuery.data
+          : undefined)
+      );
+    },
+    [annotationFramesQuery.data, currentWindowStartMs, queryClient, videoId],
+  );
+
+  const updateActiveLog = useCallback(
+    (currentTimeSeconds: number) => {
+      if (sortedLogs.length === 0) {
+        setActiveLog(undefined);
+        return;
+      }
+
+      const currentLog = sortedLogs.findLast(
+        (log) => log.frame.timestamp_seconds <= currentTimeSeconds,
+      );
+      const nextActiveLog = currentLog ?? sortedLogs[0];
+
+      setActiveLog((previous) =>
+        previous?.id === nextActiveLog.id ? previous : nextActiveLog,
+      );
+    },
+    [sortedLogs],
+  );
+
+  const getVisibleDetections = useCallback(
+    (currentTimeSeconds: number) => {
+      const currentTimeMs = currentTimeSeconds * 1000;
+      const targetWindowStartMs = getWindowStartMs(currentTimeMs);
+
+      if (targetWindowStartMs !== currentWindowStartMs) {
+        setCurrentWindowStartMs(targetWindowStartMs);
+      }
+
+      const windowData = getWindowData(targetWindowStartMs);
+
+      if (!windowData) {
+        return [];
+      }
+
+      if (
+        windowData.window.has_more_after &&
+        currentTimeMs >=
+          windowData.window.end_time_ms - ANNOTATION_PREFETCH_THRESHOLD_MS
+      ) {
+        prefetchWindow(windowData.window.end_time_ms);
+      }
+
+      const frameDurationMs = fps > 0 ? 1000 / fps : 0;
+      const currentFrame = windowData.frames.find((frame) => {
+        const frameEndMs =
+          frame.end_timestamp_ms ?? frame.timestamp_ms + frameDurationMs;
+
+        return (
+          currentTimeMs >= frame.timestamp_ms && currentTimeMs < frameEndMs
+        );
+      });
+
+      return currentFrame?.detections ?? [];
+    },
+    [currentWindowStartMs, fps, getWindowData, prefetchWindow],
+  );
+
+  const updateVisibleDetections = useCallback(
+    (currentTimeSeconds: number) => {
+      const nextDetections = getVisibleDetections(currentTimeSeconds);
+      const nextIds = nextDetections.map((detection) => detection.id).join('|');
+
+      if (nextIds === visibleDetectionIdsRef.current) return;
+
+      visibleDetectionIdsRef.current = nextIds;
+      setVisibleDetections(nextDetections);
+    },
+    [getVisibleDetections],
+  );
+
+  const handleSeek = useCallback(
+    (log: DetectionResultLog) => {
+      const video = videoRef.current;
+      if (!video) return;
+
+      const targetTimeSeconds = log.frame.timestamp_seconds;
+      const targetWindowStartMs = getWindowStartMs(targetTimeSeconds * 1000);
+
+      prefetchWindow(targetWindowStartMs);
+      setCurrentWindowStartMs(targetWindowStartMs);
+      shouldPauseAfterSeekRef.current = true;
+      video.pause();
+      video.currentTime = targetTimeSeconds;
+      setActiveLog(log);
+      updateVisibleDetections(targetTimeSeconds);
+    },
+    [prefetchWindow, updateVisibleDetections, videoRef],
+  );
+
+  const handleSeeked = useCallback(() => {
+    const video = videoRef.current;
+    if (!video || !shouldPauseAfterSeekRef.current) return;
+
+    shouldPauseAfterSeekRef.current = false;
+    video.pause();
+  }, [videoRef]);
+
+  const handleTimeUpdate = useCallback(() => {
+    const video = videoRef.current;
+    if (!video) return;
+
+    updateActiveLog(video.currentTime);
+  }, [updateActiveLog, videoRef]);
+
+  const handleVideoFrame = useCallback(
+    (mediaTime: number) => {
+      updateActiveLog(mediaTime);
+      updateVisibleDetections(mediaTime);
+    },
+    [updateActiveLog, updateVisibleDetections],
+  );
+
+  return {
+    activeLog,
+    annotationFramesQuery,
+    handleSeek,
+    handleSeeked,
+    handleTimeUpdate,
+    handleVideoFrame,
+    sortedLogs,
+    visibleDetections,
+  };
+}
--- a/src/components/videoPlayerSection.tsx
+++ b/src/components/videoPlayerSection.tsx
@@ -1,36 +1,60 @@
 'use client';

-import { useMemo, useState } from 'react';
-import { ImageIcon } from 'lucide-react';
+import { useRef } from 'react';
+import type { MediaPlayerInstance } from '@vidstack/react';
+import { Film } from 'lucide-react';
+
+import { useProtectedVideoQuery } from '@/app/(modules)/results/hooks/useVideoResults';
+import type { CompletedVideoResult } from '@/types';
+
+import DetectionLocationMap from './map/detectionLocationMap';
 import {
  Card,
  CardContent,
  CardDescription,
  CardHeader,
  CardTitle,
-} from '@/components/ui/card';
-import { CompletedVideoResult } from '@/types';
-import DetectionLocationMap from './map/detectionLocationMap';
+} from './ui/card';
 import AnnotatedDetectionImage from './video/annotatedDetectionImage';
+import AnnotatedVideoPlayer from './video/annotatedVideoPlayer';
 import CurrentDetectionBar from './video/currentDetectionBar';
 import DetectionLogs from './video/detectionLogs';
 import ResultStatsGrid from './video/resultStatsGrid';
+import { useVideoAnnotationPlayback } from './video/useVideoAnnotationPlayback';

 type VideoPlayerSectionProps = {
  data: CompletedVideoResult;
 };

 export default function VideoPlayerSection({ data }: VideoPlayerSectionProps) {
-  const sortedLogs = useMemo(
-    () =>
-      [...data.logs].sort(
-        (a, b) => a.frame.timestamp_seconds - b.frame.timestamp_seconds,
-      ),
-    [data.logs],
-  );
-  const [selectedLogId, setSelectedLogId] = useState(sortedLogs[0]?.id);
-  const activeLog =
-    sortedLogs.find((log) => log.id === selectedLogId) ?? sortedLogs[0];
+  const videoRef = useRef<MediaPlayerInstance | null>(null);
+  const {
+    activeLog,
+    handleSeek,
+    handleSeeked,
+    handleTimeUpdate,
+    handleVideoFrame,
+    sortedLogs,
+    visibleDetections,
+  } = useVideoAnnotationPlayback({
+    videoId: data.video_id,
+    logs: data.logs,
+    fps: data.summary.fps,
+    videoRef,
+  });
+  const {
+    videoUrl,
+    isLoading: isVideoLoading,
+    isError: isVideoError,
+    refetch: refetchVideo,
+  } = useProtectedVideoQuery(data.raw_video_url);
+  const mediaAspectRatio =
+    data.summary.video_width > 0 && data.summary.video_height > 0
+      ? `${data.summary.video_width} / ${data.summary.video_height}`
+      : '16 / 9';
+  const activeMapLabel = activeLog
+    ? `${activeLog.detection.display_name} - Frame ${activeLog.frame.number}`
+    : 'Detection';

  return (
    <div className="space-y-6">
@@ -38,14 +62,15 @@ export default function VideoPlayerSection({ data }: VideoPlayerSectionProps) {
        <CardHeader className="border-b pb-4">
          <div className="flex items-center gap-3">
            <div className="rounded bg-secondary p-2">
-              <ImageIcon className="h-5 w-5 text-primary" />
+              <Film className="h-5 w-5 text-primary" />
            </div>
            <div>
              <CardTitle className="text-lg font-bold">
-                Detection Preview
+                Detection Playback
              </CardTitle>
              <CardDescription className="text-xs">
-                Selected detection image with its bounding box
+                Raw video with frontend annotation overlay and selected
+                detection location
              </CardDescription>
            </div>
          </div>
@@ -53,17 +78,48 @@ export default function VideoPlayerSection({ data }: VideoPlayerSectionProps) {
        <CardContent className="pt-6">
          <div className="grid grid-cols-1 gap-6 lg:grid-cols-3">
            <div className="space-y-6 lg:col-span-2">
-              <AnnotatedDetectionImage
-                detection={activeLog}
-                videoWidth={data.summary.video_width}
-                videoHeight={data.summary.video_height}
-              />
+              <div className="space-y-2">
+                <p className="text-xs font-medium text-muted-foreground">
+                  Detection Video
+                </p>
+                <AnnotatedVideoPlayer
+                  videoRef={videoRef}
+                  logs={sortedLogs}
+                  visibleDetections={visibleDetections}
+                  videoWidth={data.summary.video_width}
+                  videoHeight={data.summary.video_height}
+                  videoUrl={videoUrl}
+                  isLoading={isVideoLoading}
+                  isError={isVideoError}
+                  onRetry={() => void refetchVideo()}
+                  onTimeUpdate={handleTimeUpdate}
+                  onVideoFrame={handleVideoFrame}
+                  onSeeked={handleSeeked}
+                />
+              </div>

-              <DetectionLocationMap
-                latitude={activeLog?.location.latitude ?? null}
-                longitude={activeLog?.location.longitude ?? null}
-                label={activeLog?.detection.display_name ?? 'Detection'}
-              />
+              <div className="grid gap-4 xl:grid-cols-[minmax(0,1.65fr)_minmax(280px,1fr)]">
+                <div className="space-y-2">
+                  <p className="text-xs font-medium text-muted-foreground">
+                    Annotated Image
+                  </p>
+                  <AnnotatedDetectionImage
+                    detection={activeLog}
+                    videoWidth={data.summary.video_width}
+                    videoHeight={data.summary.video_height}
+                  />
+                </div>
+
+                <DetectionLocationMap
+                  latitude={activeLog?.location.latitude ?? null}
+                  longitude={activeLog?.location.longitude ?? null}
+                  label={activeMapLabel}
+                  title="Location on Map"
+                  variant="plain"
+                  aspectRatio={mediaAspectRatio}
+                  showCoordinates={false}
+                />
+              </div>

              <CurrentDetectionBar activeLog={activeLog} data={data} />
            </div>
@@ -71,7 +127,7 @@ export default function VideoPlayerSection({ data }: VideoPlayerSectionProps) {
            <DetectionLogs
              logs={sortedLogs}
              activeLogId={activeLog?.id}
-              onSelect={(log) => setSelectedLogId(log.id)}
+              onSelect={handleSeek}
            />
          </div>