diff --git a/package-lock.json b/package-lock.json index d818001..9cef16f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,7 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "date-fns": "^4.4.0", + "leaflet": "^1.9.4", "libphonenumber-js": "^1.13.7", "lightgallery": "^2.9.0", "lucide-react": "^0.548.0", @@ -39,6 +40,7 @@ "react-day-picker": "^10.0.1", "react-dom": "19.2.0", "react-hook-form": "^7.79.0", + "react-leaflet": "^5.0.0", "recharts": "2.15.4", "sonner": "^1.7.4", "tailwind-merge": "^3.3.1", @@ -48,6 +50,7 @@ }, "devDependencies": { "@tailwindcss/postcss": "^4.1.9", + "@types/leaflet": "^1.9.21", "@types/node": "^22", "@types/react": "^19", "@types/react-dom": "^19", @@ -2890,6 +2893,17 @@ "integrity": "sha512-xnXE7wG13PI+cxieVssYXlQJuYVRhH9NBoxt3KNwzghDIA69GMm7d4wXRouHIYjE+KvS6U/MsMO73NdS2MH9ZA==", "license": "MIT" }, + "node_modules/@react-leaflet/core": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@react-leaflet/core/-/core-3.0.0.tgz", + "integrity": "sha512-3EWmekh4Nz+pGcr+xjf0KNyYfC3U2JjnkWsh0zcqaexYqmmB5ZhH37kz41JXGmKzpaMZCnPofBBm64i+YrEvGQ==", + "license": "Hippocratic-2.1", + "peerDependencies": { + "leaflet": "^1.9.0", + "react": "^19.0.0", + "react-dom": "^19.0.0" + } + }, "node_modules/@rtsao/scc": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@rtsao/scc/-/scc-1.1.0.tgz", @@ -3377,6 +3391,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/geojson": { + "version": "7946.0.16", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", + "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -3391,6 +3412,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/leaflet": { + "version": "1.9.21", + "resolved": "https://registry.npmjs.org/@types/leaflet/-/leaflet-1.9.21.tgz", + "integrity": "sha512-TbAd9DaPGSnzp6QvtYngntMZgcRk+igFELwR2N99XZn7RXUdKgsXMR+28bUO0rPsWp8MIu/f47luLIQuSLYv/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/geojson": "*" + } + }, "node_modules/@types/node": { "version": "22.19.15", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.15.tgz", @@ -6857,6 +6888,12 @@ "node": ">=0.10" } }, + "node_modules/leaflet": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/leaflet/-/leaflet-1.9.4.tgz", + "integrity": "sha512-nxS1ynzJOmOlHp+iL3FyWqK89GtNL8U8rvlMOsQdTTssxZwCXh8N2NB3GDQOL+YR3XnWyZAxwQixURb+FA74PA==", + "license": "BSD-2-Clause" + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -8068,6 +8105,20 @@ "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", "license": "MIT" }, + "node_modules/react-leaflet": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/react-leaflet/-/react-leaflet-5.0.0.tgz", + "integrity": "sha512-CWbTpr5vcHw5bt9i4zSlPEVQdTVcML390TjeDG0cK59z1ylexpqC6M1PJFjV8jD7CF+ACBFsLIDs6DRMoLEofw==", + "license": "Hippocratic-2.1", + "dependencies": { + "@react-leaflet/core": "^3.0.0" + }, + "peerDependencies": { + "leaflet": "^1.9.0", + "react": "^19.0.0", + "react-dom": "^19.0.0" + } + }, "node_modules/react-remove-scroll": { "version": "2.7.2", "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.7.2.tgz", diff --git a/package.json b/package.json index 71800d2..520c886 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "date-fns": "^4.4.0", + "leaflet": "^1.9.4", "libphonenumber-js": "^1.13.7", "lightgallery": "^2.9.0", "lucide-react": "^0.548.0", @@ -43,6 +44,7 @@ "react-day-picker": "^10.0.1", "react-dom": "19.2.0", "react-hook-form": "^7.79.0", + "react-leaflet": "^5.0.0", "recharts": "2.15.4", "sonner": "^1.7.4", "tailwind-merge": "^3.3.1", @@ -52,6 +54,7 @@ }, "devDependencies": { "@tailwindcss/postcss": "^4.1.9", + "@types/leaflet": "^1.9.21", "@types/node": "^22", "@types/react": "^19", "@types/react-dom": "^19", diff --git a/src/app/layout.tsx b/src/app/layout.tsx index 1bb4956..9fdb57d 100644 --- a/src/app/layout.tsx +++ b/src/app/layout.tsx @@ -2,6 +2,7 @@ import type { Metadata } from 'next'; import { Geist_Mono, Poppins } from 'next/font/google'; import '@vidstack/react/player/styles/default/theme.css'; import '@vidstack/react/player/styles/default/layouts/video.css'; +import 'leaflet/dist/leaflet.css'; import 'lightgallery/css/lightgallery.css'; import 'lightgallery/css/lg-thumbnail.css'; import 'lightgallery/css/lg-zoom.css'; diff --git a/src/components/annotation/boundingBoxOverlay.tsx b/src/components/annotation/boundingBoxOverlay.tsx new file mode 100644 index 0000000..dc7ff2e --- /dev/null +++ b/src/components/annotation/boundingBoxOverlay.tsx @@ -0,0 +1,66 @@ +import { getDefectVisual } from '@/constants/defectVisualConfig'; +import type { DetectionResultLog } from '@/types'; + +interface BoundingBoxOverlayProps { + detections: DetectionResultLog[]; + videoWidth: number; + videoHeight: number; +} + +export default function BoundingBoxOverlay({ + detections, + videoWidth, + videoHeight, +}: BoundingBoxOverlayProps) { + if (videoWidth <= 0 || videoHeight <= 0 || detections.length === 0) { + return null; + } + + return ( + + ); +} diff --git a/src/components/map/detectionLocationMap.tsx b/src/components/map/detectionLocationMap.tsx new file mode 100644 index 0000000..c7a2760 --- /dev/null +++ b/src/components/map/detectionLocationMap.tsx @@ -0,0 +1,113 @@ +'use client'; + +import dynamic from 'next/dynamic'; +import { MapPin } from 'lucide-react'; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from '@/components/ui/card'; + +type DetectionLocationMapProps = { + latitude: number | null; + longitude: number | null; + label: string; +}; + +type ClientDetectionLocationMapProps = { + latitude: number; + longitude: number; + label: string; +}; + +const ClientDetectionLocationMap = dynamic( + async () => { + const { CircleMarker, MapContainer, Popup, TileLayer } = + await import('react-leaflet'); + + return function ClientDetectionLocationMapInner({ + latitude, + longitude, + label, + }: ClientDetectionLocationMapProps) { + return ( + + + + {label} + + + ); + }; + }, + { + ssr: false, + loading: () =>
, + }, +); + +export default function DetectionLocationMap({ + latitude, + longitude, + label, +}: DetectionLocationMapProps) { + return ( + + +
+
+ +
+
+ + Detection Location + + + Selected log GPS coordinates + +
+
+ + + {latitude == null || longitude == null ? ( +
+ Coordinates are unavailable for the selected log. +
+ ) : ( + <> +
+ {latitude.toFixed(6)}, {longitude.toFixed(6)} +
+
+ +
+ + )} + + + ); +} diff --git a/src/components/video/annotatedDetectionImage.tsx b/src/components/video/annotatedDetectionImage.tsx new file mode 100644 index 0000000..f91ed38 --- /dev/null +++ b/src/components/video/annotatedDetectionImage.tsx @@ -0,0 +1,69 @@ +'use client'; + +import { AlertTriangle, ImageIcon, Loader2 } from 'lucide-react'; +import Image from 'next/image'; + +import { useProtectedMediaObjectUrl } from '@/hooks/useProtectedMedia'; +import type { DetectionResultLog } from '@/types'; + +import BoundingBoxOverlay from '@/components/annotation/boundingBoxOverlay'; + +interface AnnotatedDetectionImageProps { + detection?: DetectionResultLog; + videoWidth: number; + videoHeight: number; +} + +export default function AnnotatedDetectionImage({ + detection, + videoWidth, + videoHeight, +}: AnnotatedDetectionImageProps) { + const { objectUrl, isLoading, isError } = useProtectedMediaObjectUrl( + detection?.detection.context_image_url, + ); + + const aspectRatio = + videoWidth > 0 && videoHeight > 0 + ? `${videoWidth} / ${videoHeight}` + : '16 / 9'; + + return ( +
+ {!detection ? ( +
+ +

Select a detection to view its image.

+
+ ) : isLoading ? ( +
+ +

Loading detection image...

+
+ ) : isError || !objectUrl ? ( +
+ +

Failed to load the detection image.

+
+ ) : ( + <> + {`${detection.detection.display_name} + + + )} +
+ ); +} diff --git a/src/components/video/annotatedVideoPlayer.tsx b/src/components/video/annotatedVideoPlayer.tsx index 386e5c4..a02c96d 100644 --- a/src/components/video/annotatedVideoPlayer.tsx +++ b/src/components/video/annotatedVideoPlayer.tsx @@ -1,14 +1,17 @@ 'use client'; -import { RefObject } from 'react'; +import { RefObject, useEffect } from 'react'; import { AlertTriangle, Loader2 } from 'lucide-react'; import type { DetectionResultLog } from '@/types'; import { useDetectionThumbnails } from './useDetectionThumbnails'; +import BoundingBoxOverlay from '@/components/annotation/boundingBoxOverlay'; import { MediaPlayer, MediaProvider, Poster, + isVideoProvider, type MediaPlayerInstance, + useMediaProvider, } from '@vidstack/react'; import { DefaultVideoLayout, @@ -18,22 +21,54 @@ import { interface AnnotatedVideoPlayerProps { videoRef: RefObject; logs: DetectionResultLog[]; + visibleLogs: DetectionResultLog[]; + videoWidth: number; + videoHeight: number; videoUrl: string | null; isLoading: boolean; isError: boolean; onRetry: () => void; onTimeUpdate: () => void; + onVideoFrame: (mediaTime: number) => void; onSeeked: () => void; } +function VideoFrameSync({ + onVideoFrame, +}: Pick) { + const provider = useMediaProvider(); + + useEffect(() => { + if (!isVideoProvider(provider)) return; + + const video = provider.video; + let callbackId: number; + + const update = (_now: number, metadata: VideoFrameCallbackMetadata) => { + onVideoFrame(metadata.mediaTime); + callbackId = video.requestVideoFrameCallback(update); + }; + + callbackId = video.requestVideoFrameCallback(update); + + return () => video.cancelVideoFrameCallback(callbackId); + }, [onVideoFrame, provider]); + + return null; +} + export default function AnnotatedVideoPlayer({ videoRef, logs, + visibleLogs, + videoWidth, + videoHeight, videoUrl, isLoading, isError, onRetry, onTimeUpdate, + onVideoFrame, onSeeked, }: AnnotatedVideoPlayerProps) { const thumbnails = useDetectionThumbnails(logs); @@ -55,6 +90,12 @@ export default function AnnotatedVideoPlayer({ + + void; + onSelect: (log: DetectionResultLog) => void; } const formatVideoTime = (seconds: number) => { @@ -63,7 +63,7 @@ function DetectionLogThumbnail({ ); } -const DetectionLogs = ({ logs, activeLogId, onSeek }: DetectionLogsProps) => { +const DetectionLogs = ({ logs, activeLogId, onSelect }: DetectionLogsProps) => { const activeLogRef = useRef(null); useEffect(() => { @@ -100,7 +100,7 @@ const DetectionLogs = ({ logs, activeLogId, onSeek }: DetectionLogsProps) => { key={`${log.id}-${log.frame.number}`} ref={isActive ? activeLogRef : null} type="button" - onClick={() => onSeek(log)} + onClick={() => onSelect(log)} className={cn( 'w-full rounded-md border bg-card p-3 text-left transition-colors hover:border-primary', isActive && 'border-primary bg-primary/5', diff --git a/src/components/video/useDetectionPlayback.ts b/src/components/video/useDetectionPlayback.ts index 064706f..2857a8f 100644 --- a/src/components/video/useDetectionPlayback.ts +++ b/src/components/video/useDetectionPlayback.ts @@ -1,16 +1,18 @@ 'use client'; -import { RefObject, useMemo, useRef, useState } from 'react'; +import { RefObject, useCallback, useMemo, useRef, useState } from 'react'; import type { MediaPlayerInstance } from '@vidstack/react'; import { CompletedVideoResult, DetectionResultLog } from '@/types'; interface UseDetectionPlaybackParams { logs: CompletedVideoResult['logs']; + fps: number; videoRef: RefObject; } export function useDetectionPlayback({ logs, + fps, videoRef, }: UseDetectionPlaybackParams) { const shouldPauseAfterSeekRef = useRef(false); @@ -24,6 +26,36 @@ export function useDetectionPlayback({ const [activeLog, setActiveLog] = useState( sortedLogs[0], ); + const [visibleLogs, setVisibleLogs] = useState([]); + const visibleLogIdsRef = useRef(''); + + const getVisibleLogs = useCallback( + (currentTime: number) => { + const frameDuration = fps > 0 ? 1 / fps : 0; + + return sortedLogs.filter((log) => { + const start = + log.frame.start_timestamp_seconds ?? log.frame.timestamp_seconds; + const end = log.frame.end_timestamp_seconds ?? start + frameDuration; + + return currentTime >= start && currentTime < end; + }); + }, + [fps, sortedLogs], + ); + + const updateVisibleLogs = useCallback( + (currentTime: number) => { + const nextLogs = getVisibleLogs(currentTime); + const nextIds = nextLogs.map((log) => log.id).join('|'); + + if (nextIds === visibleLogIdsRef.current) return; + + visibleLogIdsRef.current = nextIds; + setVisibleLogs(nextLogs); + }, + [getVisibleLogs], + ); const handleSeek = (log: DetectionResultLog) => { const video = videoRef.current; @@ -33,6 +65,7 @@ export function useDetectionPlayback({ video.pause(); video.currentTime = log.frame.timestamp_seconds; setActiveLog(log); + updateVisibleLogs(log.frame.timestamp_seconds); }; const handleSeeked = () => { @@ -58,9 +91,11 @@ export function useDetectionPlayback({ return { activeLog, + visibleLogs, sortedLogs, handleSeek, handleSeeked, handleTimeUpdate, + handleVideoFrame: updateVisibleLogs, }; } diff --git a/src/components/videoPlayerSection.tsx b/src/components/videoPlayerSection.tsx index 9ed1ec2..5fc3998 100644 --- a/src/components/videoPlayerSection.tsx +++ b/src/components/videoPlayerSection.tsx @@ -1,8 +1,7 @@ 'use client'; -import { useRef } from 'react'; -import { Film } from 'lucide-react'; -import type { MediaPlayerInstance } from '@vidstack/react'; +import { useMemo, useState } from 'react'; +import { ImageIcon } from 'lucide-react'; import { Card, CardContent, @@ -11,30 +10,27 @@ import { CardTitle, } from '@/components/ui/card'; import { CompletedVideoResult } from '@/types'; -import { useAnnotatedVideoQuery } from '@/app/(modules)/results/hooks/useVideoResults'; -import AnnotatedVideoPlayer from './video/annotatedVideoPlayer'; +import DetectionLocationMap from './map/detectionLocationMap'; +import AnnotatedDetectionImage from './video/annotatedDetectionImage'; import CurrentDetectionBar from './video/currentDetectionBar'; import DetectionLogs from './video/detectionLogs'; import ResultStatsGrid from './video/resultStatsGrid'; -import { useDetectionPlayback } from './video/useDetectionPlayback'; type VideoPlayerSectionProps = { data: CompletedVideoResult; }; export default function VideoPlayerSection({ data }: VideoPlayerSectionProps) { - const videoRef = useRef(null); - const { activeLog, sortedLogs, handleSeek, handleSeeked, handleTimeUpdate } = - useDetectionPlayback({ - logs: data.logs, - videoRef, - }); - const { - videoUrl, - isLoading: isVideoLoading, - isError: isVideoError, - refetch: refetchVideo, - } = useAnnotatedVideoQuery(data.processed_video_url || ''); + const sortedLogs = useMemo( + () => + [...data.logs].sort( + (a, b) => a.frame.timestamp_seconds - b.frame.timestamp_seconds, + ), + [data.logs], + ); + const [selectedLogId, setSelectedLogId] = useState(sortedLogs[0]?.id); + const activeLog = + sortedLogs.find((log) => log.id === selectedLogId) ?? sortedLogs[0]; return (
@@ -42,14 +38,14 @@ export default function VideoPlayerSection({ data }: VideoPlayerSectionProps) {
- +
- Detection Playback + Detection Preview - Annotated video with backend detection logs + Selected detection image with its bounding box
@@ -57,15 +53,16 @@ export default function VideoPlayerSection({ data }: VideoPlayerSectionProps) {
- void refetchVideo()} - onTimeUpdate={handleTimeUpdate} - onSeeked={handleSeeked} + + + @@ -74,7 +71,7 @@ export default function VideoPlayerSection({ data }: VideoPlayerSectionProps) { setSelectedLogId(log.id)} />
diff --git a/src/constants/defectVisualConfig.ts b/src/constants/defectVisualConfig.ts index 3149b5f..18fde73 100644 --- a/src/constants/defectVisualConfig.ts +++ b/src/constants/defectVisualConfig.ts @@ -12,6 +12,7 @@ import { type DefectVisualConfig = { icon: LucideIcon; + boundingBoxColor: string; colorClassName: string; iconClassName: string; cardClassName: string; @@ -20,30 +21,35 @@ type DefectVisualConfig = { const DEFECT_VISUALS: Record = { manhole_cover: { icon: CircleDot, + boundingBoxColor: '#71717a', colorClassName: 'text-zinc-500', iconClassName: 'bg-zinc-500/10 text-zinc-500', cardClassName: 'bg-zinc-500/5', }, pothole: { icon: Construction, + boundingBoxColor: '#f97316', colorClassName: 'text-orange-500', iconClassName: 'bg-orange-500/10 text-orange-500', cardClassName: 'bg-orange-500/5', }, road_crack: { icon: Spline, + boundingBoxColor: '#f43f5e', colorClassName: 'text-rose-500', iconClassName: 'bg-rose-500/10 text-rose-500', cardClassName: 'bg-rose-500/5', }, sign_board: { icon: SignpostBig, + boundingBoxColor: '#3b82f6', colorClassName: 'text-blue-500', iconClassName: 'bg-blue-500/10 text-blue-500', cardClassName: 'bg-blue-500/5', }, water_puddle: { icon: Droplets, + boundingBoxColor: '#06b6d4', colorClassName: 'text-cyan-500', iconClassName: 'bg-cyan-500/10 text-cyan-500', cardClassName: 'bg-cyan-500/5', @@ -52,6 +58,7 @@ const DEFECT_VISUALS: Record = { const DEFAULT_VISUAL: DefectVisualConfig = { icon: Activity, + boundingBoxColor: '#22c55e', colorClassName: 'text-green-500', iconClassName: 'bg-green-500/10 text-green-500', cardClassName: 'bg-green-500/5', diff --git a/src/types/analysis.ts b/src/types/analysis.ts index 040cc3f..03c9e85 100644 --- a/src/types/analysis.ts +++ b/src/types/analysis.ts @@ -8,6 +8,8 @@ export type CompletedVideoResult = { summary: { fps: number; duration_seconds: number; + video_width: number; + video_height: number; total_detections: number; }; defect_counts: DetectionClassCount[]; diff --git a/src/types/detection.ts b/src/types/detection.ts index 5be2d8b..cc51430 100644 --- a/src/types/detection.ts +++ b/src/types/detection.ts @@ -5,6 +5,15 @@ export type DetectionClassCount = { unique_count: number; }; +export type DetectionBoundingBox = { + x1: number; + y1: number; + x2: number; + y2: number; + width: number; + height: number; +}; + export type DetectionResultLog = { id: string; detection: { @@ -13,10 +22,13 @@ export type DetectionResultLog = { display_name: string; confidence: number; context_image_url: string; + bounding_box: DetectionBoundingBox; }; frame: { number: number; timestamp_seconds: number; + start_timestamp_seconds?: number; + end_timestamp_seconds?: number; }; location: { latitude: number | null;