| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1,12 +1,32 @@ | |||
| 1 | 1 | import Track from 'vue-media-annotator/track'; | |
| 2 | 2 | import { headTailFeatures } from 'vue-media-annotator/headTail'; | |
| 3 | 3 | import { | |
| 4 | - autoPopulatePrompt, autoPopulateTarget, boundsIoU, closedRing, polygonBounds, orientLineLike, LINE_PROMPT_FRACTIONS, | ||
| 4 | + autoPopulatePrompt, autoPopulateTarget, boundsIoU, closedRing, interiorPromptPoints, polygonBounds, orientLineLike, | ||
| 5 | + LINE_PROMPT_FRACTIONS, | ||
| 5 | 6 | } from './autoPopulate'; | |
| 6 | 7 | ||
| 7 | 8 | it('prompts a box with its centre', () => { | |
| 8 | 9 | expect(autoPopulatePrompt({ source: 'box', bounds: [10, 20, 30, 60] })).toEqual({ points: [[20, 40]], labels: [1] }); | |
| 9 | 10 | expect(autoPopulatePrompt({ source: 'mask', polygons: [] })).toEqual({ points: [], labels: [] }); | |
| 11 | + expect(autoPopulatePrompt({ source: 'points', points: [[1, 2], [3, 4]] })).toEqual({ points: [[1, 2], [3, 4]], labels: [1, 1] }); | ||
| 12 | + }); | ||
| 13 | + | ||
| 14 | + it('samples prompt points inside a mask, spread apart and clear of its holes', () => { | ||
| 15 | + const ring: [number, number][] = [[0, 0], [90, 0], [90, 30], [0, 30]]; | ||
| 16 | + const hole: [number, number][] = [[40, 10], [50, 10], [50, 20], [40, 20]]; | ||
| 17 | + const points = interiorPromptPoints([{ exterior: ring, holes: [hole] }], 4); | ||
| 18 | + expect(points).toHaveLength(4); | ||
| 19 | + points.forEach(([x, y]) => { | ||
| 20 | + expect(x).toBeGreaterThan(0); expect(x).toBeLessThan(90); | ||
| 21 | + expect(y).toBeGreaterThan(0); expect(y).toBeLessThan(30); | ||
| 22 | + expect(x >= 40 && x <= 50 && y >= 10 && y <= 20).toBe(false); | ||
| 23 | + }); | ||
| 24 | + const xs = points.map((p) => p[0]); | ||
| 25 | + expect(Math.max(...xs) - Math.min(...xs)).toBeGreaterThan(60); | ||
| 26 | + // A thin diagonal sliver still yields points on the object, never the empty corners. | ||
| 27 | + const sliver: [number, number][] = [[0, 0], [100, 98], [100, 100], [0, 2]]; | ||
| 28 | + interiorPromptPoints([{ exterior: sliver, holes: [] }]).forEach(([x, y]) => expect(Math.abs(x - y)).toBeLessThan(3)); | ||
| 29 | + expect(interiorPromptPoints([{ exterior: [[0, 0], [1, 1]], holes: [] }])).toEqual([]); | ||
| 10 | 30 | }); | |
| 11 | 31 | ||
| 12 | 32 | it('prompts a line with foreground points spread along its length', () => { | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1,12 +1,14 @@ | |||
| 1 | 1 | import type Track from 'vue-media-annotator/track'; | |
| 2 | - import type { RectBounds } from 'vue-media-annotator/utils'; | ||
| 2 | + import { pointInPolygon, RectBounds } from 'vue-media-annotator/utils'; | ||
| 3 | 3 | import type { SegmentationPolygon } from 'dive-common/apispec'; | |
| 4 | 4 | ||
| 5 | 5 | export type Point = [number, number]; | |
| 6 | 6 | ||
| 7 | 7 | export type NewAnnotationGeometry = | |
| 8 | 8 | | { source: 'box'; bounds: RectBounds } | |
| 9 | 9 | | { source: 'line'; line: Point[] } | |
| 10 | + /** Foreground prompts already known, e.g. points warped from the other stereo camera's mask. */ | ||
| 11 | + | { source: 'points'; points: Point[] } | ||
| 10 | 12 | /** A mask that already exists on the detection (point segmentation); nothing to predict. */ | |
| 11 | 13 | | { source: 'mask'; polygons: SegmentationPolygon[] }; | |
| 12 | 14 | ||
@@ -24,6 +26,9 @@ export function autoPopulatePrompt(geometry: NewAnnotationGeometry): { points: P | |||
| 24 | 26 | return { points: [[(x0 + x1) / 2, (y0 + y1) / 2]], labels: [1] }; | |
| 25 | 27 | } | |
| 26 | 28 | if (geometry.source === 'mask') return { points: [], labels: [] }; | |
| 29 | + if (geometry.source === 'points') { | ||
| 30 | + return { points: geometry.points.map((p) => [...p] as Point), labels: geometry.points.map(() => 1) }; | ||
| 31 | + } | ||
| 27 | 32 | const { line } = geometry; | |
| 28 | 33 | if (line.length < 2) return { points: line.map((p) => [...p] as Point), labels: line.map(() => 1) }; | |
| 29 | 34 | const total = line.slice(1).reduce((sum, p, i) => sum + Math.hypot(p[0] - line[i][0], p[1] - line[i][1]), 0); | |
@@ -72,6 +77,48 @@ export function boundsIoU(a: RectBounds, b: RectBounds): number { | |||
| 72 | 77 | return union > 0 ? intersection / union : 0; | |
| 73 | 78 | } | |
| 74 | 79 | ||
| 80 | + /** Grid resolution across a mask's bounds when sampling prompt points inside it. */ | ||
| 81 | + const INTERIOR_GRID = 9; | ||
| 82 | + | ||
| 83 | + /** | ||
| 84 | + * Up to `count` points inside a mask, spread across it: the one nearest its | ||
| 85 | + * centre first, then repeatedly the candidate farthest from those chosen. | ||
| 86 | + * Points inside the object warp to the other camera far more reliably than a | ||
| 87 | + * box's corners, which sit on the background. | ||
| 88 | + */ | ||
| 89 | + export function interiorPromptPoints(polygons: SegmentationPolygon[], count = 5): Point[] { | ||
| 90 | + const xy = ([x, y]: Point) => ({ x, y }); | ||
| 91 | + const valid = polygons.filter((polygon) => polygon.exterior.length >= 3); | ||
| 92 | + if (!valid.length) return []; | ||
| 93 | + const [x0, y0, x1, y1] = polygonBounds(valid.flatMap((polygon) => polygon.exterior)); | ||
| 94 | + const inside: Point[] = []; | ||
| 95 | + for (let i = 0; i < INTERIOR_GRID; i += 1) { | ||
| 96 | + for (let j = 0; j < INTERIOR_GRID; j += 1) { | ||
| 97 | + const point: Point = [ | ||
| 98 | + x0 + ((i + 0.5) / INTERIOR_GRID) * (x1 - x0), | ||
| 99 | + y0 + ((j + 0.5) / INTERIOR_GRID) * (y1 - y0), | ||
| 100 | + ]; | ||
| 101 | + if (valid.some((polygon) => pointInPolygon(xy(point), polygon.exterior.map(xy), polygon.holes.map((h) => h.map(xy))))) { | ||
| 102 | + inside.push(point); | ||
| 103 | + } | ||
| 104 | + } | ||
| 105 | + } | ||
| 106 | + if (!inside.length) return []; | ||
| 107 | + const centre: Point = [(x0 + x1) / 2, (y0 + y1) / 2]; | ||
| 108 | + const distance = (a: Point, b: Point) => Math.hypot(a[0] - b[0], a[1] - b[1]); | ||
| 109 | + const chosen = [inside.reduce((best, p) => (distance(p, centre) < distance(best, centre) ? p : best))]; | ||
| 110 | + while (chosen.length < count && chosen.length < inside.length) { | ||
| 111 | + const next = inside | ||
| 112 | + .filter((p) => !chosen.includes(p)) | ||
| 113 | + .reduce((best, p) => { | ||
| 114 | + const gap = (q: Point) => Math.min(...chosen.map((c) => distance(q, c))); | ||
| 115 | + return gap(p) > gap(best) ? p : best; | ||
| 116 | + }); | ||
| 117 | + chosen.push(next); | ||
| 118 | + } | ||
| 119 | + return chosen; | ||
| 120 | + } | ||
| 121 | + | ||
| 75 | 122 | /** | |
| 76 | 123 | * Head/tail derived independently on each stereo camera can come out flipped; | |
| 77 | 124 | * order this pair to run the same way as the other camera's line. | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -87,6 +87,7 @@ export type NewAnnotationGeometryParams = { | |||
| 87 | 87 | } & ( | |
| 88 | 88 | | { source: 'box'; bounds: [number, number, number, number] } | |
| 89 | 89 | | { source: 'line'; line: [number, number][] } | |
| 90 | + | { source: 'points'; points: [number, number][] } | ||
| 90 | 91 | | { source: 'mask'; polygons: SegmentationPolygon[] } | |
| 91 | 92 | ); | |
| 92 | 93 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -188,6 +188,22 @@ it('derives head/tail for a point-segmented mask without predicting again', asyn | |||
| 188 | 188 | expect(lined.track.getFeature(0)[0]?.head).toEqual([2, 2]); | |
| 189 | 189 | }); | |
| 190 | 190 | ||
| 191 | + it('segments from given prompt points, boxes the mask, and reports it even when nothing is stored', async () => { | ||
| 192 | + const h = harness(); | ||
| 193 | + const onMask = vi.fn(); | ||
| 194 | + h.services.predict.mockResolvedValue({ success: true, polygons: components }); | ||
| 195 | + expect(await populateAnnotation({ | ||
| 196 | + camera: 'right', trackId: 1, frameNum: 0, source: 'points', points: [[3, 3], [25, 5]], | ||
| 197 | + }, { mask: false, points: false, onMask }, h.services)).toBe('applied'); | ||
| 198 | + expect(h.services.predict).toHaveBeenCalledWith(expect.objectContaining({ | ||
| 199 | + points: [[3, 3], [25, 5]], pointLabels: [1, 1], multimaskOutput: false, | ||
| 200 | + })); | ||
| 201 | + expect(onMask).toHaveBeenCalledWith(components); | ||
| 202 | + expect(h.track.getFeature(0)[0]?.bounds).toEqual([0, 0, 30, 10]); | ||
| 203 | + expect(h.track.getPolygonFeatures(0)).toHaveLength(0); | ||
| 204 | + expect(h.services.keypoints).not.toHaveBeenCalled(); | ||
| 205 | + }); | ||
| 206 | + | ||
| 191 | 207 | it('extracts points from all components when storing the mask is disabled', async () => { | |
| 192 | 208 | const h = harness(); | |
| 193 | 209 | h.services.predict.mockResolvedValue({ success: true, polygons: components }); | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -23,6 +23,8 @@ export interface AutoPopulateOptions { | |||
| 23 | 23 | orientLike?: [number, number][] | null; | |
| 24 | 24 | /** The box was mapped from the other stereo camera: refit it when the mask disagrees with it. */ | |
| 25 | 25 | fitBoxToMask?: boolean; | |
| 26 | + /** Receives the mask the annotation was segmented to, whether or not it is stored. */ | ||
| 27 | + onMask?: (polygons: SegmentationPolygon[]) => void; | ||
| 26 | 28 | } | |
| 27 | 29 | ||
| 28 | 30 | async function predictMask( | |
@@ -62,6 +64,7 @@ export default async function populateAnnotation( | |||
| 62 | 64 | } | |
| 63 | 65 | const target = currentTarget(); | |
| 64 | 66 | if (!target) return 'changed'; | |
| 67 | + options.onMask?.(polygons); | ||
| 65 | 68 | const { track, feature } = target; | |
| 66 | 69 | const features = feature.geometry?.features ?? []; | |
| 67 | 70 | const hasPolygon = features.some((f) => f.geometry.type === 'Polygon'); | |
@@ -78,7 +81,7 @@ export default async function populateAnnotation( | |||
| 78 | 81 | }))); | |
| 79 | 82 | } | |
| 80 | 83 | const maskBounds = polygonBounds(polygons.flatMap((polygon) => polygon.exterior)); | |
| 81 | - if (options.points && params.source === 'line') { | ||
| 84 | + if ((options.points && params.source === 'line') || params.source === 'points') { | ||
| 82 | 85 | track.setFeature({ frame: params.frameNum, keyframe: true, bounds: maskBounds }); | |
| 83 | 86 | } else if (options.fitBoxToMask && params.source === 'box' | |
| 84 | 87 | && boundsIoU(params.bounds, maskBounds) < MAPPED_BOX_MIN_IOU) { | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -16,7 +16,7 @@ import CalibrationMenu from 'dive-common/components/CalibrationMenu.vue'; | |||
| 16 | 16 | import SidebarContext from 'dive-common/components/SidebarContext.vue'; | |
| 17 | 17 | import context from 'dive-common/store/context'; | |
| 18 | 18 | import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; | |
| 19 | - import { SegmentationPredictRequest } from 'dive-common/apispec'; | ||
| 19 | + import { SegmentationPredictRequest, SegmentationPolygon } from 'dive-common/apispec'; | ||
| 20 | 20 | import { clientSettings } from 'dive-common/store/settings'; | |
| 21 | 21 | import { isStereoscopicDatasetConfig } from 'dive-common/multicamDisplay'; | |
| 22 | 22 | import type { | |
@@ -45,6 +45,7 @@ import { | |||
| 45 | 45 | openLink, | |
| 46 | 46 | setScoringAnnotationPreviewFile, | |
| 47 | 47 | } from 'platform/desktop/frontend/api'; | |
| 48 | + import { interiorPromptPoints, polygonBounds } from 'dive-common/use/autoPopulate'; | ||
| 48 | 49 | import populateAnnotation from '../autoPopulate'; | |
| 49 | 50 | import Export from './Export.vue'; | |
| 50 | 51 | import JobTab from './JobTab.vue'; | |
@@ -1186,7 +1187,9 @@ export default defineComponent({ | |||
| 1186 | 1187 | ||
| 1187 | 1188 | // In-flight auto-populate passes for freshly drawn shapes, so the stereo | |
| 1188 | 1189 | // transfer of the same shape can populate its mapped copy afterwards. | |
| 1189 | - const pendingAutoPopulate = new Map<string, Promise<void>>(); | ||
| 1190 | + /** Outcome of one auto-populate pass; the mask is reported even when only points were asked for. */ | ||
| 1191 | + type AutoPopulateOutcome = { status: 'applied' | 'changed' | 'failed' | 'skipped'; polygons: SegmentationPolygon[] }; | ||
| 1192 | + const pendingAutoPopulate = new Map<string, Promise<AutoPopulateOutcome>>(); | ||
| 1190 | 1193 | const autoPopulateKey = (camera: string, trackId: number, frameNum: number) => `${camera}:${trackId}:${frameNum}`; | |
| 1191 | 1194 | ||
| 1192 | 1195 | // eslint-disable-next-line @typescript-eslint/no-explicit-any | |
@@ -1785,6 +1788,9 @@ export default defineComponent({ | |||
| 1785 | 1788 | }); | |
| 1786 | 1789 | } | |
| 1787 | 1790 | } | |
| 1791 | + } else if (params.type === 'box' | ||
| 1792 | + && sourceAutoPopulate && await mapBoxThroughSourceMask(sourceAutoPopulate, params, otherCamera)) { | ||
| 1793 | + // The other camera's box came from its own segmentation. | ||
| 1788 | 1794 | } else if (params.type === 'box') { | |
| 1789 | 1795 | // Convert box bounds to 4 corner points | |
| 1790 | 1796 | const [x1, y1, x2, y2] = params.bounds; | |
@@ -2130,18 +2136,21 @@ export default defineComponent({ | |||
| 2130 | 2136 | async function autoPopulateGeometry( | |
| 2131 | 2137 | params: NewAnnotationGeometryParams, | |
| 2132 | 2138 | stereo?: { orientLike: [number, number][] | null; fitBoxToMask: boolean }, | |
| 2133 | - ) { | ||
| 2139 | + ): Promise<AutoPopulateOutcome> { | ||
| 2134 | 2140 | const { autoPopulateMask, autoPopulatePoints } = clientSettings.trackSettings.newTrackSettings; | |
| 2135 | - if (!autoPopulateMask && !autoPopulatePoints) return; | ||
| 2136 | 2141 | const cameraStore = viewerRef.value?.cameraStore; | |
| 2137 | - if (!cameraStore) return; | ||
| 2142 | + if ((!autoPopulateMask && !autoPopulatePoints) || !cameraStore) return { status: 'skipped', polygons: [] }; | ||
| 2143 | + let polygons: SegmentationPolygon[] = []; | ||
| 2138 | 2144 | // Resolve the requested camera independently of recipe initialization and | |
| 2139 | 2145 | // whichever camera is selected when asynchronous work finishes. | |
| 2140 | 2146 | const datasetId = params.camera === 'singleCam' ? props.id : `${props.id}/${params.camera}`; | |
| 2141 | 2147 | autoPopulateActive.value += 1; | |
| 2142 | 2148 | try { | |
| 2143 | 2149 | const result = await populateAnnotation(params, { | |
| 2144 | - mask: autoPopulateMask, points: autoPopulatePoints, ...stereo, | ||
| 2150 | + mask: autoPopulateMask, | ||
| 2151 | + points: autoPopulatePoints, | ||
| 2152 | + onMask: (mask) => { polygons = mask; }, | ||
| 2153 | + ...stereo, | ||
| 2145 | 2154 | }, { | |
| 2146 | 2155 | getTrack: () => cameraStore.getPossibleTrack(params.trackId, params.camera), | |
| 2147 | 2156 | getMedia: async () => { | |
@@ -2166,10 +2175,12 @@ export default defineComponent({ | |||
| 2166 | 2175 | if (result === 'changed' && cameraStore.getPossibleTrack(params.trackId, params.camera)) { | |
| 2167 | 2176 | autoPopulateMessage.value = `Auto-populate skipped for track ${params.trackId}: the annotation changed while the request was running.`; | |
| 2168 | 2177 | } | |
| 2178 | + return { status: result, polygons }; | ||
| 2169 | 2179 | } catch (err) { | |
| 2170 | 2180 | const message = err instanceof Error ? err.message : String(err); | |
| 2171 | 2181 | autoPopulateMessage.value = `Auto-populate — track ${params.trackId}, ${params.camera}, frame ${params.frameNum}: ${message}`; | |
| 2172 | 2182 | console.warn('[AutoPopulate] Could not populate the new annotation:', err); | |
| 2183 | + return { status: 'failed', polygons }; | ||
| 2173 | 2184 | } finally { | |
| 2174 | 2185 | autoPopulateActive.value -= 1; | |
| 2175 | 2186 | } | |
@@ -2181,15 +2192,15 @@ export default defineComponent({ | |||
| 2181 | 2192 | * so a derived head/tail can follow the source's direction. | |
| 2182 | 2193 | */ | |
| 2183 | 2194 | async function autoPopulateOtherCamera( | |
| 2184 | - sourceJob: Promise<void>, | ||
| 2195 | + sourceJob: Promise<unknown>, | ||
| 2185 | 2196 | sourceCamera: string, | |
| 2186 | 2197 | mapped: NewAnnotationGeometryParams, | |
| 2187 | - ) { | ||
| 2198 | + ): Promise<AutoPopulateOutcome> { | ||
| 2188 | 2199 | await sourceJob; | |
| 2189 | 2200 | const cameraStore = viewerRef.value?.cameraStore; | |
| 2190 | - if (!cameraStore) return; | ||
| 2201 | + if (!cameraStore) return { status: 'skipped', polygons: [] }; | ||
| 2191 | 2202 | const sourceTrack = cameraStore.getPossibleTrack(mapped.trackId, sourceCamera); | |
| 2192 | - await autoPopulateGeometry(mapped, { | ||
| 2203 | + const outcome = await autoPopulateGeometry(mapped, { | ||
| 2193 | 2204 | orientLike: getStereoLineEndpoints(sourceTrack, mapped.frameNum), | |
| 2194 | 2205 | fitBoxToMask: true, | |
| 2195 | 2206 | }); | |
@@ -2200,6 +2211,45 @@ export default defineComponent({ | |||
| 2200 | 2211 | console.warn('[Stereo] Measurement update failed:', err); | |
| 2201 | 2212 | } | |
| 2202 | 2213 | } | |
| 2214 | + return outcome; | ||
| 2215 | + } | ||
| 2216 | + | ||
| 2217 | + /** | ||
| 2218 | + * With auto-segmentation on, a new box reaches the other camera through | ||
| 2219 | + * its mask rather than its corners: a few points inside the source mask | ||
| 2220 | + * are warped (their disparities are stable, unlike corners that sit on the | ||
| 2221 | + * background), the other camera is segmented from them, and its box is | ||
| 2222 | + * that mask's bounds. False means fall back to warping the corners. | ||
| 2223 | + */ | ||
| 2224 | + async function mapBoxThroughSourceMask( | ||
| 2225 | + sourceJob: Promise<AutoPopulateOutcome>, | ||
| 2226 | + params: { camera: string; trackId: number; frameNum: number }, | ||
| 2227 | + otherCamera: string, | ||
| 2228 | + ): Promise<boolean> { | ||
| 2229 | + const source = await sourceJob; | ||
| 2230 | + const prompts = interiorPromptPoints(source.polygons); | ||
| 2231 | + if (!prompts.length) return false; | ||
| 2232 | + const response = await stereoTransferPoints({ points: prompts }); | ||
| 2233 | + const warped = (response.transferredPoints ?? []) | ||
| 2234 | + .filter((p): p is [number, number] => !!p && Number.isFinite(p[0]) && Number.isFinite(p[1])); | ||
| 2235 | + const cameraStore = viewerRef.value?.cameraStore; | ||
| 2236 | + if (!response.success || !warped.length || !cameraStore) return false; | ||
| 2237 | + const track = getOrCreateStereoTrack(cameraStore, params.trackId, params.camera, otherCamera, params.frameNum); | ||
| 2238 | + if (!track) return false; | ||
| 2239 | + // A placeholder for the segmentation to replace; it needs a feature to target. | ||
| 2240 | + track.setFeature({ | ||
| 2241 | + frame: params.frameNum, | ||
| 2242 | + flick: 0, | ||
| 2243 | + bounds: polygonBounds(warped), | ||
| 2244 | + keyframe: true, | ||
| 2245 | + interpolate: false, | ||
| 2246 | + }); | ||
| 2247 | + const outcome = await autoPopulateOtherCamera(Promise.resolve(), params.camera, { | ||
| 2248 | + camera: otherCamera, trackId: params.trackId, frameNum: params.frameNum, source: 'points', points: warped, | ||
| 2249 | + }); | ||
| 2250 | + if (outcome.status === 'applied' || outcome.status === 'changed') return true; | ||
| 2251 | + track.deleteFeature(params.frameNum); | ||
| 2252 | + return false; | ||
| 2203 | 2253 | } | |
| 2204 | 2254 | ||
| 2205 | 2255 | /** A brand-new box or line just got drawn, or a brand-new mask point-segmented. */ | |
@@ -2208,7 +2258,7 @@ export default defineComponent({ | |||
| 2208 | 2258 | if (!autoPopulateMask && !autoPopulatePoints) return; | |
| 2209 | 2259 | if (params.source === 'mask' && !autoPopulatePoints) return; | |
| 2210 | 2260 | const key = autoPopulateKey(params.camera, params.trackId, params.frameNum); | |
| 2211 | - const job: Promise<void> = autoPopulateGeometry(params).finally(() => { | ||
| 2261 | + const job: Promise<AutoPopulateOutcome> = autoPopulateGeometry(params).finally(() => { | ||
| 2212 | 2262 | if (pendingAutoPopulate.get(key) === job) pendingAutoPopulate.delete(key); | |
| 2213 | 2263 | }); | |
| 2214 | 2264 | pendingAutoPopulate.set(key, job); | |
| Back | FazBrowse Home | New Git URL |
0 commit comments