FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Merge PR #1941 stereo box mapping through the source mask · Kitware/dive@d8f2ff6 · GitHub

Repository navigation

Commit d8f2ff6

Browse files
committed
Merge PR #1941 stereo box mapping through the source mask
2 parents 8d4384d + 69cebf5 commit d8f2ff6

6 files changed

Lines changed: 151 additions & 14 deletions

File tree

‎client/dive-common/use/autoPopulate.spec.ts‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,32 @@
11
import Track from 'vue-media-annotator/track';
22
import { headTailFeatures } from 'vue-media-annotator/headTail';
33
import {
4-
autoPopulatePrompt, autoPopulateTarget, boundsIoU, closedRing, polygonBounds, orientLineLike, LINE_PROMPT_FRACTIONS,
4+
autoPopulatePrompt, autoPopulateTarget, boundsIoU, closedRing, interiorPromptPoints, polygonBounds, orientLineLike,
5+
LINE_PROMPT_FRACTIONS,
56
} from './autoPopulate';
67

78
it('prompts a box with its centre', () => {
89
expect(autoPopulatePrompt({ source: 'box', bounds: [10, 20, 30, 60] })).toEqual({ points: [[20, 40]], labels: [1] });
910
expect(autoPopulatePrompt({ source: 'mask', polygons: [] })).toEqual({ points: [], labels: [] });
11+
expect(autoPopulatePrompt({ source: 'points', points: [[1, 2], [3, 4]] })).toEqual({ points: [[1, 2], [3, 4]], labels: [1, 1] });
12+
});
13+
14+
it('samples prompt points inside a mask, spread apart and clear of its holes', () => {
15+
const ring: [number, number][] = [[0, 0], [90, 0], [90, 30], [0, 30]];
16+
const hole: [number, number][] = [[40, 10], [50, 10], [50, 20], [40, 20]];
17+
const points = interiorPromptPoints([{ exterior: ring, holes: [hole] }], 4);
18+
expect(points).toHaveLength(4);
19+
points.forEach(([x, y]) => {
20+
expect(x).toBeGreaterThan(0); expect(x).toBeLessThan(90);
21+
expect(y).toBeGreaterThan(0); expect(y).toBeLessThan(30);
22+
expect(x >= 40 && x <= 50 && y >= 10 && y <= 20).toBe(false);
23+
});
24+
const xs = points.map((p) => p[0]);
25+
expect(Math.max(...xs) - Math.min(...xs)).toBeGreaterThan(60);
26+
// A thin diagonal sliver still yields points on the object, never the empty corners.
27+
const sliver: [number, number][] = [[0, 0], [100, 98], [100, 100], [0, 2]];
28+
interiorPromptPoints([{ exterior: sliver, holes: [] }]).forEach(([x, y]) => expect(Math.abs(x - y)).toBeLessThan(3));
29+
expect(interiorPromptPoints([{ exterior: [[0, 0], [1, 1]], holes: [] }])).toEqual([]);
1030
});
1131

1232
it('prompts a line with foreground points spread along its length', () => {

‎client/dive-common/use/autoPopulate.ts‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
import type Track from 'vue-media-annotator/track';
2-
import type { RectBounds } from 'vue-media-annotator/utils';
2+
import { pointInPolygon, RectBounds } from 'vue-media-annotator/utils';
33
import type { SegmentationPolygon } from 'dive-common/apispec';
44

55
export type Point = [number, number];
66

77
export type NewAnnotationGeometry =
88
| { source: 'box'; bounds: RectBounds }
99
| { source: 'line'; line: Point[] }
10+
/** Foreground prompts already known, e.g. points warped from the other stereo camera's mask. */
11+
| { source: 'points'; points: Point[] }
1012
/** A mask that already exists on the detection (point segmentation); nothing to predict. */
1113
| { source: 'mask'; polygons: SegmentationPolygon[] };
1214

@@ -24,6 +26,9 @@ export function autoPopulatePrompt(geometry: NewAnnotationGeometry): { points: P
2426
return { points: [[(x0 + x1) / 2, (y0 + y1) / 2]], labels: [1] };
2527
}
2628
if (geometry.source === 'mask') return { points: [], labels: [] };
29+
if (geometry.source === 'points') {
30+
return { points: geometry.points.map((p) => [...p] as Point), labels: geometry.points.map(() => 1) };
31+
}
2732
const { line } = geometry;
2833
if (line.length < 2) return { points: line.map((p) => [...p] as Point), labels: line.map(() => 1) };
2934
const total = line.slice(1).reduce((sum, p, i) => sum + Math.hypot(p[0] - line[i][0], p[1] - line[i][1]), 0);
@@ -72,6 +77,48 @@ export function boundsIoU(a: RectBounds, b: RectBounds): number {
7277
return union > 0 ? intersection / union : 0;
7378
}
7479

80+
/** Grid resolution across a mask's bounds when sampling prompt points inside it. */
81+
const INTERIOR_GRID = 9;
82+
83+
/**
84+
* Up to `count` points inside a mask, spread across it: the one nearest its
85+
* centre first, then repeatedly the candidate farthest from those chosen.
86+
* Points inside the object warp to the other camera far more reliably than a
87+
* box's corners, which sit on the background.
88+
*/
89+
export function interiorPromptPoints(polygons: SegmentationPolygon[], count = 5): Point[] {
90+
const xy = ([x, y]: Point) => ({ x, y });
91+
const valid = polygons.filter((polygon) => polygon.exterior.length >= 3);
92+
if (!valid.length) return [];
93+
const [x0, y0, x1, y1] = polygonBounds(valid.flatMap((polygon) => polygon.exterior));
94+
const inside: Point[] = [];
95+
for (let i = 0; i < INTERIOR_GRID; i += 1) {
96+
for (let j = 0; j < INTERIOR_GRID; j += 1) {
97+
const point: Point = [
98+
x0 + ((i + 0.5) / INTERIOR_GRID) * (x1 - x0),
99+
y0 + ((j + 0.5) / INTERIOR_GRID) * (y1 - y0),
100+
];
101+
if (valid.some((polygon) => pointInPolygon(xy(point), polygon.exterior.map(xy), polygon.holes.map((h) => h.map(xy))))) {
102+
inside.push(point);
103+
}
104+
}
105+
}
106+
if (!inside.length) return [];
107+
const centre: Point = [(x0 + x1) / 2, (y0 + y1) / 2];
108+
const distance = (a: Point, b: Point) => Math.hypot(a[0] - b[0], a[1] - b[1]);
109+
const chosen = [inside.reduce((best, p) => (distance(p, centre) < distance(best, centre) ? p : best))];
110+
while (chosen.length < count && chosen.length < inside.length) {
111+
const next = inside
112+
.filter((p) => !chosen.includes(p))
113+
.reduce((best, p) => {
114+
const gap = (q: Point) => Math.min(...chosen.map((c) => distance(q, c)));
115+
return gap(p) > gap(best) ? p : best;
116+
});
117+
chosen.push(next);
118+
}
119+
return chosen;
120+
}
121+
75122
/**
76123
* Head/tail derived independently on each stereo camera can come out flipped;
77124
* order this pair to run the same way as the other camera's line.

‎client/dive-common/use/useModeManager.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ export type NewAnnotationGeometryParams = {
8787
} & (
8888
| { source: 'box'; bounds: [number, number, number, number] }
8989
| { source: 'line'; line: [number, number][] }
90+
| { source: 'points'; points: [number, number][] }
9091
| { source: 'mask'; polygons: SegmentationPolygon[] }
9192
);
9293

‎client/platform/desktop/frontend/autoPopulate.spec.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,22 @@ it('derives head/tail for a point-segmented mask without predicting again', asyn
188188
expect(lined.track.getFeature(0)[0]?.head).toEqual([2, 2]);
189189
});
190190

191+
it('segments from given prompt points, boxes the mask, and reports it even when nothing is stored', async () => {
192+
const h = harness();
193+
const onMask = vi.fn();
194+
h.services.predict.mockResolvedValue({ success: true, polygons: components });
195+
expect(await populateAnnotation({
196+
camera: 'right', trackId: 1, frameNum: 0, source: 'points', points: [[3, 3], [25, 5]],
197+
}, { mask: false, points: false, onMask }, h.services)).toBe('applied');
198+
expect(h.services.predict).toHaveBeenCalledWith(expect.objectContaining({
199+
points: [[3, 3], [25, 5]], pointLabels: [1, 1], multimaskOutput: false,
200+
}));
201+
expect(onMask).toHaveBeenCalledWith(components);
202+
expect(h.track.getFeature(0)[0]?.bounds).toEqual([0, 0, 30, 10]);
203+
expect(h.track.getPolygonFeatures(0)).toHaveLength(0);
204+
expect(h.services.keypoints).not.toHaveBeenCalled();
205+
});
206+
191207
it('extracts points from all components when storing the mask is disabled', async () => {
192208
const h = harness();
193209
h.services.predict.mockResolvedValue({ success: true, polygons: components });

‎client/platform/desktop/frontend/autoPopulate.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ export interface AutoPopulateOptions {
2323
orientLike?: [number, number][] | null;
2424
/** The box was mapped from the other stereo camera: refit it when the mask disagrees with it. */
2525
fitBoxToMask?: boolean;
26+
/** Receives the mask the annotation was segmented to, whether or not it is stored. */
27+
onMask?: (polygons: SegmentationPolygon[]) => void;
2628
}
2729

2830
async function predictMask(
@@ -62,6 +64,7 @@ export default async function populateAnnotation(
6264
}
6365
const target = currentTarget();
6466
if (!target) return 'changed';
67+
options.onMask?.(polygons);
6568
const { track, feature } = target;
6669
const features = feature.geometry?.features ?? [];
6770
const hasPolygon = features.some((f) => f.geometry.type === 'Polygon');
@@ -78,7 +81,7 @@ export default async function populateAnnotation(
7881
})));
7982
}
8083
const maskBounds = polygonBounds(polygons.flatMap((polygon) => polygon.exterior));
81-
if (options.points && params.source === 'line') {
84+
if ((options.points && params.source === 'line') || params.source === 'points') {
8285
track.setFeature({ frame: params.frameNum, keyframe: true, bounds: maskBounds });
8386
} else if (options.fitBoxToMask && params.source === 'box'
8487
&& boundsIoU(params.bounds, maskBounds) < MAPPED_BOX_MIN_IOU) {

‎client/platform/desktop/frontend/components/ViewerLoader.vue‎

Lines changed: 61 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import CalibrationMenu from 'dive-common/components/CalibrationMenu.vue';
1616
import SidebarContext from 'dive-common/components/SidebarContext.vue';
1717
import context from 'dive-common/store/context';
1818
import { usePrompt } from 'dive-common/vue-utilities/prompt-service';
19-
import { SegmentationPredictRequest } from 'dive-common/apispec';
19+
import { SegmentationPredictRequest, SegmentationPolygon } from 'dive-common/apispec';
2020
import { clientSettings } from 'dive-common/store/settings';
2121
import { isStereoscopicDatasetConfig } from 'dive-common/multicamDisplay';
2222
import type {
@@ -45,6 +45,7 @@ import {
4545
openLink,
4646
setScoringAnnotationPreviewFile,
4747
} from 'platform/desktop/frontend/api';
48+
import { interiorPromptPoints, polygonBounds } from 'dive-common/use/autoPopulate';
4849
import populateAnnotation from '../autoPopulate';
4950
import Export from './Export.vue';
5051
import JobTab from './JobTab.vue';
@@ -1186,7 +1187,9 @@ export default defineComponent({
11861187
11871188
// In-flight auto-populate passes for freshly drawn shapes, so the stereo
11881189
// transfer of the same shape can populate its mapped copy afterwards.
1189-
const pendingAutoPopulate = new Map<string, Promise<void>>();
1190+
/** Outcome of one auto-populate pass; the mask is reported even when only points were asked for. */
1191+
type AutoPopulateOutcome = { status: 'applied' | 'changed' | 'failed' | 'skipped'; polygons: SegmentationPolygon[] };
1192+
const pendingAutoPopulate = new Map<string, Promise<AutoPopulateOutcome>>();
11901193
const autoPopulateKey = (camera: string, trackId: number, frameNum: number) => `${camera}:${trackId}:${frameNum}`;
11911194
11921195
// eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -1785,6 +1788,9 @@ export default defineComponent({
17851788
});
17861789
}
17871790
}
1791+
} else if (params.type === 'box'
1792+
&& sourceAutoPopulate && await mapBoxThroughSourceMask(sourceAutoPopulate, params, otherCamera)) {
1793+
// The other camera's box came from its own segmentation.
17881794
} else if (params.type === 'box') {
17891795
// Convert box bounds to 4 corner points
17901796
const [x1, y1, x2, y2] = params.bounds;
@@ -2130,18 +2136,21 @@ export default defineComponent({
21302136
async function autoPopulateGeometry(
21312137
params: NewAnnotationGeometryParams,
21322138
stereo?: { orientLike: [number, number][] | null; fitBoxToMask: boolean },
2133-
) {
2139+
): Promise<AutoPopulateOutcome> {
21342140
const { autoPopulateMask, autoPopulatePoints } = clientSettings.trackSettings.newTrackSettings;
2135-
if (!autoPopulateMask && !autoPopulatePoints) return;
21362141
const cameraStore = viewerRef.value?.cameraStore;
2137-
if (!cameraStore) return;
2142+
if ((!autoPopulateMask && !autoPopulatePoints) || !cameraStore) return { status: 'skipped', polygons: [] };
2143+
let polygons: SegmentationPolygon[] = [];
21382144
// Resolve the requested camera independently of recipe initialization and
21392145
// whichever camera is selected when asynchronous work finishes.
21402146
const datasetId = params.camera === 'singleCam' ? props.id : `${props.id}/${params.camera}`;
21412147
autoPopulateActive.value += 1;
21422148
try {
21432149
const result = await populateAnnotation(params, {
2144-
mask: autoPopulateMask, points: autoPopulatePoints, ...stereo,
2150+
mask: autoPopulateMask,
2151+
points: autoPopulatePoints,
2152+
onMask: (mask) => { polygons = mask; },
2153+
...stereo,
21452154
}, {
21462155
getTrack: () => cameraStore.getPossibleTrack(params.trackId, params.camera),
21472156
getMedia: async () => {
@@ -2166,10 +2175,12 @@ export default defineComponent({
21662175
if (result === 'changed' && cameraStore.getPossibleTrack(params.trackId, params.camera)) {
21672176
autoPopulateMessage.value = `Auto-populate skipped for track ${params.trackId}: the annotation changed while the request was running.`;
21682177
}
2178+
return { status: result, polygons };
21692179
} catch (err) {
21702180
const message = err instanceof Error ? err.message : String(err);
21712181
autoPopulateMessage.value = `Auto-populate — track ${params.trackId}, ${params.camera}, frame ${params.frameNum}: ${message}`;
21722182
console.warn('[AutoPopulate] Could not populate the new annotation:', err);
2183+
return { status: 'failed', polygons };
21732184
} finally {
21742185
autoPopulateActive.value -= 1;
21752186
}
@@ -2181,15 +2192,15 @@ export default defineComponent({
21812192
* so a derived head/tail can follow the source's direction.
21822193
*/
21832194
async function autoPopulateOtherCamera(
2184-
sourceJob: Promise<void>,
2195+
sourceJob: Promise<unknown>,
21852196
sourceCamera: string,
21862197
mapped: NewAnnotationGeometryParams,
2187-
) {
2198+
): Promise<AutoPopulateOutcome> {
21882199
await sourceJob;
21892200
const cameraStore = viewerRef.value?.cameraStore;
2190-
if (!cameraStore) return;
2201+
if (!cameraStore) return { status: 'skipped', polygons: [] };
21912202
const sourceTrack = cameraStore.getPossibleTrack(mapped.trackId, sourceCamera);
2192-
await autoPopulateGeometry(mapped, {
2203+
const outcome = await autoPopulateGeometry(mapped, {
21932204
orientLike: getStereoLineEndpoints(sourceTrack, mapped.frameNum),
21942205
fitBoxToMask: true,
21952206
});
@@ -2200,6 +2211,45 @@ export default defineComponent({
22002211
console.warn('[Stereo] Measurement update failed:', err);
22012212
}
22022213
}
2214+
return outcome;
2215+
}
2216+
2217+
/**
2218+
* With auto-segmentation on, a new box reaches the other camera through
2219+
* its mask rather than its corners: a few points inside the source mask
2220+
* are warped (their disparities are stable, unlike corners that sit on the
2221+
* background), the other camera is segmented from them, and its box is
2222+
* that mask's bounds. False means fall back to warping the corners.
2223+
*/
2224+
async function mapBoxThroughSourceMask(
2225+
sourceJob: Promise<AutoPopulateOutcome>,
2226+
params: { camera: string; trackId: number; frameNum: number },
2227+
otherCamera: string,
2228+
): Promise<boolean> {
2229+
const source = await sourceJob;
2230+
const prompts = interiorPromptPoints(source.polygons);
2231+
if (!prompts.length) return false;
2232+
const response = await stereoTransferPoints({ points: prompts });
2233+
const warped = (response.transferredPoints ?? [])
2234+
.filter((p): p is [number, number] => !!p && Number.isFinite(p[0]) && Number.isFinite(p[1]));
2235+
const cameraStore = viewerRef.value?.cameraStore;
2236+
if (!response.success || !warped.length || !cameraStore) return false;
2237+
const track = getOrCreateStereoTrack(cameraStore, params.trackId, params.camera, otherCamera, params.frameNum);
2238+
if (!track) return false;
2239+
// A placeholder for the segmentation to replace; it needs a feature to target.
2240+
track.setFeature({
2241+
frame: params.frameNum,
2242+
flick: 0,
2243+
bounds: polygonBounds(warped),
2244+
keyframe: true,
2245+
interpolate: false,
2246+
});
2247+
const outcome = await autoPopulateOtherCamera(Promise.resolve(), params.camera, {
2248+
camera: otherCamera, trackId: params.trackId, frameNum: params.frameNum, source: 'points', points: warped,
2249+
});
2250+
if (outcome.status === 'applied' || outcome.status === 'changed') return true;
2251+
track.deleteFeature(params.frameNum);
2252+
return false;
22032253
}
22042254
22052255
/** A brand-new box or line just got drawn, or a brand-new mask point-segmented. */
@@ -2208,7 +2258,7 @@ export default defineComponent({
22082258
if (!autoPopulateMask && !autoPopulatePoints) return;
22092259
if (params.source === 'mask' && !autoPopulatePoints) return;
22102260
const key = autoPopulateKey(params.camera, params.trackId, params.frameNum);
2211-
const job: Promise<void> = autoPopulateGeometry(params).finally(() => {
2261+
const job: Promise<AutoPopulateOutcome> = autoPopulateGeometry(params).finally(() => {
22122262
if (pendingAutoPopulate.get(key) === job) pendingAutoPopulate.delete(key);
22132263
});
22142264
pendingAutoPopulate.set(key, job);

0 commit comments

Comments
 (0)

Back | FazBrowse Home | New Git URL