canvas: the marquee hinting follows a rotated view like the release does (#2732)

The live ants for the Rectangular/Elliptical Marquee (and Object Selection)
were an axis-aligned screen box built from the two projected corners of the
drag. A rotated view projects the drag's document rectangle onto a rotated
one, and the box through two of its corners is a different shape in a
different place: on a 30 degree view a corner landed 21 x 50 px away from
where the selection it previews was committed. The release was always right,
only the hinting was drawn elsewhere (#2512).

marquee_drag_ants builds the outline in document space and maps every point
through the view instead: four corners for the rectangle, the ellipse
inscribed in it for the elliptical marquee, with as many vertices as its size
on screen asks for. Every point is snapped to the half-pixel grid like every
other ants outline (marching_ants_segments), so the lines stay crisp at any
zoom. Unrotated the mapping is only a scale, and the corners come out where
they always did.

Tests drag a marquee in a 30 degree view for both tools and check the drawn
outline against the selection the release commits: each corner is the mapped
corner of the dragged rectangle, and each point of the ellipse maps back onto
it.

Co-authored-by: switch87 <8284204+switch87@users.noreply.github.com>

crates/ui-egui/src/canvas.rs

@@ -217,6 +217,47 @@ pub fn marquee_px(a: [f64; 2], b: [f64; 2]) -> [f64; 4] {
     [a[0].min(b[0]).floor(), a[1].min(b[1]).floor(), a[0].max(b[0]).ceil(), a[1].max(b[1]).ceil()]
 }
 
+/// The marching ants of a marquee drag: its rectangle, or the ellipse inscribed in it, built in
+/// document space and mapped point by point through the view, so the hinting lands where the
+/// release commits the selection.
+///
+/// A rotated view projects the rectangle onto a rotated one, and an axis-aligned screen box
+/// through two of its corners is a different shape in a different place: the commit was always
+/// right, only the hinting was drawn elsewhere (#2512). Every point is snapped to the half-pixel
+/// grid like every other ants outline (`marching_ants_segments`), so the lines stay crisp at any
+/// zoom (#172).
+pub(crate) fn marquee_ants(xf: &ViewXform, tool: Tool, a: [f64; 2], b: [f64; 2]) -> Vec<Pos2> {
+    let (x0, x1) = (a[0].min(b[0]), a[0].max(b[0]));
+    let (y0, y1) = (a[1].min(b[1]), a[1].max(b[1]));
+    let at = |x: f64, y: f64| {
+        let p = xf.to_screen(x as f32, y as f32);
+        pos2(p.x.round() + 0.5, p.y.round() + 0.5)
+    };
+    if tool != Tool::EllipseMarquee {
+        return vec![at(x0, y0), at(x1, y0), at(x1, y1), at(x0, y1)];
+    }
+    // As many vertices as the same ellipse would get drawn on screen (`ellipse_points`).
+    let scr = Rect::from_points(&[xf.to_screen(x0 as f32, y0 as f32), xf.to_screen(x1 as f32, y1 as f32)]);
+    let n = (((scr.width() + scr.height()) * 0.25) as usize).clamp(16, 256);
+    let (cx, cy, rx, ry) = ((x0 + x1) * 0.5, (y0 + y1) * 0.5, (x1 - x0) * 0.5, (y1 - y0) * 0.5);
+    (0..n)
+        .map(|i| {
+            let t = i as f64 / n as f64 * std::f64::consts::TAU;
+            at(cx + rx * t.cos(), cy + ry * t.sin())
+        })
+        .collect()
+}
+
+/// The ants of the marquee being drawn: `marquee` when the drag has one (the options-bar style
+/// and the modifiers applied), the raw press and pointer corners otherwise. This is the whole of
+/// what the hinting draws for the selection tools, so the test in `marquee_tests` checks it
+/// against the selection the release commits (#2512).
+pub(crate) fn marquee_drag_ants(d: &Drag, xf: &ViewXform, marquee: Option<[f64; 4]>) -> Vec<Pos2> {
+    let last = d.points.last().map_or(d.start, |p| [p[0], p[1]]);
+    let (a, b) = marquee.map_or((d.start, last), |[x0, y0, x1, y1]| ([x0, y0], [x1, y1]));
+    marquee_ants(xf, d.tool, a, b)
+}
+
 /// The size readout shown beside the cursor while dragging a marquee: the width and height values.
 pub fn marquee_readout(r: [f64; 4]) -> [String; 2] {
     [format!("{} px", r[2] - r[0]), format!("{} px", r[3] - r[1])]
@@ -3669,15 +3710,9 @@ fn draw_drag_preview(app: &mut PhotocraftApp, painter: &egui::Painter, xf: &View
         // Nothing selected yet: no outline, as the release would deselect.
         Tool::RectMarquee | Tool::EllipseMarquee if marquee.is_none() => {}
         Tool::RectMarquee | Tool::EllipseMarquee | Tool::ObjectSelection => {
-            // Marching ants, visible on any pixels (#172).
-            let (a, b) = marquee.map_or((d.start, last), |[x0, y0, x1, y1]| ([x0, y0], [x1, y1]));
-            let r = Rect::from_two_pos(xf.to_screen(a[0] as f32, a[1] as f32), xf.to_screen(b[0] as f32, b[1] as f32));
-            let r = Rect::from_min_max(r.min.round() + vec2(0.5, 0.5), r.max.round() + vec2(0.5, 0.5));
-            let pts = if d.tool == Tool::EllipseMarquee {
-                crate::tool_feedback::ellipse_points(r)
-            } else {
-                vec![r.left_top(), r.right_top(), r.right_bottom(), r.left_bottom()]
-            };
+            // Marching ants, visible on any pixels (#172): built in document space and mapped
+            // corner by corner, so a rotated view shows them where the release commits (#2512).
+            let pts = marquee_drag_ants(d, xf, marquee);
             crate::tool_feedback::draw_ants(painter, &pts, true);
         }
         // Patch / Content-Aware Move dragging the selection: its outline follows the pointer.

crates/ui-egui/src/marquee_tests.rs

@@ -2,7 +2,7 @@
 //! ⌥ draws from the centre (both together work too), a "W × H px" readout follows the cursor, and
 //! modifiers held before the drag still pick add/subtract (#188) instead of constraining.
 
-use egui::{Modifiers, PointerButton, Pos2, vec2};
+use egui::{Modifiers, PointerButton, Pos2, pos2, vec2};
 use egui_kittest::Harness;
 use egui_kittest::kittest::Queryable;
 use photocraft_geom::Rect;
@@ -40,17 +40,20 @@ fn harness(tool: Tool) -> Harness<'static, PhotocraftApp> {
     h
 }
 
-fn screen(h: &Harness<'static, PhotocraftApp>, x: f32, y: f32) -> Pos2 {
+fn view_xform(h: &Harness<'static, PhotocraftApp>) -> ViewXform {
     let app = h.state();
     let v = &app.ui.views[0];
-    let xf = ViewXform {
+    ViewXform {
         rect: crate::rulers::content_rect(app, app.last_canvas_rect),
         zoom: v.zoom,
         center: v.center,
         flip: app.ui.view.flip_horizontal,
         rotation: v.rotation,
-    };
-    xf.to_screen(x, y)
+    }
+}
+
+fn screen(h: &Harness<'static, PhotocraftApp>, x: f32, y: f32) -> Pos2 {
+    view_xform(h).to_screen(x, y)
 }
 
 fn mods(h: &mut Harness<'static, PhotocraftApp>, m: Modifiers) {
@@ -184,6 +187,58 @@ fn the_live_outline_snaps_to_pixels_and_matches_the_committed_selection() {
     }
 }
 
+/// #2512: the hinting must follow the same transform as the release. With a rotated view the
+/// marquee projects onto a rotated rectangle, and an axis-aligned screen box drawn through its
+/// two diagonal corners is a different shape somewhere else entirely.
+#[test]
+fn the_live_outline_matches_the_committed_selection_in_a_rotated_view() {
+    for tool in [Tool::RectMarquee, Tool::EllipseMarquee] {
+        let mut h = harness(tool);
+        let v = &mut h.state_mut().ui.views[0];
+        v.rotation = 30.0;
+        h.run_steps(2);
+        press_at(&mut h, 50.0, 50.0, Modifiers::NONE);
+        move_to(&mut h, 150.0, 90.0);
+        let xf = view_xform(&h);
+        let (live, pts) = {
+            let app = h.state();
+            let d = app.drag.as_ref().expect("the drag is on");
+            let live = crate::canvas::marquee_preview_px(&app.ui.tool_options, d, app.point_zoom()).expect("a marquee is dragged");
+            // Exactly what the canvas paints for this drag.
+            (live, crate::canvas::marquee_drag_ants(d, &xf, Some(live)))
+        };
+        release_at(&mut h, 150.0, 90.0, Modifiers::NONE);
+        let sel = selection(&h);
+        // The pointer round-trips through f32 screen points, so a corner can land a hair over a
+        // pixel edge; what matters is that the rotation moves neither the selection nor the
+        // hinting, and that the release commits what was shown.
+        assert!(
+            (sel.x0 as f64 - live[0]).abs() <= 1.0
+                && (sel.y0 as f64 - live[1]).abs() <= 1.0
+                && (sel.x1 as f64 - live[2]).abs() <= 1.0
+                && (sel.y1 as f64 - live[3]).abs() <= 1.0,
+            "{tool:?}: the release commits what the hinting showed: {sel:?} vs {live:?}"
+        );
+        if tool == Tool::RectMarquee {
+            // Each corner is a corner of the dragged rectangle, mapped: an axis-aligned box
+            // through the two diagonal corners puts all four elsewhere.
+            let snap = |p: Pos2| pos2(p.x.round() + 0.5, p.y.round() + 0.5);
+            let want = [[live[0], live[1]], [live[2], live[1]], [live[2], live[3]], [live[0], live[3]]].map(|[x, y]| snap(xf.to_screen(x as f32, y as f32)));
+            for (i, (p, w)) in pts.iter().zip(want).enumerate() {
+                assert!((p.x - w.x).abs() < 0.01 && (p.y - w.y).abs() < 0.01, "{tool:?}: corner {i} at {p:?}, the dragged rectangle's corner is at {w:?}");
+            }
+        } else {
+            // Every point maps back onto the ellipse the drag drew.
+            let (cx, cy, rx, ry) = ((live[0] + live[2]) * 0.5, (live[1] + live[3]) * 0.5, (live[2] - live[0]) * 0.5, (live[3] - live[1]) * 0.5);
+            for p in &pts {
+                let [x, y] = xf.to_doc(*p);
+                let e = ((x - cx) / rx).powi(2) + ((y - cy) / ry).powi(2);
+                assert!((e - 1.0).abs() < 0.08, "{tool:?}: {p:?} maps back to ({x},{y}), off the dragged ellipse (e = {e})");
+            }
+        }
+    }
+}
+
 /// 800 %, snapping off, the view centred on (30, 30): the pointer lands between pixel edges.
 fn zoomed(tool: Tool) -> Harness<'static, PhotocraftApp> {
     let mut h = harness(tool);