denise: correct out-of-range BPLCON0/BPLCON2 mode and priority decoding

src/video/bitplane/output.rs

@@ -229,6 +229,26 @@ pub(super) fn denise_playfield_output(
     }
 
     if control.hold_and_modify() {
+        if control.dual_playfield() {
+            // Invalid HAM + dual-playfield combination. Denise resolves the
+            // dual-playfield colour index and then runs it through the HAM
+            // logic: the HAM control code still comes from the raw plane-5/6
+            // bits, but the value nibble (and the "set" palette index) is the
+            // dual-playfield-resolved index, not the raw plane bits (vAmiga
+            // translateDPF writes mBuffer with the resolved index, then
+            // colorizeHAM takes the control from dBuffer bits 4-5). No real
+            // software sets both bits; regression coverage is vAmigaTS
+            // Denise/BPLCON0/modes4 and invprio3.
+            let (pf_mask, color_idx) = dual_playfield_pixel(idx, control);
+            let ham_code = ((idx >> 4) & 0x03) << 4 | ((color_idx as u8) & 0x0F);
+            let previous = rgb24_to_rgb12_hi(*ham_color);
+            *ham_color = rgb12_to_rgb24(ham6_rgb12(palette, ham_code, previous));
+            return DenisePlayfieldOutput {
+                color: *ham_color,
+                color_latch: palette.get(color_idx).copied().unwrap_or(0),
+                pf_mask,
+            };
+        }
         let previous = rgb24_to_rgb12_hi(*ham_color);
         *ham_color = rgb12_to_rgb24(ham6_rgb12(palette, idx, previous));
         return DenisePlayfieldOutput {
@@ -250,6 +270,19 @@ pub(super) fn denise_playfield_output(
         };
     }
 
+    // A single playfield whose BPLCON2 PF2 priority code is programmed out of
+    // range (5-7) eliminates the four low bitplanes wherever the fifth
+    // bitplane is set, keeping only bitplanes 5-6, and forces the pixel to
+    // background sprite priority (vAmiga translateSPF; the quirk does not
+    // happen in HAM mode, already returned above). Real software only uses
+    // codes 0-4, so valid single-playfield content is unaffected.
+    let invalid_pf2_priority = control.playfield_priority_code(2) > 4;
+    let (idx, pf_mask) = if invalid_pf2_priority {
+        let idx = if idx & 0x10 != 0 { idx & 0x30 } else { idx };
+        (idx, 0)
+    } else {
+        (idx, u8::from(idx != 0) * 2)
+    };
     let color_latch = palette[(idx as usize) & 0x1F];
     let color = rgb12_to_rgb24(palette_index_to_rgb12(
         palette,
@@ -260,7 +293,7 @@ pub(super) fn denise_playfield_output(
     DenisePlayfieldOutput {
         color,
         color_latch,
-        pf_mask: u8::from(idx != 0) * 2,
+        pf_mask,
     }
 }
 
@@ -355,13 +388,22 @@ pub(super) fn dual_playfield_pixel(idx: u8, control: ControlState) -> (u8, usize
         pf2 |= (idx >> 4) & 0x08;
     }
     let pf2_offset = control.pf2_palette_offset();
-    match (pf1, pf2) {
-        (0, 0) => (0, 0),
-        (pf, 0) => (1, pf as usize),
-        (0, pf) => (2, pf2_offset + pf as usize),
-        (_, pf2) if control.pf2_priority() => (2, pf2_offset + pf2 as usize),
-        (pf1, _) => (1, pf1 as usize),
+    let (winner, pf_mask, color_idx) = match (pf1, pf2) {
+        (0, 0) => return (0, 0),
+        (pf, 0) => (1u8, 1u8, pf as usize),
+        (0, pf) => (2, 2, pf2_offset + pf as usize),
+        (_, pf2) if control.pf2_priority() => (2, 2, pf2_offset + pf2 as usize),
+        (pf1, _) => (1, 1, pf1 as usize),
+    };
+    // A playfield whose BPLCON2 priority code is programmed out of range
+    // (> 4) is drawn transparent: the winning field's pixels collapse to the
+    // background rather than showing the field behind it (vAmiga zPF returns
+    // 0 for codes 5-7, which masks that field's index to 0). Real software
+    // only uses codes 0-4, so valid dual-playfield content is unaffected.
+    if control.playfield_priority_code(winner) > 4 {
+        return (0, 0);
     }
+    (pf_mask, color_idx)
 }
 
 pub(super) fn half_brite_rgb12(color: u16) -> u16 {

src/video/bitplane/tests.rs

@@ -4460,6 +4460,66 @@ fn dual_playfield_uses_separate_palette_banks_and_priority() {
     assert_eq!(dual_playfield_palette_index(0b000011, pf2_priority), 9);
 }
 
+#[test]
+fn dual_playfield_out_of_range_priority_draws_the_field_transparent() {
+    // A dual playfield whose BPLCON2 priority code is programmed out of range
+    // (5-7) is drawn transparent: the winning field's pixels collapse to the
+    // background instead of revealing the field behind it (vAmiga zPF returns
+    // 0 for codes 5-7). vAmigaTS Denise/BPLCON0/invprio. Valid codes (0-4)
+    // are unaffected, so real dual-playfield content is unchanged.
+    let invalid_pf2 = ControlState {
+        bplcon0: 0x6400, // 4 planes, dual playfield
+        bplcon2: 0x0038, // PF2 priority code 7 (invalid), PF1 code 0
+        bplcon3: BPLCON3_PF2OF_DEFAULT,
+        ..ControlState::default()
+    };
+    // A PF2-only pixel is transparent (the winning field's priority is out of
+    // range); a PF1-only pixel still resolves (PF1 code 0 is valid).
+    assert_eq!(dual_playfield_pixel(0b0000_0010, invalid_pf2), (0, 0));
+    assert_eq!(dual_playfield_pixel(0b0000_0001, invalid_pf2), (1, 1));
+
+    // With a valid PF2 priority the same PF2 pixel resolves normally.
+    let valid = ControlState {
+        bplcon2: 0x0004, // PF2 code 0, PF1 code 4 (both valid)
+        ..invalid_pf2
+    };
+    assert_eq!(dual_playfield_pixel(0b0000_0010, valid), (2, 9));
+}
+
+#[test]
+fn single_playfield_out_of_range_priority_eliminates_low_bitplanes() {
+    // A single playfield with an out-of-range BPLCON2 PF2 priority code keeps
+    // only bitplanes 5-6 wherever bitplane 5 is set (eliminating the four low
+    // planes) and forces background sprite priority (vAmiga translateSPF).
+    // Valid codes leave the pixel untouched.
+    let mut palette = Palette::new();
+    palette.write_ocs(0x10, 0x0F00);
+    palette.write_ocs(0x13, 0x000F);
+
+    let invalid = ControlState {
+        bplcon0: 0x5000, // 5 planes, single playfield (no HAM/dual/EHB)
+        bplcon2: 0x0038, // PF2 priority code 7 (invalid)
+        ..ControlState::default()
+    };
+    let valid = ControlState {
+        bplcon2: 0x0004, // PF1 code 4, PF2 code 0 (both valid)
+        ..invalid
+    };
+
+    // Planes 1,2,5 set: the invalid priority eliminates planes 1-2, leaving
+    // the plane-5-only index, so the pixel matches a valid render of 0x10.
+    let mut h = 0u32;
+    let invalid_13 = denise_playfield_output(invalid, palette, 0x13, &mut h);
+    let mut h = 0u32;
+    let valid_10 = denise_playfield_output(valid, palette, 0x10, &mut h);
+    let mut h = 0u32;
+    let valid_13 = denise_playfield_output(valid, palette, 0x13, &mut h);
+    assert_eq!(invalid_13.color, valid_10.color);
+    assert_ne!(invalid_13.color, valid_13.color);
+    assert_eq!(invalid_13.pf_mask, 0);
+    assert_eq!(valid_13.pf_mask, 2);
+}
+
 #[test]
 fn aga_dual_playfield_decodes_bitplane7_into_pf1_fourth_bit() {
     // AGA Lisa dual playfield gives each field four bits: bitplane 7
@@ -5448,6 +5508,47 @@ fn denise_playfield_output_selects_ehb_ham_and_dual_playfield_colors() {
     );
 }
 
+#[test]
+fn ham_dual_playfield_runs_the_resolved_index_through_ham() {
+    // The invalid HAM + dual-playfield combination (both BPLCON0 bits set --
+    // no real software does this) resolves the dual-playfield colour index
+    // and then runs it through the HAM logic: the HAM control code comes from
+    // the raw plane 5/6 bits while the value nibble is the resolved index.
+    // Matches vAmiga (translateDPF writes the resolved index, colorizeHAM
+    // takes the control from the raw bitplane bits); exact on vAmigaTS
+    // Denise/BPLCON0/modes4.
+    let mut palette = Palette::new();
+    palette.write_ocs(9, 0x0456);
+    let control = ControlState {
+        bplcon0: 0x6C00, // BPU6 + HAM + dual playfield
+        bplcon3: BPLCON3_PF2OF_DEFAULT,
+        ..ControlState::default()
+    };
+    assert!(control.hold_and_modify() && control.dual_playfield());
+
+    // Planes 5 and 6 clear -> HAM "set": plane 2 resolves to PF2 index 1 at
+    // the default PF2 palette offset 8 (entry 9), shown directly, exactly as
+    // the plain dual-playfield path would.
+    let mut ham_color = rgb12_to_rgb24(0x0ABC);
+    assert_eq!(
+        denise_playfield_output(control, palette, 0x02, &mut ham_color),
+        DenisePlayfieldOutput {
+            color: rgb12_to_rgb24(0x0456),
+            color_latch: 0x0456,
+            pf_mask: 2,
+        }
+    );
+
+    // Plane 5 set supplies the HAM "modify blue" control code; the value
+    // nibble is the resolved PF1 index (planes 1/3/5 -> plane 5 only -> 4),
+    // so only the blue nibble of the held colour changes.
+    let mut ham_color = rgb12_to_rgb24(0x0ABC);
+    assert_eq!(
+        denise_playfield_output(control, palette, 0x12, &mut ham_color).color,
+        rgb12_to_rgb24(0x0AB4),
+    );
+}
+
 /// Plan 3.3: the Lisa resolution path. BPLAM XORs the pixel index,
 /// HAM8 modifies six bits per component, EHB halves in 8-bit space,
 /// and palette lookups read the full 24-bit banked store.