diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 20f0991b8..93a2ed6e4 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -475,6 +475,7 @@ pub struct LastPublishedZone { #[derive(Deserialize, Serialize, Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum Progress { Waiting, + Restoring, Loading, LoadedReview, HaltLoaded, diff --git a/crates/cli/src/commands/zone.rs b/crates/cli/src/commands/zone.rs index afc1f14a8..9a71fc8aa 100644 --- a/crates/cli/src/commands/zone.rs +++ b/crates/cli/src/commands/zone.rs @@ -640,7 +640,7 @@ impl Zone { // Output information per step progressed until the first still // in-progress/aborted step or show all steps if all have completed. println!(""); - print_status(zone.progress, &zone, &policy); + print_status(&zone, &policy); if zone.last_published.is_some() { println!(""); @@ -703,8 +703,11 @@ impl Zone { } } -pub fn print_status(current: Progress, zone: &ZoneStatus, policy: &PolicyInfo) { - let progress = match zone.progress { +pub fn print_status(zone: &ZoneStatus, policy: &PolicyInfo) { + let current = zone.progress; + + let progress = match current { + Progress::Restoring => "restoring", Progress::Waiting => "idle", Progress::Loading => "loading", Progress::LoadedReview => "waiting for loaded review", @@ -717,7 +720,7 @@ pub fn print_status(current: Progress, zone: &ZoneStatus, policy: &PolicyInfo) { println!("status: {}{progress}{}", ansi::BLUE, ansi::RESET); - if current == Progress::Waiting { + if matches!(current, Progress::Waiting | Progress::Restoring) { return; } diff --git a/crates/zonedata/src/restorer.rs b/crates/zonedata/src/restorer.rs index 3c5c68cba..2203cfec1 100644 --- a/crates/zonedata/src/restorer.rs +++ b/crates/zonedata/src/restorer.rs @@ -197,6 +197,11 @@ impl LoadedZoneRestorer { .find(|inst| inst.soa.is_some()) .map(LoadedZoneReader::new) } + + /// The diff from the preceding loaded instance to the current one. + pub fn take_diff(&mut self) -> Option> { + self.diff.take() + } } impl LoadedZoneRestorer { @@ -424,6 +429,20 @@ impl SignedZoneRestorer { pub fn clear(&mut self) { // Initialize the absolute data. + // SAFETY: As per the caller, 'loaded[index]' will not be + // accessed elsewhere for the lifetime of 'self', and so is sound to + // access mutably. + let next = unsafe { &mut *self.data.loaded[self.index as usize].get() }; + next.soa = None; + next.records.clear(); + + // SAFETY: As per the caller, 'loaded[!index]' will not be + // accessed elsewhere for the lifetime of 'self', and so is sound to + // access mutably. + let curr = unsafe { &mut *self.data.loaded[!self.index as usize].get() }; + curr.soa = None; + curr.records.clear(); + // SAFETY: As per the caller, 'signed[index]' will not be // accessed elsewhere for the lifetime of 'self', and so is sound to // access mutably. @@ -469,6 +488,11 @@ impl SignedZoneRestorer { .find(|inst| inst.soa.is_some()) .map(|inst| SignedZoneReader::new(curr_loaded, inst)) } + + /// The diff from the preceding signed instance to the current one. + pub fn take_diff(&mut self) -> Option> { + self.diff.take() + } } impl SignedZoneRestorer { diff --git a/crates/zonedata/src/storage/mod.rs b/crates/zonedata/src/storage/mod.rs index d979c46de..8cf62f863 100644 --- a/crates/zonedata/src/storage/mod.rs +++ b/crates/zonedata/src/storage/mod.rs @@ -31,6 +31,32 @@ mod transitions; /// reviewed, and switched to. While it requires `&mut` access to be modified, /// it is designed to live in a (synchronous) mutex -- expensive operations on /// the zone are always achievable without `&mut` access. +/// +/// ```text +/// ╔═══════════════════╗ finish ╔═══════════════════╗ finish +/// ║ RestoringLoaded ╠═════════▶ RestoringSigned ╠════════╗ +/// ╚══╤════════════════╝ ╚══╤════════════════╝ ║ +/// │ abandon │ abandon ║ +/// ┌──▼─────────────────────────────▼─────────────────────────▼─────────┐ +/// │ Passive │ +/// └─────────╥────────────────────────────────────────────────────────▲─┘ +/// ║ │ mark_complete +/// ┌─────────║──────────────────────────────────────────────────────────┐ ╔═════════════╗ start ╔════════════════════╗ approve +/// │ ║ Cleaning <═══════════════║ Switching <═══════════║ PersistingSigned <═══════════════════════════════════════════════════════════════════▲ +/// └─────────║──────▲─────────────────────────────────────────────────▲─┘ ╚═════════════╝ ╚════════════════════╝ ║ +/// ║ │ │ stop_review ║ +/// ║ │ ┌─────────────────────┐ stop_review ┌────────────────────┐ give_up ║ +/// load ║ │ give_up │ CleanLoadedPending <──────────────────────────│ CleanWholePending <─────────────────────────────────────────────────────┐ ║ +/// ║ │ └────▲────────────────┘ └────────────────────┘ │ ║ +/// ▼ │ │ give_up │ ║ +/// ╔═════════╧═╗ finish ╔══════════════════════════╗ start ╔═══════════════════╗ approve ╔════════════════════╗ complete ╔══▼═════════════╗ finish ╔═══════════════════════╗ start ╔══════════════════╗ +/// ║ Loading ╠════════▶ ReviewingLoadedPending ╠═══════▶ ReviewingLoaded ╠═════════▶ PersistingLoaded ╠══════════▶ Signing ╠════════▶ ReviewSignedPending ╠═══════▶ ReviewingSigned ║ +/// ╚═══════════╝ ╚══════════════════════════╝ ╚═══════════════════╝ ╚════════════════════╝ ╚══╤═════════▲═══╝ ╚═══════════════════════╝ ╚════════╤═════════╝ +/// retry │ │ complete retry │ +/// ┌──▼─────────┴────┐ stop_review ┌──────────▼──────────┐ +/// │ CleaningSigned <─────────────────────────────────────│ CleanSignedPending │ +/// └─────────────────┘ └─────────────────────┘ +/// ``` pub enum ZoneDataStorage { /// The loaded instance of the zone is being restored. RestoringLoaded(RestoringLoadedStorage), diff --git a/crates/zonedata/src/storage/transitions.rs b/crates/zonedata/src/storage/transitions.rs index 02831a430..92dcdd6be 100644 --- a/crates/zonedata/src/storage/transitions.rs +++ b/crates/zonedata/src/storage/transitions.rs @@ -138,8 +138,7 @@ impl RestoringSignedStorage { LoadedZoneReviewer, SignedZoneReviewer, ZoneViewer, - SignedZoneBuilder, - SigningStorage, + PassiveStorage, ) { assert!( Arc::ptr_eq(restorer.data(), &self.data), @@ -147,11 +146,11 @@ impl RestoringSignedStorage { ); restorer.clear(); - let Self { data, - curr_loaded_index, + curr_loaded_index: _, } = self; + let curr_loaded_index = false; let curr_signed_index = false; let loaded_reviewer = @@ -166,18 +165,14 @@ impl RestoringSignedStorage { ) }; let viewer = unsafe { ZoneViewer::new(data.clone(), curr_loaded_index, curr_signed_index) }; - let builder = unsafe { - SignedZoneBuilder::new(data.clone(), curr_loaded_index, !curr_signed_index, None) - }; - let storage = SigningStorage { + let storage = PassiveStorage { data, curr_loaded_index, curr_signed_index, - loaded_diff: None, }; - (loaded_reviewer, signed_reviewer, viewer, builder, storage) + (loaded_reviewer, signed_reviewer, viewer, storage) } } diff --git a/integration-tests/system-tests.yml b/integration-tests/system-tests.yml index 7e3fb8755..a2c43b201 100644 --- a/integration-tests/system-tests.yml +++ b/integration-tests/system-tests.yml @@ -301,3 +301,18 @@ jobs: accept-xfr-from: ${{ matrix.accept-xfr-from }} downstream-expects-tsig: ${{ matrix.downstream-expects-tsig }} downstream-uses-tsig: ${{ matrix.downstream-uses-tsig }} + + persist-zone: + name: Added zone should still exist after restart. + runs-on: ubuntu-latest + strategy: + matrix: + rust: [stable] + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/set-build-profile + with: + build-profile: ${{ inputs.build-profile }} + - uses: ./integration-tests/tests/persist-zone + with: + log-level: ${{ inputs.log-level }} diff --git a/integration-tests/tests/persist-zone/action.yml b/integration-tests/tests/persist-zone/action.yml new file mode 100644 index 000000000..36cc80614 --- /dev/null +++ b/integration-tests/tests/persist-zone/action.yml @@ -0,0 +1,687 @@ +# Making reusable composite actions documented at +# https://docs.github.com/en/actions/tutorials/create-actions/create-a-composite-action#creating-a-composite-action-within-the-same-repository +name: 'Added zone should still exist after restart.' +description: 'Added zone should still exist after restart.' +defaults: + # see: https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#defaultsrunshell + run: + shell: bash --noprofile --norc -eo pipefail -x {0} +inputs: + log-level: + description: The level of logging that Cascade should output. + required: false + default: debug + type: choice + options: + - error + - warning + - info + - debug + - trace +runs: + using: "composite" + steps: + - uses: ./.github/actions/prepare-systest-env + + - uses: ./.github/actions/setup-and-start-cascade + with: + log-level: ${{ inputs.log-level }} + + # ------------------------------------------------------------------------ + # 0. Add a zone that cannot be loaded but does not fail `zone add` + # immediately so that Cascade has the zone but no content for it, + # then stop Cascade and restart it, to verify that Cascade can handle + # restoration without any persisted state paths to restore from. + # ------------------------------------------------------------------------ + + - name: Add a non-existing upstream zone + run: | + cascade zone add --policy default --source 127.0.0.1:1055 i.dont.exist + + - name: Stop Cascade + run: | + pkill cascaded + + - name: Wait for Cascade to exit again + run: | + timeout=10 # seconds + start=$(date +%s) + until ! cascade health; do + if (($(date +%s) > (start + timeout))); then + cascade health + echo "::error:: timeout: health check did not indicate Cascade had stopped" + exit 1 + fi + sleep 1 + done + + - name: Restart Cascade + run: | + CASCADE_DIR=${{ github.workspace }}/cascade-dir + cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log" + + - name: Wait for Cascade to become healthy + run: | + timeout=10 # seconds + start=$(date +%s) + until cascade health; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: health check did not indicate Cascade had started" + exit 1 + fi + sleep 1 + done + + - name: Check that Cascade still knows the zone + run: | + timeout=10 # seconds + start=$(date +%s) + until cascade zone list | grep -q i.dont.exist; do + if (($(date +%s) > (start + timeout))); then + cascade zone list + echo "::error:: timeout: Cascade no longer knows the zone" + exit 1 + fi + sleep 1 + done + + - name: Delete the zone + run: | + cascade zone remove i.dont.exist + + # ------------------------------------------------------------------------ + # 1. Add an upstream NSD source zone and sign it. + # ------------------------------------------------------------------------ + + - name: Add a zone served by the NSD primary + run: | + cascade zone add --policy default --source 127.0.0.1:1055 example.test + + # By default Casade policy uses the 'date-counter' serial policy which + # makes it possible to predict the serial number it will generate. + - name: Predict the SOA SERIAL that Cascade will generate for the signed zone + id: expected-serial + run: | + EXPECTED_SERIAL=$(date +'%Y%m%d00') + echo "serial=${EXPECTED_SERIAL}" >> "$GITHUB_OUTPUT" + + - name: Wait for Cascade to serve the signed zone + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + timeout=10 # seconds + start=$(date +%s) + until dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR | grep -q "${EXPECTED_SERIAL}" ; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: signed zone did not become available for AXFR with expected SOA '${EXPECTED_SERIAL}'" + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR + exit 1 + fi + sleep 1 + done + + - name: Write the signed zone to a file so we can compare it later. + run: | + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR > example.test.signed.log + + # ------------------------------------------------------------------------ + # 2. Stop and restart Cascade so we can test that it: + # - Restores the signed zone correctly from persisted data on disk. + # - Does NOT re-sign the already signed zone, i.e. an AXFR of the + # published zone should be identical to the AXFR of the published + # zone that we saved to disk in the step above. + # ------------------------------------------------------------------------ + + - name: Stop Cascade again + run: | + pkill cascaded + + - name: Wait for Cascade to exit again + run: | + timeout=10 # seconds + start=$(date +%s) + until ! cascade health; do + if (($(date +%s) > (start + timeout))); then + cascade health + echo "::error:: timeout: health check did not indicate Cascade had stopped" + exit 1 + fi + sleep 1 + done + + # Cascade should reload the signed zone data from the data it persisted + # to disk. + - name: Start Cascade again + run: | + CASCADE_DIR=${{ github.workspace }}/cascade-dir + cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log" + + - name: Wait for Cascade to become healthy again + run: | + timeout=10 # seconds + start=$(date +%s) + until cascade health; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: health check did not indicate Cascade had started" + exit 1 + fi + sleep 1 + done + + - name: Check that Cascade still knows the zone + run: | + timeout=10 # seconds + start=$(date +%s) + until cascade zone list | grep -q example.test; do + if (($(date +%s) > (start + timeout))); then + cascade zone list + echo "::error:: timeout: Cascade no longer knows the zone" + exit 1 + fi + sleep 1 + done + + - name: Wait for Cascade to serve the signed zone + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + timeout=10 # seconds + start=$(date +%s) + + until dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR | grep -q "${EXPECTED_SERIAL}" ; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: signed zone did not become available for AXFR with expected SOA '${EXPECTED_SERIAL}'" + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR + exit 1 + fi + sleep 1 + done + + - name: Check that Cascade did NOT re-sign the zone + run: | + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR > example.test.signed2.log + diff -u example.test.signed.log example.test.signed2.log + + # ------------------------------------------------------------------------ + # 3. Delete all persisted zone data files to verify that Cascade can + # handle restoration with persisted state paths that refer to missing + # files. + # ------------------------------------------------------------------------ + + - name: Stop Cascade again + run: | + pkill cascaded + + - name: Wait for Cascade to exit again + run: | + timeout=10 # seconds + start=$(date +%s) + until ! cascade health; do + if (($(date +%s) > (start + timeout))); then + cascade health + echo "::error:: timeout: health check did not indicate Cascade had stopped" + exit 1 + fi + sleep 1 + done + + - name: Delete the persisted zone data that Cascade wrote to disk + run: | + CASCADE_DIR=${{ github.workspace }}/cascade-dir + rm "${CASCADE_DIR}"/zone-state/example.test.loaded.* + rm "${CASCADE_DIR}"/zone-state/example.test.signed.* + + - name: Start Cascade a 3rd time + run: | + CASCADE_DIR=${{ github.workspace }}/cascade-dir + cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log" + + - name: Wait for Cascade to become healthy a 3rd time + run: | + timeout=10 # seconds + start=$(date +%s) + until cascade health; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: health check did not indicate Cascade had started" + exit 1 + fi + sleep 1 + done + + - name: Check that Cascade still knows the zone + run: | + timeout=10 # seconds + start=$(date +%s) + until cascade zone list | grep -q example.test; do + if (($(date +%s) > (start + timeout))); then + cascade zone list + echo "::error:: timeout: Cascade no longer knows the zone" + exit 1 + fi + sleep 1 + done + + - name: Wait for Cascade to serve the re-signed zone with a newer serial number + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + timeout=10 # seconds + start=$(date +%s) + let EXPECTED_SERIAL=$(( EXPECTED_SERIAL + 1 )) + + until dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR | grep -q "${EXPECTED_SERIAL}" ; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: signed zone did not become available for AXFR with expected SOA '${EXPECTED_SERIAL}'" + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR + exit 1 + fi + sleep 1 + done + + - name: Save the signed zone for later comparison + run: | + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR > example.test.signed3.log + + - name: Check that Cascade DID resign the zone, not just bump the serial number + run: | + if diff -u example.test.signed2.log example.test.signed3.log > unexpected-diff.log; then + echo "::error:: Cascade should have re-signed the zone" + exit 1 + fi + + # ------------------------------------------------------------------------ + # 4. Update the zone at the source and tell NSD to reload it. + # This should cause Cascade to generate, persist and serve an IXFR + # diff for serial 01 -> 02. Note: it cannot serve the diff from + # 00 -> 01 because that diff was lost and cannot be recreated because + # a full zone signing needed to be done again and could not re-use the + # 00 serial number. + # ------------------------------------------------------------------------ + + - name: Edit the source zone and tell NSD to reload it. + run: | + ZONE_DIR=$(integration-tests/scripts/get-default-path.sh test-env:nameserver-base-dir) + ZONE_FILE="${ZONE_DIR}/nsd-primary/zones/example.test.primary-zone" + perl -pi -e 's/1 ; serial/12345 ; serial/' ${ZONE_FILE} + perl -pi -e 's/www A 169.254.1.1/www A 192.168.0.1/' ${ZONE_FILE} + ./integration-tests/scripts/manage-test-environment.sh control nsd-primary reload example.test + + - name: Wait for Cascade to load the changed zone + run: | + timeout=10 # seconds + start=$(date +%s) + + until dig +noall +answer @127.0.0.1 -p 4540 example.test SOA | grep -q 12345 ; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: signed zone did not become available for AXFR with expected SOA SERIAL 12345" + dig +noall +answer @127.0.0.1 -p 4540 example.test SOA + exit 1 + fi + sleep 1 + done + + - name: Wait for Cascade to serve the re-signed zone with a newer serial number + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + timeout=10 # seconds + start=$(date +%s) + let EXPECTED_SERIAL=$(( EXPECTED_SERIAL + 2 )) + + until dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR | grep -Fq "${EXPECTED_SERIAL}" ; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: signed zone did not become available for AXFR with expected SOA '${EXPECTED_SERIAL}'" + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR + exit 1 + fi + sleep 1 + done + + - name: Verify that the modified A record is served in the published zone + run: | + # Cascade can only answer SOA and XFR queries at the moment so we + # cannot query it for the A record we are interested in. + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR > example.test.final.log + grep -F www.example.test example.test.final.log | grep -Fq "192.168.0.1" + + - name: Dump IXFR ${{ steps.expected-serial.outputs.serial }} + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + let CLIENT_SERIAL=$(( EXPECTED_SERIAL + 0 )) + OUT_FILE="example.test.ixfr.from-${CLIENT_SERIAL}.log" + dig +noall +answer @127.0.0.1 -p 4542 -t ixfr=${CLIENT_SERIAL} example.test > "${OUT_FILE}" + + + # ------------------------------------------------------------------------ + # 5. Update the zone at the source and tell NSD to reload it again. + # This should create a second IXFR diff. + # ------------------------------------------------------------------------ + + - name: Edit the source zone and tell NSD to reload it. + run: | + ZONE_DIR=$(integration-tests/scripts/get-default-path.sh test-env:nameserver-base-dir) + ZONE_FILE="${ZONE_DIR}/nsd-primary/zones/example.test.primary-zone" + perl -pi -e 's/12345 ; serial/45678 ; serial/' ${ZONE_FILE} + perl -pi -e 's/www A 192.168.0.1/www A 192.168.0.2/' ${ZONE_FILE} + ./integration-tests/scripts/manage-test-environment.sh control nsd-primary reload example.test + + - name: Wait for Cascade to load the changed zone + run: | + timeout=10 # seconds + start=$(date +%s) + + until dig +noall +answer @127.0.0.1 -p 4540 example.test SOA | grep -q 45678 ; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: signed zone did not become available for AXFR with expected SOA SERIAL 12345" + dig +noall +answer @127.0.0.1 -p 4540 example.test SOA + exit 1 + fi + sleep 1 + done + + - name: Wait for Cascade to serve the re-signed zone with a newer serial number + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + timeout=10 # seconds + start=$(date +%s) + let EXPECTED_SERIAL=$(( EXPECTED_SERIAL + 3 )) + + until dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR | grep -Fq "${EXPECTED_SERIAL}" ; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: signed zone did not become available for AXFR with expected SOA '${EXPECTED_SERIAL}'" + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR + exit 1 + fi + sleep 1 + done + + - name: Verify that the modified A record is served in the published zone + run: | + # Cascade can only answer SOA and XFR queries at the moment so we + # cannot query it for the A record we are interested in. + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR > example.test.final.log + grep -F www.example.test example.test.final.log | grep -Fq "192.168.0.2" + + # Note: Cascade cannot serve the diff from 00 -> 01 because that diff was + # lost and cannot be recreated because a full zone signing needed to be + # done again. This creates only a snapshot of the zone, not a diff against + # a previous signed zone. + - name: "Verify that IXFR for 00 -> 01 is NOT available" + env: + CLIENT_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + # Attempting to get the diff from 00 -> 01 should return an AXFR + # (denoted by a single leading SOA and a single trailing SOA) + let EXPECTED_SERIAL=$(( CLIENT_SERIAL + 3 )) + OUT_FILE="example.test.ixfr.from-${CLIENT_SERIAL}.log" + dig +noall +answer @127.0.0.1 -p 4542 -t ixfr=${CLIENT_SERIAL} example.test > "${OUT_FILE}" + + NUM_SOA=$(grep -E 'example.test.\s+5\s+IN\s+SOA\s+ns1.example.test.\smail.example.test.\s'${EXPECTED_SERIAL}'\s60\s60\s3600\s5' "${OUT_FILE}" | wc -l) + if [[ ${NUM_SOA} -ne 2 ]]; then + echo "::error:: Expected 2 SOA RRs but got ${NUM_SOA} RRs." + exit 1 + fi + + - name: "Verify that IXFR for 01 -> 03 is available" + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + let CLIENT_SERIAL=$(( EXPECTED_SERIAL + 1 )) + OUT_FILE="example.test.ixfr.from-${CLIENT_SERIAL}.log" + EXPECTED_OUT_FILE="expected.out" + + let SER_1=$(( EXPECTED_SERIAL + 1)) + let SER_2=$(( EXPECTED_SERIAL + 2)) + let SER_3=$(( EXPECTED_SERIAL + 3)) + + cat >"${EXPECTED_OUT_FILE}" < "${OUT_FILE}" + sed -e 's/\s\+/ /g' "${OUT_FILE}" | grep -F '5 IN SOA' > diff.in + diff -u diff.in "${EXPECTED_OUT_FILE}" + + - name: "Verify that IXFR for 02 -> 03 is available" + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + let CLIENT_SERIAL=$(( EXPECTED_SERIAL + 2 )) + OUT_FILE="example.test.ixfr.from-${CLIENT_SERIAL}.log" + EXPECTED_OUT_FILE="expected.out" + + let SER_2=$(( EXPECTED_SERIAL + 2)) + let SER_3=$(( EXPECTED_SERIAL + 3)) + + cat >"${EXPECTED_OUT_FILE}" < "${OUT_FILE}" + sed -e 's/\s\+/ /g' "${OUT_FILE}" | grep -F '5 IN SOA' > diff.in + diff -u diff.in "${EXPECTED_OUT_FILE}" + + - name: Write the signed zone to a file so we can compare it later. + run: | + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR > example.test.signed4.log + + # ------------------------------------------------------------------------ + # 6. Stop and restart Cascade again so we can test that it still serves + # the same IXFRs as before. + # ------------------------------------------------------------------------ + + - name: Stop Cascade again + run: | + pkill cascaded + + - name: Wait for Cascade to exit again + run: | + timeout=10 # seconds + start=$(date +%s) + until ! cascade health; do + if (($(date +%s) > (start + timeout))); then + cascade health + echo "::error:: timeout: health check did not indicate Cascade had stopped" + exit 1 + fi + sleep 1 + done + + - name: Start Cascade a 4th time + run: | + CASCADE_DIR=${{ github.workspace }}/cascade-dir + cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log" + + - name: Wait for Cascade to become healthy a 4th time + run: | + timeout=10 # seconds + start=$(date +%s) + until cascade health; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: health check did not indicate Cascade had started" + exit 1 + fi + sleep 1 + done + + - name: Check that Cascade still knows the zone + run: | + timeout=10 # seconds + start=$(date +%s) + until cascade zone list | grep -q example.test; do + if (($(date +%s) > (start + timeout))); then + cascade zone list + echo "::error:: timeout: Cascade no longer knows the zone" + exit 1 + fi + sleep 1 + done + + - name: Wait for Cascade to serve the re-signed zone with a newer serial number + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + timeout=10 # seconds + start=$(date +%s) + let EXPECTED_SERIAL=$(( EXPECTED_SERIAL + 3 )) + + until dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR | grep -q "${EXPECTED_SERIAL}" ; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: signed zone did not become available for AXFR with expected SOA '${EXPECTED_SERIAL}'" + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR + exit 1 + fi + sleep 1 + done + + - name: Check that Cascade did NOT re-sign the zone + run: | + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR > example.test.signed5.log + diff -u example.test.signed4.log example.test.signed5.log + + - name: "Verify again that IXFR for 01 -> 03 is available" + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + let CLIENT_SERIAL=$(( EXPECTED_SERIAL + 1 )) + OUT_FILE="example.test.ixfr.from-${CLIENT_SERIAL}.log" + EXPECTED_OUT_FILE="expected.out" + + let SER_1=$(( EXPECTED_SERIAL + 1)) + let SER_2=$(( EXPECTED_SERIAL + 2)) + let SER_3=$(( EXPECTED_SERIAL + 3)) + + cat >"${EXPECTED_OUT_FILE}" < "${OUT_FILE}" + sed -e 's/\s\+/ /g' "${OUT_FILE}" | grep -F '5 IN SOA' > diff.in + diff -u diff.in "${EXPECTED_OUT_FILE}" + + - name: "Verify again that IXFR for 02 -> 03 is available" + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + let CLIENT_SERIAL=$(( EXPECTED_SERIAL + 2 )) + OUT_FILE="example.test.ixfr.from-${CLIENT_SERIAL}.log" + EXPECTED_OUT_FILE="expected.out" + + let SER_2=$(( EXPECTED_SERIAL + 2)) + let SER_3=$(( EXPECTED_SERIAL + 3)) + + cat >"${EXPECTED_OUT_FILE}" < "${OUT_FILE}" + sed -e 's/\s\+/ /g' "${OUT_FILE}" | grep -F '5 IN SOA' > diff.in + diff -u diff.in "${EXPECTED_OUT_FILE}" + + # ------------------------------------------------------------------------ + # 7. Delete all persisted signed zone data files to verify that Cascade + # can handle partial restoration for a zone that was previously signed + # at least once. + # ------------------------------------------------------------------------ + + - name: Stop Cascade again + run: | + pkill cascaded + + - name: Wait for Cascade to exit again + run: | + timeout=10 # seconds + start=$(date +%s) + until ! cascade health; do + if (($(date +%s) > (start + timeout))); then + cascade health + echo "::error:: timeout: health check did not indicate Cascade had stopped" + exit 1 + fi + sleep 1 + done + + - name: Delete the persisted signed zone data that Cascade wrote to disk + run: | + CASCADE_DIR=${{ github.workspace }}/cascade-dir + rm "${CASCADE_DIR}"/zone-state/example.test.signed.* + + - name: Start Cascade again + run: | + CASCADE_DIR=${{ github.workspace }}/cascade-dir + cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log" + + - name: Wait for Cascade to become healthy again + run: | + timeout=10 # seconds + start=$(date +%s) + until cascade health; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: health check did not indicate Cascade had started" + exit 1 + fi + sleep 1 + done + + - name: Check that Cascade still knows the zone + run: | + timeout=10 # seconds + start=$(date +%s) + until cascade zone list | grep -q example.test; do + if (($(date +%s) > (start + timeout))); then + cascade zone list + echo "::error:: timeout: Cascade no longer knows the zone" + exit 1 + fi + sleep 1 + done + + - name: Wait for Cascade to serve the re-signed zone with a newer serial number + env: + EXPECTED_SERIAL: ${{ steps.expected-serial.outputs.serial }} + run: | + timeout=10 # seconds + start=$(date +%s) + let EXPECTED_SERIAL=$(( EXPECTED_SERIAL + 4 )) + + until dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR | grep -q "${EXPECTED_SERIAL}" ; do + if (($(date +%s) > (start + timeout))); then + echo "::error:: timeout: signed zone did not become available for AXFR with expected SOA '${EXPECTED_SERIAL}'" + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR + exit 1 + fi + sleep 1 + done + + - name: Save the signed zone for later comparison + run: | + dig +noall +answer @127.0.0.1 -p 4542 example.test AXFR > example.test.signed7.log + + - name: Check that Cascade DID resign the zone, not just bump the serial number + run: | + if diff -u example.test.signed2.log example.test.signed7.log > unexpected-diff.log; then + echo "::error:: Cascade should have re-signed the zone" + exit 1 + fi + + - name: Print log files on any failure in this job + uses: ./.github/actions/print-logfiles + if: failure() diff --git a/src/loader/mod.rs b/src/loader/mod.rs index 39c8c7e44..3b7416b1e 100644 --- a/src/loader/mod.rs +++ b/src/loader/mod.rs @@ -51,16 +51,27 @@ impl Loader { /// Initialize the loader, synchronously. pub fn init(center: &Arc
, state: &mut State) { - // Enqueue refreshes for all known zones. + // Enqueue refreshes for all known upstream zones. for zone in &state.zones { let mut state = zone.0.state.lock().unwrap(); - ZoneHandle { - zone: &zone.0, - state: &mut state, - center, + match state.loader.source { + Source::None => { /* Nothing to do */ } + Source::Zonefile { .. } => { + // Don't enqueue a refresh for zones sourced from disk + // as the operator may be in the middle of editing the + // zonefile and thus we require zonefiles to be reloaded + // explicitly via `cascade zone reload`. + } + Source::Server { .. } => { + ZoneHandle { + zone: &zone.0, + state: &mut state, + center, + } + .loader() + .enqueue_refresh(false); + } } - .loader() - .enqueue_refresh(false); } } diff --git a/src/persistence/mod.rs b/src/persistence/mod.rs index 85311f245..62fd0807f 100644 --- a/src/persistence/mod.rs +++ b/src/persistence/mod.rs @@ -2,7 +2,95 @@ //! //! The zone persister saves the data for loaded and signed zones to disk, so //! that Cascade can seamlessly resume operation after a crash / restart. At -//! startup, it tries to restore data for all known zones. +//! startup it tries to restore data for all known zones. +//! +//! When re-starting Cascade in-memory zone and IXFR diff data will be lost +//! unless persisted and restored. This module implements persistence +//! and restoration using files on disk stored in the zone-state directory +//! alongside the JSON '.db' zone state files. +//! +//! # Data format +//! +//! Data is persisted as AXFR and IXFR message ANSWER sections in wire format. +//! +//! # Persistence +//! +//! Persistence is invoked immediately after zone approval, either because of +//! a successful review hook, or no review hook at all, or because the +//! operator overrode a failed review hook. +//! +//! Persisted data is stored separately for records received while loading the +//! zone vs changes that occur to the zone as a result of (re)signing it. +//! +//! For both loaded and signed changes, persistence stores an initial snapshot +//! and a sequence of zero or more diffs: +//! - A loaded snapshot file is written immediately after approval of the +//! initial version of a zone is received, whether from disk or via +//! XFR-in. This file has the name `.loaded.0`. +//! - A loaded diff file is written each time the input zone is reloaded, +//! whether due to XFR-in or due to reloading of the input file from disk. +//! Loaded diff files are named `.loaded.N` where N > 0 and +//! increases by one each time a new diff is persisted. +//! - A signed snapshot file is written immediately after approval of the +//! first signed version of the zone resulting from full zone signing. +//! This file has the name `.signed.0`. +//! - A signed diff file is written each time the zone is re-signed, whether +//! due to changes in the input zone or changes in the signing keys or the +//! replacing of signatures for records whose signatures need refreshing. +//! Signed diff files are named `.signed.N` where N > 0 and +//! increases by one each time a new diff is persisted. +//! +//! After a diff is persisted successfully: +//! - The diff is stored in memory alongside the zone in +//! `StorageState::diffs` so that it can be served in response to an IXFR +//! request from a downstream nameserver. +//! - The path that the diff file was written to is appended to +//! `ZoneState::persisted_loaded_diff_paths` or +//! `ZoneState::persisted_signed_diff_paths` and the zone state is +//! immediately saved to disk. +//! +//! # Panics +//! +//! If persistence fails due to an I/O error this will cause Cascade to panic. +//! If the underlying storage that Cascade depends on is not reliable we have +//! no way of knowing what else may be failing and abort as it is not safe to +//! continue under such circumstances. +//! +//! # Restoration +//! +//! Zones are created in memory at Cascade startup in storage state +//! `RestoringLoaded`. If the zone loader attempts to start loading the zone +//! the load will fail because the zone storage is not yet in the `Passive` +//! state. Instead a refresh will be enqueud and acted upon once the zone +//! storage enters the `Passive` state. +//! +//! On startup Cascade starts a zone restorer which will attempt to restore +//! all known zones. If restoration fails the zone will move to the `Passive` +//! state and any subsequent load of data will be handled as usual. As long +//! as the last used serial number was successfully persisted to state and +//! restored from state the newly signed zone will receive a higher serial +//! number than the last published zone that we failed to restore. Failure +//! to restore also results in deletion of all persisted data for the zone +//! and updating of the state to clear the paths to the no longer existing +//! persisted data files. Failure to remove a persisted data file will be +//! logged as a WARNing and Cascade will continue. +//! +//! Restoration of a zone is achieved by replacing the current (empty) zone +//! content with the loaded snapshot, then applying each loaded diff file +//! to the snapshot one at a time. The diffs are also kept in-memory for +//! responding to IXFR requests from downstream nameservers. The signed +//! snapshot and diffs are also restored like this. +//! +//! Any diff that was available at a review server will have been lost. +//! However as only approved data gets persisted, there should be no need +//! to still be able to query the review server for an IXFR diff after +//! Cascsade restarts. +//! +//! TODO: What happens if loaded data is approved and persisted, but +//! Cascade is terminated before signing occurs. In such a case if restore +//! is done as described above signing can occur as usual, but will a +//! signed review hook be able to query the loaded review server for the +//! loaded diff? use std::sync::Arc; diff --git a/src/persistence/persist.rs b/src/persistence/persist.rs index 983ae2c2b..7e3d84d69 100644 --- a/src/persistence/persist.rs +++ b/src/persistence/persist.rs @@ -1,12 +1,23 @@ //! Persisting zone data. -use std::sync::Arc; +use std::{ + fs::File, + io::{BufWriter, ErrorKind, Write}, + path::Path, + sync::Arc, +}; use cascade_zonedata::{ DiffData, LoadedZonePersisted, LoadedZonePersister, SignedZonePersisted, SignedZonePersister, }; -use crate::{center::Center, zone::Zone}; +use domain::new::base::wire::{BuildBytes, TruncationError}; +use tracing::{trace, warn}; + +use crate::{ + center::Center, + zone::{Zone, ZoneHandle, save_state_now}, +}; /// Persist the data for a loaded instance of a zone. #[tracing::instrument( @@ -19,7 +30,62 @@ pub fn persist_loaded( center: &Arc
, persister: LoadedZonePersister, ) -> LoadedZonePersisted { - let _ = center; + if !persister.loaded_diff().is_empty() { + // Determine the path to write to and update the record of written + // paths here as we don't want to give responsibility for working with + // ZoneState to the persistence crate. Accumulate a set of diffs per + // unsigned and signed zone, each stored at a path one suffixed by an + // index which rises by one when persisted. + // TODO: Don't keep an unlimited number of diffs. + // TODO: Compact diffs when idle? + let destination = { + let mut state = zone.state.lock().unwrap(); + let handle = ZoneHandle { + zone, + state: &mut state, + center, + }; + let next_idx = handle.state.persisted_loaded_diff_paths.len(); + let destination = center + .config + .zone_state_dir + .join(format!("{}.loaded.{next_idx}", zone.name)); + + handle + .state + .persisted_loaded_diff_paths + .push(destination.clone().into()); + + destination + }; + + // Update the set of persisted zone data file paths BEFORE writing + // the new file because if we crash/lose power between writing the new + // file and storing the updated set of paths: + // - We would not know when restoring that a diff is missing. + // This isn't ultimately a problem as we also would not ever + // have served the updated zone from the publication server as + // persistence has to complete before publication occurs. + // - The new file would be unused but left behind on disk. It may + // later be overwritten by a new diff but until then it would + // be unused and also not removed by cleaning of diff files that + // occurs if restoration fails, as the path would not be known + // to us. + + // TODO: Saving state first then writing the file could lead to a + // situation where a zone that was published consisting of a snapshot + // and N diffs would be discarded if the path to the N+1th diff was + // recorded in state and persisted but actually writing the N+1th + // diff file failed. On restore the file would be looked for and not + // found and all loaded and signed content including diffs would be + // discarded at that point. One way around this could be to track + // (e.g. using an Option field) that this path is in the process of + // being written but has not yet finished yet, and then on restore if + // the Option is Some the referred to path can just be deleted. + save_state_now(center, zone); + + persist_to_file(destination.as_std_path(), persister.loaded_diff().clone()); + } // Store the loaded diff in-memory for serving IXFR out. @@ -28,10 +94,10 @@ pub fn persist_loaded( // Only store a diff if something has changed compared to the previous // version of the loaded zone, otherwise this is not a diff to a previous // version of the zone but actually a snapshot of the zone after having - // been loaded for the first time. If the SOA serial didn't change (which - // is legal for a loaded zone) don't store a diff because the IXFR protocol - // requires a SOA serial number change so we won't be able to serve the diff - // anyway. + // been loaded for the first time. If the SOA serial didn't change + // (which is legal for a loaded zone) don't store a diff because the IXFR + // protocol requires a SOA serial number change so we won't be able to + // serve the diff anyway. if !loaded_diff.is_empty() && loaded_diff.removed_soa.is_some() { // Store anything that changed when the zone was re-loaded, i.e. // unsigned zone content changes. Note that the SOA SERIAL is not @@ -40,6 +106,19 @@ pub fn persist_loaded( let mut state = zone.state.lock().unwrap(); let loaded_only_diff = (persister.loaded_diff().clone(), DiffData::new().into()); + trace!( + "Storing IXFR in-memory diff for SOA loaded serial -{:?}:+{:?}", + loaded_only_diff + .0 + .removed_soa + .as_ref() + .map(|soa_rr| soa_rr.rdata.serial), + loaded_only_diff + .0 + .added_soa + .as_ref() + .map(|soa_rr| soa_rr.rdata.serial), + ); state.storage.diffs.push(loaded_only_diff); } @@ -57,36 +136,267 @@ pub fn persist_signed( center: &Arc
, persister: SignedZonePersister, ) -> SignedZonePersisted { - let _ = center; + if !persister.signed_diff().is_empty() { + let loaded_diff = persister.loaded_diff(); + let signed_diff = persister.signed_diff(); - // Store the signed diff in-memory for serving IXFR out. - // - // Only store a diff if the SOA from the previous version of the signed - // zone was removed and a new one added, otherwise this is not a diff to a - // previous version of the zone but actually a snapshot of the zone after - // having been signed for the first time. - let loaded_diff = persister.loaded_diff(); - let signed_diff = persister.signed_diff(); + // Determine the path to write to and update the record of written + // paths here as we don't want to give responsibility for working with + // ZoneState to the persistence crate. Accumulate a set of diffs per + // unsigned and signed zone, each stored at a path one suffixed by an + // index which rises by one when persisted. + // TODO: Don't keep an unlimited number of diffs. + // TODO: Compact diffs when idle? + let destination = { + let mut state = zone.state.lock().unwrap(); + let handle = ZoneHandle { + zone, + state: &mut state, + center, + }; + let next_idx = handle.state.persisted_signed_diff_paths.len(); + let destination = center + .config + .zone_state_dir + .join(format!("{}.signed.{next_idx}", zone.name)); - if signed_diff.removed_soa.is_some() && signed_diff.removed_soa != signed_diff.added_soa { - let mut state = zone.state.lock().unwrap(); + handle + .state + .persisted_signed_diff_paths + .push(destination.clone().into()); - // Store anything that changed when the zone was re-loaded, i.e. - // unsigned zone content changes. Note that the SOA SERIAL is not - // required to change unless using 'keep' policy and so we should not - // require the SOA to have been removed and a new one added. + destination + }; + + // Update the set of persisted zone data file paths BEFORE writing + // the new file because if we crash/lose power between writing the new + // file and storing the updated set of paths: + // - We would not know when restoring that a diff is missing. + // This isn't ultimately a problem as we also would not ever + // have served the updated zone from the publication server as + // persistence has to complete before publication occurs. + // - The new file would be unused but left behind on disk. It may + // later be overwritten by a new diff but until then it would + // be unused and also not removed by cleaning of diff files that + // occurs if restoration fails, as the path would not be known + // to us. + + // TODO: Saving state first then writing the file could lead to a + // situation where a zone that was published consisting of a snapshot + // and N diffs would be discarded if the path to the N+1th diff was + // recorded in state and persisted but actually writing the N+1th + // diff file failed. On restore the file would be looked for and not + // found and all loaded and signed content including diffs would be + // discarded at that point. One way around this could be to track + // (e.g. using an Option field) that this path is in the process of + // being written but has not yet finished yet, and then on restore if + // the Option is Some the referred to path can just be deleted. + save_state_now(center, zone); + + // Write the diff to disk as a binary AXFR snapshot or binary IXFR + // diff. + persist_to_file(destination.as_std_path(), signed_diff.clone()); + + // Store the diffs in-memory for serving IXFR out. + // + // Only store a diff if the SOA from the previous version of the + // signed zone was removed and a new one added, otherwise this is not + // a diff to a previous version of the zone but actually a snapshot of + // the zone after having been signed for the first time. + if signed_diff.removed_soa.is_some() && signed_diff.removed_soa != signed_diff.added_soa { + // Store anything that changed when the zone was re-loaded, i.e. + // unsigned zone content changes. Note that the SOA SERIAL is not + // required to change unless using 'keep' policy and so we should + // not require the SOA to have been removed and a new one added. + + // Store anything that changed when the zone was re-signed, i.e. + // changes DNSSEC RRs that can be caused by unsigned content + // changes or changing from NSEC <-> NSEC3 or using a new key + // to sign with or just regenerating signatures to avoid them + // expiring. Signed zones MUST always have a new SOA SERIAL + // compared to the previous version of the signed zone. + + let mut state = zone.state.lock().unwrap(); - // Store anything that changed when the zone was re-signed, i.e. - // changes DNSSEC RRs that can be caused by unsigned content changes - // or changing from NSEC <-> NSEC3 or using a new key to sign with or - // just regenerating signatures to avoid them expiring. Signed zones - // MUST always have a new SOA SERIAL compared to the previous version - // of the signed zone. + // If we have a new signed diff to store because records in the + // loaded part of the zone changed, e.g. due to changes in the + // zone content or receipt of a changed DNSKEY set from the key + // manager, then the loaded diff will have been stored in-memory + // by loaded zone persistence, but the corresponding signed diff + // will not yet have been stored in-memory, we have to do that + // now. In this case we have to update the last stored in-memory + // diff. We drop the partial diff and push a replacement full + // diff instead. + // + // Alternatively if we have a new signed diff to store because + // records in the signed part of the zone changed, e.g. due to + // signature re-generation to ensure that existing signatures + // don't expire, then there will be no corresponding loaded diff + // yet in-memory. In this case we have to push an entirely new + // diff to the in-memory collection without dropping an existing + // diff first. - let loaded_diff = loaded_diff.cloned().unwrap_or(DiffData::new().into()); - let complete_diff = (loaded_diff, signed_diff.clone()); - state.storage.diffs.push(complete_diff); + let mut action = "Storing new"; + if let Some(potentially_partial_diff) = state.storage.diffs.last() + && potentially_partial_diff.1.is_empty() + { + // Remove the partial diff, it will be replaced by a + // complete diff. + let _partial_diff = state.storage.diffs.pop(); + action = "Updating existing"; + } + + let loaded_diff = loaded_diff.cloned().unwrap_or(DiffData::new().into()); + trace!( + "{action} IXFR in-memory diff for SOA loaded serial -{:?}:+{:?} -> signed serial -{:?}:+{:?}", + loaded_diff + .removed_soa + .as_ref() + .map(|soa_rr| soa_rr.rdata.serial), + loaded_diff + .added_soa + .as_ref() + .map(|soa_rr| soa_rr.rdata.serial), + signed_diff + .removed_soa + .as_ref() + .map(|soa_rr| soa_rr.rdata.serial), + signed_diff + .added_soa + .as_ref() + .map(|soa_rr| soa_rr.rdata.serial), + ); + state.storage.diffs.push((loaded_diff, signed_diff.clone())); + } } persister.mark_complete() } + +//------------ persist_to_file() ---------------------------------------------- + +fn persist_to_file(destination: &Path, diff: Arc) { + // Write the diff in AXFR / IXFR wire format to disk. + let f = match File::create_new(destination) { + Ok(f) => f, + Err(err) if err.kind() == ErrorKind::AlreadyExists => { + // This is not expected. When persisting the zone data to a file + // we save Cascade zone state "now" so that the persisted paths in + // use are known on next restart, so we should know this path was + // in use and be attempting to write to a different non-existing + // path. If for some reason zone state was not persisted after the + // persisted zone data file was created, e.g. a power outage in + // combination with a change to persistence logic compared to how + // it is at the time of writing so that zone state was not ensured + // to be persisted before proceeding, that could cause this. + warn!( + "Overwriting existing persisted zone data file at '{}'.", + destination.display() + ); + File::create(destination).unwrap_or_else(|err| { + panic!( + "Failed to persist zone data to '{}': {err}", + destination.display() + ); + }) + } + Err(err) => { + panic!( + "Failed to persist zone data to '{}': {err}", + destination.display() + ); + } + }; + let mut f = BufWriter::new(f); + + let mut buf = vec![0u8; 1024]; + + fn write_rr(buf: &mut Vec, rr: &RR, mut writer: W) + where + RR: std::ops::Deref, + W: Write, + { + let buf_len = buf.len(); + + let num_bytes_to_write = match rr.build_bytes(buf) { + Ok(unused_buf_part) => { + // Build succeeded, now determine how many bytes were built. + buf_len - unused_buf_part.len() + } + Err(TruncationError) => { + // Build failed due to insufficient buffer space, resize the + // buffer to be large enough then redo the build which should + // not fail this time. + let num_required_bytes = rr.built_bytes_size(); + buf.resize(num_required_bytes, 0u8); + rr.build_bytes(buf).unwrap(); + num_required_bytes + } + }; + + // Failure to write would be imply there is a serious problem with + // the environment in which Cascade is running, and being unable to + // persist a zone means we can't allow downstreams to consume it as if + // we get terminated we will not be able to serve the same zone again + // (which would only actually be a real problem if we are unable to + // bump the serial). So, treat a write failure as fatal. + writer.write_all(&buf[0..num_bytes_to_write]).unwrap(); + } + + let added_soa = diff.added_soa.clone().unwrap(); + + // IXFR format has the form: + // - New SOA + // - Old SOA + // - Deleted records + // - New SOA + // - Added records + // - New SOA + // + // AXFR format has the form: + // - New SOA + // - Records + // - New SOA + // + // Write AXFR if no records were deleted by the diff, else write IXFR. + + write_rr(&mut buf, &added_soa, &mut f); + + // Start deleted records block by writing the old SOA, if any. + if let Some(removed_soa) = &diff.removed_soa { + write_rr(&mut buf, removed_soa, &mut f); + + // Write the deleted records. + for r in &diff.removed_records { + write_rr(&mut buf, r, &mut f); + } + + // Start added records block by writing the new SOA + write_rr(&mut buf, &added_soa, &mut f); + } + + // Write the added records. + for r in &diff.added_records { + write_rr(&mut buf, r, &mut f); + } + + // Finish the AXFR/IXFR by writing the new SOA again + write_rr(&mut buf, &added_soa, &mut f); + + trace!( + "Persisted zone to file '{}': SOA {:?} -> {:?}: {} records removed, {} records added", + destination.display(), + diff.removed_soa.as_ref().map(|v| v.rdata.serial), + diff.added_soa.as_ref().map(|v| v.rdata.serial), + if !diff.removed_records.is_empty() { + diff.removed_records.len() + 1 + } else { + 0 + }, + if !diff.added_records.is_empty() { + diff.added_records.len() + 1 + } else { + 0 + }, + ); +} diff --git a/src/persistence/restore.rs b/src/persistence/restore.rs index bbfbb709d..690bc5fcd 100644 --- a/src/persistence/restore.rs +++ b/src/persistence/restore.rs @@ -1,12 +1,35 @@ //! Restoring persisted zone data. -use std::{io, sync::Arc}; +use std::{ + fs::File, + io::{self, BufReader, Read}, + path::Path, + sync::Arc, +}; -use cascade_zonedata::{LoadedZoneRestorer, SignedZoneRestorer}; +use cascade_zonedata::{ + DiffData, LoadedZonePatcher, LoadedZoneRestorer, RegularRecord, SignedZonePatcher, + SignedZoneRestorer, SoaRecord, +}; +use domain::{ + new::{ + base::{ + RType, Record, Serial, + name::{NameBuf, RevNameBuf}, + parse::{ParseMessageBytes, SplitMessageBytes}, + }, + rdata::{BoxedRecordData, Soa}, + }, + utils::dst::UnsizedCopy, +}; +use tracing::{info, trace}; use crate::{center::Center, zone::Zone}; /// Restore the loaded instance data of a zone. +/// +/// Returns Ok(true) if data was stored, Ok(false) if there was nothing to +/// restore, or Err(..) on error. #[tracing::instrument( level = "trace", skip_all, @@ -16,13 +39,102 @@ pub fn restore_loaded( zone: &Arc, center: &Arc
, restorer: &mut LoadedZoneRestorer, -) -> io::Result<()> { - // TODO - let _ = (zone, center, restorer); - Err(io::Error::other("not yet implemented")) +) -> io::Result { + let mut state = zone.state.lock().unwrap(); + if state.persisted_loaded_diff_paths.is_empty() { + return io::Result::Ok(false); + } + + info!("Restoring loaded zone from persisted data"); + state.storage.diffs.clear(); + + // Determine the paths to read from. Each zone is persisted as an AXFR + // plus zero or more IXFRs. The restorer takes a base path ending in an + // unsigned integer number and loads that file plus N more, where the + // final number in the path is replaced by the previous number plus one + // each time. + let loaded_source = center + .config + .zone_state_dir + .join(format!("{}.loaded.0", zone.name)); + let count = state.persisted_loaded_diff_paths.len(); + let mut buf = Vec::::new(); + drop(state); + + // Process the initial "loaded" AXFR wire format dump. + let (soa, records) = + load_axfr_wire_dump(loaded_source.as_std_path(), &mut buf).map_err(|err| { + io::Error::other(format!("Loading snapshot '{loaded_source}' failed: {err}")) + })?; + let mut loaded_replacer = restorer.fill().ok_or(io::Error::other( + "Internal error: Could not acquire replacer".to_string(), + ))?; + loaded_replacer.set_soa(soa.clone()).unwrap(); + loaded_replacer.set_records(records).unwrap(); + loaded_replacer.apply().unwrap(); + trace!( + "Restored loaded snapshot for SOA serial {} for zone '{}' from file '{loaded_source}'", + soa.rdata.serial, zone.name + ); + + if count == 1 { + return io::Result::Ok(true); + } + + let mut source = loaded_source.to_path_buf(); + let mut all_serials = vec![]; + + for i in 1..count { + let mut loaded_patcher = restorer + .patch() + .ok_or(io::Error::other("Internal error: Patch failed".to_string()))?; + source.set_extension(i.to_string()); + + let (start_serial, end_serial) = + load_ixfr_wire_dump(source.as_std_path(), &mut buf, |event| { + apply_ixfr_event_to_loaded_data(&mut loaded_patcher, event); + }) + .map_err(|err| io::Error::other(format!("Loading diff '{source}' failed: {err}",)))?; + + loaded_patcher.next_patchset().map_err(|err| { + io::Error::other(format!("Internal error: Next patchset failed: {err}")) + })?; + + loaded_patcher + .apply() + .map_err(|err| io::Error::other(format!("Internal error: Apply failed: {err}")))?; + + if let Some(diff) = restorer.take_diff() { + // Store the loaded diff to be used as part of serving an IXFR. + let mut state = zone.state.lock().unwrap(); + state + .storage + .diffs + .push((diff.into(), DiffData::new().into())); + + trace!( + "Stored IXFR loaded diff for SOA serial {} from file '{loaded_source}': serial {start_serial} -> {end_serial}", + soa.rdata.serial, + ); + } + + let start_serial: u32 = start_serial.into(); + let end_serial: u32 = end_serial.into(); + all_serials.push((start_serial, end_serial)); + } + trace!( + "Restored loaded diff for SOA serial {} for zone '{}' from file '{loaded_source}' with diff serials: {all_serials:?}", + soa.rdata.serial, zone.name + ); + + info!("Restored loaded zone snapshot and {} diffs", count - 1); + io::Result::Ok(true) } /// Restore the loaded instance data of a zone. +/// +/// Returns Ok(true) if data was stored, Ok(false) if there was nothing to +/// restore, or Err(..) on error. #[tracing::instrument( level = "trace", skip_all, @@ -32,8 +144,374 @@ pub fn restore_signed( zone: &Arc, center: &Arc
, restorer: &mut SignedZoneRestorer, -) -> io::Result<()> { - // TODO - let _ = (zone, center, restorer); - Err(io::Error::other("not yet implemented")) +) -> io::Result { + let state = zone.state.lock().unwrap(); + if state.persisted_signed_diff_paths.is_empty() { + return io::Result::Ok(false); + } + + // Determine the paths to read from. Each zone is persisted as an AXFR + // plus zero or more IXFRs. The restorer takes a base path ending in an + // unsigned integer number and loads that file plus N more, where the + // final number in the path is replaced by the previous number plus one + // each time. + let signed_source = center + .config + .zone_state_dir + .join(format!("{}.signed.0", zone.name)); + let count = state.persisted_signed_diff_paths.len(); + let mut buf = Vec::::new(); + drop(state); + + // Process the initial "signed" AXFR wire format dump. + let (soa, records) = + load_axfr_wire_dump(signed_source.as_std_path(), &mut buf).map_err(|err| { + io::Error::other(format!( + "Loading snapshot from '{signed_source}' failed: {err}" + )) + })?; + let mut signed_replacer = restorer.fill().ok_or(io::Error::other( + "Internal error: Could not acquire replacer".to_string(), + ))?; + signed_replacer.set_soa(soa.clone()).unwrap(); + signed_replacer.set_records(records).unwrap(); + signed_replacer.apply().unwrap(); + trace!( + "Restored signed snapshot for SOA serial {} for zone '{}' from file '{signed_source}'", + soa.rdata.serial, zone.name + ); + + if count == 1 { + return io::Result::Ok(true); + } + + // Process zero or more "signed" IXFR wire format dumps. + let mut source = signed_source.to_path_buf(); + let mut all_serials = vec![]; + + // Load each diff and apply it to the zone, retrieving a single DiffData + // per signed diff. Store each signed DiffData alongside the corresponding + // loaded DiffData that was restored earlier in restore_loaded(). These + // DiffData's will be used to respond to IXFR requests, while at the same + // time also building up the entire signed zone that should be served for + // AXFR requests. + for i in 1..count { + let mut signed_patcher = restorer + .patch() + .ok_or(io::Error::other("Internal error: Patch failed".to_string()))?; + source.set_extension(i.to_string()); + + let (start_serial, end_serial) = + load_ixfr_wire_dump(source.as_std_path(), &mut buf, |event| { + apply_ixfr_event_to_signed_data(&mut signed_patcher, event); + }) + .map_err(|err| io::Error::other(format!("Loading diff '{source}' failed: {err}")))?; + + signed_patcher.next_patchset().map_err(|err| { + io::Error::other(format!("Internal error: Next patchset failed: {err}")) + })?; + + signed_patcher + .apply() + .map_err(|err| io::Error::other(format!("Internal error: Apply failed: {err}")))?; + + if let Some(signed_diff) = restorer.take_diff() { + let mut state = zone.state.lock().unwrap(); + // Get the diff pair (loaded diff and missing signed diff) that + // this signed diff needs to be inserted into. If the signed diff + // was caused by incremental signing then a loaded diff won't have + // been available to restore, we need to use an empty loaded diff + // in that case. + if let Some(partial_diff) = state.storage.diffs.get_mut(i - 1) { + // Insert the signed diff alongside the loaded diff, unless + // the signed diff already unexpectedly exists. + assert!(partial_diff.1.is_empty()); + partial_diff.1 = signed_diff.into(); + trace!( + "Updated signed part of IXFR in-memory diff for SOA loaded serial -{:?}:+{:?} -> signed serial -{:?}:+{:?} from file '{signed_source}'", + partial_diff + .0 + .removed_soa + .as_ref() + .map(|soa_rr| soa_rr.rdata.serial), + partial_diff + .0 + .added_soa + .as_ref() + .map(|soa_rr| soa_rr.rdata.serial), + partial_diff + .1 + .removed_soa + .as_ref() + .map(|soa_rr| soa_rr.rdata.serial), + partial_diff + .1 + .added_soa + .as_ref() + .map(|soa_rr| soa_rr.rdata.serial), + ); + } else { + let loaded_diff = Arc::new(DiffData::new()); + trace!( + "Storing IXFR in-memory diff for SOA loaded serial -{:?}:+{:?} -> signed serial -{:?}:+{:?} from file '{signed_source}'", + loaded_diff + .removed_soa + .as_ref() + .map(|soa_rr| soa_rr.rdata.serial), + loaded_diff + .added_soa + .as_ref() + .map(|soa_rr| soa_rr.rdata.serial), + signed_diff + .removed_soa + .as_ref() + .map(|soa_rr| soa_rr.rdata.serial), + signed_diff + .added_soa + .as_ref() + .map(|soa_rr| soa_rr.rdata.serial), + ); + state.storage.diffs.push((loaded_diff, signed_diff.into())); + } + } + + let start_serial: u32 = start_serial.into(); + let end_serial: u32 = end_serial.into(); + all_serials.push((start_serial, end_serial)); + } + trace!( + "Restored signed diff for SOA serial {} for zone '{}' from file '{signed_source}' with diff serials: {all_serials:?}", + soa.rdata.serial, zone.name + ); + + info!("Restored signed zone snapshot and {} diffs", count - 1); + io::Result::Ok(true) +} + +fn parse_rr( + buf: &[u8], + pos: usize, +) -> Result<(Record, usize), String> { + Record::::split_message_bytes(buf, pos) + .map_err(|err| format!("Invalid wire format RR: {err}")) +} + +fn parse_soa(buf: &[u8], pos: usize) -> Result<(SoaRecord, Soa, usize), String> { + let (first_rr, rest) = Record::>::split_message_bytes(buf, pos) + .map_err(|err| { + format!("Failed to parse record of persisted XFR dump as a SOA record: {err}") + })?; + + if first_rr.rtype != RType::SOA { + return Err(format!( + "Persisted XFR dump record has RTYPE '{}' which is not a SOA RR.", + first_rr.rtype.code + )); + } + + // Save the SOA rdata for comparison later, before we convert it from + // using NameBuf typed fields to having Box typed fields (which + // is the format we store resource records in longer term in memory). + let soa_rdata = first_rr.rdata.clone(); + let soa_rr = SoaRecord(first_rr.transform_ref( + |name: &RevNameBuf| (*name).unsized_copy_into(), + |data: &Soa| data.map_names_by_ref(|name| (*name).unsized_copy_into()), + )); + + Ok((soa_rr, soa_rdata, rest)) +} + +fn load_file_into_memory(source: &Path, buf: &mut Vec) -> std::io::Result { + buf.clear(); + BufReader::new(File::open(source)?).read_to_end(buf) +} + +fn load_axfr_wire_dump( + source: &Path, + buf: &mut Vec, +) -> io::Result<(SoaRecord, Vec)> { + load_file_into_memory(source, buf)?; + + let (start_soa, start_soa_rdata, mut rest) = parse_soa(buf, 0).map_err(|err| { + io::Error::other(format!( + "Failed to parse persisted snapshot initial SOA: {err}" + )) + })?; + + let mut records = vec![]; + loop { + // Parse remaining resource records from the current read + // index in the buffer, each time receiving back the 'rest' + // index at which parsing should start on the next iteration + // of the loop. + let r; + (r, rest) = parse_rr(buf, rest).map_err(|err| { + io::Error::other(format!( + "Failed to parse persisted snapshot resource record at pos {rest}: {err}" + )) + })?; + + // If the parsed record is a SOA it should be identical to + // the SOA record that started the AXFR dump and signals the + // end of the dump. + // + // TODO: If the SOA is not at the apex should we keep it and + // keep going? + if r.rtype == RType::SOA { + // An AXFR ends with a SOA RR identical to the start SOA. + // Since we persisted the AXFR wire dump to disk it is a + // very unexpected error if it does not match the starting + // SOA. + if (r.rname.as_ref() == &*start_soa.0.rname + || r.rclass == start_soa.0.rclass + || r.ttl == start_soa.ttl) + && let Ok(soa_rdata) = Soa::::parse_message_bytes(r.rdata.bytes(), 0) + && soa_rdata == start_soa_rdata + { + break; + } + } + + let r = r.transform(|name| name.unsized_copy_into(), |data| data); + records.push(RegularRecord(r)); + } + + Ok((start_soa, records)) +} + +fn load_ixfr_wire_dump( + source: &Path, + buf: &mut Vec, + mut rr_handler: F, +) -> io::Result<(Serial, Serial)> +where + F: FnMut(IxfrEvent), +{ + load_file_into_memory(source, buf)?; + + let buf = buf.as_slice(); + let (start_soa, start_soa_rdata, mut rest) = parse_soa(buf, 0).map_err(|err| { + io::Error::other(format!("Failed to parse persisted diff initial SOA: {err}")) + })?; + + let mut oldest_soa = None; + + // Parse one or more diff sequences. + loop { + // Parse one diff sequence: + // "Each difference sequence represents one update to the + // zone (one SOA serial change) consisting of deleted RRs + // and added RRs. The first RR of the deleted RRs is the + // older SOA RR and the first RR of the added RRs is the + // newer SOA RR." + // Iterate over a sequence of resource records to remove + // followed by a sequence of resource records to add, + + // Read the first deleted RR which should be a SOA RR. + let r; + (r, rest) = parse_rr(buf, rest).map_err(|err| { + io::Error::other(format!( + "Failed to parse persisted diff resource record at pos {rest}: {err}" + )) + })?; + + if r.rtype != RType::SOA { + // If this is the first RR of the sequence it MUST + // be a SOA RR. + return Err(io::Error::other(format!( + "Expected first record of persisted diff remove sequence to be a SOA RR but found RTYPE {}", + r.rtype.code + ))); + } + + if oldest_soa.is_none() { + let soa = Soa::::parse_message_bytes(r.rdata.bytes(), 0).unwrap(); + oldest_soa = Some(soa); + } + + let r = RegularRecord(r.transform(|name| name.unsized_copy_into(), |data| data)); + rr_handler(IxfrEvent::Remove(r)); + + // Read more removed RRs until a SOA signals the end of the + // removed RRs and the start of the added RRs. + loop { + let r; + (r, rest) = parse_rr(buf, rest).map_err(|err| { + io::Error::other(format!( + "Failed to parse persisted diff removed resource record at pos {rest}: {err}" + )) + })?; + + let r = RegularRecord(r.transform(|name| name.unsized_copy_into(), |data| data)); + if r.rtype == RType::SOA { + rr_handler(IxfrEvent::Add(r)); + break; + } else { + rr_handler(IxfrEvent::Remove(r)); + } + } + + // Read more added RRs until a SOA signals the end of added RRs. + loop { + let r; + (r, rest) = parse_rr(buf, rest).map_err(|err| { + io::Error::other(format!( + "Failed to parse persisted diff added resource record at pos {rest}: {err}" + )) + })?; + + let r = + RegularRecord(r.transform(|name| name.unsized_copy_into(), |data| data.clone())); + if r.rtype == RType::SOA { + if is_same_soa(&start_soa, &start_soa_rdata, &r) { + // This SOA signals the end of the IXFR dump. + return Ok((oldest_soa.unwrap().serial, start_soa.rdata.serial)); + } + + rr_handler(IxfrEvent::EndOfUpdate); + rr_handler(IxfrEvent::Remove(r)); + break; + } else { + rr_handler(IxfrEvent::Add(r)); + } + } + } +} + +fn is_same_soa(start_soa: &SoaRecord, start_soa_rdata: &Soa, r: &RegularRecord) -> bool { + if (r.rtype == RType::SOA + && r.rname.as_ref() == &*start_soa.0.rname + && r.rclass == start_soa.0.rclass + && r.ttl == start_soa.ttl) + && let Ok(soa_rdata) = Soa::::parse_message_bytes(r.rdata.bytes(), 0) + { + return &soa_rdata == start_soa_rdata; + } + false +} + +enum IxfrEvent { + Remove(RegularRecord), + Add(RegularRecord), + EndOfUpdate, +} + +fn apply_ixfr_event_to_loaded_data(patcher: &mut LoadedZonePatcher<'_>, event: IxfrEvent) { + match event { + IxfrEvent::Remove(r) if r.rtype == RType::SOA => patcher.remove_soa(r.into()).unwrap(), + IxfrEvent::Remove(r) => patcher.remove(r).unwrap(), + IxfrEvent::Add(r) if r.rtype == RType::SOA => patcher.add_soa(r.into()).unwrap(), + IxfrEvent::Add(r) => patcher.add(r).unwrap(), + IxfrEvent::EndOfUpdate => patcher.next_patchset().unwrap(), + } +} + +fn apply_ixfr_event_to_signed_data(patcher: &mut SignedZonePatcher<'_>, event: IxfrEvent) { + match event { + IxfrEvent::Remove(r) if r.rtype == RType::SOA => patcher.remove_soa(r.into()).unwrap(), + IxfrEvent::Remove(r) => patcher.remove(r).unwrap(), + IxfrEvent::Add(r) if r.rtype == RType::SOA => patcher.add_soa(r.into()).unwrap(), + IxfrEvent::Add(r) => patcher.add(r).unwrap(), + IxfrEvent::EndOfUpdate => patcher.next_patchset().unwrap(), + } } diff --git a/src/persistence/zone.rs b/src/persistence/zone.rs index d5383c50d..812d7bdde 100644 --- a/src/persistence/zone.rs +++ b/src/persistence/zone.rs @@ -2,13 +2,15 @@ use std::sync::Arc; -use cascade_zonedata::{LoadedZonePersister, LoadedZoneRestorer, SignedZonePersister}; -use tracing::{debug, info, trace, trace_span}; +use cascade_zonedata::{ + LoadedZonePersister, LoadedZoneRestorer, SignedZonePersister, SignedZoneRestorer, +}; +use tracing::{debug, info, trace, trace_span, warn}; use crate::{ center::Center, util::BackgroundTasks, - zone::{Zone, ZoneHandle, ZoneState}, + zone::{Zone, ZoneHandle, ZoneState, save_state_now}, }; //----------- ZonePersistenceHandle -------------------------------------------- @@ -40,7 +42,7 @@ impl ZonePersistenceHandle<'_> { /// A background task will be spawned to restore the zone's data (for both /// the loaded and signed instances). #[tracing::instrument( - level = "trace", + level = "info", skip_all, fields(zone = %self.zone.name), )] @@ -57,21 +59,20 @@ impl ZonePersistenceHandle<'_> { // Try to restore the loaded instance. let mut restorer = restorer; let restored = match super::restore_loaded(&zone, ¢er, &mut restorer) { - Ok(()) => restorer.finish().unwrap_or_else(|_| { - unreachable!( - "'restore_loaded()' always completes restoration on successful return" - ) - }), - Err(_) => { - trace!("Abandoning loaded restoration"); - let mut state = zone.state.lock().unwrap(); - let mut handle = ZoneHandle { - zone: &zone, - state: &mut state, - center: ¢er, - }; - handle.storage().abandon_loaded_restoration(restorer); - handle.state.persistence.ongoing.finish(); + Ok(true) => { + // Data was restored. Use it. + restorer.finish().unwrap_or_else(|_| { + unreachable!("Loaded zone restoration should have built new zone data") + }) + } + Ok(false) => { + // There was nothing to restore. + abandon_loaded_restoration(¢er, &zone, restorer); + return; + } + Err(err) => { + warn!("Abandoning loaded restoration: {err}"); + abandon_loaded_restoration(¢er, &zone, restorer); return; } }; @@ -89,27 +90,26 @@ impl ZonePersistenceHandle<'_> { // Try to restore the signed instance. let restored = match super::restore_signed(&zone, ¢er, &mut restorer) { - Ok(()) => restorer.finish().unwrap_or_else(|_| { - unreachable!( - "'restore_signed()' always completes restoration on successful return" - ) - }), - Err(_) => { - trace!("Abandoning signed restoration"); - let mut state = zone.state.lock().unwrap(); - let mut handle = ZoneHandle { - zone: &zone, - state: &mut state, - center: ¢er, - }; - handle.storage().abandon_signed_restoration(restorer); - handle.state.persistence.ongoing.finish(); + Ok(true) => { + // Data was restored. Use it. + restorer.finish().unwrap_or_else(|_| { + unreachable!("Signed zone restoration should have built new zone data") + }) + } + Ok(false) => { + // There was nothing to restore. + abandon_signed_restoration(¢er, &zone, restorer); + return; + } + Err(err) => { + warn!("Abandoning signed restoration: {err}"); + abandon_signed_restoration(¢er, &zone, restorer); return; } }; - info!("Restored the zone's persisted data"); let mut state = zone.state.lock().unwrap(); + trace!("Restored diffs: {:?}", state.persisted_loaded_diff_paths); let mut handle = ZoneHandle { zone: &zone, state: &mut state, @@ -138,8 +138,8 @@ impl ZonePersistenceHandle<'_> { .ongoing .spawn_blocking(span, move || { debug!("Persisting the loaded instance"); - let persisted = super::persist_loaded(&zone, ¢er, persister); + debug!("Persisting the loaded instance completed"); // NOTE: The outer function, which is spawning the background // task, has a lock of the zone state. Thus, the following lock @@ -175,8 +175,8 @@ impl ZonePersistenceHandle<'_> { .ongoing .spawn_blocking(span, move || { debug!("Persisting the signed instance"); - let persisted = super::persist_signed(&zone, ¢er, persister); + debug!("Persisting the signed instance completed"); // NOTE: The outer function, which is spawning the background // task, has a lock of the zone state. Thus, the following lock @@ -195,6 +195,87 @@ impl ZonePersistenceHandle<'_> { } } +fn abandon_loaded_restoration( + center: &Arc
, + zone: &Arc, + restorer: LoadedZoneRestorer, +) { + reset_state_due_to_abandoned_restore(center, zone); + let mut state = zone.state.lock().unwrap(); + let mut handle = ZoneHandle { + zone, + state: &mut state, + center, + }; + handle.storage().abandon_loaded_restoration(restorer); + handle.state.persistence.ongoing.finish(); +} + +fn abandon_signed_restoration( + center: &Arc
, + zone: &Arc, + restorer: SignedZoneRestorer, +) { + reset_state_due_to_abandoned_restore(center, zone); + let mut state = zone.state.lock().unwrap(); + let mut handle = ZoneHandle { + zone, + state: &mut state, + center, + }; + handle.storage().abandon_signed_restoration(restorer); + handle.state.persistence.ongoing.finish(); +} + +fn reset_state_due_to_abandoned_restore(center: &Arc
, zone: &Arc) { + { + let mut state = zone.state.lock().unwrap(); + clear_persisted_zone_data(center, &mut state); + + // In case this zone was signed in the past we have to make sure that + // any attempt to enqueue a re-signing operation will be skipped as + // doing so will fail due to the lack of loaded zone content. + // TODO: Find a better way to prevent this issue as changing the + // min_expiration timestamps is a very indirect and non-obvious way of + // preventing re-signing. + state.min_expiration = None; + state.next_min_expiration = None; + + // Also remove any already enqueued signing operation that is blocked + // by the ongoing restore as it will otherwise immediately start once + // the restore completes. + state.signer.cancel_enqueued_signing_operations(); + } + save_state_now(center, zone); +} + +fn clear_persisted_zone_data(center: &Center, state: &mut ZoneState) { + // We can't use the persisted data so remove the paths from state, remove + // the corresponding files on disk and remove any diffs that we loaded + // into memory. + for p in state + .persisted_loaded_diff_paths + .iter() + .chain(state.persisted_signed_diff_paths.iter()) + { + if p.exists() && p.starts_with(center.config.zone_state_dir.as_std_path()) { + info!( + "Removing unusable persisted zone data file '{}'", + p.display() + ); + if let Err(err) = std::fs::remove_file(p) { + warn!( + "Failed to remove unusable persisted zone data file '{}': {err}", + p.display() + ); + } + } + } + state.persisted_loaded_diff_paths.clear(); + state.persisted_signed_diff_paths.clear(); + state.storage.diffs.clear(); +} + //----------- PersistenceState ------------------------------------------------- /// State related to data persistence for a zone. diff --git a/src/server/service.rs b/src/server/service.rs index 8bef675f7..438cbd566 100644 --- a/src/server/service.rs +++ b/src/server/service.rs @@ -179,17 +179,17 @@ mod compat { fn is_permitted( zone: &ServedZone, - old_request: &Request, Option>>, + request: &Request, Option>>, ) -> bool { let zone_state = zone.handle.state.lock().unwrap(); if tracing::enabled!(Level::TRACE) { - let tsig_key = old_request.metadata().as_ref().map(|key| key.name()); + let tsig_key = request.metadata().as_ref().map(|key| key.name()); trace!( "Received request {} from {} for {} in zone {} with TSIG key {tsig_key:?}", - old_request.message().header().id(), - old_request.client_addr().ip(), - old_request + request.message().header().id(), + request.client_addr().ip(), + request .message() .qtype() .map(|rtype| rtype.to_string()) @@ -205,11 +205,11 @@ mod compat { { // If at least one ACL was specified, enforce it. if !acls.is_empty() { - let wanted_tsig_key_name = old_request.metadata().as_ref().map(|key| key.name()); + let wanted_tsig_key_name = request.metadata().as_ref().map(|key| key.name()); for acl in acls { // Does the client address match the allowed address? - if acl.addr.ip() == old_request.client_addr().ip() { + if acl.addr.ip() == request.client_addr().ip() { // Is the request signed with the right TSIG key? if acl.tsig_key_name.as_ref() == wanted_tsig_key_name { // Allow the request. @@ -229,9 +229,9 @@ mod compat { }; debug!( "Rejecting request {} from {} for {} in zone {}: access denied{extra}", - old_request.message().header().id(), - old_request.client_addr().ip(), - old_request + request.message().header().id(), + request.client_addr().ip(), + request .message() .qtype() .map(|rtype| rtype.to_string()) diff --git a/src/signer/zone.rs b/src/signer/zone.rs index 46a6f3dd7..d64cbd6a8 100644 --- a/src/signer/zone.rs +++ b/src/signer/zone.rs @@ -254,6 +254,13 @@ pub struct SignerState { pub enqueued_resign: Option, } +impl SignerState { + pub fn cancel_enqueued_signing_operations(&mut self) { + self.enqueued_new_sign = None; + self.enqueued_resign = None; + } +} + //----------- EnqueuedSign ----------------------------------------------------- /// An enqueued sign of a zone. diff --git a/src/units/http_server.rs b/src/units/http_server.rs index 08cc3f67d..df8393101 100644 --- a/src/units/http_server.rs +++ b/src/units/http_server.rs @@ -474,7 +474,13 @@ impl HttpServer { .map(|r| Serial::from(u32::from(r.rdata.serial))); progress = match zone_state.machine { - ZoneStateMachine::Waiting(..) => Progress::Waiting, + ZoneStateMachine::Waiting(..) => { + if zone_state.storage.is_restoring() { + Progress::Restoring + } else { + Progress::Waiting + } + } ZoneStateMachine::Loading(..) => Progress::Loading, ZoneStateMachine::LoadedReview(..) => Progress::LoadedReview, ZoneStateMachine::HaltLoaded(..) => Progress::HaltLoaded, diff --git a/src/zone/machine.rs b/src/zone/machine.rs index e320956ac..4074dedf5 100644 --- a/src/zone/machine.rs +++ b/src/zone/machine.rs @@ -265,28 +265,6 @@ impl<'a> ZoneHandle<'a> { self.signer().enqueue_new_sign(builder); } - /// Begin signing a restored loaded instance. - /// - /// This is called when the zone is restored from persisted state, the data - /// for its last known loaded instance is restored successfully, but the - /// data for its last known signed instance could not be restored (or none - /// existed). It directly initiates signing. - pub(crate) fn start_sign_after_restore( - &mut self, - builder: cascade_zonedata::SignedZoneBuilder, - ) { - // Force the state machine to the signing state. - let (transition, state) = self.state.machine.transition(); - let ZoneStateMachine::Waiting(waiting) = state else { - panic!("'start_sign_after_restore()' called when the zone was not passive"); - }; - transition.move_to(ZoneStateMachine::Signing( - waiting.start_sign_after_restore(), - )); - - self.signer().enqueue_new_sign(builder); - } - pub(crate) fn finish_signing(&mut self, built: cascade_zonedata::SignedZoneBuilt) { let (transition, state) = self.state.machine.transition(); @@ -542,9 +520,9 @@ impl Waiting { Loading {} } - fn start_sign_after_restore(self) -> Signing { - Signing {} - } + // fn start_sign_after_restore(self) -> Signing { + // Signing {} + // } fn start_resign(self) -> Signing { Signing {} diff --git a/src/zone/mod.rs b/src/zone/mod.rs index 699af28f4..d764ab168 100644 --- a/src/zone/mod.rs +++ b/src/zone/mod.rs @@ -1,6 +1,7 @@ //! Zone-specific state and management. use std::collections::HashSet; +use std::path::PathBuf; use std::{ borrow::Borrow, cmp::Ordering, @@ -258,6 +259,18 @@ pub struct ZoneState { /// History of interesting events that occurred for this zone. pub history: Vec, + /// Locations of persisted unsigned zone diffs to enable IXFR from + /// the upstream to resume on restart, and to enable a complete latest + /// unsigned version of the zone to be reconstituted. + // TODO: Move into `PersistenceState`. + pub persisted_loaded_diff_paths: Vec, + + /// Locations of persisted signed zone diffs to ensure IXFR out toward + /// downstreams is still possible after restart, and to enable a complete + /// latest signed version of the zone to be reconsituted. + // TODO: Move into `PersistenceState`. + pub persisted_signed_diff_paths: Vec, + /// Loading new versions of the zone. pub loader: LoaderState, @@ -321,11 +334,13 @@ impl Default for ZoneState { signer: Default::default(), storage: Default::default(), persistence: Default::default(), + persisted_loaded_diff_paths: Default::default(), + persisted_signed_diff_paths: Default::default(), } } } -#[derive(Debug)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct LastPublished { pub loaded_serial: Serial, pub signed_serial: Serial, diff --git a/src/zone/state/mod.rs b/src/zone/state/mod.rs index f68c58962..c9cd83ef2 100644 --- a/src/zone/state/mod.rs +++ b/src/zone/state/mod.rs @@ -84,6 +84,7 @@ impl Spec { match self { Self::V1(v1::Spec { policy, + last_published, source, min_expiration, next_min_expiration, @@ -94,6 +95,8 @@ impl Spec { last_signature_refresh, previous_serial, history, + persisted_loaded_diffs, + persisted_signed_diffs, }) => { let loader = LoaderState { source: source @@ -113,6 +116,7 @@ impl Spec { Ok(ZoneState { policy, + last_published, min_expiration, next_min_expiration, apex_remove, @@ -123,6 +127,8 @@ impl Spec { previous_serial, loader, history, + persisted_loaded_diff_paths: persisted_loaded_diffs, + persisted_signed_diff_paths: persisted_signed_diffs, ..Default::default() }) } diff --git a/src/zone/state/v1.rs b/src/zone/state/v1.rs index cd0561bbc..5c97292dd 100644 --- a/src/zone/state/v1.rs +++ b/src/zone/state/v1.rs @@ -2,6 +2,7 @@ use std::collections::HashSet; use std::net::SocketAddr; +use std::path::PathBuf; use std::time::Duration; use camino::Utf8Path; @@ -15,7 +16,7 @@ use crate::loader::Source; use crate::policy::file::v1::{NameserverCommsSpec, OutboundSpec}; use crate::policy::{AutoConfig, DsAlgorithm, KeyParameters}; use crate::tsig::TsigStore; -use crate::zone::HistoryItem; +use crate::zone::{HistoryItem, LastPublished}; use crate::{ policy::{ KeyManagerPolicy, LoaderPolicy, PolicyVersion, ReviewPolicy, ServerPolicy, @@ -38,6 +39,9 @@ pub struct Spec { /// version of the policy that is not yet in use. pub policy: Option, + /// Metadata related to the last published zone version. + pub last_published: Option, + /// The source of the zone. pub source: ZoneLoadSourceSpec, @@ -91,6 +95,16 @@ pub struct Spec { /// History of interesting events that occurred for this zone. pub history: Vec, + + /// Locations of persisted unsigned zone diffs to enable IXFR from + /// the upstream to resume on restart, and to enable a complete latest + /// unsigned version of the zone to be reconstituted. + pub persisted_loaded_diffs: Vec, + + /// Locations of persisted signed zone diffs to ensure IXFR out toward + /// downstreams is still possible after restart, and to enable a complete + /// latest signed version of the zone to be reconsituted. + pub persisted_signed_diffs: Vec, } //--- Conversion @@ -100,6 +114,7 @@ impl Spec { pub fn build(zone: &ZoneState) -> Self { Self { policy: zone.policy.as_ref().map(|p| PolicySpec::build(p)), + last_published: zone.last_published.clone(), source: ZoneLoadSourceSpec::build(&zone.loader.source), min_expiration: zone.min_expiration, next_min_expiration: zone.next_min_expiration, @@ -110,6 +125,8 @@ impl Spec { last_signature_refresh: zone.last_signature_refresh.clone(), previous_serial: zone.previous_serial, history: zone.history.clone(), + persisted_loaded_diffs: zone.persisted_loaded_diff_paths.clone(), + persisted_signed_diffs: zone.persisted_signed_diff_paths.clone(), } } } diff --git a/src/zone/storage.rs b/src/zone/storage.rs index 418c3f2b0..98803a346 100644 --- a/src/zone/storage.rs +++ b/src/zone/storage.rs @@ -633,18 +633,16 @@ impl StorageZoneHandle<'_> { /// Abandon the ongoing signed-instance restore. /// - /// Any intermediate zone data for the signed instance is wiped. The - /// restored loaded instance is preserved. The zone is moved to the signing - /// state. It is registered against Cascade's zone servers and a signing - /// operation is enqueued. + /// Any intermediate zone data is cleared and the zone is moved to the + /// passive state. It is registered against Cascade's zone servers. pub fn abandon_signed_restoration(&mut self, restorer: SignedZoneRestorer) { // Examine the current state. - let (loaded_reviewer, signed_reviewer, viewer, builder); + let (loaded_reviewer, signed_reviewer, viewer); match transition(&mut self.state.storage.machine) { (transition, ZoneDataStorage::RestoringSigned(s)) => { let new_s; - (loaded_reviewer, signed_reviewer, viewer, builder, new_s) = s.abandon(restorer); - transition.move_to(ZoneDataStorage::Signing(new_s)); + (loaded_reviewer, signed_reviewer, viewer, new_s) = s.abandon(restorer); + transition.move_to(ZoneDataStorage::Passive(new_s)); } _ => unreachable!( @@ -657,8 +655,8 @@ impl StorageZoneHandle<'_> { SignedReviewServer::add_zone(self.center, self.zone.clone(), signed_reviewer); PublicationServer::add_zone(self.center, self.zone.clone(), viewer); - // Initiate a new signing operation. - self.zone().start_sign_after_restore(builder); + // Send a notification that the state machine is now passive. + self.on_passive(); } } @@ -959,6 +957,14 @@ impl StorageState { } } + /// Is this zone currently being restored from persistent storage? + pub fn is_restoring(&self) -> bool { + matches!( + self.machine, + ZoneDataStorage::RestoringLoaded(_) | ZoneDataStorage::RestoringSigned(_) + ) + } + /// Get the current loaded diff, if any. pub fn current_loaded_diff(&self) -> Option> { self.machine.loaded_diff()