diff --git a/Cargo.toml b/Cargo.toml index 5a68df1..144900f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,11 @@ [workspace] -members = ["accumulator", "hintfile", "node"] +members = ["accumulator", "node"] default-members = ["accumulator"] resolver = "2" [workspace.dependencies] bitcoin = { git = "https://github.com/rust-bitcoin/rust-bitcoin", default-features = false, rev = "16cc257c3695dea0e7301a5fa9cab44b8ed60598" } +hintsfile = { git = "https://github.com/rustaceanrob/hintsfile.git" } +# hintsfile = { path = "../hintsfile/" } kernel = { package = "bitcoinkernel", git = "https://github.com/alexanderwiederin/rust-bitcoinkernel.git", rev = "353533221e3ba91d672418eab1ae7b83a61214f9" } p2p = { package = "bitcoin-p2p", git = "https://github.com/2140-dev/bitcoin-p2p.git", rev = "a8b3af8dfc32a06eadb867b07afd537ebf347104" } diff --git a/README.md b/README.md index 27f9b49..a9640cf 100644 --- a/README.md +++ b/README.md @@ -5,5 +5,4 @@ This repository is a collection of crates related to a SwiftSync node implementation. Some crates will be SwiftSync-specific, while others may have broader use cases. - `accumulator`: A hash-based SwiftSync accumulator used to add and subtrack elements from a set. -- `hintfile`: Read a hints file from a server. - `node`: Perform fast IBD using a SwiftSync hints file. diff --git a/hintfile/Cargo.toml b/hintfile/Cargo.toml deleted file mode 100644 index d0c0360..0000000 --- a/hintfile/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "hintfile" -version = "0.1.0" -edition = "2021" - - diff --git a/hintfile/README.md b/hintfile/README.md deleted file mode 100644 index a04a2cb..0000000 --- a/hintfile/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# SwiftSync hint file - -This crate is capable of generating a `.hints` file to use as a SwiftSync client. You will need to have Bitcoin Core synced to a recent block or either mainnet or Signet. diff --git a/hintfile/src/lib.rs b/hintfile/src/lib.rs deleted file mode 100644 index 9c43d3b..0000000 --- a/hintfile/src/lib.rs +++ /dev/null @@ -1,152 +0,0 @@ -use std::{ - collections::BTreeMap, - fs::File, - io::{self, Read, Seek, SeekFrom, Write}, -}; - -type BlockHeight = u32; -type FilePos = u64; - -pub fn write_compact_size(value: u64, writer: &mut W) -> Result<(), io::Error> { - match value { - 0..=0xFC => { - writer.write_all(&[value as u8]) // Cast ok because of match. - } - 0xFD..=0xFFFF => { - let v = (value as u16).to_le_bytes(); // Cast ok because of match. - writer.write_all(&[0xFD, v[0], v[1]]) - } - 0x10000..=0xFFFFFFFF => { - let v = (value as u32).to_le_bytes(); // Cast ok because of match. - writer.write_all(&[0xFE, v[0], v[1], v[2], v[3]]) - } - _ => panic!("unexpected large offset"), - } -} - -pub fn read_compact_size(reader: &mut R) -> Result { - let mut buf: [u8; 1] = [0; 1]; - reader.read_exact(&mut buf)?; - let prefix = buf[0]; - match prefix { - 0xFD => { - let mut buf: [u8; 2] = [0; 2]; - reader.read_exact(&mut buf)?; - Ok(u16::from_le_bytes(buf) as u64) - } - 0xFE => { - let mut buf: [u8; 4] = [0; 4]; - reader.read_exact(&mut buf)?; - Ok(u32::from_le_bytes(buf) as u64) - } - 0..=0xFC => Ok(prefix as u64), - _ => panic!("unexpected large offset"), - } -} - -#[derive(Debug)] -pub struct Hints { - map: BTreeMap, - file: File, - stop_height: BlockHeight, -} - -impl Hints { - // # Panics - // - // Panics when expected data is not present, or the hintfile overflows the maximum blockheight - pub fn from_file(mut file: File) -> Self { - let mut map = BTreeMap::new(); - let mut magic = [0; 4]; - file.read_exact(&mut magic).unwrap(); - assert_eq!(magic, [0x55, 0x54, 0x58, 0x4f]); - let mut ver = [0; 1]; - file.read_exact(&mut ver).unwrap(); - if u8::from_le_bytes(ver) != 0x00 { - panic!("Unsupported file version."); - } - let mut stop_height = [0; 4]; - file.read_exact(&mut stop_height).expect("empty file"); - let stop_height = BlockHeight::from_le_bytes(stop_height); - for _ in 1..=stop_height { - let mut height = [0; 4]; - file.read_exact(&mut height) - .expect("expected kv pair does not exist."); - let height = BlockHeight::from_le_bytes(height); - let mut file_pos = [0; 8]; - file.read_exact(&mut file_pos) - .expect("expected kv pair does not exist."); - let file_pos = FilePos::from_le_bytes(file_pos); - map.insert(height, file_pos); - } - Self { - map, - file, - stop_height, - } - } - - /// Get the stop height of the hint file. - pub fn stop_height(&self) -> BlockHeight { - self.stop_height - } - - /// # Panics - /// - /// If there are no offset present at that height, aka an overflow, or the entry has already - /// been fetched. - pub fn get_indexes(&mut self, height: BlockHeight) -> Vec { - let file_pos = self - .map - .get(&height) - .cloned() - .expect("block height overflow"); - self.file - .seek(SeekFrom::Start(file_pos)) - .expect("missing file position."); - let mut bits_arr = [0; 4]; - self.file.read_exact(&mut bits_arr).unwrap(); - let mut unspents = Vec::new(); - let num_bits = u32::from_le_bytes(bits_arr); - let mut curr_byte: u8 = 0; - for bit_pos in 0..num_bits { - let leftovers = bit_pos % 8; - if leftovers == 0 { - let mut single_byte_arr = [0; 1]; - self.file.read_exact(&mut single_byte_arr).unwrap(); - curr_byte = u8::from_le_bytes(single_byte_arr); - } - if ((curr_byte >> leftovers) & 0x01) == 0x01 { - unspents.push(bit_pos as u64); - } - } - unspents - } -} - -#[cfg(test)] -mod tests { - use crate::{read_compact_size, write_compact_size}; - - #[test] - fn deser_roundtrip() { - let mut buf = Vec::new(); - let less: u8 = 0xFB; - write_compact_size(less as u64, &mut buf).unwrap(); - let read_cs = read_compact_size(&mut buf.as_slice()).unwrap(); - let cast_less = read_cs as u8; - assert_eq!(less, cast_less); - let mut buf = Vec::new(); - let median: u16 = 0xFFF; - write_compact_size(median as u64, &mut buf).unwrap(); - let read_cs = read_compact_size(&mut buf.as_slice()).unwrap(); - let cast_median = read_cs as u16; - assert_eq!(median, cast_median); - let mut buf = Vec::new(); - let more: u32 = 0xFFFFF; - write_compact_size(more as u64, &mut buf).unwrap(); - let read_cs = read_compact_size(&mut buf.as_slice()).unwrap(); - let cast_more = read_cs as u32; - assert_eq!(more, cast_more); - } -} diff --git a/node/Cargo.toml b/node/Cargo.toml index 0f2639d..54390da 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -8,7 +8,7 @@ build = "build.rs" accumulator = { path = "../accumulator/" } bitcoin = { workspace = true } kernel = { workspace = true } -hintfile = { path = "../hintfile/" } +hintsfile = { workspace = true } p2p = { workspace = true } configure_me = "0.4.0" diff --git a/node/src/bin/ibd.rs b/node/src/bin/ibd.rs index ac2081a..1c4c0af 100644 --- a/node/src/bin/ibd.rs +++ b/node/src/bin/ibd.rs @@ -6,7 +6,7 @@ use std::{ }; use bitcoin::Network; -use hintfile::Hints; +use hintsfile::Hintsfile; use kernel::{ChainstateManager, ChainstateManagerOptions, ContextBuilder}; use node::{ @@ -41,8 +41,8 @@ fn main() { tracing::subscriber::set_global_default(subscriber).unwrap(); let hintfile_start_time = Instant::now(); tracing::info!("Reading in {hint_path}"); - let hintfile = File::open(hint_path).expect("invalid hintfile path"); - let hints = Hints::from_file(hintfile); + let mut hintfile = File::open(hint_path).expect("invalid hintfile path"); + let hints = Hintsfile::from_reader(&mut hintfile); let stop_height = hints.stop_height(); elapsed_time(hintfile_start_time); tracing::info!("Syncing to height {}", hints.stop_height()); diff --git a/node/src/lib.rs b/node/src/lib.rs index fb3ce8a..3cc6528 100644 --- a/node/src/lib.rs +++ b/node/src/lib.rs @@ -19,7 +19,7 @@ use bitcoin::{ transaction::TransactionExt, BlockHash, Network, OutPoint, }; -use hintfile::Hints; +use hintsfile::Hintsfile; use kernel::{ChainType, ChainstateManager}; use p2p::{ dns::DnsQueryExt, @@ -177,7 +177,7 @@ pub fn get_blocks_for_range( network: Network, block_dir: Option, chain: Arc, - hints: Arc>, + hints: Arc>, peers: Arc>>, updater: Sender, hashes: Arc>>>, @@ -234,9 +234,13 @@ pub fn get_blocks_for_range( .block_index_by_hash(kernal_hash) .expect("header is in best chain."); let block_height = block_index.height().unsigned_abs(); - let unspent_indexes: HashSet = { - let mut hint_ref = hints.lock().unwrap(); - hint_ref.get_indexes(block_height).into_iter().collect() + let unspent_indexes: HashSet = { + let hint_ref = hints.lock().unwrap(); + hint_ref + .indices_at_height(block_height) + .expect("hints should exist") + .into_iter() + .collect() }; if let Some(block_dir) = block_dir.as_ref() { let file_path = block_dir.join(format!("{hash}.block"));