From 8f1c0cad9671d53c1b88c654b8f37648186cc072 Mon Sep 17 00:00:00 2001 From: Disservin Date: Tue, 30 Sep 2025 20:58:03 +0200 Subject: [PATCH 1/2] directly read into buffer --- src/common/compressed_training_file_reader.rs | 8 ++++++++ src/reader/compressed_reader.rs | 9 +++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/common/compressed_training_file_reader.rs b/src/common/compressed_training_file_reader.rs index 9c7111e..63f76ff 100644 --- a/src/common/compressed_training_file_reader.rs +++ b/src/common/compressed_training_file_reader.rs @@ -52,6 +52,14 @@ impl CompressedTrainingDataFileReader { Ok(data) } + pub fn read_next_chunk_into(&mut self, buffer: &mut Vec) -> Result<()> { + let header = self.read_chunk_header()?; + buffer.resize(header.chunk_size as usize, 0); + self.file.read_exact(buffer)?; + self.read_bytes += header.chunk_size as u64; + Ok(()) + } + fn read_chunk_header(&mut self) -> Result
{ let mut buf = [0u8; HEADER_SIZE]; diff --git a/src/reader/compressed_reader.rs b/src/reader/compressed_reader.rs index 0a263a1..e1b6812 100644 --- a/src/reader/compressed_reader.rs +++ b/src/reader/compressed_reader.rs @@ -97,10 +97,11 @@ impl CompressedTrainingDataEntryReader { reader.is_end = true; return Err(CompressedReaderError::EndOfFile); } else { - reader.chunk = match reader.input_file.as_mut().unwrap().read_next_chunk() { - Ok(chunk) => chunk, - Err(e) => return Err(CompressedReaderError::BinpackError(e)), - }; + reader + .input_file + .as_mut() + .unwrap() + .read_next_chunk_into(&mut reader.chunk)?; } Ok(reader) From 95fb32bb605a0a280ac391b9dd0afb321e2483c5 Mon Sep 17 00:00:00 2001 From: Disservin Date: Tue, 30 Sep 2025 21:50:40 +0200 Subject: [PATCH 2/2] update --- src/common/compressed_training_file_reader.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/compressed_training_file_reader.rs b/src/common/compressed_training_file_reader.rs index 63f76ff..8e5d444 100644 --- a/src/common/compressed_training_file_reader.rs +++ b/src/common/compressed_training_file_reader.rs @@ -52,7 +52,7 @@ impl CompressedTrainingDataFileReader { Ok(data) } - pub fn read_next_chunk_into(&mut self, buffer: &mut Vec) -> Result<()> { + pub fn read_next_chunk_into(&mut self, buffer: &mut Vec) -> Result<()> { let header = self.read_chunk_header()?; buffer.resize(header.chunk_size as usize, 0); self.file.read_exact(buffer)?;