diff --git a/cli/src/commands/scan.rs b/cli/src/commands/scan.rs index 62755fcd..3810940b 100644 --- a/cli/src/commands/scan.rs +++ b/cli/src/commands/scan.rs @@ -237,7 +237,7 @@ pub fn exec_scan(args: &ArgMatches, config: &Config) -> anyhow::Result<()> { } let file = File::open(rules_path) - .with_context(|| format!("can not open {:?}", &rules_path))?; + .with_context(|| format!("can not open {:?}", rules_path))?; let rules = Rules::deserialize_from(file)?; @@ -340,7 +340,7 @@ pub fn exec_scan(args: &ArgMatches, config: &Config) -> anyhow::Result<()> { let path = file_path.display().to_string(); scanner.console_log(move |msg| { output - .send(Message::Error(format!("{}: {}", &path.paint(Yellow), msg.paint(Yellow)))) + .send(Message::Error(format!("{}: {}", path.paint(Yellow), msg.paint(Yellow)))) .unwrap(); }); } @@ -374,7 +374,7 @@ pub fn exec_scan(args: &ArgMatches, config: &Config) -> anyhow::Result<()> { let scan_results = scanner .scan_file_with_options(file_path.as_path(), scan_options) - .with_context(|| format!("scanning {:?}", &file_path)); + .with_context(|| format!("scanning {:?}", file_path)); state .files_in_progress @@ -743,7 +743,7 @@ mod output_handler { output .send(Message::Info(format!( "{}: {}", - &file_path.display().to_string(), + file_path.display(), scan_results.len() ))) .unwrap(); diff --git a/lib/build.rs b/lib/build.rs index bee9b4dd..9ee06f21 100644 --- a/lib/build.rs +++ b/lib/build.rs @@ -371,10 +371,10 @@ fn generate_proto_code() { }; let path = fs::canonicalize(&path) - .with_context(|| format!("`{:?}`", &path)) + .with_context(|| format!("`{:?}`", path)) .expect("can not read file"); - println!("cargo:warning=using extra proto: {:?}", &path); + println!("cargo:warning=using extra proto: {:?}", path); let base_path = path.with_file_name(""); diff --git a/lib/src/compiler/mod.rs b/lib/src/compiler/mod.rs index acc9ca21..2774caee 100644 --- a/lib/src/compiler/mod.rs +++ b/lib/src/compiler/mod.rs @@ -830,6 +830,7 @@ impl<'a> Compiler<'a> { header_constraints: self.header_constraints, regex_sets: self.regex_sets, fast_scan_patterns: self.fast_scan_patterns, + rules_profiling_enabled: cfg!(feature = "rules-profiling"), }; rules.build_ac_automaton(); diff --git a/lib/src/compiler/report.rs b/lib/src/compiler/report.rs index e21dd18f..fccd5b3b 100644 --- a/lib/src/compiler/report.rs +++ b/lib/src/compiler/report.rs @@ -138,7 +138,7 @@ impl Report { Some((line, column)) => (line, column), None => panic!( "can't find line and column for span {span} in code:\n{}", - &cache_entry.code + cache_entry.code ), }; diff --git a/lib/src/compiler/rules.rs b/lib/src/compiler/rules.rs index d0d9c3f4..75c35fea 100644 --- a/lib/src/compiler/rules.rs +++ b/lib/src/compiler/rules.rs @@ -33,7 +33,7 @@ const MAGIC: &[u8] = b"YARA-X\0\0"; /// /// This version is incremented every time a change is made to the binary /// format in a way that breaks backwards compatibility. -const SERIALIZATION_VERSION: u32 = 3; +const SERIALIZATION_VERSION: u32 = 4; /// Aho-Corasick automaton bundled with an optional Teddy scanner if the /// number of patterns is low enough. If the Teddy scanner is present, and @@ -189,6 +189,14 @@ pub struct Rules { /// as count `#a`, offset `@a`, length `!a`, anchored checks, or loop /// equivalents), it cannot be fast-scanned. pub(in crate::compiler) fast_scan_patterns: bitvec::vec::BitVec, + + /// Indicates whether the rules were compiled with rules profiling enabled. + /// + /// This flag is checked during deserialization to ensure that the YARA-X + /// binary performing deserialization is built with the same profiling + /// settings as the compiled rules. A mismatch will result in a + /// deserialization error. + pub(in crate::compiler) rules_profiling_enabled: bool, } impl Rules { @@ -262,6 +270,16 @@ impl Rules { #[cfg(feature = "logging")] info!("Deserialization time: {:?}", Instant::elapsed(&start)); + let profiling_enabled = cfg!(feature = "rules-profiling"); + + if rules.rules_profiling_enabled != profiling_enabled { + return Err(SerializationError::InvalidWASM(anyhow!( + "rules profiling mismatch: rules compiled with profiling enabled = {}, but YARA-X compiled with profiling enabled = {}", + rules.rules_profiling_enabled, + profiling_enabled + ))); + } + // `rules.compiled_wasm_mod` can be `None` for two reasons: // // 1- The rules were serialized without compiled rules (i.e: the diff --git a/lib/src/compiler/tests/mod.rs b/lib/src/compiler/tests/mod.rs index d875546c..3d7b17dc 100644 --- a/lib/src/compiler/tests/mod.rs +++ b/lib/src/compiler/tests/mod.rs @@ -28,7 +28,7 @@ fn serialization() { // `DecodeError`. let mut data = Vec::new(); data.extend(b"YARA-X\0\0"); - data.extend(3u32.to_le_bytes()); + data.extend(4u32.to_le_bytes()); data.extend(b"foo"); assert!(matches!( @@ -48,6 +48,17 @@ fn serialization() { SerializationError::InvalidVersion { expected: _, actual: 0 } )); + // A mismatched rules profiling flag should produce an InvalidWASM error. + let mut rules = compile(r#"rule test { condition: true }"#).unwrap(); + rules.rules_profiling_enabled = !rules.rules_profiling_enabled; + + let rules_bytes = rules.serialize().unwrap(); + + assert!(matches!( + Rules::deserialize(&rules_bytes).err().unwrap(), + SerializationError::InvalidWASM(_) + )); + let rules = compile(r#"rule test { strings: $a = "foo" condition: $a }"#) .unwrap() .serialize() diff --git a/lib/src/modules/dotnet/parser.rs b/lib/src/modules/dotnet/parser.rs index 3914ca62..7feaaf7a 100644 --- a/lib/src/modules/dotnet/parser.rs +++ b/lib/src/modules/dotnet/parser.rs @@ -1549,21 +1549,9 @@ impl<'a> Dotnet<'a> { |(flags, name, namespace, extends, _field_list, method_list)| { TypeDef { flags, - name: name.and_then(|v| { - if v.is_empty() { - None - } else { - Some(v) - } - }), + name: name.filter(|&v| !v.is_empty()), // The namespace can be an empty string (""), - namespace: namespace.and_then(|v| { - if v.is_empty() { - None - } else { - Some(v) - } - }), + namespace: namespace.filter(|&v| !v.is_empty()), method_list, extends, } @@ -1956,13 +1944,7 @@ impl<'a> Dotnet<'a> { // cases return `None` instead. Empty strings are against // the specification, but it happens with files like: // 756684f4017ba7e931a26724ae61606b16b5f8cc84ed38a260a34e50c5016f59 - culture: culture.and_then(|v| { - if v.is_empty() { - None - } else { - Some(v) - } - }), + culture: culture.filter(|&v| !v.is_empty()), version: Version { major_version, minor_version, @@ -2009,13 +1991,7 @@ impl<'a> Dotnet<'a> { )| AssemblyRef { // public_key_or_token sometimes have an empty string (""), // in such cases return `None` instead. - public_key_or_token: public_key_or_token.and_then(|v| { - if v.is_empty() { - None - } else { - Some(v) - } - }), + public_key_or_token: public_key_or_token.filter(|&v| !v.is_empty()), name, version: Version { major_version, diff --git a/lib/src/modules/hash/mod.rs b/lib/src/modules/hash/mod.rs index 60a7f7bf..d478560c 100644 --- a/lib/src/modules/hash/mod.rs +++ b/lib/src/modules/hash/mod.rs @@ -216,10 +216,10 @@ fn crc_str(ctx: &ScanContext, s: RuntimeString) -> Option { #[inline] fn checksum32(data: &[u8]) -> u32 { let mut sum = 0_u64; - let mut chunks = data.chunks_exact(8); + let (chunks, remainder) = data.as_chunks::<8>(); - for chunk in &mut chunks { - let x = u64::from_le_bytes(chunk.try_into().unwrap()); + for chunk in chunks { + let x = u64::from_le_bytes(*chunk); let pairs = (x & 0x00ff_00ff_00ff_00ff) + ((x >> 8) & 0x00ff_00ff_00ff_00ff); let quads = (pairs & 0x0000_ffff_0000_ffff) @@ -228,7 +228,7 @@ fn checksum32(data: &[u8]) -> u32 { } let mut checksum = sum as u32; - for byte in chunks.remainder() { + for byte in remainder { checksum = checksum.wrapping_add(*byte as u32); } checksum diff --git a/lib/src/modules/math.rs b/lib/src/modules/math.rs index 51864a0d..797dabe8 100644 --- a/lib/src/modules/math.rs +++ b/lib/src/modules/math.rs @@ -424,7 +424,7 @@ fn monte_carlo_pi(data: &[u8]) -> Option { let mut inmont = 0; let mut mcount = 0; - for chunk in data.chunks_exact(6) { + for chunk in data.as_chunks::<6>().0 { let mut mx = 0.0_f64; let mut my = 0.0_f64; diff --git a/lib/src/modules/utils/asn1.rs b/lib/src/modules/utils/asn1.rs index 4218deb2..e3ca5a8b 100644 --- a/lib/src/modules/utils/asn1.rs +++ b/lib/src/modules/utils/asn1.rs @@ -674,7 +674,9 @@ fn string_from_utf16be(v: &[u8]) -> Option { } let codepoints = v - .chunks_exact(2) + .as_chunks::<2>() + .0 + .iter() .map(|chunk| u16::from_be_bytes([chunk[0], chunk[1]])); let x: String = diff --git a/lib/src/re/fast/compiler.rs b/lib/src/re/fast/compiler.rs index f6afc7fd..3471e694 100644 --- a/lib/src/re/fast/compiler.rs +++ b/lib/src/re/fast/compiler.rs @@ -60,7 +60,7 @@ impl Compiler { } else { best_range_in_bytes(bytes) }; - atoms.push((Some(bytes), mask, range, quality)); + atoms.push((bytes, mask, range, quality)); }; // Iterate the pieces, looking for the one that contains the best @@ -100,14 +100,15 @@ impl Compiler { } } } - PatternPiece::JumpExact(..) | PatternPiece::Jump(..) => { - piece_atoms.push((None, None, 0..0, i32::MIN)) - } + PatternPiece::JumpExact(..) | PatternPiece::Jump(..) => {} }; // Find the quality of the worst piece. - let quality = - *piece_atoms.iter().map(|(_, _, _, q)| q).min().unwrap(); + let quality = *piece_atoms + .iter() + .map(|(_, _, _, q)| q) + .min() + .unwrap_or(&i32::MIN); // If the quality of the worst piece is higher than the current // best quality, replace the best atoms and best quality. @@ -164,11 +165,11 @@ impl Compiler { None => return Err(Error::FastIncompatible), }; - for (best_bytes, best_mask, best_range, _) in best_atoms { - match (best_bytes, best_mask) { - (Some(bytes), Some(mask)) => { + for (pattern, mask, range, _) in best_atoms { + match (pattern, mask) { + (pattern, Some(mask)) => { let masked_atom = - MaskedAtom::from_slice_range(bytes, mask, best_range) + MaskedAtom::from_slice_range(pattern, mask, range) .ok_or(Error::FastIncompatible)?; for atom in masked_atom.mask_combinations() { atoms.push(RegexpAtom { @@ -178,14 +179,12 @@ impl Compiler { }) } } - (Some(bytes), None) => atoms.push(RegexpAtom { - atom: Atom::from_slice_range(bytes, best_range) + (pattern, None) => atoms.push(RegexpAtom { + atom: Atom::from_slice_range(pattern, range) .ok_or(Error::FastIncompatible)?, fwd_code: Some(FwdCodeLoc::from(fwd_code_start)), bck_code: bck_code_start.map(BckCodeLoc::from), }), - (None, None) => {} - _ => unreachable!(), } } diff --git a/lib/src/re/fast/fastvm.rs b/lib/src/re/fast/fastvm.rs index 35ef96a0..ee8e6a21 100644 --- a/lib/src/re/fast/fastvm.rs +++ b/lib/src/re/fast/fastvm.rs @@ -405,7 +405,8 @@ impl FastVM<'_> { // Iterate the input in chunks of two bytes, where the first one // must match a byte in the literal, and the second one is the // interleaved zero. - for (chunk, byte) in izip!(input.chunks_exact(2), literal.iter()) { + let (chunks, _) = input.as_chunks::<2>(); + for (chunk, byte) in izip!(chunks, literal.iter()) { if chunk[0] != *byte || chunk[1] != 0 { return false; } diff --git a/lib/src/re/fast/mod.rs b/lib/src/re/fast/mod.rs index 24aff997..bf0ed8aa 100644 --- a/lib/src/re/fast/mod.rs +++ b/lib/src/re/fast/mod.rs @@ -4,8 +4,8 @@ to [PikeVM], accompanied by a compiler designed to generate code for it. [FastVM] closely resembles [PikeVM], albeit with certain limitations. It exclusively supports regular expressions adhering to the following rules: -- No repetitions are allowed, except when the repeated pattern is any byte. So, - `.*` and `.{1,3}` are permitted, but `a*` and `a{1,3}` are not. +- No repetitions are allowed, except when the repeated pattern is any byte. + So, `.*` and `.{1,3}` are permitted, but `a*` and `a{1,3}` are not. - Character classes are disallowed unless they can be represented as masked bytes. For example, `[a-z]` is not supported, but `[Aa]` is, as it can be diff --git a/lib/src/re/hir.rs b/lib/src/re/hir.rs index a1ab1339..228141a4 100644 --- a/lib/src/re/hir.rs +++ b/lib/src/re/hir.rs @@ -241,17 +241,6 @@ impl Hir { let mut mask = Vec::new(); let mut stack = vec![&self.inner]; - fn is_any_byte(hir: ®ex_syntax::hir::Hir) -> bool { - match hir.kind() { - HirKind::Class(Class::Bytes(class)) => { - class.ranges().len() == 1 - && class.ranges()[0].start() == 0 - && class.ranges()[0].end() == 255 - } - _ => false, - } - } - while let Some(hir) = stack.pop() { match hir.kind() { HirKind::Literal(lit) => { @@ -260,7 +249,7 @@ impl Hir { mask.extend(repeat_n(0xff, bytes.len())); } HirKind::Repetition(Repetition { min, max, sub, .. }) => { - if *max == Some(*min) && is_any_byte(sub.as_ref()) { + if *max == Some(*min) && any_byte(sub.kind()) { let count = *min as usize; pattern.extend(repeat_n(0x00, count)); mask.extend(repeat_n(0x00, count)); @@ -275,12 +264,9 @@ impl Hir { } } HirKind::Class(Class::Bytes(class)) => { - if let Some(hex_byte) = class_to_masked_byte(class) { - pattern.push(hex_byte.value); - mask.push(hex_byte.mask); - } else { - return None; - } + let hex_byte = class_to_masked_byte(class)?; + pattern.push(hex_byte.value); + mask.push(hex_byte.mask); } HirKind::Capture(group) => { stack.push(&group.sub); diff --git a/lib/src/re/thompson/compiler.rs b/lib/src/re/thompson/compiler.rs index 266cf981..b8957bb7 100644 --- a/lib/src/re/thompson/compiler.rs +++ b/lib/src/re/thompson/compiler.rs @@ -1999,7 +1999,7 @@ fn optimize_seq(mut seq: Seq) -> Option { return Some(seq); } - for (_, bitmap) in map.iter_mut() { + for bitmap in map.values_mut() { bitmap.set(0, true); } diff --git a/lib/src/re/thompson/pikevm.rs b/lib/src/re/thompson/pikevm.rs index fec89e61..1c251e62 100644 --- a/lib/src/re/thompson/pikevm.rs +++ b/lib/src/re/thompson/pikevm.rs @@ -409,7 +409,7 @@ pub(crate) fn epsilon_closure( | Instr::ClassBitmap(_) | Instr::ClassRanges(_) | Instr::Match => { - if closure.iter().find(|(i, _)| *i == ip).is_none() { + if !closure.iter().any(|(i, _)| *i == ip) { closure.push((ip, rep_count)); } } diff --git a/lib/src/scanner/matches.rs b/lib/src/scanner/matches.rs index 3d5f3fd9..53c5a4e7 100644 --- a/lib/src/scanner/matches.rs +++ b/lib/src/scanner/matches.rs @@ -274,7 +274,7 @@ impl PatternMatches { self.matches.clear(); self.capacity = 0; } else { - for (_, matches) in self.matches.iter_mut() { + for matches in self.matches.values_mut() { matches.clear(); } } diff --git a/lib/src/scanner/mod.rs b/lib/src/scanner/mod.rs index 9de2fb89..f8cde64a 100644 --- a/lib/src/scanner/mod.rs +++ b/lib/src/scanner/mod.rs @@ -654,17 +654,14 @@ impl<'r> Scanner<'r> { ctx.user_provided_module_outputs.remove(root_struct_name) { module_output = Some(output); - } else { - if let Some(main_res) = module.main_fn(&mut mod_ctx, data) { - module_output = Some(main_res.map_err(|err| { - ScanError::ModuleError { - module: module_name.to_string(), - err, - } + } else if let Some(main_res) = module.main_fn(&mut mod_ctx, data) { + module_output = + Some(main_res.map_err(|err| ScanError::ModuleError { + module: module_name.to_string(), + err, })?); - } else { - module_output = None; - } + } else { + module_output = None; } if let Some(module_output) = &module_output { diff --git a/lib/src/tests/mod.rs b/lib/src/tests/mod.rs index 94c4f3db..3360ff26 100644 --- a/lib/src/tests/mod.rs +++ b/lib/src/tests/mod.rs @@ -1268,32 +1268,60 @@ fn hex_patterns() { // https://github.com/VirusTotal/yara-x/issues/383 pattern_match!( r#"{ - ( 2D A? FF FF FF | - 81 E8 ( A? | B? | C? | D? ) FF | - 81 E9 ( A? | B? | C? | D? ) FF | - 81 EA ( A? | B? | C? | D? ) FF | - 81 EB ( A? | B? | C? | D? ) FF | - 81 ED ( A? | B? | C? | D? ) FF | - 81 EE ( A? | B? | C? | D? ) FF | - 81 EF ( A? | B? | C? | D? ) FF | - 6A 2? 03 ?? 24 ( 58 | 59 | 5A | 5B | 5D | 5E | 5F ) | - 6A 3? 03 ?? 24 ( 58 | 59 | 5A | 5B | 5D | 5E | 5F ) | - 6A 4? 03 ?? 24 ( 58 | 59 | 5A | 5B | 5D | 5E | 5F ) | - 6A 5? 03 ?? 24 ( 58 | 59 | 5A | 5B | 5D | 5E | 5F ) | - 83 C0 ( 2? | 3? | 4? | 5? ) | - 83 C1 ( 2? | 3? | 4? | 5? ) | - 83 C2 ( 2? | 3? | 4? | 5? ) | - 83 C3 ( 2? | 3? | 4? | 5? ) | - 8D 40 ( 2? | 3? | 4? | 5? ) | - 8D 49 ( 2? | 3? | 4? | 5? ) | - 8D 6D ( 2? | 3? | 4? | 5? ) | - 8D 76 ( 2? | 3? | 4? | 5? ) | - 8D 7F ( 2? | 3? | 4? | 5? ) - ) - }"#, + ( 2D A? FF FF FF | + 81 E8 ( A? | B? | C? | D? ) FF | + 81 E9 ( A? | B? | C? | D? ) FF | + 81 EA ( A? | B? | C? | D? ) FF | + 81 EB ( A? | B? | C? | D? ) FF | + 81 ED ( A? | B? | C? | D? ) FF | + 81 EE ( A? | B? | C? | D? ) FF | + 81 EF ( A? | B? | C? | D? ) FF | + 6A 2? 03 ?? 24 ( 58 | 59 | 5A | 5B | 5D | 5E | 5F ) | + 6A 3? 03 ?? 24 ( 58 | 59 | 5A | 5B | 5D | 5E | 5F ) | + 6A 4? 03 ?? 24 ( 58 | 59 | 5A | 5B | 5D | 5E | 5F ) | + 6A 5? 03 ?? 24 ( 58 | 59 | 5A | 5B | 5D | 5E | 5F ) | + 83 C0 ( 2? | 3? | 4? | 5? ) | + 83 C1 ( 2? | 3? | 4? | 5? ) | + 83 C2 ( 2? | 3? | 4? | 5? ) | + 83 C3 ( 2? | 3? | 4? | 5? ) | + 8D 40 ( 2? | 3? | 4? | 5? ) | + 8D 49 ( 2? | 3? | 4? | 5? ) | + 8D 6D ( 2? | 3? | 4? | 5? ) | + 8D 76 ( 2? | 3? | 4? | 5? ) | + 8D 7F ( 2? | 3? | 4? | 5? ) + ) + }"#, &[0x2D, 0xA0, 0xFF, 0xFF, 0xFF], &[0x2D, 0xA0, 0xFF, 0xFF, 0xFF] ); + + pattern_match!( + r#"{ 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 ?? }"#, + &[ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, + ], + &[ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, + ] + ); + + pattern_false!( + r#"{ 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 ?? }"#, + &[ + 0x01, 0x02, 0x03, 0xff, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13 + ] + ); + + pattern_false!( + r#"{ 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 ?? }"#, + &[ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, + ] + ); } #[test] diff --git a/parser/src/ast/ascii_tree.rs b/parser/src/ast/ascii_tree.rs index bf7b4d10..5a494be8 100644 --- a/parser/src/ast/ascii_tree.rs +++ b/parser/src/ast/ascii_tree.rs @@ -379,10 +379,7 @@ fn build_tree_for_expr(expr: &Expr, children: Vec) -> Tree { .iter() .flat_map(|d| { vec![ - Leaf(vec![format!( - "{}", - d.identifier.name - )]), + Leaf(vec![d.identifier.name.to_string()]), children_iter.next().unwrap(), ] }) diff --git a/parser/src/parser/mod.rs b/parser/src/parser/mod.rs index 9b80fcc8..f619423d 100644 --- a/parser/src/parser/mod.rs +++ b/parser/src/parser/mod.rs @@ -318,15 +318,13 @@ impl<'src> ParserImpl<'src> { // } // let token_pos = loop { - match self.tokens.peek_token(i) { - Some(token) => { - if token.is_trivia() { - i += 1; - } else { - break i; - } + { + let token = self.tokens.peek_token(i)?; + if token.is_trivia() { + i += 1; + } else { + break i; } - None => return None, } }; self.tokens.peek_token(token_pos)