diff --git a/fmt/src/testdata/default_tests/test37.formatted b/fmt/src/testdata/default_tests/test37.formatted new file mode 100644 index 00000000..eeea144d --- /dev/null +++ b/fmt/src/testdata/default_tests/test37.formatted @@ -0,0 +1,7 @@ +rule test { + strings: + $c1 = { s00 01 1a 2b } + + condition: + any of them +} diff --git a/fmt/src/testdata/default_tests/test37.unformatted b/fmt/src/testdata/default_tests/test37.unformatted new file mode 100644 index 00000000..f4906e07 --- /dev/null +++ b/fmt/src/testdata/default_tests/test37.unformatted @@ -0,0 +1,6 @@ +rule test { + strings: + $c1 = { s00 01 1a 2b } + condition: + any of them +} \ No newline at end of file diff --git a/fmt/src/tokens/mod.rs b/fmt/src/tokens/mod.rs index ec670e83..b80c321e 100644 --- a/fmt/src/tokens/mod.rs +++ b/fmt/src/tokens/mod.rs @@ -497,6 +497,30 @@ where } loop { match self.events.next()? { + // When an ERROR node is encountered, we want to preserve its + // original source text exactly as it is in the input, without + // applying any formatting to its contents. To do this, we + // extract the original bytes from the span of the ERROR node, + // skip all parsing events inside this node, and return a + // single literal token containing the original text. + Event::Begin { kind: SyntaxKind::ERROR, span } => { + let error_bytes = &self.source[span.range()]; + let mut depth = 1; + while depth > 0 { + match self.events.next()? { + Event::Begin { + kind: SyntaxKind::ERROR, .. + } => { + depth += 1; + } + Event::End { kind: SyntaxKind::ERROR, .. } => { + depth -= 1; + } + _ => {} + } + } + return Some(Token::Literal(error_bytes)); + } Event::Begin { kind, .. } => return Some(Token::Begin(kind)), Event::End { kind, .. } => return Some(Token::End(kind)), Event::Token { kind, span } => { diff --git a/lib/src/compiler/ir/tests/mod.rs b/lib/src/compiler/ir/tests/mod.rs index 958d54c4..91d92c4b 100644 --- a/lib/src/compiler/ir/tests/mod.rs +++ b/lib/src/compiler/ir/tests/mod.rs @@ -134,3 +134,328 @@ fn ir() { } }); } + +#[test] +fn replace_child() { + use super::{ + FieldAccess, ForIn, ForOf, ForVars, FuncCall, Iterable, Lookup, + MatchAnchor, OfExprTuple, OfPatternSet, Quantifier, Range, With, + }; + use crate::compiler::context::Var; + use crate::compiler::{ExprId, PatternIdx, RegexSetId}; + use crate::types::{FuncSignature, MangledFnName, Type}; + + let c1 = ExprId::from(1); + let c2 = ExprId::from(2); + let repl = ExprId::from(3); + let dummy_var = Var::new(0, Type::Integer, 0); + let make_for_vars = || ForVars { + n: dummy_var, + i: dummy_var, + max_count: dummy_var, + count: dummy_var, + item: dummy_var, + }; + + // Expr::Const, Expr::Filesize, Expr::Symbol do nothing and do not panic + let mut expr = Expr::Filesize; + expr.replace_child(c1, repl); + assert!(matches!(expr, Expr::Filesize)); + + // Expr::Not, Expr::Minus, Expr::Defined, Expr::BitwiseNot + let mut expr = Expr::Not { operand: c1 }; + expr.replace_child(c1, repl); + assert!(matches!(expr, Expr::Not { operand } if operand == repl)); + + let mut expr = Expr::Minus { operand: c1, is_float: false }; + expr.replace_child(c1, repl); + assert!(matches!(expr, Expr::Minus { operand, .. } if operand == repl)); + + let mut expr = Expr::Defined { operand: c1 }; + expr.replace_child(c1, repl); + assert!(matches!(expr, Expr::Defined { operand } if operand == repl)); + + let mut expr = Expr::BitwiseNot { operand: c1 }; + expr.replace_child(c1, repl); + assert!(matches!(expr, Expr::BitwiseNot { operand } if operand == repl)); + + // Expr::And, Expr::Or, Expr::Add, Expr::Sub, Expr::Mul, Expr::Div, Expr::Mod + let mut expr = Expr::And { operands: vec![c1, c2] }; + expr.replace_child(c1, repl); + assert!( + matches!(expr, Expr::And { operands } if operands == vec![repl, c2]) + ); + + let mut expr = Expr::Or { operands: vec![c1, c2] }; + expr.replace_child(c1, repl); + assert!( + matches!(expr, Expr::Or { operands } if operands == vec![repl, c2]) + ); + + let mut expr = Expr::Add { operands: vec![c1, c2], is_float: false }; + expr.replace_child(c1, repl); + assert!( + matches!(expr, Expr::Add { operands, .. } if operands == vec![repl, c2]) + ); + + let mut expr = Expr::Sub { operands: vec![c1, c2], is_float: false }; + expr.replace_child(c1, repl); + assert!( + matches!(expr, Expr::Sub { operands, .. } if operands == vec![repl, c2]) + ); + + let mut expr = Expr::Mul { operands: vec![c1, c2], is_float: false }; + expr.replace_child(c1, repl); + assert!( + matches!(expr, Expr::Mul { operands, .. } if operands == vec![repl, c2]) + ); + + let mut expr = Expr::Div { operands: vec![c1, c2], is_float: false }; + expr.replace_child(c1, repl); + assert!( + matches!(expr, Expr::Div { operands, .. } if operands == vec![repl, c2]) + ); + + let mut expr = Expr::Mod { operands: vec![c1, c2] }; + expr.replace_child(c1, repl); + assert!( + matches!(expr, Expr::Mod { operands, .. } if operands == vec![repl, c2]) + ); + + // Binary Operators + let mut expr = Expr::Eq { lhs: c1, rhs: c2 }; + expr.replace_child(c1, repl); + assert!(matches!(expr, Expr::Eq { lhs, rhs } if lhs == repl && rhs == c2)); + + let mut expr = Expr::Eq { lhs: c1, rhs: c2 }; + expr.replace_child(c2, repl); + assert!(matches!(expr, Expr::Eq { lhs, rhs } if lhs == c1 && rhs == repl)); + + // MatchesMany + let mut expr = + Expr::MatchesMany { lhs: c1, regex_set: RegexSetId::from(0) }; + expr.replace_child(c1, repl); + assert!(matches!(expr, Expr::MatchesMany { lhs, .. } if lhs == repl)); + + // PatternMatch (Anchor At/In) + let mut expr = Expr::PatternMatch { + pattern: PatternIdx::from(0), + anchor: MatchAnchor::At(c1), + }; + expr.replace_child(c1, repl); + assert!( + matches!(expr, Expr::PatternMatch { anchor: MatchAnchor::At(expr), .. } if expr == repl) + ); + + let mut expr = Expr::PatternMatch { + pattern: PatternIdx::from(0), + anchor: MatchAnchor::In(Range { lower_bound: c1, upper_bound: c2 }), + }; + expr.replace_child(c1, repl); + assert!( + matches!(expr, Expr::PatternMatch { anchor: MatchAnchor::In(Range { lower_bound, upper_bound }), .. } if lower_bound == repl && upper_bound == c2) + ); + + let mut expr = Expr::PatternMatch { + pattern: PatternIdx::from(0), + anchor: MatchAnchor::In(Range { lower_bound: c1, upper_bound: c2 }), + }; + expr.replace_child(c2, repl); + assert!( + matches!(expr, Expr::PatternMatch { anchor: MatchAnchor::In(Range { lower_bound, upper_bound }), .. } if lower_bound == c1 && upper_bound == repl) + ); + + // PatternCount + let mut expr = Expr::PatternCount { + pattern: PatternIdx::from(0), + range: Some(Range { lower_bound: c1, upper_bound: c2 }), + }; + expr.replace_child(c1, repl); + assert!( + matches!(expr, Expr::PatternCount { range: Some(Range { lower_bound, upper_bound }), .. } if lower_bound == repl && upper_bound == c2) + ); + + let mut expr = Expr::PatternCount { + pattern: PatternIdx::from(0), + range: Some(Range { lower_bound: c1, upper_bound: c2 }), + }; + expr.replace_child(c2, repl); + assert!( + matches!(expr, Expr::PatternCount { range: Some(Range { lower_bound, upper_bound }), .. } if lower_bound == c1 && upper_bound == repl) + ); + + // PatternOffset, PatternLength + let mut expr = + Expr::PatternOffset { pattern: PatternIdx::from(0), index: Some(c1) }; + expr.replace_child(c1, repl); + assert!( + matches!(expr, Expr::PatternOffset { index: Some(idx), .. } if idx == repl) + ); + + let mut expr = + Expr::PatternLength { pattern: PatternIdx::from(0), index: Some(c1) }; + expr.replace_child(c1, repl); + assert!( + matches!(expr, Expr::PatternLength { index: Some(idx), .. } if idx == repl) + ); + + // With + let mut expr = Expr::With(Box::new(With { + type_value: TypeValue::Unknown, + declarations: vec![(dummy_var, c1)], + body: c2, + })); + expr.replace_child(c1, repl); + assert!( + matches!(&expr, Expr::With(with) if with.declarations[0].1 == repl && with.body == c2) + ); + + let mut expr = Expr::With(Box::new(With { + type_value: TypeValue::Unknown, + declarations: vec![(dummy_var, c1)], + body: c2, + })); + expr.replace_child(c2, repl); + assert!( + matches!(&expr, Expr::With(with) if with.declarations[0].1 == c1 && with.body == repl) + ); + + // FieldAccess + let mut expr = Expr::FieldAccess(Box::new(FieldAccess { + operands: vec![c1, c2], + type_value: TypeValue::Unknown, + })); + expr.replace_child(c1, repl); + assert!( + matches!(&expr, Expr::FieldAccess(fa) if fa.operands == vec![repl, c2]) + ); + + // FuncCall + let mut expr = Expr::FuncCall(Box::new(FuncCall { + object: Some(c1), + args: vec![c2], + signature: std::rc::Rc::new(FuncSignature { + mangled_name: MangledFnName::from(""), + args: vec![], + result: TypeValue::Unknown, + doc: None, + }), + })); + expr.replace_child(c1, repl); + assert!( + matches!(&expr, Expr::FuncCall(fc) if fc.object == Some(repl) && fc.args == vec![c2]) + ); + + let mut expr = Expr::FuncCall(Box::new(FuncCall { + object: Some(c1), + args: vec![c2], + signature: std::rc::Rc::new(FuncSignature { + mangled_name: MangledFnName::from(""), + args: vec![], + result: TypeValue::Unknown, + doc: None, + }), + })); + expr.replace_child(c2, repl); + assert!( + matches!(&expr, Expr::FuncCall(fc) if fc.object == Some(c1) && fc.args == vec![repl]) + ); + + // OfExprTuple + let mut expr = Expr::OfExprTuple(Box::new(OfExprTuple { + quantifier: Quantifier::Percentage(c1), + items: vec![c2], + for_vars: make_for_vars(), + anchor: MatchAnchor::At(c1), + })); + expr.replace_child(c1, repl); + assert!( + matches!(&expr, Expr::OfExprTuple(of) if matches!(of.anchor, MatchAnchor::At(a) if a == repl)) + ); + + let mut expr = Expr::OfExprTuple(Box::new(OfExprTuple { + quantifier: Quantifier::Percentage(c1), + items: vec![c2], + for_vars: make_for_vars(), + anchor: MatchAnchor::At(c1), + })); + expr.replace_child(c2, repl); + assert!(matches!(&expr, Expr::OfExprTuple(of) if of.items == vec![repl])); + + // OfPatternSet + let mut expr = Expr::OfPatternSet(Box::new(OfPatternSet { + quantifier: Quantifier::Percentage(c1), + items: vec![], + for_vars: make_for_vars(), + anchor: MatchAnchor::At(c2), + })); + expr.replace_child(c2, repl); + assert!( + matches!(&expr, Expr::OfPatternSet(of) if matches!(of.anchor, MatchAnchor::At(a) if a == repl)) + ); + + // ForOf + let mut expr = Expr::ForOf(Box::new(ForOf { + quantifier: Quantifier::Percentage(c1), + for_vars: make_for_vars(), + pattern_set: vec![], + body: c2, + })); + expr.replace_child(c1, repl); + assert!( + matches!(&expr, Expr::ForOf(fo) if matches!(fo.quantifier, Quantifier::Percentage(q) if q == repl)) + ); + + let mut expr = Expr::ForOf(Box::new(ForOf { + quantifier: Quantifier::Percentage(c1), + for_vars: make_for_vars(), + pattern_set: vec![], + body: c2, + })); + expr.replace_child(c2, repl); + assert!(matches!(&expr, Expr::ForOf(fo) if fo.body == repl)); + + // ForIn (iterable Range/ExprTuple/Expr) + let mut expr = Expr::ForIn(Box::new(ForIn { + quantifier: Quantifier::Percentage(c1), + variables: vec![], + for_vars: make_for_vars(), + iterable: Iterable::Range(Range { lower_bound: c1, upper_bound: c2 }), + body: c2, + })); + expr.replace_child(c1, repl); + assert!( + matches!(&expr, Expr::ForIn(fi) if matches!(fi.quantifier, Quantifier::Percentage(q) if q == repl)) + ); + + let mut expr = Expr::ForIn(Box::new(ForIn { + quantifier: Quantifier::Percentage(c1), + variables: vec![], + for_vars: make_for_vars(), + iterable: Iterable::Range(Range { lower_bound: c1, upper_bound: c2 }), + body: c2, + })); + expr.replace_child(c2, repl); + assert!(matches!(&expr, Expr::ForIn(fi) if fi.body == repl)); + + // Lookup + let mut expr = Expr::Lookup(Box::new(Lookup { + type_value: TypeValue::Unknown, + primary: c1, + index: c2, + })); + expr.replace_child(c1, repl); + assert!( + matches!(&expr, Expr::Lookup(l) if l.primary == repl && l.index == c2) + ); + + let mut expr = Expr::Lookup(Box::new(Lookup { + type_value: TypeValue::Unknown, + primary: c1, + index: c2, + })); + expr.replace_child(c2, repl); + assert!( + matches!(&expr, Expr::Lookup(l) if l.primary == c1 && l.index == repl) + ); +} diff --git a/lib/src/compiler/ir/tests/testdata/16.cse.ir b/lib/src/compiler/ir/tests/testdata/16.cse.ir new file mode 100644 index 00000000..636c23d4 --- /dev/null +++ b/lib/src/compiler/ir/tests/testdata/16.cse.ir @@ -0,0 +1,137 @@ +RULE test_1 + 10: FOR_IN -- hash: 0xd210e99877f4fa84 + n: Var { frame_id: 1, ty: integer, index: 0 } + i: Var { frame_id: 1, ty: integer, index: 1 } + max_count: Var { frame_id: 1, ty: integer, index: 2 } + count: Var { frame_id: 1, ty: integer, index: 3 } + item: Var { frame_id: 1, ty: unknown, index: 4 } + 0: CONST integer(1) + 1: CONST integer(2) + 9: FOR_IN -- hash: 0x39ccd6c06865e36d + n: Var { frame_id: 2, ty: integer, index: 7 } + i: Var { frame_id: 2, ty: integer, index: 8 } + max_count: Var { frame_id: 2, ty: integer, index: 9 } + count: Var { frame_id: 2, ty: integer, index: 10 } + item: Var { frame_id: 2, ty: unknown, index: 11 } + 2: CONST integer(1) + 3: CONST integer(2) + 8: EQ -- hash: 0x209cbc2ded9b10e6 + 4: SYMBOL Var { var: Var { frame_id: 2, ty: integer, index: 12 }, type_value: integer(unknown) } + 7: ADD -- hash: 0x8c4973b1fbd58946 + 5: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 5 }, type_value: integer(unknown) } + 6: CONST integer(1) + +RULE test_2 + 8: FOR_OF -- hash: 0xe4a08e7456292b47 + n: Var { frame_id: 1, ty: integer, index: 0 } + i: Var { frame_id: 1, ty: integer, index: 1 } + max_count: Var { frame_id: 1, ty: integer, index: 2 } + count: Var { frame_id: 1, ty: integer, index: 3 } + item: Var { frame_id: 1, ty: integer, index: 4 } + 7: FOR_IN -- hash: 0x6ed2fae041710947 + n: Var { frame_id: 2, ty: integer, index: 5 } + i: Var { frame_id: 2, ty: integer, index: 6 } + max_count: Var { frame_id: 2, ty: integer, index: 7 } + count: Var { frame_id: 2, ty: integer, index: 8 } + item: Var { frame_id: 2, ty: unknown, index: 9 } + 0: CONST integer(1) + 1: CONST integer(2) + 6: EQ -- hash: 0x773b9bba17dfad2a + 2: SYMBOL Var { var: Var { frame_id: 2, ty: integer, index: 10 }, type_value: integer(unknown) } + 5: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 4: CONST integer(1) + +RULE test_3 + 9: AND -- hash: 0x76ad43078646820d + 8: FOR_IN -- hash: 0x33ea248cbadf6b9 + n: Var { frame_id: 1, ty: integer, index: 0 } + i: Var { frame_id: 1, ty: integer, index: 1 } + max_count: Var { frame_id: 1, ty: integer, index: 2 } + count: Var { frame_id: 1, ty: integer, index: 3 } + item: Var { frame_id: 1, ty: unknown, index: 4 } + 1: CONST integer(1) + 2: CONST integer(2) + 7: EQ -- hash: 0xba743229e1c9a9c + 3: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 5 }, type_value: integer(unknown) } + 6: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 5: CONST integer(1) + +RULE test_4 + 9: OR -- hash: 0x47d223471b2405d4 + 8: FOR_IN -- hash: 0x33ea248cbadf6b9 + n: Var { frame_id: 1, ty: integer, index: 0 } + i: Var { frame_id: 1, ty: integer, index: 1 } + max_count: Var { frame_id: 1, ty: integer, index: 2 } + count: Var { frame_id: 1, ty: integer, index: 3 } + item: Var { frame_id: 1, ty: unknown, index: 4 } + 1: CONST integer(1) + 2: CONST integer(2) + 7: EQ -- hash: 0xba743229e1c9a9c + 3: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 5 }, type_value: integer(unknown) } + 6: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 5: CONST integer(1) + +RULE test_5 + 8: NOT -- hash: 0xa58862c7f168fe46 + 7: FOR_IN -- hash: 0x33ea248cbadf6b9 + n: Var { frame_id: 1, ty: integer, index: 0 } + i: Var { frame_id: 1, ty: integer, index: 1 } + max_count: Var { frame_id: 1, ty: integer, index: 2 } + count: Var { frame_id: 1, ty: integer, index: 3 } + item: Var { frame_id: 1, ty: unknown, index: 4 } + 0: CONST integer(1) + 1: CONST integer(2) + 6: EQ -- hash: 0xba743229e1c9a9c + 2: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 5 }, type_value: integer(unknown) } + 5: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 4: CONST integer(1) + +RULE test_6 + 10: OF -- hash: 0x24cc8e76ae19e17f + 0: CONST integer(1) + 9: CONST boolean(true) + 8: FOR_IN -- hash: 0x6ed2fae041710947 + n: Var { frame_id: 2, ty: integer, index: 5 } + i: Var { frame_id: 2, ty: integer, index: 6 } + max_count: Var { frame_id: 2, ty: integer, index: 7 } + count: Var { frame_id: 2, ty: integer, index: 8 } + item: Var { frame_id: 2, ty: unknown, index: 9 } + 1: CONST integer(1) + 2: CONST integer(2) + 7: EQ -- hash: 0x773b9bba17dfad2a + 3: SYMBOL Var { var: Var { frame_id: 2, ty: integer, index: 10 }, type_value: integer(unknown) } + 6: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 5: CONST integer(1) + +RULE test_7 + 11: EQ -- hash: 0x3dae225ca107ebac + 9: FN_CALL math.to_number@b:b@i -- hash: 0xdcca860b2fe0bb09 + 8: FOR_IN -- hash: 0x33ea248cbadf6b9 + n: Var { frame_id: 1, ty: integer, index: 0 } + i: Var { frame_id: 1, ty: integer, index: 1 } + max_count: Var { frame_id: 1, ty: integer, index: 2 } + count: Var { frame_id: 1, ty: integer, index: 3 } + item: Var { frame_id: 1, ty: unknown, index: 4 } + 1: CONST integer(1) + 2: CONST integer(2) + 7: EQ -- hash: 0xba743229e1c9a9c + 3: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 5 }, type_value: integer(unknown) } + 6: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 5: CONST integer(1) + 10: CONST integer(1) + +RULE test_8 + 8: DEFINED -- hash: 0x27daaa3b4b5e6f93 + 7: FOR_IN -- hash: 0x33ea248cbadf6b9 + n: Var { frame_id: 1, ty: integer, index: 0 } + i: Var { frame_id: 1, ty: integer, index: 1 } + max_count: Var { frame_id: 1, ty: integer, index: 2 } + count: Var { frame_id: 1, ty: integer, index: 3 } + item: Var { frame_id: 1, ty: unknown, index: 4 } + 0: CONST integer(1) + 1: CONST integer(2) + 6: EQ -- hash: 0xba743229e1c9a9c + 2: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 5 }, type_value: integer(unknown) } + 5: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 4: CONST integer(1) + diff --git a/lib/src/compiler/ir/tests/testdata/16.hoisting.ir b/lib/src/compiler/ir/tests/testdata/16.hoisting.ir new file mode 100644 index 00000000..5bf3da52 --- /dev/null +++ b/lib/src/compiler/ir/tests/testdata/16.hoisting.ir @@ -0,0 +1,153 @@ +RULE test_1 + 10: FOR_IN -- hash: 0xf0de15212626240d + n: Var { frame_id: 1, ty: integer, index: 0 } + i: Var { frame_id: 1, ty: integer, index: 1 } + max_count: Var { frame_id: 1, ty: integer, index: 2 } + count: Var { frame_id: 1, ty: integer, index: 3 } + item: Var { frame_id: 1, ty: unknown, index: 4 } + 0: CONST integer(1) + 1: CONST integer(2) + 12: WITH -- hash: 0x92d04716bb2575d0 + 11: ADD -- hash: 0x8c4973b1fbd58946 + 5: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 5 }, type_value: integer(unknown) } + 6: CONST integer(1) + 9: FOR_IN -- hash: 0x841c4369b9d14bf7 + n: Var { frame_id: 2, ty: integer, index: 8 } + i: Var { frame_id: 2, ty: integer, index: 9 } + max_count: Var { frame_id: 2, ty: integer, index: 10 } + count: Var { frame_id: 2, ty: integer, index: 11 } + item: Var { frame_id: 2, ty: unknown, index: 12 } + 2: CONST integer(1) + 3: CONST integer(2) + 8: EQ -- hash: 0x2645e81ead56f981 + 4: SYMBOL Var { var: Var { frame_id: 2, ty: integer, index: 13 }, type_value: integer(unknown) } + 7: SYMBOL Var { var: Var { frame_id: 0, ty: integer, index: 7 }, type_value: integer(unknown) } + +RULE test_2 + 8: FOR_OF -- hash: 0xfc7c823bf3d2e50 + n: Var { frame_id: 1, ty: integer, index: 0 } + i: Var { frame_id: 1, ty: integer, index: 1 } + max_count: Var { frame_id: 1, ty: integer, index: 2 } + count: Var { frame_id: 1, ty: integer, index: 3 } + item: Var { frame_id: 1, ty: integer, index: 4 } + 10: WITH -- hash: 0x292eb9310b1059ca + 9: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 4: CONST integer(1) + 7: FOR_IN -- hash: 0x88da6dcf4a806eda + n: Var { frame_id: 2, ty: integer, index: 6 } + i: Var { frame_id: 2, ty: integer, index: 7 } + max_count: Var { frame_id: 2, ty: integer, index: 8 } + count: Var { frame_id: 2, ty: integer, index: 9 } + item: Var { frame_id: 2, ty: unknown, index: 10 } + 0: CONST integer(1) + 1: CONST integer(2) + 6: EQ -- hash: 0x2b0412843e061c64 + 2: SYMBOL Var { var: Var { frame_id: 2, ty: integer, index: 11 }, type_value: integer(unknown) } + 5: SYMBOL Var { var: Var { frame_id: 0, ty: integer, index: 5 }, type_value: integer(unknown) } + +RULE test_3 + 9: AND -- hash: 0x368be42e0979068b + 11: WITH -- hash: 0x87e6897f6e291aed + 10: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 5: CONST integer(1) + 8: FOR_IN -- hash: 0xe7923e1dad992ffd + n: Var { frame_id: 1, ty: integer, index: 1 } + i: Var { frame_id: 1, ty: integer, index: 2 } + max_count: Var { frame_id: 1, ty: integer, index: 3 } + count: Var { frame_id: 1, ty: integer, index: 4 } + item: Var { frame_id: 1, ty: unknown, index: 5 } + 1: CONST integer(1) + 2: CONST integer(2) + 7: EQ -- hash: 0x89bbe2d2a11edd87 + 3: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 6 }, type_value: integer(unknown) } + 6: SYMBOL Var { var: Var { frame_id: 0, ty: integer, index: 0 }, type_value: integer(unknown) } + +RULE test_4 + 9: OR -- hash: 0x70c302683e93aa6a + 11: WITH -- hash: 0x87e6897f6e291aed + 10: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 5: CONST integer(1) + 8: FOR_IN -- hash: 0xe7923e1dad992ffd + n: Var { frame_id: 1, ty: integer, index: 1 } + i: Var { frame_id: 1, ty: integer, index: 2 } + max_count: Var { frame_id: 1, ty: integer, index: 3 } + count: Var { frame_id: 1, ty: integer, index: 4 } + item: Var { frame_id: 1, ty: unknown, index: 5 } + 1: CONST integer(1) + 2: CONST integer(2) + 7: EQ -- hash: 0x89bbe2d2a11edd87 + 3: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 6 }, type_value: integer(unknown) } + 6: SYMBOL Var { var: Var { frame_id: 0, ty: integer, index: 0 }, type_value: integer(unknown) } + +RULE test_5 + 8: NOT -- hash: 0xfc54c5f3d45e62ab + 10: WITH -- hash: 0x87e6897f6e291aed + 9: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 4: CONST integer(1) + 7: FOR_IN -- hash: 0xe7923e1dad992ffd + n: Var { frame_id: 1, ty: integer, index: 1 } + i: Var { frame_id: 1, ty: integer, index: 2 } + max_count: Var { frame_id: 1, ty: integer, index: 3 } + count: Var { frame_id: 1, ty: integer, index: 4 } + item: Var { frame_id: 1, ty: unknown, index: 5 } + 0: CONST integer(1) + 1: CONST integer(2) + 6: EQ -- hash: 0x89bbe2d2a11edd87 + 2: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 6 }, type_value: integer(unknown) } + 5: SYMBOL Var { var: Var { frame_id: 0, ty: integer, index: 0 }, type_value: integer(unknown) } + +RULE test_6 + 10: OF -- hash: 0xb2e61e7cd7d603ab + 0: CONST integer(1) + 9: CONST boolean(true) + 12: WITH -- hash: 0x292eb9310b1059ca + 11: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 5: CONST integer(1) + 8: FOR_IN -- hash: 0x88da6dcf4a806eda + n: Var { frame_id: 2, ty: integer, index: 6 } + i: Var { frame_id: 2, ty: integer, index: 7 } + max_count: Var { frame_id: 2, ty: integer, index: 8 } + count: Var { frame_id: 2, ty: integer, index: 9 } + item: Var { frame_id: 2, ty: unknown, index: 10 } + 1: CONST integer(1) + 2: CONST integer(2) + 7: EQ -- hash: 0x2b0412843e061c64 + 3: SYMBOL Var { var: Var { frame_id: 2, ty: integer, index: 11 }, type_value: integer(unknown) } + 6: SYMBOL Var { var: Var { frame_id: 0, ty: integer, index: 5 }, type_value: integer(unknown) } + +RULE test_7 + 11: EQ -- hash: 0x2f1b62e477b4c5b + 9: FN_CALL math.to_number@b:b@i -- hash: 0x70da20af1e6f5a7 + 13: WITH -- hash: 0x87e6897f6e291aed + 12: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 5: CONST integer(1) + 8: FOR_IN -- hash: 0xe7923e1dad992ffd + n: Var { frame_id: 1, ty: integer, index: 1 } + i: Var { frame_id: 1, ty: integer, index: 2 } + max_count: Var { frame_id: 1, ty: integer, index: 3 } + count: Var { frame_id: 1, ty: integer, index: 4 } + item: Var { frame_id: 1, ty: unknown, index: 5 } + 1: CONST integer(1) + 2: CONST integer(2) + 7: EQ -- hash: 0x89bbe2d2a11edd87 + 3: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 6 }, type_value: integer(unknown) } + 6: SYMBOL Var { var: Var { frame_id: 0, ty: integer, index: 0 }, type_value: integer(unknown) } + 10: CONST integer(1) + +RULE test_8 + 8: DEFINED -- hash: 0xceca50c5ed7d96d4 + 10: WITH -- hash: 0x87e6897f6e291aed + 9: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 4: CONST integer(1) + 7: FOR_IN -- hash: 0xe7923e1dad992ffd + n: Var { frame_id: 1, ty: integer, index: 1 } + i: Var { frame_id: 1, ty: integer, index: 2 } + max_count: Var { frame_id: 1, ty: integer, index: 3 } + count: Var { frame_id: 1, ty: integer, index: 4 } + item: Var { frame_id: 1, ty: unknown, index: 5 } + 0: CONST integer(1) + 1: CONST integer(2) + 6: EQ -- hash: 0x89bbe2d2a11edd87 + 2: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 6 }, type_value: integer(unknown) } + 5: SYMBOL Var { var: Var { frame_id: 0, ty: integer, index: 0 }, type_value: integer(unknown) } + diff --git a/lib/src/compiler/ir/tests/testdata/16.in b/lib/src/compiler/ir/tests/testdata/16.in new file mode 100644 index 00000000..7c6a37a5 --- /dev/null +++ b/lib/src/compiler/ir/tests/testdata/16.in @@ -0,0 +1,51 @@ +import "math" + +rule test_1 { + condition: + for any i in (1..2) : ( + for any j in (1..2) : ( + j == (i + 1) + ) + ) +} + +rule test_2 { + strings: + $a = "abc" + condition: + for any of ($a) : ( + for any i in (1..2) : ( + i == math.abs(1) + ) + ) +} + +rule test_3 { + condition: + true and (for any i in (1..2) : ( i == math.abs(1) )) +} + +rule test_4 { + condition: + false or (for any i in (1..2) : ( i == math.abs(1) )) +} + +rule test_5 { + condition: + not (for any i in (1..2) : ( i == math.abs(1) )) +} + +rule test_6 { + condition: + 1 of ( (for any i in (1..2) : ( i == math.abs(1) )), true ) +} + +rule test_7 { + condition: + math.to_number(for any i in (1..2) : ( i == math.abs(1) )) == 1 +} + +rule test_8 { + condition: + defined (for any i in (1..2) : ( i == math.abs(1) )) +} diff --git a/lib/src/compiler/ir/tests/testdata/16.ir b/lib/src/compiler/ir/tests/testdata/16.ir new file mode 100644 index 00000000..636c23d4 --- /dev/null +++ b/lib/src/compiler/ir/tests/testdata/16.ir @@ -0,0 +1,137 @@ +RULE test_1 + 10: FOR_IN -- hash: 0xd210e99877f4fa84 + n: Var { frame_id: 1, ty: integer, index: 0 } + i: Var { frame_id: 1, ty: integer, index: 1 } + max_count: Var { frame_id: 1, ty: integer, index: 2 } + count: Var { frame_id: 1, ty: integer, index: 3 } + item: Var { frame_id: 1, ty: unknown, index: 4 } + 0: CONST integer(1) + 1: CONST integer(2) + 9: FOR_IN -- hash: 0x39ccd6c06865e36d + n: Var { frame_id: 2, ty: integer, index: 7 } + i: Var { frame_id: 2, ty: integer, index: 8 } + max_count: Var { frame_id: 2, ty: integer, index: 9 } + count: Var { frame_id: 2, ty: integer, index: 10 } + item: Var { frame_id: 2, ty: unknown, index: 11 } + 2: CONST integer(1) + 3: CONST integer(2) + 8: EQ -- hash: 0x209cbc2ded9b10e6 + 4: SYMBOL Var { var: Var { frame_id: 2, ty: integer, index: 12 }, type_value: integer(unknown) } + 7: ADD -- hash: 0x8c4973b1fbd58946 + 5: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 5 }, type_value: integer(unknown) } + 6: CONST integer(1) + +RULE test_2 + 8: FOR_OF -- hash: 0xe4a08e7456292b47 + n: Var { frame_id: 1, ty: integer, index: 0 } + i: Var { frame_id: 1, ty: integer, index: 1 } + max_count: Var { frame_id: 1, ty: integer, index: 2 } + count: Var { frame_id: 1, ty: integer, index: 3 } + item: Var { frame_id: 1, ty: integer, index: 4 } + 7: FOR_IN -- hash: 0x6ed2fae041710947 + n: Var { frame_id: 2, ty: integer, index: 5 } + i: Var { frame_id: 2, ty: integer, index: 6 } + max_count: Var { frame_id: 2, ty: integer, index: 7 } + count: Var { frame_id: 2, ty: integer, index: 8 } + item: Var { frame_id: 2, ty: unknown, index: 9 } + 0: CONST integer(1) + 1: CONST integer(2) + 6: EQ -- hash: 0x773b9bba17dfad2a + 2: SYMBOL Var { var: Var { frame_id: 2, ty: integer, index: 10 }, type_value: integer(unknown) } + 5: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 4: CONST integer(1) + +RULE test_3 + 9: AND -- hash: 0x76ad43078646820d + 8: FOR_IN -- hash: 0x33ea248cbadf6b9 + n: Var { frame_id: 1, ty: integer, index: 0 } + i: Var { frame_id: 1, ty: integer, index: 1 } + max_count: Var { frame_id: 1, ty: integer, index: 2 } + count: Var { frame_id: 1, ty: integer, index: 3 } + item: Var { frame_id: 1, ty: unknown, index: 4 } + 1: CONST integer(1) + 2: CONST integer(2) + 7: EQ -- hash: 0xba743229e1c9a9c + 3: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 5 }, type_value: integer(unknown) } + 6: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 5: CONST integer(1) + +RULE test_4 + 9: OR -- hash: 0x47d223471b2405d4 + 8: FOR_IN -- hash: 0x33ea248cbadf6b9 + n: Var { frame_id: 1, ty: integer, index: 0 } + i: Var { frame_id: 1, ty: integer, index: 1 } + max_count: Var { frame_id: 1, ty: integer, index: 2 } + count: Var { frame_id: 1, ty: integer, index: 3 } + item: Var { frame_id: 1, ty: unknown, index: 4 } + 1: CONST integer(1) + 2: CONST integer(2) + 7: EQ -- hash: 0xba743229e1c9a9c + 3: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 5 }, type_value: integer(unknown) } + 6: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 5: CONST integer(1) + +RULE test_5 + 8: NOT -- hash: 0xa58862c7f168fe46 + 7: FOR_IN -- hash: 0x33ea248cbadf6b9 + n: Var { frame_id: 1, ty: integer, index: 0 } + i: Var { frame_id: 1, ty: integer, index: 1 } + max_count: Var { frame_id: 1, ty: integer, index: 2 } + count: Var { frame_id: 1, ty: integer, index: 3 } + item: Var { frame_id: 1, ty: unknown, index: 4 } + 0: CONST integer(1) + 1: CONST integer(2) + 6: EQ -- hash: 0xba743229e1c9a9c + 2: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 5 }, type_value: integer(unknown) } + 5: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 4: CONST integer(1) + +RULE test_6 + 10: OF -- hash: 0x24cc8e76ae19e17f + 0: CONST integer(1) + 9: CONST boolean(true) + 8: FOR_IN -- hash: 0x6ed2fae041710947 + n: Var { frame_id: 2, ty: integer, index: 5 } + i: Var { frame_id: 2, ty: integer, index: 6 } + max_count: Var { frame_id: 2, ty: integer, index: 7 } + count: Var { frame_id: 2, ty: integer, index: 8 } + item: Var { frame_id: 2, ty: unknown, index: 9 } + 1: CONST integer(1) + 2: CONST integer(2) + 7: EQ -- hash: 0x773b9bba17dfad2a + 3: SYMBOL Var { var: Var { frame_id: 2, ty: integer, index: 10 }, type_value: integer(unknown) } + 6: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 5: CONST integer(1) + +RULE test_7 + 11: EQ -- hash: 0x3dae225ca107ebac + 9: FN_CALL math.to_number@b:b@i -- hash: 0xdcca860b2fe0bb09 + 8: FOR_IN -- hash: 0x33ea248cbadf6b9 + n: Var { frame_id: 1, ty: integer, index: 0 } + i: Var { frame_id: 1, ty: integer, index: 1 } + max_count: Var { frame_id: 1, ty: integer, index: 2 } + count: Var { frame_id: 1, ty: integer, index: 3 } + item: Var { frame_id: 1, ty: unknown, index: 4 } + 1: CONST integer(1) + 2: CONST integer(2) + 7: EQ -- hash: 0xba743229e1c9a9c + 3: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 5 }, type_value: integer(unknown) } + 6: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 5: CONST integer(1) + 10: CONST integer(1) + +RULE test_8 + 8: DEFINED -- hash: 0x27daaa3b4b5e6f93 + 7: FOR_IN -- hash: 0x33ea248cbadf6b9 + n: Var { frame_id: 1, ty: integer, index: 0 } + i: Var { frame_id: 1, ty: integer, index: 1 } + max_count: Var { frame_id: 1, ty: integer, index: 2 } + count: Var { frame_id: 1, ty: integer, index: 3 } + item: Var { frame_id: 1, ty: unknown, index: 4 } + 0: CONST integer(1) + 1: CONST integer(2) + 6: EQ -- hash: 0xba743229e1c9a9c + 2: SYMBOL Var { var: Var { frame_id: 1, ty: integer, index: 5 }, type_value: integer(unknown) } + 5: FN_CALL math.abs@x:i@i -- hash: 0x8fe1d7c818e3e998 + 4: CONST integer(1) + diff --git a/lib/src/compiler/tests/mod.rs b/lib/src/compiler/tests/mod.rs index 3d7b17dc..95491182 100644 --- a/lib/src/compiler/tests/mod.rs +++ b/lib/src/compiler/tests/mod.rs @@ -1,4 +1,3 @@ -use std::collections::Bound; use std::fs; use std::io::Write; use std::mem::size_of; @@ -6,7 +5,7 @@ use std::mem::size_of; use pretty_assertions::assert_eq; use serde_json::json; -use crate::compiler::{FilesizeBounds, SubPattern, VarStack, linters}; +use crate::compiler::{SubPattern, VarStack, linters}; use crate::errors::{SerializationError, VariableError}; use crate::types::Type; use crate::{Compiler, Rules, Scanner, SourceCode, compile}; @@ -1418,53 +1417,290 @@ fn test_warnings() { } } +fn check_filesize_bound( + condition: &str, + matching_sizes: &[usize], + non_matching_sizes: &[usize], +) { + let source = format!( + r#" + rule test {{ + strings: + $a = "foo" + condition: + $a and ({}) + }} + "#, + condition + ); + let rules = compile(source.as_str()).unwrap(); + + for &size in matching_sizes { + let mut data = vec![b' '; size]; + if size >= 3 { + data[0..3].copy_from_slice(b"foo"); + } + let mut scanner = Scanner::new(&rules); + let results = scanner.scan(&data).unwrap(); + assert_eq!( + results.matching_rules().len(), + 1, + "Expected match for condition '{}' with size {}", + condition, + size + ); + } + + for &size in non_matching_sizes { + let mut data = vec![b' '; size]; + if size >= 3 { + data[0..3].copy_from_slice(b"foo"); + } + let mut scanner = Scanner::new(&rules); + let results = scanner.scan(&data).unwrap(); + assert_eq!( + results.matching_rules().len(), + 0, + "Expected NO match for condition '{}' with size {}", + condition, + size + ); + } +} + #[test] fn test_filesize_bounds() { - let mut bounds = FilesizeBounds::default(); + // 1. filesize > 10 (Excluded start 10) + check_filesize_bound("filesize > 10", &[11, 20], &[10, 9, 3]); - // Initially unbounded. - assert_eq!(bounds, FilesizeBounds::from(..)); + // 2. filesize >= 10 (Included start 10) + check_filesize_bound("filesize >= 10", &[10, 11, 20], &[9, 3]); - bounds.min_end(Bound::Included(1000)); - // Now the end must be <= 1000. - assert_eq!(bounds, FilesizeBounds::from(..=1000)); + // 3. filesize < 10 (Excluded end 10) + check_filesize_bound("filesize < 10", &[9, 3], &[10, 11]); - bounds.min_end(Bound::Excluded(1000)); - // Now the end must be < 1000, 1000 was excluded. - assert_eq!(bounds, FilesizeBounds::from(..1000)); + // 4. filesize <= 10 (Included end 10) + check_filesize_bound("filesize <= 10", &[10, 9, 3], &[11, 20]); - bounds.min_end(Bound::Excluded(2000)); - // The bounds remain the same, the previous call didn't change the - // bounds as the existing bounds were already more restrictive. - assert_eq!(bounds, FilesizeBounds::from(..1000)); + // 5. 10 < filesize (Excluded start 10, reversed) + check_filesize_bound("10 < filesize", &[11, 20], &[10, 9, 3]); - bounds.max_start(Bound::Included(1)); - assert_eq!(bounds, FilesizeBounds::from(1..1000)); + // 6. 10 <= filesize (Included start 10, reversed) + check_filesize_bound("10 <= filesize", &[10, 11, 20], &[9, 3]); - bounds.max_start(Bound::Excluded(1)); - assert_eq!( - bounds, - FilesizeBounds::from((Bound::Excluded(1), Bound::Excluded(1000))) + // 7. 10 > filesize (Excluded end 10, reversed) + check_filesize_bound("10 > filesize", &[9, 3], &[10, 11]); + + // 8. 10 >= filesize (Included end 10, reversed) + check_filesize_bound("10 >= filesize", &[10, 9, 3], &[11, 20]); + + // 9. filesize > 10 and filesize < 20 + check_filesize_bound( + "filesize > 10 and filesize < 20", + &[11, 15, 19], + &[10, 20, 21, 3], + ); + + // 10. max_start merging: (Included, Included) with new > current + check_filesize_bound( + "filesize >= 10 and filesize >= 20", + &[20, 25], + &[19, 10, 9, 3], + ); + + // 11. max_start merging: (Included, Excluded) with new >= current + check_filesize_bound( + "filesize >= 10 and filesize > 10", + &[11, 15], + &[10, 9, 3], + ); + check_filesize_bound( + "filesize >= 10 and filesize > 20", + &[21, 25], + &[20, 19, 10, 3], + ); + + // 12. max_start merging: (Excluded, Included) with new > current + check_filesize_bound( + "filesize > 10 and filesize >= 20", + &[20, 25], + &[19, 10, 9, 3], + ); + + // 13. max_start merging: (Excluded, Excluded) with new > current + check_filesize_bound( + "filesize > 10 and filesize > 20", + &[21, 25], + &[20, 10, 9, 3], + ); + + // 14. min_end merging: (Included, Included) with new < current + check_filesize_bound( + "filesize <= 100 and filesize <= 90", + &[90, 85, 3], + &[91, 100], + ); + + // 15. min_end merging: (Included, Excluded) with new <= current + check_filesize_bound( + "filesize <= 100 and filesize < 100", + &[99, 90, 3], + &[100, 101], + ); + check_filesize_bound( + "filesize <= 100 and filesize < 90", + &[89, 80, 3], + &[90, 100], + ); + + // 16. min_end merging: (Excluded, Included) with new < current + check_filesize_bound( + "filesize < 100 and filesize <= 90", + &[90, 85, 3], + &[91, 100], + ); + + // 17. min_end merging: (Excluded, Excluded) with new < current + check_filesize_bound( + "filesize < 100 and filesize < 90", + &[89, 80, 3], + &[90, 100], + ); +} + +fn check_header_constraint( + condition: &str, + matching_headers: &[&[u8]], + non_matching_headers: &[&[u8]], +) { + let source = format!( + r#" + rule test {{ + strings: + $a = "foo" + condition: + $a and ({}) + }} + "#, + condition + ); + let rules = compile(source.as_str()).unwrap(); + + for &header in matching_headers { + let mut data = header.to_vec(); + if data.len() < 3 { + data.extend(vec![b' '; 3 - data.len()]); + } + data.extend(b"foo"); + let mut scanner = Scanner::new(&rules); + let results = scanner.scan(&data).unwrap(); + assert_eq!( + results.matching_rules().len(), + 1, + "Expected match for header condition '{}' with prefix {:?}", + condition, + header + ); + } + + for &header in non_matching_headers { + let mut data = header.to_vec(); + if data.len() < 3 { + data.extend(vec![b' '; 3 - data.len()]); + } + data.extend(b"foo"); + let mut scanner = Scanner::new(&rules); + let results = scanner.scan(&data).unwrap(); + assert_eq!( + results.matching_rules().len(), + 0, + "Expected NO match for header condition '{}' with prefix {:?}", + condition, + header + ); + } +} + +#[test] +fn test_header_constraints() { + // 1. Constrained + check_header_constraint( + "uint32(0) == 0x464c457f", + &[b"\x7FELF"], + &[b"MZ\x00\x00"], + ); + + // 2. Unsatisfiable + check_header_constraint( + "uint8(0) == 1 and uint8(0) == 2", + &[], + &[b"\x01", b"\x02", b"\x03"], ); } #[test] -fn test_error_on_slow_pattern() { +fn test_rules_warnings() { let mut compiler = Compiler::new(); - compiler.error_on_slow_pattern(true); + compiler + .add_source( + r#" + import "math" + import "math" + rule test { condition: true } + "#, + ) + .unwrap(); + let rules = compiler.build(); + assert!(!rules.warnings().is_empty()); +} - let err = compiler +#[test] +fn test_rules_debug() { + let mut compiler = Compiler::new(); + compiler.add_source("rule test { condition: true }").unwrap(); + let rules = compiler.build(); + + let debug_str = format!("{:?}", rules); + assert!(debug_str.contains("RuleId(0)")); + assert!(debug_str.contains("name: test")); +} + +#[test] +fn test_rules_serialization() { + let mut compiler = Compiler::new(); + compiler.add_source("rule test { condition: true }").unwrap(); + let rules = compiler.build(); + + let serialized = rules.serialize().unwrap(); + let deserialized = Rules::deserialize_from(&serialized[..]).unwrap(); + assert_eq!(deserialized.iter().len(), 1); + + let mut iter = deserialized.iter(); + assert_eq!(iter.len(), 1); + let r = iter.next().unwrap(); + assert_eq!(r.identifier(), "test"); + assert!(iter.next().is_none()); +} + +#[test] +fn test_rules_matches_many() { + let mut compiler = Compiler::new(); + compiler + .define_global("str_foo", "foo") + .unwrap() .add_source( r#" rule test { - strings: - $a = "a" condition: - $a + str_foo matches /ba[rs]/ or str_foo matches /ba[zt]/ } "#, ) - .unwrap_err(); - - assert!(matches!(err, crate::errors::CompileError::SlowPattern(_))); + .unwrap(); + let rules = compiler.build(); + // Scan using these rules to trigger get_regex_set at scan time + let mut scanner = Scanner::new(&rules); + scanner.set_global("str_foo", "bar").unwrap(); + let scan_results = scanner.scan(&[]).unwrap(); + assert_eq!(scan_results.matching_rules().len(), 1); } diff --git a/lib/src/modules/macho/tests/mod.rs b/lib/src/modules/macho/tests/mod.rs index 73aa9eac..b368c26c 100644 --- a/lib/src/modules/macho/tests/mod.rs +++ b/lib/src/modules/macho/tests/mod.rs @@ -21,7 +21,17 @@ fn test_macho_module() { "src/modules/macho/tests/testdata/8962a76d0aeaee3326cf840de11543c8beebeb768e712bd3b754b5cd3e151356.in.zip", ); - let linker_options_data = create_binary_from_zipped_ihex("src/modules/macho/tests/testdata/f3cde7740370819a974d1bc7fbeae1946382e3377e64f3162bcc3a5cb34828b7.in.zip"); + let linker_options_data = create_binary_from_zipped_ihex( + "src/modules/macho/tests/testdata/f3cde7740370819a974d1bc7fbeae1946382e3377e64f3162bcc3a5cb34828b7.in.zip", + ); + + let arm64_macho_test_data = create_binary_from_zipped_ihex( + "src/modules/macho/tests/testdata/39addb273998cffe5362fc69047a3ecbf499d1b1b95ac6937785c8c17a9a1515.in.zip", + ); + + let ppc64_macho_test_data = create_binary_from_zipped_ihex( + "src/modules/macho/tests/testdata/24536f0d970ca2e02b998d21e51e062b84f9b4f31af8bb5addd5b0de6e0013ab.in.zip", + ); rule_true!( r#" @@ -561,4 +571,47 @@ fn test_macho_module() { "#, &linker_options_data ); + + rule_true!( + r#" + import "macho" + rule macho_test { + condition: + macho.magic == 0xcffaedfe and + macho.cputype == 0x100000c and + macho.cpusubtype == 0x0 and + macho.filetype == 1 and + macho.ncmds == 5 and + macho.sizeofcmds == 1656 and + macho.flags == 0x2000 and + macho.number_of_segments == 1 and + macho.build_version.platform == 11 and + macho.build_version.minos == "1.0.0" and + macho.build_version.sdk == "26.2.0" and + macho.has_export("_$s10Foundation4DataV15_RepresentationOWOe") and + macho.has_import("_$s10Foundation4DataV19base64EncodedString7optionsSSSo27NSDataBase64EncodingOptionsV_tF") + } + "#, + &arm64_macho_test_data + ); + + rule_true!( + r#" + import "macho" + rule macho_test { + condition: + macho.magic == 0xfeedfacf and + macho.cputype == macho.CPU_TYPE_POWERPC64 and + macho.cpusubtype == 0x0 and + macho.filetype == 1 and + macho.ncmds == 3 and + macho.sizeofcmds == 656 and + macho.flags == 0x2000 and + macho.number_of_segments == 1 and + macho.has_export("_choose_tmpdir") and + macho.has_import("_abort") + } + "#, + &ppc64_macho_test_data + ); } diff --git a/lib/src/modules/macho/tests/testdata/24536f0d970ca2e02b998d21e51e062b84f9b4f31af8bb5addd5b0de6e0013ab.in.zip b/lib/src/modules/macho/tests/testdata/24536f0d970ca2e02b998d21e51e062b84f9b4f31af8bb5addd5b0de6e0013ab.in.zip new file mode 100644 index 00000000..4462145d Binary files /dev/null and b/lib/src/modules/macho/tests/testdata/24536f0d970ca2e02b998d21e51e062b84f9b4f31af8bb5addd5b0de6e0013ab.in.zip differ diff --git a/lib/src/modules/macho/tests/testdata/24536f0d970ca2e02b998d21e51e062b84f9b4f31af8bb5addd5b0de6e0013ab.out b/lib/src/modules/macho/tests/testdata/24536f0d970ca2e02b998d21e51e062b84f9b4f31af8bb5addd5b0de6e0013ab.out new file mode 100644 index 00000000..948d8a0c --- /dev/null +++ b/lib/src/modules/macho/tests/testdata/24536f0d970ca2e02b998d21e51e062b84f9b4f31af8bb5addd5b0de6e0013ab.out @@ -0,0 +1,333 @@ +magic: 0xfeedfacf +cputype: 0x1000012 +cpusubtype: 0x0 +filetype: 1 +ncmds: 3 +sizeofcmds: 656 +flags: 0x2000 +reserved: 0 +number_of_segments: 1 +symtab: + symoff: 2968 + nsyms: 93 + stroff: 4456 + strsize: 788 + entries: + - "/Users/gkhanna/build2/powerpc-apple-darwin8.8.0/ppc64/libiberty/" + - "../../../../gcc-4.2-20061007/libiberty/make-temp-file.c" + - "gcc2_compiled." + - "_memoized_tmpdir" + - "_vartmp" + - "_usrtmp" + - "_tmp" + - "choose_tmpdir:F(0,1)=*(0,2)=r(0,2);0;127;" + - "char:t(0,2)" + - "base:r(0,3)=*(0,4)=k(0,2)" + - "tmpdir:r(0,1)" + - "len:r(0,5)=r(0,5);0;037777777777;" + - "unsigned int:t(0,5)" + - "make_temp_file:F(0,1)" + - "suffix:P(0,3)" + - "base:r(0,3)" + - "suffix_len:r(0,6)=r(0,6);-2147483648;2147483647;" + - "int:t(0,6)" + - "tmp:S(0,7)=ar(0,8)=@s64;r(0,8);0;01777777777777777777777;;0;04;(0,4)" + - "long unsigned int:t(0,9)=@s64;r(0,9);0;01777777777777777777777;" + - "usrtmp:S(0,10)=ar(0,8);0;010;(0,4)" + - "vartmp:S(0,11)=ar(0,8);0;010;(0,4)" + - "memoized_tmpdir:S(0,1)" + - "_choose_tmpdir" + - "_make_temp_file" + - "_abort" + - "_access" + - "_close" + - "_getenv" + - "_mkstemps" + - "_strcpy" + - "_strlen" + - "_xmalloc" + - "dyld_stub_binding_helper" + nlists: + - n_strx: 122 + n_type: 100 + n_sect: 1 + n_desc: 2 + n_value: 0 + - n_strx: 187 + n_type: 100 + n_sect: 1 + n_desc: 2 + n_value: 0 + - n_strx: 243 + n_type: 60 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 258 + n_type: 14 + n_sect: 4 + n_desc: 0 + n_value: 1216 + - n_strx: 275 + n_type: 14 + n_sect: 3 + n_desc: 0 + n_value: 856 + - n_strx: 283 + n_type: 14 + n_sect: 3 + n_desc: 0 + n_value: 872 + - n_strx: 291 + n_type: 14 + n_sect: 3 + n_desc: 0 + n_value: 888 + - n_strx: 296 + n_type: 36 + n_sect: 1 + n_desc: 98 + n_value: 0 + - n_strx: 338 + n_type: 128 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 350 + n_type: 64 + n_sect: 0 + n_desc: 99 + n_value: 28 + - n_strx: 376 + n_type: 64 + n_sect: 0 + n_desc: 100 + n_value: 30 + - n_strx: 390 + n_type: 64 + n_sect: 0 + n_desc: 101 + n_value: 29 + - n_strx: 424 + n_type: 128 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 447 + n_type: 36 + n_sect: 1 + n_desc: 149 + n_value: 528 + - n_strx: 469 + n_type: 64 + n_sect: 0 + n_desc: 148 + n_value: 27 + - n_strx: 483 + n_type: 64 + n_sect: 0 + n_desc: 150 + n_value: 26 + - n_strx: 495 + n_type: 64 + n_sect: 0 + n_desc: 152 + n_value: 28 + - n_strx: 544 + n_type: 128 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 558 + n_type: 38 + n_sect: 3 + n_desc: 77 + n_value: 888 + - n_strx: 627 + n_type: 128 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 691 + n_type: 38 + n_sect: 3 + n_desc: 78 + n_value: 872 + - n_strx: 726 + n_type: 38 + n_sect: 3 + n_desc: 80 + n_value: 856 + - n_strx: 761 + n_type: 40 + n_sect: 4 + n_desc: 83 + n_value: 1216 + - n_strx: 1 + n_type: 15 + n_sect: 1 + n_desc: 0 + n_value: 0 + - n_strx: 16 + n_type: 15 + n_sect: 1 + n_desc: 0 + n_value: 528 + - n_strx: 100 + n_type: 1 + n_sect: 0 + n_desc: 1 + n_value: 0 + - n_strx: 57 + n_type: 1 + n_sect: 0 + n_desc: 1 + n_value: 0 + - n_strx: 107 + n_type: 1 + n_sect: 0 + n_desc: 1 + n_value: 0 + - n_strx: 82 + n_type: 1 + n_sect: 0 + n_desc: 1 + n_value: 0 + - n_strx: 90 + n_type: 1 + n_sect: 0 + n_desc: 1 + n_value: 0 + - n_strx: 65 + n_type: 1 + n_sect: 0 + n_desc: 1 + n_value: 0 + - n_strx: 114 + n_type: 1 + n_sect: 0 + n_desc: 1 + n_value: 0 + - n_strx: 73 + n_type: 1 + n_sect: 0 + n_desc: 1 + n_value: 0 + - n_strx: 32 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 +dysymtab: + ilocalsym: 0 + nlocalsym: 82 + iextdefsym: 82 + nextdefsym: 2 + tocoff: 84 + ntoc: 9 + modtaboff: 0 + nmodtab: 0 + extrefsymoff: 0 + nextrefsyms: 0 + indirectsymoff: 0 + nindirectsyms: 0 + extreloff: 2904 + nextrel: 16 + locreloff: 0 + nlocrel: 0 +segments: + - segname: "" + vmaddr: 0x0 + vmsize: 0x4c8 + fileoff: 688 + filesize: 1216 + maxprot: 0x7 + initprot: 0x7 + nsects: 6 + flags: 0x0 + sections: + - segname: "__TEXT" + sectname: "__text" + addr: 0x0 + size: 0x310 + offset: 688 + align: 4 + reloff: 1904 + nreloc: 85 + flags: 0x80000400 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__TEXT" + sectname: "__cstring" + addr: 0x310 + size: 0x41 + offset: 1472 + align: 3 + reloff: 0 + nreloc: 0 + flags: 0x2 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__TEXT" + sectname: "__const" + addr: 0x358 + size: 0x25 + offset: 1544 + align: 3 + reloff: 0 + nreloc: 0 + flags: 0x0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__DATA" + sectname: "__bss" + addr: 0x4c0 + size: 0x8 + offset: 0 + align: 3 + reloff: 0 + nreloc: 0 + flags: 0x1 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__TEXT" + sectname: "__picsymbolstub1" + addr: 0x380 + size: 0x100 + offset: 1584 + align: 5 + reloff: 2584 + nreloc: 32 + flags: 0x80000408 + reserved1: 0 + reserved2: 32 + reserved3: 0 + - segname: "__DATA" + sectname: "__la_symbol_ptr" + addr: 0x480 + size: 0x40 + offset: 1840 + align: 2 + reloff: 2840 + nreloc: 8 + flags: 0x7 + reserved1: 8 + reserved2: 0 + reserved3: 0 +exports: + - "_choose_tmpdir" + - "_make_temp_file" +imports: + - "_abort" + - "_access" + - "_close" + - "_getenv" + - "_mkstemps" + - "_strcpy" + - "_strlen" + - "_xmalloc" + - "dyld_stub_binding_helper" \ No newline at end of file diff --git a/lib/src/modules/macho/tests/testdata/39addb273998cffe5362fc69047a3ecbf499d1b1b95ac6937785c8c17a9a1515.in.zip b/lib/src/modules/macho/tests/testdata/39addb273998cffe5362fc69047a3ecbf499d1b1b95ac6937785c8c17a9a1515.in.zip new file mode 100644 index 00000000..e11c1dc4 Binary files /dev/null and b/lib/src/modules/macho/tests/testdata/39addb273998cffe5362fc69047a3ecbf499d1b1b95ac6937785c8c17a9a1515.in.zip differ diff --git a/lib/src/modules/macho/tests/testdata/39addb273998cffe5362fc69047a3ecbf499d1b1b95ac6937785c8c17a9a1515.out b/lib/src/modules/macho/tests/testdata/39addb273998cffe5362fc69047a3ecbf499d1b1b95ac6937785c8c17a9a1515.out new file mode 100644 index 00000000..1fa1bd02 --- /dev/null +++ b/lib/src/modules/macho/tests/testdata/39addb273998cffe5362fc69047a3ecbf499d1b1b95ac6937785c8c17a9a1515.out @@ -0,0 +1,1031 @@ +magic: 0xcffaedfe +cputype: 0x100000c +cpusubtype: 0x0 +filetype: 1 +ncmds: 5 +sizeofcmds: 1656 +flags: 0x2000 +reserved: 0 +number_of_segments: 1 +symtab: + symoff: 5744 + nsyms: 115 + stroff: 7584 + strsize: 3296 + entries: + - "ltmp0" + - "ltmp1" + - "lCPI1_0" + - "l_OBJC_CLASS_REF_$_RCAVrpek" + - "L_selector(init)" + - "L_selector(setVrpw:)" + - "L_selector(vrpqArray)" + - "L_selector(addObject:)" + - "l_OBJC_CLASS_REF_$_RCAVrpek_Vrpel" + - "L_selector(data)" + - "l_metadata" + - "_$s19RecaptchaEnterprise5RCedfO5rcecg_5rcgqh5rcgqiys6UInt32V_AA5RCcrzCSaySo14RCAWspeu_WspduCGtKFZySayypSgGcfU_TA" + - "l_objectdestroy" + - "ltmp2" + - "l_.str.19.RecaptchaEnterprise" + - "ltmp3" + - "l_.str.5.RCedf" + - "ltmp4" + - "ltmp5" + - "ltmp6" + - "l_got.$s19RecaptchaEnterprise5RCecfMp" + - "ltmp7" + - "ltmp8" + - "l__swift5_reflection_descriptor" + - "ltmp9" + - "ltmp10" + - "L_selector_data(init)" + - "ltmp11" + - "L_selector_data(setVrpw:)" + - "L_selector_data(vrpqArray)" + - "L_selector_data(addObject:)" + - "L_selector_data(data)" + - "ltmp12" + - "l_$s19RecaptchaEnterprise5RCedfOAA5RCecfAAHc" + - "ltmp13" + - "l_$s19RecaptchaEnterprise5RCedfOHn" + - "ltmp14" + - "l_llvm.swift_module_hash" + - "ltmp15" + - "ltmp16" + - "ltmp17" + - "_$s10Foundation4DataV15_RepresentationOWOe" + - "_$s19RecaptchaEnterprise5RCcnaOACs5ErrorAAWL" + - "_$s19RecaptchaEnterprise5RCcnaOACs5ErrorAAWl" + - "_$s19RecaptchaEnterprise5RCedfO5rcecg_5rcgqh5rcgqiys6UInt32V_AA5RCcrzCSaySo14RCAWspeu_WspduCGtKFZTf4nnnd_n" + - "_$s19RecaptchaEnterprise5RCedfO5rcecg_5rcgqh5rcgqiys6UInt32V_AA5RCcrzCSaySo14RCAWspeu_WspduCGtKFZySayypSgGcfU_" + - "_$s19RecaptchaEnterprise5RCedfOAA5RCecfA2aDP5rcecg_5rcgqh5rcgqiys6UInt32V_AA5RCcrzCSaySo14RCAWspeu_WspduCGtKFZTW" + - "_$s19RecaptchaEnterprise5RCedfOAA5RCecfAAMc" + - "_$s19RecaptchaEnterprise5RCedfOAA5RCecfAAWP" + - "_$s19RecaptchaEnterprise5RCedfOMF" + - "_$s19RecaptchaEnterprise5RCedfOMa" + - "_$s19RecaptchaEnterprise5RCedfOMf" + - "_$s19RecaptchaEnterprise5RCedfOMn" + - "_$s19RecaptchaEnterprise5RCedfON" + - "_$s19RecaptchaEnterpriseMXM" + - "_$sSSWOh" + - "_$ss23_ContiguousArrayStorageCySSGMR" + - "_$ss23_ContiguousArrayStorageCySSGMd" + - "_$sypSgMR" + - "_$sypSgMd" + - "_$sypSgWOc" + - "_$sypSgWOh" + - "_$sypWOb" + - "_$sypWOc" + - "___swift_destroy_boxed_opaque_existential_0" + - "___swift_instantiateConcreteTypeFromMangledNameV2" + - "___swift_reflection_version" + - "_symbolic SS" + - "_symbolic _____ 19RecaptchaEnterprise5RCcrzC" + - "_symbolic _____ 19RecaptchaEnterprise5RCedfO" + - "_symbolic _____ySSG s23_ContiguousArrayStorageC" + - "_symbolic ypSg" + - "_$s10Foundation4DataV19base64EncodedString7optionsSSSo27NSDataBase64EncodingOptionsV_tF" + - "_$s10Foundation4DataV36_unconditionallyBridgeFromObjectiveCyACSo6NSDataCSgFZ" + - "_$s19RecaptchaEnterprise5RCcnaON" + - "_$s19RecaptchaEnterprise5RCcnaOs5ErrorAAMc" + - "_$s19RecaptchaEnterprise5RCcrzCMn" + - "_$s19RecaptchaEnterprise5RCecfMp" + - "_$s19RecaptchaEnterprise5RCevuC5rcewoyxSo14RCAWspeu_WspduCKlFSS_Tg5" + - "_$s19RecaptchaEnterprise5RCevuC5rcewoyxSo14RCAWspeu_WspduCKlFs5Int32V_Tg5" + - "_$s19RecaptchaEnterprise5RCfzyO5rcfzz5rcgaa5rcgsfyps5Int32V_ySayypSgGctKFZ" + - "_$s19RecaptchaEnterprise5RCgarC5rcgauyySS_SSdtF" + - "_$sSDyq_Sgxciss6UInt32V_ypTg5" + - "_$sSS10FoundationE19_bridgeToObjectiveCSo8NSStringCyF" + - "_$sSS10describingSSx_tclufC" + - "_$sSSN" + - "_$ss12_ArrayBufferV19_getElementSlowPathyyXlSiFyXl_Ts5" + - "_$ss18_CocoaArrayWrapperV8endIndexSivg" + - "_$ss23_ContiguousArrayStorageCMn" + - "_$sypN" + - "_$sytWV" + - "_OBJC_CLASS_$_RCAVrpek" + - "_OBJC_CLASS_$_RCAVrpek_Vrpel" + - "_objc_allocWithZone" + - "_objc_msgSend" + - "_objc_release_x21" + - "_objc_release_x23" + - "_objc_release_x24" + - "_objc_release_x25" + - "_objc_release_x26" + - "_objc_retainAutoreleasedReturnValue" + - "_objc_retain_x21" + - "_swift_allocError" + - "_swift_allocObject" + - "_swift_beginAccess" + - "_swift_bridgeObjectRelease" + - "_swift_deallocObject" + - "_swift_endAccess" + - "_swift_getTypeByMangledNameInContext2" + - "_swift_getWitnessTable" + - "_swift_initStackObject" + - "_swift_release" + - "_swift_retain" + - "_swift_setDeallocating" + - "_swift_willThrow" + nlists: + - n_strx: 2999 + n_type: 14 + n_sect: 1 + n_desc: 0 + n_value: 0 + - n_strx: 2951 + n_type: 14 + n_sect: 2 + n_desc: 0 + n_value: 1808 + - n_strx: 3049 + n_type: 14 + n_sect: 2 + n_desc: 0 + n_value: 1808 + - n_strx: 670 + n_type: 14 + n_sect: 10 + n_desc: 0 + n_value: 2128 + - n_strx: 3113 + n_type: 14 + n_sect: 12 + n_desc: 0 + n_value: 2184 + - n_strx: 3191 + n_type: 14 + n_sect: 12 + n_desc: 0 + n_value: 2192 + - n_strx: 3064 + n_type: 14 + n_sect: 12 + n_desc: 0 + n_value: 2200 + - n_strx: 3238 + n_type: 14 + n_sect: 12 + n_desc: 0 + n_value: 2208 + - n_strx: 568 + n_type: 14 + n_sect: 10 + n_desc: 0 + n_value: 2136 + - n_strx: 3152 + n_type: 14 + n_sect: 12 + n_desc: 0 + n_value: 2216 + - n_strx: 1285 + n_type: 14 + n_sect: 5 + n_desc: 0 + n_value: 1984 + - n_strx: 2361 + n_type: 14 + n_sect: 1 + n_desc: 0 + n_value: 1352 + - n_strx: 1 + n_type: 14 + n_sect: 1 + n_desc: 0 + n_value: 1308 + - n_strx: 2888 + n_type: 14 + n_sect: 3 + n_desc: 0 + n_value: 1824 + - n_strx: 905 + n_type: 14 + n_sect: 3 + n_desc: 0 + n_value: 1824 + - n_strx: 2819 + n_type: 14 + n_sect: 4 + n_desc: 0 + n_value: 1892 + - n_strx: 820 + n_type: 14 + n_sect: 3 + n_desc: 0 + n_value: 1844 + - n_strx: 2788 + n_type: 14 + n_sect: 5 + n_desc: 0 + n_value: 1936 + - n_strx: 2585 + n_type: 14 + n_sect: 6 + n_desc: 0 + n_value: 2032 + - n_strx: 2499 + n_type: 14 + n_sect: 7 + n_desc: 0 + n_value: 2064 + - n_strx: 216 + n_type: 14 + n_sect: 5 + n_desc: 0 + n_value: 2024 + - n_strx: 2486 + n_type: 14 + n_sect: 8 + n_desc: 0 + n_value: 2080 + - n_strx: 2480 + n_type: 14 + n_sect: 9 + n_desc: 0 + n_value: 2104 + - n_strx: 133 + n_type: 14 + n_sect: 9 + n_desc: 32 + n_value: 2104 + - n_strx: 2474 + n_type: 14 + n_sect: 10 + n_desc: 0 + n_value: 2128 + - n_strx: 3057 + n_type: 14 + n_sect: 11 + n_desc: 0 + n_value: 2144 + - n_strx: 3130 + n_type: 14 + n_sect: 11 + n_desc: 0 + n_value: 2144 + - n_strx: 2992 + n_type: 14 + n_sect: 12 + n_desc: 0 + n_value: 2184 + - n_strx: 3212 + n_type: 14 + n_sect: 11 + n_desc: 0 + n_value: 2149 + - n_strx: 3086 + n_type: 14 + n_sect: 11 + n_desc: 0 + n_value: 2158 + - n_strx: 3261 + n_type: 14 + n_sect: 11 + n_desc: 0 + n_value: 2168 + - n_strx: 3169 + n_type: 14 + n_sect: 11 + n_desc: 0 + n_value: 2179 + - n_strx: 2944 + n_type: 14 + n_sect: 13 + n_desc: 0 + n_value: 2224 + - n_strx: 1231 + n_type: 14 + n_sect: 13 + n_desc: 32 + n_value: 2224 + - n_strx: 2843 + n_type: 14 + n_sect: 14 + n_desc: 0 + n_value: 2228 + - n_strx: 504 + n_type: 14 + n_sect: 14 + n_desc: 32 + n_value: 2228 + - n_strx: 2812 + n_type: 14 + n_sect: 15 + n_desc: 0 + n_value: 2232 + - n_strx: 698 + n_type: 14 + n_sect: 15 + n_desc: 32 + n_value: 2232 + - n_strx: 2781 + n_type: 14 + n_sect: 16 + n_desc: 0 + n_value: 2248 + - n_strx: 2523 + n_type: 14 + n_sect: 17 + n_desc: 0 + n_value: 2256 + - n_strx: 2492 + n_type: 14 + n_sect: 18 + n_desc: 0 + n_value: 2384 + - n_strx: 1020 + n_type: 31 + n_sect: 1 + n_desc: 192 + n_value: 1728 + - n_strx: 1971 + n_type: 31 + n_sect: 8 + n_desc: 128 + n_value: 2080 + - n_strx: 602 + n_type: 31 + n_sect: 1 + n_desc: 192 + n_value: 1244 + - n_strx: 296 + n_type: 31 + n_sect: 1 + n_desc: 192 + n_value: 588 + - n_strx: 1330 + n_type: 31 + n_sect: 1 + n_desc: 192 + n_value: 16 + - n_strx: 1593 + n_type: 31 + n_sect: 1 + n_desc: 128 + n_value: 568 + - n_strx: 1187 + n_type: 31 + n_sect: 3 + n_desc: 0 + n_value: 1852 + - n_strx: 1774 + n_type: 31 + n_sect: 5 + n_desc: 0 + n_value: 1968 + - n_strx: 2206 + n_type: 31 + n_sect: 7 + n_desc: 160 + n_value: 2064 + - n_strx: 1296 + n_type: 31 + n_sect: 1 + n_desc: 0 + n_value: 0 + - n_strx: 835 + n_type: 31 + n_sect: 5 + n_desc: 128 + n_value: 1936 + - n_strx: 403 + n_type: 31 + n_sect: 4 + n_desc: 0 + n_value: 1904 + - n_strx: 1877 + n_type: 31 + n_sect: 5 + n_desc: 512 + n_value: 1952 + - n_strx: 1943 + n_type: 31 + n_sect: 4 + n_desc: 128 + n_value: 1892 + - n_strx: 734 + n_type: 31 + n_sect: 1 + n_desc: 192 + n_value: 1676 + - n_strx: 1737 + n_type: 31 + n_sect: 3 + n_desc: 128 + n_value: 1880 + - n_strx: 1087 + n_type: 31 + n_sect: 8 + n_desc: 128 + n_value: 2096 + - n_strx: 1727 + n_type: 31 + n_sect: 3 + n_desc: 128 + n_value: 1872 + - n_strx: 1077 + n_type: 31 + n_sect: 8 + n_desc: 128 + n_value: 2088 + - n_strx: 1133 + n_type: 31 + n_sect: 1 + n_desc: 192 + n_value: 1424 + - n_strx: 723 + n_type: 31 + n_sect: 1 + n_desc: 192 + n_value: 1572 + - n_strx: 1276 + n_type: 31 + n_sect: 1 + n_desc: 192 + n_value: 1792 + - n_strx: 1124 + n_type: 31 + n_sect: 1 + n_desc: 192 + n_value: 1364 + - n_strx: 3005 + n_type: 31 + n_sect: 1 + n_desc: 192 + n_value: 1644 + - n_strx: 2894 + n_type: 31 + n_sect: 1 + n_desc: 192 + n_value: 1504 + - n_strx: 254 + n_type: 31 + n_sect: 3 + n_desc: 160 + n_value: 1888 + - n_strx: 1714 + n_type: 31 + n_sect: 6 + n_desc: 128 + n_value: 2044 + - n_strx: 2240 + n_type: 31 + n_sect: 6 + n_desc: 128 + n_value: 2038 + - n_strx: 1818 + n_type: 31 + n_sect: 6 + n_desc: 128 + n_value: 2032 + - n_strx: 2313 + n_type: 31 + n_sect: 6 + n_desc: 128 + n_value: 2054 + - n_strx: 805 + n_type: 31 + n_sect: 6 + n_desc: 128 + n_value: 2048 + - n_strx: 2118 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 1441 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 1910 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 1144 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 437 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 183 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 2695 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 2621 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 1518 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 2070 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 2591 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 2016 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 2285 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 1870 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 2530 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 743 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 471 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 1863 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 1706 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 647 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 539 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 977 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 1063 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 2974 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 2825 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 2794 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 2763 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 2505 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 869 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 2957 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 165 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 78 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 97 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 950 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 57 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 116 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 2850 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 997 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 34 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 935 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 282 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 782 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 + - n_strx: 17 + n_type: 1 + n_sect: 0 + n_desc: 0 + n_value: 0 +dysymtab: + ilocalsym: 0 + nlocalsym: 41 + iextdefsym: 41 + nextdefsym: 31 + tocoff: 72 + ntoc: 43 + modtaboff: 0 + nmodtab: 0 + extrefsymoff: 0 + nextrefsyms: 0 + indirectsymoff: 0 + nindirectsyms: 0 + extreloff: 0 + nextrel: 0 + locreloff: 0 + nlocrel: 0 +segments: + - segname: "" + vmaddr: 0x0 + vmsize: 0x998 + fileoff: 1688 + filesize: 2456 + maxprot: 0x7 + initprot: 0x7 + nsects: 18 + flags: 0x0 + sections: + - segname: "__TEXT" + sectname: "__text" + addr: 0x0 + size: 0x710 + offset: 1688 + align: 2 + reloff: 4144 + nreloc: 130 + flags: 0x80000400 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__TEXT" + sectname: "__literal16" + addr: 0x710 + size: 0x10 + offset: 3496 + align: 4 + reloff: 0 + nreloc: 0 + flags: 0xe + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__TEXT" + sectname: "__const" + addr: 0x720 + size: 0x42 + offset: 3512 + align: 4 + reloff: 5184 + nreloc: 10 + flags: 0x0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__TEXT" + sectname: "__constg_swiftt" + addr: 0x764 + size: 0x28 + offset: 3580 + align: 2 + reloff: 5264 + nreloc: 10 + flags: 0x0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__DATA" + sectname: "__const" + addr: 0x790 + size: 0x60 + offset: 3624 + align: 3 + reloff: 5344 + nreloc: 7 + flags: 0x0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__TEXT" + sectname: "__swift5_typeref" + addr: 0x7f0 + size: 0x20 + offset: 3720 + align: 1 + reloff: 5400 + nreloc: 4 + flags: 0x0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__TEXT" + sectname: "__swift5_fieldmd" + addr: 0x810 + size: 0x10 + offset: 3752 + align: 2 + reloff: 5432 + nreloc: 2 + flags: 0x0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__DATA" + sectname: "__data" + addr: 0x820 + size: 0x18 + offset: 3768 + align: 3 + reloff: 0 + nreloc: 0 + flags: 0x0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__TEXT" + sectname: "__swift5_capture" + addr: 0x838 + size: 0x14 + offset: 3792 + align: 2 + reloff: 5448 + nreloc: 4 + flags: 0x0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__DATA" + sectname: "__objc_classrefs" + addr: 0x850 + size: 0x10 + offset: 3816 + align: 3 + reloff: 5480 + nreloc: 2 + flags: 0x10000000 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__TEXT" + sectname: "__objc_methname" + addr: 0x860 + size: 0x28 + offset: 3832 + align: 0 + reloff: 0 + nreloc: 0 + flags: 0x2 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__DATA" + sectname: "__objc_selrefs" + addr: 0x888 + size: 0x28 + offset: 3872 + align: 3 + reloff: 5496 + nreloc: 5 + flags: 0x10000005 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__TEXT" + sectname: "__swift5_proto" + addr: 0x8b0 + size: 0x4 + offset: 3912 + align: 2 + reloff: 5536 + nreloc: 2 + flags: 0x0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__TEXT" + sectname: "__swift5_types" + addr: 0x8b4 + size: 0x4 + offset: 3916 + align: 2 + reloff: 5552 + nreloc: 2 + flags: 0x0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__LLVM" + sectname: "__swift_modhash" + addr: 0x8b8 + size: 0x10 + offset: 3920 + align: 0 + reloff: 0 + nreloc: 0 + flags: 0x0 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__DATA" + sectname: "__objc_imageinfo" + addr: 0x8c8 + size: 0x8 + offset: 3936 + align: 0 + reloff: 0 + nreloc: 0 + flags: 0x10000000 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__LD" + sectname: "__compact_unwind" + addr: 0x8d0 + size: 0x80 + offset: 3944 + align: 3 + reloff: 5568 + nreloc: 4 + flags: 0x2000000 + reserved1: 0 + reserved2: 0 + reserved3: 0 + - segname: "__TEXT" + sectname: "__eh_frame" + addr: 0x950 + size: 0x48 + offset: 4072 + align: 3 + reloff: 5600 + nreloc: 2 + flags: 0x6800000b + reserved1: 0 + reserved2: 0 + reserved3: 0 +build_version: + platform: 11 + minos: "1.0.0" + sdk: "26.2.0" + ntools: 0 +exports: + - "_$s10Foundation4DataV15_RepresentationOWOe" + - "_$s19RecaptchaEnterprise5RCcnaOACs5ErrorAAWL" + - "_$s19RecaptchaEnterprise5RCcnaOACs5ErrorAAWl" + - "_$s19RecaptchaEnterprise5RCedfO5rcecg_5rcgqh5rcgqiys6UInt32V_AA5RCcrzCSaySo14RCAWspeu_WspduCGtKFZTf4nnnd_n" + - "_$s19RecaptchaEnterprise5RCedfO5rcecg_5rcgqh5rcgqiys6UInt32V_AA5RCcrzCSaySo14RCAWspeu_WspduCGtKFZySayypSgGcfU_" + - "_$s19RecaptchaEnterprise5RCedfOAA5RCecfA2aDP5rcecg_5rcgqh5rcgqiys6UInt32V_AA5RCcrzCSaySo14RCAWspeu_WspduCGtKFZTW" + - "_$s19RecaptchaEnterprise5RCedfOAA5RCecfAAMc" + - "_$s19RecaptchaEnterprise5RCedfOAA5RCecfAAWP" + - "_$s19RecaptchaEnterprise5RCedfOMF" + - "_$s19RecaptchaEnterprise5RCedfOMa" + - "_$s19RecaptchaEnterprise5RCedfOMf" + - "_$s19RecaptchaEnterprise5RCedfOMn" + - "_$s19RecaptchaEnterprise5RCedfON" + - "_$s19RecaptchaEnterpriseMXM" + - "_$sSSWOh" + - "_$ss23_ContiguousArrayStorageCySSGMR" + - "_$ss23_ContiguousArrayStorageCySSGMd" + - "_$sypSgMR" + - "_$sypSgMd" + - "_$sypSgWOc" + - "_$sypSgWOh" + - "_$sypWOb" + - "_$sypWOc" + - "___swift_destroy_boxed_opaque_existential_0" + - "___swift_instantiateConcreteTypeFromMangledNameV2" + - "___swift_reflection_version" + - "_symbolic SS" + - "_symbolic _____ 19RecaptchaEnterprise5RCcrzC" + - "_symbolic _____ 19RecaptchaEnterprise5RCedfO" + - "_symbolic _____ySSG s23_ContiguousArrayStorageC" + - "_symbolic ypSg" +imports: + - "_$s10Foundation4DataV19base64EncodedString7optionsSSSo27NSDataBase64EncodingOptionsV_tF" + - "_$s10Foundation4DataV36_unconditionallyBridgeFromObjectiveCyACSo6NSDataCSgFZ" + - "_$s19RecaptchaEnterprise5RCcnaON" + - "_$s19RecaptchaEnterprise5RCcnaOs5ErrorAAMc" + - "_$s19RecaptchaEnterprise5RCcrzCMn" + - "_$s19RecaptchaEnterprise5RCecfMp" + - "_$s19RecaptchaEnterprise5RCevuC5rcewoyxSo14RCAWspeu_WspduCKlFSS_Tg5" + - "_$s19RecaptchaEnterprise5RCevuC5rcewoyxSo14RCAWspeu_WspduCKlFs5Int32V_Tg5" + - "_$s19RecaptchaEnterprise5RCfzyO5rcfzz5rcgaa5rcgsfyps5Int32V_ySayypSgGctKFZ" + - "_$s19RecaptchaEnterprise5RCgarC5rcgauyySS_SSdtF" + - "_$sSDyq_Sgxciss6UInt32V_ypTg5" + - "_$sSS10FoundationE19_bridgeToObjectiveCSo8NSStringCyF" + - "_$sSS10describingSSx_tclufC" + - "_$sSSN" + - "_$ss12_ArrayBufferV19_getElementSlowPathyyXlSiFyXl_Ts5" + - "_$ss18_CocoaArrayWrapperV8endIndexSivg" + - "_$ss23_ContiguousArrayStorageCMn" + - "_$sypN" + - "_$sytWV" + - "_OBJC_CLASS_$_RCAVrpek" + - "_OBJC_CLASS_$_RCAVrpek_Vrpel" + - "_objc_allocWithZone" + - "_objc_msgSend" + - "_objc_release_x21" + - "_objc_release_x23" + - "_objc_release_x24" + - "_objc_release_x25" + - "_objc_release_x26" + - "_objc_retainAutoreleasedReturnValue" + - "_objc_retain_x21" + - "_swift_allocError" + - "_swift_allocObject" + - "_swift_beginAccess" + - "_swift_bridgeObjectRelease" + - "_swift_deallocObject" + - "_swift_endAccess" + - "_swift_getTypeByMangledNameInContext2" + - "_swift_getWitnessTable" + - "_swift_initStackObject" + - "_swift_release" + - "_swift_retain" + - "_swift_setDeallocating" + - "_swift_willThrow" \ No newline at end of file diff --git a/lib/src/teddy/generic.rs b/lib/src/teddy/generic.rs index 039cda66..fbe51611 100644 --- a/lib/src/teddy/generic.rs +++ b/lib/src/teddy/generic.rs @@ -1,34 +1,9 @@ -#![allow(dead_code)] #![allow(unused_variables)] use super::vector::{FatVector, Vector}; use std::collections::BTreeMap; use std::fmt::Debug; use std::sync::Arc; -pub(crate) trait Pointer { - unsafe fn distance(self, origin: Self) -> usize; - fn as_usize(self) -> usize; -} -impl Pointer for *const T { - #[inline(always)] - unsafe fn distance(self, origin: *const T) -> usize { - (self as usize).saturating_sub(origin as usize) - } - #[inline(always)] - fn as_usize(self) -> usize { - self as usize - } -} -impl Pointer for *mut T { - #[inline(always)] - unsafe fn distance(self, origin: *mut T) -> usize { - unsafe { (self as *const T).distance(origin as *const T) } - } - #[inline(always)] - fn as_usize(self) -> usize { - self as usize - } -} pub type PatternID = u32; #[derive(Clone, Debug)] @@ -181,13 +156,6 @@ impl Slim { } } - /// Returns the approximate total amount of heap used by this type, in - /// units of bytes. - #[inline(always)] - pub(crate) fn memory_usage(&self) -> usize { - self.teddy.memory_usage() - } - /// Returns the minimum length, in bytes, that a haystack must be in order /// to use it with this searcher. #[inline(always)] @@ -197,73 +165,6 @@ impl Slim { } impl Slim { - /// Look for an occurrences of the patterns in this finder in the haystack - /// given by the `start` and `end` pointers. - /// - /// If no match could be found, then `None` is returned. - /// - /// # Safety - /// - /// The given pointers representing the haystack must be valid to read - /// from. They must also point to a region of memory that is at least the - /// minimum length required by this searcher. - /// - /// Callers must ensure that this is okay to call in the current target for - /// the current CPU. - #[inline(always)] - pub(crate) unsafe fn find( - &self, - start: *const u8, - end: *const u8, - ) -> Option { - unsafe { - let len = end.distance(start); - debug_assert!(len >= self.minimum_len()); - let mut cur = start; - while cur <= end.sub(V::BYTES) { - if let Some(m) = self.find_one(cur, end) { - return Some(m); - } - cur = cur.add(V::BYTES); - } - if cur < end { - cur = end.sub(V::BYTES); - if let Some(m) = self.find_one(cur, end) { - return Some(m); - } - } - None - } - } - - /// Look for a match starting at the `V::BYTES` at and after `cur`. If - /// there isn't one, then `None` is returned. - /// - /// # Safety - /// - /// The given pointers representing the haystack must be valid to read - /// from. They must also point to a region of memory that is at least the - /// minimum length required by this searcher. - /// - /// Callers must ensure that this is okay to call in the current target for - /// the current CPU. - #[inline(always)] - unsafe fn find_one( - &self, - cur: *const u8, - end: *const u8, - ) -> Option { - unsafe { - let c = self.candidate(cur); - if !c.is_zero() - && let Some(m) = self.teddy.verify(cur, end, c) - { - return Some(m); - } - None - } - } - /// Look for a candidate match (represented as a vector) starting at the /// `V::BYTES` at and after `cur`. If there isn't one, then a vector with /// all bits set to zero is returned. @@ -282,57 +183,43 @@ impl Slim { Mask::members1(chunk, self.masks) } } -} -impl Slim { - /// See Slim::find. #[inline(always)] - pub(crate) unsafe fn find( + pub(crate) unsafe fn find_overlapping( &self, start: *const u8, end: *const u8, - ) -> Option { + callback: &mut dyn FnMut(Match), + ) { unsafe { - let len = end.distance(start); - debug_assert!(len >= self.minimum_len()); - let mut cur = start.add(1); - let mut prev0 = V::splat(0xFF); + if (end as usize - start as usize) < self.minimum_len() { + return; + } + let mut cur = start; while cur <= end.sub(V::BYTES) { - if let Some(m) = self.find_one(cur, end, &mut prev0) { - return Some(m); + let c = self.candidate(cur); + if !c.is_zero() { + self.teddy.verify_all(cur, end, c, callback); } cur = cur.add(V::BYTES); } if cur < end { + let prev_bound = cur; cur = end.sub(V::BYTES); - prev0 = V::splat(0xFF); - if let Some(m) = self.find_one(cur, end, &mut prev0) { - return Some(m); + let c = self.candidate(cur); + if !c.is_zero() { + self.teddy.verify_all(cur, end, c, &mut |m| { + if m.start >= prev_bound { + callback(m); + } + }); } } - None - } - } - - /// See Slim::find_one. - #[inline(always)] - unsafe fn find_one( - &self, - cur: *const u8, - end: *const u8, - prev0: &mut V, - ) -> Option { - unsafe { - let c = self.candidate(cur, prev0); - if !c.is_zero() - && let Some(m) = self.teddy.verify(cur.sub(1), end, c) - { - return Some(m); - } - None } } +} +impl Slim { /// See Slim::candidate. #[inline(always)] unsafe fn candidate(&self, cur: *const u8, prev0: &mut V) -> V { @@ -348,61 +235,6 @@ impl Slim { } impl Slim { - /// See Slim::find. - #[inline(always)] - pub(crate) unsafe fn find( - &self, - start: *const u8, - end: *const u8, - ) -> Option { - unsafe { - let len = end.distance(start); - debug_assert!(len >= self.minimum_len()); - let mut cur = start.add(2); - let mut prev0 = V::splat(0xFF); - let mut prev1 = V::splat(0xFF); - while cur <= end.sub(V::BYTES) { - if let Some(m) = - self.find_one(cur, end, &mut prev0, &mut prev1) - { - return Some(m); - } - cur = cur.add(V::BYTES); - } - if cur < end { - cur = end.sub(V::BYTES); - prev0 = V::splat(0xFF); - prev1 = V::splat(0xFF); - if let Some(m) = - self.find_one(cur, end, &mut prev0, &mut prev1) - { - return Some(m); - } - } - None - } - } - - /// See Slim::find_one. - #[inline(always)] - unsafe fn find_one( - &self, - cur: *const u8, - end: *const u8, - prev0: &mut V, - prev1: &mut V, - ) -> Option { - unsafe { - let c = self.candidate(cur, prev0, prev1); - if !c.is_zero() - && let Some(m) = self.teddy.verify(cur.sub(2), end, c) - { - return Some(m); - } - None - } - } - /// See Slim::candidate. #[inline(always)] unsafe fn candidate( @@ -425,64 +257,6 @@ impl Slim { } impl Slim { - /// See Slim::find. - #[inline(always)] - pub(crate) unsafe fn find( - &self, - start: *const u8, - end: *const u8, - ) -> Option { - unsafe { - let len = end.distance(start); - debug_assert!(len >= self.minimum_len()); - let mut cur = start.add(3); - let mut prev0 = V::splat(0xFF); - let mut prev1 = V::splat(0xFF); - let mut prev2 = V::splat(0xFF); - while cur <= end.sub(V::BYTES) { - if let Some(m) = - self.find_one(cur, end, &mut prev0, &mut prev1, &mut prev2) - { - return Some(m); - } - cur = cur.add(V::BYTES); - } - if cur < end { - cur = end.sub(V::BYTES); - prev0 = V::splat(0xFF); - prev1 = V::splat(0xFF); - prev2 = V::splat(0xFF); - if let Some(m) = - self.find_one(cur, end, &mut prev0, &mut prev1, &mut prev2) - { - return Some(m); - } - } - None - } - } - - /// See Slim::find_one. - #[inline(always)] - unsafe fn find_one( - &self, - cur: *const u8, - end: *const u8, - prev0: &mut V, - prev1: &mut V, - prev2: &mut V, - ) -> Option { - unsafe { - let c = self.candidate(cur, prev0, prev1, prev2); - if !c.is_zero() - && let Some(m) = self.teddy.verify(cur.sub(3), end, c) - { - return Some(m); - } - None - } - } - /// See Slim::candidate. #[inline(always)] unsafe fn candidate( @@ -544,13 +318,6 @@ impl Fat { } } - /// Returns the approximate total amount of heap used by this type, in - /// units of bytes. - #[inline(always)] - pub(crate) fn memory_usage(&self) -> usize { - self.teddy.memory_usage() - } - /// Returns the minimum length, in bytes, that a haystack must be in order /// to use it with this searcher. #[inline(always)] @@ -560,73 +327,6 @@ impl Fat { } impl Fat { - /// Look for an occurrences of the patterns in this finder in the haystack - /// given by the `start` and `end` pointers. - /// - /// If no match could be found, then `None` is returned. - /// - /// # Safety - /// - /// The given pointers representing the haystack must be valid to read - /// from. They must also point to a region of memory that is at least the - /// minimum length required by this searcher. - /// - /// Callers must ensure that this is okay to call in the current target for - /// the current CPU. - #[inline(always)] - pub(crate) unsafe fn find( - &self, - start: *const u8, - end: *const u8, - ) -> Option { - unsafe { - let len = end.distance(start); - debug_assert!(len >= self.minimum_len()); - let mut cur = start; - while cur <= end.sub(V::Half::BYTES) { - if let Some(m) = self.find_one(cur, end) { - return Some(m); - } - cur = cur.add(V::Half::BYTES); - } - if cur < end { - cur = end.sub(V::Half::BYTES); - if let Some(m) = self.find_one(cur, end) { - return Some(m); - } - } - None - } - } - - /// Look for a match starting at the `V::BYTES` at and after `cur`. If - /// there isn't one, then `None` is returned. - /// - /// # Safety - /// - /// The given pointers representing the haystack must be valid to read - /// from. They must also point to a region of memory that is at least the - /// minimum length required by this searcher. - /// - /// Callers must ensure that this is okay to call in the current target for - /// the current CPU. - #[inline(always)] - unsafe fn find_one( - &self, - cur: *const u8, - end: *const u8, - ) -> Option { - unsafe { - let c = self.candidate(cur); - if !c.is_zero() - && let Some(m) = self.teddy.verify(cur, end, c) - { - return Some(m); - } - None - } - } - /// Look for a candidate match (represented as a vector) starting at the /// `V::BYTES` at and after `cur`. If there isn't one, then a vector with /// all bits set to zero is returned. @@ -648,54 +348,6 @@ impl Fat { } impl Fat { - /// See `Fat::find`. - #[inline(always)] - pub(crate) unsafe fn find( - &self, - start: *const u8, - end: *const u8, - ) -> Option { - unsafe { - let len = end.distance(start); - debug_assert!(len >= self.minimum_len()); - let mut cur = start.add(1); - let mut prev0 = V::splat(0xFF); - while cur <= end.sub(V::Half::BYTES) { - if let Some(m) = self.find_one(cur, end, &mut prev0) { - return Some(m); - } - cur = cur.add(V::Half::BYTES); - } - if cur < end { - cur = end.sub(V::Half::BYTES); - prev0 = V::splat(0xFF); - if let Some(m) = self.find_one(cur, end, &mut prev0) { - return Some(m); - } - } - None - } - } - - /// See `Fat::find_one`. - #[inline(always)] - unsafe fn find_one( - &self, - cur: *const u8, - end: *const u8, - prev0: &mut V, - ) -> Option { - unsafe { - let c = self.candidate(cur, prev0); - if !c.is_zero() - && let Some(m) = self.teddy.verify(cur.sub(1), end, c) - { - return Some(m); - } - None - } - } - /// See `Fat::candidate`. #[inline(always)] unsafe fn candidate(&self, cur: *const u8, prev0: &mut V) -> V { @@ -711,61 +363,6 @@ impl Fat { } impl Fat { - /// See `Fat::find`. - #[inline(always)] - pub(crate) unsafe fn find( - &self, - start: *const u8, - end: *const u8, - ) -> Option { - unsafe { - let len = end.distance(start); - debug_assert!(len >= self.minimum_len()); - let mut cur = start.add(2); - let mut prev0 = V::splat(0xFF); - let mut prev1 = V::splat(0xFF); - while cur <= end.sub(V::Half::BYTES) { - if let Some(m) = - self.find_one(cur, end, &mut prev0, &mut prev1) - { - return Some(m); - } - cur = cur.add(V::Half::BYTES); - } - if cur < end { - cur = end.sub(V::Half::BYTES); - prev0 = V::splat(0xFF); - prev1 = V::splat(0xFF); - if let Some(m) = - self.find_one(cur, end, &mut prev0, &mut prev1) - { - return Some(m); - } - } - None - } - } - - /// See `Fat::find_one`. - #[inline(always)] - unsafe fn find_one( - &self, - cur: *const u8, - end: *const u8, - prev0: &mut V, - prev1: &mut V, - ) -> Option { - unsafe { - let c = self.candidate(cur, prev0, prev1); - if !c.is_zero() - && let Some(m) = self.teddy.verify(cur.sub(2), end, c) - { - return Some(m); - } - None - } - } - /// See `Fat::candidate`. #[inline(always)] unsafe fn candidate( @@ -788,64 +385,6 @@ impl Fat { } impl Fat { - /// See `Fat::find`. - #[inline(always)] - pub(crate) unsafe fn find( - &self, - start: *const u8, - end: *const u8, - ) -> Option { - unsafe { - let len = end.distance(start); - debug_assert!(len >= self.minimum_len()); - let mut cur = start.add(3); - let mut prev0 = V::splat(0xFF); - let mut prev1 = V::splat(0xFF); - let mut prev2 = V::splat(0xFF); - while cur <= end.sub(V::Half::BYTES) { - if let Some(m) = - self.find_one(cur, end, &mut prev0, &mut prev1, &mut prev2) - { - return Some(m); - } - cur = cur.add(V::Half::BYTES); - } - if cur < end { - cur = end.sub(V::Half::BYTES); - prev0 = V::splat(0xFF); - prev1 = V::splat(0xFF); - prev2 = V::splat(0xFF); - if let Some(m) = - self.find_one(cur, end, &mut prev0, &mut prev1, &mut prev2) - { - return Some(m); - } - } - None - } - } - - /// See `Fat::find_one`. - #[inline(always)] - unsafe fn find_one( - &self, - cur: *const u8, - end: *const u8, - prev0: &mut V, - prev1: &mut V, - prev2: &mut V, - ) -> Option { - unsafe { - let c = self.candidate(cur, prev0, prev1, prev2); - if !c.is_zero() - && let Some(m) = self.teddy.verify(cur.sub(3), end, c) - { - return Some(m); - } - None - } - } - /// See `Fat::candidate`. #[inline(always)] unsafe fn candidate( @@ -966,71 +505,6 @@ impl Teddy { t } - /// Verify whether there are any matches starting at or after `cur` in the - /// haystack. The candidate chunk given should correspond to 8-bit bitsets - /// for N buckets. - /// - /// # Safety - /// - /// The given pointers representing the haystack must be valid to read - /// from. - #[inline(always)] - unsafe fn verify64( - &self, - cur: *const u8, - end: *const u8, - mut candidate_chunk: u64, - ) -> Option { - unsafe { - while candidate_chunk != 0 { - let bit = candidate_chunk.trailing_zeros() as usize; - candidate_chunk &= !(1 << bit); - - let cur = cur.add(bit / BUCKETS); - let bucket = bit % BUCKETS; - if let Some(m) = self.verify_bucket(cur, end, bucket) { - return Some(m); - } - } - None - } - } - - /// Verify whether there are any matches starting at `at` in the given - /// `haystack` corresponding only to patterns in the given bucket. - /// - /// # Safety - /// - /// The given pointers representing the haystack must be valid to read - /// from. - /// - /// The bucket index must be less than or equal to `self.buckets.len()`. - #[inline(always)] - unsafe fn verify_bucket( - &self, - cur: *const u8, - end: *const u8, - bucket: usize, - ) -> Option { - unsafe { - debug_assert!(bucket < self.buckets.len()); - // SAFETY: The caller must ensure that the bucket index is correct. - for pid in self.buckets.get_unchecked(bucket).iter().copied() { - // SAFETY: This is safe because we are guaranteed that every - // index in a Teddy bucket is a valid index into `pats`, by - // construction. - debug_assert!((pid as usize) < self.patterns.len()); - let pat = self.patterns.get_unchecked(pid); - if pat.is_prefix_raw(cur, end) { - let start = cur; - let end = start.add(pat.len()); - return Some(Match { pid, start, end }); - } - } - None - } - } - /// Returns the total number of masks required by the patterns in this /// Teddy searcher. /// @@ -1045,122 +519,6 @@ impl Teddy { fn mask_len(&self) -> usize { core::cmp::min(4, self.patterns.minimum_len()) } - - /// Returns the approximate total amount of heap used by this type, in - /// units of bytes. - fn memory_usage(&self) -> usize { - // This is an upper bound rather than a precise accounting. No - // particular reason, other than it's probably very close to actual - // memory usage in practice. - self.patterns.len() * core::mem::size_of::() - } -} - -impl Teddy<8> { - /// Runs the verification routine for "slim" Teddy. - /// - /// The candidate given should be a collection of 8-bit bitsets (one bitset - /// per lane), where the ith bit is set in the jth lane if and only if the - /// byte occurring at `at + j` in `cur` is in the bucket `i`. - /// - /// # Safety - /// - /// Callers must ensure that this is okay to call in the current target for - /// the current CPU. - /// - /// The given pointers must be valid to read from. - #[inline(always)] - unsafe fn verify( - &self, - mut cur: *const u8, - end: *const u8, - candidate: V, - ) -> Option { - unsafe { - debug_assert!(!candidate.is_zero()); - // Convert the candidate into 64-bit chunks, and then verify each of - // those chunks. - candidate.for_each_64bit_lane( - #[inline(always)] - |_, chunk| { - let result = self.verify64(cur, end, chunk); - cur = cur.add(8); - result - }, - ) - } - } -} - -impl Teddy<16> { - /// Runs the verification routine for "fat" Teddy. - /// - /// The candidate given should be a collection of 8-bit bitsets (one bitset - /// per lane), where the ith bit is set in the jth lane if and only if the - /// byte occurring at `at + (j < 16 ? j : j - 16)` in `cur` is in the - /// bucket `j < 16 ? i : i + 8`. - /// - /// # Safety - /// - /// Callers must ensure that this is okay to call in the current target for - /// the current CPU. - /// - /// The given pointers must be valid to read from. - #[inline(always)] - unsafe fn verify( - &self, - mut cur: *const u8, - end: *const u8, - candidate: V, - ) -> Option { - unsafe { - // This is a bit tricky, but we basically want to convert our - // candidate, which looks like this (assuming a 256-bit vector): - // - // a31 a30 ... a17 a16 a15 a14 ... a01 a00 - // - // where each a(i) is an 8-bit bitset corresponding to the activated - // buckets, to this - // - // a31 a15 a30 a14 a29 a13 ... a18 a02 a17 a01 a16 a00 - // - // Namely, for Fat Teddy, the high 128-bits of the candidate correspond - // to the same bytes in the haystack in the low 128-bits (so we only - // scan 16 bytes at a time), but are for buckets 8-15 instead of 0-7. - // - // The verification routine wants to look at all potentially matching - // buckets before moving on to the next lane. So for example, both - // a16 and a00 both correspond to the first byte in our window; a00 - // contains buckets 0-7 and a16 contains buckets 8-15. Specifically, - // a16 should be checked before a01. So the transformation shown above - // allows us to use our normal verification procedure with one small - // change: we treat each bitset as 16 bits instead of 8 bits. - debug_assert!(!candidate.is_zero()); - - // Swap the 128-bit lanes in the candidate vector. - let swapped = candidate.swap_halves(); - // Interleave the bytes from the low 128-bit lanes, starting with - // cand first. - let r1 = candidate.interleave_low_8bit_lanes(swapped); - // Interleave the bytes from the high 128-bit lanes, starting with - // cand first. - let r2 = candidate.interleave_high_8bit_lanes(swapped); - // Now just take the 2 low 64-bit integers from both r1 and r2. We - // can drop the high 64-bit integers because they are a mirror image - // of the low 64-bit integers. All we care about are the low 128-bit - // lanes of r1 and r2. Combined, they contain all our 16-bit bitsets - // laid out in the desired order, as described above. - r1.for_each_low_64bit_lane( - r2, - #[inline(always)] - |_, chunk| { - let result = self.verify64(cur, end, chunk); - cur = cur.add(4); - result - }, - ) - } - } } /// A vector generic mask for the low and high nybbles in a set of patterns. @@ -1641,42 +999,6 @@ impl Teddy<16> { } } -impl Slim { - #[inline(always)] - pub(crate) unsafe fn find_overlapping( - &self, - start: *const u8, - end: *const u8, - callback: &mut dyn FnMut(Match), - ) { - unsafe { - if (end as usize - start as usize) < self.minimum_len() { - return; - } - let mut cur = start; - while cur <= end.sub(V::BYTES) { - let c = self.candidate(cur); - if !c.is_zero() { - self.teddy.verify_all(cur, end, c, callback); - } - cur = cur.add(V::BYTES); - } - if cur < end { - let prev_bound = cur; - cur = end.sub(V::BYTES); - let c = self.candidate(cur); - if !c.is_zero() { - self.teddy.verify_all(cur, end, c, &mut |m| { - if m.start >= prev_bound { - callback(m); - } - }); - } - } - } - } -} - impl Slim { #[inline(always)] pub(crate) unsafe fn find_overlapping( @@ -1962,27 +1284,6 @@ mod tests { use super::*; use std::sync::Arc; - #[test] - fn pointer_trait() { - let arr = [0u8; 10]; - let p1 = &arr[0] as *const u8; - let p2 = &arr[5] as *const u8; - unsafe { - assert_eq!(p2.distance(p1), 5); - assert_eq!(p1.distance(p2), 0); - } - assert_eq!(p1.as_usize(), p1 as usize); - - let mut arr_mut = [0u8; 10]; - let m1 = &mut arr_mut[0] as *mut u8; - let m2 = &mut arr_mut[5] as *mut u8; - unsafe { - assert_eq!(m2.distance(m1), 5); - assert_eq!(m1.distance(m2), 0); - } - assert_eq!(m1.as_usize(), m1 as usize); - } - #[test] fn patterns_and_pattern() { let by_id = vec![b"abc".to_vec(), b"d".to_vec()]; @@ -2074,7 +1375,6 @@ mod tests { }); let teddy_8 = Teddy::<8>::new(pats.clone()); assert_eq!(teddy_8.mask_len(), 3); - assert!(teddy_8.memory_usage() > 0); let teddy_16 = Teddy::<16>::new(pats); assert_eq!(teddy_16.mask_len(), 3); @@ -2105,34 +1405,6 @@ mod tests { let _ = Teddy::<4>::new(pats); } - #[test] - fn teddy_verify64() { - let pats = Arc::new(Patterns { - by_id: vec![b"abc".to_vec()], - minimum_len: 3, - }); - let teddy = Teddy::<8>::new(pats); - let haystack = b"abcdefg"; - let start = haystack.as_ptr(); - let end = unsafe { start.add(haystack.len()) }; - - let candidate_chunk = 1 << 7; - unsafe { - let m = teddy.verify64(start, end, candidate_chunk); - assert!(m.is_some()); - let m = m.unwrap(); - assert_eq!(m.pattern(), 0); - assert_eq!(m.start(), start); - assert_eq!(m.end(), start.add(3)); - } - - let candidate_chunk_no_match = 1 << 0; - unsafe { - let m = teddy.verify64(start, end, candidate_chunk_no_match); - assert!(m.is_none()); - } - } - #[test] fn teddy_verify64_all() { let pats = Arc::new(Patterns { @@ -2172,29 +1444,31 @@ mod tests { minimum_len: 4, }); unsafe { + macro_rules! assert_finds { + ($s:expr, $start:expr, $end:expr) => {{ + let mut found = false; + $s.find_overlapping($start, $end, &mut |_| { + found = true + }); + assert!(found); + }}; + } + let s1 = Slim::<__m128i, 1>::new(pats.clone()); let s2 = Slim::<__m128i, 2>::new(pats.clone()); let s3 = Slim::<__m128i, 3>::new(pats.clone()); let s4 = Slim::<__m128i, 4>::new(pats.clone()); - assert!(s1.memory_usage() > 0); assert!(s1.minimum_len() >= 4); let haystack = pad64(b"xyzabcdefgh"); let start = haystack.as_ptr(); let end = start.add(haystack.len()); - assert!(s1.find(start, end).is_some()); - s1.find_overlapping(start, end, &mut |_| {}); - - assert!(s2.find(start, end).is_some()); - s2.find_overlapping(start, end, &mut |_| {}); - - assert!(s3.find(start, end).is_some()); - s3.find_overlapping(start, end, &mut |_| {}); - - assert!(s4.find(start, end).is_some()); - s4.find_overlapping(start, end, &mut |_| {}); + assert_finds!(s1, start, end); + assert_finds!(s2, start, end); + assert_finds!(s3, start, end); + assert_finds!(s4, start, end); let short_haystack = b"abc"; let s_start = short_haystack.as_ptr(); @@ -2209,15 +1483,10 @@ mod tests { let t_start = tail_haystack.as_ptr(); let t_end = t_start.add(tail_haystack.len()); - assert!(s1.find(t_start, t_end).is_some()); - assert!(s2.find(t_start, t_end).is_some()); - assert!(s3.find(t_start, t_end).is_some()); - assert!(s4.find(t_start, t_end).is_some()); - - s1.find_overlapping(t_start, t_end, &mut |_| {}); - s2.find_overlapping(t_start, t_end, &mut |_| {}); - s3.find_overlapping(t_start, t_end, &mut |_| {}); - s4.find_overlapping(t_start, t_end, &mut |_| {}); + assert_finds!(s1, t_start, t_end); + assert_finds!(s2, t_start, t_end); + assert_finds!(s3, t_start, t_end); + assert_finds!(s4, t_start, t_end); if std::is_x86_feature_detected!("avx2") { let f1 = Fat::<__m256i, 1>::new(pats.clone()); @@ -2225,35 +1494,22 @@ mod tests { let f3 = Fat::<__m256i, 3>::new(pats.clone()); let f4 = Fat::<__m256i, 4>::new(pats.clone()); - assert!(f1.memory_usage() > 0); assert!(f1.minimum_len() >= 4); - assert!(f1.find(start, end).is_some()); - f1.find_overlapping(start, end, &mut |_| {}); - - assert!(f2.find(start, end).is_some()); - f2.find_overlapping(start, end, &mut |_| {}); - - assert!(f3.find(start, end).is_some()); - f3.find_overlapping(start, end, &mut |_| {}); - - assert!(f4.find(start, end).is_some()); - f4.find_overlapping(start, end, &mut |_| {}); + assert_finds!(f1, start, end); + assert_finds!(f2, start, end); + assert_finds!(f3, start, end); + assert_finds!(f4, start, end); f1.find_overlapping(s_start, s_end, &mut |_| {}); f2.find_overlapping(s_start, s_end, &mut |_| {}); f3.find_overlapping(s_start, s_end, &mut |_| {}); f4.find_overlapping(s_start, s_end, &mut |_| {}); - assert!(f1.find(t_start, t_end).is_some()); - assert!(f2.find(t_start, t_end).is_some()); - assert!(f3.find(t_start, t_end).is_some()); - assert!(f4.find(t_start, t_end).is_some()); - - f1.find_overlapping(t_start, t_end, &mut |_| {}); - f2.find_overlapping(t_start, t_end, &mut |_| {}); - f3.find_overlapping(t_start, t_end, &mut |_| {}); - f4.find_overlapping(t_start, t_end, &mut |_| {}); + assert_finds!(f1, t_start, t_end); + assert_finds!(f2, t_start, t_end); + assert_finds!(f3, t_start, t_end); + assert_finds!(f4, t_start, t_end); } } } @@ -2281,29 +1537,31 @@ mod tests { minimum_len: 4, }); unsafe { + macro_rules! assert_finds { + ($s:expr, $start:expr, $end:expr) => {{ + let mut found = false; + $s.find_overlapping($start, $end, &mut |_| { + found = true + }); + assert!(found); + }}; + } + let s1 = Slim::::new(pats.clone()); let s2 = Slim::::new(pats.clone()); let s3 = Slim::::new(pats.clone()); let s4 = Slim::::new(pats.clone()); - assert!(s1.memory_usage() > 0); assert!(s1.minimum_len() >= 4); let haystack = pad64(b"xyzabcdefgh"); let start = haystack.as_ptr(); let end = start.add(haystack.len()); - assert!(s1.find(start, end).is_some()); - s1.find_overlapping(start, end, &mut |_| {}); - - assert!(s2.find(start, end).is_some()); - s2.find_overlapping(start, end, &mut |_| {}); - - assert!(s3.find(start, end).is_some()); - s3.find_overlapping(start, end, &mut |_| {}); - - assert!(s4.find(start, end).is_some()); - s4.find_overlapping(start, end, &mut |_| {}); + assert_finds!(s1, start, end); + assert_finds!(s2, start, end); + assert_finds!(s3, start, end); + assert_finds!(s4, start, end); let short_haystack = b"abc"; let s_start = short_haystack.as_ptr(); @@ -2318,15 +1576,10 @@ mod tests { let t_start = tail_haystack.as_ptr(); let t_end = t_start.add(tail_haystack.len()); - assert!(s1.find(t_start, t_end).is_some()); - assert!(s2.find(t_start, t_end).is_some()); - assert!(s3.find(t_start, t_end).is_some()); - assert!(s4.find(t_start, t_end).is_some()); - - s1.find_overlapping(t_start, t_end, &mut |_| {}); - s2.find_overlapping(t_start, t_end, &mut |_| {}); - s3.find_overlapping(t_start, t_end, &mut |_| {}); - s4.find_overlapping(t_start, t_end, &mut |_| {}); + assert_finds!(s1, t_start, t_end); + assert_finds!(s2, t_start, t_end); + assert_finds!(s3, t_start, t_end); + assert_finds!(s4, t_start, t_end); } } } diff --git a/ls/src/configuration.rs b/ls/src/configuration.rs index c9671ee5..6ea4cf52 100644 --- a/ls/src/configuration.rs +++ b/ls/src/configuration.rs @@ -17,7 +17,7 @@ pub struct Config { /// This structure represents settings for the YARA-X formatter. #[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] +#[serde(rename_all = "camelCase", default)] pub(crate) struct FormattingConfiguration { pub align_metadata: bool, pub align_patterns: bool, @@ -61,3 +61,56 @@ pub struct MetadataValidationRule { #[serde(default)] pub regex: Option, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_default_config() { + let config = Config::default(); + assert!(config.code_formatting.align_metadata); + assert!(config.code_formatting.align_patterns); + assert!(config.code_formatting.indent_section_headers); + assert!(config.code_formatting.indent_section_contents); + assert!(!config.code_formatting.newline_before_curly_brace); + assert!(!config.code_formatting.empty_line_before_section_header); + assert!(!config.code_formatting.empty_line_after_section_header); + assert!(config.metadata_validation.is_empty()); + assert!(config.rule_name_validation.is_none()); + assert!(!config.cache_workspace); + } + + #[test] + fn test_deserialize_config() { + let json = r#"{ + "codeFormatting": { + "alignMetadata": false, + "newlineBeforeCurlyBrace": true + }, + "metadataValidation": [ + { + "identifier": "author", + "required": true, + "type": "string", + "regex": "^[A-Za-z ]+$" + } + ], + "ruleNameValidation": "^[a-z_]+$", + "cacheWorkspace": true + }"#; + + let config: Config = serde_json::from_str(json).unwrap(); + assert!(!config.code_formatting.align_metadata); + assert!(config.code_formatting.align_patterns); // default is true + assert!(config.code_formatting.newline_before_curly_brace); + assert_eq!(config.metadata_validation.len(), 1); + let rule = &config.metadata_validation[0]; + assert_eq!(rule.identifier, "author"); + assert!(rule.required); + assert_eq!(rule.ty.as_deref(), Some("string")); + assert_eq!(rule.regex.as_deref(), Some("^[A-Za-z ]+$")); + assert_eq!(config.rule_name_validation.as_deref(), Some("^[a-z_]+$")); + assert!(config.cache_workspace); + } +} diff --git a/ls/src/documents/document.rs b/ls/src/documents/document.rs index 7159a596..e58cbd0b 100644 --- a/ls/src/documents/document.rs +++ b/ls/src/documents/document.rs @@ -37,3 +37,31 @@ impl Document { self.text = text; } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_document_new_and_update() { + let uri = Url::parse("file:///test.yar").unwrap(); + let mut doc = Document::new( + uri.clone(), + "rule r1 { condition: true }".to_string(), + ); + + assert_eq!(doc.uri, uri); + assert_eq!(doc.text, "rule r1 { condition: true }"); + assert_eq!( + doc.cst.root().kind(), + yara_x_parser::cst::SyntaxKind::SOURCE_FILE + ); + + doc.update("rule r2 { condition: false }".to_string()); + assert_eq!(doc.text, "rule r2 { condition: false }"); + assert_eq!( + doc.cst.root().kind(), + yara_x_parser::cst::SyntaxKind::SOURCE_FILE + ); + } +} diff --git a/ls/src/documents/storage.rs b/ls/src/documents/storage.rs index cd1390fd..97def547 100644 --- a/ls/src/documents/storage.rs +++ b/ls/src/documents/storage.rs @@ -401,3 +401,129 @@ impl DocumentStorage { } } } + +#[cfg(test)] +mod tests { + use super::*; + use yara_x_parser::cst::NodeOrToken; + + #[test] + fn test_storage_basic_and_caching() { + let storage = DocumentStorage::new(); + let uri = Url::parse("file:///project/test.yar").unwrap(); + + storage.insert(uri.clone(), "rule r1 { condition: true }".to_string()); + assert!(storage.get(&uri).is_some()); + + storage + .update(uri.clone(), "rule r1 { condition: false }".to_string()); + assert_eq!( + storage.get(&uri).unwrap().text, + "rule r1 { condition: false }" + ); + + // Remove with cache enabled + storage.remove(&uri, true); + assert!(storage.get(&uri).is_none()); + assert!(storage.cached.contains_key(&uri)); + + // Reinsert should pick up from cache + storage.insert(uri.clone(), "rule r1 { condition: true }".to_string()); + assert!(storage.get(&uri).is_some()); + assert!(!storage.cached.contains_key(&uri)); + + // Remove without cache + storage.remove(&uri, false); + assert!(storage.get(&uri).is_none()); + assert!(!storage.cached.contains_key(&uri)); + } + + #[test] + fn test_find_rule_definition_and_occurrences_in_storage() { + let storage = DocumentStorage::new(); + let uri1 = Url::parse("file:///project/base.yar").unwrap(); + let uri2 = Url::parse("file:///project/main.yar").unwrap(); + + storage.insert( + uri1.clone(), + "rule base_rule { condition: true }".to_string(), + ); + storage.insert( + uri2.clone(), + "include \"base.yar\"\nrule main_rule { condition: base_rule }" + .to_string(), + ); + + let main_doc = storage.get(&uri2).unwrap(); + let mut stack = vec![NodeOrToken::Node(main_doc.cst.root())]; + let mut base_ident = None; + while let Some(nt) = stack.pop() { + match nt { + NodeOrToken::Node(n) => stack.extend(n.children_with_tokens()), + NodeOrToken::Token(t) => { + if t.text() == "base_rule" { + base_ident = Some(t); + break; + } + } + } + } + let base_ident = base_ident.unwrap(); + + drop(main_doc); // drop Ref before calling storage methods + + let def = storage.find_rule_definition(&uri2, &base_ident); + assert!(def.is_some()); + let (node, def_uri) = def.unwrap(); + assert_eq!(def_uri, uri1); + assert_eq!(node.kind(), SyntaxKind::RULE_DECL); + + let occ = storage.find_rule_occurrences(&uri2, &base_ident).unwrap(); + assert_eq!(occ.definition.0, uri1); + assert!(occ.usages.contains_key(&uri2)); + assert_eq!(occ.usages[&uri2].len(), 1); + } + + #[test] + fn test_included_rules() { + let storage = DocumentStorage::new(); + let uri1 = Url::parse("file:///project/inc.yar").unwrap(); + let uri2 = Url::parse("file:///project/main.yar").unwrap(); + + storage.insert( + uri1.clone(), + "rule inc_rule { condition: true }".to_string(), + ); + storage.insert( + uri2.clone(), + "include \"inc.yar\"\nrule main_rule { condition: true }" + .to_string(), + ); + + let main_doc = storage.get(&uri2).unwrap(); + let root = main_doc.cst.root(); + drop(main_doc); + + let included = storage.included_rules(root, &uri2); + assert_eq!(included.len(), 1); + assert_eq!(included[0].0, "inc.yar"); + assert_eq!(included[0].1.text(), "inc_rule"); + } + + #[cfg(not(target_family = "wasm"))] + #[test] + fn test_react_watched_files_changes() { + let storage = DocumentStorage::new(); + let uri = Url::parse("file:///project/deleted.yar").unwrap(); + storage + .cached + .insert(uri.clone(), CST::from("rule test { condition: true }")); + + storage.react_watched_files_changes(vec![FileEvent { + uri: uri.clone(), + typ: FileChangeType::DELETED, + }]); + + assert!(!storage.cached.contains_key(&uri)); + } +} diff --git a/ls/src/features/diagnostics.rs b/ls/src/features/diagnostics.rs index 93d4ad44..63baf3ce 100644 --- a/ls/src/features/diagnostics.rs +++ b/ls/src/features/diagnostics.rs @@ -196,8 +196,12 @@ pub fn compiler_diagnostics( ), )); } - _ => {} + _ => { + compiler.add_linter(linter); + } }; + } else { + compiler.add_linter(linter); } } diff --git a/ls/src/lib.rs b/ls/src/lib.rs index 96689c60..bc1db065 100644 --- a/ls/src/lib.rs +++ b/ls/src/lib.rs @@ -14,7 +14,7 @@ mod features; mod server; mod tracing; mod utils; -#[cfg(target_family = "wasm")] +#[cfg(any(target_family = "wasm", test))] mod worker; #[cfg(test)] diff --git a/ls/src/tests/mod.rs b/ls/src/tests/mod.rs index 5ab41ce6..96866e14 100644 --- a/ls/src/tests/mod.rs +++ b/ls/src/tests/mod.rs @@ -32,7 +32,7 @@ use crate::server::YARALanguageServer; struct ClientState; -async fn lsp_test(f: F) +async fn lsp_test(initialization_options: Option, f: F) where R: Future, F: Fn(ServerSocket) -> R, @@ -81,6 +81,7 @@ where }), ..Default::default() }, + initialization_options, ..Default::default() }) .await @@ -124,7 +125,7 @@ async fn close_document>(s: &ServerSocket, path: P) { .expect("DidOpenTextDocument notification failed"); } -async fn test_lsp_request, R: Request>(path: P) +async fn lsp_request, R: Request>(path: P) where R::Result: serde::Serialize + serde::de::DeserializeOwned + Debug, { @@ -132,7 +133,17 @@ where let abs_path = path.canonicalize().unwrap(); let test_dir = Url::from_file_path(abs_path.parent().unwrap()).unwrap(); - lsp_test(async |server_socket| { + let config_path = path.with_extension("config.json"); + let initialization_options = if config_path.exists() { + Some( + serde_json::from_str(&fs::read_to_string(&config_path).unwrap()) + .unwrap(), + ) + } else { + None + }; + + lsp_test(initialization_options, async |server_socket| { open_document(&server_socket, &abs_path).await; let mut mint = goldenfile::Mint::new("."); @@ -236,178 +247,191 @@ fn sort_json(value: &mut Value) { #[tokio::test] async fn selection_range() { - test_lsp_request::<_, SelectionRangeRequest>("selectionrange1.yar").await; - test_lsp_request::<_, SelectionRangeRequest>("selectionrange2.yar").await; - test_lsp_request::<_, SelectionRangeRequest>("selectionrange3.yar").await; - test_lsp_request::<_, SelectionRangeRequest>("selectionrange4.yar").await; - test_lsp_request::<_, SelectionRangeRequest>("selectionrange5.yar").await; - test_lsp_request::<_, SelectionRangeRequest>("selectionrange6.yar").await; - test_lsp_request::<_, SelectionRangeRequest>("selectionrange7.yar").await; + lsp_request::<_, SelectionRangeRequest>("selectionrange1.yar").await; + lsp_request::<_, SelectionRangeRequest>("selectionrange2.yar").await; + lsp_request::<_, SelectionRangeRequest>("selectionrange3.yar").await; + lsp_request::<_, SelectionRangeRequest>("selectionrange4.yar").await; + lsp_request::<_, SelectionRangeRequest>("selectionrange5.yar").await; + lsp_request::<_, SelectionRangeRequest>("selectionrange6.yar").await; + lsp_request::<_, SelectionRangeRequest>("selectionrange7.yar").await; } #[tokio::test] async fn rename() { - test_lsp_request::<_, Rename>("rename1.yar").await; - test_lsp_request::<_, Rename>("rename2.yar").await; - test_lsp_request::<_, Rename>("rename3.yar").await; - test_lsp_request::<_, Rename>("rename4.yar").await; - test_lsp_request::<_, Rename>("rename5.yar").await; - test_lsp_request::<_, Rename>("rename6.yar").await; - test_lsp_request::<_, Rename>("rename7.yar").await; - test_lsp_request::<_, Rename>("rename8.yar").await; - test_lsp_request::<_, Rename>("rename9.yar").await; - test_lsp_request::<_, Rename>("rename10.yar").await; + lsp_request::<_, Rename>("rename1.yar").await; + lsp_request::<_, Rename>("rename2.yar").await; + lsp_request::<_, Rename>("rename3.yar").await; + lsp_request::<_, Rename>("rename4.yar").await; + lsp_request::<_, Rename>("rename5.yar").await; + lsp_request::<_, Rename>("rename6.yar").await; + lsp_request::<_, Rename>("rename7.yar").await; + lsp_request::<_, Rename>("rename8.yar").await; + lsp_request::<_, Rename>("rename9.yar").await; + lsp_request::<_, Rename>("rename10.yar").await; } #[tokio::test] async fn references() { - test_lsp_request::<_, References>("references1.yar").await; - test_lsp_request::<_, References>("references2.yar").await; - test_lsp_request::<_, References>("references3.yar").await; - test_lsp_request::<_, References>("references4.yar").await; - test_lsp_request::<_, References>("references5.yar").await; - test_lsp_request::<_, References>("references6.yar").await; - test_lsp_request::<_, References>("references7.yar").await; - test_lsp_request::<_, References>("references8.yar").await; - test_lsp_request::<_, References>("references9.yar").await; + lsp_request::<_, References>("references1.yar").await; + lsp_request::<_, References>("references2.yar").await; + lsp_request::<_, References>("references3.yar").await; + lsp_request::<_, References>("references4.yar").await; + lsp_request::<_, References>("references5.yar").await; + lsp_request::<_, References>("references6.yar").await; + lsp_request::<_, References>("references7.yar").await; + lsp_request::<_, References>("references8.yar").await; + lsp_request::<_, References>("references9.yar").await; } #[tokio::test] async fn goto_definition() { - test_lsp_request::<_, GotoDefinition>("goto1.yar").await; - test_lsp_request::<_, GotoDefinition>("goto2.yar").await; - test_lsp_request::<_, GotoDefinition>("goto3.yar").await; - test_lsp_request::<_, GotoDefinition>("goto4.yar").await; - test_lsp_request::<_, GotoDefinition>("goto5.yar").await; - test_lsp_request::<_, GotoDefinition>("goto6.yar").await; - test_lsp_request::<_, GotoDefinition>("goto7.yar").await; - test_lsp_request::<_, GotoDefinition>("goto8.yar").await; + lsp_request::<_, GotoDefinition>("goto1.yar").await; + lsp_request::<_, GotoDefinition>("goto2.yar").await; + lsp_request::<_, GotoDefinition>("goto3.yar").await; + lsp_request::<_, GotoDefinition>("goto4.yar").await; + lsp_request::<_, GotoDefinition>("goto5.yar").await; + lsp_request::<_, GotoDefinition>("goto6.yar").await; + lsp_request::<_, GotoDefinition>("goto7.yar").await; + lsp_request::<_, GotoDefinition>("goto8.yar").await; } #[tokio::test] async fn hover() { - test_lsp_request::<_, HoverRequest>("hover1.yar").await; - test_lsp_request::<_, HoverRequest>("hover2.yar").await; - test_lsp_request::<_, HoverRequest>("hover3.yar").await; - test_lsp_request::<_, HoverRequest>("hover4.yar").await; - test_lsp_request::<_, HoverRequest>("hover5.yar").await; - test_lsp_request::<_, HoverRequest>("hover6.yar").await; - test_lsp_request::<_, HoverRequest>("hover7.yar").await; - test_lsp_request::<_, HoverRequest>("hover8.yar").await; - test_lsp_request::<_, HoverRequest>("hover9.yar").await; - test_lsp_request::<_, HoverRequest>("hover10.yar").await; - test_lsp_request::<_, HoverRequest>("hover11.yar").await; + lsp_request::<_, HoverRequest>("hover1.yar").await; + lsp_request::<_, HoverRequest>("hover2.yar").await; + lsp_request::<_, HoverRequest>("hover3.yar").await; + lsp_request::<_, HoverRequest>("hover4.yar").await; + lsp_request::<_, HoverRequest>("hover5.yar").await; + lsp_request::<_, HoverRequest>("hover6.yar").await; + lsp_request::<_, HoverRequest>("hover7.yar").await; + lsp_request::<_, HoverRequest>("hover8.yar").await; + lsp_request::<_, HoverRequest>("hover9.yar").await; + lsp_request::<_, HoverRequest>("hover10.yar").await; + lsp_request::<_, HoverRequest>("hover11.yar").await; + lsp_request::<_, HoverRequest>("hover12.yar").await; } #[tokio::test] async fn document_symbols() { - test_lsp_request::<_, DocumentSymbolRequest>("symbols1.yar").await; - test_lsp_request::<_, DocumentSymbolRequest>("symbols2.yar").await; - test_lsp_request::<_, DocumentSymbolRequest>("symbols3.yar").await; - test_lsp_request::<_, DocumentSymbolRequest>("symbols4.yar").await; - test_lsp_request::<_, DocumentSymbolRequest>("symbols5.yar").await; - test_lsp_request::<_, DocumentSymbolRequest>("symbols6.yar").await; - test_lsp_request::<_, DocumentSymbolRequest>("symbols7.yar").await; + lsp_request::<_, DocumentSymbolRequest>("symbols1.yar").await; + lsp_request::<_, DocumentSymbolRequest>("symbols2.yar").await; + lsp_request::<_, DocumentSymbolRequest>("symbols3.yar").await; + lsp_request::<_, DocumentSymbolRequest>("symbols4.yar").await; + lsp_request::<_, DocumentSymbolRequest>("symbols5.yar").await; + lsp_request::<_, DocumentSymbolRequest>("symbols6.yar").await; + lsp_request::<_, DocumentSymbolRequest>("symbols7.yar").await; + lsp_request::<_, DocumentSymbolRequest>("symbols8.yar").await; } #[tokio::test] async fn document_highlights() { - test_lsp_request::<_, DocumentHighlightRequest>("highlights1.yar").await; - test_lsp_request::<_, DocumentHighlightRequest>("highlights2.yar").await; - test_lsp_request::<_, DocumentHighlightRequest>("highlights3.yar").await; - test_lsp_request::<_, DocumentHighlightRequest>("highlights4.yar").await; - test_lsp_request::<_, DocumentHighlightRequest>("highlights5.yar").await; - test_lsp_request::<_, DocumentHighlightRequest>("highlights6.yar").await; - test_lsp_request::<_, DocumentHighlightRequest>("highlights7.yar").await; - test_lsp_request::<_, DocumentHighlightRequest>("highlights8.yar").await; - test_lsp_request::<_, DocumentHighlightRequest>("highlights9.yar").await; + lsp_request::<_, DocumentHighlightRequest>("highlights1.yar").await; + lsp_request::<_, DocumentHighlightRequest>("highlights2.yar").await; + lsp_request::<_, DocumentHighlightRequest>("highlights3.yar").await; + lsp_request::<_, DocumentHighlightRequest>("highlights4.yar").await; + lsp_request::<_, DocumentHighlightRequest>("highlights5.yar").await; + lsp_request::<_, DocumentHighlightRequest>("highlights6.yar").await; + lsp_request::<_, DocumentHighlightRequest>("highlights7.yar").await; + lsp_request::<_, DocumentHighlightRequest>("highlights8.yar").await; + lsp_request::<_, DocumentHighlightRequest>("highlights9.yar").await; } #[tokio::test] async fn document_diagnostics() { - test_lsp_request::<_, DocumentDiagnosticRequest>("diagnostics1.yar").await; - test_lsp_request::<_, DocumentDiagnosticRequest>("diagnostics2.yar").await; - test_lsp_request::<_, DocumentDiagnosticRequest>("diagnostics3.yar").await; - test_lsp_request::<_, DocumentDiagnosticRequest>("diagnostics4.yar").await; - test_lsp_request::<_, DocumentDiagnosticRequest>("diagnostics5.yar").await; - test_lsp_request::<_, DocumentDiagnosticRequest>("diagnostics6.yar").await; - test_lsp_request::<_, DocumentDiagnosticRequest>("diagnostics7.yar").await; - test_lsp_request::<_, DocumentDiagnosticRequest>("diagnostics8.yar").await; + lsp_request::<_, DocumentDiagnosticRequest>("diagnostics1.yar").await; + lsp_request::<_, DocumentDiagnosticRequest>("diagnostics2.yar").await; + lsp_request::<_, DocumentDiagnosticRequest>("diagnostics3.yar").await; + lsp_request::<_, DocumentDiagnosticRequest>("diagnostics4.yar").await; + lsp_request::<_, DocumentDiagnosticRequest>("diagnostics5.yar").await; + lsp_request::<_, DocumentDiagnosticRequest>("diagnostics6.yar").await; + lsp_request::<_, DocumentDiagnosticRequest>("diagnostics7.yar").await; + lsp_request::<_, DocumentDiagnosticRequest>("diagnostics8.yar").await; + lsp_request::<_, DocumentDiagnosticRequest>("diagnostics9.yar").await; + lsp_request::<_, DocumentDiagnosticRequest>("diagnostics10.yar").await; } #[tokio::test] async fn completion() { - test_lsp_request::<_, Completion>("completion1.yar").await; + lsp_request::<_, Completion>("completion1.yar").await; #[cfg(not(feature = "magic-module"))] - test_lsp_request::<_, Completion>("completion2.yar").await; + lsp_request::<_, Completion>("completion2.yar").await; - test_lsp_request::<_, Completion>("completion3.yar").await; - test_lsp_request::<_, Completion>("completion4.yar").await; - test_lsp_request::<_, Completion>("completion5.yar").await; - test_lsp_request::<_, Completion>("completion6.yar").await; - test_lsp_request::<_, Completion>("completion7.yar").await; + lsp_request::<_, Completion>("completion3.yar").await; + lsp_request::<_, Completion>("completion4.yar").await; + lsp_request::<_, Completion>("completion5.yar").await; + lsp_request::<_, Completion>("completion6.yar").await; + lsp_request::<_, Completion>("completion7.yar").await; - test_lsp_request::<_, Completion>("completion8.yar").await; + lsp_request::<_, Completion>("completion8.yar").await; - test_lsp_request::<_, Completion>("completion9.yar").await; + lsp_request::<_, Completion>("completion9.yar").await; #[cfg(not(feature = "magic-module"))] - test_lsp_request::<_, Completion>("completion10.yar").await; + lsp_request::<_, Completion>("completion10.yar").await; #[cfg(not(feature = "magic-module"))] - test_lsp_request::<_, Completion>("completion11.yar").await; + lsp_request::<_, Completion>("completion11.yar").await; #[cfg(not(feature = "magic-module"))] - test_lsp_request::<_, Completion>("completion12.yar").await; + lsp_request::<_, Completion>("completion12.yar").await; #[cfg(not(feature = "magic-module"))] - test_lsp_request::<_, Completion>("completion13.yar").await; + lsp_request::<_, Completion>("completion13.yar").await; #[cfg(not(feature = "magic-module"))] - test_lsp_request::<_, Completion>("completion14.yar").await; + lsp_request::<_, Completion>("completion14.yar").await; #[cfg(not(feature = "magic-module"))] - test_lsp_request::<_, Completion>("completion15.yar").await; + lsp_request::<_, Completion>("completion15.yar").await; - test_lsp_request::<_, Completion>("completion16.yar").await; - test_lsp_request::<_, Completion>("completion17.yar").await; + lsp_request::<_, Completion>("completion16.yar").await; + lsp_request::<_, Completion>("completion17.yar").await; + lsp_request::<_, Completion>("completion18.yar").await; + lsp_request::<_, Completion>("completion19.yar").await; + lsp_request::<_, Completion>("completion20.yar").await; + lsp_request::<_, Completion>("completion21.yar").await; + lsp_request::<_, Completion>("completion22.yar").await; + + #[cfg(not(feature = "magic-module"))] + lsp_request::<_, Completion>("completion23.yar").await; + + #[cfg(feature = "magic-module")] + lsp_request::<_, Completion>("completion24.yar").await; + + #[cfg(feature = "magic-module")] + lsp_request::<_, Completion>("completion25.yar").await; } #[tokio::test] async fn formatting() { - test_lsp_request::<_, Formatting>("formatting1.yar").await; + lsp_request::<_, Formatting>("formatting1.yar").await; } #[tokio::test] async fn code_action() { - test_lsp_request::<_, CodeActionRequest>("code_action.yar").await; + lsp_request::<_, CodeActionRequest>("code_action.yar").await; } #[tokio::test] async fn semantic_tokens() { - test_lsp_request::<_, SemanticTokensFullRequest>("semantic_tokens1.yar") - .await; - - test_lsp_request::<_, SemanticTokensFullRequest>("semantic_tokens2.yar") - .await; + lsp_request::<_, SemanticTokensFullRequest>("semantic_tokens1.yar").await; + lsp_request::<_, SemanticTokensFullRequest>("semantic_tokens2.yar").await; } #[tokio::test] async fn semantic_tokens_range() { - test_lsp_request::<_, SemanticTokensRangeRequest>( - "semantic_tokens_range.yar", - ) - .await; + lsp_request::<_, SemanticTokensRangeRequest>("semantic_tokens_range.yar") + .await; } #[tokio::test] async fn signature_help() { - test_lsp_request::<_, SignatureHelpRequest>("signature_help1.yar").await; - test_lsp_request::<_, SignatureHelpRequest>("signature_help2.yar").await; + lsp_request::<_, SignatureHelpRequest>("signature_help1.yar").await; + lsp_request::<_, SignatureHelpRequest>("signature_help2.yar").await; } #[tokio::test] async fn inlay_hint() { - test_lsp_request::<_, InlayHintRequest>("inlay_hint.yar").await; + lsp_request::<_, InlayHintRequest>("inlay_hint.yar").await; } diff --git a/ls/src/tests/testdata/completion18.request.json b/ls/src/tests/testdata/completion18.request.json new file mode 100644 index 00000000..a0fb36ed --- /dev/null +++ b/ls/src/tests/testdata/completion18.request.json @@ -0,0 +1,9 @@ +{ + "textDocument": { + "uri": "${test_dir}/completion18.yar" + }, + "position": { + "line": 5, + "character": 5 + } +} diff --git a/ls/src/tests/testdata/completion18.response.json b/ls/src/tests/testdata/completion18.response.json new file mode 100644 index 00000000..fc49f45d --- /dev/null +++ b/ls/src/tests/testdata/completion18.response.json @@ -0,0 +1,16 @@ +[ + { + "kind": 1, + "label": "my_str", + "labelDetails": { + "description": "Pattern" + } + }, + { + "kind": 1, + "label": "other", + "labelDetails": { + "description": "Pattern" + } + } +] \ No newline at end of file diff --git a/ls/src/tests/testdata/completion18.yar b/ls/src/tests/testdata/completion18.yar new file mode 100644 index 00000000..222c0104 --- /dev/null +++ b/ls/src/tests/testdata/completion18.yar @@ -0,0 +1,7 @@ +rule test_pattern_ident { + strings: + $my_str = "hello" + $other = "world" + condition: + $ +} diff --git a/ls/src/tests/testdata/completion19.request.json b/ls/src/tests/testdata/completion19.request.json new file mode 100644 index 00000000..ed0c3e31 --- /dev/null +++ b/ls/src/tests/testdata/completion19.request.json @@ -0,0 +1,9 @@ +{ + "textDocument": { + "uri": "${test_dir}/completion19.yar" + }, + "position": { + "line": 5, + "character": 5 + } +} diff --git a/ls/src/tests/testdata/completion19.response.json b/ls/src/tests/testdata/completion19.response.json new file mode 100644 index 00000000..fc49f45d --- /dev/null +++ b/ls/src/tests/testdata/completion19.response.json @@ -0,0 +1,16 @@ +[ + { + "kind": 1, + "label": "my_str", + "labelDetails": { + "description": "Pattern" + } + }, + { + "kind": 1, + "label": "other", + "labelDetails": { + "description": "Pattern" + } + } +] \ No newline at end of file diff --git a/ls/src/tests/testdata/completion19.yar b/ls/src/tests/testdata/completion19.yar new file mode 100644 index 00000000..58bac15a --- /dev/null +++ b/ls/src/tests/testdata/completion19.yar @@ -0,0 +1,7 @@ +rule test_pattern_count { + strings: + $my_str = "hello" + $other = "world" + condition: + # +} diff --git a/ls/src/tests/testdata/completion20.request.json b/ls/src/tests/testdata/completion20.request.json new file mode 100644 index 00000000..e52e60b9 --- /dev/null +++ b/ls/src/tests/testdata/completion20.request.json @@ -0,0 +1,9 @@ +{ + "textDocument": { + "uri": "${test_dir}/completion20.yar" + }, + "position": { + "line": 5, + "character": 5 + } +} diff --git a/ls/src/tests/testdata/completion20.response.json b/ls/src/tests/testdata/completion20.response.json new file mode 100644 index 00000000..fc49f45d --- /dev/null +++ b/ls/src/tests/testdata/completion20.response.json @@ -0,0 +1,16 @@ +[ + { + "kind": 1, + "label": "my_str", + "labelDetails": { + "description": "Pattern" + } + }, + { + "kind": 1, + "label": "other", + "labelDetails": { + "description": "Pattern" + } + } +] \ No newline at end of file diff --git a/ls/src/tests/testdata/completion20.yar b/ls/src/tests/testdata/completion20.yar new file mode 100644 index 00000000..6246909b --- /dev/null +++ b/ls/src/tests/testdata/completion20.yar @@ -0,0 +1,7 @@ +rule test_pattern_offset { + strings: + $my_str = "hello" + $other = "world" + condition: + @ +} diff --git a/ls/src/tests/testdata/completion21.request.json b/ls/src/tests/testdata/completion21.request.json new file mode 100644 index 00000000..8f665ae3 --- /dev/null +++ b/ls/src/tests/testdata/completion21.request.json @@ -0,0 +1,13 @@ +{ + "textDocument": { + "uri": "${test_dir}/completion21.yar" + }, + "position": { + "line": 2, + "character": 6 + }, + "context": { + "triggerKind": 2, + "triggerCharacter": "." + } +} diff --git a/ls/src/tests/testdata/completion21.response.json b/ls/src/tests/testdata/completion21.response.json new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/ls/src/tests/testdata/completion21.response.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/ls/src/tests/testdata/completion21.yar b/ls/src/tests/testdata/completion21.yar new file mode 100644 index 00000000..650a9dc9 --- /dev/null +++ b/ls/src/tests/testdata/completion21.yar @@ -0,0 +1,4 @@ +rule test_dot_no_keywords { + condition: + 1. +} diff --git a/ls/src/tests/testdata/completion22.request.json b/ls/src/tests/testdata/completion22.request.json new file mode 100644 index 00000000..3eac7f82 --- /dev/null +++ b/ls/src/tests/testdata/completion22.request.json @@ -0,0 +1,9 @@ +{ + "textDocument": { + "uri": "${test_dir}/completion22.yar" + }, + "position": { + "line": 2, + "character": 5 + } +} diff --git a/ls/src/tests/testdata/completion22.response.json b/ls/src/tests/testdata/completion22.response.json new file mode 100644 index 00000000..ca5ff4c5 --- /dev/null +++ b/ls/src/tests/testdata/completion22.response.json @@ -0,0 +1,74 @@ +[ + { + "kind": 14, + "label": "and" + }, + { + "kind": 14, + "label": "or" + }, + { + "kind": 14, + "label": "all" + }, + { + "kind": 14, + "label": "any" + }, + { + "kind": 14, + "label": "none" + }, + { + "kind": 14, + "label": "of" + }, + { + "insertText": "at ${1:expression}", + "insertTextFormat": 2, + "kind": 14, + "label": "at" + }, + { + "insertText": "in ${1:}..${2:}", + "insertTextFormat": 2, + "kind": 14, + "label": "in" + }, + { + "kind": 14, + "label": "filesize" + }, + { + "kind": 14, + "label": "entrypoint" + }, + { + "kind": 14, + "label": "true" + }, + { + "kind": 14, + "label": "false" + }, + { + "kind": 14, + "label": "not" + }, + { + "kind": 14, + "label": "defined" + }, + { + "insertText": "for ${1:quantifier} ${2:iterable} : ( ${3:expression} )", + "insertTextFormat": 2, + "kind": 14, + "label": "for" + }, + { + "insertText": "with ${1:declarations} : ( ${3:expression} )", + "insertTextFormat": 2, + "kind": 14, + "label": "with" + } +] \ No newline at end of file diff --git a/ls/src/tests/testdata/completion22.yar b/ls/src/tests/testdata/completion22.yar new file mode 100644 index 00000000..b696b07d --- /dev/null +++ b/ls/src/tests/testdata/completion22.yar @@ -0,0 +1,4 @@ +rule test_other_token { + condition: + ( +} diff --git a/ls/src/tests/testdata/completion23.request.json b/ls/src/tests/testdata/completion23.request.json new file mode 100644 index 00000000..2898425a --- /dev/null +++ b/ls/src/tests/testdata/completion23.request.json @@ -0,0 +1,9 @@ +{ + "textDocument": { + "uri": "${test_dir}/completion23.yar" + }, + "position": { + "line": 3, + "character": 9 + } +} diff --git a/ls/src/tests/testdata/completion23.response.json b/ls/src/tests/testdata/completion23.response.json new file mode 100644 index 00000000..acb28010 --- /dev/null +++ b/ls/src/tests/testdata/completion23.response.json @@ -0,0 +1,430 @@ +[ + { + "kind": 6, + "label": "rule_one", + "labelDetails": { + "description": "Rule" + } + }, + { + "kind": 6, + "label": "rule_two", + "labelDetails": { + "description": "Rule" + } + }, + { + "kind": 14, + "label": "and" + }, + { + "kind": 14, + "label": "or" + }, + { + "kind": 14, + "label": "all" + }, + { + "kind": 14, + "label": "any" + }, + { + "kind": 14, + "label": "none" + }, + { + "kind": 14, + "label": "of" + }, + { + "insertText": "at ${1:expression}", + "insertTextFormat": 2, + "kind": 14, + "label": "at" + }, + { + "insertText": "in ${1:}..${2:}", + "insertTextFormat": 2, + "kind": 14, + "label": "in" + }, + { + "kind": 14, + "label": "filesize" + }, + { + "kind": 14, + "label": "entrypoint" + }, + { + "kind": 14, + "label": "true" + }, + { + "kind": 14, + "label": "false" + }, + { + "kind": 14, + "label": "not" + }, + { + "kind": 14, + "label": "defined" + }, + { + "insertText": "for ${1:quantifier} ${2:iterable} : ( ${3:expression} )", + "insertTextFormat": 2, + "kind": 14, + "label": "for" + }, + { + "insertText": "with ${1:declarations} : ( ${3:expression} )", + "insertTextFormat": 2, + "kind": 14, + "label": "with" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"amo\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "amo" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"console\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "console" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"crx\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "crx" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"cuckoo\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "cuckoo" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"dex\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "dex" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"dotnet\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "dotnet" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"elf\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "elf" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"hash\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "hash" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"lnk\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "lnk" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"macho\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "macho" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"math\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "math" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"pe\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "pe" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"string\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "string" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"test_proto2\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "test_proto2" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"test_proto3\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "test_proto3" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"time\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "time" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"vt\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "vt" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"zip\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "zip" + } +] \ No newline at end of file diff --git a/ls/src/tests/testdata/completion23.yar b/ls/src/tests/testdata/completion23.yar new file mode 100644 index 00000000..2245dba0 --- /dev/null +++ b/ls/src/tests/testdata/completion23.yar @@ -0,0 +1,5 @@ +rule rule_one { condition: true } +rule rule_two { + condition: + rule_ +} diff --git a/ls/src/tests/testdata/completion24.request.json b/ls/src/tests/testdata/completion24.request.json new file mode 100644 index 00000000..dc55e49a --- /dev/null +++ b/ls/src/tests/testdata/completion24.request.json @@ -0,0 +1,9 @@ +{ + "textDocument": { + "uri": "${test_dir}/completion24.yar" + }, + "position": { + "line": 2, + "character": 7 + } +} diff --git a/ls/src/tests/testdata/completion24.response.json b/ls/src/tests/testdata/completion24.response.json new file mode 100644 index 00000000..9c25cd5a --- /dev/null +++ b/ls/src/tests/testdata/completion24.response.json @@ -0,0 +1,423 @@ +[ + { + "kind": 6, + "label": "test_magic_import", + "labelDetails": { + "description": "Rule" + } + }, + { + "kind": 14, + "label": "and" + }, + { + "kind": 14, + "label": "or" + }, + { + "kind": 14, + "label": "all" + }, + { + "kind": 14, + "label": "any" + }, + { + "kind": 14, + "label": "none" + }, + { + "kind": 14, + "label": "of" + }, + { + "insertText": "at ${1:expression}", + "insertTextFormat": 2, + "kind": 14, + "label": "at" + }, + { + "insertText": "in ${1:}..${2:}", + "insertTextFormat": 2, + "kind": 14, + "label": "in" + }, + { + "kind": 14, + "label": "filesize" + }, + { + "kind": 14, + "label": "entrypoint" + }, + { + "kind": 14, + "label": "true" + }, + { + "kind": 14, + "label": "false" + }, + { + "kind": 14, + "label": "not" + }, + { + "kind": 14, + "label": "defined" + }, + { + "insertText": "for ${1:quantifier} ${2:iterable} : ( ${3:expression} )", + "insertTextFormat": 2, + "kind": 14, + "label": "for" + }, + { + "insertText": "with ${1:declarations} : ( ${3:expression} )", + "insertTextFormat": 2, + "kind": 14, + "label": "with" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"console\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "console" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"crx\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "crx" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"cuckoo\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "cuckoo" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"dex\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "dex" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"dotnet\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "dotnet" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"elf\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "elf" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"hash\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "hash" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"lnk\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "lnk" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"macho\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "macho" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"magic\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "magic" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"math\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "math" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"pe\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "pe" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"string\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "string" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"test_proto2\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "test_proto2" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"test_proto3\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "test_proto3" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"time\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "time" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"vt\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "vt" + }, + { + "additionalTextEdits": [ + { + "newText": "import \"zip\"\n", + "range": { + "end": { + "character": 0, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + } + ], + "kind": 9, + "label": "zip" + } +] \ No newline at end of file diff --git a/ls/src/tests/testdata/completion24.yar b/ls/src/tests/testdata/completion24.yar new file mode 100644 index 00000000..d6500509 --- /dev/null +++ b/ls/src/tests/testdata/completion24.yar @@ -0,0 +1,4 @@ +rule test_magic_import { + condition: + mag +} diff --git a/ls/src/tests/testdata/completion25.request.json b/ls/src/tests/testdata/completion25.request.json new file mode 100644 index 00000000..59e98142 --- /dev/null +++ b/ls/src/tests/testdata/completion25.request.json @@ -0,0 +1,13 @@ +{ + "textDocument": { + "uri": "${test_dir}/completion25.yar" + }, + "position": { + "line": 4, + "character": 10 + }, + "context": { + "triggerKind": 2, + "triggerCharacter": "." + } +} diff --git a/ls/src/tests/testdata/completion25.response.json b/ls/src/tests/testdata/completion25.response.json new file mode 100644 index 00000000..2b4cd222 --- /dev/null +++ b/ls/src/tests/testdata/completion25.response.json @@ -0,0 +1,28 @@ +[ + { + "documentation": { + "kind": "markdown", + "value": "## `mime_type() -> string`\n\n Returns the MIME type provided by libmagic." + }, + "insertText": "mime_type()", + "insertTextFormat": 2, + "kind": 2, + "label": "mime_type()", + "labelDetails": { + "description": "func()" + } + }, + { + "documentation": { + "kind": "markdown", + "value": "## `type() -> string`\n\n Returns the file type provided by libmagic." + }, + "insertText": "type()", + "insertTextFormat": 2, + "kind": 2, + "label": "type()", + "labelDetails": { + "description": "func()" + } + } +] \ No newline at end of file diff --git a/ls/src/tests/testdata/completion25.yar b/ls/src/tests/testdata/completion25.yar new file mode 100644 index 00000000..baf6f341 --- /dev/null +++ b/ls/src/tests/testdata/completion25.yar @@ -0,0 +1,6 @@ +import "magic" + +rule test_magic_fields { + condition: + magic. +} diff --git a/ls/src/tests/testdata/diagnostics10.config.json b/ls/src/tests/testdata/diagnostics10.config.json new file mode 100644 index 00000000..a7bbfdc5 --- /dev/null +++ b/ls/src/tests/testdata/diagnostics10.config.json @@ -0,0 +1,34 @@ +{ + "metadataValidation": [ + { + "identifier": "description", + "type": "string" + }, + { + "identifier": "id", + "type": "string", + "regex": "^[A-Z]{3}-[0-9]{3}$" + }, + { + "identifier": "version", + "type": "integer" + }, + { + "identifier": "score", + "type": "float" + }, + { + "identifier": "enabled", + "type": "bool" + }, + { + "identifier": "created", + "type": "date" + }, + { + "identifier": "updated", + "type": "date", + "format": "%d/%m/%Y" + } + ] +} diff --git a/ls/src/tests/testdata/diagnostics10.request.json b/ls/src/tests/testdata/diagnostics10.request.json new file mode 100644 index 00000000..b2fe0b10 --- /dev/null +++ b/ls/src/tests/testdata/diagnostics10.request.json @@ -0,0 +1,5 @@ +{ + "textDocument": { + "uri": "${test_dir}/diagnostics10.yar" + } +} diff --git a/ls/src/tests/testdata/diagnostics10.response.json b/ls/src/tests/testdata/diagnostics10.response.json new file mode 100644 index 00000000..594f9e9b --- /dev/null +++ b/ls/src/tests/testdata/diagnostics10.response.json @@ -0,0 +1,257 @@ +{ + "items": [ + { + "code": "invalid_metadata", + "data": { + "patches": [] + }, + "message": "metadata `description` is not valid", + "range": { + "end": { + "character": 21, + "line": 2 + }, + "start": { + "character": 18, + "line": 2 + } + }, + "relatedInformation": [ + { + "location": { + "range": { + "end": { + "character": 21, + "line": 2 + }, + "start": { + "character": 18, + "line": 2 + } + }, + "uri": "${test_dir}/diagnostics10.yar" + }, + "message": "`description` must be a `string`" + } + ], + "severity": 2 + }, + { + "code": "invalid_metadata", + "data": { + "patches": [] + }, + "message": "metadata `id` is not valid", + "range": { + "end": { + "character": 17, + "line": 3 + }, + "start": { + "character": 9, + "line": 3 + } + }, + "relatedInformation": [ + { + "location": { + "range": { + "end": { + "character": 17, + "line": 3 + }, + "start": { + "character": 9, + "line": 3 + } + }, + "uri": "${test_dir}/diagnostics10.yar" + }, + "message": "`id` must be a string and match the pattern `^[A-Z]{3}-[0-9]{3}$`" + } + ], + "severity": 2 + }, + { + "code": "invalid_metadata", + "data": { + "patches": [] + }, + "message": "metadata `version` is not valid", + "range": { + "end": { + "character": 19, + "line": 4 + }, + "start": { + "character": 14, + "line": 4 + } + }, + "relatedInformation": [ + { + "location": { + "range": { + "end": { + "character": 19, + "line": 4 + }, + "start": { + "character": 14, + "line": 4 + } + }, + "uri": "${test_dir}/diagnostics10.yar" + }, + "message": "`version` must be a `integer`" + } + ], + "severity": 2 + }, + { + "code": "invalid_metadata", + "data": { + "patches": [] + }, + "message": "metadata `score` is not valid", + "range": { + "end": { + "character": 14, + "line": 5 + }, + "start": { + "character": 12, + "line": 5 + } + }, + "relatedInformation": [ + { + "location": { + "range": { + "end": { + "character": 14, + "line": 5 + }, + "start": { + "character": 12, + "line": 5 + } + }, + "uri": "${test_dir}/diagnostics10.yar" + }, + "message": "`score` must be a `float`" + } + ], + "severity": 2 + }, + { + "code": "invalid_metadata", + "data": { + "patches": [] + }, + "message": "metadata `enabled` is not valid", + "range": { + "end": { + "character": 19, + "line": 6 + }, + "start": { + "character": 14, + "line": 6 + } + }, + "relatedInformation": [ + { + "location": { + "range": { + "end": { + "character": 19, + "line": 6 + }, + "start": { + "character": 14, + "line": 6 + } + }, + "uri": "${test_dir}/diagnostics10.yar" + }, + "message": "`enabled` must be a `bool`" + } + ], + "severity": 2 + }, + { + "code": "invalid_metadata", + "data": { + "patches": [] + }, + "message": "metadata `created` is not valid", + "range": { + "end": { + "character": 26, + "line": 7 + }, + "start": { + "character": 14, + "line": 7 + } + }, + "relatedInformation": [ + { + "location": { + "range": { + "end": { + "character": 26, + "line": 7 + }, + "start": { + "character": 14, + "line": 7 + } + }, + "uri": "${test_dir}/diagnostics10.yar" + }, + "message": "`created` must be a `date` with format `%Y-%m-%d`" + } + ], + "severity": 2 + }, + { + "code": "invalid_metadata", + "data": { + "patches": [] + }, + "message": "metadata `updated` is not valid", + "range": { + "end": { + "character": 26, + "line": 8 + }, + "start": { + "character": 14, + "line": 8 + } + }, + "relatedInformation": [ + { + "location": { + "range": { + "end": { + "character": 26, + "line": 8 + }, + "start": { + "character": 14, + "line": 8 + } + }, + "uri": "${test_dir}/diagnostics10.yar" + }, + "message": "`updated` must be a `date` with format `%d/%m/%Y`" + } + ], + "severity": 2 + } + ], + "kind": "full" +} \ No newline at end of file diff --git a/ls/src/tests/testdata/diagnostics10.yar b/ls/src/tests/testdata/diagnostics10.yar new file mode 100644 index 00000000..664e388c --- /dev/null +++ b/ls/src/tests/testdata/diagnostics10.yar @@ -0,0 +1,12 @@ +rule test_meta { + meta: + description = 123 + id = "bug123" + version = "one" + score = 10 + enabled = "yes" + created = "03-07-2026" + updated = "2026-07-03" + condition: + filesize > 0 +} diff --git a/ls/src/tests/testdata/diagnostics9.config.json b/ls/src/tests/testdata/diagnostics9.config.json new file mode 100644 index 00000000..4fee6d4a --- /dev/null +++ b/ls/src/tests/testdata/diagnostics9.config.json @@ -0,0 +1,9 @@ +{ + "ruleNameValidation": "^[A-Z][a-zA-Z0-9_]*$", + "metadataValidation": [ + { + "identifier": "author", + "required": true + } + ] +} diff --git a/ls/src/tests/testdata/diagnostics9.request.json b/ls/src/tests/testdata/diagnostics9.request.json new file mode 100644 index 00000000..783cd982 --- /dev/null +++ b/ls/src/tests/testdata/diagnostics9.request.json @@ -0,0 +1,5 @@ +{ + "textDocument": { + "uri": "${test_dir}/diagnostics9.yar" + } +} diff --git a/ls/src/tests/testdata/diagnostics9.response.json b/ls/src/tests/testdata/diagnostics9.response.json new file mode 100644 index 00000000..f4c16eee --- /dev/null +++ b/ls/src/tests/testdata/diagnostics9.response.json @@ -0,0 +1,77 @@ +{ + "items": [ + { + "code": "invalid_rule_name", + "data": { + "patches": [] + }, + "message": "rule name does not match regex `^[A-Z][a-zA-Z0-9_]*$`", + "range": { + "end": { + "character": 18, + "line": 0 + }, + "start": { + "character": 5, + "line": 0 + } + }, + "relatedInformation": [ + { + "location": { + "range": { + "end": { + "character": 18, + "line": 0 + }, + "start": { + "character": 5, + "line": 0 + } + }, + "uri": "${test_dir}/diagnostics9.yar" + }, + "message": "this rule name does not match regex `^[A-Z][a-zA-Z0-9_]*$`" + } + ], + "severity": 2 + }, + { + "code": "missing_metadata", + "data": { + "patches": [] + }, + "message": "required metadata is missing", + "range": { + "end": { + "character": 18, + "line": 0 + }, + "start": { + "character": 5, + "line": 0 + } + }, + "relatedInformation": [ + { + "location": { + "range": { + "end": { + "character": 18, + "line": 0 + }, + "start": { + "character": 5, + "line": 0 + } + }, + "uri": "${test_dir}/diagnostics9.yar" + }, + "message": "required metadata `author` not found" + } + ], + "severity": 2 + } + ], + "kind": "full" +} \ No newline at end of file diff --git a/ls/src/tests/testdata/diagnostics9.yar b/ls/src/tests/testdata/diagnostics9.yar new file mode 100644 index 00000000..d3f8663c --- /dev/null +++ b/ls/src/tests/testdata/diagnostics9.yar @@ -0,0 +1,4 @@ +rule bad_rule_name { + condition: + filesize > 0 +} diff --git a/ls/src/tests/testdata/hover12.request.json b/ls/src/tests/testdata/hover12.request.json new file mode 100644 index 00000000..7c1463c7 --- /dev/null +++ b/ls/src/tests/testdata/hover12.request.json @@ -0,0 +1,9 @@ +{ + "textDocument": { + "uri": "${test_dir}/hover12.yar" + }, + "position": { + "line": 4, + "character": 10 + } +} diff --git a/ls/src/tests/testdata/hover12.response.json b/ls/src/tests/testdata/hover12.response.json new file mode 100644 index 00000000..1e9137b7 --- /dev/null +++ b/ls/src/tests/testdata/hover12.response.json @@ -0,0 +1,6 @@ +{ + "contents": { + "kind": "markdown", + "value": "### `max(a: integer, b: integer) -> integer`\n\n***\n\n Returns the maximum of two integers.\n\n***\n\n" + } +} \ No newline at end of file diff --git a/ls/src/tests/testdata/hover12.yar b/ls/src/tests/testdata/hover12.yar new file mode 100644 index 00000000..4c05d577 --- /dev/null +++ b/ls/src/tests/testdata/hover12.yar @@ -0,0 +1,6 @@ +import "math" + +rule test_hover_func_doc { + condition: + math.max(1, 2) == 2 +} diff --git a/ls/src/tests/testdata/symbols6.request.json b/ls/src/tests/testdata/symbols6.request.json index b07d8aab..900f7cb2 100644 --- a/ls/src/tests/testdata/symbols6.request.json +++ b/ls/src/tests/testdata/symbols6.request.json @@ -1,5 +1,5 @@ { "textDocument": { - "uri": "${test_dir}/documentsymbols6.yar" + "uri": "${test_dir}/symbols6.yar" } } diff --git a/ls/src/tests/testdata/symbols6.response.json b/ls/src/tests/testdata/symbols6.response.json index ec747fa4..a028e300 100644 --- a/ls/src/tests/testdata/symbols6.response.json +++ b/ls/src/tests/testdata/symbols6.response.json @@ -1 +1,125 @@ -null \ No newline at end of file +[ + { + "children": [ + { + "kind": 15, + "name": "$pattern_one", + "range": { + "end": { + "character": 28, + "line": 2 + }, + "start": { + "character": 4, + "line": 2 + } + }, + "selectionRange": { + "end": { + "character": 28, + "line": 2 + }, + "start": { + "character": 4, + "line": 2 + } + } + }, + { + "kind": 15, + "name": "$pattern_two", + "range": { + "end": { + "character": 28, + "line": 3 + }, + "start": { + "character": 4, + "line": 3 + } + }, + "selectionRange": { + "end": { + "character": 28, + "line": 3 + }, + "start": { + "character": 4, + "line": 3 + } + } + }, + { + "kind": 15, + "name": "$pattern_three", + "range": { + "end": { + "character": 30, + "line": 4 + }, + "start": { + "character": 4, + "line": 4 + } + }, + "selectionRange": { + "end": { + "character": 30, + "line": 4 + }, + "start": { + "character": 4, + "line": 4 + } + } + }, + { + "kind": 17, + "name": "condition", + "range": { + "end": { + "character": 8, + "line": 6 + }, + "start": { + "character": 4, + "line": 6 + } + }, + "selectionRange": { + "end": { + "character": 8, + "line": 6 + }, + "start": { + "character": 4, + "line": 6 + } + } + } + ], + "detail": "rule", + "kind": 12, + "name": "multiple_pattern", + "range": { + "end": { + "character": 21, + "line": 0 + }, + "start": { + "character": 5, + "line": 0 + } + }, + "selectionRange": { + "end": { + "character": 21, + "line": 0 + }, + "start": { + "character": 5, + "line": 0 + } + } + } +] \ No newline at end of file diff --git a/ls/src/tests/testdata/symbols7.request.json b/ls/src/tests/testdata/symbols7.request.json index 2ace4e1e..72f6de06 100644 --- a/ls/src/tests/testdata/symbols7.request.json +++ b/ls/src/tests/testdata/symbols7.request.json @@ -1,5 +1,5 @@ { "textDocument": { - "uri": "${test_dir}/documentsymbols7.yar" + "uri": "${test_dir}/symbols7.yar" } } diff --git a/ls/src/tests/testdata/symbols7.response.json b/ls/src/tests/testdata/symbols7.response.json index ec747fa4..fb174495 100644 --- a/ls/src/tests/testdata/symbols7.response.json +++ b/ls/src/tests/testdata/symbols7.response.json @@ -1 +1,325 @@ -null \ No newline at end of file +[ + { + "detail": "include", + "kind": 1, + "name": "foo", + "range": { + "end": { + "character": 13, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + }, + "selectionRange": { + "end": { + "character": 13, + "line": 0 + }, + "start": { + "character": 0, + "line": 0 + } + } + }, + { + "detail": "import", + "kind": 2, + "name": "bar", + "range": { + "end": { + "character": 12, + "line": 2 + }, + "start": { + "character": 0, + "line": 2 + } + }, + "selectionRange": { + "end": { + "character": 12, + "line": 2 + }, + "start": { + "character": 0, + "line": 2 + } + } + }, + { + "children": [ + { + "kind": 14, + "name": "description", + "range": { + "end": { + "character": 41, + "line": 7 + }, + "start": { + "character": 4, + "line": 7 + } + }, + "selectionRange": { + "end": { + "character": 41, + "line": 7 + }, + "start": { + "character": 4, + "line": 7 + } + } + }, + { + "kind": 17, + "name": "condition", + "range": { + "end": { + "character": 8, + "line": 9 + }, + "start": { + "character": 4, + "line": 9 + } + }, + "selectionRange": { + "end": { + "character": 8, + "line": 9 + }, + "start": { + "character": 4, + "line": 9 + } + } + } + ], + "detail": "rule", + "kind": 12, + "name": "no_pattern", + "range": { + "end": { + "character": 15, + "line": 5 + }, + "start": { + "character": 5, + "line": 5 + } + }, + "selectionRange": { + "end": { + "character": 15, + "line": 5 + }, + "start": { + "character": 5, + "line": 5 + } + } + }, + { + "children": [ + { + "kind": 15, + "name": "$pattern", + "range": { + "end": { + "character": 24, + "line": 14 + }, + "start": { + "character": 4, + "line": 14 + } + }, + "selectionRange": { + "end": { + "character": 24, + "line": 14 + }, + "start": { + "character": 4, + "line": 14 + } + } + }, + { + "kind": 17, + "name": "condition", + "range": { + "end": { + "character": 8, + "line": 16 + }, + "start": { + "character": 4, + "line": 16 + } + }, + "selectionRange": { + "end": { + "character": 8, + "line": 16 + }, + "start": { + "character": 4, + "line": 16 + } + } + } + ], + "detail": "rule", + "kind": 12, + "name": "single_pattern", + "range": { + "end": { + "character": 19, + "line": 12 + }, + "start": { + "character": 5, + "line": 12 + } + }, + "selectionRange": { + "end": { + "character": 19, + "line": 12 + }, + "start": { + "character": 5, + "line": 12 + } + } + }, + { + "children": [ + { + "kind": 15, + "name": "$pattern_one", + "range": { + "end": { + "character": 28, + "line": 21 + }, + "start": { + "character": 4, + "line": 21 + } + }, + "selectionRange": { + "end": { + "character": 28, + "line": 21 + }, + "start": { + "character": 4, + "line": 21 + } + } + }, + { + "kind": 15, + "name": "$pattern_two", + "range": { + "end": { + "character": 28, + "line": 22 + }, + "start": { + "character": 4, + "line": 22 + } + }, + "selectionRange": { + "end": { + "character": 28, + "line": 22 + }, + "start": { + "character": 4, + "line": 22 + } + } + }, + { + "kind": 15, + "name": "$pattern_three", + "range": { + "end": { + "character": 30, + "line": 23 + }, + "start": { + "character": 4, + "line": 23 + } + }, + "selectionRange": { + "end": { + "character": 30, + "line": 23 + }, + "start": { + "character": 4, + "line": 23 + } + } + }, + { + "kind": 17, + "name": "condition", + "range": { + "end": { + "character": 8, + "line": 25 + }, + "start": { + "character": 4, + "line": 25 + } + }, + "selectionRange": { + "end": { + "character": 8, + "line": 25 + }, + "start": { + "character": 4, + "line": 25 + } + } + } + ], + "detail": "rule", + "kind": 12, + "name": "multiple_pattern", + "range": { + "end": { + "character": 21, + "line": 19 + }, + "start": { + "character": 5, + "line": 19 + } + }, + "selectionRange": { + "end": { + "character": 21, + "line": 19 + }, + "start": { + "character": 5, + "line": 19 + } + } + } +] \ No newline at end of file diff --git a/ls/src/tests/testdata/symbols8.request.json b/ls/src/tests/testdata/symbols8.request.json new file mode 100644 index 00000000..f8bcf8de --- /dev/null +++ b/ls/src/tests/testdata/symbols8.request.json @@ -0,0 +1,5 @@ +{ + "textDocument": { + "uri": "${test_dir}/symbols8.yar" + } +} diff --git a/ls/src/tests/testdata/symbols8.response.json b/ls/src/tests/testdata/symbols8.response.json new file mode 100644 index 00000000..d621ad7f --- /dev/null +++ b/ls/src/tests/testdata/symbols8.response.json @@ -0,0 +1,125 @@ +[ + { + "children": [ + { + "kind": 14, + "name": "author", + "range": { + "end": { + "character": 20, + "line": 2 + }, + "start": { + "character": 4, + "line": 2 + } + }, + "selectionRange": { + "end": { + "character": 20, + "line": 2 + }, + "start": { + "character": 4, + "line": 2 + } + } + }, + { + "kind": 14, + "name": "version", + "range": { + "end": { + "character": 15, + "line": 3 + }, + "start": { + "character": 4, + "line": 3 + } + }, + "selectionRange": { + "end": { + "character": 15, + "line": 3 + }, + "start": { + "character": 4, + "line": 3 + } + } + }, + { + "kind": 14, + "name": "description", + "range": { + "end": { + "character": 43, + "line": 4 + }, + "start": { + "character": 4, + "line": 4 + } + }, + "selectionRange": { + "end": { + "character": 43, + "line": 4 + }, + "start": { + "character": 4, + "line": 4 + } + } + }, + { + "kind": 17, + "name": "condition", + "range": { + "end": { + "character": 8, + "line": 6 + }, + "start": { + "character": 4, + "line": 6 + } + }, + "selectionRange": { + "end": { + "character": 8, + "line": 6 + }, + "start": { + "character": 4, + "line": 6 + } + } + } + ], + "detail": "rule", + "kind": 12, + "name": "rule_with_metadata", + "range": { + "end": { + "character": 23, + "line": 0 + }, + "start": { + "character": 5, + "line": 0 + } + }, + "selectionRange": { + "end": { + "character": 23, + "line": 0 + }, + "start": { + "character": 5, + "line": 0 + } + } + } +] \ No newline at end of file diff --git a/ls/src/tests/testdata/symbols8.yar b/ls/src/tests/testdata/symbols8.yar new file mode 100644 index 00000000..5113fadc --- /dev/null +++ b/ls/src/tests/testdata/symbols8.yar @@ -0,0 +1,8 @@ +rule rule_with_metadata { + meta: + author = "Alice" + version = 1 + description = "Test rule with metadata" + condition: + true +} diff --git a/ls/src/utils/cst_traversal.rs b/ls/src/utils/cst_traversal.rs index 9e103267..aa90ecd1 100644 --- a/ls/src/utils/cst_traversal.rs +++ b/ls/src/utils/cst_traversal.rs @@ -436,3 +436,135 @@ pub fn get_includes(root: &Node, base: &Url) -> Vec { }); includes } + +#[cfg(test)] +mod tests { + use super::*; + use async_lsp::lsp_types::Position; + + fn find_tokens( + node: &Node, + target: &str, + ) -> Vec> { + let mut tokens = Vec::new(); + let mut stack = vec![NodeOrToken::Node(node.clone())]; + while let Some(nt) = stack.pop() { + match nt { + NodeOrToken::Node(n) => stack.extend(n.children_with_tokens()), + NodeOrToken::Token(t) => { + if t.text() == target { + tokens.push(t); + } + } + } + } + tokens + } + + #[test] + fn test_token_and_ident_at_position() { + let text = "rule test_rule {\n strings:\n $my_str = \"hello\"\n condition:\n $my_str and test_rule\n}"; + let cst = CST::from(text); + + let pos = Position::new(0, 6); + let token = ident_at_position(&cst, pos).expect("ident found"); + assert_eq!(token.text(), "test_rule"); + assert_eq!(token.kind(), SyntaxKind::IDENT); + + let pos_past = Position::new(0, 14); + let token = + ident_at_position(&cst, pos_past).expect("ident found past end"); + assert_eq!(token.text(), "test_rule"); + + let pos_str = Position::new(2, 6); + let token = + ident_at_position(&cst, pos_str).expect("pattern ident found"); + assert_eq!(token.text(), "$my_str"); + assert_eq!(token.kind(), SyntaxKind::PATTERN_IDENT); + } + + #[test] + fn test_rule_and_pattern_queries() { + let text = "rule r1 {\n strings:\n $s1 = \"abc\"\n $s2 = \"xyz\"\n condition:\n #s1 > 0 and @s2[1] == 0\n}"; + let cst = CST::from(text); + let root = cst.root(); + + let r1_token = find_tokens(&root, "r1").pop().unwrap(); + + let rule_node = rule_from_ident(&root, &r1_token).expect("rule node"); + assert_eq!(rule_node.kind(), SyntaxKind::RULE_DECL); + assert_eq!( + rule_containing_token(&r1_token).unwrap().kind(), + SyntaxKind::RULE_DECL + ); + + let hash_s1_token = find_tokens(&root, "#s1").pop().unwrap(); + + let pattern_def = pattern_from_ident(&rule_node, &hash_s1_token) + .expect("pattern def"); + assert_eq!(pattern_def.kind(), SyntaxKind::PATTERN_DEF); + + let usages = pattern_usages(&rule_node, &hash_s1_token) + .expect("pattern usages"); + assert_eq!(usages.len(), 1); + assert_eq!(usages[0].text(), "#s1"); + } + + #[test] + fn test_rule_usages() { + let text = + "rule helper { condition: true }\nrule main { condition: helper }"; + let cst = CST::from(text); + let root = cst.root(); + + let helper_ident = find_tokens(&root, "helper").pop().unwrap(); + + let usages = rule_usages(&root, &helper_ident).expect("usages"); + assert_eq!(usages.len(), 1); + assert_eq!(usages[0].text(), "helper"); + } + + #[test] + fn test_with_and_for_declarations() { + let text = "rule loop_test {\n condition:\n for any x in (1..10) : ( x == 5 ) and with y = 2 : ( y == 2 )\n}"; + let cst = CST::from(text); + let root = cst.root(); + + let x_tokens = find_tokens(&root, "x"); + let x_in_cond = x_tokens.first().unwrap(); // due to stack pop, tokens might be reversed or forward; let's pick one that works or test each + + let (decl_ident, decl_node) = + find_declaration(x_in_cond).expect("finds x declaration"); + assert_eq!(decl_ident.text(), "x"); + assert_eq!(decl_node.kind(), SyntaxKind::FOR_EXPR); + + let idents_for: Vec<_> = idents_declared_by_for(&decl_node).collect(); + assert_eq!(idents_for.len(), 1); + assert_eq!(idents_for[0].text(), "x"); + + let y_tokens = find_tokens(&root, "y"); + let y_in_cond = y_tokens.first().unwrap(); + + let (y_decl, y_node) = + find_declaration(y_in_cond).expect("finds y declaration"); + assert_eq!(y_decl.text(), "y"); + assert_eq!(y_node.kind(), SyntaxKind::WITH_EXPR); + + let occ = occurrences_in_with_for(&y_node, &y_decl) + .expect("occurrences of y"); + assert_eq!(occ.len(), 1); + assert_eq!(occ[0].text(), "y"); + } + + #[test] + fn test_get_includes() { + let text = "include \"common.yar\"\ninclude \"sub/rules.yar\"\nrule foo { condition: true }"; + let cst = CST::from(text); + let base = Url::parse("file:///project/main.yar").unwrap(); + + let incs = get_includes(&cst.root(), &base); + assert_eq!(incs.len(), 2); + assert_eq!(incs[0].as_str(), "file:///project/common.yar"); + assert_eq!(incs[1].as_str(), "file:///project/sub/rules.yar"); + } +} diff --git a/ls/src/utils/line_index.rs b/ls/src/utils/line_index.rs index 38d2f5c6..d807ed04 100644 --- a/ls/src/utils/line_index.rs +++ b/ls/src/utils/line_index.rs @@ -53,3 +53,40 @@ impl LineIndex { Span(start..end) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_line_index_conversions() { + let text = "first line\nsecond line\nthird"; + let index = LineIndex::new(text); + + // Offset 0 -> line 0, char 0 + let pos = index.offset_to_position(0); + assert_eq!(pos.line, 0); + assert_eq!(pos.character, 0); + + // Offset 10 ('\n' at end of first line) + let pos = index.offset_to_position(10); + assert_eq!(pos.line, 0); + assert_eq!(pos.character, 10); + + // Offset 11 (start of second line) + let pos = index.offset_to_position(11); + assert_eq!(pos.line, 1); + assert_eq!(pos.character, 0); + + // Span roundtrip + let span = Span(11..22); // "second line" + let range = index.span_to_range(span.clone()); + assert_eq!(range.start.line, 1); + assert_eq!(range.start.character, 0); + assert_eq!(range.end.line, 1); + assert_eq!(range.end.character, 11); + + let roundtrip_span = index.range_to_span(range); + assert_eq!(roundtrip_span, span); + } +} diff --git a/ls/src/utils/modules.rs b/ls/src/utils/modules.rs index 60f23956..f69641f0 100644 --- a/ls/src/utils/modules.rs +++ b/ls/src/utils/modules.rs @@ -355,3 +355,107 @@ pub fn ty_to_string(ty: &Type) -> String { } } } + +#[cfg(test)] +mod tests { + use super::*; + use yara_x_parser::cst::{CST, NodeOrToken}; + + fn find_tokens( + node: &Node, + target: &str, + ) -> Vec> { + let mut tokens = Vec::new(); + let mut stack = vec![NodeOrToken::Node(node.clone())]; + while let Some(nt) = stack.pop() { + match nt { + NodeOrToken::Node(n) => stack.extend(n.children_with_tokens()), + NodeOrToken::Token(t) => { + if t.text() == target { + tokens.push(t); + } + } + } + } + tokens + } + + #[test] + fn test_ty_to_string() { + assert_eq!(ty_to_string(&Type::Integer), "integer"); + assert_eq!(ty_to_string(&Type::Float), "float"); + assert_eq!(ty_to_string(&Type::Bool), "bool"); + assert_eq!(ty_to_string(&Type::String), "string"); + assert_eq!(ty_to_string(&Type::Regexp), "regexp"); + assert_eq!( + ty_to_string(&Type::Array(Box::new(Type::Integer))), + "array" + ); + assert_eq!( + ty_to_string(&Type::Map( + Box::new(Type::String), + Box::new(Type::Integer) + )), + "map" + ); + } + + #[test] + fn test_find_matching_left_bracket() { + let text = "rule test { condition: arr[1][2] == 0 }"; + let cst = CST::from(text); + let root = cst.root(); + + let mut stack = vec![NodeOrToken::Node(root)]; + let mut r_bracket = None; + while let Some(nt) = stack.pop() { + match nt { + NodeOrToken::Node(n) => stack.extend(n.children_with_tokens()), + NodeOrToken::Token(t) => { + if t.kind() == SyntaxKind::R_BRACKET { + r_bracket = Some(t); + break; + } + } + } + } + + let l_bracket = find_matching_left_bracket(&r_bracket.unwrap()) + .expect("matching left bracket"); + assert_eq!(l_bracket.kind(), SyntaxKind::L_BRACKET); + } + + #[test] + fn test_get_type_and_from_expr() { + let text = "import \"pe\"\nrule test { condition: pe.characteristics == 0 and pe.sections[0].name == \".text\" }"; + let cst = CST::from(text); + let root = cst.root(); + + let char_token = find_tokens(&root, "characteristics").pop().unwrap(); + let ty = get_type(&char_token).expect("characteristics type"); + assert!(matches!(ty, Type::Integer)); + + let name_token = find_tokens(&root, "name").first().unwrap().clone(); + let name_ty = get_type(&name_token).expect("sections[0].name type"); + assert!(matches!(name_ty, Type::String)); + } + + #[test] + fn test_get_type_from_declaration_loop() { + let text = "import \"pe\"\nrule test {\n condition:\n for any s in pe.sections : ( s.name == \".text\" ) and with e = pe.entry_point : ( e > 0 )\n}"; + let cst = CST::from(text); + let root = cst.root(); + + let s_tokens = find_tokens(&root, "s"); + let s_token = s_tokens.first().unwrap(); + + let s_ty = get_type(s_token).expect("type of s in for loop"); + assert!(matches!(s_ty, Type::Struct(_))); + + let name_tokens = find_tokens(&root, "name"); + let name_token = name_tokens.first().unwrap(); + + let name_ty = get_type(name_token).expect("type of s.name in loop"); + assert!(matches!(name_ty, Type::String)); + } +} diff --git a/ls/src/utils/position.rs b/ls/src/utils/position.rs index 3b679055..627e3a03 100644 --- a/ls/src/utils/position.rs +++ b/ls/src/utils/position.rs @@ -26,3 +26,46 @@ pub(crate) fn node_to_range(node: &Node) -> Option { Some(Range { start, end }) } + +#[cfg(test)] +mod tests { + use super::*; + use yara_x_parser::cst::CST; + + #[test] + fn test_position_conversions() { + let text = "rule foo { condition: true }"; + let cst = CST::from(text); + let root = cst.root(); + + let range = node_to_range(&root).expect("should produce range"); + assert_eq!(range.start.line, 0); + assert_eq!(range.start.character, 0); + assert_eq!(range.end.line, 0); + assert_eq!(range.end.character, 28); + + // Find the "foo" token + let mut stack = vec![yara_x_parser::cst::NodeOrToken::Node(root)]; + let mut foo_token = None; + while let Some(nt) = stack.pop() { + match nt { + yara_x_parser::cst::NodeOrToken::Node(n) => { + stack.extend(n.children_with_tokens()) + } + yara_x_parser::cst::NodeOrToken::Token(t) => { + if t.text() == "foo" { + foo_token = Some(t); + break; + } + } + } + } + let foo_token = foo_token.expect("foo token found"); + + let token_range = token_to_range(&foo_token).expect("token range"); + assert_eq!(token_range.start.line, 0); + assert_eq!(token_range.start.character, 5); + assert_eq!(token_range.end.line, 0); + assert_eq!(token_range.end.character, 8); + } +} diff --git a/ls/src/worker.rs b/ls/src/worker.rs index e62b950a..265d6f78 100644 --- a/ls/src/worker.rs +++ b/ls/src/worker.rs @@ -1,27 +1,40 @@ //! Browser worker entry point for the YARA language server. +#[cfg(target_family = "wasm")] use std::pin::Pin; +#[cfg(target_family = "wasm")] use std::task::{Context, Poll}; +#[cfg(target_family = "wasm")] use futures::StreamExt; +#[cfg(target_family = "wasm")] use futures::channel::mpsc::{UnboundedReceiver, unbounded}; +#[cfg(target_family = "wasm")] use futures::io::{AsyncRead, AsyncWrite}; +#[cfg(target_family = "wasm")] use js_sys::JSON; +#[cfg(target_family = "wasm")] use wasm_bindgen::JsCast; +#[cfg(target_family = "wasm")] use wasm_bindgen::prelude::*; +#[cfg(target_family = "wasm")] use wasm_bindgen_futures::spawn_local; +#[cfg(target_family = "wasm")] use web_sys::{DedicatedWorkerGlobalScope, MessageEvent}; +#[cfg(target_family = "wasm")] struct WorkerReader { receiver: UnboundedReceiver>, buffer: Vec, } +#[cfg(target_family = "wasm")] struct WorkerWriter { scope: DedicatedWorkerGlobalScope, buffer: Vec, } +#[cfg(target_family = "wasm")] impl AsyncRead for WorkerReader { fn poll_read( mut self: Pin<&mut Self>, @@ -46,6 +59,7 @@ impl AsyncRead for WorkerReader { } } +#[cfg(target_family = "wasm")] impl AsyncWrite for WorkerWriter { fn poll_write( mut self: Pin<&mut Self>, @@ -126,6 +140,7 @@ fn take_lsp_message(buffer: &mut Vec) -> Option { Some(String::from_utf8_lossy(&body).into_owned()) } +#[cfg(target_family = "wasm")] fn event_data_as_text(event: &MessageEvent) -> Option { if let Some(text) = event.data().as_string() { return Some(text); @@ -140,6 +155,7 @@ fn event_data_as_text(event: &MessageEvent) -> Option { /// `Content-Length` framing expected by [`crate::serve`]. Outgoing LSP /// messages are parsed and posted back as JSON values when possible, or /// as raw strings otherwise. +#[cfg(target_family = "wasm")] #[wasm_bindgen(js_name = "runWorkerServer")] pub fn run_worker_server() -> Result<(), JsValue> { console_error_panic_hook::set_once(); @@ -174,3 +190,59 @@ pub fn run_worker_server() -> Result<(), JsValue> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_push_lsp_message() { + let mut buf = Vec::new(); + push_lsp_message(&mut buf, b"hello"); + assert_eq!(buf, b"Content-Length: 5\r\n\r\nhello"); + } + + #[test] + fn test_find_header_end() { + assert_eq!( + find_header_end(b"Content-Length: 10\r\n\r\nbody"), + Some(18) + ); + assert_eq!(find_header_end(b"Content-Length: 10\r\n"), None); + assert_eq!(find_header_end(b"short"), None); + } + + #[test] + fn test_parse_content_length() { + assert_eq!(parse_content_length(b"Content-Length: 42"), Some(42)); + assert_eq!( + parse_content_length(b"content-length: 100\r\nOther: foo"), + Some(100) + ); + assert_eq!(parse_content_length(b"Content-Length: abc"), None); + assert_eq!(parse_content_length(b"Other: 10"), None); + } + + #[test] + fn test_take_lsp_message() { + let mut buf = Vec::new(); + assert_eq!(take_lsp_message(&mut buf), None); + + // Incomplete body + buf.extend_from_slice(b"Content-Length: 10\r\n\r\nshort"); + assert_eq!(take_lsp_message(&mut buf), None); + + // Complete message + buf.extend_from_slice(b"body!"); + assert_eq!(take_lsp_message(&mut buf), Some("shortbody!".to_string())); + assert!(buf.is_empty()); + + // Chained messages + buf.extend_from_slice( + b"Content-Length: 3\r\n\r\nfooContent-Length: 3\r\n\r\nbar", + ); + assert_eq!(take_lsp_message(&mut buf), Some("foo".to_string())); + assert_eq!(take_lsp_message(&mut buf), Some("bar".to_string())); + assert!(buf.is_empty()); + } +}