diff --git a/.github/upstream-workflows/code_health.yaml b/.github/upstream-workflows/code_health.yaml index a865881b..5b0ef304 100644 --- a/.github/upstream-workflows/code_health.yaml +++ b/.github/upstream-workflows/code_health.yaml @@ -7,7 +7,7 @@ jobs: name: Clippy runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: dtolnay/rust-toolchain@master with: toolchain: stable @@ -18,7 +18,7 @@ jobs: name: Rustfmt runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: dtolnay/rust-toolchain@master with: toolchain: stable diff --git a/.github/upstream-workflows/coverage.yaml b/.github/upstream-workflows/coverage.yaml index 8822efac..9338668d 100644 --- a/.github/upstream-workflows/coverage.yaml +++ b/.github/upstream-workflows/coverage.yaml @@ -19,7 +19,7 @@ jobs: CARGO_TERM_COLOR: always steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install toolchain uses: dtolnay/rust-toolchain@master @@ -38,7 +38,7 @@ jobs: run: cargo llvm-cov --features=magic-module,rules-profiling --workspace --lib --lcov --output-path lcov.info - name: Upload coverage to Codecov - uses: codecov/codecov-action@0565863a31f2c772f9f0395002a31e3f06189574 # v5.4.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} files: lcov.info diff --git a/.github/upstream-workflows/golang.yaml b/.github/upstream-workflows/golang.yaml index 51767591..9460adfc 100644 --- a/.github/upstream-workflows/golang.yaml +++ b/.github/upstream-workflows/golang.yaml @@ -14,15 +14,15 @@ jobs: strategy: fail-fast: false matrix: - go-version: [ '1.19', '1.20', '1.21.x', '1.22.x', '1.23.x', '1.24.x', '1.25.x' ] + go-version: [ '1.19', '1.20', '1.21.x', '1.22.x', '1.23.x', '1.24.x', '1.25.x', '1.26.x' ] os: [ ubuntu-latest, macos-latest ] runs-on: ${{ matrix.os }} steps: - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup Go ${{ matrix.go-version }} - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: go-version: ${{ matrix.go-version }} diff --git a/.github/upstream-workflows/pr_title.yaml b/.github/upstream-workflows/pr_title.yaml index aba05d3e..2fa53c98 100644 --- a/.github/upstream-workflows/pr_title.yaml +++ b/.github/upstream-workflows/pr_title.yaml @@ -13,6 +13,6 @@ jobs: permissions: statuses: write steps: - - uses: amannn/action-semantic-pull-request@fdd4d3ddf614fbcd8c29e4b106d3bbe0cb2c605d # v6.0.1 + - uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/upstream-workflows/publish_npm_package.yaml b/.github/upstream-workflows/publish_npm_package.yaml index fdb59efb..3a080e9d 100644 --- a/.github/upstream-workflows/publish_npm_package.yaml +++ b/.github/upstream-workflows/publish_npm_package.yaml @@ -13,7 +13,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master @@ -22,7 +22,7 @@ jobs: target: wasm32-unknown-unknown - name: Install Node.js - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 24 registry-url: https://registry.npmjs.org/ diff --git a/.github/upstream-workflows/publish_vscode_extension.yaml b/.github/upstream-workflows/publish_vscode_extension.yaml index f41a5668..d721d3a0 100644 --- a/.github/upstream-workflows/publish_vscode_extension.yaml +++ b/.github/upstream-workflows/publish_vscode_extension.yaml @@ -41,10 +41,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.x' @@ -70,7 +70,7 @@ jobs: run: cp target/${{ matrix.target }}/release/${{ matrix.binary_name }} ls/editors/code/dist/ - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: "20" diff --git a/.github/upstream-workflows/release.yaml b/.github/upstream-workflows/release.yaml index c633f96c..21adde4a 100644 --- a/.github/upstream-workflows/release.yaml +++ b/.github/upstream-workflows/release.yaml @@ -46,7 +46,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable @@ -91,11 +91,11 @@ jobs: ./target/${{ matrix.target }}/release-lto/yara_x_capi.dll.lib \ ./target/${{ matrix.target }}/release-lto/yara_x_capi.dll else - tar czf $pkgname.gz -C target/${{ matrix.target }}/release-lto yr + tar czf $pkgname.tar.gz -C target/${{ matrix.target }}/release-lto yr fi - name: Upload artifacts - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: yr-${{ matrix.target }} path: yara-x-* @@ -133,7 +133,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: @@ -149,7 +149,7 @@ jobs: fi - name: Install Python - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' @@ -190,13 +190,13 @@ jobs: MACOSX_DEPLOYMENT_TARGET: '14.0' - name: Upload artifacts - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: pypi-${{ matrix.build }}-${{ matrix.python-version }} path: ./wheelhouse/*.whl - name: Upload artifacts - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: pypi-source-${{ strategy.job-index }} path: ./wheelhouse/*.tar.gz @@ -207,7 +207,7 @@ jobs: steps: - name: Download artifacts - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: yr-* @@ -216,7 +216,7 @@ jobs: run: ls - name: Release - uses: softprops/action-gh-release@c95fe1489396fe8a9eb87c0abf8aa5b2ef267fda # v2.2.1 + uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v3.0.0 with: draft: true files: yr-*/yara-x-* @@ -226,7 +226,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Publish crate env: @@ -258,14 +258,14 @@ jobs: id-token: write steps: - name: Download artifacts - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: pypi-* merge-multiple: true path: dist - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1.12 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 with: #repository-url: https://test.pypi.org/legacy/ skip-existing: true diff --git a/.github/upstream-workflows/site.yaml b/.github/upstream-workflows/site.yaml index 20479bd2..1c958a8c 100644 --- a/.github/upstream-workflows/site.yaml +++ b/.github/upstream-workflows/site.yaml @@ -37,10 +37,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup Node.js - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: '22' cache: 'npm' @@ -48,7 +48,7 @@ jobs: - name: Setup Pages id: pages - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 + uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 - name: Install dependencies run: | @@ -85,7 +85,7 @@ jobs: cp -R playground/dist/. site/public/playground/ - name: Upload artifact - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: ./site/public @@ -99,4 +99,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 \ No newline at end of file + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 \ No newline at end of file diff --git a/.github/upstream-workflows/tests.yaml b/.github/upstream-workflows/tests.yaml index 2bb1a6e4..5437eeaf 100644 --- a/.github/upstream-workflows/tests.yaml +++ b/.github/upstream-workflows/tests.yaml @@ -134,7 +134,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master @@ -147,7 +147,7 @@ jobs: - name: Cache cargo registry if: matrix.use_cache == true - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: | ~/.cargo/registry @@ -197,7 +197,7 @@ jobs: CARGO_TERM_COLOR: always steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master @@ -206,13 +206,13 @@ jobs: target: wasm32-unknown-unknown - name: Install Node.js - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 22 - name: Install Chrome and ChromeDriver id: setup-chrome - uses: browser-actions/setup-chrome@v2 + uses: browser-actions/setup-chrome@2e1d749697dd1612b833dba4a722266286fbefcd # v2.1.2 with: chrome-version: stable install-dependencies: true diff --git a/.github/upstream-workflows/wasm.yaml b/.github/upstream-workflows/wasm.yaml index a8e5a71b..a82da270 100644 --- a/.github/upstream-workflows/wasm.yaml +++ b/.github/upstream-workflows/wasm.yaml @@ -15,7 +15,7 @@ jobs: CARGO_TERM_COLOR: always steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 598dcbea..7b8a4539 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13", "3.14" ] os: [ ubuntu-latest, macos-latest, windows-latest ] runs-on: ${{ matrix.os }} steps: diff --git a/cli/src/commands/fmt.rs b/cli/src/commands/fmt.rs index 3ff26cbe..8b52633d 100644 --- a/cli/src/commands/fmt.rs +++ b/cli/src/commands/fmt.rs @@ -8,13 +8,14 @@ use yara_x_fmt::{Formatter, Indentation}; use crate::config::Config; use crate::help; +use crate::walk; pub fn fmt() -> Command { super::command("fmt") .about("Format YARA source files") .arg( - arg!() - .help("Path to YARA source file") + arg!() + .help("Path to YARA source file or directory") .required(true) .value_parser(value_parser!(PathBuf)) .action(ArgAction::Append), @@ -23,6 +24,14 @@ pub fn fmt() -> Command { arg!(-c --check "Run in 'check' mode") .long_help(help::FMT_CHECK_MODE), ) + .arg( + arg!(-r - -"recursive"[MAX_DEPTH]) + .help("Walk directories recursively up to a given depth") + .long_help(help::RECURSIVE_LONG_HELP) + .default_missing_value("1000") + .require_equals(true) + .value_parser(value_parser!(usize)), + ) .arg( arg!(-t - -"tab-size" ) .help("Tab size (in spaces) used in source files") @@ -33,9 +42,10 @@ pub fn fmt() -> Command { } pub fn exec_fmt(args: &ArgMatches, config: &Config) -> anyhow::Result<()> { - let files = args.get_many::("FILE").unwrap(); + let paths = args.get_many::("PATH").unwrap(); let check = args.get_flag("check"); let tab_size = args.get_one::("tab-size").unwrap(); + let recursive = args.get_one::("recursive"); let formatter = Formatter::new() .input_tab_size(*tab_size) @@ -56,27 +66,42 @@ pub fn exec_fmt(args: &ArgMatches, config: &Config) -> anyhow::Result<()> { config.fmt.rule.empty_line_after_section_header, ); - let mut modified_files: Vec<&PathBuf> = Vec::new(); + let mut modified_files: Vec = Vec::new(); - for file in files { - let input = fs::read(file.as_path())?; - let file_modified = if check { - formatter.format(input.as_slice(), io::sink())? + for path in paths { + let mut walker = walk::Walker::path(path); + if let Some(recursive) = recursive { + walker.max_depth(*recursive); } else { - let mut formatted = Cursor::new(Vec::with_capacity(input.len())); - if formatter.format(input.as_slice(), &mut formatted)? { - formatted.seek(SeekFrom::Start(0))?; - let mut output_file = File::create(file.as_path())?; - io::copy(&mut formatted, &mut output_file)?; - true - } else { - false - } - }; - - if file_modified { - modified_files.push(file); + walker.max_depth(0); } + walker.filter("**/*.yar").filter("**/*.yara"); + + walker.walk( + |file_path| { + let input = fs::read(file_path)?; + let file_modified = if check { + formatter.format(input.as_slice(), io::sink())? + } else { + let mut formatted = + Cursor::new(Vec::with_capacity(input.len())); + if formatter.format(input.as_slice(), &mut formatted)? { + formatted.seek(SeekFrom::Start(0))?; + let mut output_file = File::create(file_path)?; + io::copy(&mut formatted, &mut output_file)?; + true + } else { + false + } + }; + + if file_modified { + modified_files.push(file_path.to_path_buf()); + } + Ok(()) + }, + Err, + )?; } if !modified_files.is_empty() { diff --git a/cli/src/commands/scan.rs b/cli/src/commands/scan.rs index 57740fee..62755fcd 100644 --- a/cli/src/commands/scan.rs +++ b/cli/src/commands/scan.rs @@ -60,6 +60,9 @@ pub fn scan() -> Command { .long_help(help::COMPILED_RULES_LONG_HELP), arg!(-c --"count") .help("Print only the number of matches per file"), + arg!(--"cpu-limit" ) + .help("Limit the CPU usage of the scan (percentage from 1 to 99)") + .value_parser(value_parser!(u8).range(1..=99)), arg!(--"disable-console-logs") .help("Disable printing console log messages"), arg!(-f --"fast-scan") @@ -121,7 +124,6 @@ pub fn scan() -> Command { arg!(-a --"timeout" ) .help("Abort scanning after the given number of seconds") .value_parser(value_parser!(u64).range(1..)) - ])) } @@ -179,6 +181,8 @@ pub fn exec_scan(args: &ArgMatches, config: &Config) -> anyhow::Result<()> { let compiled_rules = args.get_flag("compiled-rules"); let profiling = args.get_flag("profiling"); let num_threads = args.get_one::("threads"); + + let cpu_limit = args.get_one::("cpu-limit"); let skip_larger = args.get_one::("skip-larger"); let disable_console_logs = args.get_flag("disable-console-logs"); let scan_list = args.get_flag("scan-list"); @@ -264,6 +268,10 @@ pub fn exec_scan(args: &ArgMatches, config: &Config) -> anyhow::Result<()> { w.num_threads(*num_threads); } + if let Some(limit) = cpu_limit { + w.cpu_limit(*limit); + } + if let Some(max_file_size) = skip_larger { w.metadata_filter(|metadata| metadata.len() <= *max_file_size); } diff --git a/cli/src/tests/fmt.rs b/cli/src/tests/fmt.rs index 7398dfe1..78aa1c03 100644 --- a/cli/src/tests/fmt.rs +++ b/cli/src/tests/fmt.rs @@ -53,3 +53,46 @@ fn utf8_error() { .stderr("error: invalid UTF-8 at [0..1]\n") .code(1); } + +#[test] +fn fmt_directory() { + let temp_dir = TempDir::new().unwrap(); + let subdir = temp_dir.child("subdir"); + subdir.create_dir_all().unwrap(); + + let file1 = temp_dir.child("rule1.yar"); + let file2 = subdir.child("rule2.yar"); + + file1.write_str("rule test1 { condition: true }").unwrap(); + file2.write_str("rule test2 { condition: true }").unwrap(); + + // By default without -r/--recursive, only the top-level directory files are formatted. + Command::new(cargo_bin!("yr")) + .arg("fmt") + .arg(temp_dir.path()) + .assert() + .code(1); // file1 should be modified. + + // So now file1 is formatted, but file2 should still be unformatted. + Command::new(cargo_bin!("yr")) + .arg("fmt") + .arg(temp_dir.path()) + .assert() + .code(0); // Top-level files are already formatted, so no changes. + + // With -r/--recursive, the subdirectories are also processed, so file2 will be formatted. + Command::new(cargo_bin!("yr")) + .arg("fmt") + .arg("-r") + .arg(temp_dir.path()) + .assert() + .code(1); // file2 in subdir should be modified. + + // Subsequent format runs should find no modified files. + Command::new(cargo_bin!("yr")) + .arg("fmt") + .arg("-r") + .arg(temp_dir.path()) + .assert() + .code(0); +} diff --git a/cli/src/tests/scan.rs b/cli/src/tests/scan.rs index 66e2041b..44bcde3d 100644 --- a/cli/src/tests/scan.rs +++ b/cli/src/tests/scan.rs @@ -440,3 +440,14 @@ fn fast_scan() { .success() .stdout(predicate::str::contains("foo src/tests/testdata/dummy.file")); } + +#[test] +fn cpu_limit() { + Command::new(cargo_bin!("yr")) + .arg("scan") + .arg("--cpu-limit=50") + .arg("src/tests/testdata/foo.yar") + .arg("src/tests/testdata/dummy.file") + .assert() + .success(); +} diff --git a/cli/src/walk.rs b/cli/src/walk.rs index 8a61d3a3..f338e148 100644 --- a/cli/src/walk.rs +++ b/cli/src/walk.rs @@ -342,6 +342,7 @@ impl<'a> Walker<'a> { /// ``` pub(crate) struct ParWalker<'a> { num_threads: Option, + cpu_limit: Option, walker: Walker<'a>, } @@ -350,7 +351,7 @@ impl<'a> ParWalker<'a> { /// /// `path` can also point to an individual file instead of a directory. pub fn path(path: &'a Path) -> Self { - Self { walker: Walker::path(path), num_threads: None } + Self { walker: Walker::path(path), num_threads: None, cpu_limit: None } } /// Creates a [`ParWalker`] that walks the files listed in a text file @@ -358,7 +359,11 @@ impl<'a> ParWalker<'a> { /// /// `path` points to the text file that contains the paths to be walked. pub fn file_list(path: &'a Path) -> Self { - Self { walker: Walker::file_list(path), num_threads: None } + Self { + walker: Walker::file_list(path), + num_threads: None, + cpu_limit: None, + } } /// Sets the number of threads used. @@ -370,6 +375,12 @@ impl<'a> ParWalker<'a> { self } + /// Sets the target CPU limit percentage. + pub fn cpu_limit(&mut self, limit: u8) -> &mut Self { + self.cpu_limit = Some(limit); + self + } + /// Sets a maximum depth while traversing the directory tree. /// /// When the maximum depth is 0 only the files that reside in the given @@ -429,6 +440,8 @@ impl<'a> ParWalker<'a> { thread::available_parallelism().map(usize::from).unwrap_or(32) }; + let cpu_limit = self.cpu_limit; + crossbeam::scope(|s| { let mut threads = Vec::with_capacity(num_threads); @@ -453,12 +466,37 @@ impl<'a> ParWalker<'a> { threads.push(s.spawn(move |_| { let mut per_thread_obj = init(&state, &msg_send); for path in paths_recv { + let start_time = Instant::now(); let res = action( &state, &msg_send, path.to_path_buf(), &mut per_thread_obj, ); + let t_active = start_time.elapsed(); + + if let Some(limit) = cpu_limit { + if limit < 100 { + // Calculate the required sleep duration to limit + // CPU usage to the target percentage. + // + // Let T_active be the elapsed time scanning the + // file. Let T_sleep be the sleep time. The target + // utilization percentage is P. + // + // P = 100 * T_active / (T_active + T_sleep) + // P * (T_active + T_sleep) = 100 * T_active + // P * T_sleep = (100 - P) * T_active + // T_sleep = T_active * (100 - P) / P + let t_sleep = t_active.mul_f64( + (100.0 - limit as f64) / limit as f64, + ); + if !t_sleep.is_zero() { + thread::sleep(t_sleep); + } + } + } + if let Err(err) = res && error(err, &msg_send).is_err() { diff --git a/go/compiler_test.go b/go/compiler_test.go index da8d61a7..3c85322b 100644 --- a/go/compiler_test.go +++ b/go/compiler_test.go @@ -27,8 +27,8 @@ func TestNamespaces(t *testing.T) { func TestGlobals(t *testing.T) { c, err := NewCompiler() assert.NoError(t, err) - x := map[string]interface{}{"a": map[string]interface{}{"a": "b"}, "b": "d"} - y := []interface{}{"z"} + x := map[string]any{"a": map[string]any{"a": "b"}, "b": "d"} + y := []any{"z"} c.DefineGlobal("test_hashmap", x) c.DefineGlobal("A", "b") @@ -43,7 +43,7 @@ func TestGlobals(t *testing.T) { assert.Len(t, ScanResults.MatchingRules(), 3) s.SetGlobal("A", "c") - s.SetGlobal("test_arr", []interface{}{"f"}) + s.SetGlobal("test_arr", []any{"f"}) ScanResults, _ = s.Scan([]byte{}) assert.Len(t, ScanResults.MatchingRules(), 1) } @@ -151,7 +151,7 @@ func TestSerialization(t *testing.T) { func TestVariables(t *testing.T) { r, _ := Compile( "rule test { condition: var == 1234 }", - Globals(map[string]interface{}{"var": 1234})) + Globals(map[string]any{"var": 1234})) scanResults, _ := NewScanner(r).Scan([]byte{}) assert.Len(t, scanResults.MatchingRules(), 1) diff --git a/go/scanner_test.go b/go/scanner_test.go index 219343d3..ba8aa328 100644 --- a/go/scanner_test.go +++ b/go/scanner_test.go @@ -51,7 +51,7 @@ func TestScanner2(t *testing.T) { func TestScanner3(t *testing.T) { r, _ := Compile( `rule t { condition: var_bool }`, - Globals(map[string]interface{}{"var_bool": true})) + Globals(map[string]any{"var_bool": true})) s := NewScanner(r) scanResults, _ := s.Scan([]byte{}) @@ -65,7 +65,7 @@ func TestScanner3(t *testing.T) { func TestScanner4(t *testing.T) { r, _ := Compile( `rule t { condition: var_int == 1}`, - Globals(map[string]interface{}{"var_int": 0})) + Globals(map[string]any{"var_int": 0})) s := NewScanner(r) scanResults, _ := s.Scan([]byte{}) @@ -84,7 +84,6 @@ func TestScanner4(t *testing.T) { assert.Len(t, scanResults.MatchingRules(), 1) } - func TestScanFile(t *testing.T) { r, _ := Compile(`rule t { strings: $bar = "bar" condition: $bar }`) s := NewScanner(r) @@ -103,7 +102,6 @@ func TestScanFile(t *testing.T) { assert.Len(t, matchingRules, 1) } - func TestScannerTimeout(t *testing.T) { r, _ := Compile("rule t { strings: $a = /a(.*)*a/ condition: $a }") s := NewScanner(r) diff --git a/site/content/docs/cli/commands.md b/site/content/docs/cli/commands.md index 2f90672f..ee45cf91 100644 --- a/site/content/docs/cli/commands.md +++ b/site/content/docs/cli/commands.md @@ -98,6 +98,18 @@ Prints the number of matching rules per file. Instead of printing the names of the rules that matches each file, it prints the number the total number of rules matching each file. +### --cpu-limit \ + +Limit the CPU usage of the scan (percentage from 1 to 99). + +This option dynamically restricts CPU utilization per scan thread to the +specified percentage. The scanner achieves this by measuring the exact +duration spent scanning each file and introducing a sleep delay before +moving to the next file. + +This is useful for running background scan tasks on production servers +or multi-user systems without saturating CPU capacity. + ### --define Defines external variables. @@ -462,16 +474,35 @@ This command is similar in spirit to other code formatting tools like `gofmt` and `rustfmt`. ``` -yr fmt ... +yr fmt ... ``` +The path can be either a file or directory. If a directory is used, every `.yar` +or `.yara` file contained in the directory will be formated. + +### -r, --recursive=[MAX_DEPTH] + +Walk directories recursively. When is a directory, this option enables +recursive directory traversal. You can optionally specify a `MAX_DEPTH` to +limit how deep the traversal goes: + +Examples: + +``` +--recursive formats nested subdirectories with no limits. +--recursive=0 formats only the files in (no subdirectories) +--recursive=3 formats up to 3 levels deep, including nested subdirectories +``` + +If --recursive is not specified, the default behavior is equivalent to --recursive=0. + ### --check, -c Run in "check" mode. Doesn't modify any file, but exits error code 0 if the files are formatted correctly and no change is necessary, or error code 1 if otherwise. -### -t, --tab-size \\ +### -t, --tab-size \ Tab size (in spaces) used in source files diff --git a/site/hugo_stats.json b/site/hugo_stats.json index b627090f..15340303 100644 --- a/site/hugo_stats.json +++ b/site/hugo_stats.json @@ -321,6 +321,7 @@ "--tag-tag", "--threads-num_threads", "--timeout-seconds", + "-r---recursivemax_depth", "-t---tab-size-num_spaces", "-what-about-the-original-yara", "-x---module-data-modulefile", @@ -485,6 +486,7 @@ "exportsfn_regex", "exportsordinal", "extracting-file-paths", + "fast_scanbool", "fat_header", "fatarch", "fewer-timeouts", @@ -785,6 +787,7 @@ "yrx_scanner_clear_profiling_data", "yrx_scanner_create", "yrx_scanner_destroy", + "yrx_scanner_fast_scan", "yrx_scanner_finish", "yrx_scanner_iter_slowest_rules", "yrx_scanner_on_console_log",