From 9e1056fcb17d71c117e66bc22da00cc2f8fc8477 Mon Sep 17 00:00:00 2001 From: Denis Avvakumov Date: Wed, 3 Dec 2025 13:43:27 +0200 Subject: [PATCH] Add presume-avx2 feature and expand CI coverage --- .github/workflows/ci.yml | 43 +++++++++++++ .gitignore | 3 +- Cargo.lock | 10 +-- Cargo.toml | 3 +- README.md | 6 ++ build.rs | 91 ++++++++++++++++++++------ scripts/ci_utils.py | 59 +++++++++++++++++ scripts/verify_avx.py | 85 +++++++++++++++++++++++++ scripts/verify_system_lib.py | 120 +++++++++++++++++++++++++++++++++++ 9 files changed, 395 insertions(+), 25 deletions(-) create mode 100644 scripts/ci_utils.py create mode 100644 scripts/verify_avx.py create mode 100644 scripts/verify_system_lib.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a77df32..7a020f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,3 +62,46 @@ jobs: run: choco install ffmpeg --no-progress -y - name: cargo test run: cargo test + + avx-presume: + name: AVX presume feature + runs-on: ubuntu-latest + needs: [fmt, clippy, check, tests] + steps: + - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + - name: Verify AVX presume gating + run: python scripts/verify_avx.py + + system-lib: + name: System Library + runs-on: ubuntu-latest + needs: [fmt, clippy, check, tests] + steps: + - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y pkg-config ffmpeg curl build-essential autoconf automake libtool cmake + - name: Verify system lib build and tests + run: python scripts/verify_system_lib.py + + dred: + name: DRED (bundled) + runs-on: ubuntu-latest + needs: [fmt, clippy, check, tests] + steps: + - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + - name: Install deps (ffmpeg + wget for model download) + run: | + sudo apt-get update + sudo apt-get install -y ffmpeg wget + - name: Build with dred + run: cargo build --features dred + - name: Test with dred + run: cargo test --features dred diff --git a/.gitignore b/.gitignore index 79bd861..ed15ebd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target .vscode/ - +__pycache__/ +*/__pycache__/ diff --git a/Cargo.lock b/Cargo.lock index e331500..88aca2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,9 +139,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.177" +version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" [[package]] name = "libloading" @@ -161,9 +161,9 @@ checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "log" -version = "0.4.28" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "memchr" @@ -195,7 +195,7 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "opus-codec" -version = "0.1.0" +version = "0.1.1" dependencies = [ "bindgen", "cmake", diff --git a/Cargo.toml b/Cargo.toml index 49ea14b..2f21a4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "opus-codec" -version = "0.1.0" +version = "0.1.1" edition = "2024" authors = ["Denis Avvakumov"] description = "Safe Rust bindings for the Opus audio codec" @@ -22,6 +22,7 @@ pkg-config = "0.3" default = [] dred = [] system-lib = [] +presume-avx2 = [] [dev-dependencies] tempfile = "3.23.0" diff --git a/README.md b/README.md index 0213e12..f9b4773 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,12 @@ Safe Rust wrappers around libopus for encoding/decoding Opus audio, with tests that validate core functionality against ffmpeg. +## Features + +- `presume-avx2`: Build the bundled libopus with `OPUS_X86_PRESUME_AVX2` on x86/x86_64 targets, assuming AVX/AVX2/FMA support. Ignored when linking against a system libopus. +- `dred`: Enable libopus DRED support (downloads the model when building the bundled library). The bundled DRED build currently assumes a Unix-like host with `sh`, `wget`, and `tar`, it is not supported on Windows. +- `system-lib`: Link against a system-provided libopus instead of the bundled sources. + ## License This crate is licensed under either of diff --git a/build.rs b/build.rs index abcd7ed..21c6d57 100644 --- a/build.rs +++ b/build.rs @@ -1,6 +1,45 @@ use std::env; fn main() { + emit_rerun_directives(); + let opts = BuildOptions::from_env(); + + if opts.use_system_lib { + handle_system_lib(&opts); + } else { + build_bundled_and_link(&opts); + } + + generate_bindings(); +} + +struct BuildOptions { + use_system_lib: bool, + dred_enabled: bool, + presume_avx: bool, + target_arch: String, + avx_allowed: bool, +} + +impl BuildOptions { + fn from_env() -> Self { + let use_system_lib = env::var("CARGO_FEATURE_SYSTEM_LIB").is_ok(); + let dred_enabled = env::var("CARGO_FEATURE_DRED").is_ok(); + let presume_avx = env::var("CARGO_FEATURE_PRESUME_AVX2").is_ok(); + let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default(); + let avx_allowed = presume_avx && matches!(target_arch.as_str(), "x86" | "x86_64"); + + Self { + use_system_lib, + dred_enabled, + presume_avx, + target_arch, + avx_allowed, + } + } +} + +fn emit_rerun_directives() { println!("cargo:rerun-if-changed=opus/include/opus.h"); println!("cargo:rerun-if-changed=opus/include/opus_defines.h"); println!("cargo:rerun-if-changed=opus/include/opus_types.h"); @@ -9,30 +48,40 @@ fn main() { println!("cargo:rerun-if-changed=build.rs"); println!("cargo:rerun-if-changed=opus/dnn/download_model.sh"); println!("cargo:rerun-if-env-changed=CARGO_FEATURE_SYSTEM_LIB"); + println!("cargo:rerun-if-env-changed=CARGO_FEATURE_PRESUME_AVX2"); +} - let use_system_lib = env::var("CARGO_FEATURE_SYSTEM_LIB").is_ok(); - let dred_enabled = env::var("CARGO_FEATURE_DRED").is_ok(); +fn handle_system_lib(opts: &BuildOptions) { + if opts.dred_enabled { + println!( + "cargo:warning=system-lib feature enabled; ensure the system libopus includes DRED support" + ); + } + if opts.presume_avx { + println!( + "cargo:warning=presume-avx2 feature enabled; ensure the system libopus was built with OPUS_X86_PRESUME_AVX2" + ); + } + link_system_lib(); +} - if use_system_lib { - if dred_enabled { - println!( - "cargo:warning=system-lib feature enabled; ensure the system libopus includes DRED support" - ); - } - link_system_lib(); - } else { - if dred_enabled { - ensure_dred_assets(); - } - let dst = build_bundled(dred_enabled); - println!("cargo:rustc-link-search=native={}/lib", dst.display()); - println!("cargo:rustc-link-lib=static=opus"); +fn build_bundled_and_link(opts: &BuildOptions) { + if opts.dred_enabled { + ensure_dred_assets(); + } + if opts.presume_avx && !opts.avx_allowed { + println!( + "cargo:warning=presume-avx2 feature only applies to x86/x86_64 targets; ignoring for {}", + opts.target_arch + ); } - generate_bindings(); + let dst = build_bundled(opts.dred_enabled, opts.avx_allowed); + println!("cargo:rustc-link-search=native={}/lib", dst.display()); + println!("cargo:rustc-link-lib=static=opus"); } -fn build_bundled(dred_enabled: bool) -> std::path::PathBuf { +fn build_bundled(dred_enabled: bool, presume_avx: bool) -> std::path::PathBuf { let mut config = cmake::Config::new("opus"); config.profile("Release"); @@ -56,6 +105,12 @@ fn build_bundled(dred_enabled: bool) -> std::path::PathBuf { .define("OPUS_DISABLE_INTRINSICS", "OFF") .define("CMAKE_POSITION_INDEPENDENT_CODE", "ON"); + if presume_avx { + config + .define("OPUS_X86_PRESUME_AVX2", "ON") + .define("OPUS_X86_MAY_HAVE_AVX2", "ON"); + } + config.build() } diff --git a/scripts/ci_utils.py b/scripts/ci_utils.py new file mode 100644 index 0000000..836d3e9 --- /dev/null +++ b/scripts/ci_utils.py @@ -0,0 +1,59 @@ +import os +import subprocess +import sys +from contextlib import contextmanager +from pathlib import Path +from typing import Dict, List, Optional, Union + +@contextmanager +def group(name: str): + """Group output in GitHub Actions.""" + # Only print group markers if running in GitHub Actions or if forced + # But for now, we'll always print them as they are harmless in local terminals + print(f"::group::{name}") + try: + yield + finally: + print("::endgroup::") + +def run( + cmd: List[str], + env: Optional[Dict[str, str]] = None, + cwd: Optional[Union[str, Path]] = None, + check: bool = True, + capture_output: bool = False, +) -> subprocess.CompletedProcess: + """Run a command with optional grouping and error handling.""" + cmd_str = " ".join(str(c) for c in cmd) + + # Don't group if capturing output, as it's likely an internal check + should_group = not capture_output + + if should_group: + print(f"::group::{cmd_str}") + + try: + run_env = os.environ.copy() + if env: + run_env.update(env) + + result = subprocess.run( + cmd, + env=run_env, + cwd=cwd, + check=check, + text=True, + capture_output=capture_output + ) + return result + except subprocess.CalledProcessError as e: + if should_group: + print(f"Command failed with exit code {e.returncode}") + raise + finally: + if should_group: + print("::endgroup::") + +def fail(msg: str) -> None: + """Exit with an error message.""" + sys.exit(f"Error: {msg}") diff --git a/scripts/verify_avx.py b/scripts/verify_avx.py new file mode 100644 index 0000000..6673422 --- /dev/null +++ b/scripts/verify_avx.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +""" +Build and verify AVX-presume gating for the bundled opus library. + +- Builds a generic target (no presume feature) and expects no AVX flag/instructions. +- Builds a presume target (presume-avx2 feature) and expects AVX flag/instructions. +""" + +from pathlib import Path +from typing import Optional, Tuple +import ci_utils + + +PRESUME_FLAG = "OPUS_X86_PRESUME_AVX2:BOOL=ON" + + +def newest_build_dir(target_dir: Path) -> Optional[Path]: + build_root = target_dir / "release" / "build" + if not build_root.exists(): + return None + # Find all opus-codec-* directories + candidates = [p for p in build_root.glob("opus-codec-*") if p.is_dir()] + if not candidates: + return None + # Return the most recently modified one + return max(candidates, key=lambda p: p.stat().st_mtime) + + +def find_artifacts(base: Path) -> Tuple[Optional[Path], Optional[Path]]: + # Recursive search for artifacts + caches = list(base.rglob("CMakeCache.txt")) + objs = list(base.rglob("bands.c.o")) + return (caches[0] if caches else None), (objs[0] if objs else None) + + +def verify(target_dir: str, features: str, expect_flag: bool, expect_avx: bool) -> None: + # Build + cmd = ["cargo", "build", "--release"] + if features: + cmd += ["--features", features] + + ci_utils.run(cmd, env={"CARGO_TARGET_DIR": target_dir}) + + # Verify + target = Path(target_dir) + build_dir = newest_build_dir(target) + if not build_dir: + ci_utils.fail(f"build dir not found under {target_dir}") + + print(f"Checking build dir: {build_dir}") + + cache, obj = find_artifacts(target) + if not cache or not obj: + print(f"Artifacts missing for {target_dir}") + print("Found CMakeCache.txt:", [str(p) for p in target.rglob("CMakeCache.txt")]) + print("Found bands.c.o:", [str(p) for p in target.rglob("bands.c.o")]) + ci_utils.fail("Missing required build artifacts") + + # Check CMake cache for flag + cache_content = cache.read_text() + flag_present = PRESUME_FLAG in cache_content + if flag_present != expect_flag: + ci_utils.fail( + f"AVX presume flag mismatch in {cache}: expected={expect_flag}, got={flag_present}" + ) + + # Check object file for AVX instructions + disasm = ci_utils.run( + ["objdump", "-d", str(obj)], capture_output=True + ).stdout + + has_avx = "ymm" in disasm + if has_avx != expect_avx: + ci_utils.fail( + f"AVX instructions mismatch in {obj}: expected={expect_avx}, got={has_avx}" + ) + + +def main() -> None: + verify("target/ci-generic", "", False, False) + verify("target/ci-presume", "presume-avx2", True, True) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_system_lib.py b/scripts/verify_system_lib.py new file mode 100644 index 0000000..c54add1 --- /dev/null +++ b/scripts/verify_system_lib.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +""" +Verify system libopus usage: + +- Ensures pkg-config reports libopus 1.5.2. +- Builds and tests with the `system-lib` feature. +""" + +import subprocess +from pathlib import Path +from typing import Optional + +import ci_utils + +DEB_URLS = { + "dev": [ + "https://deb.debian.org/debian/pool/main/o/opus/libopus-dev_1.5.2-2_amd64.deb", + "https://mirrors.edge.kernel.org/debian/pool/main/o/opus/libopus-dev_1.5.2-2_amd64.deb", + ], + "runtime": [ + "https://deb.debian.org/debian/pool/main/o/opus/libopus0_1.5.2-2_amd64.deb", + "https://mirrors.edge.kernel.org/debian/pool/main/o/opus/libopus0_1.5.2-2_amd64.deb", + ], +} + +EXPECTED_VERSION = "1.5.2" + + +def pkg_config_version() -> Optional[str]: + try: + out = ci_utils.run( + ["pkg-config", "--modversion", "opus"], capture_output=True + ).stdout + return out.strip() + except subprocess.CalledProcessError as exc: + print(f"pkg-config failed: {exc}") + return None + + +def download_first(urls, dest: Path) -> bool: + for u in urls: + try: + ci_utils.run(["curl", "-fLsS", u, "-o", str(dest)]) + print(f"Downloaded {u}") + return True + except subprocess.CalledProcessError: + print(f"Download failed from {u}, trying next mirror...") + return False + + +def install_debs_if_needed() -> None: + ver = pkg_config_version() + if ver == EXPECTED_VERSION: + print(f"libopus already at {EXPECTED_VERSION}") + return + + # Fast path: try the packaged version first. + with ci_utils.group("Install libopus-dev from apt"): + try: + ci_utils.run(["sudo", "apt-get", "update"]) + ci_utils.run(["sudo", "apt-get", "install", "-y", "libopus-dev"]) + except subprocess.CalledProcessError: + print("apt-get install libopus-dev failed, will try deb mirrors") + + ver_after_apt = pkg_config_version() + if ver_after_apt == EXPECTED_VERSION: + print(f"libopus at {EXPECTED_VERSION} after apt install") + return + + # Try downloading Debian packages on Ubuntu runners. + # Only proceed if /etc/os-release indicates Ubuntu. + os_release = Path("/etc/os-release") + if os_release.exists(): + data = os_release.read_text().lower() + if "ubuntu" not in data: + ci_utils.fail( + f"Expected libopus {EXPECTED_VERSION} but found {ver_after_apt}; not on Ubuntu, aborting" + ) + else: + ci_utils.fail( + f"Expected libopus {EXPECTED_VERSION} but found {ver_after_apt}; /etc/os-release missing" + ) + + runtime_deb = Path("/tmp/libopus0.deb") + dev_deb = Path("/tmp/libopus-dev.deb") + + with ci_utils.group("Download libopus debs"): + ok = download_first(DEB_URLS["runtime"], runtime_deb) and download_first( + DEB_URLS["dev"], dev_deb + ) + if not ok: + ci_utils.fail("Failed to download libopus debs from all mirrors") + + with ci_utils.group("Install libopus debs"): + try: + ci_utils.run(["sudo", "dpkg", "-i", str(runtime_deb), str(dev_deb)]) + except subprocess.CalledProcessError: + print("dpkg failed, trying apt-get install -f") + ci_utils.run(["sudo", "apt-get", "install", "-f", "-y"]) + + ver_after = pkg_config_version() + if ver_after != EXPECTED_VERSION: + ci_utils.fail( + f"After deb install, expected libopus {EXPECTED_VERSION} but found {ver_after}" + ) + + +def main() -> None: + install_debs_if_needed() + ver = pkg_config_version() + print(f"pkg-config opus version: {ver}") + if ver != EXPECTED_VERSION: + ci_utils.fail(f"Expected libopus {EXPECTED_VERSION} but found {ver}") + + ci_utils.run(["cargo", "build", "--features", "system-lib"]) + ci_utils.run(["cargo", "test", "--features", "system-lib"]) + + +if __name__ == "__main__": + main()