Update testing improve poetry

.githooks/pre-push

@@ -8,14 +8,13 @@ trap 'exit 1' ERR
 # Run unit tests
 poetry run pytest
 
-# Test with test packages locally
-poetry run python -m ossprey --dry-run-safe --package test/npm_simple_math/ --mode npm
-poetry run python -m ossprey --dry-run-safe --package test/yarn_simple_math/ --mode yarn
-poetry run python -m ossprey --dry-run-safe --package test/yarn_massive_math/ --mode yarn
-poetry run python -m ossprey --dry-run-safe --package test/python_simple_math/ --mode python-requirements
+# Run Smoke tests (fast, non-network subset)
+# For the full smoke suite including network/slow tests, use:
+#   poetry run pytest test/smoke/test_smoke.py
+poetry run pytest test/smoke/test_smoke.py -m "smoke and not network and not slow"
 
 # Test malicious detection (should fail with exit code 1)
-if poetry run python -m ossprey --dry-run-malicious --package test/npm_simple_math/ --mode npm; then
+if poetry run python -m ossprey --dry-run-malicious --package test/test_packages/npm_simple_math/ --mode npm; then
     echo "ERROR: Malicious package detection failed - should have detected malware!"
     exit 1
 else

.github/workflows/test.yml

@@ -33,6 +33,10 @@ jobs:
 
       - name: Run Runtime Tests
         run: |
-          poetry run python -m ossprey --dry-run-safe --package test/npm_simple_math/ --mode npm
-          poetry run python -m ossprey --dry-run-safe --package test/yarn_simple_math/ --mode yarn
-          poetry run python -m ossprey --dry-run-safe --package test/python_simple_math/ --mode python-requirements
+          poetry run python -m ossprey --dry-run-safe --package test/test_packages/npm_simple_math/ --mode npm
+          poetry run python -m ossprey --dry-run-safe --package test/test_packages/yarn_simple_math/ --mode yarn
+          poetry run python -m ossprey --dry-run-safe --package test/test_packages/python_simple_math/ --mode python-requirements
+
+      - name: Run smoke tests
+        run: |
+          poetry run pytest test/smoke/test_smoke.py -m "not network and not slow"

ossprey/sbom_python.py

@@ -13,6 +13,8 @@
 from ossbom.model.ossbom import OSSBOM
 from ossbom.model.component import Component
 
+from ossprey.virtualenv import VirtualEnv
+
 logger = logging.getLogger(__name__)
 
 
@@ -34,11 +36,11 @@ def create_sbom_from_requirements(requirements_file: str) -> OSSBOM:
         cmd = get_cyclonedx_binary()
         # This command generates an SBOM for the active virtual environment in JSON format
         result = subprocess.run(
-            [cmd, 'requirements', requirements_file],
+            [cmd, "requirements", requirements_file],
             check=True,
             capture_output=True,
             text=True,
-            env=os.environ.copy()
+            env=os.environ.copy(),
         )
 
         ret = result.stdout
@@ -60,7 +62,7 @@ def create_sbom_from_requirements(requirements_file: str) -> OSSBOM:
 def update_sbom_from_requirements(ossbom: OSSBOM, requirements_file: str) -> OSSBOM:
     sbom = create_sbom_from_requirements(requirements_file)
     ossbom.add_components(sbom.get_components())
-    
+
     return ossbom
 
 
@@ -70,11 +72,11 @@ def create_sbom_from_env() -> OSSBOM:
         cmd = get_cyclonedx_binary()
         # This command generates an SBOM for the active virtual environment in JSON format
         result = subprocess.run(
-            [cmd, 'environment'],
+            [cmd, "environment"],
             check=True,
             capture_output=True,
             text=True,
-            env=os.environ.copy()
+            env=os.environ.copy(),
         )
 
         ret = result.stdout
@@ -110,16 +112,35 @@ def get_poetry_purls_from_lock(lockfile: str = "poetry.lock") -> list[PackageURL
 def update_sbom_from_poetry(ossbom: OSSBOM, package_dir: str) -> OSSBOM:
 
     if not os.path.exists(os.path.join(package_dir, "poetry.lock")):
-        # Run poetry install to generate the poetry.lock file
+        # Run poetry install to generate the poetry.lock file
         try:
-            subprocess.run(['poetry', 'install'], cwd=package_dir, check=True)
+            subprocess.run(["poetry", "install"], cwd=package_dir, check=True)
         except subprocess.CalledProcessError as e:
             logger.error(f"Error running poetry install: {e}")
             raise e
-        
+
     # Get the packages from the poetry.lock file
     purls = get_poetry_purls_from_lock(os.path.join(package_dir, "poetry.lock"))
 
-    ossbom.add_components([Component.create(name=purl.name, version=purl.version, source="poetry", type="pypi") for purl in purls])
+    ossbom.add_components(
+        [
+            Component.create(
+                name=purl.name, version=purl.version, source="poetry", type="pypi"
+            )
+            for purl in purls
+        ]
+    )
+
+    return ossbom
+
+
+def update_sbom_from_virtualenv(ossbom: OSSBOM, package_name: str) -> OSSBOM:
+    venv = VirtualEnv()
+    try:
+        venv.install_package(package_name)
+        requirements_file = venv.create_requirements_file_from_env()
 
+        ossbom = update_sbom_from_requirements(ossbom, requirements_file)
+    finally:
+        venv.exit()
     return ossbom

ossprey/scan.py

@@ -2,16 +2,18 @@
 import json
 import logging
 import os
-from packageurl import PackageURL
+from subprocess import CalledProcessError
 
 from ossprey.environment import get_environment_details
-from ossprey.log import init_logging
 from ossprey.modes import get_modes, get_all_modes
-from ossprey.sbom_python import update_sbom_from_requirements, update_sbom_from_poetry
+from ossprey.sbom_python import (
+    update_sbom_from_requirements,
+    update_sbom_from_poetry,
+    update_sbom_from_virtualenv,
+)
 from ossprey.sbom_javascript import update_sbom_from_npm, update_sbom_from_yarn
 from ossprey.sbom_filesystem import update_sbom_from_filesystem
 from ossprey.ossprey import Ossprey
-from ossprey.virtualenv import VirtualEnv
 
 from ossbom.converters.factory import SBOMConverterFactory
 from ossbom.model.ossbom import OSSBOM
@@ -20,6 +22,49 @@
 logger = logging.getLogger(__name__)
 
 
+def scan_python(modes: list[str], sbom: OSSBOM, package_name: str) -> OSSBOM:
+
+    if "python-requirements" in modes:
+        sbom = update_sbom_from_requirements(sbom, package_name + "/requirements.txt")
+
+    if "poetry" in modes:
+        try:
+            sbom = update_sbom_from_poetry(sbom, package_name)
+        except CalledProcessError as e:
+            logger.error(
+                "Error running poetry scan, skipping poetry scan, attempting pipenv scan instead"
+            )
+            logger.debug(e.stderr)
+            logger.debug("--")
+            logger.debug(e.stdout)
+
+            # Appending pipenv to modes to attempt to get some python dependencies if poetry fails
+            modes.append("pipenv")
+
+    if "pipenv" in modes:
+        sbom = update_sbom_from_virtualenv(sbom, package_name)
+
+    return sbom
+
+
+def scan_javascript(modes: list[str], sbom: OSSBOM, package_name: str) -> OSSBOM:
+    if "npm" in modes:
+        sbom = update_sbom_from_npm(sbom, package_name)
+
+    if "yarn" in modes:
+        sbom = update_sbom_from_yarn(sbom, package_name)
+
+    return sbom
+
+
+def scan_filesystem(modes: list[str], sbom: OSSBOM, package_name: str) -> OSSBOM:
+
+    if "fs" in modes:
+        sbom = update_sbom_from_filesystem(sbom, package_name)
+
+    return sbom
+
+
 def scan(
     package_name: str,
     mode: str = "auto",
@@ -50,37 +95,16 @@ def scan(
     if any(mode not in get_all_modes() for mode in modes) or len(modes) == 0:
         raise Exception("Invalid scanning method: " + str(modes))
 
-    if "pipenv" in modes:
-        venv = VirtualEnv()
-        venv.enter()
-
-        venv.install_package(package_name)
-        requirements_file = venv.create_requirements_file_from_env()
-
-        sbom = update_sbom_from_requirements(sbom, requirements_file)
-
-        venv.exit()
-
-    if "python-requirements" in modes:
-        sbom = update_sbom_from_requirements(sbom, package_name + "/requirements.txt")
-
-    if "poetry" in modes:
-        sbom = update_sbom_from_poetry(sbom, package_name)
-
-    if "npm" in modes:
-        sbom = update_sbom_from_npm(sbom, package_name)
-
-    if "yarn" in modes:
-        sbom = update_sbom_from_yarn(sbom, package_name)
-
-    if "fs" in modes:
-        sbom = update_sbom_from_filesystem(sbom, package_name)
+    # Enrich the sbom based on the scans provided
+    sbom = scan_python(modes, sbom, package_name)
+    sbom = scan_javascript(modes, sbom, package_name)
+    sbom = scan_filesystem(modes, sbom, package_name)
 
     # Update sbom to contain the local environment
     env = get_environment_details(package_name)
     sbom.update_environment(env)
 
-    logger.info(f"Scanning {len(sbom.get_components())}")
+    logger.info(f"Scanning {len(sbom.get_components())} components")
 
     if not local_scan:
         ossprey = Ossprey(url, api_key)

ossprey/test_main.py

@@ -7,7 +7,7 @@
 
 def test_main_function(monkeypatch, capsys):
     monkeypatch.setattr("sys.argv", ["script.py"])
-    monkeypatch.setenv("INPUT_PACKAGE", "test/python_simple_math")
+    monkeypatch.setenv("INPUT_PACKAGE", "test/test_packages/python_simple_math")
     monkeypatch.setenv("INPUT_MODE", "python-requirements")
     monkeypatch.setenv("INPUT_DRY_RUN_SAFE", "True")
 
@@ -23,7 +23,7 @@ def test_main_function(monkeypatch, capsys):
 
 def test_main_function_with_output(monkeypatch, capsys):
     monkeypatch.setattr("sys.argv", ["script.py"])
-    monkeypatch.setenv("INPUT_PACKAGE", "test/python_simple_math")
+    monkeypatch.setenv("INPUT_PACKAGE", "test/test_packages/python_simple_math")
     monkeypatch.setenv("INPUT_MODE", "python-requirements")
     monkeypatch.setenv("INPUT_DRY_RUN_SAFE", "True")
     monkeypatch.setenv("INPUT_OUTPUT", "sbom_output.json")
@@ -44,12 +44,12 @@ def test_main_function_with_output(monkeypatch, capsys):
         os.remove("sbom_output.json")
 
 
-@pytest.mark.parametrize("soft_error, expected_ret", [
-    ("True", 0),
-    ("False", 1)])
+@pytest.mark.parametrize("soft_error, expected_ret", [("True", 0), ("False", 1)])
 def test_main_function_soft_error(monkeypatch, soft_error, expected_ret):
     monkeypatch.setattr("sys.argv", ["script.py"])
-    monkeypatch.setenv("INPUT_PACKAGE", "test/python_simple_math_no_exist")
+    monkeypatch.setenv(
+        "INPUT_PACKAGE", "test/test_packages/python_simple_math_no_exist"
+    )
     monkeypatch.setenv("INPUT_MODE", "python-requirements")
     monkeypatch.setenv("INPUT_DRY_RUN_SAFE", "True")
     monkeypatch.setenv("INPUT_SOFT_ERROR", soft_error)
@@ -59,3 +59,13 @@ def test_main_function_soft_error(monkeypatch, soft_error, expected_ret):
 
     assert excinfo.value.code == expected_ret
 
+
+def test_main_function_with_broken_poetry(monkeypatch, capsys):
+    monkeypatch.setattr("sys.argv", ["script.py"])
+    monkeypatch.setenv("INPUT_PACKAGE", "test/test_packages/poetry_broken_simple_math")
+    monkeypatch.setenv("INPUT_DRY_RUN_SAFE", "True")
+
+    with pytest.raises(SystemExit) as excinfo:
+        main()
+
+    assert excinfo.value.code == 0

ossprey/test_mode_fs.py

@@ -21,7 +21,7 @@ def cleanup():
     "docker_folder, expected_packages, not_expected_packages, no_of_packages",
     [
         (
-            "../test/docker_js_simple_math",
+            "../test/test_packages/docker_js_simple_math",
             {
                 "npm": ["lodash", "axios", "@types/ms", "ms"],
                 "github": ["ossprey/example_malicious_javascript"],
@@ -32,7 +32,7 @@ def cleanup():
             509,
         ),
         (
-            "../test/docker_py_simple_math",
+            "../test/test_packages/docker_py_simple_math",
             {
                 "pypi": ["fastapi", "uvicorn", "solana", "solders", "pydantic"],
                 "github": ["ossprey/example_malicious_python"],

ossprey/test_sbom_python.py

@@ -4,14 +4,18 @@
 import pytest
 from unittest.mock import patch
 
-from ossprey.sbom_python import create_sbom_from_env, create_sbom_from_requirements, get_cyclonedx_binary
+from ossprey.sbom_python import (
+    create_sbom_from_env,
+    create_sbom_from_requirements,
+    get_cyclonedx_binary,
+)
 from ossprey.virtualenv import VirtualEnv
 
 
 def test_get_sbom():
     sbom = create_sbom_from_env()
 
-    assert sbom.format == 'OSSBOM'
+    assert sbom.format == "OSSBOM"
 
 
 def test_get_sbom_from_venv():
@@ -20,16 +24,16 @@ def test_get_sbom_from_venv():
     venv.enter()
 
     # Install a package
-    venv.install_package('numpy')
+    venv.install_package("numpy")
 
     requirements_file = venv.create_requirements_file_from_env()
 
     # Get the SBOM
     sbom = create_sbom_from_requirements(requirements_file)
 
-    assert sbom.format == 'OSSBOM'
+    assert sbom.format == "OSSBOM"
     assert len(sbom.components) == 1
-    assert any(map(lambda x: x.name == 'numpy', sbom.components.values()))
+    assert any(map(lambda x: x.name == "numpy", sbom.components.values()))
 
 
 def test_get_sbom_from_venv_local_package():
@@ -38,16 +42,16 @@ def test_get_sbom_from_venv_local_package():
     venv.enter()
 
     # Install a package
-    venv.install_package('test/python_simple_math')
-  
+    venv.install_package("test/test_packages/python_simple_math")
+
     requirements_file = venv.create_requirements_file_from_env()
 
     # Get the SBOM
     sbom = create_sbom_from_requirements(requirements_file)
 
-    assert sbom.format == 'OSSBOM'
+    assert sbom.format == "OSSBOM"
     assert len(sbom.components) == 7
-    assert any(map(lambda x: x.name == 'simple_math', sbom.components.values()))
+    assert any(map(lambda x: x.name == "simple_math", sbom.components.values()))
 
 
 @patch("shutil.which")