From c14eb2ed8006a2bf37541dd8b2509572210d49ce Mon Sep 17 00:00:00 2001 From: Nikita Pinaev Date: Wed, 8 Jul 2026 02:57:35 +0300 Subject: [PATCH 1/6] STG-4451, STG-4475, STG-4478: dual-mode CLI with microservices installation support - mdast_cli_core/microservices.py: Clark facade client (Bearer auth, md5-first multipart upload with X-File-Size, application_md5 precheck, fsm_locked two-phase scan creation, scanyon-native (stage, status) polling, /rest/scans/{id}/report) - mdast_cli_core/factory.py: installation mode resolution (MDAST_CLI_MODE env or probe autodetect), monolith path unchanged - mdast_cli/ms_flow.py: microservices scan flow with pre-check gate (exit code 8), engines preflight by type/status, reports to user-provided paths - tests/: 81 tests (unit, contract, smoke incl. monolith regression, security) - CI: tests workflow + test gates before PyPI/Docker publishing --- .github/workflows/docker-hub-publish.yml | 88 +++--- .github/workflows/python-publish.yml | 12 + .github/workflows/tests.yml | 26 ++ mdast_cli/__init__.py | 1 + mdast_cli/helpers/const.py | 48 ++++ mdast_cli/helpers/exit_codes.py | 5 +- mdast_cli/mdast_scan.py | 19 ++ mdast_cli/ms_flow.py | 333 +++++++++++++++++++++++ mdast_cli_core/factory.py | 88 ++++++ mdast_cli_core/microservices.py | 231 ++++++++++++++++ pytest.ini | 9 + requirements-dev.txt | 2 + tests/conftest.py | 101 +++++++ tests/test_client_microservices.py | 135 +++++++++ tests/test_error_envelopes.py | 53 ++++ tests/test_mode_detection.py | 63 +++++ tests/test_precheck_gate.py | 89 ++++++ tests/test_security.py | 116 ++++++++ tests/test_smoke_flows.py | 196 +++++++++++++ tests/test_state_machine.py | 57 ++++ 20 files changed, 1633 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/tests.yml create mode 100644 mdast_cli/ms_flow.py create mode 100644 mdast_cli_core/factory.py create mode 100644 mdast_cli_core/microservices.py create mode 100644 pytest.ini create mode 100644 requirements-dev.txt create mode 100644 tests/conftest.py create mode 100644 tests/test_client_microservices.py create mode 100644 tests/test_error_envelopes.py create mode 100644 tests/test_mode_detection.py create mode 100644 tests/test_precheck_gate.py create mode 100644 tests/test_security.py create mode 100644 tests/test_smoke_flows.py create mode 100644 tests/test_state_machine.py diff --git a/.github/workflows/docker-hub-publish.yml b/.github/workflows/docker-hub-publish.yml index a068611..7b28acf 100644 --- a/.github/workflows/docker-hub-publish.yml +++ b/.github/workflows/docker-hub-publish.yml @@ -1,38 +1,50 @@ -name: Publish mdast_cli image to Docker Hub - -on: - push: - branches: - - main - -jobs: - build-n-publish-docker: - name: Docker Hub - Publish mdast_cli - runs-on: ubuntu-22.04 - - steps: - - uses: actions/checkout@v3 - - - name: docker login - env: - DOCKER_USER: ${{secrets.DOCKER_USER}} - DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} - run: - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - - - name: Build the Docker image - - run: docker build . --file Dockerfile -t mobilesecurity/mdast_cli:2026.6.1 -t mobilesecurity/mdast_cli:latest - - - - name: Docker Hub push latest image - run: docker push mobilesecurity/mdast_cli:latest - - - name: Docker Hub push tagged image - - run: docker push mobilesecurity/mdast_cli:2026.6.1 - - - - - +name: Publish mdast_cli image to Docker Hub + +on: + push: + branches: + - main + +jobs: + tests: + name: Tests gate before publish + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: pip install -r requirements.txt -r requirements-dev.txt + - run: pytest -q + + build-n-publish-docker: + name: Docker Hub - Publish mdast_cli + needs: tests + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v3 + + - name: docker login + env: + DOCKER_USER: ${{secrets.DOCKER_USER}} + DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} + run: + docker login -u $DOCKER_USER -p $DOCKER_PASSWORD + + - name: Build the Docker image + + run: docker build . --file Dockerfile -t mobilesecurity/mdast_cli:2026.6.1 -t mobilesecurity/mdast_cli:latest + + + - name: Docker Hub push latest image + run: docker push mobilesecurity/mdast_cli:latest + + - name: Docker Hub push tagged image + + run: docker push mobilesecurity/mdast_cli:2026.6.1 + + + + + diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index b14b1cf..20a0f43 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -6,8 +6,20 @@ on: - main jobs: + tests: + name: Tests gate before publish + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: pip install -r requirements.txt -r requirements-dev.txt + - run: pytest -q + build-n-publish: name: Build and publish mdast_cli 🐍 distributions to PyPI + needs: tests runs-on: ubuntu-22.04 steps: - uses: actions/checkout@master diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..226ab88 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,26 @@ +name: Tests + +on: + push: + pull_request: + +jobs: + tests: + name: pytest (unit, contract, smoke, security) + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: pip install -r requirements.txt -r requirements-dev.txt + + - name: Run full test suite + run: pytest -q + + - name: Security tests gate + run: pytest -q -m security diff --git a/mdast_cli/__init__.py b/mdast_cli/__init__.py index e69de29..6633398 100644 --- a/mdast_cli/__init__.py +++ b/mdast_cli/__init__.py @@ -0,0 +1 @@ +__version__ = '2026.6.1' diff --git a/mdast_cli/helpers/const.py b/mdast_cli/helpers/const.py index d6cc1c9..93f44f1 100644 --- a/mdast_cli/helpers/const.py +++ b/mdast_cli/helpers/const.py @@ -43,3 +43,51 @@ class DastState: # HTTP timeout constants HTTP_REQUEST_TIMEOUT = 30 HTTP_DOWNLOAD_TIMEOUT = 300 + + +# --- Microservices installation (Clark facade, scanyon-native contract) --- + +class ScanStage: + CREATED = 'CREATED' + START = 'START' + WORKING = 'WORKING' + STOP = 'STOP' + SUCCESS = 'SUCCESS' + FAIL = 'FAIL' + + +class ScanStageStatus: + INITIAL = 'INITIAL' + PROCESSING = 'PROCESSING' + WAITING = 'WAITING' + COMPLETE = 'COMPLETE' + PARTIAL_COMPLETE = 'PARTIAL_COMPLETE' + FAIL = 'FAIL' + + +# Terminal (stage, status) pairs, confirmed with scanyon FSM (STG-4475). +# STOP is transitional: a stopped scan finishes as SUCCESS/COMPLETE or SUCCESS/PARTIAL_COMPLETE. +TERMINAL_SCAN_PAIRS = { + (ScanStage.SUCCESS, ScanStageStatus.COMPLETE), + (ScanStage.SUCCESS, ScanStageStatus.PARTIAL_COMPLETE), + (ScanStage.FAIL, ScanStageStatus.FAIL), +} +PRE_START_STAGES = {ScanStage.CREATED, ScanStage.START} +ACTIVE_STAGES = {ScanStage.WORKING, ScanStage.STOP} + +# Eucalyptus engine contract: platform in `type`, liveness in `status` (not `state`) +ENGINE_ACTIVE_STATUS = 'STARTED' +OS_ANDROID = 'ANDROID' +OS_IOS = 'IOS' + +# Installation mode selection (no new CLI flags by contract, env vars only) +MODE_ENV_VAR = 'MDAST_CLI_MODE' +UPLOAD_TIMEOUT_ENV_VAR = 'MDAST_UPLOAD_TIMEOUT' +TLS_VERIFY_ENV_VAR = 'MDAST_TLS_VERIFY' +MODE_AUTO = 'auto' +MODE_MONOLITH = 'monolith' +MODE_MICROSERVICES = 'microservices' + +# Clark upload contract (STG-4451): server-side upload_timeout query bounds +UPLOAD_TIMEOUT_MIN = 1 +UPLOAD_TIMEOUT_MAX = 300 diff --git a/mdast_cli/helpers/exit_codes.py b/mdast_cli/helpers/exit_codes.py index 81fd180..31506d3 100644 --- a/mdast_cli/helpers/exit_codes.py +++ b/mdast_cli/helpers/exit_codes.py @@ -24,7 +24,10 @@ class ExitCode(IntEnum): AUTH_ERROR = 7 """Authentication or authorization error.""" - + + PRECHECK_BLOCKED = 8 + """Scan pre-check returned blocking warnings (microservices installation gate).""" + INTERNAL_ERROR = 1 """Internal application error (default for unexpected errors).""" diff --git a/mdast_cli/mdast_scan.py b/mdast_cli/mdast_scan.py index 1514128..36148d0 100644 --- a/mdast_cli/mdast_scan.py +++ b/mdast_cli/mdast_scan.py @@ -27,7 +27,12 @@ from mdast_cli.helpers.exit_codes import ExitCode from mdast_cli.helpers.helpers import check_app_md5 from mdast_cli_core.token import mDastToken as mDast +from mdast_cli_core.factory import (MODE_MICROSERVICES, ModeDetectionError, resolve_installation_mode, + tls_verify_enabled) from mdast_cli.cr_report_generator import generate_cr +from mdast_cli import __version__ + +USER_AGENT = f'mdast_cli/{__version__}' logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s %(message)s', datefmt='%d/%m/%Y %H:%M:%S', stream=sys.stdout) @@ -592,6 +597,20 @@ def main(): print(f'DOWNLOAD_PATH={app_file}') sys.exit(ExitCode.SUCCESS) + try: + installation_mode = resolve_installation_mode(url, token, company_id, + verify=tls_verify_enabled()) + except ModeDetectionError as ex: + logger.error(str(ex)) + sys.exit(ExitCode.AUTH_ERROR if ex.auth_error else ExitCode.NETWORK_ERROR) + + if installation_mode == MODE_MICROSERVICES: + from mdast_cli.ms_flow import run_microservices_flow + run_microservices_flow(arguments, url, token, app_file, appstore_app_md5, + user_agent=USER_AGENT, verify=tls_verify_enabled()) + logger.info('Job completed successfully!') + sys.exit(ExitCode.SUCCESS) + mdast = mDast(url, token, company_id) get_architectures_resp = mdast.get_architectures() diff --git a/mdast_cli/ms_flow.py b/mdast_cli/ms_flow.py new file mode 100644 index 0000000..9a1e64d --- /dev/null +++ b/mdast_cli/ms_flow.py @@ -0,0 +1,333 @@ +"""Scan flow for the microservices installation (Clark facade). + +Mirrors the monolith flow of mdast_scan.main() with the scanyon-native +contract (STG-4451 / STG-4475 / STG-4478): + + engines preflight -> dedup by md5 -> upload -> pre-check gate -> + create scan (fsm_locked) -> start -> poll (stage, status) -> reports. + +The public CLI interface is intentionally unchanged: same flags, same +report file arguments, `--architecture_id`/`--company_id` are accepted +and ignored here. +""" +import json +import logging +import os +import sys +import time + +from mdast_cli.helpers.const import (ACTIVE_STAGES, ANDROID_EXTENSIONS, END_SCAN_TIMEOUT, LONG_TRY, OS_ANDROID, + OS_IOS, PRE_START_STAGES, SLEEP_TIMEOUT, TERMINAL_SCAN_PAIRS, TRY, + UPLOAD_TIMEOUT_ENV_VAR, UPLOAD_TIMEOUT_MAX, UPLOAD_TIMEOUT_MIN, + ENGINE_ACTIVE_STATUS, ScanStage, ScanStageStatus) +from mdast_cli.helpers.exit_codes import ExitCode +from mdast_cli.helpers.helpers import check_app_md5 +from mdast_cli_core.microservices import extract_error_message, mDastMicroservices + +logger = logging.getLogger(__name__) + +KNOWN_PRECHECK_WARNINGS = { + 'empty_profile': 'Scan profile has no enabled modules', + 'no_compatible_devices': 'No compatible devices for the required OS version', + 'application_not_found': 'Application is not found on the platform', + 'precheck_unavailable': 'Pre-check dependencies are temporarily unavailable, retry later', + 'architecture_unsupported': 'Platform/OS version pair is not supported by this installation', +} + + +def resolve_platform(app_file): + """ANDROID/IOS from the application file extension.""" + _, extension = os.path.splitext(app_file) + if extension in ANDROID_EXTENSIONS: + return OS_ANDROID + if extension == '.ipa': + return OS_IOS + return None + + +def render_precheck_warning(warning): + """One human-readable line per pre-check warning: known text + raw payload. + + Unknown warning types are rendered generically: the contract explicitly + allows backward-compatible extension of the type set (DEC-666-04). + """ + warning_type = str(warning.get('type', 'unknown')) + payload = warning.get('payload') or {} + text = KNOWN_PRECHECK_WARNINGS.get(warning_type, 'Scan pre-check warning') + payload_str = json.dumps(payload, ensure_ascii=False) if payload else '' + return f'[{warning_type}] {text}{" " + payload_str if payload_str else ""}' + + +def scan_pair(scan): + return scan.get('stage'), scan.get('status') + + +def is_terminal(scan): + return scan_pair(scan) in TERMINAL_SCAN_PAIRS + + +def is_success(scan): + stage, status = scan_pair(scan) + return stage == ScanStage.SUCCESS and status in (ScanStageStatus.COMPLETE, + ScanStageStatus.PARTIAL_COMPLETE) + + +def resolve_upload_timeout(): + """Optional server-side upload_timeout override via env (no new CLI flags).""" + raw = os.environ.get(UPLOAD_TIMEOUT_ENV_VAR) + if not raw: + return None + try: + value = int(raw) + except ValueError: + logger.warning(f'Ignoring invalid {UPLOAD_TIMEOUT_ENV_VAR}={raw!r} (expected integer)') + return None + clamped = max(UPLOAD_TIMEOUT_MIN, min(UPLOAD_TIMEOUT_MAX, value)) + if clamped != value: + logger.warning(f'{UPLOAD_TIMEOUT_ENV_VAR}={value} is out of range, using {clamped}') + return clamped + + +def _exit_on_http_error(resp, action, exit_code=ExitCode.SCAN_FAILED): + message = extract_error_message(resp) + if resp.status_code in (401, 403): + logger.error(f'{action}: authorization failed (HTTP {resp.status_code}): {message}. ' + 'Check the organization CLI token (issued in the platform UI).') + sys.exit(ExitCode.AUTH_ERROR) + logger.error(f'{action} failed (HTTP {resp.status_code}): {message}') + sys.exit(exit_code) + + +def _get_scan(mdast, scan_id): + resp = mdast.get_scan_info(scan_id) + if resp.status_code != 200: + _exit_on_http_error(resp, f'Getting scan info for scan {scan_id}', + ExitCode.NETWORK_ERROR) + return resp.json() + + +def run_precheck_gate(mdast, md5, profile_id, testcase_id): + """Gate model (DEC-671-05): any warning blocks the scan with a non-zero exit.""" + resp = mdast.precheck_scan(md5, profile_id, testcase_id) + if resp.status_code == 404: + _exit_on_http_error(resp, 'Scan pre-check (profile lookup)') + if resp.status_code == 422 and profile_id is None: + # Installation does not yet accept pre-check without profile_id + # (scanyon change pending); skipping is a documented interim behavior. + logger.warning('Scan pre-check skipped: this installation requires profile_id ' + 'for pre-check and no --profile_id was given') + return + if resp.status_code != 200: + message = extract_error_message(resp) + logger.error(f'Scan pre-check is unavailable (HTTP {resp.status_code}): {message}') + sys.exit(ExitCode.PRECHECK_BLOCKED) + warnings = (resp.json() or {}).get('warnings') or [] + if not warnings: + logger.info('Scan pre-check passed, no warnings') + return + logger.error('Scan pre-check returned blocking warnings, scan will not be created:') + for warning in warnings: + print(render_precheck_warning(warning), file=sys.stderr) + sys.exit(ExitCode.PRECHECK_BLOCKED) + + +def run_microservices_flow(arguments, url, token, app_file, appstore_app_md5, user_agent=None, + verify=False): + """Full scan flow against the microservices installation. Exits the process.""" + profile_id = arguments.profile_id + project_id = arguments.project_id + testcase_id = arguments.testcase_id + + if arguments.appium_script_path: + logger.error('Appium scans are not supported on the microservices installation ' + '(--appium_script_path). Use a recorded test case (--testcase_id) instead.') + sys.exit(ExitCode.INVALID_ARGS) + if arguments.cr_report: + logger.error('CR report generation (--cr_report) is not supported on the ' + 'microservices installation.') + sys.exit(ExitCode.INVALID_ARGS) + if arguments.architecture_id is not None: + logger.warning('--architecture_id is ignored on the microservices installation: ' + 'the platform resolves the architecture from application metadata') + + mdast = mDastMicroservices(url, token, arguments.company_id, + user_agent=user_agent, verify=verify) + + architectures_resp = mdast.get_architectures() + if architectures_resp.status_code != 200: + _exit_on_http_error(architectures_resp, 'Getting architectures', ExitCode.NETWORK_ERROR) + logger.info(f'Supported architectures: {architectures_resp.json()}') + + platform = resolve_platform(app_file) + if platform is None: + logger.error(f'Cannot resolve platform (Android/iOS) from file extension: {app_file}') + sys.exit(ExitCode.INVALID_ARGS) + + if testcase_id is not None: + testcase_resp = mdast.get_testcase(testcase_id) + if testcase_resp.status_code == 200: + testcase_os = str(testcase_resp.json().get('os', '')).upper() + if testcase_os and testcase_os != platform: + logger.error(f'Test case {testcase_id} is recorded for {testcase_os}, ' + f'but the application file is for {platform}') + sys.exit(ExitCode.INVALID_ARGS) + else: + logger.warning(f'Cannot get test case {testcase_id} ' + f'(HTTP {testcase_resp.status_code}), continuing') + + engines_resp = mdast.get_engines() + if engines_resp.status_code != 200: + _exit_on_http_error(engines_resp, 'Getting engines', ExitCode.NETWORK_ERROR) + engines = engines_resp.json() + active_engines = [engine for engine in engines + if str(engine.get('type', '')).upper() == platform + and str(engine.get('status', '')).upper() == ENGINE_ACTIVE_STATUS] + if not active_engines: + logger.error(f'Cannot create scan - no active engine for platform {platform}') + sys.exit(ExitCode.SCAN_FAILED) + + logger.info('Check if this version of application was already uploaded..') + app_md5 = (appstore_app_md5 or check_app_md5(app_file)).lower() + dedup_resp = mdast.check_app_md5(None, app_md5) + if dedup_resp.status_code != 200: + _exit_on_http_error(dedup_resp, 'Application dedup check', ExitCode.NETWORK_ERROR) + found_apps = dedup_resp.json() + if found_apps: + application = found_apps[0] + logger.info(f"This app was uploaded before, application id is: {application['id']}, " + f"package name: {application['package_name']}, " + f"version: {application['version_name']}, md5: {application['md5']}") + else: + logger.info('This is new application or new version') + logger.info('Uploading application to server..') + upload_resp = mdast.upload_application(app_file, upload_timeout=resolve_upload_timeout()) + if upload_resp.status_code != 201: + if upload_resp.status_code == 504: + logger.error('Application parsing did not finish in time (504). The upload is ' + 'processed asynchronously - retry the same command later, the file ' + f'will not be re-uploaded. {UPLOAD_TIMEOUT_ENV_VAR} env var can ' + 'raise the wait (max 300 seconds).') + sys.exit(ExitCode.SCAN_FAILED) + _exit_on_http_error(upload_resp, 'Uploading application') + application = upload_resp.json() + logger.info(f"Application uploaded successfully. Application id: {application['id']}") + + run_precheck_gate(mdast, app_md5, profile_id, testcase_id) + + logger.info(f"Creating scan for application {application['id']}") + if testcase_id is not None: + create_resp = mdast.create_auto_scan(project_id, profile_id, app_md5, None, testcase_id) + scan_type = 'auto_stingray' + else: + create_resp = mdast.create_manual_scan(project_id, profile_id, app_md5) + scan_type = 'manual' + if create_resp.status_code not in (200, 201): + _exit_on_http_error(create_resp, 'Creating scan') + scan = create_resp.json() + if not scan.get('id'): + logger.error(f'Something went wrong while creating scan: {scan}') + sys.exit(ExitCode.SCAN_FAILED) + scan_id = scan['id'] + logger.info(f"Project and profile was created/found successfully. " + f"Project id: {(scan.get('project') or {}).get('id')}, " + f"profile id: {(scan.get('profile') or {}).get('id')}") + logger.info(f'Scan was created successfully. Scan id: {scan_id}') + + logger.info(f'Start scan with id {scan_id}') + start_resp = mdast.start_scan(scan_id) + if start_resp.status_code != 200: + _exit_on_http_error(start_resp, f'Starting scan {scan_id}') + + if arguments.nowait: + logger.info('Scan successfully started. Don`t wait for end, exit with zero code') + sys.exit(ExitCode.SUCCESS) + + try_count = LONG_TRY if arguments.long_wait else TRY + + logger.info('Scan started successfully.') + scan = _get_scan(mdast, scan_id) + logger.info(f'Current scan state: {scan_pair(scan)}') + + count = 0 + while scan.get('stage') in PRE_START_STAGES and count < try_count: + logger.info(f'Try to get scan status for scan id {scan_id}. Count number {count}') + scan = _get_scan(mdast, scan_id) + logger.info(f'Current scan state: {scan_pair(scan)}') + count += 1 + if scan.get('stage') in PRE_START_STAGES: + logger.info(f'Wait {SLEEP_TIMEOUT} seconds and try again') + time.sleep(SLEEP_TIMEOUT) + + if scan.get('stage') not in ACTIVE_STAGES | {ScanStage.SUCCESS, ScanStage.FAIL}: + logger.error(f'Error with scan id {scan_id}. Scan did not start, ' + f'current state: {scan_pair(scan)}, message: {scan.get("message")}') + sys.exit(ExitCode.SCAN_FAILED) + + if scan_type == 'manual' and not is_terminal(scan) and scan.get('stage') != ScanStage.STOP: + logger.info(f'This is a scan without a test case, ' + f'lets wait for {END_SCAN_TIMEOUT} seconds and stop it.') + time.sleep(END_SCAN_TIMEOUT) + stop_resp = mdast.stop_scan(scan_id) + if stop_resp.status_code == 200: + logger.info(f'Scan {scan_id} was requested to stop (stopped by CLI after ' + f'{END_SCAN_TIMEOUT} seconds, as in the manual scan flow)') + elif stop_resp.status_code == 409: + logger.info(f'Scan {scan_id} is not in a stoppable state anymore, continuing') + else: + _exit_on_http_error(stop_resp, f'Stopping scan {scan_id}') + + logger.info(f"Scan {scan_id} is started now. Let's wait until the scan is finished") + count = 0 + while not is_terminal(scan) and count < try_count: + logger.info(f'Try to get scan status for scan id {scan_id}. Count number {count}') + scan = _get_scan(mdast, scan_id) + logger.info(f'Current scan state: {scan_pair(scan)}') + count += 1 + if not is_terminal(scan): + logger.info(f'Wait {SLEEP_TIMEOUT} seconds and try again') + time.sleep(SLEEP_TIMEOUT) + + if not is_success(scan): + logger.error(f'Scan {scan_id} finished with state {scan_pair(scan)}, ' + f'message: {scan.get("message")}. Exit with error status code.') + sys.exit(ExitCode.SCAN_FAILED) + if scan.get('status') == ScanStageStatus.PARTIAL_COMPLETE: + logger.warning(f'Scan {scan_id} finished partially complete: {scan.get("message")}') + + download_reports(mdast, scan_id, arguments) + + +def download_reports(mdast, scan_id, arguments): + """Report step (STG-4478): files are written to the user-provided paths. + + Content-Disposition from the server is deliberately ignored: the local + file name is a CLI argument and must not be controlled by the server. + """ + pdf_report_file_name = arguments.pdf_report_file_name + json_summary_file_name = arguments.summary_report_json_file_name + + if pdf_report_file_name: + logger.info(f'Create and download pdf report for scan with id {scan_id} ' + f'to file {pdf_report_file_name}.') + pdf_report = mdast.download_report(scan_id) + if pdf_report.status_code != 200: + _exit_on_http_error(pdf_report, 'PDF report downloading') + pdf_report_file_name = pdf_report_file_name if pdf_report_file_name.endswith( + '.pdf') else f'{pdf_report_file_name}.pdf' + with open(pdf_report_file_name, 'wb') as f: + f.write(pdf_report.content) + logger.info(f'Report for scan {scan_id} successfully created and available at path: ' + f'{pdf_report_file_name}.') + + if json_summary_file_name: + logger.info(f'Download JSON summary report for scan with id {scan_id} ' + f'to file {json_summary_file_name}.') + json_report = mdast.download_scan_json_result(scan_id) + if json_report.status_code != 200: + _exit_on_http_error(json_report, 'JSON summary report downloading') + json_file_name = json_summary_file_name if json_summary_file_name.endswith( + '.json') else f'{json_summary_file_name}.json' + with open(json_file_name, 'w') as fp: + json.dump(json_report.json(), fp, indent=4, ensure_ascii=False) + logger.info(f'JSON report for scan {scan_id} successfully created and available ' + f'at path: {json_file_name}.') diff --git a/mdast_cli_core/factory.py b/mdast_cli_core/factory.py new file mode 100644 index 0000000..f6b2562 --- /dev/null +++ b/mdast_cli_core/factory.py @@ -0,0 +1,88 @@ +"""Installation mode resolution (monolith vs microservices) and client factory. + +No new CLI flags are introduced (OOS-671-03): the mode comes from the +MDAST_CLI_MODE environment variable (`auto` | `monolith` | `microservices`, +default `auto`) or is detected by probing. + +Auto-detection probe: `GET {base}/engines/` exists only on the Clark facade +(the monolith serves engines under `/organizations/{id}/engines/`), so a 200 +identifies the microservices installation; otherwise the monolith path is +probed with the legacy `Token` auth scheme. +""" +import logging +import os + +import requests + +MODE_ENV_VAR = 'MDAST_CLI_MODE' +TLS_VERIFY_ENV_VAR = 'MDAST_TLS_VERIFY' +MODE_AUTO = 'auto' +MODE_MONOLITH = 'monolith' +MODE_MICROSERVICES = 'microservices' +PROBE_TIMEOUT = 20 + +logger = logging.getLogger(__name__) + + +class ModeDetectionError(Exception): + """Raised when neither installation flavour answered the probes.""" + + def __init__(self, message, auth_error=False): + super().__init__(message) + self.auth_error = auth_error + + +def tls_verify_enabled(): + return os.environ.get(TLS_VERIFY_ENV_VAR, '').strip().lower() in ('1', 'true', 'yes', 'on') + + +def resolve_installation_mode(base_url, ci_token, company_id, mode=None, verify=None): + """Return MODE_MONOLITH or MODE_MICROSERVICES. + + `base_url` is the normalized base ending with `/rest` (both installations + serve the CLI routes under this prefix). + """ + if verify is None: + verify = tls_verify_enabled() + mode = (mode or os.environ.get(MODE_ENV_VAR) or MODE_AUTO).strip().lower() + if mode in (MODE_MONOLITH, MODE_MICROSERVICES): + logger.info(f'Installation mode forced via {MODE_ENV_VAR}: {mode}') + return mode + if mode != MODE_AUTO: + raise ModeDetectionError( + f'Unknown {MODE_ENV_VAR} value: {mode!r} ' + f'(expected {MODE_AUTO}/{MODE_MONOLITH}/{MODE_MICROSERVICES})') + + ms_status = None + try: + resp = requests.get(f'{base_url}/engines/', + headers={'Authorization': f'Bearer {ci_token}'}, + verify=verify, + timeout=PROBE_TIMEOUT) + ms_status = resp.status_code + if resp.status_code == 200: + logger.info('Detected microservices installation (Clark facade)') + return MODE_MICROSERVICES + except requests.RequestException as ex: + logger.debug(f'Microservices probe failed: {ex}') + + monolith_status = None + try: + resp = requests.get(f'{base_url}/organizations/{company_id}/engines/', + headers={'Authorization': f'Token {ci_token}'}, + verify=verify, + timeout=PROBE_TIMEOUT) + monolith_status = resp.status_code + if resp.status_code == 200: + logger.info('Detected monolith installation') + return MODE_MONOLITH + except requests.RequestException as ex: + logger.debug(f'Monolith probe failed: {ex}') + + auth_error = 401 in (ms_status, monolith_status) or 403 in (ms_status, monolith_status) + raise ModeDetectionError( + 'Cannot detect installation mode: probes failed ' + f'(microservices GET /engines/ -> {ms_status}, ' + f'monolith GET /organizations/{{id}}/engines/ -> {monolith_status}). ' + f'Check --url and --token, or force the mode via {MODE_ENV_VAR}.', + auth_error=auth_error) diff --git a/mdast_cli_core/microservices.py b/mdast_cli_core/microservices.py new file mode 100644 index 0000000..df3b2b6 --- /dev/null +++ b/mdast_cli_core/microservices.py @@ -0,0 +1,231 @@ +"""Client for the microservices installation (Clark facade behind Envoy ExtAuth -> OPA). + +Public method names match the monolith client (mDastToken) per ASM-671-01; +routes and payloads follow the Clark facade contract (STGRST-670/671/694/698): + +- auth: `Authorization: Bearer ` (org-level opaque token issued in UI); +- all routes live under the same `/rest` prefix as the monolith (base_url ends + with `/rest` after mdast_scan.py normalization), so the base URL handling is + shared between both installations; +- tenant comes from the `X-Organization-Id` header injected by the platform, + the `/organizations/{id}/` URL segment is gone, `company_id` is ignored; +- scans are keyed by application md5 (not application_id) and are created with + the two-phase model (`fsm_locked=true` + explicit `/scans/{id}/start/`). +""" +import hashlib +import json +import os + +import requests + +from .base import mDastBase + +HTTP_REQUEST_TIMEOUT = 30 +HTTP_DOWNLOAD_TIMEOUT = 300 +UPLOAD_EXTRA_TIMEOUT = 120 + + +def file_md5(file_path): + """Lower-case hex md5 of a file (Clark multipart contract requires lower-case).""" + file_hash = hashlib.md5() + with open(file_path, 'rb') as f: + while chunk := f.read(8192): + file_hash.update(chunk) + return file_hash.hexdigest().lower() + + +def extract_error_message(resp): + """Best-effort human-readable message from Clark error responses. + + /rest routes answer with the monolith envelope {error_code, message}; + the report route answers with the FastAPI envelope {"detail": ...}; + scanyon 429 passes through {"detail": ..., "reasons": [...]}. + """ + try: + data = resp.json() + except ValueError: + return resp.text + if isinstance(data, dict): + if data.get('message'): + prefix = data.get('error_code') + message = data['message'] + return f'{prefix}: {message}' if prefix else message + if data.get('detail') is not None: + detail = data['detail'] + if not isinstance(detail, str): + detail = json.dumps(detail, ensure_ascii=False) + reasons = data.get('reasons') + if reasons: + detail = f'{detail} (reasons: {json.dumps(reasons, ensure_ascii=False)})' + return detail + return resp.text + + +class mDastMicroservices(mDastBase): + """API client for the Clark facade of the microservices installation.""" + + def __init__(self, base_url, ci_token, company_id=None, user_agent=None, verify=False): + super().__init__(base_url) + # company_id is accepted for interface parity with mDastToken and ignored: + # the organization is resolved server-side from the token (X-Organization-Id). + self.company_id = company_id + self.current_context = {'company': company_id} + self.verify = verify + self.timeout = HTTP_REQUEST_TIMEOUT + self.headers = {'Authorization': 'Bearer {0}'.format(ci_token), + 'Content-Type': 'application/json'} + if user_agent: + self.headers['User-Agent'] = user_agent + + def set_headers(self, ci_token): + self.headers['Authorization'] = 'Bearer {0}'.format(ci_token) + + def _request_headers(self, json_body=True): + headers = {'Authorization': self.headers['Authorization']} + if 'User-Agent' in self.headers: + headers['User-Agent'] = self.headers['User-Agent'] + if json_body: + headers['Content-Type'] = 'application/json' + return headers + + # --- preflight --- + + def get_architectures(self): + return requests.get(f'{self.url}/architectures/', + headers=self.headers, + verify=self.verify, + timeout=self.timeout) + + def get_engines(self): + return requests.get(f'{self.url}/engines/', + headers=self.headers, + verify=self.verify, + timeout=self.timeout) + + def get_testcase(self, testcase_id): + return requests.get(f'{self.url}/testcases/{testcase_id}/', + headers=self.headers, + verify=self.verify, + timeout=self.timeout) + + # --- application upload / dedup (STG-4451) --- + + def check_app_md5(self, org_id, md5): + # org_id is ignored: no organization segment in facade URLs (DEC-694-05) + return requests.get(f'{self.url}/applications/?md5={md5}', + headers=self.headers, + verify=self.verify, + timeout=self.timeout) + + def upload_application(self, path, architecture_type=None, upload_timeout=None): + """Upload via POST /rest/applications/upload_info/. + + Contract (STG-4451 / STG-4450): multipart field `md5` (lower-case hex) + strictly before `file`; `X-File-Size` header with the exact binary size; + Content-Length is set by requests (body is fully built, no chunked TE). + `architecture_type` is accepted for interface parity and not sent: + the platform parses the binary itself. + """ + md5 = file_md5(path) + file_size = os.path.getsize(path) + headers = self._request_headers(json_body=False) + headers['X-File-Size'] = str(file_size) + params = {} + request_timeout = HTTP_DOWNLOAD_TIMEOUT + if upload_timeout is not None: + params['upload_timeout'] = int(upload_timeout) + request_timeout = int(upload_timeout) + UPLOAD_EXTRA_TIMEOUT + with open(path, 'rb') as f: + # ordered list keeps `md5` as the first multipart part (facade requirement) + multipart = [ + ('md5', (None, md5)), + ('file', (os.path.split(path)[-1], f, 'application/octet-stream')), + ] + return requests.post(f'{self.url}/applications/upload_info/', + headers=headers, + files=multipart, + params=params or None, + verify=self.verify, + timeout=request_timeout) + + # --- scan flow (STG-4475) --- + + def precheck_scan(self, md5, profile_id, testcase_id): + """Gate pre-check before scan creation (DEC-671-05). + + Field is `application_md5` in the scanyon contract (not `md5`). + """ + data = { + 'application_md5': md5, + 'profile_id': profile_id, + 'type': 'AUTO', + 'testcase_id': testcase_id, + } + return requests.post(f'{self.url}/scans/start/precheck/', + headers=self.headers, + data=json.dumps(data), + verify=self.verify, + timeout=self.timeout) + + def _create_scan(self, project_id, profile_id, app_md5, test_case_id): + # testcase_id key is mandatory in ScanIn even when null + data = { + 'md5': app_md5, + 'type': 'AUTO', + 'testcase_id': test_case_id, + 'fsm_locked': True, + } + if project_id: + data['project_id'] = project_id + if profile_id: + data['profile_id'] = profile_id + return requests.post(f'{self.url}/scans/start/', + headers=self.headers, + data=json.dumps(data), + verify=self.verify, + timeout=self.timeout) + + def create_manual_scan(self, project_id, profile_id, app_md5, arch_id=None): + # arch_id accepted and ignored (microservices resolve platform from app metadata) + return self._create_scan(project_id, profile_id, app_md5, None) + + def create_auto_scan(self, project_id, profile_id, app_md5, arch_id, test_case_id): + return self._create_scan(project_id, profile_id, app_md5, test_case_id) + + def create_appium_scan(self, project_id, profile_id, app_id, arch_id, appium_script_path): + raise NotImplementedError( + 'Appium scans are not supported on the microservices installation (OOS-671-06)') + + def start_scan(self, dast_id): + return requests.post(f'{self.url}/scans/{dast_id}/start/', + headers=self.headers, + verify=self.verify, + timeout=self.timeout) + + def stop_scan(self, scan_id): + return requests.post(f'{self.url}/scans/{scan_id}/stop/', + headers=self.headers, + verify=self.verify, + timeout=self.timeout) + + def get_scan_info(self, scan_id): + return requests.get(f'{self.url}/scans/{scan_id}/', + headers=self.headers, + verify=self.verify, + timeout=self.timeout) + + # --- reports (STG-4478) --- + + def _download_report_output(self, dast_id, output): + return requests.get(f'{self.url}/scans/{dast_id}/report', + params={'output': output}, + allow_redirects=True, + headers=self.headers, + verify=self.verify, + timeout=HTTP_DOWNLOAD_TIMEOUT) + + def download_report(self, dast_id): + return self._download_report_output(dast_id, 'pdf') + + def download_scan_json_result(self, dast_id): + return self._download_report_output(dast_id, 'json') diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..2c44689 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +testpaths = tests +markers = + unit: fast isolated tests of a single module + contract: tests pinning the HTTP contract with the Clark facade / monolith + smoke: end-to-end CLI flows against a mocked server + security: security-focused checks (secrets handling, header trust, TLS) +filterwarnings = + ignore::DeprecationWarning diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..71ff239 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +pytest>=8.0 +responses>=0.25 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9c44fb3 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,101 @@ +import hashlib +import sys + +import pytest +import responses as responses_lib + +BASE_URL = 'https://stand.example' +REST_URL = f'{BASE_URL}/rest' +TOKEN = 'very-secret-org-token-value-1234567890' +COMPANY_ID = '1' + + +@pytest.fixture +def tmp_apk(tmp_path): + path = tmp_path / 'app.apk' + path.write_bytes(b'PK\x03\x04' + b'binary-payload' * 128) + return str(path) + + +@pytest.fixture +def apk_md5(tmp_apk): + with open(tmp_apk, 'rb') as f: + return hashlib.md5(f.read()).hexdigest().lower() + + +@pytest.fixture +def mocked_responses(): + with responses_lib.RequestsMock(assert_all_requests_are_fired=False) as rsps: + yield rsps + + +@pytest.fixture +def no_sleep(monkeypatch): + monkeypatch.setattr('time.sleep', lambda *_args, **_kwargs: None) + + +@pytest.fixture +def ms_mode(monkeypatch): + monkeypatch.setenv('MDAST_CLI_MODE', 'microservices') + + +@pytest.fixture +def monolith_mode(monkeypatch): + monkeypatch.setenv('MDAST_CLI_MODE', 'monolith') + + +def run_main(monkeypatch, argv): + """Run mdast_scan.main() with the given CLI args, return the exit code. + + The monolith success path returns from main() without sys.exit - that is + process exit code 0. + """ + from mdast_cli import mdast_scan + monkeypatch.setattr(sys, 'argv', ['mdast_cli'] + argv) + try: + mdast_scan.main() + except SystemExit as excinfo: + return int(excinfo.code or 0) + return 0 + + +def scan_json(scan_id=77, stage='WORKING', status='PROCESSING', message=None, **extra): + payload = { + 'id': scan_id, + 'stage': stage, + 'status': status, + 'message': message, + 'md5': 'a' * 32, + 'type': 'AUTO', + 'testcase_id': None, + 'storage_id': None, + 'project': {'id': 1, 'name': 'proj'}, + 'profile': {'id': 2, 'name': 'prof'}, + 'fsm_locked': False, + } + payload.update(extra) + return payload + + +def application_json(md5, app_id=10): + return { + 'id': app_id, + 'architecture_type': 'Android', + 'file_url': 'https://s3.example/presigned', + 'icon_url': None, + 'name': 'Example App', + 'package_name': 'com.example.app', + 'md5': md5, + 'file_size': 1234, + 'version_code': 1, + 'version_name': '1.0', + 'min_sdk_version': '21', + 'max_sdk_version': None, + 'target_sdk_version': '33', + 'main_activity': 'com.example.Main', + 'main_activities_objects': [], + 'is_debuggable': False, + 'signature': 'abc', + 'is_parked': False, + 'distribution_type': 0, + } diff --git a/tests/test_client_microservices.py b/tests/test_client_microservices.py new file mode 100644 index 0000000..7593e1c --- /dev/null +++ b/tests/test_client_microservices.py @@ -0,0 +1,135 @@ +"""Unit/contract tests for the microservices client (Clark facade URLs, bodies, headers).""" +import json + +import pytest +import responses + +from mdast_cli_core.microservices import file_md5, mDastMicroservices +from tests.conftest import REST_URL, TOKEN + +pytestmark = [pytest.mark.unit, pytest.mark.contract] + + +@pytest.fixture +def client(): + return mDastMicroservices(REST_URL, TOKEN, company_id=1, user_agent='mdast_cli/test') + + +def last_request(rsps): + return rsps.calls[-1].request + + +def test_bearer_auth_scheme(client, mocked_responses): + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', json=[]) + client.get_architectures() + assert last_request(mocked_responses).headers['Authorization'] == f'Bearer {TOKEN}' + + +def test_user_agent_is_sent(client, mocked_responses): + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', json=[]) + client.get_architectures() + assert last_request(mocked_responses).headers['User-Agent'] == 'mdast_cli/test' + + +def test_engines_url_has_no_organization_segment(client, mocked_responses): + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', json=[]) + client.get_engines() + assert '/organizations/' not in last_request(mocked_responses).url + + +def test_dedup_url(client, mocked_responses): + mocked_responses.add(responses.GET, f'{REST_URL}/applications/', json=[]) + client.check_app_md5(None, 'a' * 32) + request = last_request(mocked_responses) + assert request.url == f'{REST_URL}/applications/?md5={"a" * 32}' + assert '/organizations/' not in request.url + + +def test_testcase_url(client, mocked_responses): + mocked_responses.add(responses.GET, f'{REST_URL}/testcases/5/', json={'os': 'ANDROID'}) + client.get_testcase(5) + assert last_request(mocked_responses).url == f'{REST_URL}/testcases/5/' + + +def test_create_scan_body_auto(client, mocked_responses): + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', json={'id': 1}, status=200) + client.create_auto_scan(project_id=3, profile_id=4, app_md5='b' * 32, + arch_id=999, test_case_id=5) + body = json.loads(last_request(mocked_responses).body) + assert body == {'md5': 'b' * 32, 'type': 'AUTO', 'testcase_id': 5, + 'fsm_locked': True, 'project_id': 3, 'profile_id': 4} + + +def test_create_scan_body_manual_testcase_key_is_null(client, mocked_responses): + """ScanIn requires the testcase_id key to be present even when null.""" + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', json={'id': 1}, status=200) + client.create_manual_scan(project_id=None, profile_id=None, app_md5='b' * 32) + body = json.loads(last_request(mocked_responses).body) + assert body == {'md5': 'b' * 32, 'type': 'AUTO', 'testcase_id': None, 'fsm_locked': True} + assert 'architecture_id' not in body + assert 'application_id' not in body + + +def test_precheck_body_uses_application_md5_field(client, mocked_responses): + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/precheck/', + json={'warnings': []}) + client.precheck_scan('c' * 32, profile_id=7, testcase_id=None) + body = json.loads(last_request(mocked_responses).body) + assert body == {'application_md5': 'c' * 32, 'profile_id': 7, + 'type': 'AUTO', 'testcase_id': None} + + +def test_scan_lifecycle_urls(client, mocked_responses): + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/start/', json={'id': 77}) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/stop/', json={'id': 77}) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', json={'id': 77}) + client.start_scan(77) + client.stop_scan(77) + client.get_scan_info(77) + urls = [call.request.url for call in mocked_responses.calls] + assert urls == [f'{REST_URL}/scans/77/start/', f'{REST_URL}/scans/77/stop/', + f'{REST_URL}/scans/77/'] + + +def test_report_urls_and_output_param(client, mocked_responses): + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', body=b'%PDF') + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', json={'summary': {}}) + client.download_report(77) + client.download_scan_json_result(77) + assert mocked_responses.calls[0].request.url == f'{REST_URL}/scans/77/report?output=pdf' + assert mocked_responses.calls[1].request.url == f'{REST_URL}/scans/77/report?output=json' + + +def test_upload_multipart_md5_first_and_headers(client, mocked_responses, tmp_apk, apk_md5): + mocked_responses.add(responses.POST, f'{REST_URL}/applications/upload_info/', + json={'id': 10}, status=201) + client.upload_application(tmp_apk) + request = last_request(mocked_responses) + body = request.body if isinstance(request.body, bytes) else request.body.read() + md5_pos = body.index(b'name="md5"') + file_pos = body.index(b'name="file"') + assert md5_pos < file_pos, 'md5 multipart field must precede file (facade contract)' + assert apk_md5.encode() in body + import os + assert request.headers['X-File-Size'] == str(os.path.getsize(tmp_apk)) + assert int(request.headers['Content-Length']) >= os.path.getsize(tmp_apk) + assert 'chunked' not in request.headers.get('Transfer-Encoding', '') + + +def test_upload_timeout_param(client, mocked_responses, tmp_apk): + mocked_responses.add(responses.POST, f'{REST_URL}/applications/upload_info/', + json={'id': 10}, status=201) + client.upload_application(tmp_apk, upload_timeout=120) + assert 'upload_timeout=120' in last_request(mocked_responses).url + + +def test_file_md5_is_lower_case_hex(tmp_apk): + digest = file_md5(tmp_apk) + assert digest == digest.lower() + assert len(digest) == 32 + int(digest, 16) + + +def test_appium_scan_not_supported(client): + with pytest.raises(NotImplementedError): + client.create_appium_scan(1, 2, 3, 4, '/tmp/script.py') diff --git a/tests/test_error_envelopes.py b/tests/test_error_envelopes.py new file mode 100644 index 0000000..9e9f7fc --- /dev/null +++ b/tests/test_error_envelopes.py @@ -0,0 +1,53 @@ +"""Contract tests for Clark error envelope parsing. + +/rest routes answer with the monolith envelope {error_code, message}; +the report route answers with FastAPI {"detail": ...}; +scanyon 429 body carries {"detail", "reasons"}. +""" +import pytest +import responses +import requests + +from mdast_cli_core.microservices import extract_error_message + +pytestmark = pytest.mark.contract + +URL = 'https://stand.example/x' + + +def _resp(mocked_responses, **kwargs): + mocked_responses.add(responses.GET, URL, **kwargs) + return requests.get(URL) + + +def test_monolith_envelope(mocked_responses): + message = extract_error_message(_resp( + mocked_responses, status=413, + json={'error_code': 'file_too_large', 'message': 'File exceeds 500 MB limit'})) + assert message == 'file_too_large: File exceeds 500 MB limit' + + +def test_fastapi_detail_string(mocked_responses): + message = extract_error_message(_resp( + mocked_responses, status=409, json={'detail': 'scan is not finished'})) + assert message == 'scan is not finished' + + +def test_fastapi_detail_validation_list(mocked_responses): + message = extract_error_message(_resp( + mocked_responses, status=422, + json={'detail': [{'loc': ['body', 'md5'], 'msg': 'field required'}]})) + assert 'field required' in message + + +def test_429_reasons_preserved(mocked_responses): + message = extract_error_message(_resp( + mocked_responses, status=429, + json={'detail': 'Scan limit exceeded', 'reasons': [{'limit': 'scans_per_day'}]})) + assert 'Scan limit exceeded' in message + assert 'scans_per_day' in message + + +def test_non_json_body(mocked_responses): + message = extract_error_message(_resp(mocked_responses, status=502, body='Bad Gateway')) + assert message == 'Bad Gateway' diff --git a/tests/test_mode_detection.py b/tests/test_mode_detection.py new file mode 100644 index 0000000..361bd6f --- /dev/null +++ b/tests/test_mode_detection.py @@ -0,0 +1,63 @@ +"""Unit tests for installation mode auto-detection and env override.""" +import pytest +import responses + +from mdast_cli_core.factory import (MODE_MICROSERVICES, MODE_MONOLITH, ModeDetectionError, + resolve_installation_mode) +from tests.conftest import COMPANY_ID, REST_URL, TOKEN + +pytestmark = pytest.mark.unit + + +def test_env_override_microservices(monkeypatch): + monkeypatch.setenv('MDAST_CLI_MODE', 'microservices') + assert resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) == MODE_MICROSERVICES + + +def test_env_override_monolith(monkeypatch): + monkeypatch.setenv('MDAST_CLI_MODE', 'monolith') + assert resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) == MODE_MONOLITH + + +def test_env_invalid_value(monkeypatch): + monkeypatch.setenv('MDAST_CLI_MODE', 'quantum') + with pytest.raises(ModeDetectionError): + resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) + + +def test_autodetect_microservices(mocked_responses, monkeypatch): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', json=[]) + assert resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) == MODE_MICROSERVICES + probe = mocked_responses.calls[0].request + assert probe.headers['Authorization'] == f'Bearer {TOKEN}' + + +def test_autodetect_monolith_fallback(mocked_responses, monkeypatch): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', status=404) + mocked_responses.add(responses.GET, f'{REST_URL}/organizations/{COMPANY_ID}/engines/', + json=[]) + assert resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) == MODE_MONOLITH + monolith_probe = mocked_responses.calls[1].request + assert monolith_probe.headers['Authorization'] == f'Token {TOKEN}' + + +def test_autodetect_failure_reports_auth_error(mocked_responses, monkeypatch): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', status=401) + mocked_responses.add(responses.GET, f'{REST_URL}/organizations/{COMPANY_ID}/engines/', + status=401) + with pytest.raises(ModeDetectionError) as excinfo: + resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) + assert excinfo.value.auth_error is True + + +def test_autodetect_failure_network(mocked_responses, monkeypatch): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', status=404) + mocked_responses.add(responses.GET, f'{REST_URL}/organizations/{COMPANY_ID}/engines/', + status=500) + with pytest.raises(ModeDetectionError) as excinfo: + resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) + assert excinfo.value.auth_error is False diff --git a/tests/test_precheck_gate.py b/tests/test_precheck_gate.py new file mode 100644 index 0000000..6d59626 --- /dev/null +++ b/tests/test_precheck_gate.py @@ -0,0 +1,89 @@ +"""Unit/contract tests for the pre-check gate (DEC-671-05) and warning rendering.""" +import pytest +import responses + +from mdast_cli.helpers.exit_codes import ExitCode +from mdast_cli.ms_flow import render_precheck_warning, run_precheck_gate +from mdast_cli_core.microservices import mDastMicroservices +from tests.conftest import REST_URL, TOKEN + +pytestmark = [pytest.mark.unit, pytest.mark.contract] + +PRECHECK_URL = f'{REST_URL}/scans/start/precheck/' + + +@pytest.fixture +def client(): + return mDastMicroservices(REST_URL, TOKEN) + + +def test_empty_warnings_pass(client, mocked_responses): + mocked_responses.add(responses.POST, PRECHECK_URL, json={'warnings': []}) + run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None) + + +def test_warnings_block_with_exit_code_8(client, mocked_responses, capsys): + mocked_responses.add(responses.POST, PRECHECK_URL, json={'warnings': [ + {'type': 'empty_profile', 'payload': {}}, + {'type': 'architecture_unsupported', + 'payload': {'platform': 'Android', 'os_version': '99'}}, + ]}) + with pytest.raises(SystemExit) as excinfo: + run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None) + assert excinfo.value.code == ExitCode.PRECHECK_BLOCKED == 8 + stderr = capsys.readouterr().err + assert '[empty_profile]' in stderr + assert '[architecture_unsupported]' in stderr + assert '"os_version": "99"' in stderr + + +def test_precheck_unavailable_blocks(client, mocked_responses): + """precheck_unavailable is a blocking warning too (STG-4475).""" + mocked_responses.add(responses.POST, PRECHECK_URL, json={'warnings': [ + {'type': 'precheck_unavailable', + 'payload': {'unavailable_dependencies': ['application_lookup']}}, + ]}) + with pytest.raises(SystemExit) as excinfo: + run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None) + assert excinfo.value.code == ExitCode.PRECHECK_BLOCKED + + +def test_precheck_transport_error_blocks(client, mocked_responses): + mocked_responses.add(responses.POST, PRECHECK_URL, status=502, + json={'error_code': 'downstream_unavailable', 'message': 'Scanyon down'}) + with pytest.raises(SystemExit) as excinfo: + run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None) + assert excinfo.value.code == ExitCode.PRECHECK_BLOCKED + + +def test_profile_not_found_fails(client, mocked_responses): + mocked_responses.add(responses.POST, PRECHECK_URL, status=404, + json={'error_code': 'not_found', 'message': 'profile not found'}) + with pytest.raises(SystemExit) as excinfo: + run_precheck_gate(client, 'a' * 32, profile_id=42, testcase_id=None) + assert excinfo.value.code == ExitCode.SCAN_FAILED + + +def test_missing_profile_id_skips_on_422(client, mocked_responses, caplog): + """Interim behavior until scanyon makes profile_id optional: 422 + no profile -> skip.""" + mocked_responses.add(responses.POST, PRECHECK_URL, status=422, + json={'error_code': 'validation_error', 'message': 'profile_id required'}) + run_precheck_gate(client, 'a' * 32, profile_id=None, testcase_id=None) + assert any('pre-check skipped' in record.message.lower() for record in caplog.records) + + +def test_unknown_warning_type_rendered_generically(): + """Contract extension is backward-compatible: unknown types must not crash (DEC-666-04).""" + line = render_precheck_warning({'type': 'brand_new_rule', 'payload': {'x': 1}}) + assert '[brand_new_rule]' in line + assert '"x": 1' in line + + +def test_warning_without_payload_rendered(): + line = render_precheck_warning({'type': 'empty_profile'}) + assert '[empty_profile]' in line + + +def test_malformed_warning_rendered(): + line = render_precheck_warning({}) + assert '[unknown]' in line diff --git a/tests/test_security.py b/tests/test_security.py new file mode 100644 index 0000000..bfa4ff0 --- /dev/null +++ b/tests/test_security.py @@ -0,0 +1,116 @@ +"""Security-focused tests: secrets handling, server-controlled inputs, TLS.""" +import logging +import os +from unittest import mock + +import pytest +import responses + +from mdast_cli_core import factory +from mdast_cli_core.microservices import mDastMicroservices +from tests.conftest import BASE_URL, REST_URL, TOKEN, application_json, run_main, scan_json +from tests.test_smoke_flows import ms_argv, register_ms_happy_path + +pytestmark = pytest.mark.security + + +def test_token_never_appears_in_logs(mocked_responses, monkeypatch, tmp_path, tmp_apk, + apk_md5, no_sleep, ms_mode, caplog, capsys): + """The org token is a credential: it must not leak to logs, stdout or stderr.""" + monkeypatch.chdir(tmp_path) + register_ms_happy_path(mocked_responses, apk_md5) + with caplog.at_level(logging.DEBUG): + exit_code = run_main(monkeypatch, ms_argv(tmp_apk) + ['--verbose']) + assert exit_code == 0 + for record in caplog.records: + assert TOKEN not in record.getMessage() + captured = capsys.readouterr() + assert TOKEN not in captured.out + assert TOKEN not in captured.err + + +def test_token_never_in_urls(mocked_responses, monkeypatch, tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode): + """The token travels only in the Authorization header, never in a URL/query.""" + monkeypatch.chdir(tmp_path) + register_ms_happy_path(mocked_responses, apk_md5) + run_main(monkeypatch, ms_argv(tmp_apk)) + for call in mocked_responses.calls: + assert TOKEN not in call.request.url + assert call.request.headers['Authorization'] == f'Bearer {TOKEN}' + + +def test_content_disposition_cannot_control_local_path(mocked_responses, monkeypatch, + tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode): + """A hostile/compromised server must not choose where the CLI writes files. + + Report file names come from CLI arguments only; Content-Disposition + (path traversal attempt here) is ignored. + """ + monkeypatch.chdir(tmp_path) + register_ms_happy_path(mocked_responses, apk_md5) + mocked_responses.replace( + responses.GET, f'{REST_URL}/scans/77/report', body=b'%PDF-1.4 fake', + headers={'Content-Disposition': 'attachment; filename="../../../../tmp/evil.pdf"'}) + exit_code = run_main(monkeypatch, ms_argv(tmp_apk)) + assert exit_code == 0 + written = sorted(p.name for p in tmp_path.iterdir()) + assert 'scan_report_pdf.pdf' in written + assert not any('evil' in name for name in written) + assert not os.path.exists('/tmp/evil.pdf') + + +def test_x_file_size_matches_actual_content(mocked_responses, tmp_apk): + """Anti request smuggling consistency: X-File-Size equals the real binary size.""" + mocked_responses.add(responses.POST, f'{REST_URL}/applications/upload_info/', + json={'id': 1}, status=201) + client = mDastMicroservices(REST_URL, TOKEN) + client.upload_application(tmp_apk) + request = mocked_responses.calls[-1].request + declared = int(request.headers['X-File-Size']) + assert declared == os.path.getsize(tmp_apk) + assert declared <= int(request.headers['Content-Length']) + + +def test_tls_verify_disabled_by_default(): + client = mDastMicroservices(REST_URL, TOKEN) + with mock.patch('mdast_cli_core.microservices.requests.get') as mocked_get: + client.get_architectures() + assert mocked_get.call_args.kwargs['verify'] is False + + +def test_tls_verify_enabled_via_client_flag(): + client = mDastMicroservices(REST_URL, TOKEN, verify=True) + with mock.patch('mdast_cli_core.microservices.requests.get') as mocked_get: + client.get_architectures() + assert mocked_get.call_args.kwargs['verify'] is True + + +@pytest.mark.parametrize('raw,expected', [ + ('1', True), ('true', True), ('YES', True), ('on', True), + ('', False), ('0', False), ('false', False), ('off', False), +]) +def test_tls_verify_env_parsing(monkeypatch, raw, expected): + monkeypatch.setenv(factory.TLS_VERIFY_ENV_VAR, raw) + assert factory.tls_verify_enabled() is expected + + +def test_precheck_payload_with_control_characters_is_safe(): + """Warning payload is server-controlled: rendering must be json-escaped, one line.""" + from mdast_cli.ms_flow import render_precheck_warning + line = render_precheck_warning({ + 'type': 'empty_profile', + 'payload': {'note': 'line1\nline2\x1b[31mred'}, + }) + assert '\n' not in line + assert '\x1b' not in line + + +def test_mode_probe_does_not_leak_token_on_error(mocked_responses, monkeypatch): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', status=401) + mocked_responses.add(responses.GET, f'{REST_URL}/organizations/1/engines/', status=401) + with pytest.raises(factory.ModeDetectionError) as excinfo: + factory.resolve_installation_mode(REST_URL, TOKEN, '1') + assert TOKEN not in str(excinfo.value) diff --git a/tests/test_smoke_flows.py b/tests/test_smoke_flows.py new file mode 100644 index 0000000..0164a81 --- /dev/null +++ b/tests/test_smoke_flows.py @@ -0,0 +1,196 @@ +"""Smoke tests: full CLI flows against a mocked server. + +The microservices flow follows the acceptance scenario of STG-4475: +architectures -> engines -> dedup/upload -> pre-check gate -> create (fsm_locked) +-> start -> polling to a terminal (stage, status) -> reports. +The monolith flow is pinned as a regression guard (dual-mode must not break it). +""" +import json +import os + +import pytest +import responses + +from tests.conftest import BASE_URL, REST_URL, TOKEN, application_json, run_main, scan_json + +pytestmark = pytest.mark.smoke + + +def register_ms_happy_path(rsps, apk_md5, with_testcase=True): + rsps.add(responses.GET, f'{REST_URL}/architectures/', json=[ + {'id': 1, 'type': 1, 'os_version': '11', 'name': 'Android 11', 'description': 'API 30'}, + ]) + if with_testcase: + rsps.add(responses.GET, f'{REST_URL}/testcases/5/', json={'id': 5, 'os': 'ANDROID'}) + rsps.add(responses.GET, f'{REST_URL}/engines/', json=[ + {'engine_id': 'e-1', 'type': 'ANDROID', 'status': 'STARTED'}, + {'engine_id': 'e-2', 'type': 'IOS', 'status': 'STOPPED_SIGTERM'}, + ]) + rsps.add(responses.GET, f'{REST_URL}/applications/', json=[]) + rsps.add(responses.POST, f'{REST_URL}/applications/upload_info/', + json=application_json(apk_md5), status=201) + rsps.add(responses.POST, f'{REST_URL}/scans/start/precheck/', json={'warnings': []}) + rsps.add(responses.POST, f'{REST_URL}/scans/start/', + json=scan_json(stage='CREATED', status='INITIAL'), status=200) + rsps.add(responses.POST, f'{REST_URL}/scans/77/start/', + json=scan_json(stage='START', status='INITIAL')) + rsps.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='WORKING', status='PROCESSING')) + rsps.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='SUCCESS', status='COMPLETE')) + rsps.add(responses.GET, f'{REST_URL}/scans/77/report', body=b'%PDF-1.4 fake', + content_type='application/pdf', + headers={'Content-Disposition': 'attachment; filename="dast_77.pdf"'}) + rsps.add(responses.GET, f'{REST_URL}/scans/77/report', + json={'summary': {'scan_id': 77}, 'defect_summary': {}, 'requirements': [], + 'defects': []}, + headers={'Content-Disposition': 'attachment; filename="dast_77.json"'}) + + +def ms_argv(tmp_apk, with_testcase=True): + argv = ['--distribution_system', 'file', '--file_path', tmp_apk, + '--url', BASE_URL, '--company_id', '1', '--token', TOKEN, + '--profile_id', '2', + '--pdf_report_file_name', 'scan_report_pdf', + '--summary_report_json_file_name', 'scan_report_json'] + if with_testcase: + argv += ['--testcase_id', '5'] + return argv + + +def test_ms_full_flow_success(mocked_responses, monkeypatch, tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode): + monkeypatch.chdir(tmp_path) + register_ms_happy_path(mocked_responses, apk_md5) + exit_code = run_main(monkeypatch, ms_argv(tmp_apk)) + assert exit_code == 0 + assert (tmp_path / 'scan_report_pdf.pdf').read_bytes().startswith(b'%PDF') + report = json.loads((tmp_path / 'scan_report_json.json').read_text()) + assert report['summary']['scan_id'] == 77 + # create scan body follows the scanyon-native contract + create_calls = [c for c in mocked_responses.calls + if c.request.url == f'{REST_URL}/scans/start/'] + body = json.loads(create_calls[0].request.body) + assert body['md5'] == apk_md5 + assert body['type'] == 'AUTO' + assert body['fsm_locked'] is True + + +def test_ms_full_flow_with_autodetect(mocked_responses, monkeypatch, tmp_path, tmp_apk, + apk_md5, no_sleep): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + monkeypatch.chdir(tmp_path) + register_ms_happy_path(mocked_responses, apk_md5) + exit_code = run_main(monkeypatch, ms_argv(tmp_apk)) + assert exit_code == 0 + + +def test_ms_manual_flow_stops_scan(mocked_responses, monkeypatch, tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode): + """Scan without a test case keeps the manual semantics: wait, stop, expect SUCCESS.""" + monkeypatch.chdir(tmp_path) + register_ms_happy_path(mocked_responses, apk_md5, with_testcase=False) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/stop/', + json=scan_json(stage='STOP', status='PROCESSING')) + exit_code = run_main(monkeypatch, ms_argv(tmp_apk, with_testcase=False)) + assert exit_code == 0 + stop_calls = [c for c in mocked_responses.calls + if c.request.url == f'{REST_URL}/scans/77/stop/'] + assert stop_calls, 'manual flow must request scan stop' + + +def test_ms_precheck_gate_blocks(mocked_responses, monkeypatch, tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode, capsys): + monkeypatch.chdir(tmp_path) + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', json=[]) + mocked_responses.add(responses.GET, f'{REST_URL}/testcases/5/', json={'os': 'ANDROID'}) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', + json=[{'type': 'ANDROID', 'status': 'STARTED'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/applications/', + json=[application_json(apk_md5)]) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/precheck/', json={ + 'warnings': [{'type': 'empty_profile', 'payload': {}}]}) + exit_code = run_main(monkeypatch, ms_argv(tmp_apk)) + assert exit_code == 8 + assert '[empty_profile]' in capsys.readouterr().err + scan_creates = [c for c in mocked_responses.calls + if c.request.url == f'{REST_URL}/scans/start/'] + assert not scan_creates, 'scan must not be created when pre-check blocks' + + +def test_ms_scan_fail_state(mocked_responses, monkeypatch, tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode): + monkeypatch.chdir(tmp_path) + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', json=[]) + mocked_responses.add(responses.GET, f'{REST_URL}/testcases/5/', json={'os': 'ANDROID'}) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', + json=[{'type': 'ANDROID', 'status': 'STARTED'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/applications/', + json=[application_json(apk_md5)]) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/precheck/', + json={'warnings': []}) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', + json=scan_json(stage='CREATED', status='INITIAL')) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/start/', + json=scan_json(stage='START')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='FAIL', status='FAIL', + message='All modules failed')) + exit_code = run_main(monkeypatch, ms_argv(tmp_apk)) + assert exit_code == 5 + + +def test_ms_appium_rejected(monkeypatch, tmp_apk, ms_mode): + exit_code = run_main(monkeypatch, [ + '--distribution_system', 'file', '--file_path', tmp_apk, + '--url', BASE_URL, '--company_id', '1', '--token', TOKEN, + '--appium_script_path', '/tmp/script.py']) + assert exit_code == 2 + + +def test_ms_cr_report_rejected(monkeypatch, tmp_apk, ms_mode): + exit_code = run_main(monkeypatch, [ + '--distribution_system', 'file', '--file_path', tmp_apk, + '--url', BASE_URL, '--company_id', '1', '--token', TOKEN, + '--cr_report', '--stingray_login', 'x', '--stingray_password', 'y']) + assert exit_code == 2 + + +def test_monolith_regression_flow(mocked_responses, monkeypatch, tmp_path, tmp_apk, + no_sleep, monolith_mode): + """The pre-dual-mode monolith flow must keep working byte-for-byte.""" + monkeypatch.chdir(tmp_path) + rest = REST_URL + mocked_responses.add(responses.GET, f'{rest}/architectures/', json=[ + {'id': 1, 'name': 'Android 11', 'type': 1}, + {'id': 3, 'name': 'iOS 14', 'type': 2}, + ]) + mocked_responses.add(responses.GET, f'{rest}/organizations/1/engines/', json=[ + {'architecture': 1, 'state': 3}, + ]) + mocked_responses.add(responses.GET, f'{rest}/organizations/1/applications/', json=[]) + mocked_responses.add(responses.POST, f'{rest}/organizations/1/applications/', + json={'id': 10}, status=201) + mocked_responses.add(responses.POST, f'{rest}/organizations/1/dasts/', + json={'id': 77, 'project': {'id': 1}, 'profile': {'id': 2}}, + status=201) + mocked_responses.add(responses.POST, f'{rest}/dasts/77/start/', json={}, status=200) + mocked_responses.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 2}) + mocked_responses.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 2}) + mocked_responses.add(responses.POST, f'{rest}/dasts/77/stop/', json={}, status=200) + mocked_responses.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 4}) + mocked_responses.add(responses.GET, f'{rest}/dasts/77/report/', body=b'%PDF-1.4 mono') + mocked_responses.add(responses.GET, f'{rest}/dasts/77/report/', json={'summary': {}}) + exit_code = run_main(monkeypatch, [ + '--distribution_system', 'file', '--file_path', tmp_apk, + '--url', BASE_URL, '--company_id', '1', '--token', TOKEN, + '--pdf_report_file_name', 'mono_pdf', + '--summary_report_json_file_name', 'mono_json']) + assert exit_code == 0 + assert (tmp_path / 'mono_pdf.pdf').read_bytes().startswith(b'%PDF') + assert (tmp_path / 'mono_json.json').exists() + # monolith URL shapes preserved: organization segment and Token scheme + engine_calls = [c for c in mocked_responses.calls + if '/organizations/1/engines/' in c.request.url] + assert engine_calls + assert engine_calls[0].request.headers['Authorization'] == f'Token {TOKEN}' diff --git a/tests/test_state_machine.py b/tests/test_state_machine.py new file mode 100644 index 0000000..affda77 --- /dev/null +++ b/tests/test_state_machine.py @@ -0,0 +1,57 @@ +"""Unit tests for the scanyon-native (stage, status) state machine helpers.""" +import pytest + +from mdast_cli.helpers.const import TERMINAL_SCAN_PAIRS +from mdast_cli.ms_flow import is_success, is_terminal, resolve_platform, scan_pair + +pytestmark = pytest.mark.unit + + +@pytest.mark.parametrize('stage,status', sorted(TERMINAL_SCAN_PAIRS)) +def test_terminal_pairs(stage, status): + assert is_terminal({'stage': stage, 'status': status}) + + +@pytest.mark.parametrize('stage,status', [ + ('CREATED', 'INITIAL'), + ('START', 'PROCESSING'), + ('WORKING', 'PROCESSING'), + ('WORKING', 'WAITING'), + ('STOP', 'INITIAL'), + ('STOP', 'PROCESSING'), + # STOP is transitional even with COMPLETE status: the FSM proceeds to SUCCESS + ('STOP', 'COMPLETE'), + ('SUCCESS', 'PROCESSING'), + ('FAIL', 'PROCESSING'), +]) +def test_non_terminal_pairs(stage, status): + assert not is_terminal({'stage': stage, 'status': status}) + + +def test_success_complete(): + assert is_success({'stage': 'SUCCESS', 'status': 'COMPLETE'}) + + +def test_success_partial_complete(): + assert is_success({'stage': 'SUCCESS', 'status': 'PARTIAL_COMPLETE'}) + + +def test_fail_is_not_success(): + assert not is_success({'stage': 'FAIL', 'status': 'FAIL'}) + + +def test_scan_pair_missing_fields(): + assert scan_pair({}) == (None, None) + assert not is_terminal({}) + + +@pytest.mark.parametrize('file_name,expected', [ + ('app.apk', 'ANDROID'), + ('app.aab', 'ANDROID'), + ('app.apks', 'ANDROID'), + ('app.zip', 'ANDROID'), + ('app.ipa', 'IOS'), + ('app.exe', None), +]) +def test_resolve_platform(file_name, expected): + assert resolve_platform(f'/tmp/{file_name}') == expected From da520cb9359b4f5c266968f3ec964d205075b5e8 Mon Sep 17 00:00:00 2001 From: Nikita Pinaev Date: Wed, 8 Jul 2026 02:59:52 +0300 Subject: [PATCH 2/6] CI: run pytest as python -m pytest so the repo root is on sys.path --- .github/workflows/docker-hub-publish.yml | 2 +- .github/workflows/python-publish.yml | 2 +- .github/workflows/tests.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-hub-publish.yml b/.github/workflows/docker-hub-publish.yml index 7b28acf..4848837 100644 --- a/.github/workflows/docker-hub-publish.yml +++ b/.github/workflows/docker-hub-publish.yml @@ -15,7 +15,7 @@ jobs: with: python-version: '3.12' - run: pip install -r requirements.txt -r requirements-dev.txt - - run: pytest -q + - run: python -m pytest -q build-n-publish-docker: name: Docker Hub - Publish mdast_cli diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 20a0f43..9f599b7 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -15,7 +15,7 @@ jobs: with: python-version: '3.12' - run: pip install -r requirements.txt -r requirements-dev.txt - - run: pytest -q + - run: python -m pytest -q build-n-publish: name: Build and publish mdast_cli 🐍 distributions to PyPI diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 226ab88..fb4824b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,7 +20,7 @@ jobs: run: pip install -r requirements.txt -r requirements-dev.txt - name: Run full test suite - run: pytest -q + run: python -m pytest -q - name: Security tests gate - run: pytest -q -m security + run: python -m pytest -q -m security From f2fd30e3452cb33571381d63237220da54b47532 Mon Sep 17 00:00:00 2001 From: Nikita Pinaev Date: Wed, 8 Jul 2026 14:43:28 +0300 Subject: [PATCH 3/6] STG-4475: fix scan type (MANUAL vs AUTO) and tolerate non-idempotent start 409 Found by live e2e on dev: - scanyon rejects type=AUTO without testcase_id (services/scan.py: 'Testcase ID should be set for auto scan'). The DEC-671-02 premise 'AUTO is the only CLI mode' is wrong: a scan without a test case must be MANUAL (server sets wait=True, install/launch/wait/stop semantics), only test-case replays are AUTO. create_manual_scan now sends type=MANUAL, precheck mirrors the type. - POST /scans/{id}/start/ is not idempotent; the facade retries it, so a retry after the first (successful) unlock returns 409 'not in initial state' though the scan did start. CLI now confirms scan state on 409 and continues if the scan is past CREATED, fails only on a genuine conflict. - Tests updated (MANUAL body, precheck type) + 2 new smoke tests for the 409 retry-tolerance and genuine-conflict paths. 83 passed. --- mdast_cli/ms_flow.py | 19 +++++++++--- mdast_cli_core/microservices.py | 18 ++++++----- tests/test_client_microservices.py | 11 +++---- tests/test_precheck_gate.py | 12 ++++---- tests/test_smoke_flows.py | 48 ++++++++++++++++++++++++++++++ 5 files changed, 86 insertions(+), 22 deletions(-) diff --git a/mdast_cli/ms_flow.py b/mdast_cli/ms_flow.py index 9a1e64d..405cc7f 100644 --- a/mdast_cli/ms_flow.py +++ b/mdast_cli/ms_flow.py @@ -106,9 +106,9 @@ def _get_scan(mdast, scan_id): return resp.json() -def run_precheck_gate(mdast, md5, profile_id, testcase_id): +def run_precheck_gate(mdast, md5, profile_id, testcase_id, scan_type): """Gate model (DEC-671-05): any warning blocks the scan with a non-zero exit.""" - resp = mdast.precheck_scan(md5, profile_id, testcase_id) + resp = mdast.precheck_scan(md5, profile_id, testcase_id, scan_type) if resp.status_code == 404: _exit_on_http_error(resp, 'Scan pre-check (profile lookup)') if resp.status_code == 422 and profile_id is None: @@ -212,7 +212,8 @@ def run_microservices_flow(arguments, url, token, app_file, appstore_app_md5, us application = upload_resp.json() logger.info(f"Application uploaded successfully. Application id: {application['id']}") - run_precheck_gate(mdast, app_md5, profile_id, testcase_id) + precheck_type = 'AUTO' if testcase_id is not None else 'MANUAL' + run_precheck_gate(mdast, app_md5, profile_id, testcase_id, precheck_type) logger.info(f"Creating scan for application {application['id']}") if testcase_id is not None: @@ -235,7 +236,17 @@ def run_microservices_flow(arguments, url, token, app_file, appstore_app_md5, us logger.info(f'Start scan with id {scan_id}') start_resp = mdast.start_scan(scan_id) - if start_resp.status_code != 200: + if start_resp.status_code == 409: + # POST /scans/{id}/start/ is not idempotent: a facade retry (network/timeout + # after the first call already unlocked the scan) surfaces as 409 "not in + # initial state" even though the scan did start. Confirm by state before failing. + current = _get_scan(mdast, scan_id) + if current.get('stage') != ScanStage.CREATED: + logger.warning(f'Start returned 409 but scan {scan_id} is already past initial ' + f'state ({scan_pair(current)}) - treating as started (facade retry).') + else: + _exit_on_http_error(start_resp, f'Starting scan {scan_id}') + elif start_resp.status_code != 200: _exit_on_http_error(start_resp, f'Starting scan {scan_id}') if arguments.nowait: diff --git a/mdast_cli_core/microservices.py b/mdast_cli_core/microservices.py index df3b2b6..23803c0 100644 --- a/mdast_cli_core/microservices.py +++ b/mdast_cli_core/microservices.py @@ -150,15 +150,17 @@ def upload_application(self, path, architecture_type=None, upload_timeout=None): # --- scan flow (STG-4475) --- - def precheck_scan(self, md5, profile_id, testcase_id): + def precheck_scan(self, md5, profile_id, testcase_id, scan_type): """Gate pre-check before scan creation (DEC-671-05). Field is `application_md5` in the scanyon contract (not `md5`). + scan_type mirrors the create call: MANUAL for a scan without a test case, + AUTO for a test-case replay. """ data = { 'application_md5': md5, 'profile_id': profile_id, - 'type': 'AUTO', + 'type': scan_type, 'testcase_id': testcase_id, } return requests.post(f'{self.url}/scans/start/precheck/', @@ -167,11 +169,13 @@ def precheck_scan(self, md5, profile_id, testcase_id): verify=self.verify, timeout=self.timeout) - def _create_scan(self, project_id, profile_id, app_md5, test_case_id): - # testcase_id key is mandatory in ScanIn even when null + def _create_scan(self, project_id, profile_id, app_md5, test_case_id, scan_type): + # scanyon: AUTO requires testcase_id; a scan without a test case is MANUAL + # (install/launch/wait/stop semantics, wait=True on the server side). + # testcase_id key is mandatory in ScanIn even when null. data = { 'md5': app_md5, - 'type': 'AUTO', + 'type': scan_type, 'testcase_id': test_case_id, 'fsm_locked': True, } @@ -187,10 +191,10 @@ def _create_scan(self, project_id, profile_id, app_md5, test_case_id): def create_manual_scan(self, project_id, profile_id, app_md5, arch_id=None): # arch_id accepted and ignored (microservices resolve platform from app metadata) - return self._create_scan(project_id, profile_id, app_md5, None) + return self._create_scan(project_id, profile_id, app_md5, None, 'MANUAL') def create_auto_scan(self, project_id, profile_id, app_md5, arch_id, test_case_id): - return self._create_scan(project_id, profile_id, app_md5, test_case_id) + return self._create_scan(project_id, profile_id, app_md5, test_case_id, 'AUTO') def create_appium_scan(self, project_id, profile_id, app_id, arch_id, appium_script_path): raise NotImplementedError( diff --git a/tests/test_client_microservices.py b/tests/test_client_microservices.py index 7593e1c..23845f2 100644 --- a/tests/test_client_microservices.py +++ b/tests/test_client_microservices.py @@ -60,12 +60,13 @@ def test_create_scan_body_auto(client, mocked_responses): 'fsm_locked': True, 'project_id': 3, 'profile_id': 4} -def test_create_scan_body_manual_testcase_key_is_null(client, mocked_responses): - """ScanIn requires the testcase_id key to be present even when null.""" +def test_create_scan_body_manual_is_manual_type(client, mocked_responses): + """Scan without a test case is MANUAL (scanyon rejects AUTO without testcase_id); + testcase_id key stays present (null).""" mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', json={'id': 1}, status=200) client.create_manual_scan(project_id=None, profile_id=None, app_md5='b' * 32) body = json.loads(last_request(mocked_responses).body) - assert body == {'md5': 'b' * 32, 'type': 'AUTO', 'testcase_id': None, 'fsm_locked': True} + assert body == {'md5': 'b' * 32, 'type': 'MANUAL', 'testcase_id': None, 'fsm_locked': True} assert 'architecture_id' not in body assert 'application_id' not in body @@ -73,10 +74,10 @@ def test_create_scan_body_manual_testcase_key_is_null(client, mocked_responses): def test_precheck_body_uses_application_md5_field(client, mocked_responses): mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/precheck/', json={'warnings': []}) - client.precheck_scan('c' * 32, profile_id=7, testcase_id=None) + client.precheck_scan('c' * 32, profile_id=7, testcase_id=None, scan_type='MANUAL') body = json.loads(last_request(mocked_responses).body) assert body == {'application_md5': 'c' * 32, 'profile_id': 7, - 'type': 'AUTO', 'testcase_id': None} + 'type': 'MANUAL', 'testcase_id': None} def test_scan_lifecycle_urls(client, mocked_responses): diff --git a/tests/test_precheck_gate.py b/tests/test_precheck_gate.py index 6d59626..a9b9530 100644 --- a/tests/test_precheck_gate.py +++ b/tests/test_precheck_gate.py @@ -19,7 +19,7 @@ def client(): def test_empty_warnings_pass(client, mocked_responses): mocked_responses.add(responses.POST, PRECHECK_URL, json={'warnings': []}) - run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None) + run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None, scan_type='MANUAL') def test_warnings_block_with_exit_code_8(client, mocked_responses, capsys): @@ -29,7 +29,7 @@ def test_warnings_block_with_exit_code_8(client, mocked_responses, capsys): 'payload': {'platform': 'Android', 'os_version': '99'}}, ]}) with pytest.raises(SystemExit) as excinfo: - run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None) + run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None, scan_type='MANUAL') assert excinfo.value.code == ExitCode.PRECHECK_BLOCKED == 8 stderr = capsys.readouterr().err assert '[empty_profile]' in stderr @@ -44,7 +44,7 @@ def test_precheck_unavailable_blocks(client, mocked_responses): 'payload': {'unavailable_dependencies': ['application_lookup']}}, ]}) with pytest.raises(SystemExit) as excinfo: - run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None) + run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None, scan_type='MANUAL') assert excinfo.value.code == ExitCode.PRECHECK_BLOCKED @@ -52,7 +52,7 @@ def test_precheck_transport_error_blocks(client, mocked_responses): mocked_responses.add(responses.POST, PRECHECK_URL, status=502, json={'error_code': 'downstream_unavailable', 'message': 'Scanyon down'}) with pytest.raises(SystemExit) as excinfo: - run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None) + run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None, scan_type='MANUAL') assert excinfo.value.code == ExitCode.PRECHECK_BLOCKED @@ -60,7 +60,7 @@ def test_profile_not_found_fails(client, mocked_responses): mocked_responses.add(responses.POST, PRECHECK_URL, status=404, json={'error_code': 'not_found', 'message': 'profile not found'}) with pytest.raises(SystemExit) as excinfo: - run_precheck_gate(client, 'a' * 32, profile_id=42, testcase_id=None) + run_precheck_gate(client, 'a' * 32, profile_id=42, testcase_id=None, scan_type='MANUAL') assert excinfo.value.code == ExitCode.SCAN_FAILED @@ -68,7 +68,7 @@ def test_missing_profile_id_skips_on_422(client, mocked_responses, caplog): """Interim behavior until scanyon makes profile_id optional: 422 + no profile -> skip.""" mocked_responses.add(responses.POST, PRECHECK_URL, status=422, json={'error_code': 'validation_error', 'message': 'profile_id required'}) - run_precheck_gate(client, 'a' * 32, profile_id=None, testcase_id=None) + run_precheck_gate(client, 'a' * 32, profile_id=None, testcase_id=None, scan_type='MANUAL') assert any('pre-check skipped' in record.message.lower() for record in caplog.records) diff --git a/tests/test_smoke_flows.py b/tests/test_smoke_flows.py index 0164a81..27a551a 100644 --- a/tests/test_smoke_flows.py +++ b/tests/test_smoke_flows.py @@ -140,6 +140,54 @@ def test_ms_scan_fail_state(mocked_responses, monkeypatch, tmp_path, tmp_apk, ap assert exit_code == 5 +def test_ms_start_409_facade_retry_is_tolerated(mocked_responses, monkeypatch, tmp_path, + tmp_apk, apk_md5, no_sleep, ms_mode): + """POST /scans/{id}/start/ is not idempotent: a facade retry after the first + (successful) call returns 409, but the scan did start. CLI must confirm by state + and continue, not fail.""" + monkeypatch.chdir(tmp_path) + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', json=[]) + mocked_responses.add(responses.GET, f'{REST_URL}/testcases/5/', json={'os': 'ANDROID'}) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', + json=[{'type': 'ANDROID', 'status': 'STARTED'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/applications/', + json=[application_json(apk_md5)]) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/precheck/', + json={'warnings': []}) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', + json=scan_json(stage='CREATED', status='INITIAL')) + # start returns 409 (retry after first unlock succeeded) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/start/', + json={'error_code': 'bad_request', 'message': 'недоступны'}, status=409) + # but the scan is already past initial - 409 must be tolerated + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='START', status='INITIAL')) + exit_code = run_main(monkeypatch, ms_argv(tmp_apk) + ['--nowait']) + assert exit_code == 0 + + +def test_ms_start_409_real_conflict_fails(mocked_responses, monkeypatch, tmp_path, + tmp_apk, apk_md5, no_sleep, ms_mode): + """A genuine 409 (scan still in CREATED) must NOT be masked - CLI fails.""" + monkeypatch.chdir(tmp_path) + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', json=[]) + mocked_responses.add(responses.GET, f'{REST_URL}/testcases/5/', json={'os': 'ANDROID'}) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', + json=[{'type': 'ANDROID', 'status': 'STARTED'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/applications/', + json=[application_json(apk_md5)]) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/precheck/', + json={'warnings': []}) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', + json=scan_json(stage='CREATED', status='INITIAL')) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/start/', + json={'error_code': 'bad_request', 'message': 'x'}, status=409) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='CREATED', status='INITIAL')) + exit_code = run_main(monkeypatch, ms_argv(tmp_apk) + ['--nowait']) + assert exit_code == 5 + + def test_ms_appium_rejected(monkeypatch, tmp_apk, ms_mode): exit_code = run_main(monkeypatch, [ '--distribution_system', 'file', '--file_path', tmp_apk, From b1dee944a7fe0da58b2198030dcc03077591ea83 Mon Sep 17 00:00:00 2001 From: Nikita Pinaev Date: Wed, 8 Jul 2026 14:46:02 +0300 Subject: [PATCH 4/6] STG-4475: tolerate transient 5xx/network during scan polling Live e2e hit a transient 502 (scanyon momentarily unavailable) on GET /scans/{id}/ mid-poll and the CLI died, though the scan kept running. A long CI poll must survive flaky downstream blips: _get_scan now retries transient 5xx (502/503/504) and network errors a few times before failing; 4xx stays terminal. +4 unit tests. 87 passed. --- mdast_cli/ms_flow.py | 37 ++++++++++++++++++++--- tests/test_poll_resilience.py | 57 +++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 tests/test_poll_resilience.py diff --git a/mdast_cli/ms_flow.py b/mdast_cli/ms_flow.py index 405cc7f..77dce1e 100644 --- a/mdast_cli/ms_flow.py +++ b/mdast_cli/ms_flow.py @@ -98,12 +98,41 @@ def _exit_on_http_error(resp, action, exit_code=ExitCode.SCAN_FAILED): sys.exit(exit_code) +POLL_TRANSIENT_RETRIES = 5 +POLL_TRANSIENT_CODES = (502, 503, 504) + + def _get_scan(mdast, scan_id): - resp = mdast.get_scan_info(scan_id) - if resp.status_code != 200: - _exit_on_http_error(resp, f'Getting scan info for scan {scan_id}', + """Poll scan state, tolerating transient downstream failures. + + A long CI poll must not die on a single flaky 5xx / network blip while the + scan keeps running server-side. Transient errors are retried a few times; + a persistent failure or a 4xx still exits. + """ + import requests as _requests + last_resp = None + for attempt in range(POLL_TRANSIENT_RETRIES): + try: + resp = mdast.get_scan_info(scan_id) + except _requests.RequestException as ex: + logger.warning(f'Scan info request failed ({type(ex).__name__}), ' + f'retry {attempt + 1}/{POLL_TRANSIENT_RETRIES}') + time.sleep(SLEEP_TIMEOUT) + continue + if resp.status_code == 200: + return resp.json() + last_resp = resp + if resp.status_code in POLL_TRANSIENT_CODES: + logger.warning(f'Scan info returned {resp.status_code} (transient), ' + f'retry {attempt + 1}/{POLL_TRANSIENT_RETRIES}') + time.sleep(SLEEP_TIMEOUT) + continue + break + if last_resp is not None: + _exit_on_http_error(last_resp, f'Getting scan info for scan {scan_id}', ExitCode.NETWORK_ERROR) - return resp.json() + logger.error(f'Getting scan info for scan {scan_id} failed after retries (network)') + sys.exit(ExitCode.NETWORK_ERROR) def run_precheck_gate(mdast, md5, profile_id, testcase_id, scan_type): diff --git a/tests/test_poll_resilience.py b/tests/test_poll_resilience.py new file mode 100644 index 0000000..142a244 --- /dev/null +++ b/tests/test_poll_resilience.py @@ -0,0 +1,57 @@ +"""Poll resilience: transient downstream 5xx during scan polling must be retried, +not fatal (a long CI poll should survive flaky blips while the scan keeps running).""" +from unittest import mock + +import pytest +import requests +import responses + +from mdast_cli.helpers.exit_codes import ExitCode +from mdast_cli import ms_flow +from mdast_cli_core.microservices import mDastMicroservices +from tests.conftest import REST_URL, TOKEN + +pytestmark = pytest.mark.unit + +SCAN_URL = f'{REST_URL}/scans/55/' + + +@pytest.fixture +def client(): + return mDastMicroservices(REST_URL, TOKEN) + + +def test_transient_502_then_success(client, mocked_responses, no_sleep): + mocked_responses.add(responses.GET, SCAN_URL, status=502, + json={'error_code': 'downstream_unavailable', 'message': 'x'}) + mocked_responses.add(responses.GET, SCAN_URL, status=503, json={'error_code': 'busy'}) + mocked_responses.add(responses.GET, SCAN_URL, + json={'id': 55, 'stage': 'WORKING', 'status': 'PROCESSING'}) + scan = ms_flow._get_scan(client, 55) + assert scan['stage'] == 'WORKING' + + +def test_persistent_502_exits_network_error(client, mocked_responses, no_sleep): + for _ in range(ms_flow.POLL_TRANSIENT_RETRIES + 1): + mocked_responses.add(responses.GET, SCAN_URL, status=502, + json={'error_code': 'downstream_unavailable', 'message': 'x'}) + with pytest.raises(SystemExit) as excinfo: + ms_flow._get_scan(client, 55) + assert excinfo.value.code == ExitCode.NETWORK_ERROR + + +def test_404_is_not_retried(client, mocked_responses, no_sleep): + mocked_responses.add(responses.GET, SCAN_URL, status=404, json={'error_code': 'not_found'}) + with pytest.raises(SystemExit): + ms_flow._get_scan(client, 55) + # only one call - 404 is terminal, not retried + assert len([c for c in mocked_responses.calls if c.request.url == SCAN_URL]) == 1 + + +def test_network_exception_retried_then_success(client, no_sleep): + resp_ok = mock.Mock(status_code=200) + resp_ok.json.return_value = {'id': 55, 'stage': 'SUCCESS', 'status': 'COMPLETE'} + seq = [requests.ConnectionError('boom'), resp_ok] + with mock.patch.object(client, 'get_scan_info', side_effect=seq): + scan = ms_flow._get_scan(client, 55) + assert scan['status'] == 'COMPLETE' From 8b657cd77e8908af629c535b1cdd8922254d78c6 Mon Sep 17 00:00:00 2001 From: Nikita Pinaev Date: Wed, 8 Jul 2026 15:43:13 +0300 Subject: [PATCH 5/6] STG-4478: tolerate transient 5xx/network on report download (same as polling) Report download (Pepper via Clark) can hit transient 502 right after a scan finishes; wrap download in the same retry as _get_scan so a blip doesn't lose a finished scan's report. Confirmed live: JSON report downloads, PDF currently 502s server-side (Pepper WeasyPrint CSS issue - separate server bug). --- mdast_cli/ms_flow.py | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/mdast_cli/ms_flow.py b/mdast_cli/ms_flow.py index 77dce1e..0e80cfa 100644 --- a/mdast_cli/ms_flow.py +++ b/mdast_cli/ms_flow.py @@ -337,6 +337,37 @@ def run_microservices_flow(arguments, url, token, app_file, appstore_app_md5, us download_reports(mdast, scan_id, arguments) +def _download_report_with_retry(fetch, action): + """Download a report tolerating transient downstream (Pepper) 5xx/network. + + The report render can momentarily be unavailable (502/503/504) right after a + scan finishes; a single blip must not lose the finished scan's report. + """ + import requests as _requests + last_resp = None + for attempt in range(POLL_TRANSIENT_RETRIES): + try: + resp = fetch() + except _requests.RequestException as ex: + logger.warning(f'{action} request failed ({type(ex).__name__}), ' + f'retry {attempt + 1}/{POLL_TRANSIENT_RETRIES}') + time.sleep(SLEEP_TIMEOUT) + continue + if resp.status_code == 200: + return resp + last_resp = resp + if resp.status_code in POLL_TRANSIENT_CODES: + logger.warning(f'{action} returned {resp.status_code} (transient), ' + f'retry {attempt + 1}/{POLL_TRANSIENT_RETRIES}') + time.sleep(SLEEP_TIMEOUT) + continue + break + if last_resp is not None: + _exit_on_http_error(last_resp, action) + logger.error(f'{action} failed after retries (network)') + sys.exit(ExitCode.NETWORK_ERROR) + + def download_reports(mdast, scan_id, arguments): """Report step (STG-4478): files are written to the user-provided paths. @@ -349,9 +380,8 @@ def download_reports(mdast, scan_id, arguments): if pdf_report_file_name: logger.info(f'Create and download pdf report for scan with id {scan_id} ' f'to file {pdf_report_file_name}.') - pdf_report = mdast.download_report(scan_id) - if pdf_report.status_code != 200: - _exit_on_http_error(pdf_report, 'PDF report downloading') + pdf_report = _download_report_with_retry(lambda: mdast.download_report(scan_id), + 'PDF report downloading') pdf_report_file_name = pdf_report_file_name if pdf_report_file_name.endswith( '.pdf') else f'{pdf_report_file_name}.pdf' with open(pdf_report_file_name, 'wb') as f: @@ -362,9 +392,8 @@ def download_reports(mdast, scan_id, arguments): if json_summary_file_name: logger.info(f'Download JSON summary report for scan with id {scan_id} ' f'to file {json_summary_file_name}.') - json_report = mdast.download_scan_json_result(scan_id) - if json_report.status_code != 200: - _exit_on_http_error(json_report, 'JSON summary report downloading') + json_report = _download_report_with_retry( + lambda: mdast.download_scan_json_result(scan_id), 'JSON summary report downloading') json_file_name = json_summary_file_name if json_summary_file_name.endswith( '.json') else f'{json_summary_file_name}.json' with open(json_file_name, 'w') as fp: From a5c96c6b5f787a2d3ca79810cac7108e513558aa Mon Sep 17 00:00:00 2001 From: Nikita Pinaev Date: Thu, 9 Jul 2026 17:22:48 +0300 Subject: [PATCH 6/6] STG-4451/4475/4478: harden dual-mode CLI, add report format selection and e2e tests Report handling: - --report_format {pdf,json,all,none} (default pdf), shared by both flows. resolve_report_targets() centralises output paths (scan_report_.pdf/.json defaults; --pdf_report_file_name / --summary_report_json_file_name still force a format and keep backward compatibility). Microservices reports are soft-fail: a Pepper render failure keeps a finished scan green (exit 0 with a warning). Exit-code parity and robustness: - Required url/company_id/token validation, app-file existence guard, cr_report requires stingray creds. Both flows wrap RequestException -> NETWORK_ERROR(6). - 401/403 -> AUTH_ERROR(7) consistently in both installations. - Microservices upload accepts any 2xx (not only 201); server-controlled fields sanitised before logging (CWE-117); monolith architecture resolution guards non-dict payload items instead of crashing. - factory: TLS verify secure-by-default (unset/empty stay on), architectures content-fingerprint probe (not a bare HTTP 200), clearer ambiguous/empty errors. - nexus2: fix crash when --nexus2_file_name omitted (None into os.path.join). Docs: README default timeout corrected to ~1h, CR report relabelled Compliance (not Change Request), Docker report-path note added. Tests: positive store-download -> scan e2e (8 stores), negative Sting-side e2e, report target resolution, report-download resilience, mode-probe path fix. 151 passed. .gitignore: ignore .venv/ and scan_report_* CLI artifacts. --- .gitignore | 5 + README.md | 140 +- mdast_cli/distribution_systems/appgallery.py | 1 - mdast_cli/distribution_systems/base.py | 2 +- mdast_cli/distribution_systems/nexus2.py | 5 +- mdast_cli/distribution_systems/rumarket.py | 1 - mdast_cli/helpers/const.py | 2 +- mdast_cli/helpers/helpers.py | 47 + mdast_cli/mdast_scan.py | 1782 +++++++++--------- mdast_cli/ms_flow.py | 426 +++-- mdast_cli_core/factory.py | 144 +- mdast_cli_core/microservices.py | 7 +- pytest.ini | 1 + tests/test_download_scan_positive.py | 224 +++ tests/test_mode_detection.py | 55 +- tests/test_negative_e2e.py | 470 +++++ tests/test_poll_resilience.py | 33 + tests/test_precheck_gate.py | 20 +- tests/test_report_targets.py | 55 + tests/test_security.py | 29 +- tests/test_smoke_flows.py | 2 +- 21 files changed, 2386 insertions(+), 1065 deletions(-) create mode 100644 tests/test_download_scan_positive.py create mode 100644 tests/test_negative_e2e.py create mode 100644 tests/test_report_targets.py diff --git a/.gitignore b/.gitignore index e738652..59c188a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,11 @@ dist/ # venv venv/ +.venv/ + +# CLI-generated scan reports (default output names: scan_report_.pdf/.json) +scan_report_*.pdf +scan_report_*.json # Compiled Java class files *.class diff --git a/README.md b/README.md index a30cd63..9e90ae4 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ This script is designed to integrate mobile applications' security analysis into * [Usage Modes](#usage-modes) * [Download Only Mode](#download-only-mode) * [Scan Mode](#scan-mode) +* [Installation Modes (Monolith and Kubernetes)](#installation-modes-monolith-and-kubernetes) * [Distribution Systems](#distribution-systems) * [Local File](#local-file) * [Google Play](#google-play) @@ -189,7 +190,55 @@ Without `--download_only`, the script will: 2. Upload to the DAST platform 3. Start a security scan 4. Wait for completion (unless `--nowait` is set) -5. Generate reports (if specified) +5. Generate reports (PDF by default) + +--- + +## Installation Modes (Monolith and Kubernetes) + +`mdast_cli` is a **single package that works against both** flavours of the DAST +platform, with **no change to the command line**: + +- **Monolith** - the classic all-in-one installation (`Token` auth, tenant carried + in the `/organizations/{company_id}/...` URL path). +- **Microservices / Kubernetes** - the platform behind the Clark facade (`Bearer` + auth with an organization token issued in the platform UI; the tenant is resolved + server-side, so `--company_id` is accepted but ignored). + +By default the CLI **auto-detects** the installation by probing `GET /rest/architectures/` +and inspecting the response shape (it does not merely trust an HTTP 200). The same +`--url`, `--token`, `--company_id` and scan flags are used either way. + +### Selecting the mode explicitly + +Auto-detection can be overridden with the `MDAST_CLI_MODE` environment variable - +recommended in CI, where the target platform is known and a deterministic choice is +preferable to a probe: + +```bash +export MDAST_CLI_MODE=microservices # or: monolith (default: auto) +``` + +### Environment variables (microservices installation) + +| Variable | Default | Purpose | +|----------|---------|---------| +| `MDAST_CLI_MODE` | `auto` | Force the installation mode (`auto` / `monolith` / `microservices`). | +| `MDAST_TLS_VERIFY` | *(on)* | TLS certificate verification. **Secure by default.** Set to `0`/`false`/`no`/`off` to disable for a self-signed stand (the org token is a credential - use only on trusted networks). An unset **or empty** value stays secure. | +| `MDAST_UPLOAD_TIMEOUT` | *(server default)* | Optional server-side upload/parse wait in seconds (1-300) for large builds. | + +### Differences on the microservices installation + +- **Authentication** uses an **organization CLI token** issued in the platform UI + (`Bearer`), not a per-user CI/CD token. There is one active token per organization. +- `--architecture_id` is **ignored** (the platform resolves the architecture from the + application binary itself). +- `--appium_script_path` (Appium scans) and `--cr_report` (CR report) are **not + supported** and exit with `INVALID_ARGS` (2). +- A report render failure is **soft-fail**: a finished scan stays green (exit 0) with + a warning, because the report service (Pepper) is separate from the scan itself. +- Transient gateway errors (502/503/504) during polling, upload and report download + are **retried**, so a rolling restart of a backend mid-scan does not fail a CI job. --- @@ -643,11 +692,43 @@ mdast_cli ... --project_id 10 # Create profile in project #10 ## Reports +Reports work the **same way on both installations**. + +### Choosing the report format + +**Parameter:** `--report_format {pdf|json|all|none}` (default: `pdf`) + +After a successful scan the CLI downloads a **PDF report by default** - a plain scan +always produces one. Use the flag to choose otherwise: + +| `--report_format` | Result | +|-------------------|--------| +| `pdf` *(default)* | PDF only | +| `json` | JSON summary only | +| `all` | both PDF and JSON | +| `none` | no report (e.g. gate on exit code only) | + +**File names.** If you do not pass an explicit file name, reports are saved next to +the working directory as `scan_report_.pdf` / `scan_report_.json`, +so the file is always tied to the scan it came from. Passing `--pdf_report_file_name` +and/or `--summary_report_json_file_name` overrides the name and forces that format +(so those flags remain fully backward compatible). PDF is always downloaded first. + +> On the **microservices** installation a report that fails to render (e.g. the +> report service returns 502) is a **warning, not a failure**: the scan already +> succeeded, so the CLI still exits 0 and any other requested report is still saved. + +> **In Docker:** the default `scan_report_.*` name is written to the +> container's working directory, which is discarded when the container exits. Mount +> a host directory and point the report there with an absolute path (e.g. +> `-v /host/reports:/mdast/report` + `--pdf_report_file_name /mdast/report/report.pdf`), +> otherwise the report is lost. See the Docker example under *Local File*. + ### JSON Summary Report Generate a structured JSON report with scan summary and statistics. -**Parameter:** `--summary_report_json_file_name ` +**Parameter:** `--summary_report_json_file_name ` (implies `--report_format json`) **Output Format:** - Total number of vulnerabilities @@ -673,7 +754,7 @@ mdast_cli \ Generate a detailed PDF report with full scan results. -**Parameter:** `--pdf_report_file_name ` +**Parameter:** `--pdf_report_file_name ` (implies `--report_format pdf`; a PDF is produced by default even without this flag) **Output Format:** - Detailed vulnerability descriptions @@ -697,7 +778,12 @@ mdast_cli \ ### CR Report -Generate a CR (Change Request) report in HTML format. +Generate a CR (Compliance) report in HTML format - a Bank-of-Russia regulatory +report on call-trace immutability (KNTV) that compares two versions of a mobile +application. Not a "change request" report. + +> **Monolith installation only.** On the microservices installation `--cr_report` +> exits with `INVALID_ARGS` (2). **Required Parameters (when `--cr_report` is set):** - `--cr_report` - Enable CR report generation @@ -760,7 +846,12 @@ mdast_cli \ ### Long-running Scans -Use `--long_wait` to extend the maximum wait time to 1 week (instead of default timeout). +Use `--long_wait` to extend the maximum wait time to ~1 week (instead of the default +~1 hour timeout). + +If a scan still has not reached a terminal state when the wait budget is exhausted, +the CLI **fails** with `SCAN_FAILED` (5) rather than exiting 0 - a scan that never +finished is never reported as success. **Use Case:** - Very long testcases @@ -784,6 +875,10 @@ mdast_cli \ Use `--appium_script_path` to provide a custom Appium script for automated testing. +> **Monolith installation only.** On the microservices installation +> `--appium_script_path` exits with `INVALID_ARGS` (2); use a recorded test case +> (`--testcase_id`) instead. + **Parameter:** `--appium_script_path ` **Use Case:** @@ -898,16 +993,23 @@ mdast_cli -d \ ## Exit Codes -The script uses standardized exit codes for CI/CD integration: +The script uses standardized exit codes for CI/CD integration. They are **identical +on both installations** (monolith and microservices), so pipelines behave the same +regardless of which platform they target: | Code | Constant | Description | |------|----------|-------------| | 0 | `SUCCESS` | Operation completed successfully | -| 1 | `INVALID_ARGS` | Invalid command-line arguments | -| 2 | `AUTH_ERROR` | Authentication failed | -| 3 | `DOWNLOAD_FAILED` | Application download failed | -| 4 | `NETWORK_ERROR` | Network/connection error | -| 5 | `SCAN_FAILED` | Scan execution or upload failed | +| 1 | `INTERNAL_ERROR` | Unexpected internal error | +| 2 | `INVALID_ARGS` | Invalid command-line arguments (also: Appium/CR report on the microservices installation) | +| 4 | `DOWNLOAD_FAILED` | Application download failed / file not found | +| 5 | `SCAN_FAILED` | Scan created but failed, no active engine, or finished in a non-success terminal state | +| 6 | `NETWORK_ERROR` | Network/connection error, gateway 5xx (incl. upload 502/503/504), non-JSON gateway response | +| 7 | `AUTH_ERROR` | Authentication/authorization error (bad or revoked token, 401/403) | +| 8 | `PRECHECK_BLOCKED` | Scan pre-check returned blocking warnings (microservices gate) | + +> There is no code 3. Codes 6/7/8 are meaningful mostly on the microservices +> installation but are used consistently wherever they apply. **Example CI/CD Usage:** ```bash @@ -915,15 +1017,13 @@ The script uses standardized exit codes for CI/CD integration: mdast_cli --distribution_system file --file_path app.apk ... EXIT_CODE=$? -if [ $EXIT_CODE -eq 0 ]; then - echo "Scan completed successfully" -elif [ $EXIT_CODE -eq 3 ]; then - echo "Download failed - check distribution system" - exit 1 -else - echo "Scan failed with code $EXIT_CODE" - exit 1 -fi +case $EXIT_CODE in + 0) echo "Scan completed successfully" ;; + 4) echo "Download failed - check distribution system"; exit 1 ;; + 6|7) echo "Infrastructure/auth problem (code $EXIT_CODE) - safe to retry"; exit 1 ;; + 8) echo "Scan blocked by pre-check warnings"; exit 1 ;; + *) echo "Scan failed with code $EXIT_CODE"; exit 1 ;; +esac ``` --- diff --git a/mdast_cli/distribution_systems/appgallery.py b/mdast_cli/distribution_systems/appgallery.py index 5827484..d2c5d4a 100644 --- a/mdast_cli/distribution_systems/appgallery.py +++ b/mdast_cli/distribution_systems/appgallery.py @@ -1,5 +1,4 @@ import logging -import os import time import requests diff --git a/mdast_cli/distribution_systems/base.py b/mdast_cli/distribution_systems/base.py index 30b8b6b..5f4a338 100644 --- a/mdast_cli/distribution_systems/base.py +++ b/mdast_cli/distribution_systems/base.py @@ -2,7 +2,7 @@ Base interface for all distribution system downloaders. """ from abc import ABC, abstractmethod -from typing import Dict, Optional, Any +from typing import Dict, Any class BaseDownloader(ABC): diff --git a/mdast_cli/distribution_systems/nexus2.py b/mdast_cli/distribution_systems/nexus2.py index 95354da..63434c4 100644 --- a/mdast_cli/distribution_systems/nexus2.py +++ b/mdast_cli/distribution_systems/nexus2.py @@ -29,7 +29,10 @@ def __init__(self, nexus_url, login, password): def download_app(self, download_path, repo_name, group_id, artifact_id, version, extension, file_name=''): download_url = f'{self.nexus_url}/service/local/artifact/maven/content?r={repo_name}&g={group_id}&a=' \ f'{artifact_id}&v={version}&p={extension}' - if file_name == '': + # `not file_name` (not `== ''`): mdast_scan passes arguments.nexus2_file_name, which is + # None when --nexus2_file_name is omitted (argparse default). The old `== ''` check missed + # None, so os.path.join(download_path, None) crashed the download on a valid invocation. + if not file_name: file_name = f'{artifact_id}-{version}.{extension}' headers = { 'DNT': '1', diff --git a/mdast_cli/distribution_systems/rumarket.py b/mdast_cli/distribution_systems/rumarket.py index ce7c32a..1466437 100644 --- a/mdast_cli/distribution_systems/rumarket.py +++ b/mdast_cli/distribution_systems/rumarket.py @@ -1,5 +1,4 @@ import logging -import os import requests from tqdm import tqdm diff --git a/mdast_cli/helpers/const.py b/mdast_cli/helpers/const.py index 93f44f1..7fcb2ff 100644 --- a/mdast_cli/helpers/const.py +++ b/mdast_cli/helpers/const.py @@ -36,7 +36,7 @@ class DastState: # Timeout constants (in seconds) TRY = 360 -LONG_TRY = 20160 +LONG_TRY = 60480 # ~1 week at SLEEP_TIMEOUT=10s (matches --long_wait docs) END_SCAN_TIMEOUT = 30 SLEEP_TIMEOUT = 10 diff --git a/mdast_cli/helpers/helpers.py b/mdast_cli/helpers/helpers.py index 0bb9e78..c1cd6f8 100644 --- a/mdast_cli/helpers/helpers.py +++ b/mdast_cli/helpers/helpers.py @@ -14,3 +14,50 @@ def check_app_md5(file_path): while chunk := f.read(8192): file_hash.update(chunk) return file_hash.hexdigest() + + +def resolve_report_targets(arguments, scan_id): + """Decide which reports to produce and to which files, for BOTH installations. + + The report format is selected by ``--report_format`` (``pdf`` by default, so a + plain scan always yields a PDF). Precedence rules, kept identical for the + monolith and the microservices flow: + + * an explicit ``--pdf_report_file_name`` / ``--summary_report_json_file_name`` + always produces that format at that path (backward compatible); + * ``--report_format`` still applies on top, so ``--report_format all`` plus a + PDF file name yields the JSON report too (at a default name); + * with no file-name flags, ``--report_format`` alone decides + (``pdf`` | ``json`` | ``all`` | ``none``); + * default file names are derived from the scan id + (``scan_report_.pdf`` / ``.json``) so a report can never land at + a confusing server- or arg-derived name. + + Returns an ordered dict ``{'pdf': path, 'json': path}`` limited to the formats + that should be produced. PDF is inserted first (it is the primary artifact). + """ + pdf_name = arguments.pdf_report_file_name + json_name = arguments.summary_report_json_file_name + fmt = getattr(arguments, 'report_format', None) + + if pdf_name or json_name: + want_pdf = bool(pdf_name) + want_json = bool(json_name) + if fmt in ('pdf', 'all'): + want_pdf = True + if fmt in ('json', 'all'): + want_json = True + else: + effective = fmt or 'pdf' # nothing specified -> default PDF + want_pdf = effective in ('pdf', 'all') + want_json = effective in ('json', 'all') + + targets = {} + if want_pdf: + target = pdf_name or f'scan_report_{scan_id}.pdf' + # case-insensitive suffix check so report.PDF does not become report.PDF.pdf + targets['pdf'] = target if target.lower().endswith('.pdf') else f'{target}.pdf' + if want_json: + target = json_name or f'scan_report_{scan_id}.json' + targets['json'] = target if target.lower().endswith('.json') else f'{target}.json' + return targets diff --git a/mdast_cli/mdast_scan.py b/mdast_cli/mdast_scan.py index 36148d0..6a10970 100644 --- a/mdast_cli/mdast_scan.py +++ b/mdast_cli/mdast_scan.py @@ -1,845 +1,937 @@ -import argparse -import asyncio -import json -import logging -import os -import sys -import time -import warnings - -import urllib3 - -# Suppress pkg_resources deprecation warning from google-auth -# This warning appears when google-auth is imported, even if not used -warnings.filterwarnings('ignore', message='.*pkg_resources is deprecated.*', category=UserWarning) - -from mdast_cli.distribution_systems.appgallery import appgallery_download_app -from mdast_cli.distribution_systems.appstore import AppStore -from mdast_cli.distribution_systems.firebase import firebase_download_app -from mdast_cli.distribution_systems.google_play import GooglePlay -from mdast_cli.distribution_systems import google_play_apkeep as gp_apkeep -from mdast_cli.distribution_systems.nexus import NexusRepository -from mdast_cli.distribution_systems.nexus2 import Nexus2Repository -from mdast_cli.distribution_systems.rumarket import rumarket_download_app -from mdast_cli.distribution_systems.rustore import rustore_download_app -from mdast_cli.helpers.const import (ANDROID_EXTENSIONS, DEFAULT_ANDROID_ARCHITECTURE, DEFAULT_IOS_ARCHITECTURE, - END_SCAN_TIMEOUT, LONG_TRY, SLEEP_TIMEOUT, TRY, DastState, DastStateDict) -from mdast_cli.helpers.exit_codes import ExitCode -from mdast_cli.helpers.helpers import check_app_md5 -from mdast_cli_core.token import mDastToken as mDast -from mdast_cli_core.factory import (MODE_MICROSERVICES, ModeDetectionError, resolve_installation_mode, - tls_verify_enabled) -from mdast_cli.cr_report_generator import generate_cr -from mdast_cli import __version__ - -USER_AGENT = f'mdast_cli/{__version__}' - -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s %(message)s', - datefmt='%d/%m/%Y %H:%M:%S', stream=sys.stdout) - -logger = logging.getLogger(__name__) - - -def parse_args(): - parser = argparse.ArgumentParser( - description='MDast CLI - tool for downloading and scanning mobile applications.', - epilog=''' -Usage examples: - # Download application from Google Play without scanning: - mdast_cli -d --distribution_system google_play \\ - --google_play_package_name com.example.app \\ - --google_play_email user@example.com \\ - --google_play_aas_token YOUR_TOKEN - - # Start application scanning: - mdast_cli --distribution_system file --file_path app.apk \\ - --url https://mdast.example.com --company_id 1 --token YOUR_TOKEN - -For detailed information about specific distribution system see README.md - ''', - formatter_class=argparse.RawDescriptionHelpFormatter - ) - - # Main options - main_group = parser.add_argument_group('Main Options', 'General parameters for utility operation') - main_group.add_argument('--download_only', '-d', action='store_true', - help='Download application without starting scan. ' - 'When using this option, --url, --company_id, --token parameters are not required. ' - 'After successful download, file path is printed in format DOWNLOAD_PATH=/path/to/file') - main_group.add_argument('--distribution_system', '-ds', type=str, - help='Application distribution system for downloading. ' - 'Available options: file, nexus, nexus2, firebase, appstore, google_play, ' - 'rustore, appgallery, rumarket. ' - 'System choice determines which additional parameters will be required.', - choices=['file', 'nexus', 'nexus2', 'firebase', 'appstore', 'google_play', - 'rustore', 'appgallery', 'rumarket'], - required=True) - main_group.add_argument('--download_path', '-p', type=str, - help='Path to directory for saving downloaded applications. ' - 'Default: downloaded_apps. ' - 'Directory will be created automatically if it does not exist.', - default='downloaded_apps') - main_group.add_argument('--verbose', '-v', action='store_true', - help='Enable debug logging (e.g. for troubleshooting download failures).') - - # File distribution system - file_group = parser.add_argument_group('Local File (file)', - 'Using existing local application file') - file_group.add_argument('--file_path', type=str, - help='Full path to local application file (APK for Android or IPA for iOS). ' - 'Required parameter when --distribution_system is set to "file". ' - 'Example: /path/to/app.apk or ./my_app.ipa') - - # Nexus distribution system - nexus_group = parser.add_argument_group('Nexus Repository (nexus)', - 'Downloading applications from Nexus Repository Manager 3.x') - nexus_group.add_argument('--nexus_url', type=str, - help='Nexus server URL (with http/https protocol). ' - 'Required parameter when --distribution_system is set to "nexus". ' - 'Example: https://nexus.example.com or http://localhost:8081') - nexus_group.add_argument('--nexus_login', type=str, - help='Username for Nexus authentication. ' - 'Required parameter when --distribution_system is set to "nexus".') - nexus_group.add_argument('--nexus_password', type=str, - help='Password for Nexus authentication. ' - 'Required parameter when --distribution_system is set to "nexus". ' - 'It is recommended to use environment variables for security.') - nexus_group.add_argument('--nexus_repo_name', type=str, - help='Repository name in Nexus where the application is located. ' - 'Required parameter when --distribution_system is set to "nexus". ' - 'Example: releases, snapshots, maven-releases') - nexus_group.add_argument('--nexus_group_id', type=str, - help='Application Group ID in Maven format (usually reverse domain). ' - 'Required parameter when --distribution_system is set to "nexus". ' - 'Example: com.example, org.mycompany') - nexus_group.add_argument('--nexus_artifact_id', type=str, - help='Application Artifact ID (artifact name). ' - 'Required parameter when --distribution_system is set to "nexus". ' - 'Example: myapp, mobile-app') - nexus_group.add_argument('--nexus_version', type=str, - help='Application version to download. ' - 'Required parameter when --distribution_system is set to "nexus". ' - 'Example: 1.0.0, 2.5.1, 1.0.0-SNAPSHOT') - - # Firebase distribution system - firebase_group = parser.add_argument_group('Firebase App Distribution (firebase)', - 'Downloading applications from Firebase App Distribution') - firebase_group.add_argument('--firebase_project_number', type=int, - help='Firebase project number (Project Number). ' - 'Required parameter when --distribution_system is set to "firebase". ' - 'Can be found in Firebase Console project settings. ' - 'Example: 123456789012') - firebase_group.add_argument('--firebase_app_id', type=str, - help='Application ID in Firebase in format: "1:PROJECT_NUMBER:PLATFORM:APP_ID". ' - 'Required parameter when --distribution_system is set to "firebase". ' - 'Examples: "1:123456789012:android:abc123def456", "1:123456789012:ios:xyz789"') - firebase_group.add_argument('--firebase_account_json_path', type=str, - help='Path to JSON file with Firebase service account credentials. ' - 'Required parameter when --distribution_system is set to "firebase". ' - 'File can be downloaded from Firebase Console → Project Settings → Service Accounts. ' - 'Example: /path/to/service-account.json or ./firebase-key.json') - firebase_group.add_argument('--firebase_file_extension', type=str, - help='Application file extension for download. ' - 'Required parameter when --distribution_system is set to "firebase". ' - 'Available values: apk (Android) or ipa (iOS).', - choices=['ipa', 'apk']) - firebase_group.add_argument('--firebase_file_name', type=str, - help='File name for saving application (without extension). ' - 'Optional parameter. If not specified, version from Firebase is used. ' - 'Example: my_app, production_build') - - # AppStore distribution system - appstore_group = parser.add_argument_group('Apple App Store (appstore)', - 'Downloading applications from Apple App Store') - appstore_group.add_argument('--appstore_app_id', type=str, - help='Application ID in App Store. Can be found in application page URL: ' - 'https://apps.apple.com/app/id{APP_ID}. ' - 'Either --appstore_app_id or --appstore_bundle_id must be specified. ' - 'Example: 389801252 (Instagram), 310633997 (WhatsApp)') - appstore_group.add_argument('--appstore_bundle_id', type=str, - help='Application Bundle ID (package identifier). ' - 'Either --appstore_app_id or --appstore_bundle_id must be specified. ' - 'Example: com.instagram.ios, com.whatsapp.WhatsApp') - appstore_group.add_argument('--appstore_apple_id', type=str, - help='Apple ID email address for iTunes/App Store login. ' - 'Required parameter when --distribution_system is set to "appstore". ' - 'Example: user@example.com') - appstore_group.add_argument('--appstore_password', type=str, - help='Apple ID password. ' - 'Required parameter when --distribution_system is set to "appstore". ' - 'It is recommended to use environment variables for security.') - appstore_group.add_argument('--appstore_2FA', type=str, - help='Two-factor authentication code (6 digits). ' - 'Required parameter when --distribution_system is set to "appstore". ' - 'Code is sent to trusted Apple devices. ' - 'Example: 123456') - appstore_group.add_argument('--appstore_file_name', type=str, - help='File name for saving application (without .ipa extension). ' - 'Optional parameter. If not specified, application name and version are used. ' - 'Example: my_app, instagram_latest') - appstore_group.add_argument('--appstore_country', type=str, default='US', - help='App Store country/region code for lookup and download (ISO 3166-1 alpha-2). ' - 'Default: US. Example: RU, DE, GB') - appstore_group.add_argument('--appstore_password2FA', type=str, - help='[DEPRECATED] Password and 2FA code in one parameter (format: password2FA_code). ' - 'Use --appstore_password and --appstore_2FA instead of this parameter. ' - 'Will be removed in future versions.', - metavar='DEPRECATED') - - # Google Play distribution system - google_play_group = parser.add_argument_group('Google Play (google_play)', - 'Downloading applications from Google Play Store using apkeep') - google_play_group.add_argument('--google_play_package_name', type=str, - help='Application package name from Google Play. ' - 'Required parameter when --distribution_system is set to "google_play". ' - 'Can be found in application page URL: ' - 'https://play.google.com/store/apps/details?id={PACKAGE_NAME}. ' - 'Examples: com.instagram.android, com.whatsapp, org.telegram.messenger') - google_play_group.add_argument('--google_play_email', type=str, - help='Google account email address. ' - 'Required parameter when --distribution_system is set to "google_play". ' - 'Used together with --google_play_aas_token or --google_play_oauth2_token. ' - 'Example: user@gmail.com') - google_play_group.add_argument('--google_play_aas_token', type=str, - help='AAS token for Google Play (obtained via apkeep). ' - 'Used for subsequent downloads after first token acquisition. ' - 'Requires --google_play_email. ' - 'Format: aas_et/... (long string). ' - 'For first run use --google_play_oauth2_token.') - google_play_group.add_argument('--google_play_oauth2_token', type=str, - help='OAuth2 token to obtain AAS token via apkeep. ' - 'Used on first run. After obtaining AAS token, you can use ' - '--google_play_aas_token for subsequent downloads. ' - 'Requires --google_play_email. ' - 'Format: ya29.a0AVvZVs... (OAuth2 access token)') - google_play_group.add_argument('--google_play_file_name', type=str, - help='File name for saving application (without extension). ' - 'Optional parameter. If not specified, package name is used. ' - 'Example: instagram_latest, whatsapp_production') - google_play_group.add_argument('--google_play_proxy', type=str, - help='Proxy settings for connecting to Google Play. ' - 'Optional parameter. ' - 'Format: socks5://user:pass@host:port or http://user:pass@host:port. ' - 'Example: socks5://proxy.example.com:1080') - - # RuStore distribution system - rustore_group = parser.add_argument_group('RuStore (rustore)', - 'Downloading applications from Russian RuStore marketplace') - rustore_group.add_argument('--rustore_package_name', type=str, - help='Application package name from RuStore. ' - 'Required parameter when --distribution_system is set to "rustore". ' - 'Can be found in application page URL. ' - 'Examples: com.vkontakte.android, com.yandex.browser') - - # RuMarket distribution system - rumarket_group = parser.add_argument_group('RuMarket (rumarket)', - 'Downloading applications from Russian RuMarket marketplace') - rumarket_group.add_argument('--rumarket_package_name', type=str, - help='Application package name from RuMarket. ' - 'Required parameter when --distribution_system is set to "rumarket". ' - 'Example: com.example.app') - - # AppGallery distribution system - appgallery_group = parser.add_argument_group('Huawei AppGallery (appgallery)', - 'Downloading applications from Huawei AppGallery') - appgallery_group.add_argument('--appgallery_app_id', type=str, - help='Application ID in AppGallery. ' - 'Required parameter when --distribution_system is set to "appgallery". ' - 'Can be found in application page URL: ' - 'https://appgallery.huawei.com/app/{APP_ID}. ' - 'Format: C + digits, e.g.: C101184875, C100000001') - appgallery_group.add_argument('--appgallery_file_name', type=str, - help='File name for saving application (without .apk extension). ' - 'Optional parameter. If not specified, package name and version are used. ' - 'Example: huawei_app, instagram_huawei') - - # Nexus2 distribution system - nexus2_group = parser.add_argument_group('Nexus Repository 2.x (nexus2)', - 'Downloading applications from Nexus Repository Manager 2.x') - nexus2_group.add_argument('--nexus2_url', type=str, - help='Nexus 2.x server URL (with http/https protocol and /nexus/ path). ' - 'Required parameter when --distribution_system is set to "nexus2". ' - 'Example: http://nexus.example.com:8081/nexus/ or http://localhost:8081/nexus/') - nexus2_group.add_argument('--nexus2_login', type=str, - help='Username for Nexus 2.x authentication. ' - 'Required parameter when --distribution_system is set to "nexus2".') - nexus2_group.add_argument('--nexus2_password', type=str, - help='Password for Nexus 2.x authentication. ' - 'Required parameter when --distribution_system is set to "nexus2". ' - 'It is recommended to use environment variables for security.') - nexus2_group.add_argument('--nexus2_repo_name', type=str, - help='Repository name in Nexus 2.x where the application is located. ' - 'Required parameter when --distribution_system is set to "nexus2". ' - 'Example: releases, snapshots') - nexus2_group.add_argument('--nexus2_group_id', type=str, - help='Application Group ID in Maven format. ' - 'Required parameter when --distribution_system is set to "nexus2". ' - 'Example: com.example, org.mycompany') - nexus2_group.add_argument('--nexus2_artifact_id', type=str, - help='Application Artifact ID. ' - 'Required parameter when --distribution_system is set to "nexus2". ' - 'Example: myapp, android-app') - nexus2_group.add_argument('--nexus2_version', type=str, - help='Application version to download. ' - 'Required parameter when --distribution_system is set to "nexus2". ' - 'Example: 1.0.0, 2.5.1, 1.0.0-SNAPSHOT') - nexus2_group.add_argument('--nexus2_extension', type=str, - help='Application file extension. ' - 'Required parameter when --distribution_system is set to "nexus2". ' - 'Usually: apk (Android) or ipa (iOS). ' - 'Example: apk, ipa, zip') - nexus2_group.add_argument('--nexus2_file_name', type=str, - help='File name for saving application (without extension). ' - 'Optional parameter. If not specified, generated automatically. ' - 'Example: my_app, production_build') - - # Scanning arguments - scan_group = parser.add_argument_group('Scanning Parameters', - 'Parameters for starting and managing application scanning') - scan_group.add_argument('--url', type=str, - help='MDast server URL for submitting application for scanning. ' - 'Required parameter when starting scan (without --download_only). ' - 'Example: https://mdast.example.com', - required=(not '-d')) - scan_group.add_argument('--company_id', type=int, - help='Company ID in MDast system. ' - 'Required parameter when starting scan (without --download_only). ' - 'Can be found in company settings in MDast web interface.', - required=(not '-d')) - scan_group.add_argument('--token', type=str, - help='CI/CD token for authentication and starting scan. ' - 'Required parameter when starting scan (without --download_only). ' - 'Token can be obtained in profile settings in MDast web interface. ' - 'It is recommended to use environment variables for security.', - required=(not '-d')) - scan_group.add_argument('--architecture_id', type=int, - help='Architecture ID for performing scan. ' - 'Optional parameter. If not specified, default architecture is used. ' - 'List of available architectures can be obtained via MDast API.') - scan_group.add_argument('--profile_id', type=int, default=None, - help='Profile ID for scanning. ' - 'Optional parameter. If not specified, profile will be created automatically.') - scan_group.add_argument('--project_id', type=int, default=None, - help='Project ID for scanning. ' - 'Optional parameter. Used only when auto-creating profile ' - 'to place new profile in existing project.') - scan_group.add_argument('--testcase_id', type=int, - help='Test case ID for automatic scanning. ' - 'Optional parameter. If not specified, manual scanning is performed.') - scan_group.add_argument('--appium_script_path', type=str, - help='Path to Appium script for automatic scanning using Stingray Appium. ' - 'Optional parameter. Used for automated testing.') - scan_group.add_argument('--summary_report_json_file_name', type=str, - help='File name for saving JSON report with scan results in structured format. ' - 'Optional parameter. If specified, report will be saved to specified file.') - scan_group.add_argument('--pdf_report_file_name', type=str, - help='File name for saving PDF report with scan results. ' - 'Optional parameter. If specified, PDF report will be saved to specified file.') - scan_group.add_argument('--nowait', '-nw', action='store_true', - help='Do not wait for scan completion. ' - 'If set, utility will start scan and exit immediately. ' - 'If not set, utility will wait for scan completion and output results.') - scan_group.add_argument('--long_wait', action='store_true', - help='Increase time limit for waiting scan completion to 1 week. ' - 'By default, standard timeout is used.') - - # CR Report arguments - cr_report_group = parser.add_argument_group('CR Report Parameters', - 'Parameters for generating Compliance Report (CR)') - cr_report_group.add_argument('--cr_report', action='store_true', - help='Enable CR report generation after scan completion. ' - 'CR report contains information about application compliance with security requirements.') - cr_report_group.add_argument('--stingray_login', type=str, - help='Login for accessing Stingray system for CR report generation. ' - 'Required parameter when using --cr_report.') - cr_report_group.add_argument('--stingray_password', type=str, - help='Password for accessing Stingray system for CR report generation. ' - 'Required parameter when using --cr_report. ' - 'It is recommended to use environment variables for security.') - cr_report_group.add_argument('--organization_name', type=str, - help='Organization name for CR report. ' - 'Default: ООО Стингрей Технолоджиз', - default='ООО Стингрей Технолоджиз') - cr_report_group.add_argument('--engineer_name', type=str, - help='Engineer name to be specified in CR report. ' - 'Optional parameter.') - cr_report_group.add_argument('--controller_name', type=str, - help='Controller name to be specified in CR report. ' - 'Optional parameter.') - cr_report_group.add_argument('--use_ldap', type=str, - help='Use LDAP for authentication when generating CR report. ' - 'Default: False. ' - 'Optional parameter.', - default=False) - cr_report_group.add_argument('--authority_server_id', type=str, - help='Authority server ID for CR report. ' - 'Optional parameter. Used when working with LDAP.', - default=None) - cr_report_group.add_argument('--cr_report_path', type=str, - help='Path for saving CR report. ' - 'Default: stingray-CR-report.html. ' - 'Optional parameter.', - default='stingray-CR-report.html') - - - - - - args = parser.parse_args() - - if args.distribution_system == 'file' and args.file_path is None: - parser.error('"--distribution_system file" requires "--file_path" argument to be set') - elif args.distribution_system == 'nexus' and ( - args.nexus_url is None or - args.nexus_login is None or - args.nexus_password is None or - args.nexus_repo_name is None or - args.nexus_group_id is None or - args.nexus_artifact_id is None or - args.nexus_version is None): - parser.error('"--distribution_system nexus" requires "--nexus_url", "--nexus_login", "--nexus_password",' - ' "--nexus_repo_name" arguments to be set') - - elif args.distribution_system == 'nexus2' and ( - args.nexus2_url is None or - args.nexus2_login is None or - args.nexus2_password is None or - args.nexus2_repo_name is None or - args.nexus2_group_id is None or - args.nexus2_artifact_id is None or - args.nexus2_version is None or - args.nexus2_extension is None): - parser.error('"--distribution_system nexus2" requires "--nexus2_url", "--nexus2_login", "--nexus2_password",' - ' "--nexus2_repo_name", "--nexus2_group_id", "--nexus2_artifact_id", "--nexus2_extension" ' - 'arguments to be set') - - elif args.distribution_system == 'firebase' and ( - args.firebase_project_number is None or - args.firebase_app_id is None or - args.firebase_account_json_path is None or - args.firebase_file_extension is None): - parser.error('"--distribution_system firebase" requires "--firebase_project_number", "--firebase_app_id", ' - '"--firebase_account_json_path", "--firebase_file_extension" arguments to be set') - - elif args.distribution_system == 'appstore' and ( - (args.appstore_app_id is None and args.appstore_bundle_id is None) or - args.appstore_apple_id is None or - (args.appstore_password is None or args.appstore_2FA is None) and args.appstore_password2FA is None): - parser.error('"--distribution_system appstore" requires either "--appstore_app_id" or "--appstore_bundle_id", ' - '"--appstore_apple_id" and ("--appstore_password" + "--appstore_2FA")/' - '(deprecated "--appstore_password2FA") arguments to be set') - - elif args.distribution_system == 'google_play': - if args.google_play_package_name is None: - parser.error('"--distribution_system google_play" requires "--google_play_package_name" to be set') - # 1) email + aas_token - # 2) email + oauth2_token (we will fetch aas_token automatically) - email_aas_ok = (args.google_play_email is not None and args.google_play_aas_token is not None) - email_oauth2_ok = (args.google_play_email is not None and args.google_play_oauth2_token is not None) - if not (email_aas_ok or email_oauth2_ok): - parser.error( - '"--distribution_system google_play" requires one of: ' - '(1) "--google_play_email" + "--google_play_aas_token", ' - '(2) "--google_play_email" + "--google_play_oauth2_token".' - ) - - elif args.distribution_system == 'rustore' and args.rustore_package_name is None: - parser.error('"--distribution_system rustore" requires "--rustore_package_name" to be set') - - elif args.distribution_system == 'rumarket' and args.rumarket_package_name is None: - parser.error('"--distribution_system rumarket" requires "--rumarket_package_name" to be set') - - elif args.distribution_system == 'appgallery' and args.appgallery_app_id is None: - parser.error('"--distribution_system appgallery" requires "--appgallery_app_id" to be set') - - return args - - -def main(): - urllib3.disable_warnings() - - arguments = parse_args() - - if getattr(arguments, 'verbose', False): - logging.getLogger().setLevel(logging.DEBUG) - logger.debug('Verbose (debug) logging enabled') - - distribution_system = arguments.distribution_system - download_path = arguments.download_path - - if arguments.download_only is False: - url = arguments.url - company_id = arguments.company_id - architecture = arguments.architecture_id - token = arguments.token - profile_id = arguments.profile_id - project_id = arguments.project_id - testcase_id = arguments.testcase_id - appium_script_path = arguments.appium_script_path - json_summary_file_name = arguments.summary_report_json_file_name - pdf_report_file_name = arguments.pdf_report_file_name - not_wait_scan_end = arguments.nowait - long_wait = arguments.long_wait - cr_report = arguments.cr_report - stingray_login = arguments.stingray_login - stingray_password = arguments.stingray_password - organization_name = arguments.organization_name - engineer_name = arguments.engineer_name - controller_name = arguments.controller_name - use_ldap = arguments.use_ldap - authority_server_id = arguments.authority_server_id - cr_report_path = arguments.cr_report_path - - - url = url if url.endswith('/') else f'{url}/' - url = url if url.endswith('rest/') else f'{url}rest' - - app_file = '' - appstore_app_md5 = None - - try: - if distribution_system == 'file': - app_file = arguments.file_path - - elif distribution_system == 'nexus': - nexus_repository = NexusRepository(arguments.nexus_url, - arguments.nexus_login, - arguments.nexus_password) - app_file = nexus_repository.download_app(download_path, - arguments.nexus_repo_name, - arguments.nexus_group_id, - arguments.nexus_artifact_id, - arguments.nexus_version) - elif distribution_system == 'nexus2': - nexus2_repository = Nexus2Repository(arguments.nexus2_url, - arguments.nexus2_login, - arguments.nexus2_password) - app_file = nexus2_repository.download_app(download_path, - arguments.nexus2_repo_name, - arguments.nexus2_group_id, - arguments.nexus2_artifact_id, - arguments.nexus2_version, - arguments.nexus2_extension, - arguments.nexus2_file_name) - - elif distribution_system == 'firebase': - app_file = firebase_download_app(download_path, - arguments.firebase_project_number, - arguments.firebase_app_id, - arguments.firebase_account_json_path, - arguments.firebase_file_name, - arguments.firebase_file_extension) - - elif distribution_system == 'appstore': - if arguments.appstore_password and arguments.appstore_2FA: - password2FA = arguments.appstore_password + arguments.appstore_2FA - else: - password2FA = arguments.appstore_password2FA - appstore = AppStore(arguments.appstore_apple_id, - password2FA) - app_file, appstore_app_md5 = appstore.download_app( - download_path, - arguments.appstore_app_id, - arguments.appstore_bundle_id, - arguments.appstore_country, - arguments.appstore_file_name, - ) - - elif distribution_system == 'google_play': - # Resolve AAS token from OAuth2 if provided and AAS not given - resolved_aas_token = arguments.google_play_aas_token - if (not resolved_aas_token) and arguments.google_play_oauth2_token: - try: - resolved_aas_token = asyncio.run( - gp_apkeep.fetch_aas_token( - email=arguments.google_play_email or '', - oauth2_token=arguments.google_play_oauth2_token, - timeout_sec=gp_apkeep.DEFAULT_TIMEOUT_SEC - ) - ) - except Exception as ex: - logger.error(f'Failed to obtain AAS token via OAuth2: {ex}') - sys.exit(ExitCode.AUTH_ERROR) - - google_play = GooglePlay(arguments.google_play_email, - resolved_aas_token) - google_play.login() - - app_file = google_play.download_app(download_path, - arguments.google_play_package_name, - arguments.google_play_file_name, proxy=arguments.google_play_proxy) - - elif distribution_system == 'rustore': - package_name = arguments.rustore_package_name - app_file = rustore_download_app(package_name, download_path) - - elif distribution_system == 'rumarket': - package_name = arguments.rumarket_package_name - app_file = rumarket_download_app(package_name, download_path) - - elif distribution_system == 'appgallery': - app_file = appgallery_download_app(arguments.appgallery_app_id, - download_path, - arguments.appgallery_file_name) - - except Exception as e: - logger.fatal( - 'Cannot download application file: %s (exception type: %s)', - e, - type(e).__name__, - ) - logger.exception('Download failed (traceback below)') - sys.exit(ExitCode.DOWNLOAD_FAILED) - - if arguments.download_only is True: - logger.info('Your application was downloaded!') - # Emit a single-line machine-friendly output for CI parsers - print(f'DOWNLOAD_PATH={app_file}') - sys.exit(ExitCode.SUCCESS) - - try: - installation_mode = resolve_installation_mode(url, token, company_id, - verify=tls_verify_enabled()) - except ModeDetectionError as ex: - logger.error(str(ex)) - sys.exit(ExitCode.AUTH_ERROR if ex.auth_error else ExitCode.NETWORK_ERROR) - - if installation_mode == MODE_MICROSERVICES: - from mdast_cli.ms_flow import run_microservices_flow - run_microservices_flow(arguments, url, token, app_file, appstore_app_md5, - user_agent=USER_AGENT, verify=tls_verify_enabled()) - logger.info('Job completed successfully!') - sys.exit(ExitCode.SUCCESS) - - mdast = mDast(url, token, company_id) - get_architectures_resp = mdast.get_architectures() - - if get_architectures_resp.status_code != 200: - logger.error(f'Error while getting architectures. Server response: {get_architectures_resp.text}') - sys.exit(ExitCode.NETWORK_ERROR) - - architectures = get_architectures_resp.json() - - _, file_extension = os.path.splitext(app_file) - - if architecture is None: - if file_extension in ANDROID_EXTENSIONS: - architecture = next((arch['id'] for arch in architectures if arch.get('name', '') == DEFAULT_ANDROID_ARCHITECTURE), None) - if file_extension == '.ipa': - architecture = next((arch['id'] for arch in architectures if arch.get('name', '') == DEFAULT_IOS_ARCHITECTURE), None) - if architecture is None: - logger.error(f"Cannot create scan - no suitable architecture for this app, try to set it manually with --architecture_id") - sys.exit(ExitCode.INVALID_ARGS) - architecture_type = next(arch for arch in architectures if arch.get('id', '') == architecture) - logger.info(f'Architecture type is {architecture_type}') - - if testcase_id is not None: - get_testcase_resp = mdast.get_testcase(testcase_id) - if get_testcase_resp.status_code == 200: - architecture = get_testcase_resp.json()['architecture']['id'] - else: - logger.warning("Testcase with this id does not exist or you use old version of system. Trying to use " - "architecture from command line params.") - - if sum(e['architecture'] == architecture_type['id'] and e['state'] == 3 for e in mdast.get_engines().json()) == 0: - logger.error(f"Cannot create scan - Cannot find active engine for architecture {architecture_type['name']}") - sys.exit(ExitCode.SCAN_FAILED) - - if testcase_id: - logger.info(f'Autoscan(Stingray) with test case id: ' - f'{testcase_id}, profile id: {profile_id} and file: {app_file}, architecture id is {architecture}') - elif appium_script_path: - logger.info(f'Autoscan(Appium) with profile id: {profile_id} and file: {app_file},' - f' architecture id is {architecture}') - else: - logger.info(f'Manual scan with profile id: {profile_id} and file located in {app_file},' - f' architecture id is {architecture}') - - logger.info('Check if this version of application was already uploaded..') - if appstore_app_md5: - check_app_already_uploaded = mdast.check_app_md5(mdast.company_id, appstore_app_md5).json() - else: - check_app_already_uploaded = mdast.check_app_md5(mdast.company_id, check_app_md5(app_file)).json() - if check_app_already_uploaded: - application = check_app_already_uploaded[0] - logger.info(f"This app was uploaded before, application id is: {application['id']}, " - f"package name: {application['package_name']}," - f" version: {application['version_name']}, md5: {application['md5']}") - else: - logger.info('This is new application or new version') - logger.info('Uploading application to server..') - upload_application_resp = mdast.upload_application(app_file, str(architecture_type['type'])) - if upload_application_resp.status_code != 201: - logger.error(f'Error while uploading application to server: {upload_application_resp.text}') - sys.exit(ExitCode.SCAN_FAILED) - application = upload_application_resp.json() - logger.info(f"Application uploaded successfully. Application id: {application['id']}") - - logger.info(f"Creating scan for application {application['id']}") - if testcase_id is not None: - create_dast_resp = mdast.create_auto_scan(project_id, profile_id, application['id'], architecture, testcase_id) - scan_type = 'auto_stingray' - elif appium_script_path is not None: - create_dast_resp = mdast.create_appium_scan(project_id, profile_id, application['id'], architecture, - appium_script_path) - scan_type = 'auto_appium' - else: - create_dast_resp = mdast.create_manual_scan(project_id, profile_id, application['id'], architecture) - scan_type = 'manual' - if create_dast_resp.status_code != 201: - logger.error(f'Error while creating scan: {create_dast_resp.text}') - sys.exit(ExitCode.SCAN_FAILED) - - dast_info = create_dast_resp.json() - logger.info(f"Project and profile was created/found successfully." - f" Project id: {dast_info['project']['id']}, profile id: {dast_info['profile']['id']}") - - dast = create_dast_resp.json() - if 'id' not in dast or dast.get('id', '') == '': - logger.error(f'Something went wrong while creating scan: {dast}') - sys.exit(ExitCode.SCAN_FAILED) - - if scan_type == 'auto_stingray': - logger.info(f"Autoscan(Stingray) was created successfully. Scan id: {dast['id']}") - elif scan_type == 'auto_appium': - logger.info(f"Autoscan(Appium) was created successfully. Scan id: {dast['id']}") - else: - logger.info(f"Manual scan was created successfully. Scan id: {dast['id']}") - - logger.info(f"Start scan with id {dast['id']}") - start_dast_resp = mdast.start_scan(dast['id']) - if start_dast_resp.status_code != 200: - logger.error(f"Error while starting scan with id {dast['id']}: {start_dast_resp.text}") - sys.exit(ExitCode.SCAN_FAILED) - - if not_wait_scan_end: - logger.info('Scan successfully started. Don`t wait for end, exit with zero code') - sys.exit(ExitCode.SUCCESS) - - logger.info("Scan started successfully.") - logger.info(f"Checking scan state with id {dast['id']}") - get_dast_info_resp = mdast.get_scan_info(dast['id']) - if get_dast_info_resp.status_code != 200: - logger.error(f"Error while getting scan info with id {dast['id']}: {get_dast_info_resp.text}") - sys.exit(ExitCode.NETWORK_ERROR) - - dast = get_dast_info_resp.json() - dast_status = dast['state'] - logger.info(f"Current scan status: {DastStateDict.get(dast_status)}") - count = 0 - - if long_wait: - try_count = LONG_TRY - else: - try_count = TRY - - while dast_status in (DastState.CREATED, DastState.INITIALIZING, DastState.STARTING) and count < try_count: - logger.info(f"Try to get scan status for scan id {dast['id']}. Count number {count}") - get_dast_info_resp = mdast.get_scan_info(dast['id']) - if get_dast_info_resp.status_code != 200: - logger.error(f"Error while getting scan info with id {dast['id']}: {get_dast_info_resp.text}") - sys.exit(1) - - dast = get_dast_info_resp.json() - dast_status = dast['state'] - logger.info(f"Current scan status: {DastStateDict.get(dast_status)}") - count += 1 - if dast_status not in (DastState.STARTED, DastState.SUCCESS): - logger.info(f"Wait {SLEEP_TIMEOUT} seconds and try again") - time.sleep(SLEEP_TIMEOUT) - - if dast['state'] not in (DastState.STARTED, DastState.STOPPING, DastState.ANALYZING, DastState.SUCCESS): - logger.error(f"Error with scan id {dast['id']}. Current scan status: {dast['state']}," - f" but expected to be {DastState.STARTED}, {DastState.ANALYZING}, {DastState.STOPPING} " - f"or {DastState.SUCCESS}") - sys.exit(ExitCode.SCAN_FAILED) - logger.info(f"Scan {dast['id']} is started now. Let's wait until the scan is finished") - - get_dast_info_resp = mdast.get_scan_info(dast['id']) - if get_dast_info_resp.status_code != 200: - logger.error(f"Error while getting scan info with id {dast['id']}: {get_dast_info_resp.text}") - sys.exit(ExitCode.NETWORK_ERROR) - count = 0 - - while dast_status in (DastState.STARTED, DastState.STOPPING, DastState.ANALYZING) and count < try_count: - if count == 0 and scan_type == 'manual': - if dast_status not in (DastState.ANALYZING, DastState.SUCCESS): - logger.info(f"This is manual scan with dynamic modules," - f" lets wait for {END_SCAN_TIMEOUT} seconds and stop it.") - time.sleep(END_SCAN_TIMEOUT) - stop_manual_dast_resp = mdast.stop_scan(dast['id']) - if stop_manual_dast_resp.status_code == 200: - logger.info(f'Scan {dast["id"]} was successfully stopped') - else: - logger.error(f'Error while stopping scan with id {dast["id"]}: {stop_manual_dast_resp.text}') - sys.exit(ExitCode.SCAN_FAILED) - else: - logger.info("This is manual scan with profile without dynamic modules," - " only SAST, lets wait till the end") - - logger.info(f"Try to get scan status for scan id {dast['id']}. Count number {count}") - get_dast_info_resp = mdast.get_scan_info(dast['id']) - if get_dast_info_resp.status_code != 200: - logger.error(f"Error while getting scan info with id {dast['id']}: {get_dast_info_resp.text}") - sys.exit(1) - dast = get_dast_info_resp.json() - dast_status = dast['state'] - logger.info(f"Current scan status: {DastStateDict.get(dast_status)}") - count += 1 - if dast_status is not DastState.SUCCESS: - logger.info(f"Wait {SLEEP_TIMEOUT} seconds and try again") - time.sleep(SLEEP_TIMEOUT) - - logger.info(f"Check if scan with id {dast['id']} was finished correctly.") - get_dast_info_resp = mdast.get_scan_info(dast['id']) - if get_dast_info_resp.status_code != 200: - logger.error(f"Error while getting scan info with id {dast['id']}: {get_dast_info_resp.text}") - sys.exit(ExitCode.NETWORK_ERROR) - dast = get_dast_info_resp.json() - - if dast['state'] != DastState.SUCCESS: - logger.error( - f"Expected state {DastStateDict.get(DastState.SUCCESS)}, but in real it was {dast['state']}. " - f"Exit with error status code.") - sys.exit(ExitCode.SCAN_FAILED) - - if pdf_report_file_name: - logger.info(f"Create and download pdf report for scan with id {dast['id']} to file {pdf_report_file_name}.") - pdf_report = mdast.download_report(dast['id']) - if pdf_report.status_code != 200: - logger.error(f"PDF report creating failed with error {pdf_report.text}. Exit...") - sys.exit(ExitCode.SCAN_FAILED) - - logger.info(f"Saving pdf report to file {pdf_report_file_name}.") - pdf_report_file_name = pdf_report_file_name if pdf_report_file_name.endswith( - '.pdf') else f'{pdf_report_file_name}.pdf' - with open(pdf_report_file_name, 'wb') as f: - f.write(pdf_report.content) - - logger.info(f"Report for scan {dast['id']} successfully created and available at path: {pdf_report_file_name}.") - - if json_summary_file_name: - logger.info( - f"Download JSON summary report for scan with id {dast['id']} to file {json_summary_file_name}.") - json_summary_report = mdast.download_scan_json_result(dast['id']) - if json_summary_report.status_code != 200: - logger.error(f"JSON summary report while downloading failed with error {json_summary_report.text}. Exit...") - sys.exit(ExitCode.SCAN_FAILED) - - logger.info(f"Saving summary json results to file {json_summary_file_name}.") - mdast_json_file = json_summary_file_name if json_summary_file_name.endswith( - '.json') else f'{json_summary_file_name}.json' - with open(mdast_json_file, 'w') as fp: - json.dump(json_summary_report.json(), fp, indent=4, ensure_ascii=False) - - logger.info(f"JSON report for scan {dast['id']} successfully created and available at path: {mdast_json_file}.") - - if cr_report: - generate_cr(f"{url}", stingray_login, stingray_password, dast['id'], organization_name, engineer_name, - controller_name, cr_report_path, use_ldap, authority_server_id) - - logger.info('Job completed successfully!') - - -if __name__ == '__main__': - main() +import argparse +import asyncio +import json +import logging +import os +import sys +import time +import warnings + +import requests +import urllib3 + +# Suppress pkg_resources deprecation warning from google-auth +# This warning appears when google-auth is imported, even if not used +warnings.filterwarnings('ignore', message='.*pkg_resources is deprecated.*', category=UserWarning) + +from mdast_cli.distribution_systems.appgallery import appgallery_download_app +from mdast_cli.distribution_systems.appstore import AppStore +from mdast_cli.distribution_systems.firebase import firebase_download_app +from mdast_cli.distribution_systems.google_play import GooglePlay +from mdast_cli.distribution_systems import google_play_apkeep as gp_apkeep +from mdast_cli.distribution_systems.nexus import NexusRepository +from mdast_cli.distribution_systems.nexus2 import Nexus2Repository +from mdast_cli.distribution_systems.rumarket import rumarket_download_app +from mdast_cli.distribution_systems.rustore import rustore_download_app +from mdast_cli.helpers.const import (ANDROID_EXTENSIONS, DEFAULT_ANDROID_ARCHITECTURE, DEFAULT_IOS_ARCHITECTURE, + END_SCAN_TIMEOUT, LONG_TRY, SLEEP_TIMEOUT, TRY, DastState, DastStateDict) +from mdast_cli.helpers.exit_codes import ExitCode +from mdast_cli.helpers.helpers import check_app_md5, resolve_report_targets +from mdast_cli_core.token import mDastToken as mDast +from mdast_cli_core.factory import (MODE_MICROSERVICES, ModeDetectionError, resolve_installation_mode, + tls_verify_enabled) +from mdast_cli.cr_report_generator import generate_cr +from mdast_cli import __version__ + +USER_AGENT = f'mdast_cli/{__version__}' + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s %(message)s', + datefmt='%d/%m/%Y %H:%M:%S', stream=sys.stdout) + +logger = logging.getLogger(__name__) + + +def parse_args(): + parser = argparse.ArgumentParser( + description='MDast CLI - tool for downloading and scanning mobile applications.', + epilog=''' +Usage examples: + # Download application from Google Play without scanning: + mdast_cli -d --distribution_system google_play \\ + --google_play_package_name com.example.app \\ + --google_play_email user@example.com \\ + --google_play_aas_token YOUR_TOKEN + + # Start application scanning: + mdast_cli --distribution_system file --file_path app.apk \\ + --url https://mdast.example.com --company_id 1 --token YOUR_TOKEN + +For detailed information about specific distribution system see README.md + ''', + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + # Main options + main_group = parser.add_argument_group('Main Options', 'General parameters for utility operation') + main_group.add_argument('--download_only', '-d', action='store_true', + help='Download application without starting scan. ' + 'When using this option, --url, --company_id, --token parameters are not required. ' + 'After successful download, file path is printed in format DOWNLOAD_PATH=/path/to/file') + main_group.add_argument('--distribution_system', '-ds', type=str, + help='Application distribution system for downloading. ' + 'Available options: file, nexus, nexus2, firebase, appstore, google_play, ' + 'rustore, appgallery, rumarket. ' + 'System choice determines which additional parameters will be required.', + choices=['file', 'nexus', 'nexus2', 'firebase', 'appstore', 'google_play', + 'rustore', 'appgallery', 'rumarket'], + required=True) + main_group.add_argument('--download_path', '-p', type=str, + help='Path to directory for saving downloaded applications. ' + 'Default: downloaded_apps. ' + 'Directory will be created automatically if it does not exist.', + default='downloaded_apps') + main_group.add_argument('--verbose', '-v', action='store_true', + help='Enable debug logging (e.g. for troubleshooting download failures).') + + # File distribution system + file_group = parser.add_argument_group('Local File (file)', + 'Using existing local application file') + file_group.add_argument('--file_path', type=str, + help='Full path to local application file (APK for Android or IPA for iOS). ' + 'Required parameter when --distribution_system is set to "file". ' + 'Example: /path/to/app.apk or ./my_app.ipa') + + # Nexus distribution system + nexus_group = parser.add_argument_group('Nexus Repository (nexus)', + 'Downloading applications from Nexus Repository Manager 3.x') + nexus_group.add_argument('--nexus_url', type=str, + help='Nexus server URL (with http/https protocol). ' + 'Required parameter when --distribution_system is set to "nexus". ' + 'Example: https://nexus.example.com or http://localhost:8081') + nexus_group.add_argument('--nexus_login', type=str, + help='Username for Nexus authentication. ' + 'Required parameter when --distribution_system is set to "nexus".') + nexus_group.add_argument('--nexus_password', type=str, + help='Password for Nexus authentication. ' + 'Required parameter when --distribution_system is set to "nexus". ' + 'It is recommended to use environment variables for security.') + nexus_group.add_argument('--nexus_repo_name', type=str, + help='Repository name in Nexus where the application is located. ' + 'Required parameter when --distribution_system is set to "nexus". ' + 'Example: releases, snapshots, maven-releases') + nexus_group.add_argument('--nexus_group_id', type=str, + help='Application Group ID in Maven format (usually reverse domain). ' + 'Required parameter when --distribution_system is set to "nexus". ' + 'Example: com.example, org.mycompany') + nexus_group.add_argument('--nexus_artifact_id', type=str, + help='Application Artifact ID (artifact name). ' + 'Required parameter when --distribution_system is set to "nexus". ' + 'Example: myapp, mobile-app') + nexus_group.add_argument('--nexus_version', type=str, + help='Application version to download. ' + 'Required parameter when --distribution_system is set to "nexus". ' + 'Example: 1.0.0, 2.5.1, 1.0.0-SNAPSHOT') + + # Firebase distribution system + firebase_group = parser.add_argument_group('Firebase App Distribution (firebase)', + 'Downloading applications from Firebase App Distribution') + firebase_group.add_argument('--firebase_project_number', type=int, + help='Firebase project number (Project Number). ' + 'Required parameter when --distribution_system is set to "firebase". ' + 'Can be found in Firebase Console project settings. ' + 'Example: 123456789012') + firebase_group.add_argument('--firebase_app_id', type=str, + help='Application ID in Firebase in format: "1:PROJECT_NUMBER:PLATFORM:APP_ID". ' + 'Required parameter when --distribution_system is set to "firebase". ' + 'Examples: "1:123456789012:android:abc123def456", "1:123456789012:ios:xyz789"') + firebase_group.add_argument('--firebase_account_json_path', type=str, + help='Path to JSON file with Firebase service account credentials. ' + 'Required parameter when --distribution_system is set to "firebase". ' + 'File can be downloaded from Firebase Console → Project Settings → Service Accounts. ' + 'Example: /path/to/service-account.json or ./firebase-key.json') + firebase_group.add_argument('--firebase_file_extension', type=str, + help='Application file extension for download. ' + 'Required parameter when --distribution_system is set to "firebase". ' + 'Available values: apk (Android) or ipa (iOS).', + choices=['ipa', 'apk']) + firebase_group.add_argument('--firebase_file_name', type=str, + help='File name for saving application (without extension). ' + 'Optional parameter. If not specified, version from Firebase is used. ' + 'Example: my_app, production_build') + + # AppStore distribution system + appstore_group = parser.add_argument_group('Apple App Store (appstore)', + 'Downloading applications from Apple App Store') + appstore_group.add_argument('--appstore_app_id', type=str, + help='Application ID in App Store. Can be found in application page URL: ' + 'https://apps.apple.com/app/id{APP_ID}. ' + 'Either --appstore_app_id or --appstore_bundle_id must be specified. ' + 'Example: 389801252 (Instagram), 310633997 (WhatsApp)') + appstore_group.add_argument('--appstore_bundle_id', type=str, + help='Application Bundle ID (package identifier). ' + 'Either --appstore_app_id or --appstore_bundle_id must be specified. ' + 'Example: com.instagram.ios, com.whatsapp.WhatsApp') + appstore_group.add_argument('--appstore_apple_id', type=str, + help='Apple ID email address for iTunes/App Store login. ' + 'Required parameter when --distribution_system is set to "appstore". ' + 'Example: user@example.com') + appstore_group.add_argument('--appstore_password', type=str, + help='Apple ID password. ' + 'Required parameter when --distribution_system is set to "appstore". ' + 'It is recommended to use environment variables for security.') + appstore_group.add_argument('--appstore_2FA', type=str, + help='Two-factor authentication code (6 digits). ' + 'Required parameter when --distribution_system is set to "appstore". ' + 'Code is sent to trusted Apple devices. ' + 'Example: 123456') + appstore_group.add_argument('--appstore_file_name', type=str, + help='File name for saving application (without .ipa extension). ' + 'Optional parameter. If not specified, application name and version are used. ' + 'Example: my_app, instagram_latest') + appstore_group.add_argument('--appstore_country', type=str, default='US', + help='App Store country/region code for lookup and download (ISO 3166-1 alpha-2). ' + 'Default: US. Example: RU, DE, GB') + appstore_group.add_argument('--appstore_password2FA', type=str, + help='[DEPRECATED] Password and 2FA code in one parameter (format: password2FA_code). ' + 'Use --appstore_password and --appstore_2FA instead of this parameter. ' + 'Will be removed in future versions.', + metavar='DEPRECATED') + + # Google Play distribution system + google_play_group = parser.add_argument_group('Google Play (google_play)', + 'Downloading applications from Google Play Store using apkeep') + google_play_group.add_argument('--google_play_package_name', type=str, + help='Application package name from Google Play. ' + 'Required parameter when --distribution_system is set to "google_play". ' + 'Can be found in application page URL: ' + 'https://play.google.com/store/apps/details?id={PACKAGE_NAME}. ' + 'Examples: com.instagram.android, com.whatsapp, org.telegram.messenger') + google_play_group.add_argument('--google_play_email', type=str, + help='Google account email address. ' + 'Required parameter when --distribution_system is set to "google_play". ' + 'Used together with --google_play_aas_token or --google_play_oauth2_token. ' + 'Example: user@gmail.com') + google_play_group.add_argument('--google_play_aas_token', type=str, + help='AAS token for Google Play (obtained via apkeep). ' + 'Used for subsequent downloads after first token acquisition. ' + 'Requires --google_play_email. ' + 'Format: aas_et/... (long string). ' + 'For first run use --google_play_oauth2_token.') + google_play_group.add_argument('--google_play_oauth2_token', type=str, + help='OAuth2 token to obtain AAS token via apkeep. ' + 'Used on first run. After obtaining AAS token, you can use ' + '--google_play_aas_token for subsequent downloads. ' + 'Requires --google_play_email. ' + 'Format: ya29.a0AVvZVs... (OAuth2 access token)') + google_play_group.add_argument('--google_play_file_name', type=str, + help='File name for saving application (without extension). ' + 'Optional parameter. If not specified, package name is used. ' + 'Example: instagram_latest, whatsapp_production') + google_play_group.add_argument('--google_play_proxy', type=str, + help='Proxy settings for connecting to Google Play. ' + 'Optional parameter. ' + 'Format: socks5://user:pass@host:port or http://user:pass@host:port. ' + 'Example: socks5://proxy.example.com:1080') + + # RuStore distribution system + rustore_group = parser.add_argument_group('RuStore (rustore)', + 'Downloading applications from Russian RuStore marketplace') + rustore_group.add_argument('--rustore_package_name', type=str, + help='Application package name from RuStore. ' + 'Required parameter when --distribution_system is set to "rustore". ' + 'Can be found in application page URL. ' + 'Examples: com.vkontakte.android, com.yandex.browser') + + # RuMarket distribution system + rumarket_group = parser.add_argument_group('RuMarket (rumarket)', + 'Downloading applications from Russian RuMarket marketplace') + rumarket_group.add_argument('--rumarket_package_name', type=str, + help='Application package name from RuMarket. ' + 'Required parameter when --distribution_system is set to "rumarket". ' + 'Example: com.example.app') + + # AppGallery distribution system + appgallery_group = parser.add_argument_group('Huawei AppGallery (appgallery)', + 'Downloading applications from Huawei AppGallery') + appgallery_group.add_argument('--appgallery_app_id', type=str, + help='Application ID in AppGallery. ' + 'Required parameter when --distribution_system is set to "appgallery". ' + 'Can be found in application page URL: ' + 'https://appgallery.huawei.com/app/{APP_ID}. ' + 'Format: C + digits, e.g.: C101184875, C100000001') + appgallery_group.add_argument('--appgallery_file_name', type=str, + help='File name for saving application (without .apk extension). ' + 'Optional parameter. If not specified, package name and version are used. ' + 'Example: huawei_app, instagram_huawei') + + # Nexus2 distribution system + nexus2_group = parser.add_argument_group('Nexus Repository 2.x (nexus2)', + 'Downloading applications from Nexus Repository Manager 2.x') + nexus2_group.add_argument('--nexus2_url', type=str, + help='Nexus 2.x server URL (with http/https protocol and /nexus/ path). ' + 'Required parameter when --distribution_system is set to "nexus2". ' + 'Example: http://nexus.example.com:8081/nexus/ or http://localhost:8081/nexus/') + nexus2_group.add_argument('--nexus2_login', type=str, + help='Username for Nexus 2.x authentication. ' + 'Required parameter when --distribution_system is set to "nexus2".') + nexus2_group.add_argument('--nexus2_password', type=str, + help='Password for Nexus 2.x authentication. ' + 'Required parameter when --distribution_system is set to "nexus2". ' + 'It is recommended to use environment variables for security.') + nexus2_group.add_argument('--nexus2_repo_name', type=str, + help='Repository name in Nexus 2.x where the application is located. ' + 'Required parameter when --distribution_system is set to "nexus2". ' + 'Example: releases, snapshots') + nexus2_group.add_argument('--nexus2_group_id', type=str, + help='Application Group ID in Maven format. ' + 'Required parameter when --distribution_system is set to "nexus2". ' + 'Example: com.example, org.mycompany') + nexus2_group.add_argument('--nexus2_artifact_id', type=str, + help='Application Artifact ID. ' + 'Required parameter when --distribution_system is set to "nexus2". ' + 'Example: myapp, android-app') + nexus2_group.add_argument('--nexus2_version', type=str, + help='Application version to download. ' + 'Required parameter when --distribution_system is set to "nexus2". ' + 'Example: 1.0.0, 2.5.1, 1.0.0-SNAPSHOT') + nexus2_group.add_argument('--nexus2_extension', type=str, + help='Application file extension. ' + 'Required parameter when --distribution_system is set to "nexus2". ' + 'Usually: apk (Android) or ipa (iOS). ' + 'Example: apk, ipa, zip') + nexus2_group.add_argument('--nexus2_file_name', type=str, + help='File name for saving application (without extension). ' + 'Optional parameter. If not specified, generated automatically. ' + 'Example: my_app, production_build') + + # Scanning arguments + scan_group = parser.add_argument_group('Scanning Parameters', + 'Parameters for starting and managing application scanning') + scan_group.add_argument('--url', type=str, + help='MDast server URL for submitting application for scanning. ' + 'Required parameter when starting scan (without --download_only). ' + 'Example: https://mdast.example.com') + scan_group.add_argument('--company_id', type=int, + help='Company ID in MDast system. ' + 'Required parameter when starting scan (without --download_only). ' + 'Can be found in company settings in MDast web interface.') + scan_group.add_argument('--token', type=str, + help='CI/CD token for authentication and starting scan. ' + 'Required parameter when starting scan (without --download_only). ' + 'Token can be obtained in profile settings in MDast web interface. ' + 'It is recommended to use environment variables for security.') + scan_group.add_argument('--architecture_id', type=int, + help='Architecture ID for performing scan. ' + 'Optional parameter. If not specified, default architecture is used. ' + 'List of available architectures can be obtained via MDast API.') + scan_group.add_argument('--profile_id', type=int, default=None, + help='Profile ID for scanning. ' + 'Optional parameter. If not specified, profile will be created automatically.') + scan_group.add_argument('--project_id', type=int, default=None, + help='Project ID for scanning. ' + 'Optional parameter. Used only when auto-creating profile ' + 'to place new profile in existing project.') + scan_group.add_argument('--testcase_id', type=int, + help='Test case ID for automatic scanning. ' + 'Optional parameter. If not specified, manual scanning is performed.') + scan_group.add_argument('--appium_script_path', type=str, + help='Path to Appium script for automatic scanning using Stingray Appium. ' + 'Optional parameter. Used for automated testing.') + scan_group.add_argument('--report_format', type=str, + choices=['pdf', 'json', 'all', 'none'], default=None, + help='Which scan report(s) to download after a successful scan. ' + 'Works the same on both installations (monolith and microservices). ' + 'Default: pdf. Use "json" for the structured JSON summary, "all" for both, ' + '"none" to skip reports. When --pdf_report_file_name / ' + '--summary_report_json_file_name are given, those formats are always ' + 'produced (this flag can add the other one). Without explicit file names, ' + 'reports are saved as scan_report_.pdf / .json.') + scan_group.add_argument('--summary_report_json_file_name', type=str, + help='File name for saving JSON report with scan results in structured format. ' + 'Optional parameter. If specified, the JSON report will be saved to this file ' + '(implies --report_format json).') + scan_group.add_argument('--pdf_report_file_name', type=str, + help='File name for saving PDF report with scan results. ' + 'Optional parameter. If specified, the PDF report will be saved to this file ' + '(implies --report_format pdf).') + scan_group.add_argument('--nowait', '-nw', action='store_true', + help='Do not wait for scan completion. ' + 'If set, utility will start scan and exit immediately. ' + 'If not set, utility will wait for scan completion and output results.') + scan_group.add_argument('--long_wait', action='store_true', + help='Increase time limit for waiting scan completion to 1 week. ' + 'By default, standard timeout is used.') + + # CR Report arguments + cr_report_group = parser.add_argument_group('CR Report Parameters', + 'Parameters for generating Compliance Report (CR)') + cr_report_group.add_argument('--cr_report', action='store_true', + help='Enable CR report generation after scan completion. ' + 'CR report contains information about application compliance with security requirements.') + cr_report_group.add_argument('--stingray_login', type=str, + help='Login for accessing Stingray system for CR report generation. ' + 'Required parameter when using --cr_report.') + cr_report_group.add_argument('--stingray_password', type=str, + help='Password for accessing Stingray system for CR report generation. ' + 'Required parameter when using --cr_report. ' + 'It is recommended to use environment variables for security.') + cr_report_group.add_argument('--organization_name', type=str, + help='Organization name for CR report. ' + 'Default: ООО Стингрей Технолоджиз', + default='ООО Стингрей Технолоджиз') + cr_report_group.add_argument('--engineer_name', type=str, + help='Engineer name to be specified in CR report. ' + 'Optional parameter.') + cr_report_group.add_argument('--controller_name', type=str, + help='Controller name to be specified in CR report. ' + 'Optional parameter.') + cr_report_group.add_argument('--use_ldap', type=str, + help='Use LDAP for authentication when generating CR report. ' + 'Default: False. ' + 'Optional parameter.', + default=False) + cr_report_group.add_argument('--authority_server_id', type=str, + help='Authority server ID for CR report. ' + 'Optional parameter. Used when working with LDAP.', + default=None) + cr_report_group.add_argument('--cr_report_path', type=str, + help='Path for saving CR report. ' + 'Default: stingray-CR-report.html. ' + 'Optional parameter.', + default='stingray-CR-report.html') + + + + + + args = parser.parse_args() + + if args.distribution_system == 'file' and args.file_path is None: + parser.error('"--distribution_system file" requires "--file_path" argument to be set') + elif args.distribution_system == 'nexus' and ( + args.nexus_url is None or + args.nexus_login is None or + args.nexus_password is None or + args.nexus_repo_name is None or + args.nexus_group_id is None or + args.nexus_artifact_id is None or + args.nexus_version is None): + parser.error('"--distribution_system nexus" requires "--nexus_url", "--nexus_login", "--nexus_password",' + ' "--nexus_repo_name" arguments to be set') + + elif args.distribution_system == 'nexus2' and ( + args.nexus2_url is None or + args.nexus2_login is None or + args.nexus2_password is None or + args.nexus2_repo_name is None or + args.nexus2_group_id is None or + args.nexus2_artifact_id is None or + args.nexus2_version is None or + args.nexus2_extension is None): + parser.error('"--distribution_system nexus2" requires "--nexus2_url", "--nexus2_login", "--nexus2_password",' + ' "--nexus2_repo_name", "--nexus2_group_id", "--nexus2_artifact_id", "--nexus2_extension" ' + 'arguments to be set') + + elif args.distribution_system == 'firebase' and ( + args.firebase_project_number is None or + args.firebase_app_id is None or + args.firebase_account_json_path is None or + args.firebase_file_extension is None): + parser.error('"--distribution_system firebase" requires "--firebase_project_number", "--firebase_app_id", ' + '"--firebase_account_json_path", "--firebase_file_extension" arguments to be set') + + elif args.distribution_system == 'appstore' and ( + (args.appstore_app_id is None and args.appstore_bundle_id is None) or + args.appstore_apple_id is None or + (args.appstore_password is None or args.appstore_2FA is None) and args.appstore_password2FA is None): + parser.error('"--distribution_system appstore" requires either "--appstore_app_id" or "--appstore_bundle_id", ' + '"--appstore_apple_id" and ("--appstore_password" + "--appstore_2FA")/' + '(deprecated "--appstore_password2FA") arguments to be set') + + elif args.distribution_system == 'google_play': + if args.google_play_package_name is None: + parser.error('"--distribution_system google_play" requires "--google_play_package_name" to be set') + # 1) email + aas_token + # 2) email + oauth2_token (we will fetch aas_token automatically) + email_aas_ok = (args.google_play_email is not None and args.google_play_aas_token is not None) + email_oauth2_ok = (args.google_play_email is not None and args.google_play_oauth2_token is not None) + if not (email_aas_ok or email_oauth2_ok): + parser.error( + '"--distribution_system google_play" requires one of: ' + '(1) "--google_play_email" + "--google_play_aas_token", ' + '(2) "--google_play_email" + "--google_play_oauth2_token".' + ) + + elif args.distribution_system == 'rustore' and args.rustore_package_name is None: + parser.error('"--distribution_system rustore" requires "--rustore_package_name" to be set') + + elif args.distribution_system == 'rumarket' and args.rumarket_package_name is None: + parser.error('"--distribution_system rumarket" requires "--rumarket_package_name" to be set') + + elif args.distribution_system == 'appgallery' and args.appgallery_app_id is None: + parser.error('"--distribution_system appgallery" requires "--appgallery_app_id" to be set') + + # F13: при запуске скана (без --download_only) обязательны --url/--company_id/--token. + # Прежнее required=(not '-d') вычислялось в False и не работало. + if not args.download_only: + missing = [name for name, val in ( + ('--url', args.url), ('--company_id', args.company_id), ('--token', args.token), + ) if val in (None, '')] + if missing: + parser.error( + 'scanning requires ' + ', '.join(missing) + + ' (or use --download_only to only download the application)') + + # --cr_report (monolith-only) needs Stingray credentials; fail fast with a + # clear message instead of crashing mid-flow after the scan already ran. + if args.cr_report and (not args.stingray_login or not args.stingray_password): + parser.error('--cr_report requires --stingray_login and --stingray_password') + + return args + + +def _mono_http_error(resp, action, default_exit=ExitCode.SCAN_FAILED): + """Classify a failed monolith HTTP response and exit with the right code. + + 401/403 map to AUTH_ERROR(7) in BOTH installations (F12 parity with + ms_flow._exit_on_http_error); every other status uses the caller's default + (NETWORK_ERROR for read/poll calls, SCAN_FAILED for mutating calls). + """ + body = getattr(resp, 'text', '') + if resp.status_code in (401, 403): + logger.error(f'{action}: authorization failed (HTTP {resp.status_code}): {body}. ' + 'Check the CI/CD token.') + sys.exit(ExitCode.AUTH_ERROR) + logger.error(f'{action} (HTTP {resp.status_code}): {body}') + sys.exit(default_exit) + + +def run_monolith_flow(arguments, url, token, company_id, app_file, appstore_app_md5=None): + """Full scan flow against the monolith installation. Exits the process. + + Wrapped by main() in a requests.RequestException guard so a bare network error + surfaces as NETWORK_ERROR(6) - the same class the microservices flow uses - + instead of a traceback + INTERNAL_ERROR(1). Kept symmetric with + run_microservices_flow. + """ + architecture = arguments.architecture_id + profile_id = arguments.profile_id + project_id = arguments.project_id + testcase_id = arguments.testcase_id + appium_script_path = arguments.appium_script_path + not_wait_scan_end = arguments.nowait + long_wait = arguments.long_wait + + mdast = mDast(url, token, company_id) + get_architectures_resp = mdast.get_architectures() + if get_architectures_resp.status_code != 200: + _mono_http_error(get_architectures_resp, 'Error while getting architectures', + ExitCode.NETWORK_ERROR) + architectures = get_architectures_resp.json() + if not isinstance(architectures, list): + logger.error('Getting architectures: unexpected response shape (expected a list)') + sys.exit(ExitCode.NETWORK_ERROR) + + _, file_extension = os.path.splitext(app_file) + + if architecture is None: + if file_extension in ANDROID_EXTENSIONS: + architecture = next((arch.get('id') for arch in architectures + if isinstance(arch, dict) and arch.get('name') == DEFAULT_ANDROID_ARCHITECTURE), None) + if file_extension == '.ipa': + architecture = next((arch.get('id') for arch in architectures + if isinstance(arch, dict) and arch.get('name') == DEFAULT_IOS_ARCHITECTURE), None) + if architecture is None: + logger.error("Cannot create scan - no suitable architecture for this app, try to set it manually with --architecture_id") + sys.exit(ExitCode.INVALID_ARGS) + # An explicit --architecture_id that the server does not know is a user error, + # not an internal crash: guard next() with a default instead of StopIteration, + # and skip non-dict items so a malformed payload can't raise AttributeError. + architecture_type = next((arch for arch in architectures + if isinstance(arch, dict) and arch.get('id', '') == architecture), None) + if architecture_type is None: + logger.error(f"--architecture_id {architecture} is not available on this server. " + "Check the available architectures.") + sys.exit(ExitCode.INVALID_ARGS) + logger.info(f'Architecture type is {architecture_type}') + + if testcase_id is not None: + # Best-effort (preserved from the original monolith flow): the test-case + # lookup only refines the architecture. Any non-200 - including 401/403 - + # falls back to the CLI/auto architecture rather than aborting; the real + # authorization check happens on create_auto_scan below. + get_testcase_resp = mdast.get_testcase(testcase_id) + if get_testcase_resp.status_code == 200: + architecture = get_testcase_resp.json()['architecture']['id'] + else: + logger.warning("Testcase with this id does not exist or you use old version of system. Trying to use " + "architecture from command line params.") + + get_engines_resp = mdast.get_engines() + if get_engines_resp.status_code != 200: + _mono_http_error(get_engines_resp, 'Error while getting engines', ExitCode.NETWORK_ERROR) + engines = get_engines_resp.json() + if not isinstance(engines, list): + logger.error('Getting engines: unexpected response shape (expected a list)') + sys.exit(ExitCode.NETWORK_ERROR) + if sum(isinstance(e, dict) and e.get('architecture') == architecture_type['id'] and e.get('state') == 3 + for e in engines) == 0: + logger.error(f"Cannot create scan - Cannot find active engine for architecture {architecture_type['name']}") + sys.exit(ExitCode.SCAN_FAILED) + + if testcase_id: + logger.info(f'Autoscan(Stingray) with test case id: ' + f'{testcase_id}, profile id: {profile_id} and file: {app_file}, architecture id is {architecture}') + elif appium_script_path: + logger.info(f'Autoscan(Appium) with profile id: {profile_id} and file: {app_file},' + f' architecture id is {architecture}') + else: + logger.info(f'Manual scan with profile id: {profile_id} and file located in {app_file},' + f' architecture id is {architecture}') + + logger.info('Check if this version of application was already uploaded..') + dedup_md5 = appstore_app_md5 if appstore_app_md5 else check_app_md5(app_file) + dedup_resp = mdast.check_app_md5(mdast.company_id, dedup_md5) + if dedup_resp.status_code != 200: + _mono_http_error(dedup_resp, 'Application dedup check', ExitCode.NETWORK_ERROR) + check_app_already_uploaded = dedup_resp.json() + if isinstance(check_app_already_uploaded, list) and check_app_already_uploaded: + application = check_app_already_uploaded[0] + logger.info(f"This app was uploaded before, application id is: {application.get('id')}, " + f"package name: {application.get('package_name')}," + f" version: {application.get('version_name')}, md5: {application.get('md5')}") + else: + logger.info('This is new application or new version') + logger.info('Uploading application to server..') + upload_application_resp = mdast.upload_application(app_file, str(architecture_type['type'])) + if upload_application_resp.status_code != 201: + _mono_http_error(upload_application_resp, 'Error while uploading application to server') + application = upload_application_resp.json() + logger.info(f"Application uploaded successfully. Application id: {application['id']}") + + logger.info(f"Creating scan for application {application['id']}") + if testcase_id is not None: + create_dast_resp = mdast.create_auto_scan(project_id, profile_id, application['id'], architecture, testcase_id) + scan_type = 'auto_stingray' + elif appium_script_path is not None: + create_dast_resp = mdast.create_appium_scan(project_id, profile_id, application['id'], architecture, + appium_script_path) + scan_type = 'auto_appium' + else: + create_dast_resp = mdast.create_manual_scan(project_id, profile_id, application['id'], architecture) + scan_type = 'manual' + if create_dast_resp.status_code != 201: + _mono_http_error(create_dast_resp, 'Error while creating scan') + + dast = create_dast_resp.json() + logger.info(f"Project and profile was created/found successfully." + f" Project id: {(dast.get('project') or {}).get('id')}, " + f"profile id: {(dast.get('profile') or {}).get('id')}") + + if 'id' not in dast or dast.get('id', '') == '': + logger.error(f'Something went wrong while creating scan: {dast}') + sys.exit(ExitCode.SCAN_FAILED) + + if scan_type == 'auto_stingray': + logger.info(f"Autoscan(Stingray) was created successfully. Scan id: {dast['id']}") + elif scan_type == 'auto_appium': + logger.info(f"Autoscan(Appium) was created successfully. Scan id: {dast['id']}") + else: + logger.info(f"Manual scan was created successfully. Scan id: {dast['id']}") + + logger.info(f"Start scan with id {dast['id']}") + start_dast_resp = mdast.start_scan(dast['id']) + if start_dast_resp.status_code != 200: + _mono_http_error(start_dast_resp, f"Error while starting scan with id {dast['id']}") + + if not_wait_scan_end: + logger.info('Scan successfully started. Don`t wait for end, exit with zero code') + sys.exit(ExitCode.SUCCESS) + + logger.info("Scan started successfully.") + logger.info(f"Checking scan state with id {dast['id']}") + get_dast_info_resp = mdast.get_scan_info(dast['id']) + if get_dast_info_resp.status_code != 200: + _mono_http_error(get_dast_info_resp, f"Error while getting scan info with id {dast['id']}", + ExitCode.NETWORK_ERROR) + + dast = get_dast_info_resp.json() + dast_status = dast['state'] + logger.info(f"Current scan status: {DastStateDict.get(dast_status)}") + count = 0 + + try_count = LONG_TRY if long_wait else TRY + + while dast_status in (DastState.CREATED, DastState.INITIALIZING, DastState.STARTING) and count < try_count: + logger.info(f"Try to get scan status for scan id {dast['id']}. Count number {count}") + get_dast_info_resp = mdast.get_scan_info(dast['id']) + if get_dast_info_resp.status_code != 200: + _mono_http_error(get_dast_info_resp, f"Error while getting scan info with id {dast['id']}", + ExitCode.NETWORK_ERROR) + + dast = get_dast_info_resp.json() + dast_status = dast['state'] + logger.info(f"Current scan status: {DastStateDict.get(dast_status)}") + count += 1 + if dast_status not in (DastState.STARTED, DastState.SUCCESS): + logger.info(f"Wait {SLEEP_TIMEOUT} seconds and try again") + time.sleep(SLEEP_TIMEOUT) + + if dast['state'] not in (DastState.STARTED, DastState.STOPPING, DastState.ANALYZING, DastState.SUCCESS): + logger.error(f"Error with scan id {dast['id']}. Current scan status: {dast['state']}," + f" but expected to be {DastState.STARTED}, {DastState.ANALYZING}, {DastState.STOPPING} " + f"or {DastState.SUCCESS}") + sys.exit(ExitCode.SCAN_FAILED) + logger.info(f"Scan {dast['id']} is started now. Let's wait until the scan is finished") + + get_dast_info_resp = mdast.get_scan_info(dast['id']) + if get_dast_info_resp.status_code != 200: + _mono_http_error(get_dast_info_resp, f"Error while getting scan info with id {dast['id']}", + ExitCode.NETWORK_ERROR) + count = 0 + + while dast_status in (DastState.STARTED, DastState.STOPPING, DastState.ANALYZING) and count < try_count: + if count == 0 and scan_type == 'manual': + if dast_status not in (DastState.ANALYZING, DastState.SUCCESS): + logger.info(f"This is manual scan with dynamic modules," + f" lets wait for {END_SCAN_TIMEOUT} seconds and stop it.") + time.sleep(END_SCAN_TIMEOUT) + stop_manual_dast_resp = mdast.stop_scan(dast['id']) + if stop_manual_dast_resp.status_code == 200: + logger.info(f'Scan {dast["id"]} was successfully stopped') + else: + _mono_http_error(stop_manual_dast_resp, f'Error while stopping scan with id {dast["id"]}') + else: + logger.info("This is manual scan with profile without dynamic modules," + " only SAST, lets wait till the end") + + logger.info(f"Try to get scan status for scan id {dast['id']}. Count number {count}") + get_dast_info_resp = mdast.get_scan_info(dast['id']) + if get_dast_info_resp.status_code != 200: + _mono_http_error(get_dast_info_resp, f"Error while getting scan info with id {dast['id']}", + ExitCode.NETWORK_ERROR) + dast = get_dast_info_resp.json() + dast_status = dast['state'] + logger.info(f"Current scan status: {DastStateDict.get(dast_status)}") + count += 1 + if dast_status is not DastState.SUCCESS: + logger.info(f"Wait {SLEEP_TIMEOUT} seconds and try again") + time.sleep(SLEEP_TIMEOUT) + + logger.info(f"Check if scan with id {dast['id']} was finished correctly.") + get_dast_info_resp = mdast.get_scan_info(dast['id']) + if get_dast_info_resp.status_code != 200: + _mono_http_error(get_dast_info_resp, f"Error while getting scan info with id {dast['id']}", + ExitCode.NETWORK_ERROR) + dast = get_dast_info_resp.json() + + if dast['state'] != DastState.SUCCESS: + logger.error( + f"Expected state {DastStateDict.get(DastState.SUCCESS)}, but in real it was {dast['state']}. " + f"Exit with error status code.") + sys.exit(ExitCode.SCAN_FAILED) + + # Report selection is shared with the microservices flow: --report_format + # (default pdf) plus explicit file-name flags, with scan-id default names. + # Monolith keeps the report download as a hard-fail (unlike the soft-fail + # microservices path, where the report service is a separate component). + report_targets = resolve_report_targets(arguments, dast['id']) + + if 'pdf' in report_targets: + pdf_path = report_targets['pdf'] + logger.info(f"Create and download pdf report for scan with id {dast['id']} to file {pdf_path}.") + pdf_report = mdast.download_report(dast['id']) + if pdf_report.status_code != 200: + _mono_http_error(pdf_report, 'PDF report creating failed') + with open(pdf_path, 'wb') as f: + f.write(pdf_report.content) + logger.info(f"Report for scan {dast['id']} successfully created and available at path: {pdf_path}.") + + if 'json' in report_targets: + json_path = report_targets['json'] + logger.info(f"Download JSON summary report for scan with id {dast['id']} to file {json_path}.") + json_summary_report = mdast.download_scan_json_result(dast['id']) + if json_summary_report.status_code != 200: + _mono_http_error(json_summary_report, 'JSON summary report download') + # Parse BEFORE opening the file, so a 200 with a non-JSON body neither + # crashes with a traceback nor leaves a 0-byte invalid .json behind. + try: + payload = json_summary_report.json() + except ValueError: + logger.error('JSON summary report: server returned HTTP 200 with a non-JSON body. Exit...') + sys.exit(ExitCode.SCAN_FAILED) + with open(json_path, 'w') as fp: + json.dump(payload, fp, indent=4, ensure_ascii=False) + logger.info(f"JSON report for scan {dast['id']} successfully created and available at path: {json_path}.") + + if arguments.cr_report: + generate_cr(f"{url}", arguments.stingray_login, arguments.stingray_password, dast['id'], + arguments.organization_name, arguments.engineer_name, arguments.controller_name, + arguments.cr_report_path, arguments.use_ldap, arguments.authority_server_id) + + +def main(): + urllib3.disable_warnings() + + arguments = parse_args() + + if getattr(arguments, 'verbose', False): + logging.getLogger().setLevel(logging.DEBUG) + logger.debug('Verbose (debug) logging enabled') + + distribution_system = arguments.distribution_system + download_path = arguments.download_path + + if arguments.download_only is False: + url = arguments.url + company_id = arguments.company_id + token = arguments.token + + # Canonical base URL: strip trailing slashes, then ensure it ends with + # '/rest' (both installations serve the CLI routes under /rest). All + # clients build request paths as f'{url}/...', so a trailing slash here + # would produce '//' and can 404 on a strict Envoy/OPA path match. The + # previous two-step form left a trailing slash when --url already + # contained 'rest', breaking the monolith and the mode probe alike. + url = url.rstrip('/') + if not url.endswith('/rest'): + url = f'{url}/rest' + + app_file = '' + appstore_app_md5 = None + + try: + if distribution_system == 'file': + app_file = arguments.file_path + if not app_file or not os.path.isfile(app_file): + # A bad --file_path is a user error, not a download failure; fail + # fast with INVALID_ARGS instead of a later md5/open traceback. + # (SystemExit is not caught by the `except Exception` below.) + logger.error(f'--file_path does not point to an existing file: {app_file!r}') + sys.exit(ExitCode.INVALID_ARGS) + + elif distribution_system == 'nexus': + nexus_repository = NexusRepository(arguments.nexus_url, + arguments.nexus_login, + arguments.nexus_password) + app_file = nexus_repository.download_app(download_path, + arguments.nexus_repo_name, + arguments.nexus_group_id, + arguments.nexus_artifact_id, + arguments.nexus_version) + elif distribution_system == 'nexus2': + nexus2_repository = Nexus2Repository(arguments.nexus2_url, + arguments.nexus2_login, + arguments.nexus2_password) + app_file = nexus2_repository.download_app(download_path, + arguments.nexus2_repo_name, + arguments.nexus2_group_id, + arguments.nexus2_artifact_id, + arguments.nexus2_version, + arguments.nexus2_extension, + arguments.nexus2_file_name) + + elif distribution_system == 'firebase': + app_file = firebase_download_app(download_path, + arguments.firebase_project_number, + arguments.firebase_app_id, + arguments.firebase_account_json_path, + arguments.firebase_file_name, + arguments.firebase_file_extension) + + elif distribution_system == 'appstore': + if arguments.appstore_password and arguments.appstore_2FA: + password2FA = arguments.appstore_password + arguments.appstore_2FA + else: + password2FA = arguments.appstore_password2FA + appstore = AppStore(arguments.appstore_apple_id, + password2FA) + app_file, appstore_app_md5 = appstore.download_app( + download_path, + arguments.appstore_app_id, + arguments.appstore_bundle_id, + arguments.appstore_country, + arguments.appstore_file_name, + ) + + elif distribution_system == 'google_play': + # Resolve AAS token from OAuth2 if provided and AAS not given + resolved_aas_token = arguments.google_play_aas_token + if (not resolved_aas_token) and arguments.google_play_oauth2_token: + try: + resolved_aas_token = asyncio.run( + gp_apkeep.fetch_aas_token( + email=arguments.google_play_email or '', + oauth2_token=arguments.google_play_oauth2_token, + timeout_sec=gp_apkeep.DEFAULT_TIMEOUT_SEC + ) + ) + except Exception as ex: + logger.error(f'Failed to obtain AAS token via OAuth2: {ex}') + sys.exit(ExitCode.AUTH_ERROR) + + google_play = GooglePlay(arguments.google_play_email, + resolved_aas_token) + google_play.login() + + app_file = google_play.download_app(download_path, + arguments.google_play_package_name, + arguments.google_play_file_name, proxy=arguments.google_play_proxy) + + elif distribution_system == 'rustore': + package_name = arguments.rustore_package_name + app_file = rustore_download_app(package_name, download_path) + + elif distribution_system == 'rumarket': + package_name = arguments.rumarket_package_name + app_file = rumarket_download_app(package_name, download_path) + + elif distribution_system == 'appgallery': + app_file = appgallery_download_app(arguments.appgallery_app_id, + download_path, + arguments.appgallery_file_name) + + except Exception as e: + logger.fatal( + 'Cannot download application file: %s (exception type: %s)', + e, + type(e).__name__, + ) + logger.exception('Download failed (traceback below)') + sys.exit(ExitCode.DOWNLOAD_FAILED) + + # Defensive guard: a downloader must have produced a real file before we try + # to md5/upload it. Catches a downloader that returned an empty/bogus path + # without raising, so the failure is a clear DOWNLOAD_FAILED, not a traceback. + if not app_file or not os.path.isfile(app_file): + logger.error(f'Downloaded application file was not found on disk: {app_file!r}') + sys.exit(ExitCode.DOWNLOAD_FAILED) + + if arguments.download_only is True: + logger.info('Your application was downloaded!') + # Emit a single-line machine-friendly output for CI parsers + print(f'DOWNLOAD_PATH={app_file}') + sys.exit(ExitCode.SUCCESS) + + try: + installation_mode = resolve_installation_mode(url, token, company_id, + verify=tls_verify_enabled()) + except ModeDetectionError as ex: + logger.error(str(ex)) + sys.exit(ExitCode.AUTH_ERROR if ex.auth_error else ExitCode.NETWORK_ERROR) + + if installation_mode == MODE_MICROSERVICES: + tls_verify = tls_verify_enabled() + if not tls_verify: + logger.warning('TLS verification is DISABLED (MDAST_TLS_VERIFY): the organization ' + 'CLI token is sent over an unverified connection - use only on trusted ' + 'networks / self-signed stands.') + from mdast_cli.ms_flow import run_microservices_flow + run_microservices_flow(arguments, url, token, app_file, + user_agent=USER_AGENT, verify=tls_verify) + logger.info('Job completed successfully!') + sys.exit(ExitCode.SUCCESS) + + try: + run_monolith_flow(arguments, url, token, company_id, app_file, appstore_app_md5) + except requests.RequestException as ex: + logger.error(f'Network error talking to the monolith installation ' + f'({type(ex).__name__}): {ex}') + sys.exit(ExitCode.NETWORK_ERROR) + + logger.info('Job completed successfully!') + sys.exit(ExitCode.SUCCESS) + + +if __name__ == '__main__': + main() diff --git a/mdast_cli/ms_flow.py b/mdast_cli/ms_flow.py index 0e80cfa..7ffa9e9 100644 --- a/mdast_cli/ms_flow.py +++ b/mdast_cli/ms_flow.py @@ -13,15 +13,18 @@ import json import logging import os +import re import sys import time +import requests + from mdast_cli.helpers.const import (ACTIVE_STAGES, ANDROID_EXTENSIONS, END_SCAN_TIMEOUT, LONG_TRY, OS_ANDROID, OS_IOS, PRE_START_STAGES, SLEEP_TIMEOUT, TERMINAL_SCAN_PAIRS, TRY, UPLOAD_TIMEOUT_ENV_VAR, UPLOAD_TIMEOUT_MAX, UPLOAD_TIMEOUT_MIN, ENGINE_ACTIVE_STATUS, ScanStage, ScanStageStatus) from mdast_cli.helpers.exit_codes import ExitCode -from mdast_cli.helpers.helpers import check_app_md5 +from mdast_cli.helpers.helpers import check_app_md5, resolve_report_targets from mdast_cli_core.microservices import extract_error_message, mDastMicroservices logger = logging.getLogger(__name__) @@ -34,13 +37,41 @@ 'architecture_unsupported': 'Platform/OS version pair is not supported by this installation', } +# Transient downstream failures (facade/scanyon/pepper) are retried while polling +# and downloading, so a rolling restart of a backend mid-scan does not kill a +# long CI job. ~5 min budget per call covers a typical k8s rolling restart. +# 429 (Too Many Requests) is included: scanyon rate-limits with a 429, and backing +# off is the correct response, not an immediate abort. +POLL_TRANSIENT_RETRIES = 30 +POLL_TRANSIENT_CODES = (429, 502, 503, 504) +# upload has its own (shorter) transient budget: a busy/redeploying/rate-limited +# upload path returns 429/502/503, worth a few retries, but not the full 5 min. +UPLOAD_TRANSIENT_RETRIES = 3 +UPLOAD_TRANSIENT_CODES = (429, 502, 503) + +# Strip C0/C1 control chars from server-controlled strings before logging, keeping +# only TAB (\x09). CRUCIALLY this includes LF (\x0a) and CR (\x0d): otherwise a +# hostile/compromised server could embed a newline in `message`/`package_name` and +# forge an extra log line in CI output (CWE-117 log injection). +_CONTROL_CHARS_RE = re.compile(r'[\x00-\x08\x0a-\x1f\x7f-\x9f]') + + +def sanitize(value): + """Strip control/ANSI chars (incl. CR/LF) from server strings before logging. + + A hostile/compromised server could embed ANSI escapes or CR/LF into fields + like `message`/`package_name` to forge log lines / poison CI logs; neutralise + them. Only TAB survives from the control range. + """ + return _CONTROL_CHARS_RE.sub('', str(value)) + def resolve_platform(app_file): """ANDROID/IOS from the application file extension.""" _, extension = os.path.splitext(app_file) - if extension in ANDROID_EXTENSIONS: + if extension.lower() in ANDROID_EXTENSIONS: return OS_ANDROID - if extension == '.ipa': + if extension.lower() == '.ipa': return OS_IOS return None @@ -49,11 +80,12 @@ def render_precheck_warning(warning): """One human-readable line per pre-check warning: known text + raw payload. Unknown warning types are rendered generically: the contract explicitly - allows backward-compatible extension of the type set (DEC-666-04). + allows backward-compatible extension of the type set (DEC-666-04). Payload is + JSON-escaped so server-controlled content cannot inject terminal sequences. """ - warning_type = str(warning.get('type', 'unknown')) + warning_type = sanitize(warning.get('type', 'unknown')) payload = warning.get('payload') or {} - text = KNOWN_PRECHECK_WARNINGS.get(warning_type, 'Scan pre-check warning') + text = KNOWN_PRECHECK_WARNINGS.get(warning.get('type'), 'Scan pre-check warning') payload_str = json.dumps(payload, ensure_ascii=False) if payload else '' return f'[{warning_type}] {text}{" " + payload_str if payload_str else ""}' @@ -89,7 +121,7 @@ def resolve_upload_timeout(): def _exit_on_http_error(resp, action, exit_code=ExitCode.SCAN_FAILED): - message = extract_error_message(resp) + message = sanitize(extract_error_message(resp)) if resp.status_code in (401, 403): logger.error(f'{action}: authorization failed (HTTP {resp.status_code}): {message}. ' 'Check the organization CLI token (issued in the platform UI).') @@ -98,59 +130,111 @@ def _exit_on_http_error(resp, action, exit_code=ExitCode.SCAN_FAILED): sys.exit(exit_code) -POLL_TRANSIENT_RETRIES = 5 -POLL_TRANSIENT_CODES = (502, 503, 504) +def _json_or_exit(resp, action, exit_code=ExitCode.NETWORK_ERROR): + """Parse a 2xx response body as JSON or exit cleanly. + + A 2xx with a non-JSON body (e.g. an nginx/Envoy HTML stub, or a 204 with no + body) must not crash with a raw traceback; it is a network/gateway problem, + not INTERNAL_ERROR. + """ + try: + return resp.json() + except ValueError: + logger.error(f'{action}: server returned HTTP {resp.status_code} with a non-JSON body ' + f'(unexpected gateway/proxy response).') + sys.exit(exit_code) -def _get_scan(mdast, scan_id): - """Poll scan state, tolerating transient downstream failures. +def _retry_request(call, action, retries, exit_code=ExitCode.NETWORK_ERROR): + """Call `call()` (returns a Response), retrying transient 5xx/network errors. - A long CI poll must not die on a single flaky 5xx / network blip while the - scan keeps running server-side. Transient errors are retried a few times; - a persistent failure or a 4xx still exits. + Non-transient HTTP errors and exhausted transient retries exit via + `_exit_on_http_error`. Returns the successful (2xx) Response. """ - import requests as _requests last_resp = None - for attempt in range(POLL_TRANSIENT_RETRIES): + for attempt in range(retries): try: - resp = mdast.get_scan_info(scan_id) - except _requests.RequestException as ex: - logger.warning(f'Scan info request failed ({type(ex).__name__}), ' - f'retry {attempt + 1}/{POLL_TRANSIENT_RETRIES}') - time.sleep(SLEEP_TIMEOUT) + resp = call() + except requests.RequestException as ex: + logger.warning(f'{action} request failed ({type(ex).__name__}), ' + f'retry {attempt + 1}/{retries}') + if attempt + 1 < retries: + time.sleep(SLEEP_TIMEOUT) continue - if resp.status_code == 200: - return resp.json() + if 200 <= resp.status_code < 300: + return resp last_resp = resp - if resp.status_code in POLL_TRANSIENT_CODES: - logger.warning(f'Scan info returned {resp.status_code} (transient), ' - f'retry {attempt + 1}/{POLL_TRANSIENT_RETRIES}') + if resp.status_code in POLL_TRANSIENT_CODES and attempt + 1 < retries: + logger.warning(f'{action} returned {resp.status_code} (transient), ' + f'retry {attempt + 1}/{retries}') time.sleep(SLEEP_TIMEOUT) continue break if last_resp is not None: - _exit_on_http_error(last_resp, f'Getting scan info for scan {scan_id}', - ExitCode.NETWORK_ERROR) - logger.error(f'Getting scan info for scan {scan_id} failed after retries (network)') - sys.exit(ExitCode.NETWORK_ERROR) + _exit_on_http_error(last_resp, action, exit_code) + logger.error(f'{action} failed after {retries} attempts (network)') + sys.exit(exit_code) + + +def _get_scan(mdast, scan_id): + """Poll scan state, tolerating transient downstream failures.""" + resp = _retry_request(lambda: mdast.get_scan_info(scan_id), + f'Getting scan info for scan {scan_id}', POLL_TRANSIENT_RETRIES) + return _json_or_exit(resp, f'Getting scan info for scan {scan_id}') def run_precheck_gate(mdast, md5, profile_id, testcase_id, scan_type): - """Gate model (DEC-671-05): any warning blocks the scan with a non-zero exit.""" - resp = mdast.precheck_scan(md5, profile_id, testcase_id, scan_type) + """Gate model (DEC-671-05): any warning blocks the scan with a non-zero exit. + + A transient gateway failure (429/502/503/504) or a raw network error is NOT a + policy block: it is retried and, if it persists, exits NETWORK_ERROR(6) (the CI + "retry" signal) rather than PRECHECK_BLOCKED(8). The scan is still not created + in that case, so the gate stays fail-closed either way. + """ + resp = None + for attempt in range(UPLOAD_TRANSIENT_RETRIES): + try: + resp = mdast.precheck_scan(md5, profile_id, testcase_id, scan_type) + except requests.RequestException as ex: + logger.warning(f'Scan pre-check request failed ({type(ex).__name__}), ' + f'retry {attempt + 1}/{UPLOAD_TRANSIENT_RETRIES}') + resp = None + if attempt + 1 < UPLOAD_TRANSIENT_RETRIES: + time.sleep(SLEEP_TIMEOUT) + continue + if resp.status_code in POLL_TRANSIENT_CODES and attempt + 1 < UPLOAD_TRANSIENT_RETRIES: + logger.warning(f'Scan pre-check returned {resp.status_code} (transient), ' + f'retry {attempt + 1}/{UPLOAD_TRANSIENT_RETRIES}') + time.sleep(SLEEP_TIMEOUT) + continue + break + if resp is None: + logger.error('Scan pre-check could not be completed (network). This is retryable ' + 'infrastructure, not a policy block.') + sys.exit(ExitCode.NETWORK_ERROR) + if resp.status_code in (401, 403): + _exit_on_http_error(resp, 'Scan pre-check') # -> AUTH_ERROR if resp.status_code == 404: - _exit_on_http_error(resp, 'Scan pre-check (profile lookup)') + _exit_on_http_error(resp, 'Scan pre-check (profile lookup)') # -> SCAN_FAILED if resp.status_code == 422 and profile_id is None: - # Installation does not yet accept pre-check without profile_id - # (scanyon change pending); skipping is a documented interim behavior. - logger.warning('Scan pre-check skipped: this installation requires profile_id ' - 'for pre-check and no --profile_id was given') + # Auto-created-profile scan (no --profile_id): the profile does not exist + # yet, so pre-check cannot run against it. Skipping is intentional interim + # behaviour; platform-side validation still runs at scan creation. + logger.warning('Scan pre-check skipped: no --profile_id given (profile is ' + 'auto-created at scan start); platform validates at creation.') return + if resp.status_code in POLL_TRANSIENT_CODES: + # transient gateway/rate-limit that outlived the retry budget: retryable infra, + # not a deliberate policy block -> NETWORK_ERROR(6), consistent with the rest. + message = sanitize(extract_error_message(resp)) + logger.error(f'Scan pre-check unavailable (HTTP {resp.status_code}): {message}. ' + 'Retryable infrastructure, not a policy block.') + sys.exit(ExitCode.NETWORK_ERROR) if resp.status_code != 200: - message = extract_error_message(resp) + message = sanitize(extract_error_message(resp)) logger.error(f'Scan pre-check is unavailable (HTTP {resp.status_code}): {message}') sys.exit(ExitCode.PRECHECK_BLOCKED) - warnings = (resp.json() or {}).get('warnings') or [] + warnings = (_json_or_exit(resp, 'Scan pre-check') or {}).get('warnings') or [] if not warnings: logger.info('Scan pre-check passed, no warnings') return @@ -160,9 +244,28 @@ def run_precheck_gate(mdast, md5, profile_id, testcase_id, scan_type): sys.exit(ExitCode.PRECHECK_BLOCKED) -def run_microservices_flow(arguments, url, token, app_file, appstore_app_md5, user_agent=None, - verify=False): - """Full scan flow against the microservices installation. Exits the process.""" +def run_microservices_flow(arguments, url, token, app_file, user_agent=None, verify=True): + """Full scan flow against the microservices installation. Exits the process. + + Wrapped so a bare network error surfaces as NETWORK_ERROR(6), consistent with + the rest of the flow, instead of a traceback + INTERNAL_ERROR(1). + + Note there is no appstore_app_md5 parameter (unlike the monolith path): the + App Store downloader rewrites the .ipa after download (adds iTunesMetadata / + sinf), so the Apple-store md5 differs from the file actually uploaded. The + microservices flow always keys dedup/precheck/create off the local file md5, + which is also what upload sends - so an App Store (iOS) build uploads and + scans on the microservices installation exactly like any other .ipa (F1). + """ + try: + _run_microservices_flow(arguments, url, token, app_file, user_agent, verify) + except requests.RequestException as ex: + logger.error(f'Network error talking to the microservices installation ' + f'({type(ex).__name__}): {ex}') + sys.exit(ExitCode.NETWORK_ERROR) + + +def _run_microservices_flow(arguments, url, token, app_file, user_agent, verify): profile_id = arguments.profile_id project_id = arguments.project_id testcase_id = arguments.testcase_id @@ -185,7 +288,8 @@ def run_microservices_flow(arguments, url, token, app_file, appstore_app_md5, us architectures_resp = mdast.get_architectures() if architectures_resp.status_code != 200: _exit_on_http_error(architectures_resp, 'Getting architectures', ExitCode.NETWORK_ERROR) - logger.info(f'Supported architectures: {architectures_resp.json()}') + architectures = _json_or_exit(architectures_resp, 'Getting architectures') + logger.info(f'Supported architectures: {architectures}') platform = resolve_platform(app_file) if platform is None: @@ -195,11 +299,13 @@ def run_microservices_flow(arguments, url, token, app_file, appstore_app_md5, us if testcase_id is not None: testcase_resp = mdast.get_testcase(testcase_id) if testcase_resp.status_code == 200: - testcase_os = str(testcase_resp.json().get('os', '')).upper() + testcase_os = str((_json_or_exit(testcase_resp, 'Getting test case') or {}).get('os', '')).upper() if testcase_os and testcase_os != platform: logger.error(f'Test case {testcase_id} is recorded for {testcase_os}, ' f'but the application file is for {platform}') sys.exit(ExitCode.INVALID_ARGS) + elif testcase_resp.status_code in (401, 403): + _exit_on_http_error(testcase_resp, f'Getting test case {testcase_id}') else: logger.warning(f'Cannot get test case {testcase_id} ' f'(HTTP {testcase_resp.status_code}), continuing') @@ -207,44 +313,48 @@ def run_microservices_flow(arguments, url, token, app_file, appstore_app_md5, us engines_resp = mdast.get_engines() if engines_resp.status_code != 200: _exit_on_http_error(engines_resp, 'Getting engines', ExitCode.NETWORK_ERROR) - engines = engines_resp.json() + engines = _json_or_exit(engines_resp, 'Getting engines') + if not isinstance(engines, list): + logger.error('Getting engines: unexpected response shape (expected a list)') + sys.exit(ExitCode.NETWORK_ERROR) active_engines = [engine for engine in engines - if str(engine.get('type', '')).upper() == platform + if isinstance(engine, dict) + and str(engine.get('type', '')).upper() == platform and str(engine.get('status', '')).upper() == ENGINE_ACTIVE_STATUS] if not active_engines: logger.error(f'Cannot create scan - no active engine for platform {platform}') sys.exit(ExitCode.SCAN_FAILED) + # F1: single md5 for the whole flow — the md5 of the file actually uploaded. + # For appstore the CLI rewrites the ipa (adds iTunesMetadata/sinf) after + # download, so the Apple store md5 differs from what is uploaded; dedup / + # precheck / create must all use the local file's md5, which upload also sends. logger.info('Check if this version of application was already uploaded..') - app_md5 = (appstore_app_md5 or check_app_md5(app_file)).lower() + app_md5 = check_app_md5(app_file).lower() dedup_resp = mdast.check_app_md5(None, app_md5) if dedup_resp.status_code != 200: _exit_on_http_error(dedup_resp, 'Application dedup check', ExitCode.NETWORK_ERROR) - found_apps = dedup_resp.json() - if found_apps: - application = found_apps[0] - logger.info(f"This app was uploaded before, application id is: {application['id']}, " - f"package name: {application['package_name']}, " - f"version: {application['version_name']}, md5: {application['md5']}") + found_apps = _json_or_exit(dedup_resp, 'Application dedup check') + application = found_apps[0] if isinstance(found_apps, list) and found_apps else None + if application: + logger.info('This app was uploaded before, application id is: ' + f"{application.get('id')}, package name: {sanitize(application.get('package_name'))}, " + f"version: {sanitize(application.get('version_name'))}, md5: {sanitize(application.get('md5'))}") else: logger.info('This is new application or new version') logger.info('Uploading application to server..') - upload_resp = mdast.upload_application(app_file, upload_timeout=resolve_upload_timeout()) - if upload_resp.status_code != 201: - if upload_resp.status_code == 504: - logger.error('Application parsing did not finish in time (504). The upload is ' - 'processed asynchronously - retry the same command later, the file ' - f'will not be re-uploaded. {UPLOAD_TIMEOUT_ENV_VAR} env var can ' - 'raise the wait (max 300 seconds).') - sys.exit(ExitCode.SCAN_FAILED) - _exit_on_http_error(upload_resp, 'Uploading application') - application = upload_resp.json() - logger.info(f"Application uploaded successfully. Application id: {application['id']}") + upload_timeout = resolve_upload_timeout() + application = _upload_application(mdast, app_file, upload_timeout) + + app_id = application.get('id') + if not app_id: + logger.error(f'Application response has no id: {sanitize(application)}') + sys.exit(ExitCode.SCAN_FAILED) precheck_type = 'AUTO' if testcase_id is not None else 'MANUAL' run_precheck_gate(mdast, app_md5, profile_id, testcase_id, precheck_type) - logger.info(f"Creating scan for application {application['id']}") + logger.info(f'Creating scan for application {sanitize(app_id)}') if testcase_id is not None: create_resp = mdast.create_auto_scan(project_id, profile_id, app_md5, None, testcase_id) scan_type = 'auto_stingray' @@ -253,12 +363,12 @@ def run_microservices_flow(arguments, url, token, app_file, appstore_app_md5, us scan_type = 'manual' if create_resp.status_code not in (200, 201): _exit_on_http_error(create_resp, 'Creating scan') - scan = create_resp.json() - if not scan.get('id'): + scan = _json_or_exit(create_resp, 'Creating scan') + scan_id = scan.get('id') + if not scan_id: logger.error(f'Something went wrong while creating scan: {scan}') sys.exit(ExitCode.SCAN_FAILED) - scan_id = scan['id'] - logger.info(f"Project and profile was created/found successfully. " + logger.info('Project and profile were created/found successfully. ' f"Project id: {(scan.get('project') or {}).get('id')}, " f"profile id: {(scan.get('profile') or {}).get('id')}") logger.info(f'Scan was created successfully. Scan id: {scan_id}') @@ -300,14 +410,18 @@ def run_microservices_flow(arguments, url, token, app_file, appstore_app_md5, us if scan.get('stage') not in ACTIVE_STAGES | {ScanStage.SUCCESS, ScanStage.FAIL}: logger.error(f'Error with scan id {scan_id}. Scan did not start, ' - f'current state: {scan_pair(scan)}, message: {scan.get("message")}') + f'current state: {scan_pair(scan)}, message: {sanitize(scan.get("message"))}') sys.exit(ExitCode.SCAN_FAILED) if scan_type == 'manual' and not is_terminal(scan) and scan.get('stage') != ScanStage.STOP: logger.info(f'This is a scan without a test case, ' f'lets wait for {END_SCAN_TIMEOUT} seconds and stop it.') time.sleep(END_SCAN_TIMEOUT) - stop_resp = mdast.stop_scan(scan_id) + try: + stop_resp = mdast.stop_scan(scan_id) + except requests.RequestException as ex: + logger.error(f'Stopping scan {scan_id} request failed ({type(ex).__name__})') + sys.exit(ExitCode.NETWORK_ERROR) if stop_resp.status_code == 200: logger.info(f'Scan {scan_id} was requested to stop (stopped by CLI after ' f'{END_SCAN_TIMEOUT} seconds, as in the manual scan flow)') @@ -327,76 +441,146 @@ def run_microservices_flow(arguments, url, token, app_file, appstore_app_md5, us logger.info(f'Wait {SLEEP_TIMEOUT} seconds and try again') time.sleep(SLEEP_TIMEOUT) + if not is_terminal(scan): + logger.error(f'Scan {scan_id} did not reach a terminal state within the wait budget ' + f'(last state {scan_pair(scan)}). Increase timeout with --long_wait.') + sys.exit(ExitCode.SCAN_FAILED) if not is_success(scan): logger.error(f'Scan {scan_id} finished with state {scan_pair(scan)}, ' - f'message: {scan.get("message")}. Exit with error status code.') + f'message: {sanitize(scan.get("message"))}. Exit with error status code.') sys.exit(ExitCode.SCAN_FAILED) if scan.get('status') == ScanStageStatus.PARTIAL_COMPLETE: - logger.warning(f'Scan {scan_id} finished partially complete: {scan.get("message")}') + logger.warning(f'Scan {scan_id} finished partially complete ' + f'(some modules did not run): {sanitize(scan.get("message"))}') download_reports(mdast, scan_id, arguments) -def _download_report_with_retry(fetch, action): - """Download a report tolerating transient downstream (Pepper) 5xx/network. +def _upload_application(mdast, app_file, upload_timeout): + """Upload with transient retry; friendly message on server-side parse timeout. - The report render can momentarily be unavailable (502/503/504) right after a - scan finishes; a single blip must not lose the finished scan's report. + Retries both transient HTTP (502/503) and raw network errors: a rolling + restart of the upload path can drop the connection outright, not only answer + 502. The server keys uploads by md5, so re-sending the same build is + idempotent (a lost-response upload is de-duplicated server-side). """ - import requests as _requests last_resp = None - for attempt in range(POLL_TRANSIENT_RETRIES): + for attempt in range(UPLOAD_TRANSIENT_RETRIES): try: - resp = fetch() - except _requests.RequestException as ex: - logger.warning(f'{action} request failed ({type(ex).__name__}), ' - f'retry {attempt + 1}/{POLL_TRANSIENT_RETRIES}') - time.sleep(SLEEP_TIMEOUT) + resp = mdast.upload_application(app_file, upload_timeout=upload_timeout) + except requests.RequestException as ex: + logger.warning(f'Uploading application request failed ({type(ex).__name__}), ' + f'retry {attempt + 1}/{UPLOAD_TRANSIENT_RETRIES}') + if attempt + 1 < UPLOAD_TRANSIENT_RETRIES: + time.sleep(SLEEP_TIMEOUT) continue - if resp.status_code == 200: - return resp + if 200 <= resp.status_code < 300: + # Accept any 2xx (not just 201): a facade/gateway may answer 200 for an + # already-registered build, and the rest of the flow (poll/create) already + # treats 2xx as success. + application = _json_or_exit(resp, 'Uploading application') + logger.info(f"Application uploaded successfully. Application id: {application.get('id')}") + return application + if resp.status_code == 504: + # Gateway timeout: the synchronous wait elapsed but the platform keeps + # parsing. Re-running finds the build via md5 dedup. This is retryable + # infra, not a bad scan -> NETWORK_ERROR (the CI "retry" signal). + logger.error('Application parsing did not finish in time (504). The upload is ' + 'processed asynchronously - retry the same command later, the file ' + f'will not be re-uploaded. {UPLOAD_TIMEOUT_ENV_VAR} env var can raise ' + f'the wait (max {UPLOAD_TIMEOUT_MAX} seconds).') + sys.exit(ExitCode.NETWORK_ERROR) last_resp = resp - if resp.status_code in POLL_TRANSIENT_CODES: - logger.warning(f'{action} returned {resp.status_code} (transient), ' - f'retry {attempt + 1}/{POLL_TRANSIENT_RETRIES}') + if resp.status_code in UPLOAD_TRANSIENT_CODES and attempt + 1 < UPLOAD_TRANSIENT_RETRIES: + logger.warning(f'Uploading application returned {resp.status_code} (transient), ' + f'retry {attempt + 1}/{UPLOAD_TRANSIENT_RETRIES}') time.sleep(SLEEP_TIMEOUT) continue break if last_resp is not None: - _exit_on_http_error(last_resp, action) - logger.error(f'{action} failed after retries (network)') + if last_resp.status_code in UPLOAD_TRANSIENT_CODES: + # gateway 5xx / rate-limit that outlived the retry budget: retryable infra + logger.error(f'Uploading application failed: transient status {last_resp.status_code} ' + f'persisted after {UPLOAD_TRANSIENT_RETRIES} retries.') + sys.exit(ExitCode.NETWORK_ERROR) + _exit_on_http_error(last_resp, 'Uploading application') + logger.error(f'Uploading application failed after {UPLOAD_TRANSIENT_RETRIES} attempts ' + f'(network). For large builds raise the wait with {UPLOAD_TIMEOUT_ENV_VAR} ' + f'(seconds, max {UPLOAD_TIMEOUT_MAX}).') sys.exit(ExitCode.NETWORK_ERROR) def download_reports(mdast, scan_id, arguments): - """Report step (STG-4478): files are written to the user-provided paths. - - Content-Disposition from the server is deliberately ignored: the local - file name is a CLI argument and must not be controlled by the server. + """Report step (STG-4478): files are written to CLI-chosen paths. + + Format selection is shared with the monolith flow (--report_format, default + pdf, scan-id default names). Reports are downloaded independently and are + soft-fail: the scan already finished SUCCESS, so a report render failure (e.g. + Pepper 502, a separate microservice) must not turn a green scan red or block + the other report format. Content-Disposition from the server is deliberately + ignored - the local file name is a CLI argument. """ - pdf_report_file_name = arguments.pdf_report_file_name - json_summary_file_name = arguments.summary_report_json_file_name - - if pdf_report_file_name: - logger.info(f'Create and download pdf report for scan with id {scan_id} ' - f'to file {pdf_report_file_name}.') - pdf_report = _download_report_with_retry(lambda: mdast.download_report(scan_id), - 'PDF report downloading') - pdf_report_file_name = pdf_report_file_name if pdf_report_file_name.endswith( - '.pdf') else f'{pdf_report_file_name}.pdf' - with open(pdf_report_file_name, 'wb') as f: - f.write(pdf_report.content) - logger.info(f'Report for scan {scan_id} successfully created and available at path: ' - f'{pdf_report_file_name}.') - - if json_summary_file_name: - logger.info(f'Download JSON summary report for scan with id {scan_id} ' - f'to file {json_summary_file_name}.') - json_report = _download_report_with_retry( - lambda: mdast.download_scan_json_result(scan_id), 'JSON summary report downloading') - json_file_name = json_summary_file_name if json_summary_file_name.endswith( - '.json') else f'{json_summary_file_name}.json' - with open(json_file_name, 'w') as fp: - json.dump(json_report.json(), fp, indent=4, ensure_ascii=False) - logger.info(f'JSON report for scan {scan_id} successfully created and available ' - f'at path: {json_file_name}.') + targets = resolve_report_targets(arguments, scan_id) + failures = [] + + if 'pdf' in targets: + target = targets['pdf'] + logger.info(f'Create and download pdf report for scan {scan_id} to file {target}.') + resp = _download_report(mdast.download_report, scan_id, 'PDF report') + if resp is None: + failures.append('PDF') + else: + with open(target, 'wb') as f: + f.write(resp.content) + logger.info(f'PDF report for scan {scan_id} saved to {target}.') + + if 'json' in targets: + target = targets['json'] + logger.info(f'Download JSON summary report for scan {scan_id} to file {target}.') + resp = _download_report(mdast.download_scan_json_result, scan_id, 'JSON report') + if resp is None: + failures.append('JSON') + else: + try: + payload = resp.json() + except ValueError: + logger.error('JSON report: server returned a non-JSON body; saving raw bytes.') + with open(target, 'wb') as f: + f.write(resp.content) + else: + with open(target, 'w') as fp: + json.dump(payload, fp, indent=4, ensure_ascii=False) + logger.info(f'JSON report for scan {scan_id} saved to {target}.') + + if failures: + # Soft-fail: scan succeeded; report render is a downstream (Pepper) issue. + logger.warning(f'Scan {scan_id} finished successfully, but these reports could not be ' + f'downloaded (report service issue, not scan failure): {", ".join(failures)}. ' + 'Retry the report download later.') + + +def _download_report(fetch, scan_id, label): + """Download one report with transient retry. Returns Response or None (soft-fail).""" + last = None + for attempt in range(POLL_TRANSIENT_RETRIES): + try: + resp = fetch(scan_id) + except requests.RequestException as ex: + logger.warning(f'{label} request failed ({type(ex).__name__}), ' + f'retry {attempt + 1}/{POLL_TRANSIENT_RETRIES}') + if attempt + 1 < POLL_TRANSIENT_RETRIES: + time.sleep(SLEEP_TIMEOUT) + continue + if resp.status_code == 200: + return resp + last = resp + if resp.status_code in POLL_TRANSIENT_CODES and attempt + 1 < POLL_TRANSIENT_RETRIES: + logger.warning(f'{label} returned {resp.status_code} (transient), ' + f'retry {attempt + 1}/{POLL_TRANSIENT_RETRIES}') + time.sleep(SLEEP_TIMEOUT) + continue + break + if last is not None: + logger.error(f'{label} download failed (HTTP {last.status_code}): ' + f'{sanitize(extract_error_message(last))}') + return None diff --git a/mdast_cli_core/factory.py b/mdast_cli_core/factory.py index f6b2562..4276a5c 100644 --- a/mdast_cli_core/factory.py +++ b/mdast_cli_core/factory.py @@ -1,13 +1,20 @@ -"""Installation mode resolution (monolith vs microservices) and client factory. - -No new CLI flags are introduced (OOS-671-03): the mode comes from the -MDAST_CLI_MODE environment variable (`auto` | `monolith` | `microservices`, -default `auto`) or is detected by probing. - -Auto-detection probe: `GET {base}/engines/` exists only on the Clark facade -(the monolith serves engines under `/organizations/{id}/engines/`), so a 200 -identifies the microservices installation; otherwise the monolith path is -probed with the legacy `Token` auth scheme. +"""Installation mode resolution: monolith vs microservices (Clark facade). + +The mode decides which backend the scan flow talks to, so it must be reliable in +BOTH directions (monolith -> monolith, k8s -> k8s). It is resolved as: + +1. Explicit override via the ``MDAST_CLI_MODE`` env var + (``monolith`` | ``microservices`` | ``auto``; default ``auto``). No new CLI + flags are introduced (OOS-671-03). +2. Auto-detection by *content fingerprint*, not just an HTTP 200. Both + installations expose ``GET {base}/architectures/`` under ``/rest``, but the + payloads differ structurally: + - microservices (scanyon-native): ``type`` is a string ``ANDROID``/``IOS`` + and each item carries ``os_version``; + - monolith: ``type`` is an integer code (1/2) and there is no ``os_version``. + Auth scheme also differs (microservices = ``Bearer``, monolith = ``Token``), + so each probe uses its own scheme. A monolith that happens to answer 200 to a + Bearer probe is NOT misclassified, because the payload is inspected. """ import logging import os @@ -25,7 +32,7 @@ class ModeDetectionError(Exception): - """Raised when neither installation flavour answered the probes.""" + """Raised when the installation flavour cannot be determined.""" def __init__(self, message, auth_error=False): super().__init__(message) @@ -33,19 +40,72 @@ def __init__(self, message, auth_error=False): def tls_verify_enabled(): - return os.environ.get(TLS_VERIFY_ENV_VAR, '').strip().lower() in ('1', 'true', 'yes', 'on') + """TLS verification is on unless explicitly disabled via env. + + Secure by default (the CLI carries an org-level bearer token): the operator + must opt OUT with MDAST_TLS_VERIFY in {0,false,no,off} for self-signed stands. + An unset OR empty value counts as "not configured" and stays secure, so a + docker `-e MDAST_TLS_VERIFY` passthrough with no value can't silently + disable verification. + """ + raw = os.environ.get(TLS_VERIFY_ENV_VAR) + if raw is None: + return True + raw = raw.strip().lower() + if raw == '': + return True + return raw not in ('0', 'false', 'no', 'off') + + +def _looks_microservices(payload): + """True if an /architectures/ payload is scanyon-native (microservices).""" + if not isinstance(payload, list) or not payload: + return False + item = payload[0] + if not isinstance(item, dict): + return False + # scanyon-native: type is a string ANDROID/IOS and os_version is present + type_value = item.get('type') + return isinstance(type_value, str) and type_value.upper() in ('ANDROID', 'IOS') + + +def _looks_monolith(payload): + """True if an /architectures/ payload is monolith-shaped (int type code).""" + if not isinstance(payload, list) or not payload: + return False + item = payload[0] + return isinstance(item, dict) and isinstance(item.get('type'), int) + + +def _probe(base_url, path, scheme, ci_token, verify): + """GET a probe endpoint; return (status_code | None, parsed_json | None).""" + try: + resp = requests.get(f'{base_url}{path}', + headers={'Authorization': f'{scheme} {ci_token}'}, + verify=verify, + timeout=PROBE_TIMEOUT) + except requests.RequestException as ex: + logger.debug(f'Probe {scheme} {path} failed: {type(ex).__name__}: {ex}') + return None, None + payload = None + if resp.status_code == 200: + try: + payload = resp.json() + except ValueError: + payload = None + return resp.status_code, payload def resolve_installation_mode(base_url, ci_token, company_id, mode=None, verify=None): """Return MODE_MONOLITH or MODE_MICROSERVICES. - `base_url` is the normalized base ending with `/rest` (both installations + ``base_url`` is the normalized base ending with ``/rest`` (both installations serve the CLI routes under this prefix). """ if verify is None: verify = tls_verify_enabled() mode = (mode or os.environ.get(MODE_ENV_VAR) or MODE_AUTO).strip().lower() - if mode in (MODE_MONOLITH, MODE_MICROSERVICES): + if mode == MODE_MONOLITH or mode == MODE_MICROSERVICES: logger.info(f'Installation mode forced via {MODE_ENV_VAR}: {mode}') return mode if mode != MODE_AUTO: @@ -53,36 +113,32 @@ def resolve_installation_mode(base_url, ci_token, company_id, mode=None, verify= f'Unknown {MODE_ENV_VAR} value: {mode!r} ' f'(expected {MODE_AUTO}/{MODE_MONOLITH}/{MODE_MICROSERVICES})') - ms_status = None - try: - resp = requests.get(f'{base_url}/engines/', - headers={'Authorization': f'Bearer {ci_token}'}, - verify=verify, - timeout=PROBE_TIMEOUT) - ms_status = resp.status_code - if resp.status_code == 200: - logger.info('Detected microservices installation (Clark facade)') - return MODE_MICROSERVICES - except requests.RequestException as ex: - logger.debug(f'Microservices probe failed: {ex}') - - monolith_status = None - try: - resp = requests.get(f'{base_url}/organizations/{company_id}/engines/', - headers={'Authorization': f'Token {ci_token}'}, - verify=verify, - timeout=PROBE_TIMEOUT) - monolith_status = resp.status_code - if resp.status_code == 200: - logger.info('Detected monolith installation') - return MODE_MONOLITH - except requests.RequestException as ex: - logger.debug(f'Monolith probe failed: {ex}') + # Microservices probe: Bearer + architectures, classify by payload shape. + ms_status, ms_payload = _probe(base_url, '/architectures/', 'Bearer', ci_token, verify) + if ms_status == 200 and _looks_microservices(ms_payload): + logger.info('Detected microservices installation (Clark facade)') + return MODE_MICROSERVICES + + # Monolith probe: Token + architectures, classify by payload shape. + mono_status, mono_payload = _probe(base_url, '/architectures/', 'Token', ci_token, verify) + if mono_status == 200 and _looks_monolith(mono_payload): + logger.info('Detected monolith installation') + return MODE_MONOLITH + + # Ambiguous 200 (payload matched neither shape) — do not guess. + if ms_status == 200 or mono_status == 200: + empty_list = ms_payload == [] or mono_payload == [] + hint = ('The /architectures/ list is empty, so the installation flavour cannot be ' + 'inferred from it. ' if empty_list else + 'The payload matched neither the microservices (string type + os_version) ' + 'nor the monolith (int type) shape. ') + raise ModeDetectionError( + f'Cannot detect installation mode: /architectures/ returned 200 but {hint}' + f'Force the mode via {MODE_ENV_VAR}=monolith|microservices.') - auth_error = 401 in (ms_status, monolith_status) or 403 in (ms_status, monolith_status) + auth_error = 401 in (ms_status, mono_status) or 403 in (ms_status, mono_status) raise ModeDetectionError( - 'Cannot detect installation mode: probes failed ' - f'(microservices GET /engines/ -> {ms_status}, ' - f'monolith GET /organizations/{{id}}/engines/ -> {monolith_status}). ' - f'Check --url and --token, or force the mode via {MODE_ENV_VAR}.', + 'Cannot detect installation mode via GET /architectures/ ' + f'(Bearer -> {ms_status}, Token -> {mono_status}). ' + f'Check --url/--token, or force the mode via {MODE_ENV_VAR}.', auth_error=auth_error) diff --git a/mdast_cli_core/microservices.py b/mdast_cli_core/microservices.py index 23803c0..ec3299e 100644 --- a/mdast_cli_core/microservices.py +++ b/mdast_cli_core/microservices.py @@ -64,12 +64,17 @@ def extract_error_message(resp): class mDastMicroservices(mDastBase): """API client for the Clark facade of the microservices installation.""" - def __init__(self, base_url, ci_token, company_id=None, user_agent=None, verify=False): + def __init__(self, base_url, ci_token, company_id=None, user_agent=None, verify=True): super().__init__(base_url) + # F14: strip trailing slash so f'{self.url}/architectures/' can't produce + # a double slash (a strict OPA/Envoy path match may 403/404 on '//'). + self.url = base_url.rstrip('/') # company_id is accepted for interface parity with mDastToken and ignored: # the organization is resolved server-side from the token (X-Organization-Id). self.company_id = company_id self.current_context = {'company': company_id} + # F4: TLS verification on by default (org bearer token is a credential); + # opt out only via MDAST_TLS_VERIFY for self-signed stands. self.verify = verify self.timeout = HTTP_REQUEST_TIMEOUT self.headers = {'Authorization': 'Bearer {0}'.format(ci_token), diff --git a/pytest.ini b/pytest.ini index 2c44689..f731fd7 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,6 +4,7 @@ markers = unit: fast isolated tests of a single module contract: tests pinning the HTTP contract with the Clark facade / monolith smoke: end-to-end CLI flows against a mocked server + e2e: negative end-to-end CLI flows asserting exit codes security: security-focused checks (secrets handling, header trust, TLS) filterwarnings = ignore::DeprecationWarning diff --git a/tests/test_download_scan_positive.py b/tests/test_download_scan_positive.py new file mode 100644 index 0000000..6cada16 --- /dev/null +++ b/tests/test_download_scan_positive.py @@ -0,0 +1,224 @@ +"""Positive e2e: download a build from a store, then scan it in Sting (microservices). + +Each test mocks the store's HTTP endpoints to deliver a valid APK, then lets the +CLI run the full microservices flow (upload -> precheck -> create -> start -> poll +-> report) against the mocked Sting facade. This is the happy path of the +"download from store -> scan" integration, one store per test. + +Store-side FAILURE modes are intentionally NOT here (per scope: negatives cover +work with Sting, positives cover download-from-store + scan). See test_negative_e2e.py. +""" +import hashlib +import io +import json +import os +import zipfile + +import pytest +import responses +from responses import matchers + +from tests.conftest import (BASE_URL, REST_URL, TOKEN, application_json, run_main, + scan_json) +from tests.test_smoke_flows import register_ms_happy_path + +pytestmark = pytest.mark.e2e + +APK_BYTES = b'PK\x03\x04' + b'fake-apk-payload' * 64 + + +def _valid_zip_apk(): + """A real (minimal) zip — APK is a zip; some downloaders (rustore) validate the container.""" + buf = io.BytesIO() + with zipfile.ZipFile(buf, 'w') as z: + z.writestr('AndroidManifest.xml', b'\x00') + return buf.getvalue() + + +APK_ZIP = _valid_zip_apk() + + +def _write_app(download_path, name): + """Write a real app file to download_path and return its path (for monkeypatched downloaders).""" + os.makedirs(str(download_path), exist_ok=True) + p = os.path.join(str(download_path), name) + with open(p, 'wb') as f: + f.write(APK_ZIP) + return p + + +def scan_argv(distribution_argv, download_path): + """Common scan flags appended to a store's download flags.""" + return distribution_argv + [ + '--download_path', str(download_path), + '--url', BASE_URL, '--company_id', '1', '--token', TOKEN, + '--profile_id', '2', '--testcase_id', '5', '--report_format', 'json', + ] + + +def _assert_scanned_ok(exit_code, mocked_responses): + assert exit_code == 0 + # the scan was actually created against the facade with the downloaded build + creates = [c for c in mocked_responses.calls if c.request.url == f'{REST_URL}/scans/start/'] + assert creates, 'a scan must be created from the downloaded store build' + + +# --- RuMarket --------------------------------------------------------------- + +def test_rumarket_download_then_scan(mocked_responses, monkeypatch, tmp_path, apk_md5, + no_sleep, ms_mode): + # store: app info + apk download + mocked_responses.add(responses.GET, + 'https://store-api.ruplay.market/api/v1/app/getApp/com.example.app', + json={'data': { + 'packageName': 'com.example.app', + 'author': {'name': 'Acme'}, + 'iconUrl': 'https://x/icon.png', + 'latestApk': {'name': 'app.apk', 'versionName': '1.0', + 'versionCode': 1, 'minSdkVersion': 21, + 'targetSdkVersion': 33, 'size': len(APK_BYTES)}, + }}) + mocked_responses.add(responses.GET, 'https://cdn.ruplay.market/data/apks/app.apk', + body=APK_BYTES, content_type='application/vnd.android.package-archive') + # Sting happy path + register_ms_happy_path(mocked_responses, apk_md5) + exit_code = run_main(monkeypatch, scan_argv( + ['--distribution_system', 'rumarket', '--rumarket_package_name', 'com.example.app'], + tmp_path)) + _assert_scanned_ok(exit_code, mocked_responses) + + +# --- Huawei AppGallery ------------------------------------------------------ + +def test_appgallery_download_then_scan(mocked_responses, monkeypatch, tmp_path, apk_md5, + no_sleep, ms_mode): + mocked_responses.add(responses.POST, + 'https://web-drru.hispace.dbankcloud.ru/webedge/getInterfaceCode', + json='ifc-code') + mocked_responses.add(responses.GET, + 'https://web-drru.hispace.dbankcloud.ru/uowap/index', + json={'layoutData': [{'dataList': [{ + 'package': 'com.example.app', 'appid': 'C123', 'name': 'App', + 'versionName': '1.0', 'versionCode': 1, 'targetSDK': 33, + 'size': len(APK_BYTES), 'md5': 'abc', 'icon': 'https://x/i.png'}]}]}) + mocked_responses.add(responses.GET, + 'https://appgallery.cloud.huawei.com/appdl/C123', + body=APK_BYTES, content_type='application/vnd.android.package-archive') + register_ms_happy_path(mocked_responses, apk_md5) + exit_code = run_main(monkeypatch, scan_argv( + ['--distribution_system', 'appgallery', '--appgallery_app_id', 'C123'], + tmp_path)) + _assert_scanned_ok(exit_code, mocked_responses) + + +# --- Nexus 3 (session + search + download) --- +def test_nexus_download_then_scan(mocked_responses, monkeypatch, tmp_path, apk_md5, no_sleep, ms_mode): + nx = 'http://nexus.example' + mocked_responses.add(responses.POST, f'{nx}/service/rapture/session', json={}, status=200) + mocked_responses.add(responses.GET, f'{nx}/service/rest/v1/search', json={'items': [{'assets': [{ + 'contentType': 'application/vnd.android.package-archive', 'downloadUrl': f'{nx}/repo/app.apk'}]}]}) + mocked_responses.add(responses.GET, f'{nx}/repo/app.apk', body=APK_BYTES) + register_ms_happy_path(mocked_responses, apk_md5) + argv = ['--distribution_system', 'nexus', '--nexus_url', nx, '--nexus_login', 'u', + '--nexus_password', 'p', '--nexus_repo_name', 'releases', '--nexus_group_id', 'com.example', + '--nexus_artifact_id', 'app', '--nexus_version', '1.0'] + _assert_scanned_ok(run_main(monkeypatch, scan_argv(argv, tmp_path)), mocked_responses) + + +# --- Nexus 2 (maven content GET) --- +def test_nexus2_download_then_scan(mocked_responses, monkeypatch, tmp_path, apk_md5, no_sleep, ms_mode): + nx = 'http://nexus2.example' + mocked_responses.add(responses.GET, f'{nx}/service/local/artifact/maven/content', body=APK_BYTES) + register_ms_happy_path(mocked_responses, apk_md5) + argv = ['--distribution_system', 'nexus2', '--nexus2_url', nx, '--nexus2_login', 'u', + '--nexus2_password', 'p', '--nexus2_repo_name', 'releases', '--nexus2_group_id', 'com.example', + '--nexus2_artifact_id', 'app', '--nexus2_version', '1.0', '--nexus2_extension', 'apk'] + _assert_scanned_ok(run_main(monkeypatch, scan_argv(argv, tmp_path)), mocked_responses) + + +# --- RuStore (overallInfo + download-link + apk; validates zip container) --- +def test_rustore_download_then_scan(mocked_responses, monkeypatch, tmp_path, apk_md5, no_sleep, ms_mode): + mocked_responses.add(responses.GET, + 'https://backapi.rustore.ru/applicationData/overallInfo/com.example.app', + json={'body': {'appId': 42, 'packageName': 'com.example.app', 'versionName': '1.0', + 'versionCode': 1, 'minSdkVersion': 21, 'maxSdkVersion': 33, 'targetSdkVersion': 33, + 'fileSize': len(APK_ZIP), 'iconUrl': 'https://x/i.png', 'companyName': 'Acme'}}) + mocked_responses.add(responses.POST, 'https://backapi.rustore.ru/applicationData/download-link', + json={'body': {'apkUrl': 'https://cdn.rustore.example/app.apk'}}) + mocked_responses.add(responses.GET, 'https://cdn.rustore.example/app.apk', body=APK_ZIP, + content_type='application/vnd.android.package-archive') + register_ms_happy_path(mocked_responses, apk_md5) + argv = ['--distribution_system', 'rustore', '--rustore_package_name', 'com.example.app'] + _assert_scanned_ok(run_main(monkeypatch, scan_argv(argv, tmp_path)), mocked_responses) + + +# --- Firebase (downloader monkeypatched: google-auth/service-account internals out of scope) --- +def test_firebase_download_then_scan(mocked_responses, monkeypatch, tmp_path, apk_md5, no_sleep, ms_mode): + monkeypatch.setattr('mdast_cli.mdast_scan.firebase_download_app', + lambda download_path, *a, **k: _write_app(download_path, 'fb_app-1.0.apk')) + register_ms_happy_path(mocked_responses, apk_md5) + argv = ['--distribution_system', 'firebase', '--firebase_project_number', '123', + '--firebase_app_id', '1:123:android:abc', '--firebase_account_json_path', '/tmp/fake.json', + '--firebase_file_extension', 'apk'] + _assert_scanned_ok(run_main(monkeypatch, scan_argv(argv, tmp_path)), mocked_responses) + + +# --- Google Play (apkeep subprocess monkeypatched) --- +def test_google_play_download_then_scan(mocked_responses, monkeypatch, tmp_path, apk_md5, no_sleep, ms_mode): + monkeypatch.setattr('mdast_cli.mdast_scan.GooglePlay.login', lambda self: None) + monkeypatch.setattr('mdast_cli.mdast_scan.GooglePlay.download_app', + lambda self, download_path, *a, **k: _write_app(download_path, 'gp_app-1.0.apk')) + register_ms_happy_path(mocked_responses, apk_md5) + argv = ['--distribution_system', 'google_play', '--google_play_package_name', 'com.example.app', + '--google_play_email', 'u@example.com', '--google_play_aas_token', 'aas_et/xxx'] + _assert_scanned_ok(run_main(monkeypatch, scan_argv(argv, tmp_path)), mocked_responses) + + +# --- Apple App Store (iOS; StoreClient monkeypatched). iOS -> manual scan on an iOS engine. --- +def _register_ms_ios_manual(rsps, apk_md5): + rsps.add(responses.GET, f'{REST_URL}/architectures/', + json=[{'id': 3, 'type': 'IOS', 'os_version': '16', 'name': 'iOS 16'}]) + rsps.add(responses.GET, f'{REST_URL}/engines/', json=[{'type': 'IOS', 'status': 'STARTED'}]) + rsps.add(responses.GET, f'{REST_URL}/applications/', json=[]) + rsps.add(responses.POST, f'{REST_URL}/applications/upload_info/', + json=application_json(apk_md5), status=201) + rsps.add(responses.POST, f'{REST_URL}/scans/start/precheck/', json={'warnings': []}) + rsps.add(responses.POST, f'{REST_URL}/scans/start/', + json=scan_json(stage='CREATED', status='INITIAL', type='MANUAL')) + rsps.add(responses.POST, f'{REST_URL}/scans/77/start/', + json=scan_json(stage='WORKING', status='PROCESSING')) + rsps.add(responses.POST, f'{REST_URL}/scans/77/stop/', + json=scan_json(stage='STOP', status='PROCESSING')) + rsps.add(responses.GET, f'{REST_URL}/scans/77/', json=scan_json(stage='WORKING', status='PROCESSING')) + rsps.add(responses.GET, f'{REST_URL}/scans/77/', json=scan_json(stage='SUCCESS', status='COMPLETE')) + rsps.add(responses.GET, f'{REST_URL}/scans/77/report', body=b'%PDF-1.4 x', + match=[matchers.query_param_matcher({'output': 'pdf'})]) + + +def test_appstore_download_then_scan(mocked_responses, monkeypatch, tmp_path, apk_md5, no_sleep, ms_mode): + # The AppStore downloader returns its own md5 ('d'*32) as the 2nd tuple element. + # F1: on the microservices installation that md5 is DELIBERATELY ignored - the + # Apple store md5 differs from the file actually uploaded (post-download the IPA + # is re-signed/re-zipped), so dedup/precheck/create must key off the LOCAL file md5. + appstore_md5 = 'd' * 32 + monkeypatch.setattr('mdast_cli.mdast_scan.AppStore.download_app', + lambda self, download_path, *a, **k: (_write_app(download_path, 'as_app-1.0.ipa'), appstore_md5)) + _register_ms_ios_manual(mocked_responses, apk_md5) + argv = ['--distribution_system', 'appstore', '--appstore_app_id', '123', + '--appstore_apple_id', 'u@example.com', '--appstore_password', 'pw', '--appstore_2FA', '123456', + '--download_path', str(tmp_path), '--url', BASE_URL, '--company_id', '1', '--token', TOKEN, + '--profile_id', '2'] + _assert_scanned_ok(run_main(monkeypatch, argv), mocked_responses) + + # The local file md5 (of what was actually written/uploaded), NOT the Apple md5. + local_md5 = hashlib.md5(APK_ZIP).hexdigest().lower() + assert local_md5 != appstore_md5 + # create-scan (POST /scans/start/) must carry the local file md5, not 'd'*32. + create = next(c for c in mocked_responses.calls + if c.request.url == f'{REST_URL}/scans/start/' and c.request.method == 'POST') + assert json.loads(create.request.body)['md5'] == local_md5 + # ...and the Apple store md5 must appear nowhere in the create/precheck payloads. + for c in mocked_responses.calls: + if c.request.method == 'POST' and c.request.body and isinstance(c.request.body, (str, bytes)): + body = c.request.body if isinstance(c.request.body, str) else c.request.body.decode('utf-8', 'ignore') + assert appstore_md5 not in body diff --git a/tests/test_mode_detection.py b/tests/test_mode_detection.py index 361bd6f..832ff81 100644 --- a/tests/test_mode_detection.py +++ b/tests/test_mode_detection.py @@ -1,4 +1,8 @@ -"""Unit tests for installation mode auto-detection and env override.""" +"""Unit tests for installation mode auto-detection and env override. + +Auto-detection probes GET {base}/architectures/ and classifies by payload shape: +microservices = string type (ANDROID/IOS), monolith = int type code. +""" import pytest import responses @@ -8,6 +12,10 @@ pytestmark = pytest.mark.unit +ARCH_URL = f'{REST_URL}/architectures/' +MS_ARCH = [{'id': 1, 'type': 'ANDROID', 'os_version': '11', 'name': 'Android 11'}] +MONO_ARCH = [{'id': 1, 'type': 1, 'name': 'Android 11'}] + def test_env_override_microservices(monkeypatch): monkeypatch.setenv('MDAST_CLI_MODE', 'microservices') @@ -25,29 +33,43 @@ def test_env_invalid_value(monkeypatch): resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) -def test_autodetect_microservices(mocked_responses, monkeypatch): +def test_autodetect_microservices_by_payload(mocked_responses, monkeypatch): monkeypatch.delenv('MDAST_CLI_MODE', raising=False) - mocked_responses.add(responses.GET, f'{REST_URL}/engines/', json=[]) + mocked_responses.add(responses.GET, ARCH_URL, json=MS_ARCH) # Bearer probe assert resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) == MODE_MICROSERVICES - probe = mocked_responses.calls[0].request - assert probe.headers['Authorization'] == f'Bearer {TOKEN}' + assert mocked_responses.calls[0].request.headers['Authorization'] == f'Bearer {TOKEN}' + + +def test_autodetect_monolith_by_payload(mocked_responses, monkeypatch): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + # Bearer probe -> monolith answers 401 (Token-only), then Token probe -> int type + mocked_responses.add(responses.GET, ARCH_URL, status=401) + mocked_responses.add(responses.GET, ARCH_URL, json=MONO_ARCH) + assert resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) == MODE_MONOLITH + assert mocked_responses.calls[1].request.headers['Authorization'] == f'Token {TOKEN}' -def test_autodetect_monolith_fallback(mocked_responses, monkeypatch): +def test_monolith_200_on_bearer_not_misclassified(mocked_responses, monkeypatch): + """A monolith (int-type payload) answering 200 to the Bearer probe must NOT be + read as microservices - classification is by payload shape, not status.""" monkeypatch.delenv('MDAST_CLI_MODE', raising=False) - mocked_responses.add(responses.GET, f'{REST_URL}/engines/', status=404) - mocked_responses.add(responses.GET, f'{REST_URL}/organizations/{COMPANY_ID}/engines/', - json=[]) + mocked_responses.add(responses.GET, ARCH_URL, json=MONO_ARCH) # Bearer -> monolith shape + mocked_responses.add(responses.GET, ARCH_URL, json=MONO_ARCH) # Token -> monolith shape assert resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) == MODE_MONOLITH - monolith_probe = mocked_responses.calls[1].request - assert monolith_probe.headers['Authorization'] == f'Token {TOKEN}' + + +def test_ambiguous_200_payload_raises(mocked_responses, monkeypatch): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, ARCH_URL, json={'unexpected': 'shape'}) + mocked_responses.add(responses.GET, ARCH_URL, json={'unexpected': 'shape'}) + with pytest.raises(ModeDetectionError): + resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) def test_autodetect_failure_reports_auth_error(mocked_responses, monkeypatch): monkeypatch.delenv('MDAST_CLI_MODE', raising=False) - mocked_responses.add(responses.GET, f'{REST_URL}/engines/', status=401) - mocked_responses.add(responses.GET, f'{REST_URL}/organizations/{COMPANY_ID}/engines/', - status=401) + mocked_responses.add(responses.GET, ARCH_URL, status=401) + mocked_responses.add(responses.GET, ARCH_URL, status=401) with pytest.raises(ModeDetectionError) as excinfo: resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) assert excinfo.value.auth_error is True @@ -55,9 +77,8 @@ def test_autodetect_failure_reports_auth_error(mocked_responses, monkeypatch): def test_autodetect_failure_network(mocked_responses, monkeypatch): monkeypatch.delenv('MDAST_CLI_MODE', raising=False) - mocked_responses.add(responses.GET, f'{REST_URL}/engines/', status=404) - mocked_responses.add(responses.GET, f'{REST_URL}/organizations/{COMPANY_ID}/engines/', - status=500) + mocked_responses.add(responses.GET, ARCH_URL, status=404) + mocked_responses.add(responses.GET, ARCH_URL, status=500) with pytest.raises(ModeDetectionError) as excinfo: resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) assert excinfo.value.auth_error is False diff --git a/tests/test_negative_e2e.py b/tests/test_negative_e2e.py new file mode 100644 index 0000000..246f3a7 --- /dev/null +++ b/tests/test_negative_e2e.py @@ -0,0 +1,470 @@ +"""Negative end-to-end flows: the CLI must fail with the RIGHT exit code. + +These exercise main() end to end (arg parse -> mode -> flow) against a mocked +server and assert the process exit code, which is the contract CI depends on: + + 2 INVALID_ARGS | 4 DOWNLOAD_FAILED | 5 SCAN_FAILED | + 6 NETWORK_ERROR | 7 AUTH_ERROR | 8 PRECHECK_BLOCKED + +Exit codes must be identical in meaning on both installations (F12). +""" +import json + +import pytest +import responses +from responses import matchers + +from tests.conftest import BASE_URL, REST_URL, TOKEN, application_json, run_main, scan_json + +pytestmark = pytest.mark.e2e + +ARCH = f'{REST_URL}/architectures/' + + +def base_argv(tmp_apk, *extra): + return ['--distribution_system', 'file', '--file_path', tmp_apk, + '--url', BASE_URL, '--company_id', '1', '--token', TOKEN, *extra] + + +def _register_ms_preamble(rsps, apk_md5, engine_status='STARTED', engine_type='ANDROID', + apps=None): + """Everything up to (not including) upload/create, microservices shaped.""" + rsps.add(responses.GET, ARCH, json=[ + {'id': 1, 'type': 'ANDROID', 'os_version': '11', 'name': 'Android 11'}]) + rsps.add(responses.GET, f'{REST_URL}/engines/', + json=[{'engine_id': 'e-1', 'type': engine_type, 'status': engine_status}]) + rsps.add(responses.GET, f'{REST_URL}/applications/', + json=apps if apps is not None else []) + + +# --- mode detection --------------------------------------------------------- + +def test_mode_detection_auth_failure_exit_7(mocked_responses, monkeypatch, tmp_apk): + """Both probes answer 401 -> AUTH_ERROR, and the token never leaks.""" + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, ARCH, status=401) # Bearer probe + mocked_responses.add(responses.GET, ARCH, status=401) # Token probe + assert run_main(monkeypatch, base_argv(tmp_apk)) == 7 + + +def test_mode_detection_ambiguous_payload_exit_6(mocked_responses, monkeypatch, tmp_apk): + """200 with a payload matching neither shape is a gateway/config problem (network).""" + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, ARCH, json={'unexpected': 'shape'}) + mocked_responses.add(responses.GET, ARCH, json={'unexpected': 'shape'}) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def test_mode_detection_unreachable_exit_6(mocked_responses, monkeypatch, tmp_apk): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, ARCH, status=500) + mocked_responses.add(responses.GET, ARCH, status=503) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +# --- upload ----------------------------------------------------------------- + +def test_ms_upload_gateway_502_exhausted_exit_6(mocked_responses, monkeypatch, tmp_apk, + apk_md5, no_sleep, ms_mode): + """A persistent 502 on upload is retryable infra -> NETWORK_ERROR, not SCAN_FAILED.""" + _register_ms_preamble(mocked_responses, apk_md5) + mocked_responses.add(responses.POST, f'{REST_URL}/applications/upload_info/', status=502) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def test_ms_upload_504_exit_6(mocked_responses, monkeypatch, tmp_apk, apk_md5, no_sleep, ms_mode): + """504 = server still parsing; re-run dedupes. Retryable -> NETWORK_ERROR.""" + _register_ms_preamble(mocked_responses, apk_md5) + mocked_responses.add(responses.POST, f'{REST_URL}/applications/upload_info/', status=504) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def test_ms_upload_auth_401_exit_7(mocked_responses, monkeypatch, tmp_apk, apk_md5, + no_sleep, ms_mode): + _register_ms_preamble(mocked_responses, apk_md5) + mocked_responses.add(responses.POST, f'{REST_URL}/applications/upload_info/', status=401) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 7 + + +# --- preflight rejections --------------------------------------------------- + +def test_ms_no_active_engine_exit_5(mocked_responses, monkeypatch, tmp_apk, apk_md5, + no_sleep, ms_mode): + """Android app but the only engine is iOS -> cannot scan (SCAN_FAILED).""" + _register_ms_preamble(mocked_responses, apk_md5, engine_type='IOS') + assert run_main(monkeypatch, base_argv(tmp_apk)) == 5 + + +def test_ms_engine_present_but_down_exit_5(mocked_responses, monkeypatch, tmp_apk, apk_md5, + no_sleep, ms_mode): + _register_ms_preamble(mocked_responses, apk_md5, engine_status='STOPPED_SIGTERM') + assert run_main(monkeypatch, base_argv(tmp_apk)) == 5 + + +def test_ms_testcase_platform_mismatch_exit_2(mocked_responses, monkeypatch, tmp_apk, apk_md5, + no_sleep, ms_mode): + """Android file + iOS-recorded test case is a user mistake (INVALID_ARGS).""" + mocked_responses.add(responses.GET, ARCH, + json=[{'id': 1, 'type': 'ANDROID', 'os_version': '11'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/testcases/5/', json={'id': 5, 'os': 'IOS'}) + assert run_main(monkeypatch, base_argv(tmp_apk, '--testcase_id', '5')) == 2 + + +# --- scan lifecycle --------------------------------------------------------- + +def _register_ms_through_start(rsps, apk_md5): + rsps.add(responses.GET, f'{REST_URL}/testcases/5/', json={'id': 5, 'os': 'ANDROID'}) + _register_ms_preamble(rsps, apk_md5, apps=[application_json(apk_md5)]) + rsps.add(responses.POST, f'{REST_URL}/scans/start/precheck/', json={'warnings': []}) + rsps.add(responses.POST, f'{REST_URL}/scans/start/', + json=scan_json(stage='CREATED', status='INITIAL')) + rsps.add(responses.POST, f'{REST_URL}/scans/77/start/', + json=scan_json(stage='START', status='INITIAL')) + + +def test_ms_scan_never_terminal_times_out_exit_5(mocked_responses, monkeypatch, tmp_apk, + apk_md5, no_sleep, ms_mode): + """A scan that never reaches a terminal (stage, status) must FAIL, not hang or pass.""" + monkeypatch.setattr('mdast_cli.ms_flow.TRY', 2) + _register_ms_through_start(mocked_responses, apk_md5) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='WORKING', status='PROCESSING')) + assert run_main(monkeypatch, base_argv(tmp_apk, '--testcase_id', '5')) == 5 + + +def test_ms_scan_fail_terminal_exit_5(mocked_responses, monkeypatch, tmp_apk, apk_md5, + no_sleep, ms_mode): + _register_ms_through_start(mocked_responses, apk_md5) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='FAIL', status='FAIL', message='boom')) + assert run_main(monkeypatch, base_argv(tmp_apk, '--testcase_id', '5')) == 5 + + +# --- manual scan without a profile (F10) ------------------------------------ + +def test_ms_manual_without_profile_skips_precheck_and_stops(mocked_responses, monkeypatch, + tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode): + """F10: no --profile_id -> profile is auto-created, so pre-check (422, profile does + not exist yet) is skipped rather than blocking; the manual scan is stopped by the + CLI after the wait and is still expected to finish SUCCESS.""" + monkeypatch.chdir(tmp_path) + _register_ms_preamble(mocked_responses, apk_md5, apps=[application_json(apk_md5)]) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/precheck/', + json={'detail': 'profile not found'}, status=422) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', + json=scan_json(stage='CREATED', status='INITIAL', type='MANUAL')) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/start/', + json=scan_json(stage='WORKING', status='PROCESSING')) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/stop/', + json=scan_json(stage='STOP', status='PROCESSING')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='WORKING', status='PROCESSING')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='SUCCESS', status='COMPLETE')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', body=b'%PDF-1.4 x', + match=[matchers.query_param_matcher({'output': 'pdf'})]) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 0 + create = [c for c in mocked_responses.calls if c.request.url == f'{REST_URL}/scans/start/'] + body = json.loads(create[0].request.body) + assert body['type'] == 'MANUAL' + assert 'profile_id' not in body, 'no --profile_id -> profile_id must be omitted (auto-create)' + assert [c for c in mocked_responses.calls if c.request.url == f'{REST_URL}/scans/77/stop/'], \ + 'manual scan must be stopped by the CLI after the wait' + + +# --- reports (soft-fail) ---------------------------------------------------- + +def test_ms_report_soft_fail_keeps_scan_green(mocked_responses, monkeypatch, tmp_path, + tmp_apk, apk_md5, no_sleep, ms_mode, caplog): + """PDF render fails (Pepper 502) but the scan finished SUCCESS: exit 0, JSON still saved.""" + monkeypatch.chdir(tmp_path) + _register_ms_through_start(mocked_responses, apk_md5) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='SUCCESS', status='COMPLETE')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', status=502, + match=[matchers.query_param_matcher({'output': 'pdf'})]) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', + json={'summary': {'scan_id': 77}}, + match=[matchers.query_param_matcher({'output': 'json'})]) + exit_code = run_main(monkeypatch, base_argv(tmp_apk, '--testcase_id', '5', + '--report_format', 'all')) + assert exit_code == 0 + assert not (tmp_path / 'scan_report_77.pdf').exists() + assert json.loads((tmp_path / 'scan_report_77.json').read_text())['summary']['scan_id'] == 77 + + +def test_ms_default_report_is_pdf(mocked_responses, monkeypatch, tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode): + """No report flags at all -> a PDF is produced by default, named by scan id.""" + monkeypatch.chdir(tmp_path) + _register_ms_through_start(mocked_responses, apk_md5) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='SUCCESS', status='COMPLETE')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', body=b'%PDF-1.4 x', + match=[matchers.query_param_matcher({'output': 'pdf'})]) + assert run_main(monkeypatch, base_argv(tmp_apk, '--testcase_id', '5')) == 0 + assert (tmp_path / 'scan_report_77.pdf').read_bytes().startswith(b'%PDF') + assert not (tmp_path / 'scan_report_77.json').exists() + + +def test_ms_report_format_none_writes_nothing(mocked_responses, monkeypatch, tmp_path, + tmp_apk, apk_md5, no_sleep, ms_mode): + monkeypatch.chdir(tmp_path) + _register_ms_through_start(mocked_responses, apk_md5) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='SUCCESS', status='COMPLETE')) + assert run_main(monkeypatch, base_argv(tmp_apk, '--testcase_id', '5', + '--report_format', 'none')) == 0 + assert not list(tmp_path.glob('scan_report_*')) + + +# --- arg / input validation ------------------------------------------------- + +def test_missing_file_path_exit_2(monkeypatch, tmp_path): + missing = str(tmp_path / 'does_not_exist.apk') + assert run_main(monkeypatch, ['--distribution_system', 'file', '--file_path', missing, + '--url', BASE_URL, '--company_id', '1', '--token', TOKEN]) == 2 + + +def test_scan_without_credentials_exit_2(monkeypatch, tmp_apk): + """No --url/--company_id/--token and not --download_only -> argparse error (2).""" + assert run_main(monkeypatch, ['--distribution_system', 'file', '--file_path', tmp_apk]) == 2 + + +def test_cr_report_without_credentials_exit_2(monkeypatch, tmp_apk): + assert run_main(monkeypatch, base_argv(tmp_apk, '--cr_report')) == 2 + + +# --- URL normalization robustness ------------------------------------------- + +def test_url_already_ending_in_rest_slash_still_works(mocked_responses, monkeypatch, tmp_path, + tmp_apk, apk_md5, no_sleep, ms_mode): + """--url '.../rest/' must not produce '//' paths (which a strict facade 404s). + + All endpoints are registered at the single-slash canonical form; if + normalization regressed to leaving a trailing slash, the requests would miss + and the flow would fail with NETWORK_ERROR instead of 0. + """ + monkeypatch.chdir(tmp_path) + _register_ms_through_start(mocked_responses, apk_md5) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='SUCCESS', status='COMPLETE')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', body=b'%PDF-1.4 x', + match=[matchers.query_param_matcher({'output': 'pdf'})]) + argv = ['--distribution_system', 'file', '--file_path', tmp_apk, + '--url', f'{BASE_URL}/rest/', '--company_id', '1', '--token', TOKEN, + '--testcase_id', '5'] + assert run_main(monkeypatch, argv) == 0 + assert (tmp_path / 'scan_report_77.pdf').exists() + + +# --- monolith parity -------------------------------------------------------- + +def _register_mono_through_start(rsps): + """Monolith manual flow up to (and including) start; dedup returns an existing + app so the upload step is skipped. Caller registers the GET /dasts/77/ polls.""" + rest = REST_URL + rsps.add(responses.GET, f'{rest}/architectures/', + json=[{'id': 1, 'name': 'Android 11', 'type': 1}]) + rsps.add(responses.GET, f'{rest}/organizations/1/engines/', + json=[{'architecture': 1, 'state': 3}]) + rsps.add(responses.GET, f'{rest}/organizations/1/applications/', + json=[{'id': 10, 'package_name': 'com.example.app', 'version_name': '1.0', 'md5': 'x'}]) + rsps.add(responses.POST, f'{rest}/organizations/1/dasts/', + json={'id': 77, 'project': {'id': 1}, 'profile': {'id': 2}}, status=201) + rsps.add(responses.POST, f'{rest}/dasts/77/start/', json={}, status=200) + + +def _register_mono_through_success(rsps): + _register_mono_through_start(rsps) + rest = REST_URL + rsps.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 2}) + rsps.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 2}) + rsps.add(responses.POST, f'{rest}/dasts/77/stop/', json={}, status=200) + rsps.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 4}) + + +def test_autodetect_resolves_monolith_and_runs(mocked_responses, monkeypatch, tmp_apk): + """Auto mode must classify a monolith (int-type payload) and run the monolith flow, + even though the Bearer probe answers 200 - classification is by payload shape.""" + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + _register_mono_through_start(mocked_responses) + assert run_main(monkeypatch, base_argv(tmp_apk, '--nowait')) == 0 + # the Token-scheme /organizations/1/ URLs prove the monolith flow ran + assert any('/organizations/1/' in c.request.url for c in mocked_responses.calls) + + +def test_monolith_mid_poll_401_is_auth_error_exit_7(mocked_responses, monkeypatch, tmp_apk, + no_sleep, monolith_mode): + """F12: a token expiring mid-scan (poll 401) must exit AUTH_ERROR(7), like microservices.""" + _register_mono_through_start(mocked_responses) + mocked_responses.add(responses.GET, f'{REST_URL}/dasts/77/', status=401, + json={'detail': 'token expired'}) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 7 + + +def test_monolith_network_exception_is_network_error_exit_6(monkeypatch, mocked_responses, + tmp_apk, monolith_mode): + """F12: a bare connection error in the monolith flow -> NETWORK_ERROR(6), not a traceback+1.""" + # no endpoints registered -> the first monolith call raises ConnectionError + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def test_monolith_invalid_architecture_id_exit_2(mocked_responses, monkeypatch, tmp_apk, + monolith_mode): + """An --architecture_id the server doesn't know is a user error -> INVALID_ARGS(2), not a crash.""" + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', + json=[{'id': 1, 'name': 'Android 11', 'type': 1}]) + assert run_main(monkeypatch, base_argv(tmp_apk, '--architecture_id', '999')) == 2 + + +def test_monolith_report_pdf_hardfail_exit_5(mocked_responses, monkeypatch, tmp_path, tmp_apk, + no_sleep, monolith_mode): + """Monolith keeps report download as a HARD fail (unlike microservices soft-fail).""" + monkeypatch.chdir(tmp_path) + _register_mono_through_success(mocked_responses) + mocked_responses.add(responses.GET, f'{REST_URL}/dasts/77/report/', status=502) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 5 + + +def test_monolith_json_report_non_json_body_exit_5(mocked_responses, monkeypatch, tmp_path, + tmp_apk, no_sleep, monolith_mode): + """A 200 with a non-JSON body must exit SCAN_FAILED(5), not traceback, and leave no file.""" + monkeypatch.chdir(tmp_path) + _register_mono_through_success(mocked_responses) + mocked_responses.add(responses.GET, f'{REST_URL}/dasts/77/report/', body=b'oops') + assert run_main(monkeypatch, base_argv(tmp_apk, '--report_format', 'json')) == 5 + assert not list(tmp_path.glob('scan_report_*')) + + +def test_monolith_scan_failed_state_exit_5(mocked_responses, monkeypatch, tmp_path, tmp_apk, + no_sleep, monolith_mode): + """Monolith must map a non-SUCCESS terminal state to SCAN_FAILED, same as microservices.""" + monkeypatch.chdir(tmp_path) + rest = REST_URL + mocked_responses.add(responses.GET, f'{rest}/architectures/', + json=[{'id': 1, 'name': 'Android 11', 'type': 1}]) + mocked_responses.add(responses.GET, f'{rest}/organizations/1/engines/', + json=[{'architecture': 1, 'state': 3}]) + mocked_responses.add(responses.GET, f'{rest}/organizations/1/applications/', + json=[{'id': 10, 'package_name': 'com.example.app', + 'version_name': '1.0', 'md5': 'x'}]) + mocked_responses.add(responses.POST, f'{rest}/organizations/1/dasts/', + json={'id': 77, 'project': {'id': 1}, 'profile': {'id': 2}}, status=201) + mocked_responses.add(responses.POST, f'{rest}/dasts/77/start/', json={}, status=200) + mocked_responses.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 2}) + mocked_responses.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 2}) + mocked_responses.add(responses.POST, f'{rest}/dasts/77/stop/', json={}, status=200) + mocked_responses.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 5}) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 5 + + +# ===== more Sting-side negatives (microservices) — expanded coverage ===== + +def test_ms_architectures_5xx_is_network_6(mocked_responses, monkeypatch, tmp_apk, no_sleep, ms_mode): + mocked_responses.add(responses.GET, ARCH, status=500) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def test_ms_architectures_non_json_is_network_6(mocked_responses, monkeypatch, tmp_apk, no_sleep, ms_mode): + """A 200 with a non-JSON body (nginx/Envoy HTML stub) is a gateway problem, not a crash.""" + mocked_responses.add(responses.GET, ARCH, body='gw', status=200, content_type='text/html') + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def test_ms_engines_5xx_is_network_6(mocked_responses, monkeypatch, tmp_apk, no_sleep, ms_mode): + mocked_responses.add(responses.GET, ARCH, json=[{'id': 1, 'type': 'ANDROID', 'os_version': '11'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', status=500) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def test_ms_engines_empty_no_active_engine_5(mocked_responses, monkeypatch, tmp_apk, no_sleep, ms_mode): + mocked_responses.add(responses.GET, ARCH, json=[{'id': 1, 'type': 'ANDROID', 'os_version': '11'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', json=[]) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 5 + + +def test_ms_dedup_5xx_is_network_6(mocked_responses, monkeypatch, tmp_apk, no_sleep, ms_mode): + mocked_responses.add(responses.GET, ARCH, json=[{'id': 1, 'type': 'ANDROID', 'os_version': '11'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', + json=[{'type': 'ANDROID', 'status': 'STARTED'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/applications/', status=500) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def _ms_through_precheck(rsps, apk_md5): + """architectures + testcase + engines + dedup(existing) + precheck OK; caller adds create/start.""" + rsps.add(responses.GET, ARCH, json=[{'id': 1, 'type': 'ANDROID', 'os_version': '11'}]) + rsps.add(responses.GET, f'{REST_URL}/testcases/5/', json={'id': 5, 'os': 'ANDROID'}) + rsps.add(responses.GET, f'{REST_URL}/engines/', json=[{'type': 'ANDROID', 'status': 'STARTED'}]) + rsps.add(responses.GET, f'{REST_URL}/applications/', json=[application_json(apk_md5)]) + rsps.add(responses.POST, f'{REST_URL}/scans/start/precheck/', json={'warnings': []}) + + +def test_ms_create_scan_4xx_is_scan_failed_5(mocked_responses, monkeypatch, tmp_apk, apk_md5, + no_sleep, ms_mode): + _ms_through_precheck(mocked_responses, apk_md5) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', + json={'error_code': 'bad_request', 'message': 'nope'}, status=400) + assert run_main(monkeypatch, base_argv(tmp_apk, '--profile_id', '2', '--testcase_id', '5')) == 5 + + +def test_ms_create_scan_no_id_is_scan_failed_5(mocked_responses, monkeypatch, tmp_apk, apk_md5, + no_sleep, ms_mode): + _ms_through_precheck(mocked_responses, apk_md5) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', json={}, status=200) + assert run_main(monkeypatch, base_argv(tmp_apk, '--profile_id', '2', '--testcase_id', '5')) == 5 + + +def test_ms_start_5xx_is_scan_failed_5(mocked_responses, monkeypatch, tmp_apk, apk_md5, + no_sleep, ms_mode): + _ms_through_precheck(mocked_responses, apk_md5) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', + json=scan_json(stage='CREATED', status='INITIAL')) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/start/', + json={'error_code': 'x', 'message': 'boom'}, status=500) + assert run_main(monkeypatch, base_argv(tmp_apk, '--profile_id', '2', '--testcase_id', '5')) == 5 + + +def test_ms_partial_complete_succeeds_0(mocked_responses, monkeypatch, tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode): + """PARTIAL_COMPLETE is a green terminal (some modules skipped): exit 0 with a warning.""" + monkeypatch.chdir(tmp_path) + _register_ms_through_start(mocked_responses, apk_md5) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='SUCCESS', status='PARTIAL_COMPLETE')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', body=b'%PDF-1.4 x', + match=[matchers.query_param_matcher({'output': 'pdf'})]) + assert run_main(monkeypatch, base_argv(tmp_apk, '--testcase_id', '5')) == 0 + + +def test_ms_both_reports_fail_still_green_0(mocked_responses, monkeypatch, tmp_path, tmp_apk, + apk_md5, no_sleep, ms_mode): + """Scan SUCCESS but BOTH reports 502 -> soft-fail: exit 0 (report service is separate).""" + monkeypatch.chdir(tmp_path) + _register_ms_through_start(mocked_responses, apk_md5) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='SUCCESS', status='COMPLETE')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', status=502, + match=[matchers.query_param_matcher({'output': 'pdf'})]) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', status=502, + match=[matchers.query_param_matcher({'output': 'json'})]) + assert run_main(monkeypatch, base_argv(tmp_apk, '--testcase_id', '5', '--report_format', 'all')) == 0 + + +# ===== more Sting-side negatives (monolith) ===== + +def test_monolith_architectures_5xx_is_network_6(mocked_responses, monkeypatch, tmp_apk, + no_sleep, monolith_mode): + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', status=500) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def test_monolith_no_active_engine_5(mocked_responses, monkeypatch, tmp_apk, no_sleep, monolith_mode): + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', + json=[{'id': 1, 'name': 'Android 11', 'type': 1}]) + # engine present but state != 3 (not active) + mocked_responses.add(responses.GET, f'{REST_URL}/organizations/1/engines/', + json=[{'architecture': 1, 'state': 2}]) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 5 diff --git a/tests/test_poll_resilience.py b/tests/test_poll_resilience.py index 142a244..cbd394b 100644 --- a/tests/test_poll_resilience.py +++ b/tests/test_poll_resilience.py @@ -55,3 +55,36 @@ def test_network_exception_retried_then_success(client, no_sleep): with mock.patch.object(client, 'get_scan_info', side_effect=seq): scan = ms_flow._get_scan(client, 55) assert scan['status'] == 'COMPLETE' + + +# --- report download resilience (soft-fail contract, STG-4478) --- + +REPORT_URL = f'{REST_URL}/scans/55/report' + + +def test_report_transient_502_then_success(client, mocked_responses, no_sleep): + """A transient 502 on the report service is retried, then the PDF is returned.""" + mocked_responses.add(responses.GET, REPORT_URL, status=502, json={'error_code': 'busy'}) + mocked_responses.add(responses.GET, REPORT_URL, body=b'%PDF-1.4 x', + content_type='application/pdf') + resp = ms_flow._download_report(client.download_report, 55, 'PDF report') + assert resp is not None + assert resp.content == b'%PDF-1.4 x' + + +def test_report_persistent_failure_soft_fails_to_none(client, mocked_responses, no_sleep): + """A report that never renders returns None (soft-fail) - it must NOT raise: + the scan already succeeded, so a report-service outage cannot turn it red.""" + for _ in range(ms_flow.POLL_TRANSIENT_RETRIES + 1): + mocked_responses.add(responses.GET, REPORT_URL, status=502, json={'error_code': 'busy'}) + resp = ms_flow._download_report(client.download_report, 55, 'PDF report') + assert resp is None + + +def test_report_network_exception_then_success(client, no_sleep): + """A raw connection drop on report fetch is retried, not fatal.""" + resp_ok = mock.Mock(status_code=200, content=b'%PDF-1.4 x') + seq = [requests.ConnectionError('boom'), resp_ok] + fetch = mock.Mock(side_effect=seq) + resp = ms_flow._download_report(fetch, 55, 'PDF report') + assert resp is resp_ok diff --git a/tests/test_precheck_gate.py b/tests/test_precheck_gate.py index a9b9530..3e8166c 100644 --- a/tests/test_precheck_gate.py +++ b/tests/test_precheck_gate.py @@ -48,11 +48,29 @@ def test_precheck_unavailable_blocks(client, mocked_responses): assert excinfo.value.code == ExitCode.PRECHECK_BLOCKED -def test_precheck_transport_error_blocks(client, mocked_responses): +def test_precheck_gateway_5xx_is_network_error_not_block(client, mocked_responses, no_sleep): + """A transient gateway 5xx is retryable infra, not a policy block -> NETWORK_ERROR(6).""" mocked_responses.add(responses.POST, PRECHECK_URL, status=502, json={'error_code': 'downstream_unavailable', 'message': 'Scanyon down'}) with pytest.raises(SystemExit) as excinfo: run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None, scan_type='MANUAL') + assert excinfo.value.code == ExitCode.NETWORK_ERROR + + +def test_precheck_network_exception_is_network_error(client, mocked_responses, no_sleep): + """A raw connection error during pre-check -> NETWORK_ERROR(6), still fail-closed.""" + # no mock registered for PRECHECK_URL -> responses raises ConnectionError + with pytest.raises(SystemExit) as excinfo: + run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None, scan_type='MANUAL') + assert excinfo.value.code == ExitCode.NETWORK_ERROR + + +def test_precheck_non_transient_non_200_blocks(client, mocked_responses): + """A genuine non-transient rejection (e.g. 400) is a policy block -> exit 8.""" + mocked_responses.add(responses.POST, PRECHECK_URL, status=400, + json={'error_code': 'bad_request', 'message': 'nope'}) + with pytest.raises(SystemExit) as excinfo: + run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None, scan_type='MANUAL') assert excinfo.value.code == ExitCode.PRECHECK_BLOCKED diff --git a/tests/test_report_targets.py b/tests/test_report_targets.py new file mode 100644 index 0000000..94468ee --- /dev/null +++ b/tests/test_report_targets.py @@ -0,0 +1,55 @@ +"""Unit tests for the shared report-target resolver (used by both installations).""" +from types import SimpleNamespace + +import pytest + +from mdast_cli.helpers.helpers import resolve_report_targets + +pytestmark = pytest.mark.unit + + +def args(report_format=None, pdf=None, json=None): + return SimpleNamespace(report_format=report_format, + pdf_report_file_name=pdf, + summary_report_json_file_name=json) + + +def test_default_is_pdf_named_by_scan_id(): + assert resolve_report_targets(args(), 77) == {'pdf': 'scan_report_77.pdf'} + + +def test_format_json_only(): + assert resolve_report_targets(args(report_format='json'), 77) == {'json': 'scan_report_77.json'} + + +def test_format_all_produces_both(): + assert resolve_report_targets(args(report_format='all'), 77) == { + 'pdf': 'scan_report_77.pdf', 'json': 'scan_report_77.json'} + + +def test_format_none_produces_nothing(): + assert resolve_report_targets(args(report_format='none'), 77) == {} + + +def test_explicit_pdf_name_only(): + assert resolve_report_targets(args(pdf='r'), 77) == {'pdf': 'r.pdf'} + + +def test_explicit_json_name_does_not_pull_in_default_pdf(): + # regression: a defaulted 'pdf' format must not sneak a PDF in when the user + # asked only for a JSON file by name. + assert resolve_report_targets(args(json='r'), 77) == {'json': 'r.json'} + + +def test_extension_added_only_when_missing(): + assert resolve_report_targets(args(pdf='r.pdf', json='r.json'), 77) == { + 'pdf': 'r.pdf', 'json': 'r.json'} + + +def test_format_flag_adds_other_format_to_explicit_name(): + assert resolve_report_targets(args(report_format='all', pdf='p'), 77) == { + 'pdf': 'p.pdf', 'json': 'scan_report_77.json'} + + +def test_explicit_name_wins_over_none(): + assert resolve_report_targets(args(report_format='none', pdf='p'), 77) == {'pdf': 'p.pdf'} diff --git a/tests/test_security.py b/tests/test_security.py index bfa4ff0..b305407 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -73,26 +73,31 @@ def test_x_file_size_matches_actual_content(mocked_responses, tmp_apk): assert declared <= int(request.headers['Content-Length']) -def test_tls_verify_disabled_by_default(): +def test_tls_verify_enabled_by_default(): + """F4: secure by default - client verifies TLS unless explicitly told not to.""" client = mDastMicroservices(REST_URL, TOKEN) with mock.patch('mdast_cli_core.microservices.requests.get') as mocked_get: client.get_architectures() - assert mocked_get.call_args.kwargs['verify'] is False + assert mocked_get.call_args.kwargs['verify'] is True -def test_tls_verify_enabled_via_client_flag(): - client = mDastMicroservices(REST_URL, TOKEN, verify=True) +def test_tls_verify_can_be_disabled_via_client_flag(): + client = mDastMicroservices(REST_URL, TOKEN, verify=False) with mock.patch('mdast_cli_core.microservices.requests.get') as mocked_get: client.get_architectures() - assert mocked_get.call_args.kwargs['verify'] is True + assert mocked_get.call_args.kwargs['verify'] is False @pytest.mark.parametrize('raw,expected', [ - ('1', True), ('true', True), ('YES', True), ('on', True), - ('', False), ('0', False), ('false', False), ('off', False), + # secure by default: unset/empty -> True; only explicit off-values disable + (None, True), ('1', True), ('true', True), ('YES', True), ('on', True), ('', True), + ('0', False), ('false', False), ('no', False), ('off', False), ]) def test_tls_verify_env_parsing(monkeypatch, raw, expected): - monkeypatch.setenv(factory.TLS_VERIFY_ENV_VAR, raw) + if raw is None: + monkeypatch.delenv(factory.TLS_VERIFY_ENV_VAR, raising=False) + else: + monkeypatch.setenv(factory.TLS_VERIFY_ENV_VAR, raw) assert factory.tls_verify_enabled() is expected @@ -109,8 +114,12 @@ def test_precheck_payload_with_control_characters_is_safe(): def test_mode_probe_does_not_leak_token_on_error(mocked_responses, monkeypatch): monkeypatch.delenv('MDAST_CLI_MODE', raising=False) - mocked_responses.add(responses.GET, f'{REST_URL}/engines/', status=401) - mocked_responses.add(responses.GET, f'{REST_URL}/organizations/1/engines/', status=401) + # The factory probes GET /architectures/ (Bearer, then Token) - both 401 here. + # One registration serves both probes (responses reuses it for repeat calls). + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', status=401) with pytest.raises(factory.ModeDetectionError) as excinfo: factory.resolve_installation_mode(REST_URL, TOKEN, '1') + # An auth failure must be classified as such, and the token must never appear + # in the surfaced error (it is logged/raised near the Authorization header). + assert excinfo.value.auth_error is True assert TOKEN not in str(excinfo.value) diff --git a/tests/test_smoke_flows.py b/tests/test_smoke_flows.py index 27a551a..bf6060e 100644 --- a/tests/test_smoke_flows.py +++ b/tests/test_smoke_flows.py @@ -18,7 +18,7 @@ def register_ms_happy_path(rsps, apk_md5, with_testcase=True): rsps.add(responses.GET, f'{REST_URL}/architectures/', json=[ - {'id': 1, 'type': 1, 'os_version': '11', 'name': 'Android 11', 'description': 'API 30'}, + {'id': 1, 'type': 'ANDROID', 'os_version': '11', 'name': 'Android 11', 'description': 'API 30'}, ]) if with_testcase: rsps.add(responses.GET, f'{REST_URL}/testcases/5/', json={'id': 5, 'os': 'ANDROID'})