diff --git a/.github/workflows/docker-hub-publish.yml b/.github/workflows/docker-hub-publish.yml index a068611..4848837 100644 --- a/.github/workflows/docker-hub-publish.yml +++ b/.github/workflows/docker-hub-publish.yml @@ -1,38 +1,50 @@ -name: Publish mdast_cli image to Docker Hub - -on: - push: - branches: - - main - -jobs: - build-n-publish-docker: - name: Docker Hub - Publish mdast_cli - runs-on: ubuntu-22.04 - - steps: - - uses: actions/checkout@v3 - - - name: docker login - env: - DOCKER_USER: ${{secrets.DOCKER_USER}} - DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} - run: - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - - - name: Build the Docker image - - run: docker build . --file Dockerfile -t mobilesecurity/mdast_cli:2026.6.1 -t mobilesecurity/mdast_cli:latest - - - - name: Docker Hub push latest image - run: docker push mobilesecurity/mdast_cli:latest - - - name: Docker Hub push tagged image - - run: docker push mobilesecurity/mdast_cli:2026.6.1 - - - - - +name: Publish mdast_cli image to Docker Hub + +on: + push: + branches: + - main + +jobs: + tests: + name: Tests gate before publish + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: pip install -r requirements.txt -r requirements-dev.txt + - run: python -m pytest -q + + build-n-publish-docker: + name: Docker Hub - Publish mdast_cli + needs: tests + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v3 + + - name: docker login + env: + DOCKER_USER: ${{secrets.DOCKER_USER}} + DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} + run: + docker login -u $DOCKER_USER -p $DOCKER_PASSWORD + + - name: Build the Docker image + + run: docker build . --file Dockerfile -t mobilesecurity/mdast_cli:2026.6.1 -t mobilesecurity/mdast_cli:latest + + + - name: Docker Hub push latest image + run: docker push mobilesecurity/mdast_cli:latest + + - name: Docker Hub push tagged image + + run: docker push mobilesecurity/mdast_cli:2026.6.1 + + + + + diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index b14b1cf..9f599b7 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -6,8 +6,20 @@ on: - main jobs: + tests: + name: Tests gate before publish + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: pip install -r requirements.txt -r requirements-dev.txt + - run: python -m pytest -q + build-n-publish: name: Build and publish mdast_cli 🐍 distributions to PyPI + needs: tests runs-on: ubuntu-22.04 steps: - uses: actions/checkout@master diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..fb4824b --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,26 @@ +name: Tests + +on: + push: + pull_request: + +jobs: + tests: + name: pytest (unit, contract, smoke, security) + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: pip install -r requirements.txt -r requirements-dev.txt + + - name: Run full test suite + run: python -m pytest -q + + - name: Security tests gate + run: python -m pytest -q -m security diff --git a/.gitignore b/.gitignore index e738652..59c188a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,11 @@ dist/ # venv venv/ +.venv/ + +# CLI-generated scan reports (default output names: scan_report_.pdf/.json) +scan_report_*.pdf +scan_report_*.json # Compiled Java class files *.class diff --git a/README.md b/README.md index a30cd63..9e90ae4 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ This script is designed to integrate mobile applications' security analysis into * [Usage Modes](#usage-modes) * [Download Only Mode](#download-only-mode) * [Scan Mode](#scan-mode) +* [Installation Modes (Monolith and Kubernetes)](#installation-modes-monolith-and-kubernetes) * [Distribution Systems](#distribution-systems) * [Local File](#local-file) * [Google Play](#google-play) @@ -189,7 +190,55 @@ Without `--download_only`, the script will: 2. Upload to the DAST platform 3. Start a security scan 4. Wait for completion (unless `--nowait` is set) -5. Generate reports (if specified) +5. Generate reports (PDF by default) + +--- + +## Installation Modes (Monolith and Kubernetes) + +`mdast_cli` is a **single package that works against both** flavours of the DAST +platform, with **no change to the command line**: + +- **Monolith** - the classic all-in-one installation (`Token` auth, tenant carried + in the `/organizations/{company_id}/...` URL path). +- **Microservices / Kubernetes** - the platform behind the Clark facade (`Bearer` + auth with an organization token issued in the platform UI; the tenant is resolved + server-side, so `--company_id` is accepted but ignored). + +By default the CLI **auto-detects** the installation by probing `GET /rest/architectures/` +and inspecting the response shape (it does not merely trust an HTTP 200). The same +`--url`, `--token`, `--company_id` and scan flags are used either way. + +### Selecting the mode explicitly + +Auto-detection can be overridden with the `MDAST_CLI_MODE` environment variable - +recommended in CI, where the target platform is known and a deterministic choice is +preferable to a probe: + +```bash +export MDAST_CLI_MODE=microservices # or: monolith (default: auto) +``` + +### Environment variables (microservices installation) + +| Variable | Default | Purpose | +|----------|---------|---------| +| `MDAST_CLI_MODE` | `auto` | Force the installation mode (`auto` / `monolith` / `microservices`). | +| `MDAST_TLS_VERIFY` | *(on)* | TLS certificate verification. **Secure by default.** Set to `0`/`false`/`no`/`off` to disable for a self-signed stand (the org token is a credential - use only on trusted networks). An unset **or empty** value stays secure. | +| `MDAST_UPLOAD_TIMEOUT` | *(server default)* | Optional server-side upload/parse wait in seconds (1-300) for large builds. | + +### Differences on the microservices installation + +- **Authentication** uses an **organization CLI token** issued in the platform UI + (`Bearer`), not a per-user CI/CD token. There is one active token per organization. +- `--architecture_id` is **ignored** (the platform resolves the architecture from the + application binary itself). +- `--appium_script_path` (Appium scans) and `--cr_report` (CR report) are **not + supported** and exit with `INVALID_ARGS` (2). +- A report render failure is **soft-fail**: a finished scan stays green (exit 0) with + a warning, because the report service (Pepper) is separate from the scan itself. +- Transient gateway errors (502/503/504) during polling, upload and report download + are **retried**, so a rolling restart of a backend mid-scan does not fail a CI job. --- @@ -643,11 +692,43 @@ mdast_cli ... --project_id 10 # Create profile in project #10 ## Reports +Reports work the **same way on both installations**. + +### Choosing the report format + +**Parameter:** `--report_format {pdf|json|all|none}` (default: `pdf`) + +After a successful scan the CLI downloads a **PDF report by default** - a plain scan +always produces one. Use the flag to choose otherwise: + +| `--report_format` | Result | +|-------------------|--------| +| `pdf` *(default)* | PDF only | +| `json` | JSON summary only | +| `all` | both PDF and JSON | +| `none` | no report (e.g. gate on exit code only) | + +**File names.** If you do not pass an explicit file name, reports are saved next to +the working directory as `scan_report_.pdf` / `scan_report_.json`, +so the file is always tied to the scan it came from. Passing `--pdf_report_file_name` +and/or `--summary_report_json_file_name` overrides the name and forces that format +(so those flags remain fully backward compatible). PDF is always downloaded first. + +> On the **microservices** installation a report that fails to render (e.g. the +> report service returns 502) is a **warning, not a failure**: the scan already +> succeeded, so the CLI still exits 0 and any other requested report is still saved. + +> **In Docker:** the default `scan_report_.*` name is written to the +> container's working directory, which is discarded when the container exits. Mount +> a host directory and point the report there with an absolute path (e.g. +> `-v /host/reports:/mdast/report` + `--pdf_report_file_name /mdast/report/report.pdf`), +> otherwise the report is lost. See the Docker example under *Local File*. + ### JSON Summary Report Generate a structured JSON report with scan summary and statistics. -**Parameter:** `--summary_report_json_file_name ` +**Parameter:** `--summary_report_json_file_name ` (implies `--report_format json`) **Output Format:** - Total number of vulnerabilities @@ -673,7 +754,7 @@ mdast_cli \ Generate a detailed PDF report with full scan results. -**Parameter:** `--pdf_report_file_name ` +**Parameter:** `--pdf_report_file_name ` (implies `--report_format pdf`; a PDF is produced by default even without this flag) **Output Format:** - Detailed vulnerability descriptions @@ -697,7 +778,12 @@ mdast_cli \ ### CR Report -Generate a CR (Change Request) report in HTML format. +Generate a CR (Compliance) report in HTML format - a Bank-of-Russia regulatory +report on call-trace immutability (KNTV) that compares two versions of a mobile +application. Not a "change request" report. + +> **Monolith installation only.** On the microservices installation `--cr_report` +> exits with `INVALID_ARGS` (2). **Required Parameters (when `--cr_report` is set):** - `--cr_report` - Enable CR report generation @@ -760,7 +846,12 @@ mdast_cli \ ### Long-running Scans -Use `--long_wait` to extend the maximum wait time to 1 week (instead of default timeout). +Use `--long_wait` to extend the maximum wait time to ~1 week (instead of the default +~1 hour timeout). + +If a scan still has not reached a terminal state when the wait budget is exhausted, +the CLI **fails** with `SCAN_FAILED` (5) rather than exiting 0 - a scan that never +finished is never reported as success. **Use Case:** - Very long testcases @@ -784,6 +875,10 @@ mdast_cli \ Use `--appium_script_path` to provide a custom Appium script for automated testing. +> **Monolith installation only.** On the microservices installation +> `--appium_script_path` exits with `INVALID_ARGS` (2); use a recorded test case +> (`--testcase_id`) instead. + **Parameter:** `--appium_script_path ` **Use Case:** @@ -898,16 +993,23 @@ mdast_cli -d \ ## Exit Codes -The script uses standardized exit codes for CI/CD integration: +The script uses standardized exit codes for CI/CD integration. They are **identical +on both installations** (monolith and microservices), so pipelines behave the same +regardless of which platform they target: | Code | Constant | Description | |------|----------|-------------| | 0 | `SUCCESS` | Operation completed successfully | -| 1 | `INVALID_ARGS` | Invalid command-line arguments | -| 2 | `AUTH_ERROR` | Authentication failed | -| 3 | `DOWNLOAD_FAILED` | Application download failed | -| 4 | `NETWORK_ERROR` | Network/connection error | -| 5 | `SCAN_FAILED` | Scan execution or upload failed | +| 1 | `INTERNAL_ERROR` | Unexpected internal error | +| 2 | `INVALID_ARGS` | Invalid command-line arguments (also: Appium/CR report on the microservices installation) | +| 4 | `DOWNLOAD_FAILED` | Application download failed / file not found | +| 5 | `SCAN_FAILED` | Scan created but failed, no active engine, or finished in a non-success terminal state | +| 6 | `NETWORK_ERROR` | Network/connection error, gateway 5xx (incl. upload 502/503/504), non-JSON gateway response | +| 7 | `AUTH_ERROR` | Authentication/authorization error (bad or revoked token, 401/403) | +| 8 | `PRECHECK_BLOCKED` | Scan pre-check returned blocking warnings (microservices gate) | + +> There is no code 3. Codes 6/7/8 are meaningful mostly on the microservices +> installation but are used consistently wherever they apply. **Example CI/CD Usage:** ```bash @@ -915,15 +1017,13 @@ The script uses standardized exit codes for CI/CD integration: mdast_cli --distribution_system file --file_path app.apk ... EXIT_CODE=$? -if [ $EXIT_CODE -eq 0 ]; then - echo "Scan completed successfully" -elif [ $EXIT_CODE -eq 3 ]; then - echo "Download failed - check distribution system" - exit 1 -else - echo "Scan failed with code $EXIT_CODE" - exit 1 -fi +case $EXIT_CODE in + 0) echo "Scan completed successfully" ;; + 4) echo "Download failed - check distribution system"; exit 1 ;; + 6|7) echo "Infrastructure/auth problem (code $EXIT_CODE) - safe to retry"; exit 1 ;; + 8) echo "Scan blocked by pre-check warnings"; exit 1 ;; + *) echo "Scan failed with code $EXIT_CODE"; exit 1 ;; +esac ``` --- diff --git a/mdast_cli/__init__.py b/mdast_cli/__init__.py index e69de29..6633398 100644 --- a/mdast_cli/__init__.py +++ b/mdast_cli/__init__.py @@ -0,0 +1 @@ +__version__ = '2026.6.1' diff --git a/mdast_cli/distribution_systems/appgallery.py b/mdast_cli/distribution_systems/appgallery.py index 5827484..d2c5d4a 100644 --- a/mdast_cli/distribution_systems/appgallery.py +++ b/mdast_cli/distribution_systems/appgallery.py @@ -1,5 +1,4 @@ import logging -import os import time import requests diff --git a/mdast_cli/distribution_systems/base.py b/mdast_cli/distribution_systems/base.py index 30b8b6b..5f4a338 100644 --- a/mdast_cli/distribution_systems/base.py +++ b/mdast_cli/distribution_systems/base.py @@ -2,7 +2,7 @@ Base interface for all distribution system downloaders. """ from abc import ABC, abstractmethod -from typing import Dict, Optional, Any +from typing import Dict, Any class BaseDownloader(ABC): diff --git a/mdast_cli/distribution_systems/nexus2.py b/mdast_cli/distribution_systems/nexus2.py index 95354da..63434c4 100644 --- a/mdast_cli/distribution_systems/nexus2.py +++ b/mdast_cli/distribution_systems/nexus2.py @@ -29,7 +29,10 @@ def __init__(self, nexus_url, login, password): def download_app(self, download_path, repo_name, group_id, artifact_id, version, extension, file_name=''): download_url = f'{self.nexus_url}/service/local/artifact/maven/content?r={repo_name}&g={group_id}&a=' \ f'{artifact_id}&v={version}&p={extension}' - if file_name == '': + # `not file_name` (not `== ''`): mdast_scan passes arguments.nexus2_file_name, which is + # None when --nexus2_file_name is omitted (argparse default). The old `== ''` check missed + # None, so os.path.join(download_path, None) crashed the download on a valid invocation. + if not file_name: file_name = f'{artifact_id}-{version}.{extension}' headers = { 'DNT': '1', diff --git a/mdast_cli/distribution_systems/rumarket.py b/mdast_cli/distribution_systems/rumarket.py index ce7c32a..1466437 100644 --- a/mdast_cli/distribution_systems/rumarket.py +++ b/mdast_cli/distribution_systems/rumarket.py @@ -1,5 +1,4 @@ import logging -import os import requests from tqdm import tqdm diff --git a/mdast_cli/helpers/const.py b/mdast_cli/helpers/const.py index d6cc1c9..7fcb2ff 100644 --- a/mdast_cli/helpers/const.py +++ b/mdast_cli/helpers/const.py @@ -36,10 +36,58 @@ class DastState: # Timeout constants (in seconds) TRY = 360 -LONG_TRY = 20160 +LONG_TRY = 60480 # ~1 week at SLEEP_TIMEOUT=10s (matches --long_wait docs) END_SCAN_TIMEOUT = 30 SLEEP_TIMEOUT = 10 # HTTP timeout constants HTTP_REQUEST_TIMEOUT = 30 HTTP_DOWNLOAD_TIMEOUT = 300 + + +# --- Microservices installation (Clark facade, scanyon-native contract) --- + +class ScanStage: + CREATED = 'CREATED' + START = 'START' + WORKING = 'WORKING' + STOP = 'STOP' + SUCCESS = 'SUCCESS' + FAIL = 'FAIL' + + +class ScanStageStatus: + INITIAL = 'INITIAL' + PROCESSING = 'PROCESSING' + WAITING = 'WAITING' + COMPLETE = 'COMPLETE' + PARTIAL_COMPLETE = 'PARTIAL_COMPLETE' + FAIL = 'FAIL' + + +# Terminal (stage, status) pairs, confirmed with scanyon FSM (STG-4475). +# STOP is transitional: a stopped scan finishes as SUCCESS/COMPLETE or SUCCESS/PARTIAL_COMPLETE. +TERMINAL_SCAN_PAIRS = { + (ScanStage.SUCCESS, ScanStageStatus.COMPLETE), + (ScanStage.SUCCESS, ScanStageStatus.PARTIAL_COMPLETE), + (ScanStage.FAIL, ScanStageStatus.FAIL), +} +PRE_START_STAGES = {ScanStage.CREATED, ScanStage.START} +ACTIVE_STAGES = {ScanStage.WORKING, ScanStage.STOP} + +# Eucalyptus engine contract: platform in `type`, liveness in `status` (not `state`) +ENGINE_ACTIVE_STATUS = 'STARTED' +OS_ANDROID = 'ANDROID' +OS_IOS = 'IOS' + +# Installation mode selection (no new CLI flags by contract, env vars only) +MODE_ENV_VAR = 'MDAST_CLI_MODE' +UPLOAD_TIMEOUT_ENV_VAR = 'MDAST_UPLOAD_TIMEOUT' +TLS_VERIFY_ENV_VAR = 'MDAST_TLS_VERIFY' +MODE_AUTO = 'auto' +MODE_MONOLITH = 'monolith' +MODE_MICROSERVICES = 'microservices' + +# Clark upload contract (STG-4451): server-side upload_timeout query bounds +UPLOAD_TIMEOUT_MIN = 1 +UPLOAD_TIMEOUT_MAX = 300 diff --git a/mdast_cli/helpers/exit_codes.py b/mdast_cli/helpers/exit_codes.py index 81fd180..31506d3 100644 --- a/mdast_cli/helpers/exit_codes.py +++ b/mdast_cli/helpers/exit_codes.py @@ -24,7 +24,10 @@ class ExitCode(IntEnum): AUTH_ERROR = 7 """Authentication or authorization error.""" - + + PRECHECK_BLOCKED = 8 + """Scan pre-check returned blocking warnings (microservices installation gate).""" + INTERNAL_ERROR = 1 """Internal application error (default for unexpected errors).""" diff --git a/mdast_cli/helpers/helpers.py b/mdast_cli/helpers/helpers.py index 0bb9e78..c1cd6f8 100644 --- a/mdast_cli/helpers/helpers.py +++ b/mdast_cli/helpers/helpers.py @@ -14,3 +14,50 @@ def check_app_md5(file_path): while chunk := f.read(8192): file_hash.update(chunk) return file_hash.hexdigest() + + +def resolve_report_targets(arguments, scan_id): + """Decide which reports to produce and to which files, for BOTH installations. + + The report format is selected by ``--report_format`` (``pdf`` by default, so a + plain scan always yields a PDF). Precedence rules, kept identical for the + monolith and the microservices flow: + + * an explicit ``--pdf_report_file_name`` / ``--summary_report_json_file_name`` + always produces that format at that path (backward compatible); + * ``--report_format`` still applies on top, so ``--report_format all`` plus a + PDF file name yields the JSON report too (at a default name); + * with no file-name flags, ``--report_format`` alone decides + (``pdf`` | ``json`` | ``all`` | ``none``); + * default file names are derived from the scan id + (``scan_report_.pdf`` / ``.json``) so a report can never land at + a confusing server- or arg-derived name. + + Returns an ordered dict ``{'pdf': path, 'json': path}`` limited to the formats + that should be produced. PDF is inserted first (it is the primary artifact). + """ + pdf_name = arguments.pdf_report_file_name + json_name = arguments.summary_report_json_file_name + fmt = getattr(arguments, 'report_format', None) + + if pdf_name or json_name: + want_pdf = bool(pdf_name) + want_json = bool(json_name) + if fmt in ('pdf', 'all'): + want_pdf = True + if fmt in ('json', 'all'): + want_json = True + else: + effective = fmt or 'pdf' # nothing specified -> default PDF + want_pdf = effective in ('pdf', 'all') + want_json = effective in ('json', 'all') + + targets = {} + if want_pdf: + target = pdf_name or f'scan_report_{scan_id}.pdf' + # case-insensitive suffix check so report.PDF does not become report.PDF.pdf + targets['pdf'] = target if target.lower().endswith('.pdf') else f'{target}.pdf' + if want_json: + target = json_name or f'scan_report_{scan_id}.json' + targets['json'] = target if target.lower().endswith('.json') else f'{target}.json' + return targets diff --git a/mdast_cli/mdast_scan.py b/mdast_cli/mdast_scan.py index 1514128..6a10970 100644 --- a/mdast_cli/mdast_scan.py +++ b/mdast_cli/mdast_scan.py @@ -1,826 +1,937 @@ -import argparse -import asyncio -import json -import logging -import os -import sys -import time -import warnings - -import urllib3 - -# Suppress pkg_resources deprecation warning from google-auth -# This warning appears when google-auth is imported, even if not used -warnings.filterwarnings('ignore', message='.*pkg_resources is deprecated.*', category=UserWarning) - -from mdast_cli.distribution_systems.appgallery import appgallery_download_app -from mdast_cli.distribution_systems.appstore import AppStore -from mdast_cli.distribution_systems.firebase import firebase_download_app -from mdast_cli.distribution_systems.google_play import GooglePlay -from mdast_cli.distribution_systems import google_play_apkeep as gp_apkeep -from mdast_cli.distribution_systems.nexus import NexusRepository -from mdast_cli.distribution_systems.nexus2 import Nexus2Repository -from mdast_cli.distribution_systems.rumarket import rumarket_download_app -from mdast_cli.distribution_systems.rustore import rustore_download_app -from mdast_cli.helpers.const import (ANDROID_EXTENSIONS, DEFAULT_ANDROID_ARCHITECTURE, DEFAULT_IOS_ARCHITECTURE, - END_SCAN_TIMEOUT, LONG_TRY, SLEEP_TIMEOUT, TRY, DastState, DastStateDict) -from mdast_cli.helpers.exit_codes import ExitCode -from mdast_cli.helpers.helpers import check_app_md5 -from mdast_cli_core.token import mDastToken as mDast -from mdast_cli.cr_report_generator import generate_cr - -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s %(message)s', - datefmt='%d/%m/%Y %H:%M:%S', stream=sys.stdout) - -logger = logging.getLogger(__name__) - - -def parse_args(): - parser = argparse.ArgumentParser( - description='MDast CLI - tool for downloading and scanning mobile applications.', - epilog=''' -Usage examples: - # Download application from Google Play without scanning: - mdast_cli -d --distribution_system google_play \\ - --google_play_package_name com.example.app \\ - --google_play_email user@example.com \\ - --google_play_aas_token YOUR_TOKEN - - # Start application scanning: - mdast_cli --distribution_system file --file_path app.apk \\ - --url https://mdast.example.com --company_id 1 --token YOUR_TOKEN - -For detailed information about specific distribution system see README.md - ''', - formatter_class=argparse.RawDescriptionHelpFormatter - ) - - # Main options - main_group = parser.add_argument_group('Main Options', 'General parameters for utility operation') - main_group.add_argument('--download_only', '-d', action='store_true', - help='Download application without starting scan. ' - 'When using this option, --url, --company_id, --token parameters are not required. ' - 'After successful download, file path is printed in format DOWNLOAD_PATH=/path/to/file') - main_group.add_argument('--distribution_system', '-ds', type=str, - help='Application distribution system for downloading. ' - 'Available options: file, nexus, nexus2, firebase, appstore, google_play, ' - 'rustore, appgallery, rumarket. ' - 'System choice determines which additional parameters will be required.', - choices=['file', 'nexus', 'nexus2', 'firebase', 'appstore', 'google_play', - 'rustore', 'appgallery', 'rumarket'], - required=True) - main_group.add_argument('--download_path', '-p', type=str, - help='Path to directory for saving downloaded applications. ' - 'Default: downloaded_apps. ' - 'Directory will be created automatically if it does not exist.', - default='downloaded_apps') - main_group.add_argument('--verbose', '-v', action='store_true', - help='Enable debug logging (e.g. for troubleshooting download failures).') - - # File distribution system - file_group = parser.add_argument_group('Local File (file)', - 'Using existing local application file') - file_group.add_argument('--file_path', type=str, - help='Full path to local application file (APK for Android or IPA for iOS). ' - 'Required parameter when --distribution_system is set to "file". ' - 'Example: /path/to/app.apk or ./my_app.ipa') - - # Nexus distribution system - nexus_group = parser.add_argument_group('Nexus Repository (nexus)', - 'Downloading applications from Nexus Repository Manager 3.x') - nexus_group.add_argument('--nexus_url', type=str, - help='Nexus server URL (with http/https protocol). ' - 'Required parameter when --distribution_system is set to "nexus". ' - 'Example: https://nexus.example.com or http://localhost:8081') - nexus_group.add_argument('--nexus_login', type=str, - help='Username for Nexus authentication. ' - 'Required parameter when --distribution_system is set to "nexus".') - nexus_group.add_argument('--nexus_password', type=str, - help='Password for Nexus authentication. ' - 'Required parameter when --distribution_system is set to "nexus". ' - 'It is recommended to use environment variables for security.') - nexus_group.add_argument('--nexus_repo_name', type=str, - help='Repository name in Nexus where the application is located. ' - 'Required parameter when --distribution_system is set to "nexus". ' - 'Example: releases, snapshots, maven-releases') - nexus_group.add_argument('--nexus_group_id', type=str, - help='Application Group ID in Maven format (usually reverse domain). ' - 'Required parameter when --distribution_system is set to "nexus". ' - 'Example: com.example, org.mycompany') - nexus_group.add_argument('--nexus_artifact_id', type=str, - help='Application Artifact ID (artifact name). ' - 'Required parameter when --distribution_system is set to "nexus". ' - 'Example: myapp, mobile-app') - nexus_group.add_argument('--nexus_version', type=str, - help='Application version to download. ' - 'Required parameter when --distribution_system is set to "nexus". ' - 'Example: 1.0.0, 2.5.1, 1.0.0-SNAPSHOT') - - # Firebase distribution system - firebase_group = parser.add_argument_group('Firebase App Distribution (firebase)', - 'Downloading applications from Firebase App Distribution') - firebase_group.add_argument('--firebase_project_number', type=int, - help='Firebase project number (Project Number). ' - 'Required parameter when --distribution_system is set to "firebase". ' - 'Can be found in Firebase Console project settings. ' - 'Example: 123456789012') - firebase_group.add_argument('--firebase_app_id', type=str, - help='Application ID in Firebase in format: "1:PROJECT_NUMBER:PLATFORM:APP_ID". ' - 'Required parameter when --distribution_system is set to "firebase". ' - 'Examples: "1:123456789012:android:abc123def456", "1:123456789012:ios:xyz789"') - firebase_group.add_argument('--firebase_account_json_path', type=str, - help='Path to JSON file with Firebase service account credentials. ' - 'Required parameter when --distribution_system is set to "firebase". ' - 'File can be downloaded from Firebase Console → Project Settings → Service Accounts. ' - 'Example: /path/to/service-account.json or ./firebase-key.json') - firebase_group.add_argument('--firebase_file_extension', type=str, - help='Application file extension for download. ' - 'Required parameter when --distribution_system is set to "firebase". ' - 'Available values: apk (Android) or ipa (iOS).', - choices=['ipa', 'apk']) - firebase_group.add_argument('--firebase_file_name', type=str, - help='File name for saving application (without extension). ' - 'Optional parameter. If not specified, version from Firebase is used. ' - 'Example: my_app, production_build') - - # AppStore distribution system - appstore_group = parser.add_argument_group('Apple App Store (appstore)', - 'Downloading applications from Apple App Store') - appstore_group.add_argument('--appstore_app_id', type=str, - help='Application ID in App Store. Can be found in application page URL: ' - 'https://apps.apple.com/app/id{APP_ID}. ' - 'Either --appstore_app_id or --appstore_bundle_id must be specified. ' - 'Example: 389801252 (Instagram), 310633997 (WhatsApp)') - appstore_group.add_argument('--appstore_bundle_id', type=str, - help='Application Bundle ID (package identifier). ' - 'Either --appstore_app_id or --appstore_bundle_id must be specified. ' - 'Example: com.instagram.ios, com.whatsapp.WhatsApp') - appstore_group.add_argument('--appstore_apple_id', type=str, - help='Apple ID email address for iTunes/App Store login. ' - 'Required parameter when --distribution_system is set to "appstore". ' - 'Example: user@example.com') - appstore_group.add_argument('--appstore_password', type=str, - help='Apple ID password. ' - 'Required parameter when --distribution_system is set to "appstore". ' - 'It is recommended to use environment variables for security.') - appstore_group.add_argument('--appstore_2FA', type=str, - help='Two-factor authentication code (6 digits). ' - 'Required parameter when --distribution_system is set to "appstore". ' - 'Code is sent to trusted Apple devices. ' - 'Example: 123456') - appstore_group.add_argument('--appstore_file_name', type=str, - help='File name for saving application (without .ipa extension). ' - 'Optional parameter. If not specified, application name and version are used. ' - 'Example: my_app, instagram_latest') - appstore_group.add_argument('--appstore_country', type=str, default='US', - help='App Store country/region code for lookup and download (ISO 3166-1 alpha-2). ' - 'Default: US. Example: RU, DE, GB') - appstore_group.add_argument('--appstore_password2FA', type=str, - help='[DEPRECATED] Password and 2FA code in one parameter (format: password2FA_code). ' - 'Use --appstore_password and --appstore_2FA instead of this parameter. ' - 'Will be removed in future versions.', - metavar='DEPRECATED') - - # Google Play distribution system - google_play_group = parser.add_argument_group('Google Play (google_play)', - 'Downloading applications from Google Play Store using apkeep') - google_play_group.add_argument('--google_play_package_name', type=str, - help='Application package name from Google Play. ' - 'Required parameter when --distribution_system is set to "google_play". ' - 'Can be found in application page URL: ' - 'https://play.google.com/store/apps/details?id={PACKAGE_NAME}. ' - 'Examples: com.instagram.android, com.whatsapp, org.telegram.messenger') - google_play_group.add_argument('--google_play_email', type=str, - help='Google account email address. ' - 'Required parameter when --distribution_system is set to "google_play". ' - 'Used together with --google_play_aas_token or --google_play_oauth2_token. ' - 'Example: user@gmail.com') - google_play_group.add_argument('--google_play_aas_token', type=str, - help='AAS token for Google Play (obtained via apkeep). ' - 'Used for subsequent downloads after first token acquisition. ' - 'Requires --google_play_email. ' - 'Format: aas_et/... (long string). ' - 'For first run use --google_play_oauth2_token.') - google_play_group.add_argument('--google_play_oauth2_token', type=str, - help='OAuth2 token to obtain AAS token via apkeep. ' - 'Used on first run. After obtaining AAS token, you can use ' - '--google_play_aas_token for subsequent downloads. ' - 'Requires --google_play_email. ' - 'Format: ya29.a0AVvZVs... (OAuth2 access token)') - google_play_group.add_argument('--google_play_file_name', type=str, - help='File name for saving application (without extension). ' - 'Optional parameter. If not specified, package name is used. ' - 'Example: instagram_latest, whatsapp_production') - google_play_group.add_argument('--google_play_proxy', type=str, - help='Proxy settings for connecting to Google Play. ' - 'Optional parameter. ' - 'Format: socks5://user:pass@host:port or http://user:pass@host:port. ' - 'Example: socks5://proxy.example.com:1080') - - # RuStore distribution system - rustore_group = parser.add_argument_group('RuStore (rustore)', - 'Downloading applications from Russian RuStore marketplace') - rustore_group.add_argument('--rustore_package_name', type=str, - help='Application package name from RuStore. ' - 'Required parameter when --distribution_system is set to "rustore". ' - 'Can be found in application page URL. ' - 'Examples: com.vkontakte.android, com.yandex.browser') - - # RuMarket distribution system - rumarket_group = parser.add_argument_group('RuMarket (rumarket)', - 'Downloading applications from Russian RuMarket marketplace') - rumarket_group.add_argument('--rumarket_package_name', type=str, - help='Application package name from RuMarket. ' - 'Required parameter when --distribution_system is set to "rumarket". ' - 'Example: com.example.app') - - # AppGallery distribution system - appgallery_group = parser.add_argument_group('Huawei AppGallery (appgallery)', - 'Downloading applications from Huawei AppGallery') - appgallery_group.add_argument('--appgallery_app_id', type=str, - help='Application ID in AppGallery. ' - 'Required parameter when --distribution_system is set to "appgallery". ' - 'Can be found in application page URL: ' - 'https://appgallery.huawei.com/app/{APP_ID}. ' - 'Format: C + digits, e.g.: C101184875, C100000001') - appgallery_group.add_argument('--appgallery_file_name', type=str, - help='File name for saving application (without .apk extension). ' - 'Optional parameter. If not specified, package name and version are used. ' - 'Example: huawei_app, instagram_huawei') - - # Nexus2 distribution system - nexus2_group = parser.add_argument_group('Nexus Repository 2.x (nexus2)', - 'Downloading applications from Nexus Repository Manager 2.x') - nexus2_group.add_argument('--nexus2_url', type=str, - help='Nexus 2.x server URL (with http/https protocol and /nexus/ path). ' - 'Required parameter when --distribution_system is set to "nexus2". ' - 'Example: http://nexus.example.com:8081/nexus/ or http://localhost:8081/nexus/') - nexus2_group.add_argument('--nexus2_login', type=str, - help='Username for Nexus 2.x authentication. ' - 'Required parameter when --distribution_system is set to "nexus2".') - nexus2_group.add_argument('--nexus2_password', type=str, - help='Password for Nexus 2.x authentication. ' - 'Required parameter when --distribution_system is set to "nexus2". ' - 'It is recommended to use environment variables for security.') - nexus2_group.add_argument('--nexus2_repo_name', type=str, - help='Repository name in Nexus 2.x where the application is located. ' - 'Required parameter when --distribution_system is set to "nexus2". ' - 'Example: releases, snapshots') - nexus2_group.add_argument('--nexus2_group_id', type=str, - help='Application Group ID in Maven format. ' - 'Required parameter when --distribution_system is set to "nexus2". ' - 'Example: com.example, org.mycompany') - nexus2_group.add_argument('--nexus2_artifact_id', type=str, - help='Application Artifact ID. ' - 'Required parameter when --distribution_system is set to "nexus2". ' - 'Example: myapp, android-app') - nexus2_group.add_argument('--nexus2_version', type=str, - help='Application version to download. ' - 'Required parameter when --distribution_system is set to "nexus2". ' - 'Example: 1.0.0, 2.5.1, 1.0.0-SNAPSHOT') - nexus2_group.add_argument('--nexus2_extension', type=str, - help='Application file extension. ' - 'Required parameter when --distribution_system is set to "nexus2". ' - 'Usually: apk (Android) or ipa (iOS). ' - 'Example: apk, ipa, zip') - nexus2_group.add_argument('--nexus2_file_name', type=str, - help='File name for saving application (without extension). ' - 'Optional parameter. If not specified, generated automatically. ' - 'Example: my_app, production_build') - - # Scanning arguments - scan_group = parser.add_argument_group('Scanning Parameters', - 'Parameters for starting and managing application scanning') - scan_group.add_argument('--url', type=str, - help='MDast server URL for submitting application for scanning. ' - 'Required parameter when starting scan (without --download_only). ' - 'Example: https://mdast.example.com', - required=(not '-d')) - scan_group.add_argument('--company_id', type=int, - help='Company ID in MDast system. ' - 'Required parameter when starting scan (without --download_only). ' - 'Can be found in company settings in MDast web interface.', - required=(not '-d')) - scan_group.add_argument('--token', type=str, - help='CI/CD token for authentication and starting scan. ' - 'Required parameter when starting scan (without --download_only). ' - 'Token can be obtained in profile settings in MDast web interface. ' - 'It is recommended to use environment variables for security.', - required=(not '-d')) - scan_group.add_argument('--architecture_id', type=int, - help='Architecture ID for performing scan. ' - 'Optional parameter. If not specified, default architecture is used. ' - 'List of available architectures can be obtained via MDast API.') - scan_group.add_argument('--profile_id', type=int, default=None, - help='Profile ID for scanning. ' - 'Optional parameter. If not specified, profile will be created automatically.') - scan_group.add_argument('--project_id', type=int, default=None, - help='Project ID for scanning. ' - 'Optional parameter. Used only when auto-creating profile ' - 'to place new profile in existing project.') - scan_group.add_argument('--testcase_id', type=int, - help='Test case ID for automatic scanning. ' - 'Optional parameter. If not specified, manual scanning is performed.') - scan_group.add_argument('--appium_script_path', type=str, - help='Path to Appium script for automatic scanning using Stingray Appium. ' - 'Optional parameter. Used for automated testing.') - scan_group.add_argument('--summary_report_json_file_name', type=str, - help='File name for saving JSON report with scan results in structured format. ' - 'Optional parameter. If specified, report will be saved to specified file.') - scan_group.add_argument('--pdf_report_file_name', type=str, - help='File name for saving PDF report with scan results. ' - 'Optional parameter. If specified, PDF report will be saved to specified file.') - scan_group.add_argument('--nowait', '-nw', action='store_true', - help='Do not wait for scan completion. ' - 'If set, utility will start scan and exit immediately. ' - 'If not set, utility will wait for scan completion and output results.') - scan_group.add_argument('--long_wait', action='store_true', - help='Increase time limit for waiting scan completion to 1 week. ' - 'By default, standard timeout is used.') - - # CR Report arguments - cr_report_group = parser.add_argument_group('CR Report Parameters', - 'Parameters for generating Compliance Report (CR)') - cr_report_group.add_argument('--cr_report', action='store_true', - help='Enable CR report generation after scan completion. ' - 'CR report contains information about application compliance with security requirements.') - cr_report_group.add_argument('--stingray_login', type=str, - help='Login for accessing Stingray system for CR report generation. ' - 'Required parameter when using --cr_report.') - cr_report_group.add_argument('--stingray_password', type=str, - help='Password for accessing Stingray system for CR report generation. ' - 'Required parameter when using --cr_report. ' - 'It is recommended to use environment variables for security.') - cr_report_group.add_argument('--organization_name', type=str, - help='Organization name for CR report. ' - 'Default: ООО Стингрей Технолоджиз', - default='ООО Стингрей Технолоджиз') - cr_report_group.add_argument('--engineer_name', type=str, - help='Engineer name to be specified in CR report. ' - 'Optional parameter.') - cr_report_group.add_argument('--controller_name', type=str, - help='Controller name to be specified in CR report. ' - 'Optional parameter.') - cr_report_group.add_argument('--use_ldap', type=str, - help='Use LDAP for authentication when generating CR report. ' - 'Default: False. ' - 'Optional parameter.', - default=False) - cr_report_group.add_argument('--authority_server_id', type=str, - help='Authority server ID for CR report. ' - 'Optional parameter. Used when working with LDAP.', - default=None) - cr_report_group.add_argument('--cr_report_path', type=str, - help='Path for saving CR report. ' - 'Default: stingray-CR-report.html. ' - 'Optional parameter.', - default='stingray-CR-report.html') - - - - - - args = parser.parse_args() - - if args.distribution_system == 'file' and args.file_path is None: - parser.error('"--distribution_system file" requires "--file_path" argument to be set') - elif args.distribution_system == 'nexus' and ( - args.nexus_url is None or - args.nexus_login is None or - args.nexus_password is None or - args.nexus_repo_name is None or - args.nexus_group_id is None or - args.nexus_artifact_id is None or - args.nexus_version is None): - parser.error('"--distribution_system nexus" requires "--nexus_url", "--nexus_login", "--nexus_password",' - ' "--nexus_repo_name" arguments to be set') - - elif args.distribution_system == 'nexus2' and ( - args.nexus2_url is None or - args.nexus2_login is None or - args.nexus2_password is None or - args.nexus2_repo_name is None or - args.nexus2_group_id is None or - args.nexus2_artifact_id is None or - args.nexus2_version is None or - args.nexus2_extension is None): - parser.error('"--distribution_system nexus2" requires "--nexus2_url", "--nexus2_login", "--nexus2_password",' - ' "--nexus2_repo_name", "--nexus2_group_id", "--nexus2_artifact_id", "--nexus2_extension" ' - 'arguments to be set') - - elif args.distribution_system == 'firebase' and ( - args.firebase_project_number is None or - args.firebase_app_id is None or - args.firebase_account_json_path is None or - args.firebase_file_extension is None): - parser.error('"--distribution_system firebase" requires "--firebase_project_number", "--firebase_app_id", ' - '"--firebase_account_json_path", "--firebase_file_extension" arguments to be set') - - elif args.distribution_system == 'appstore' and ( - (args.appstore_app_id is None and args.appstore_bundle_id is None) or - args.appstore_apple_id is None or - (args.appstore_password is None or args.appstore_2FA is None) and args.appstore_password2FA is None): - parser.error('"--distribution_system appstore" requires either "--appstore_app_id" or "--appstore_bundle_id", ' - '"--appstore_apple_id" and ("--appstore_password" + "--appstore_2FA")/' - '(deprecated "--appstore_password2FA") arguments to be set') - - elif args.distribution_system == 'google_play': - if args.google_play_package_name is None: - parser.error('"--distribution_system google_play" requires "--google_play_package_name" to be set') - # 1) email + aas_token - # 2) email + oauth2_token (we will fetch aas_token automatically) - email_aas_ok = (args.google_play_email is not None and args.google_play_aas_token is not None) - email_oauth2_ok = (args.google_play_email is not None and args.google_play_oauth2_token is not None) - if not (email_aas_ok or email_oauth2_ok): - parser.error( - '"--distribution_system google_play" requires one of: ' - '(1) "--google_play_email" + "--google_play_aas_token", ' - '(2) "--google_play_email" + "--google_play_oauth2_token".' - ) - - elif args.distribution_system == 'rustore' and args.rustore_package_name is None: - parser.error('"--distribution_system rustore" requires "--rustore_package_name" to be set') - - elif args.distribution_system == 'rumarket' and args.rumarket_package_name is None: - parser.error('"--distribution_system rumarket" requires "--rumarket_package_name" to be set') - - elif args.distribution_system == 'appgallery' and args.appgallery_app_id is None: - parser.error('"--distribution_system appgallery" requires "--appgallery_app_id" to be set') - - return args - - -def main(): - urllib3.disable_warnings() - - arguments = parse_args() - - if getattr(arguments, 'verbose', False): - logging.getLogger().setLevel(logging.DEBUG) - logger.debug('Verbose (debug) logging enabled') - - distribution_system = arguments.distribution_system - download_path = arguments.download_path - - if arguments.download_only is False: - url = arguments.url - company_id = arguments.company_id - architecture = arguments.architecture_id - token = arguments.token - profile_id = arguments.profile_id - project_id = arguments.project_id - testcase_id = arguments.testcase_id - appium_script_path = arguments.appium_script_path - json_summary_file_name = arguments.summary_report_json_file_name - pdf_report_file_name = arguments.pdf_report_file_name - not_wait_scan_end = arguments.nowait - long_wait = arguments.long_wait - cr_report = arguments.cr_report - stingray_login = arguments.stingray_login - stingray_password = arguments.stingray_password - organization_name = arguments.organization_name - engineer_name = arguments.engineer_name - controller_name = arguments.controller_name - use_ldap = arguments.use_ldap - authority_server_id = arguments.authority_server_id - cr_report_path = arguments.cr_report_path - - - url = url if url.endswith('/') else f'{url}/' - url = url if url.endswith('rest/') else f'{url}rest' - - app_file = '' - appstore_app_md5 = None - - try: - if distribution_system == 'file': - app_file = arguments.file_path - - elif distribution_system == 'nexus': - nexus_repository = NexusRepository(arguments.nexus_url, - arguments.nexus_login, - arguments.nexus_password) - app_file = nexus_repository.download_app(download_path, - arguments.nexus_repo_name, - arguments.nexus_group_id, - arguments.nexus_artifact_id, - arguments.nexus_version) - elif distribution_system == 'nexus2': - nexus2_repository = Nexus2Repository(arguments.nexus2_url, - arguments.nexus2_login, - arguments.nexus2_password) - app_file = nexus2_repository.download_app(download_path, - arguments.nexus2_repo_name, - arguments.nexus2_group_id, - arguments.nexus2_artifact_id, - arguments.nexus2_version, - arguments.nexus2_extension, - arguments.nexus2_file_name) - - elif distribution_system == 'firebase': - app_file = firebase_download_app(download_path, - arguments.firebase_project_number, - arguments.firebase_app_id, - arguments.firebase_account_json_path, - arguments.firebase_file_name, - arguments.firebase_file_extension) - - elif distribution_system == 'appstore': - if arguments.appstore_password and arguments.appstore_2FA: - password2FA = arguments.appstore_password + arguments.appstore_2FA - else: - password2FA = arguments.appstore_password2FA - appstore = AppStore(arguments.appstore_apple_id, - password2FA) - app_file, appstore_app_md5 = appstore.download_app( - download_path, - arguments.appstore_app_id, - arguments.appstore_bundle_id, - arguments.appstore_country, - arguments.appstore_file_name, - ) - - elif distribution_system == 'google_play': - # Resolve AAS token from OAuth2 if provided and AAS not given - resolved_aas_token = arguments.google_play_aas_token - if (not resolved_aas_token) and arguments.google_play_oauth2_token: - try: - resolved_aas_token = asyncio.run( - gp_apkeep.fetch_aas_token( - email=arguments.google_play_email or '', - oauth2_token=arguments.google_play_oauth2_token, - timeout_sec=gp_apkeep.DEFAULT_TIMEOUT_SEC - ) - ) - except Exception as ex: - logger.error(f'Failed to obtain AAS token via OAuth2: {ex}') - sys.exit(ExitCode.AUTH_ERROR) - - google_play = GooglePlay(arguments.google_play_email, - resolved_aas_token) - google_play.login() - - app_file = google_play.download_app(download_path, - arguments.google_play_package_name, - arguments.google_play_file_name, proxy=arguments.google_play_proxy) - - elif distribution_system == 'rustore': - package_name = arguments.rustore_package_name - app_file = rustore_download_app(package_name, download_path) - - elif distribution_system == 'rumarket': - package_name = arguments.rumarket_package_name - app_file = rumarket_download_app(package_name, download_path) - - elif distribution_system == 'appgallery': - app_file = appgallery_download_app(arguments.appgallery_app_id, - download_path, - arguments.appgallery_file_name) - - except Exception as e: - logger.fatal( - 'Cannot download application file: %s (exception type: %s)', - e, - type(e).__name__, - ) - logger.exception('Download failed (traceback below)') - sys.exit(ExitCode.DOWNLOAD_FAILED) - - if arguments.download_only is True: - logger.info('Your application was downloaded!') - # Emit a single-line machine-friendly output for CI parsers - print(f'DOWNLOAD_PATH={app_file}') - sys.exit(ExitCode.SUCCESS) - - mdast = mDast(url, token, company_id) - get_architectures_resp = mdast.get_architectures() - - if get_architectures_resp.status_code != 200: - logger.error(f'Error while getting architectures. Server response: {get_architectures_resp.text}') - sys.exit(ExitCode.NETWORK_ERROR) - - architectures = get_architectures_resp.json() - - _, file_extension = os.path.splitext(app_file) - - if architecture is None: - if file_extension in ANDROID_EXTENSIONS: - architecture = next((arch['id'] for arch in architectures if arch.get('name', '') == DEFAULT_ANDROID_ARCHITECTURE), None) - if file_extension == '.ipa': - architecture = next((arch['id'] for arch in architectures if arch.get('name', '') == DEFAULT_IOS_ARCHITECTURE), None) - if architecture is None: - logger.error(f"Cannot create scan - no suitable architecture for this app, try to set it manually with --architecture_id") - sys.exit(ExitCode.INVALID_ARGS) - architecture_type = next(arch for arch in architectures if arch.get('id', '') == architecture) - logger.info(f'Architecture type is {architecture_type}') - - if testcase_id is not None: - get_testcase_resp = mdast.get_testcase(testcase_id) - if get_testcase_resp.status_code == 200: - architecture = get_testcase_resp.json()['architecture']['id'] - else: - logger.warning("Testcase with this id does not exist or you use old version of system. Trying to use " - "architecture from command line params.") - - if sum(e['architecture'] == architecture_type['id'] and e['state'] == 3 for e in mdast.get_engines().json()) == 0: - logger.error(f"Cannot create scan - Cannot find active engine for architecture {architecture_type['name']}") - sys.exit(ExitCode.SCAN_FAILED) - - if testcase_id: - logger.info(f'Autoscan(Stingray) with test case id: ' - f'{testcase_id}, profile id: {profile_id} and file: {app_file}, architecture id is {architecture}') - elif appium_script_path: - logger.info(f'Autoscan(Appium) with profile id: {profile_id} and file: {app_file},' - f' architecture id is {architecture}') - else: - logger.info(f'Manual scan with profile id: {profile_id} and file located in {app_file},' - f' architecture id is {architecture}') - - logger.info('Check if this version of application was already uploaded..') - if appstore_app_md5: - check_app_already_uploaded = mdast.check_app_md5(mdast.company_id, appstore_app_md5).json() - else: - check_app_already_uploaded = mdast.check_app_md5(mdast.company_id, check_app_md5(app_file)).json() - if check_app_already_uploaded: - application = check_app_already_uploaded[0] - logger.info(f"This app was uploaded before, application id is: {application['id']}, " - f"package name: {application['package_name']}," - f" version: {application['version_name']}, md5: {application['md5']}") - else: - logger.info('This is new application or new version') - logger.info('Uploading application to server..') - upload_application_resp = mdast.upload_application(app_file, str(architecture_type['type'])) - if upload_application_resp.status_code != 201: - logger.error(f'Error while uploading application to server: {upload_application_resp.text}') - sys.exit(ExitCode.SCAN_FAILED) - application = upload_application_resp.json() - logger.info(f"Application uploaded successfully. Application id: {application['id']}") - - logger.info(f"Creating scan for application {application['id']}") - if testcase_id is not None: - create_dast_resp = mdast.create_auto_scan(project_id, profile_id, application['id'], architecture, testcase_id) - scan_type = 'auto_stingray' - elif appium_script_path is not None: - create_dast_resp = mdast.create_appium_scan(project_id, profile_id, application['id'], architecture, - appium_script_path) - scan_type = 'auto_appium' - else: - create_dast_resp = mdast.create_manual_scan(project_id, profile_id, application['id'], architecture) - scan_type = 'manual' - if create_dast_resp.status_code != 201: - logger.error(f'Error while creating scan: {create_dast_resp.text}') - sys.exit(ExitCode.SCAN_FAILED) - - dast_info = create_dast_resp.json() - logger.info(f"Project and profile was created/found successfully." - f" Project id: {dast_info['project']['id']}, profile id: {dast_info['profile']['id']}") - - dast = create_dast_resp.json() - if 'id' not in dast or dast.get('id', '') == '': - logger.error(f'Something went wrong while creating scan: {dast}') - sys.exit(ExitCode.SCAN_FAILED) - - if scan_type == 'auto_stingray': - logger.info(f"Autoscan(Stingray) was created successfully. Scan id: {dast['id']}") - elif scan_type == 'auto_appium': - logger.info(f"Autoscan(Appium) was created successfully. Scan id: {dast['id']}") - else: - logger.info(f"Manual scan was created successfully. Scan id: {dast['id']}") - - logger.info(f"Start scan with id {dast['id']}") - start_dast_resp = mdast.start_scan(dast['id']) - if start_dast_resp.status_code != 200: - logger.error(f"Error while starting scan with id {dast['id']}: {start_dast_resp.text}") - sys.exit(ExitCode.SCAN_FAILED) - - if not_wait_scan_end: - logger.info('Scan successfully started. Don`t wait for end, exit with zero code') - sys.exit(ExitCode.SUCCESS) - - logger.info("Scan started successfully.") - logger.info(f"Checking scan state with id {dast['id']}") - get_dast_info_resp = mdast.get_scan_info(dast['id']) - if get_dast_info_resp.status_code != 200: - logger.error(f"Error while getting scan info with id {dast['id']}: {get_dast_info_resp.text}") - sys.exit(ExitCode.NETWORK_ERROR) - - dast = get_dast_info_resp.json() - dast_status = dast['state'] - logger.info(f"Current scan status: {DastStateDict.get(dast_status)}") - count = 0 - - if long_wait: - try_count = LONG_TRY - else: - try_count = TRY - - while dast_status in (DastState.CREATED, DastState.INITIALIZING, DastState.STARTING) and count < try_count: - logger.info(f"Try to get scan status for scan id {dast['id']}. Count number {count}") - get_dast_info_resp = mdast.get_scan_info(dast['id']) - if get_dast_info_resp.status_code != 200: - logger.error(f"Error while getting scan info with id {dast['id']}: {get_dast_info_resp.text}") - sys.exit(1) - - dast = get_dast_info_resp.json() - dast_status = dast['state'] - logger.info(f"Current scan status: {DastStateDict.get(dast_status)}") - count += 1 - if dast_status not in (DastState.STARTED, DastState.SUCCESS): - logger.info(f"Wait {SLEEP_TIMEOUT} seconds and try again") - time.sleep(SLEEP_TIMEOUT) - - if dast['state'] not in (DastState.STARTED, DastState.STOPPING, DastState.ANALYZING, DastState.SUCCESS): - logger.error(f"Error with scan id {dast['id']}. Current scan status: {dast['state']}," - f" but expected to be {DastState.STARTED}, {DastState.ANALYZING}, {DastState.STOPPING} " - f"or {DastState.SUCCESS}") - sys.exit(ExitCode.SCAN_FAILED) - logger.info(f"Scan {dast['id']} is started now. Let's wait until the scan is finished") - - get_dast_info_resp = mdast.get_scan_info(dast['id']) - if get_dast_info_resp.status_code != 200: - logger.error(f"Error while getting scan info with id {dast['id']}: {get_dast_info_resp.text}") - sys.exit(ExitCode.NETWORK_ERROR) - count = 0 - - while dast_status in (DastState.STARTED, DastState.STOPPING, DastState.ANALYZING) and count < try_count: - if count == 0 and scan_type == 'manual': - if dast_status not in (DastState.ANALYZING, DastState.SUCCESS): - logger.info(f"This is manual scan with dynamic modules," - f" lets wait for {END_SCAN_TIMEOUT} seconds and stop it.") - time.sleep(END_SCAN_TIMEOUT) - stop_manual_dast_resp = mdast.stop_scan(dast['id']) - if stop_manual_dast_resp.status_code == 200: - logger.info(f'Scan {dast["id"]} was successfully stopped') - else: - logger.error(f'Error while stopping scan with id {dast["id"]}: {stop_manual_dast_resp.text}') - sys.exit(ExitCode.SCAN_FAILED) - else: - logger.info("This is manual scan with profile without dynamic modules," - " only SAST, lets wait till the end") - - logger.info(f"Try to get scan status for scan id {dast['id']}. Count number {count}") - get_dast_info_resp = mdast.get_scan_info(dast['id']) - if get_dast_info_resp.status_code != 200: - logger.error(f"Error while getting scan info with id {dast['id']}: {get_dast_info_resp.text}") - sys.exit(1) - dast = get_dast_info_resp.json() - dast_status = dast['state'] - logger.info(f"Current scan status: {DastStateDict.get(dast_status)}") - count += 1 - if dast_status is not DastState.SUCCESS: - logger.info(f"Wait {SLEEP_TIMEOUT} seconds and try again") - time.sleep(SLEEP_TIMEOUT) - - logger.info(f"Check if scan with id {dast['id']} was finished correctly.") - get_dast_info_resp = mdast.get_scan_info(dast['id']) - if get_dast_info_resp.status_code != 200: - logger.error(f"Error while getting scan info with id {dast['id']}: {get_dast_info_resp.text}") - sys.exit(ExitCode.NETWORK_ERROR) - dast = get_dast_info_resp.json() - - if dast['state'] != DastState.SUCCESS: - logger.error( - f"Expected state {DastStateDict.get(DastState.SUCCESS)}, but in real it was {dast['state']}. " - f"Exit with error status code.") - sys.exit(ExitCode.SCAN_FAILED) - - if pdf_report_file_name: - logger.info(f"Create and download pdf report for scan with id {dast['id']} to file {pdf_report_file_name}.") - pdf_report = mdast.download_report(dast['id']) - if pdf_report.status_code != 200: - logger.error(f"PDF report creating failed with error {pdf_report.text}. Exit...") - sys.exit(ExitCode.SCAN_FAILED) - - logger.info(f"Saving pdf report to file {pdf_report_file_name}.") - pdf_report_file_name = pdf_report_file_name if pdf_report_file_name.endswith( - '.pdf') else f'{pdf_report_file_name}.pdf' - with open(pdf_report_file_name, 'wb') as f: - f.write(pdf_report.content) - - logger.info(f"Report for scan {dast['id']} successfully created and available at path: {pdf_report_file_name}.") - - if json_summary_file_name: - logger.info( - f"Download JSON summary report for scan with id {dast['id']} to file {json_summary_file_name}.") - json_summary_report = mdast.download_scan_json_result(dast['id']) - if json_summary_report.status_code != 200: - logger.error(f"JSON summary report while downloading failed with error {json_summary_report.text}. Exit...") - sys.exit(ExitCode.SCAN_FAILED) - - logger.info(f"Saving summary json results to file {json_summary_file_name}.") - mdast_json_file = json_summary_file_name if json_summary_file_name.endswith( - '.json') else f'{json_summary_file_name}.json' - with open(mdast_json_file, 'w') as fp: - json.dump(json_summary_report.json(), fp, indent=4, ensure_ascii=False) - - logger.info(f"JSON report for scan {dast['id']} successfully created and available at path: {mdast_json_file}.") - - if cr_report: - generate_cr(f"{url}", stingray_login, stingray_password, dast['id'], organization_name, engineer_name, - controller_name, cr_report_path, use_ldap, authority_server_id) - - logger.info('Job completed successfully!') - - -if __name__ == '__main__': - main() +import argparse +import asyncio +import json +import logging +import os +import sys +import time +import warnings + +import requests +import urllib3 + +# Suppress pkg_resources deprecation warning from google-auth +# This warning appears when google-auth is imported, even if not used +warnings.filterwarnings('ignore', message='.*pkg_resources is deprecated.*', category=UserWarning) + +from mdast_cli.distribution_systems.appgallery import appgallery_download_app +from mdast_cli.distribution_systems.appstore import AppStore +from mdast_cli.distribution_systems.firebase import firebase_download_app +from mdast_cli.distribution_systems.google_play import GooglePlay +from mdast_cli.distribution_systems import google_play_apkeep as gp_apkeep +from mdast_cli.distribution_systems.nexus import NexusRepository +from mdast_cli.distribution_systems.nexus2 import Nexus2Repository +from mdast_cli.distribution_systems.rumarket import rumarket_download_app +from mdast_cli.distribution_systems.rustore import rustore_download_app +from mdast_cli.helpers.const import (ANDROID_EXTENSIONS, DEFAULT_ANDROID_ARCHITECTURE, DEFAULT_IOS_ARCHITECTURE, + END_SCAN_TIMEOUT, LONG_TRY, SLEEP_TIMEOUT, TRY, DastState, DastStateDict) +from mdast_cli.helpers.exit_codes import ExitCode +from mdast_cli.helpers.helpers import check_app_md5, resolve_report_targets +from mdast_cli_core.token import mDastToken as mDast +from mdast_cli_core.factory import (MODE_MICROSERVICES, ModeDetectionError, resolve_installation_mode, + tls_verify_enabled) +from mdast_cli.cr_report_generator import generate_cr +from mdast_cli import __version__ + +USER_AGENT = f'mdast_cli/{__version__}' + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s %(message)s', + datefmt='%d/%m/%Y %H:%M:%S', stream=sys.stdout) + +logger = logging.getLogger(__name__) + + +def parse_args(): + parser = argparse.ArgumentParser( + description='MDast CLI - tool for downloading and scanning mobile applications.', + epilog=''' +Usage examples: + # Download application from Google Play without scanning: + mdast_cli -d --distribution_system google_play \\ + --google_play_package_name com.example.app \\ + --google_play_email user@example.com \\ + --google_play_aas_token YOUR_TOKEN + + # Start application scanning: + mdast_cli --distribution_system file --file_path app.apk \\ + --url https://mdast.example.com --company_id 1 --token YOUR_TOKEN + +For detailed information about specific distribution system see README.md + ''', + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + # Main options + main_group = parser.add_argument_group('Main Options', 'General parameters for utility operation') + main_group.add_argument('--download_only', '-d', action='store_true', + help='Download application without starting scan. ' + 'When using this option, --url, --company_id, --token parameters are not required. ' + 'After successful download, file path is printed in format DOWNLOAD_PATH=/path/to/file') + main_group.add_argument('--distribution_system', '-ds', type=str, + help='Application distribution system for downloading. ' + 'Available options: file, nexus, nexus2, firebase, appstore, google_play, ' + 'rustore, appgallery, rumarket. ' + 'System choice determines which additional parameters will be required.', + choices=['file', 'nexus', 'nexus2', 'firebase', 'appstore', 'google_play', + 'rustore', 'appgallery', 'rumarket'], + required=True) + main_group.add_argument('--download_path', '-p', type=str, + help='Path to directory for saving downloaded applications. ' + 'Default: downloaded_apps. ' + 'Directory will be created automatically if it does not exist.', + default='downloaded_apps') + main_group.add_argument('--verbose', '-v', action='store_true', + help='Enable debug logging (e.g. for troubleshooting download failures).') + + # File distribution system + file_group = parser.add_argument_group('Local File (file)', + 'Using existing local application file') + file_group.add_argument('--file_path', type=str, + help='Full path to local application file (APK for Android or IPA for iOS). ' + 'Required parameter when --distribution_system is set to "file". ' + 'Example: /path/to/app.apk or ./my_app.ipa') + + # Nexus distribution system + nexus_group = parser.add_argument_group('Nexus Repository (nexus)', + 'Downloading applications from Nexus Repository Manager 3.x') + nexus_group.add_argument('--nexus_url', type=str, + help='Nexus server URL (with http/https protocol). ' + 'Required parameter when --distribution_system is set to "nexus". ' + 'Example: https://nexus.example.com or http://localhost:8081') + nexus_group.add_argument('--nexus_login', type=str, + help='Username for Nexus authentication. ' + 'Required parameter when --distribution_system is set to "nexus".') + nexus_group.add_argument('--nexus_password', type=str, + help='Password for Nexus authentication. ' + 'Required parameter when --distribution_system is set to "nexus". ' + 'It is recommended to use environment variables for security.') + nexus_group.add_argument('--nexus_repo_name', type=str, + help='Repository name in Nexus where the application is located. ' + 'Required parameter when --distribution_system is set to "nexus". ' + 'Example: releases, snapshots, maven-releases') + nexus_group.add_argument('--nexus_group_id', type=str, + help='Application Group ID in Maven format (usually reverse domain). ' + 'Required parameter when --distribution_system is set to "nexus". ' + 'Example: com.example, org.mycompany') + nexus_group.add_argument('--nexus_artifact_id', type=str, + help='Application Artifact ID (artifact name). ' + 'Required parameter when --distribution_system is set to "nexus". ' + 'Example: myapp, mobile-app') + nexus_group.add_argument('--nexus_version', type=str, + help='Application version to download. ' + 'Required parameter when --distribution_system is set to "nexus". ' + 'Example: 1.0.0, 2.5.1, 1.0.0-SNAPSHOT') + + # Firebase distribution system + firebase_group = parser.add_argument_group('Firebase App Distribution (firebase)', + 'Downloading applications from Firebase App Distribution') + firebase_group.add_argument('--firebase_project_number', type=int, + help='Firebase project number (Project Number). ' + 'Required parameter when --distribution_system is set to "firebase". ' + 'Can be found in Firebase Console project settings. ' + 'Example: 123456789012') + firebase_group.add_argument('--firebase_app_id', type=str, + help='Application ID in Firebase in format: "1:PROJECT_NUMBER:PLATFORM:APP_ID". ' + 'Required parameter when --distribution_system is set to "firebase". ' + 'Examples: "1:123456789012:android:abc123def456", "1:123456789012:ios:xyz789"') + firebase_group.add_argument('--firebase_account_json_path', type=str, + help='Path to JSON file with Firebase service account credentials. ' + 'Required parameter when --distribution_system is set to "firebase". ' + 'File can be downloaded from Firebase Console → Project Settings → Service Accounts. ' + 'Example: /path/to/service-account.json or ./firebase-key.json') + firebase_group.add_argument('--firebase_file_extension', type=str, + help='Application file extension for download. ' + 'Required parameter when --distribution_system is set to "firebase". ' + 'Available values: apk (Android) or ipa (iOS).', + choices=['ipa', 'apk']) + firebase_group.add_argument('--firebase_file_name', type=str, + help='File name for saving application (without extension). ' + 'Optional parameter. If not specified, version from Firebase is used. ' + 'Example: my_app, production_build') + + # AppStore distribution system + appstore_group = parser.add_argument_group('Apple App Store (appstore)', + 'Downloading applications from Apple App Store') + appstore_group.add_argument('--appstore_app_id', type=str, + help='Application ID in App Store. Can be found in application page URL: ' + 'https://apps.apple.com/app/id{APP_ID}. ' + 'Either --appstore_app_id or --appstore_bundle_id must be specified. ' + 'Example: 389801252 (Instagram), 310633997 (WhatsApp)') + appstore_group.add_argument('--appstore_bundle_id', type=str, + help='Application Bundle ID (package identifier). ' + 'Either --appstore_app_id or --appstore_bundle_id must be specified. ' + 'Example: com.instagram.ios, com.whatsapp.WhatsApp') + appstore_group.add_argument('--appstore_apple_id', type=str, + help='Apple ID email address for iTunes/App Store login. ' + 'Required parameter when --distribution_system is set to "appstore". ' + 'Example: user@example.com') + appstore_group.add_argument('--appstore_password', type=str, + help='Apple ID password. ' + 'Required parameter when --distribution_system is set to "appstore". ' + 'It is recommended to use environment variables for security.') + appstore_group.add_argument('--appstore_2FA', type=str, + help='Two-factor authentication code (6 digits). ' + 'Required parameter when --distribution_system is set to "appstore". ' + 'Code is sent to trusted Apple devices. ' + 'Example: 123456') + appstore_group.add_argument('--appstore_file_name', type=str, + help='File name for saving application (without .ipa extension). ' + 'Optional parameter. If not specified, application name and version are used. ' + 'Example: my_app, instagram_latest') + appstore_group.add_argument('--appstore_country', type=str, default='US', + help='App Store country/region code for lookup and download (ISO 3166-1 alpha-2). ' + 'Default: US. Example: RU, DE, GB') + appstore_group.add_argument('--appstore_password2FA', type=str, + help='[DEPRECATED] Password and 2FA code in one parameter (format: password2FA_code). ' + 'Use --appstore_password and --appstore_2FA instead of this parameter. ' + 'Will be removed in future versions.', + metavar='DEPRECATED') + + # Google Play distribution system + google_play_group = parser.add_argument_group('Google Play (google_play)', + 'Downloading applications from Google Play Store using apkeep') + google_play_group.add_argument('--google_play_package_name', type=str, + help='Application package name from Google Play. ' + 'Required parameter when --distribution_system is set to "google_play". ' + 'Can be found in application page URL: ' + 'https://play.google.com/store/apps/details?id={PACKAGE_NAME}. ' + 'Examples: com.instagram.android, com.whatsapp, org.telegram.messenger') + google_play_group.add_argument('--google_play_email', type=str, + help='Google account email address. ' + 'Required parameter when --distribution_system is set to "google_play". ' + 'Used together with --google_play_aas_token or --google_play_oauth2_token. ' + 'Example: user@gmail.com') + google_play_group.add_argument('--google_play_aas_token', type=str, + help='AAS token for Google Play (obtained via apkeep). ' + 'Used for subsequent downloads after first token acquisition. ' + 'Requires --google_play_email. ' + 'Format: aas_et/... (long string). ' + 'For first run use --google_play_oauth2_token.') + google_play_group.add_argument('--google_play_oauth2_token', type=str, + help='OAuth2 token to obtain AAS token via apkeep. ' + 'Used on first run. After obtaining AAS token, you can use ' + '--google_play_aas_token for subsequent downloads. ' + 'Requires --google_play_email. ' + 'Format: ya29.a0AVvZVs... (OAuth2 access token)') + google_play_group.add_argument('--google_play_file_name', type=str, + help='File name for saving application (without extension). ' + 'Optional parameter. If not specified, package name is used. ' + 'Example: instagram_latest, whatsapp_production') + google_play_group.add_argument('--google_play_proxy', type=str, + help='Proxy settings for connecting to Google Play. ' + 'Optional parameter. ' + 'Format: socks5://user:pass@host:port or http://user:pass@host:port. ' + 'Example: socks5://proxy.example.com:1080') + + # RuStore distribution system + rustore_group = parser.add_argument_group('RuStore (rustore)', + 'Downloading applications from Russian RuStore marketplace') + rustore_group.add_argument('--rustore_package_name', type=str, + help='Application package name from RuStore. ' + 'Required parameter when --distribution_system is set to "rustore". ' + 'Can be found in application page URL. ' + 'Examples: com.vkontakte.android, com.yandex.browser') + + # RuMarket distribution system + rumarket_group = parser.add_argument_group('RuMarket (rumarket)', + 'Downloading applications from Russian RuMarket marketplace') + rumarket_group.add_argument('--rumarket_package_name', type=str, + help='Application package name from RuMarket. ' + 'Required parameter when --distribution_system is set to "rumarket". ' + 'Example: com.example.app') + + # AppGallery distribution system + appgallery_group = parser.add_argument_group('Huawei AppGallery (appgallery)', + 'Downloading applications from Huawei AppGallery') + appgallery_group.add_argument('--appgallery_app_id', type=str, + help='Application ID in AppGallery. ' + 'Required parameter when --distribution_system is set to "appgallery". ' + 'Can be found in application page URL: ' + 'https://appgallery.huawei.com/app/{APP_ID}. ' + 'Format: C + digits, e.g.: C101184875, C100000001') + appgallery_group.add_argument('--appgallery_file_name', type=str, + help='File name for saving application (without .apk extension). ' + 'Optional parameter. If not specified, package name and version are used. ' + 'Example: huawei_app, instagram_huawei') + + # Nexus2 distribution system + nexus2_group = parser.add_argument_group('Nexus Repository 2.x (nexus2)', + 'Downloading applications from Nexus Repository Manager 2.x') + nexus2_group.add_argument('--nexus2_url', type=str, + help='Nexus 2.x server URL (with http/https protocol and /nexus/ path). ' + 'Required parameter when --distribution_system is set to "nexus2". ' + 'Example: http://nexus.example.com:8081/nexus/ or http://localhost:8081/nexus/') + nexus2_group.add_argument('--nexus2_login', type=str, + help='Username for Nexus 2.x authentication. ' + 'Required parameter when --distribution_system is set to "nexus2".') + nexus2_group.add_argument('--nexus2_password', type=str, + help='Password for Nexus 2.x authentication. ' + 'Required parameter when --distribution_system is set to "nexus2". ' + 'It is recommended to use environment variables for security.') + nexus2_group.add_argument('--nexus2_repo_name', type=str, + help='Repository name in Nexus 2.x where the application is located. ' + 'Required parameter when --distribution_system is set to "nexus2". ' + 'Example: releases, snapshots') + nexus2_group.add_argument('--nexus2_group_id', type=str, + help='Application Group ID in Maven format. ' + 'Required parameter when --distribution_system is set to "nexus2". ' + 'Example: com.example, org.mycompany') + nexus2_group.add_argument('--nexus2_artifact_id', type=str, + help='Application Artifact ID. ' + 'Required parameter when --distribution_system is set to "nexus2". ' + 'Example: myapp, android-app') + nexus2_group.add_argument('--nexus2_version', type=str, + help='Application version to download. ' + 'Required parameter when --distribution_system is set to "nexus2". ' + 'Example: 1.0.0, 2.5.1, 1.0.0-SNAPSHOT') + nexus2_group.add_argument('--nexus2_extension', type=str, + help='Application file extension. ' + 'Required parameter when --distribution_system is set to "nexus2". ' + 'Usually: apk (Android) or ipa (iOS). ' + 'Example: apk, ipa, zip') + nexus2_group.add_argument('--nexus2_file_name', type=str, + help='File name for saving application (without extension). ' + 'Optional parameter. If not specified, generated automatically. ' + 'Example: my_app, production_build') + + # Scanning arguments + scan_group = parser.add_argument_group('Scanning Parameters', + 'Parameters for starting and managing application scanning') + scan_group.add_argument('--url', type=str, + help='MDast server URL for submitting application for scanning. ' + 'Required parameter when starting scan (without --download_only). ' + 'Example: https://mdast.example.com') + scan_group.add_argument('--company_id', type=int, + help='Company ID in MDast system. ' + 'Required parameter when starting scan (without --download_only). ' + 'Can be found in company settings in MDast web interface.') + scan_group.add_argument('--token', type=str, + help='CI/CD token for authentication and starting scan. ' + 'Required parameter when starting scan (without --download_only). ' + 'Token can be obtained in profile settings in MDast web interface. ' + 'It is recommended to use environment variables for security.') + scan_group.add_argument('--architecture_id', type=int, + help='Architecture ID for performing scan. ' + 'Optional parameter. If not specified, default architecture is used. ' + 'List of available architectures can be obtained via MDast API.') + scan_group.add_argument('--profile_id', type=int, default=None, + help='Profile ID for scanning. ' + 'Optional parameter. If not specified, profile will be created automatically.') + scan_group.add_argument('--project_id', type=int, default=None, + help='Project ID for scanning. ' + 'Optional parameter. Used only when auto-creating profile ' + 'to place new profile in existing project.') + scan_group.add_argument('--testcase_id', type=int, + help='Test case ID for automatic scanning. ' + 'Optional parameter. If not specified, manual scanning is performed.') + scan_group.add_argument('--appium_script_path', type=str, + help='Path to Appium script for automatic scanning using Stingray Appium. ' + 'Optional parameter. Used for automated testing.') + scan_group.add_argument('--report_format', type=str, + choices=['pdf', 'json', 'all', 'none'], default=None, + help='Which scan report(s) to download after a successful scan. ' + 'Works the same on both installations (monolith and microservices). ' + 'Default: pdf. Use "json" for the structured JSON summary, "all" for both, ' + '"none" to skip reports. When --pdf_report_file_name / ' + '--summary_report_json_file_name are given, those formats are always ' + 'produced (this flag can add the other one). Without explicit file names, ' + 'reports are saved as scan_report_.pdf / .json.') + scan_group.add_argument('--summary_report_json_file_name', type=str, + help='File name for saving JSON report with scan results in structured format. ' + 'Optional parameter. If specified, the JSON report will be saved to this file ' + '(implies --report_format json).') + scan_group.add_argument('--pdf_report_file_name', type=str, + help='File name for saving PDF report with scan results. ' + 'Optional parameter. If specified, the PDF report will be saved to this file ' + '(implies --report_format pdf).') + scan_group.add_argument('--nowait', '-nw', action='store_true', + help='Do not wait for scan completion. ' + 'If set, utility will start scan and exit immediately. ' + 'If not set, utility will wait for scan completion and output results.') + scan_group.add_argument('--long_wait', action='store_true', + help='Increase time limit for waiting scan completion to 1 week. ' + 'By default, standard timeout is used.') + + # CR Report arguments + cr_report_group = parser.add_argument_group('CR Report Parameters', + 'Parameters for generating Compliance Report (CR)') + cr_report_group.add_argument('--cr_report', action='store_true', + help='Enable CR report generation after scan completion. ' + 'CR report contains information about application compliance with security requirements.') + cr_report_group.add_argument('--stingray_login', type=str, + help='Login for accessing Stingray system for CR report generation. ' + 'Required parameter when using --cr_report.') + cr_report_group.add_argument('--stingray_password', type=str, + help='Password for accessing Stingray system for CR report generation. ' + 'Required parameter when using --cr_report. ' + 'It is recommended to use environment variables for security.') + cr_report_group.add_argument('--organization_name', type=str, + help='Organization name for CR report. ' + 'Default: ООО Стингрей Технолоджиз', + default='ООО Стингрей Технолоджиз') + cr_report_group.add_argument('--engineer_name', type=str, + help='Engineer name to be specified in CR report. ' + 'Optional parameter.') + cr_report_group.add_argument('--controller_name', type=str, + help='Controller name to be specified in CR report. ' + 'Optional parameter.') + cr_report_group.add_argument('--use_ldap', type=str, + help='Use LDAP for authentication when generating CR report. ' + 'Default: False. ' + 'Optional parameter.', + default=False) + cr_report_group.add_argument('--authority_server_id', type=str, + help='Authority server ID for CR report. ' + 'Optional parameter. Used when working with LDAP.', + default=None) + cr_report_group.add_argument('--cr_report_path', type=str, + help='Path for saving CR report. ' + 'Default: stingray-CR-report.html. ' + 'Optional parameter.', + default='stingray-CR-report.html') + + + + + + args = parser.parse_args() + + if args.distribution_system == 'file' and args.file_path is None: + parser.error('"--distribution_system file" requires "--file_path" argument to be set') + elif args.distribution_system == 'nexus' and ( + args.nexus_url is None or + args.nexus_login is None or + args.nexus_password is None or + args.nexus_repo_name is None or + args.nexus_group_id is None or + args.nexus_artifact_id is None or + args.nexus_version is None): + parser.error('"--distribution_system nexus" requires "--nexus_url", "--nexus_login", "--nexus_password",' + ' "--nexus_repo_name" arguments to be set') + + elif args.distribution_system == 'nexus2' and ( + args.nexus2_url is None or + args.nexus2_login is None or + args.nexus2_password is None or + args.nexus2_repo_name is None or + args.nexus2_group_id is None or + args.nexus2_artifact_id is None or + args.nexus2_version is None or + args.nexus2_extension is None): + parser.error('"--distribution_system nexus2" requires "--nexus2_url", "--nexus2_login", "--nexus2_password",' + ' "--nexus2_repo_name", "--nexus2_group_id", "--nexus2_artifact_id", "--nexus2_extension" ' + 'arguments to be set') + + elif args.distribution_system == 'firebase' and ( + args.firebase_project_number is None or + args.firebase_app_id is None or + args.firebase_account_json_path is None or + args.firebase_file_extension is None): + parser.error('"--distribution_system firebase" requires "--firebase_project_number", "--firebase_app_id", ' + '"--firebase_account_json_path", "--firebase_file_extension" arguments to be set') + + elif args.distribution_system == 'appstore' and ( + (args.appstore_app_id is None and args.appstore_bundle_id is None) or + args.appstore_apple_id is None or + (args.appstore_password is None or args.appstore_2FA is None) and args.appstore_password2FA is None): + parser.error('"--distribution_system appstore" requires either "--appstore_app_id" or "--appstore_bundle_id", ' + '"--appstore_apple_id" and ("--appstore_password" + "--appstore_2FA")/' + '(deprecated "--appstore_password2FA") arguments to be set') + + elif args.distribution_system == 'google_play': + if args.google_play_package_name is None: + parser.error('"--distribution_system google_play" requires "--google_play_package_name" to be set') + # 1) email + aas_token + # 2) email + oauth2_token (we will fetch aas_token automatically) + email_aas_ok = (args.google_play_email is not None and args.google_play_aas_token is not None) + email_oauth2_ok = (args.google_play_email is not None and args.google_play_oauth2_token is not None) + if not (email_aas_ok or email_oauth2_ok): + parser.error( + '"--distribution_system google_play" requires one of: ' + '(1) "--google_play_email" + "--google_play_aas_token", ' + '(2) "--google_play_email" + "--google_play_oauth2_token".' + ) + + elif args.distribution_system == 'rustore' and args.rustore_package_name is None: + parser.error('"--distribution_system rustore" requires "--rustore_package_name" to be set') + + elif args.distribution_system == 'rumarket' and args.rumarket_package_name is None: + parser.error('"--distribution_system rumarket" requires "--rumarket_package_name" to be set') + + elif args.distribution_system == 'appgallery' and args.appgallery_app_id is None: + parser.error('"--distribution_system appgallery" requires "--appgallery_app_id" to be set') + + # F13: при запуске скана (без --download_only) обязательны --url/--company_id/--token. + # Прежнее required=(not '-d') вычислялось в False и не работало. + if not args.download_only: + missing = [name for name, val in ( + ('--url', args.url), ('--company_id', args.company_id), ('--token', args.token), + ) if val in (None, '')] + if missing: + parser.error( + 'scanning requires ' + ', '.join(missing) + + ' (or use --download_only to only download the application)') + + # --cr_report (monolith-only) needs Stingray credentials; fail fast with a + # clear message instead of crashing mid-flow after the scan already ran. + if args.cr_report and (not args.stingray_login or not args.stingray_password): + parser.error('--cr_report requires --stingray_login and --stingray_password') + + return args + + +def _mono_http_error(resp, action, default_exit=ExitCode.SCAN_FAILED): + """Classify a failed monolith HTTP response and exit with the right code. + + 401/403 map to AUTH_ERROR(7) in BOTH installations (F12 parity with + ms_flow._exit_on_http_error); every other status uses the caller's default + (NETWORK_ERROR for read/poll calls, SCAN_FAILED for mutating calls). + """ + body = getattr(resp, 'text', '') + if resp.status_code in (401, 403): + logger.error(f'{action}: authorization failed (HTTP {resp.status_code}): {body}. ' + 'Check the CI/CD token.') + sys.exit(ExitCode.AUTH_ERROR) + logger.error(f'{action} (HTTP {resp.status_code}): {body}') + sys.exit(default_exit) + + +def run_monolith_flow(arguments, url, token, company_id, app_file, appstore_app_md5=None): + """Full scan flow against the monolith installation. Exits the process. + + Wrapped by main() in a requests.RequestException guard so a bare network error + surfaces as NETWORK_ERROR(6) - the same class the microservices flow uses - + instead of a traceback + INTERNAL_ERROR(1). Kept symmetric with + run_microservices_flow. + """ + architecture = arguments.architecture_id + profile_id = arguments.profile_id + project_id = arguments.project_id + testcase_id = arguments.testcase_id + appium_script_path = arguments.appium_script_path + not_wait_scan_end = arguments.nowait + long_wait = arguments.long_wait + + mdast = mDast(url, token, company_id) + get_architectures_resp = mdast.get_architectures() + if get_architectures_resp.status_code != 200: + _mono_http_error(get_architectures_resp, 'Error while getting architectures', + ExitCode.NETWORK_ERROR) + architectures = get_architectures_resp.json() + if not isinstance(architectures, list): + logger.error('Getting architectures: unexpected response shape (expected a list)') + sys.exit(ExitCode.NETWORK_ERROR) + + _, file_extension = os.path.splitext(app_file) + + if architecture is None: + if file_extension in ANDROID_EXTENSIONS: + architecture = next((arch.get('id') for arch in architectures + if isinstance(arch, dict) and arch.get('name') == DEFAULT_ANDROID_ARCHITECTURE), None) + if file_extension == '.ipa': + architecture = next((arch.get('id') for arch in architectures + if isinstance(arch, dict) and arch.get('name') == DEFAULT_IOS_ARCHITECTURE), None) + if architecture is None: + logger.error("Cannot create scan - no suitable architecture for this app, try to set it manually with --architecture_id") + sys.exit(ExitCode.INVALID_ARGS) + # An explicit --architecture_id that the server does not know is a user error, + # not an internal crash: guard next() with a default instead of StopIteration, + # and skip non-dict items so a malformed payload can't raise AttributeError. + architecture_type = next((arch for arch in architectures + if isinstance(arch, dict) and arch.get('id', '') == architecture), None) + if architecture_type is None: + logger.error(f"--architecture_id {architecture} is not available on this server. " + "Check the available architectures.") + sys.exit(ExitCode.INVALID_ARGS) + logger.info(f'Architecture type is {architecture_type}') + + if testcase_id is not None: + # Best-effort (preserved from the original monolith flow): the test-case + # lookup only refines the architecture. Any non-200 - including 401/403 - + # falls back to the CLI/auto architecture rather than aborting; the real + # authorization check happens on create_auto_scan below. + get_testcase_resp = mdast.get_testcase(testcase_id) + if get_testcase_resp.status_code == 200: + architecture = get_testcase_resp.json()['architecture']['id'] + else: + logger.warning("Testcase with this id does not exist or you use old version of system. Trying to use " + "architecture from command line params.") + + get_engines_resp = mdast.get_engines() + if get_engines_resp.status_code != 200: + _mono_http_error(get_engines_resp, 'Error while getting engines', ExitCode.NETWORK_ERROR) + engines = get_engines_resp.json() + if not isinstance(engines, list): + logger.error('Getting engines: unexpected response shape (expected a list)') + sys.exit(ExitCode.NETWORK_ERROR) + if sum(isinstance(e, dict) and e.get('architecture') == architecture_type['id'] and e.get('state') == 3 + for e in engines) == 0: + logger.error(f"Cannot create scan - Cannot find active engine for architecture {architecture_type['name']}") + sys.exit(ExitCode.SCAN_FAILED) + + if testcase_id: + logger.info(f'Autoscan(Stingray) with test case id: ' + f'{testcase_id}, profile id: {profile_id} and file: {app_file}, architecture id is {architecture}') + elif appium_script_path: + logger.info(f'Autoscan(Appium) with profile id: {profile_id} and file: {app_file},' + f' architecture id is {architecture}') + else: + logger.info(f'Manual scan with profile id: {profile_id} and file located in {app_file},' + f' architecture id is {architecture}') + + logger.info('Check if this version of application was already uploaded..') + dedup_md5 = appstore_app_md5 if appstore_app_md5 else check_app_md5(app_file) + dedup_resp = mdast.check_app_md5(mdast.company_id, dedup_md5) + if dedup_resp.status_code != 200: + _mono_http_error(dedup_resp, 'Application dedup check', ExitCode.NETWORK_ERROR) + check_app_already_uploaded = dedup_resp.json() + if isinstance(check_app_already_uploaded, list) and check_app_already_uploaded: + application = check_app_already_uploaded[0] + logger.info(f"This app was uploaded before, application id is: {application.get('id')}, " + f"package name: {application.get('package_name')}," + f" version: {application.get('version_name')}, md5: {application.get('md5')}") + else: + logger.info('This is new application or new version') + logger.info('Uploading application to server..') + upload_application_resp = mdast.upload_application(app_file, str(architecture_type['type'])) + if upload_application_resp.status_code != 201: + _mono_http_error(upload_application_resp, 'Error while uploading application to server') + application = upload_application_resp.json() + logger.info(f"Application uploaded successfully. Application id: {application['id']}") + + logger.info(f"Creating scan for application {application['id']}") + if testcase_id is not None: + create_dast_resp = mdast.create_auto_scan(project_id, profile_id, application['id'], architecture, testcase_id) + scan_type = 'auto_stingray' + elif appium_script_path is not None: + create_dast_resp = mdast.create_appium_scan(project_id, profile_id, application['id'], architecture, + appium_script_path) + scan_type = 'auto_appium' + else: + create_dast_resp = mdast.create_manual_scan(project_id, profile_id, application['id'], architecture) + scan_type = 'manual' + if create_dast_resp.status_code != 201: + _mono_http_error(create_dast_resp, 'Error while creating scan') + + dast = create_dast_resp.json() + logger.info(f"Project and profile was created/found successfully." + f" Project id: {(dast.get('project') or {}).get('id')}, " + f"profile id: {(dast.get('profile') or {}).get('id')}") + + if 'id' not in dast or dast.get('id', '') == '': + logger.error(f'Something went wrong while creating scan: {dast}') + sys.exit(ExitCode.SCAN_FAILED) + + if scan_type == 'auto_stingray': + logger.info(f"Autoscan(Stingray) was created successfully. Scan id: {dast['id']}") + elif scan_type == 'auto_appium': + logger.info(f"Autoscan(Appium) was created successfully. Scan id: {dast['id']}") + else: + logger.info(f"Manual scan was created successfully. Scan id: {dast['id']}") + + logger.info(f"Start scan with id {dast['id']}") + start_dast_resp = mdast.start_scan(dast['id']) + if start_dast_resp.status_code != 200: + _mono_http_error(start_dast_resp, f"Error while starting scan with id {dast['id']}") + + if not_wait_scan_end: + logger.info('Scan successfully started. Don`t wait for end, exit with zero code') + sys.exit(ExitCode.SUCCESS) + + logger.info("Scan started successfully.") + logger.info(f"Checking scan state with id {dast['id']}") + get_dast_info_resp = mdast.get_scan_info(dast['id']) + if get_dast_info_resp.status_code != 200: + _mono_http_error(get_dast_info_resp, f"Error while getting scan info with id {dast['id']}", + ExitCode.NETWORK_ERROR) + + dast = get_dast_info_resp.json() + dast_status = dast['state'] + logger.info(f"Current scan status: {DastStateDict.get(dast_status)}") + count = 0 + + try_count = LONG_TRY if long_wait else TRY + + while dast_status in (DastState.CREATED, DastState.INITIALIZING, DastState.STARTING) and count < try_count: + logger.info(f"Try to get scan status for scan id {dast['id']}. Count number {count}") + get_dast_info_resp = mdast.get_scan_info(dast['id']) + if get_dast_info_resp.status_code != 200: + _mono_http_error(get_dast_info_resp, f"Error while getting scan info with id {dast['id']}", + ExitCode.NETWORK_ERROR) + + dast = get_dast_info_resp.json() + dast_status = dast['state'] + logger.info(f"Current scan status: {DastStateDict.get(dast_status)}") + count += 1 + if dast_status not in (DastState.STARTED, DastState.SUCCESS): + logger.info(f"Wait {SLEEP_TIMEOUT} seconds and try again") + time.sleep(SLEEP_TIMEOUT) + + if dast['state'] not in (DastState.STARTED, DastState.STOPPING, DastState.ANALYZING, DastState.SUCCESS): + logger.error(f"Error with scan id {dast['id']}. Current scan status: {dast['state']}," + f" but expected to be {DastState.STARTED}, {DastState.ANALYZING}, {DastState.STOPPING} " + f"or {DastState.SUCCESS}") + sys.exit(ExitCode.SCAN_FAILED) + logger.info(f"Scan {dast['id']} is started now. Let's wait until the scan is finished") + + get_dast_info_resp = mdast.get_scan_info(dast['id']) + if get_dast_info_resp.status_code != 200: + _mono_http_error(get_dast_info_resp, f"Error while getting scan info with id {dast['id']}", + ExitCode.NETWORK_ERROR) + count = 0 + + while dast_status in (DastState.STARTED, DastState.STOPPING, DastState.ANALYZING) and count < try_count: + if count == 0 and scan_type == 'manual': + if dast_status not in (DastState.ANALYZING, DastState.SUCCESS): + logger.info(f"This is manual scan with dynamic modules," + f" lets wait for {END_SCAN_TIMEOUT} seconds and stop it.") + time.sleep(END_SCAN_TIMEOUT) + stop_manual_dast_resp = mdast.stop_scan(dast['id']) + if stop_manual_dast_resp.status_code == 200: + logger.info(f'Scan {dast["id"]} was successfully stopped') + else: + _mono_http_error(stop_manual_dast_resp, f'Error while stopping scan with id {dast["id"]}') + else: + logger.info("This is manual scan with profile without dynamic modules," + " only SAST, lets wait till the end") + + logger.info(f"Try to get scan status for scan id {dast['id']}. Count number {count}") + get_dast_info_resp = mdast.get_scan_info(dast['id']) + if get_dast_info_resp.status_code != 200: + _mono_http_error(get_dast_info_resp, f"Error while getting scan info with id {dast['id']}", + ExitCode.NETWORK_ERROR) + dast = get_dast_info_resp.json() + dast_status = dast['state'] + logger.info(f"Current scan status: {DastStateDict.get(dast_status)}") + count += 1 + if dast_status is not DastState.SUCCESS: + logger.info(f"Wait {SLEEP_TIMEOUT} seconds and try again") + time.sleep(SLEEP_TIMEOUT) + + logger.info(f"Check if scan with id {dast['id']} was finished correctly.") + get_dast_info_resp = mdast.get_scan_info(dast['id']) + if get_dast_info_resp.status_code != 200: + _mono_http_error(get_dast_info_resp, f"Error while getting scan info with id {dast['id']}", + ExitCode.NETWORK_ERROR) + dast = get_dast_info_resp.json() + + if dast['state'] != DastState.SUCCESS: + logger.error( + f"Expected state {DastStateDict.get(DastState.SUCCESS)}, but in real it was {dast['state']}. " + f"Exit with error status code.") + sys.exit(ExitCode.SCAN_FAILED) + + # Report selection is shared with the microservices flow: --report_format + # (default pdf) plus explicit file-name flags, with scan-id default names. + # Monolith keeps the report download as a hard-fail (unlike the soft-fail + # microservices path, where the report service is a separate component). + report_targets = resolve_report_targets(arguments, dast['id']) + + if 'pdf' in report_targets: + pdf_path = report_targets['pdf'] + logger.info(f"Create and download pdf report for scan with id {dast['id']} to file {pdf_path}.") + pdf_report = mdast.download_report(dast['id']) + if pdf_report.status_code != 200: + _mono_http_error(pdf_report, 'PDF report creating failed') + with open(pdf_path, 'wb') as f: + f.write(pdf_report.content) + logger.info(f"Report for scan {dast['id']} successfully created and available at path: {pdf_path}.") + + if 'json' in report_targets: + json_path = report_targets['json'] + logger.info(f"Download JSON summary report for scan with id {dast['id']} to file {json_path}.") + json_summary_report = mdast.download_scan_json_result(dast['id']) + if json_summary_report.status_code != 200: + _mono_http_error(json_summary_report, 'JSON summary report download') + # Parse BEFORE opening the file, so a 200 with a non-JSON body neither + # crashes with a traceback nor leaves a 0-byte invalid .json behind. + try: + payload = json_summary_report.json() + except ValueError: + logger.error('JSON summary report: server returned HTTP 200 with a non-JSON body. Exit...') + sys.exit(ExitCode.SCAN_FAILED) + with open(json_path, 'w') as fp: + json.dump(payload, fp, indent=4, ensure_ascii=False) + logger.info(f"JSON report for scan {dast['id']} successfully created and available at path: {json_path}.") + + if arguments.cr_report: + generate_cr(f"{url}", arguments.stingray_login, arguments.stingray_password, dast['id'], + arguments.organization_name, arguments.engineer_name, arguments.controller_name, + arguments.cr_report_path, arguments.use_ldap, arguments.authority_server_id) + + +def main(): + urllib3.disable_warnings() + + arguments = parse_args() + + if getattr(arguments, 'verbose', False): + logging.getLogger().setLevel(logging.DEBUG) + logger.debug('Verbose (debug) logging enabled') + + distribution_system = arguments.distribution_system + download_path = arguments.download_path + + if arguments.download_only is False: + url = arguments.url + company_id = arguments.company_id + token = arguments.token + + # Canonical base URL: strip trailing slashes, then ensure it ends with + # '/rest' (both installations serve the CLI routes under /rest). All + # clients build request paths as f'{url}/...', so a trailing slash here + # would produce '//' and can 404 on a strict Envoy/OPA path match. The + # previous two-step form left a trailing slash when --url already + # contained 'rest', breaking the monolith and the mode probe alike. + url = url.rstrip('/') + if not url.endswith('/rest'): + url = f'{url}/rest' + + app_file = '' + appstore_app_md5 = None + + try: + if distribution_system == 'file': + app_file = arguments.file_path + if not app_file or not os.path.isfile(app_file): + # A bad --file_path is a user error, not a download failure; fail + # fast with INVALID_ARGS instead of a later md5/open traceback. + # (SystemExit is not caught by the `except Exception` below.) + logger.error(f'--file_path does not point to an existing file: {app_file!r}') + sys.exit(ExitCode.INVALID_ARGS) + + elif distribution_system == 'nexus': + nexus_repository = NexusRepository(arguments.nexus_url, + arguments.nexus_login, + arguments.nexus_password) + app_file = nexus_repository.download_app(download_path, + arguments.nexus_repo_name, + arguments.nexus_group_id, + arguments.nexus_artifact_id, + arguments.nexus_version) + elif distribution_system == 'nexus2': + nexus2_repository = Nexus2Repository(arguments.nexus2_url, + arguments.nexus2_login, + arguments.nexus2_password) + app_file = nexus2_repository.download_app(download_path, + arguments.nexus2_repo_name, + arguments.nexus2_group_id, + arguments.nexus2_artifact_id, + arguments.nexus2_version, + arguments.nexus2_extension, + arguments.nexus2_file_name) + + elif distribution_system == 'firebase': + app_file = firebase_download_app(download_path, + arguments.firebase_project_number, + arguments.firebase_app_id, + arguments.firebase_account_json_path, + arguments.firebase_file_name, + arguments.firebase_file_extension) + + elif distribution_system == 'appstore': + if arguments.appstore_password and arguments.appstore_2FA: + password2FA = arguments.appstore_password + arguments.appstore_2FA + else: + password2FA = arguments.appstore_password2FA + appstore = AppStore(arguments.appstore_apple_id, + password2FA) + app_file, appstore_app_md5 = appstore.download_app( + download_path, + arguments.appstore_app_id, + arguments.appstore_bundle_id, + arguments.appstore_country, + arguments.appstore_file_name, + ) + + elif distribution_system == 'google_play': + # Resolve AAS token from OAuth2 if provided and AAS not given + resolved_aas_token = arguments.google_play_aas_token + if (not resolved_aas_token) and arguments.google_play_oauth2_token: + try: + resolved_aas_token = asyncio.run( + gp_apkeep.fetch_aas_token( + email=arguments.google_play_email or '', + oauth2_token=arguments.google_play_oauth2_token, + timeout_sec=gp_apkeep.DEFAULT_TIMEOUT_SEC + ) + ) + except Exception as ex: + logger.error(f'Failed to obtain AAS token via OAuth2: {ex}') + sys.exit(ExitCode.AUTH_ERROR) + + google_play = GooglePlay(arguments.google_play_email, + resolved_aas_token) + google_play.login() + + app_file = google_play.download_app(download_path, + arguments.google_play_package_name, + arguments.google_play_file_name, proxy=arguments.google_play_proxy) + + elif distribution_system == 'rustore': + package_name = arguments.rustore_package_name + app_file = rustore_download_app(package_name, download_path) + + elif distribution_system == 'rumarket': + package_name = arguments.rumarket_package_name + app_file = rumarket_download_app(package_name, download_path) + + elif distribution_system == 'appgallery': + app_file = appgallery_download_app(arguments.appgallery_app_id, + download_path, + arguments.appgallery_file_name) + + except Exception as e: + logger.fatal( + 'Cannot download application file: %s (exception type: %s)', + e, + type(e).__name__, + ) + logger.exception('Download failed (traceback below)') + sys.exit(ExitCode.DOWNLOAD_FAILED) + + # Defensive guard: a downloader must have produced a real file before we try + # to md5/upload it. Catches a downloader that returned an empty/bogus path + # without raising, so the failure is a clear DOWNLOAD_FAILED, not a traceback. + if not app_file or not os.path.isfile(app_file): + logger.error(f'Downloaded application file was not found on disk: {app_file!r}') + sys.exit(ExitCode.DOWNLOAD_FAILED) + + if arguments.download_only is True: + logger.info('Your application was downloaded!') + # Emit a single-line machine-friendly output for CI parsers + print(f'DOWNLOAD_PATH={app_file}') + sys.exit(ExitCode.SUCCESS) + + try: + installation_mode = resolve_installation_mode(url, token, company_id, + verify=tls_verify_enabled()) + except ModeDetectionError as ex: + logger.error(str(ex)) + sys.exit(ExitCode.AUTH_ERROR if ex.auth_error else ExitCode.NETWORK_ERROR) + + if installation_mode == MODE_MICROSERVICES: + tls_verify = tls_verify_enabled() + if not tls_verify: + logger.warning('TLS verification is DISABLED (MDAST_TLS_VERIFY): the organization ' + 'CLI token is sent over an unverified connection - use only on trusted ' + 'networks / self-signed stands.') + from mdast_cli.ms_flow import run_microservices_flow + run_microservices_flow(arguments, url, token, app_file, + user_agent=USER_AGENT, verify=tls_verify) + logger.info('Job completed successfully!') + sys.exit(ExitCode.SUCCESS) + + try: + run_monolith_flow(arguments, url, token, company_id, app_file, appstore_app_md5) + except requests.RequestException as ex: + logger.error(f'Network error talking to the monolith installation ' + f'({type(ex).__name__}): {ex}') + sys.exit(ExitCode.NETWORK_ERROR) + + logger.info('Job completed successfully!') + sys.exit(ExitCode.SUCCESS) + + +if __name__ == '__main__': + main() diff --git a/mdast_cli/ms_flow.py b/mdast_cli/ms_flow.py new file mode 100644 index 0000000..7ffa9e9 --- /dev/null +++ b/mdast_cli/ms_flow.py @@ -0,0 +1,586 @@ +"""Scan flow for the microservices installation (Clark facade). + +Mirrors the monolith flow of mdast_scan.main() with the scanyon-native +contract (STG-4451 / STG-4475 / STG-4478): + + engines preflight -> dedup by md5 -> upload -> pre-check gate -> + create scan (fsm_locked) -> start -> poll (stage, status) -> reports. + +The public CLI interface is intentionally unchanged: same flags, same +report file arguments, `--architecture_id`/`--company_id` are accepted +and ignored here. +""" +import json +import logging +import os +import re +import sys +import time + +import requests + +from mdast_cli.helpers.const import (ACTIVE_STAGES, ANDROID_EXTENSIONS, END_SCAN_TIMEOUT, LONG_TRY, OS_ANDROID, + OS_IOS, PRE_START_STAGES, SLEEP_TIMEOUT, TERMINAL_SCAN_PAIRS, TRY, + UPLOAD_TIMEOUT_ENV_VAR, UPLOAD_TIMEOUT_MAX, UPLOAD_TIMEOUT_MIN, + ENGINE_ACTIVE_STATUS, ScanStage, ScanStageStatus) +from mdast_cli.helpers.exit_codes import ExitCode +from mdast_cli.helpers.helpers import check_app_md5, resolve_report_targets +from mdast_cli_core.microservices import extract_error_message, mDastMicroservices + +logger = logging.getLogger(__name__) + +KNOWN_PRECHECK_WARNINGS = { + 'empty_profile': 'Scan profile has no enabled modules', + 'no_compatible_devices': 'No compatible devices for the required OS version', + 'application_not_found': 'Application is not found on the platform', + 'precheck_unavailable': 'Pre-check dependencies are temporarily unavailable, retry later', + 'architecture_unsupported': 'Platform/OS version pair is not supported by this installation', +} + +# Transient downstream failures (facade/scanyon/pepper) are retried while polling +# and downloading, so a rolling restart of a backend mid-scan does not kill a +# long CI job. ~5 min budget per call covers a typical k8s rolling restart. +# 429 (Too Many Requests) is included: scanyon rate-limits with a 429, and backing +# off is the correct response, not an immediate abort. +POLL_TRANSIENT_RETRIES = 30 +POLL_TRANSIENT_CODES = (429, 502, 503, 504) +# upload has its own (shorter) transient budget: a busy/redeploying/rate-limited +# upload path returns 429/502/503, worth a few retries, but not the full 5 min. +UPLOAD_TRANSIENT_RETRIES = 3 +UPLOAD_TRANSIENT_CODES = (429, 502, 503) + +# Strip C0/C1 control chars from server-controlled strings before logging, keeping +# only TAB (\x09). CRUCIALLY this includes LF (\x0a) and CR (\x0d): otherwise a +# hostile/compromised server could embed a newline in `message`/`package_name` and +# forge an extra log line in CI output (CWE-117 log injection). +_CONTROL_CHARS_RE = re.compile(r'[\x00-\x08\x0a-\x1f\x7f-\x9f]') + + +def sanitize(value): + """Strip control/ANSI chars (incl. CR/LF) from server strings before logging. + + A hostile/compromised server could embed ANSI escapes or CR/LF into fields + like `message`/`package_name` to forge log lines / poison CI logs; neutralise + them. Only TAB survives from the control range. + """ + return _CONTROL_CHARS_RE.sub('', str(value)) + + +def resolve_platform(app_file): + """ANDROID/IOS from the application file extension.""" + _, extension = os.path.splitext(app_file) + if extension.lower() in ANDROID_EXTENSIONS: + return OS_ANDROID + if extension.lower() == '.ipa': + return OS_IOS + return None + + +def render_precheck_warning(warning): + """One human-readable line per pre-check warning: known text + raw payload. + + Unknown warning types are rendered generically: the contract explicitly + allows backward-compatible extension of the type set (DEC-666-04). Payload is + JSON-escaped so server-controlled content cannot inject terminal sequences. + """ + warning_type = sanitize(warning.get('type', 'unknown')) + payload = warning.get('payload') or {} + text = KNOWN_PRECHECK_WARNINGS.get(warning.get('type'), 'Scan pre-check warning') + payload_str = json.dumps(payload, ensure_ascii=False) if payload else '' + return f'[{warning_type}] {text}{" " + payload_str if payload_str else ""}' + + +def scan_pair(scan): + return scan.get('stage'), scan.get('status') + + +def is_terminal(scan): + return scan_pair(scan) in TERMINAL_SCAN_PAIRS + + +def is_success(scan): + stage, status = scan_pair(scan) + return stage == ScanStage.SUCCESS and status in (ScanStageStatus.COMPLETE, + ScanStageStatus.PARTIAL_COMPLETE) + + +def resolve_upload_timeout(): + """Optional server-side upload_timeout override via env (no new CLI flags).""" + raw = os.environ.get(UPLOAD_TIMEOUT_ENV_VAR) + if not raw: + return None + try: + value = int(raw) + except ValueError: + logger.warning(f'Ignoring invalid {UPLOAD_TIMEOUT_ENV_VAR}={raw!r} (expected integer)') + return None + clamped = max(UPLOAD_TIMEOUT_MIN, min(UPLOAD_TIMEOUT_MAX, value)) + if clamped != value: + logger.warning(f'{UPLOAD_TIMEOUT_ENV_VAR}={value} is out of range, using {clamped}') + return clamped + + +def _exit_on_http_error(resp, action, exit_code=ExitCode.SCAN_FAILED): + message = sanitize(extract_error_message(resp)) + if resp.status_code in (401, 403): + logger.error(f'{action}: authorization failed (HTTP {resp.status_code}): {message}. ' + 'Check the organization CLI token (issued in the platform UI).') + sys.exit(ExitCode.AUTH_ERROR) + logger.error(f'{action} failed (HTTP {resp.status_code}): {message}') + sys.exit(exit_code) + + +def _json_or_exit(resp, action, exit_code=ExitCode.NETWORK_ERROR): + """Parse a 2xx response body as JSON or exit cleanly. + + A 2xx with a non-JSON body (e.g. an nginx/Envoy HTML stub, or a 204 with no + body) must not crash with a raw traceback; it is a network/gateway problem, + not INTERNAL_ERROR. + """ + try: + return resp.json() + except ValueError: + logger.error(f'{action}: server returned HTTP {resp.status_code} with a non-JSON body ' + f'(unexpected gateway/proxy response).') + sys.exit(exit_code) + + +def _retry_request(call, action, retries, exit_code=ExitCode.NETWORK_ERROR): + """Call `call()` (returns a Response), retrying transient 5xx/network errors. + + Non-transient HTTP errors and exhausted transient retries exit via + `_exit_on_http_error`. Returns the successful (2xx) Response. + """ + last_resp = None + for attempt in range(retries): + try: + resp = call() + except requests.RequestException as ex: + logger.warning(f'{action} request failed ({type(ex).__name__}), ' + f'retry {attempt + 1}/{retries}') + if attempt + 1 < retries: + time.sleep(SLEEP_TIMEOUT) + continue + if 200 <= resp.status_code < 300: + return resp + last_resp = resp + if resp.status_code in POLL_TRANSIENT_CODES and attempt + 1 < retries: + logger.warning(f'{action} returned {resp.status_code} (transient), ' + f'retry {attempt + 1}/{retries}') + time.sleep(SLEEP_TIMEOUT) + continue + break + if last_resp is not None: + _exit_on_http_error(last_resp, action, exit_code) + logger.error(f'{action} failed after {retries} attempts (network)') + sys.exit(exit_code) + + +def _get_scan(mdast, scan_id): + """Poll scan state, tolerating transient downstream failures.""" + resp = _retry_request(lambda: mdast.get_scan_info(scan_id), + f'Getting scan info for scan {scan_id}', POLL_TRANSIENT_RETRIES) + return _json_or_exit(resp, f'Getting scan info for scan {scan_id}') + + +def run_precheck_gate(mdast, md5, profile_id, testcase_id, scan_type): + """Gate model (DEC-671-05): any warning blocks the scan with a non-zero exit. + + A transient gateway failure (429/502/503/504) or a raw network error is NOT a + policy block: it is retried and, if it persists, exits NETWORK_ERROR(6) (the CI + "retry" signal) rather than PRECHECK_BLOCKED(8). The scan is still not created + in that case, so the gate stays fail-closed either way. + """ + resp = None + for attempt in range(UPLOAD_TRANSIENT_RETRIES): + try: + resp = mdast.precheck_scan(md5, profile_id, testcase_id, scan_type) + except requests.RequestException as ex: + logger.warning(f'Scan pre-check request failed ({type(ex).__name__}), ' + f'retry {attempt + 1}/{UPLOAD_TRANSIENT_RETRIES}') + resp = None + if attempt + 1 < UPLOAD_TRANSIENT_RETRIES: + time.sleep(SLEEP_TIMEOUT) + continue + if resp.status_code in POLL_TRANSIENT_CODES and attempt + 1 < UPLOAD_TRANSIENT_RETRIES: + logger.warning(f'Scan pre-check returned {resp.status_code} (transient), ' + f'retry {attempt + 1}/{UPLOAD_TRANSIENT_RETRIES}') + time.sleep(SLEEP_TIMEOUT) + continue + break + if resp is None: + logger.error('Scan pre-check could not be completed (network). This is retryable ' + 'infrastructure, not a policy block.') + sys.exit(ExitCode.NETWORK_ERROR) + if resp.status_code in (401, 403): + _exit_on_http_error(resp, 'Scan pre-check') # -> AUTH_ERROR + if resp.status_code == 404: + _exit_on_http_error(resp, 'Scan pre-check (profile lookup)') # -> SCAN_FAILED + if resp.status_code == 422 and profile_id is None: + # Auto-created-profile scan (no --profile_id): the profile does not exist + # yet, so pre-check cannot run against it. Skipping is intentional interim + # behaviour; platform-side validation still runs at scan creation. + logger.warning('Scan pre-check skipped: no --profile_id given (profile is ' + 'auto-created at scan start); platform validates at creation.') + return + if resp.status_code in POLL_TRANSIENT_CODES: + # transient gateway/rate-limit that outlived the retry budget: retryable infra, + # not a deliberate policy block -> NETWORK_ERROR(6), consistent with the rest. + message = sanitize(extract_error_message(resp)) + logger.error(f'Scan pre-check unavailable (HTTP {resp.status_code}): {message}. ' + 'Retryable infrastructure, not a policy block.') + sys.exit(ExitCode.NETWORK_ERROR) + if resp.status_code != 200: + message = sanitize(extract_error_message(resp)) + logger.error(f'Scan pre-check is unavailable (HTTP {resp.status_code}): {message}') + sys.exit(ExitCode.PRECHECK_BLOCKED) + warnings = (_json_or_exit(resp, 'Scan pre-check') or {}).get('warnings') or [] + if not warnings: + logger.info('Scan pre-check passed, no warnings') + return + logger.error('Scan pre-check returned blocking warnings, scan will not be created:') + for warning in warnings: + print(render_precheck_warning(warning), file=sys.stderr) + sys.exit(ExitCode.PRECHECK_BLOCKED) + + +def run_microservices_flow(arguments, url, token, app_file, user_agent=None, verify=True): + """Full scan flow against the microservices installation. Exits the process. + + Wrapped so a bare network error surfaces as NETWORK_ERROR(6), consistent with + the rest of the flow, instead of a traceback + INTERNAL_ERROR(1). + + Note there is no appstore_app_md5 parameter (unlike the monolith path): the + App Store downloader rewrites the .ipa after download (adds iTunesMetadata / + sinf), so the Apple-store md5 differs from the file actually uploaded. The + microservices flow always keys dedup/precheck/create off the local file md5, + which is also what upload sends - so an App Store (iOS) build uploads and + scans on the microservices installation exactly like any other .ipa (F1). + """ + try: + _run_microservices_flow(arguments, url, token, app_file, user_agent, verify) + except requests.RequestException as ex: + logger.error(f'Network error talking to the microservices installation ' + f'({type(ex).__name__}): {ex}') + sys.exit(ExitCode.NETWORK_ERROR) + + +def _run_microservices_flow(arguments, url, token, app_file, user_agent, verify): + profile_id = arguments.profile_id + project_id = arguments.project_id + testcase_id = arguments.testcase_id + + if arguments.appium_script_path: + logger.error('Appium scans are not supported on the microservices installation ' + '(--appium_script_path). Use a recorded test case (--testcase_id) instead.') + sys.exit(ExitCode.INVALID_ARGS) + if arguments.cr_report: + logger.error('CR report generation (--cr_report) is not supported on the ' + 'microservices installation.') + sys.exit(ExitCode.INVALID_ARGS) + if arguments.architecture_id is not None: + logger.warning('--architecture_id is ignored on the microservices installation: ' + 'the platform resolves the architecture from application metadata') + + mdast = mDastMicroservices(url, token, arguments.company_id, + user_agent=user_agent, verify=verify) + + architectures_resp = mdast.get_architectures() + if architectures_resp.status_code != 200: + _exit_on_http_error(architectures_resp, 'Getting architectures', ExitCode.NETWORK_ERROR) + architectures = _json_or_exit(architectures_resp, 'Getting architectures') + logger.info(f'Supported architectures: {architectures}') + + platform = resolve_platform(app_file) + if platform is None: + logger.error(f'Cannot resolve platform (Android/iOS) from file extension: {app_file}') + sys.exit(ExitCode.INVALID_ARGS) + + if testcase_id is not None: + testcase_resp = mdast.get_testcase(testcase_id) + if testcase_resp.status_code == 200: + testcase_os = str((_json_or_exit(testcase_resp, 'Getting test case') or {}).get('os', '')).upper() + if testcase_os and testcase_os != platform: + logger.error(f'Test case {testcase_id} is recorded for {testcase_os}, ' + f'but the application file is for {platform}') + sys.exit(ExitCode.INVALID_ARGS) + elif testcase_resp.status_code in (401, 403): + _exit_on_http_error(testcase_resp, f'Getting test case {testcase_id}') + else: + logger.warning(f'Cannot get test case {testcase_id} ' + f'(HTTP {testcase_resp.status_code}), continuing') + + engines_resp = mdast.get_engines() + if engines_resp.status_code != 200: + _exit_on_http_error(engines_resp, 'Getting engines', ExitCode.NETWORK_ERROR) + engines = _json_or_exit(engines_resp, 'Getting engines') + if not isinstance(engines, list): + logger.error('Getting engines: unexpected response shape (expected a list)') + sys.exit(ExitCode.NETWORK_ERROR) + active_engines = [engine for engine in engines + if isinstance(engine, dict) + and str(engine.get('type', '')).upper() == platform + and str(engine.get('status', '')).upper() == ENGINE_ACTIVE_STATUS] + if not active_engines: + logger.error(f'Cannot create scan - no active engine for platform {platform}') + sys.exit(ExitCode.SCAN_FAILED) + + # F1: single md5 for the whole flow — the md5 of the file actually uploaded. + # For appstore the CLI rewrites the ipa (adds iTunesMetadata/sinf) after + # download, so the Apple store md5 differs from what is uploaded; dedup / + # precheck / create must all use the local file's md5, which upload also sends. + logger.info('Check if this version of application was already uploaded..') + app_md5 = check_app_md5(app_file).lower() + dedup_resp = mdast.check_app_md5(None, app_md5) + if dedup_resp.status_code != 200: + _exit_on_http_error(dedup_resp, 'Application dedup check', ExitCode.NETWORK_ERROR) + found_apps = _json_or_exit(dedup_resp, 'Application dedup check') + application = found_apps[0] if isinstance(found_apps, list) and found_apps else None + if application: + logger.info('This app was uploaded before, application id is: ' + f"{application.get('id')}, package name: {sanitize(application.get('package_name'))}, " + f"version: {sanitize(application.get('version_name'))}, md5: {sanitize(application.get('md5'))}") + else: + logger.info('This is new application or new version') + logger.info('Uploading application to server..') + upload_timeout = resolve_upload_timeout() + application = _upload_application(mdast, app_file, upload_timeout) + + app_id = application.get('id') + if not app_id: + logger.error(f'Application response has no id: {sanitize(application)}') + sys.exit(ExitCode.SCAN_FAILED) + + precheck_type = 'AUTO' if testcase_id is not None else 'MANUAL' + run_precheck_gate(mdast, app_md5, profile_id, testcase_id, precheck_type) + + logger.info(f'Creating scan for application {sanitize(app_id)}') + if testcase_id is not None: + create_resp = mdast.create_auto_scan(project_id, profile_id, app_md5, None, testcase_id) + scan_type = 'auto_stingray' + else: + create_resp = mdast.create_manual_scan(project_id, profile_id, app_md5) + scan_type = 'manual' + if create_resp.status_code not in (200, 201): + _exit_on_http_error(create_resp, 'Creating scan') + scan = _json_or_exit(create_resp, 'Creating scan') + scan_id = scan.get('id') + if not scan_id: + logger.error(f'Something went wrong while creating scan: {scan}') + sys.exit(ExitCode.SCAN_FAILED) + logger.info('Project and profile were created/found successfully. ' + f"Project id: {(scan.get('project') or {}).get('id')}, " + f"profile id: {(scan.get('profile') or {}).get('id')}") + logger.info(f'Scan was created successfully. Scan id: {scan_id}') + + logger.info(f'Start scan with id {scan_id}') + start_resp = mdast.start_scan(scan_id) + if start_resp.status_code == 409: + # POST /scans/{id}/start/ is not idempotent: a facade retry (network/timeout + # after the first call already unlocked the scan) surfaces as 409 "not in + # initial state" even though the scan did start. Confirm by state before failing. + current = _get_scan(mdast, scan_id) + if current.get('stage') != ScanStage.CREATED: + logger.warning(f'Start returned 409 but scan {scan_id} is already past initial ' + f'state ({scan_pair(current)}) - treating as started (facade retry).') + else: + _exit_on_http_error(start_resp, f'Starting scan {scan_id}') + elif start_resp.status_code != 200: + _exit_on_http_error(start_resp, f'Starting scan {scan_id}') + + if arguments.nowait: + logger.info('Scan successfully started. Don`t wait for end, exit with zero code') + sys.exit(ExitCode.SUCCESS) + + try_count = LONG_TRY if arguments.long_wait else TRY + + logger.info('Scan started successfully.') + scan = _get_scan(mdast, scan_id) + logger.info(f'Current scan state: {scan_pair(scan)}') + + count = 0 + while scan.get('stage') in PRE_START_STAGES and count < try_count: + logger.info(f'Try to get scan status for scan id {scan_id}. Count number {count}') + scan = _get_scan(mdast, scan_id) + logger.info(f'Current scan state: {scan_pair(scan)}') + count += 1 + if scan.get('stage') in PRE_START_STAGES: + logger.info(f'Wait {SLEEP_TIMEOUT} seconds and try again') + time.sleep(SLEEP_TIMEOUT) + + if scan.get('stage') not in ACTIVE_STAGES | {ScanStage.SUCCESS, ScanStage.FAIL}: + logger.error(f'Error with scan id {scan_id}. Scan did not start, ' + f'current state: {scan_pair(scan)}, message: {sanitize(scan.get("message"))}') + sys.exit(ExitCode.SCAN_FAILED) + + if scan_type == 'manual' and not is_terminal(scan) and scan.get('stage') != ScanStage.STOP: + logger.info(f'This is a scan without a test case, ' + f'lets wait for {END_SCAN_TIMEOUT} seconds and stop it.') + time.sleep(END_SCAN_TIMEOUT) + try: + stop_resp = mdast.stop_scan(scan_id) + except requests.RequestException as ex: + logger.error(f'Stopping scan {scan_id} request failed ({type(ex).__name__})') + sys.exit(ExitCode.NETWORK_ERROR) + if stop_resp.status_code == 200: + logger.info(f'Scan {scan_id} was requested to stop (stopped by CLI after ' + f'{END_SCAN_TIMEOUT} seconds, as in the manual scan flow)') + elif stop_resp.status_code == 409: + logger.info(f'Scan {scan_id} is not in a stoppable state anymore, continuing') + else: + _exit_on_http_error(stop_resp, f'Stopping scan {scan_id}') + + logger.info(f"Scan {scan_id} is started now. Let's wait until the scan is finished") + count = 0 + while not is_terminal(scan) and count < try_count: + logger.info(f'Try to get scan status for scan id {scan_id}. Count number {count}') + scan = _get_scan(mdast, scan_id) + logger.info(f'Current scan state: {scan_pair(scan)}') + count += 1 + if not is_terminal(scan): + logger.info(f'Wait {SLEEP_TIMEOUT} seconds and try again') + time.sleep(SLEEP_TIMEOUT) + + if not is_terminal(scan): + logger.error(f'Scan {scan_id} did not reach a terminal state within the wait budget ' + f'(last state {scan_pair(scan)}). Increase timeout with --long_wait.') + sys.exit(ExitCode.SCAN_FAILED) + if not is_success(scan): + logger.error(f'Scan {scan_id} finished with state {scan_pair(scan)}, ' + f'message: {sanitize(scan.get("message"))}. Exit with error status code.') + sys.exit(ExitCode.SCAN_FAILED) + if scan.get('status') == ScanStageStatus.PARTIAL_COMPLETE: + logger.warning(f'Scan {scan_id} finished partially complete ' + f'(some modules did not run): {sanitize(scan.get("message"))}') + + download_reports(mdast, scan_id, arguments) + + +def _upload_application(mdast, app_file, upload_timeout): + """Upload with transient retry; friendly message on server-side parse timeout. + + Retries both transient HTTP (502/503) and raw network errors: a rolling + restart of the upload path can drop the connection outright, not only answer + 502. The server keys uploads by md5, so re-sending the same build is + idempotent (a lost-response upload is de-duplicated server-side). + """ + last_resp = None + for attempt in range(UPLOAD_TRANSIENT_RETRIES): + try: + resp = mdast.upload_application(app_file, upload_timeout=upload_timeout) + except requests.RequestException as ex: + logger.warning(f'Uploading application request failed ({type(ex).__name__}), ' + f'retry {attempt + 1}/{UPLOAD_TRANSIENT_RETRIES}') + if attempt + 1 < UPLOAD_TRANSIENT_RETRIES: + time.sleep(SLEEP_TIMEOUT) + continue + if 200 <= resp.status_code < 300: + # Accept any 2xx (not just 201): a facade/gateway may answer 200 for an + # already-registered build, and the rest of the flow (poll/create) already + # treats 2xx as success. + application = _json_or_exit(resp, 'Uploading application') + logger.info(f"Application uploaded successfully. Application id: {application.get('id')}") + return application + if resp.status_code == 504: + # Gateway timeout: the synchronous wait elapsed but the platform keeps + # parsing. Re-running finds the build via md5 dedup. This is retryable + # infra, not a bad scan -> NETWORK_ERROR (the CI "retry" signal). + logger.error('Application parsing did not finish in time (504). The upload is ' + 'processed asynchronously - retry the same command later, the file ' + f'will not be re-uploaded. {UPLOAD_TIMEOUT_ENV_VAR} env var can raise ' + f'the wait (max {UPLOAD_TIMEOUT_MAX} seconds).') + sys.exit(ExitCode.NETWORK_ERROR) + last_resp = resp + if resp.status_code in UPLOAD_TRANSIENT_CODES and attempt + 1 < UPLOAD_TRANSIENT_RETRIES: + logger.warning(f'Uploading application returned {resp.status_code} (transient), ' + f'retry {attempt + 1}/{UPLOAD_TRANSIENT_RETRIES}') + time.sleep(SLEEP_TIMEOUT) + continue + break + if last_resp is not None: + if last_resp.status_code in UPLOAD_TRANSIENT_CODES: + # gateway 5xx / rate-limit that outlived the retry budget: retryable infra + logger.error(f'Uploading application failed: transient status {last_resp.status_code} ' + f'persisted after {UPLOAD_TRANSIENT_RETRIES} retries.') + sys.exit(ExitCode.NETWORK_ERROR) + _exit_on_http_error(last_resp, 'Uploading application') + logger.error(f'Uploading application failed after {UPLOAD_TRANSIENT_RETRIES} attempts ' + f'(network). For large builds raise the wait with {UPLOAD_TIMEOUT_ENV_VAR} ' + f'(seconds, max {UPLOAD_TIMEOUT_MAX}).') + sys.exit(ExitCode.NETWORK_ERROR) + + +def download_reports(mdast, scan_id, arguments): + """Report step (STG-4478): files are written to CLI-chosen paths. + + Format selection is shared with the monolith flow (--report_format, default + pdf, scan-id default names). Reports are downloaded independently and are + soft-fail: the scan already finished SUCCESS, so a report render failure (e.g. + Pepper 502, a separate microservice) must not turn a green scan red or block + the other report format. Content-Disposition from the server is deliberately + ignored - the local file name is a CLI argument. + """ + targets = resolve_report_targets(arguments, scan_id) + failures = [] + + if 'pdf' in targets: + target = targets['pdf'] + logger.info(f'Create and download pdf report for scan {scan_id} to file {target}.') + resp = _download_report(mdast.download_report, scan_id, 'PDF report') + if resp is None: + failures.append('PDF') + else: + with open(target, 'wb') as f: + f.write(resp.content) + logger.info(f'PDF report for scan {scan_id} saved to {target}.') + + if 'json' in targets: + target = targets['json'] + logger.info(f'Download JSON summary report for scan {scan_id} to file {target}.') + resp = _download_report(mdast.download_scan_json_result, scan_id, 'JSON report') + if resp is None: + failures.append('JSON') + else: + try: + payload = resp.json() + except ValueError: + logger.error('JSON report: server returned a non-JSON body; saving raw bytes.') + with open(target, 'wb') as f: + f.write(resp.content) + else: + with open(target, 'w') as fp: + json.dump(payload, fp, indent=4, ensure_ascii=False) + logger.info(f'JSON report for scan {scan_id} saved to {target}.') + + if failures: + # Soft-fail: scan succeeded; report render is a downstream (Pepper) issue. + logger.warning(f'Scan {scan_id} finished successfully, but these reports could not be ' + f'downloaded (report service issue, not scan failure): {", ".join(failures)}. ' + 'Retry the report download later.') + + +def _download_report(fetch, scan_id, label): + """Download one report with transient retry. Returns Response or None (soft-fail).""" + last = None + for attempt in range(POLL_TRANSIENT_RETRIES): + try: + resp = fetch(scan_id) + except requests.RequestException as ex: + logger.warning(f'{label} request failed ({type(ex).__name__}), ' + f'retry {attempt + 1}/{POLL_TRANSIENT_RETRIES}') + if attempt + 1 < POLL_TRANSIENT_RETRIES: + time.sleep(SLEEP_TIMEOUT) + continue + if resp.status_code == 200: + return resp + last = resp + if resp.status_code in POLL_TRANSIENT_CODES and attempt + 1 < POLL_TRANSIENT_RETRIES: + logger.warning(f'{label} returned {resp.status_code} (transient), ' + f'retry {attempt + 1}/{POLL_TRANSIENT_RETRIES}') + time.sleep(SLEEP_TIMEOUT) + continue + break + if last is not None: + logger.error(f'{label} download failed (HTTP {last.status_code}): ' + f'{sanitize(extract_error_message(last))}') + return None diff --git a/mdast_cli_core/factory.py b/mdast_cli_core/factory.py new file mode 100644 index 0000000..4276a5c --- /dev/null +++ b/mdast_cli_core/factory.py @@ -0,0 +1,144 @@ +"""Installation mode resolution: monolith vs microservices (Clark facade). + +The mode decides which backend the scan flow talks to, so it must be reliable in +BOTH directions (monolith -> monolith, k8s -> k8s). It is resolved as: + +1. Explicit override via the ``MDAST_CLI_MODE`` env var + (``monolith`` | ``microservices`` | ``auto``; default ``auto``). No new CLI + flags are introduced (OOS-671-03). +2. Auto-detection by *content fingerprint*, not just an HTTP 200. Both + installations expose ``GET {base}/architectures/`` under ``/rest``, but the + payloads differ structurally: + - microservices (scanyon-native): ``type`` is a string ``ANDROID``/``IOS`` + and each item carries ``os_version``; + - monolith: ``type`` is an integer code (1/2) and there is no ``os_version``. + Auth scheme also differs (microservices = ``Bearer``, monolith = ``Token``), + so each probe uses its own scheme. A monolith that happens to answer 200 to a + Bearer probe is NOT misclassified, because the payload is inspected. +""" +import logging +import os + +import requests + +MODE_ENV_VAR = 'MDAST_CLI_MODE' +TLS_VERIFY_ENV_VAR = 'MDAST_TLS_VERIFY' +MODE_AUTO = 'auto' +MODE_MONOLITH = 'monolith' +MODE_MICROSERVICES = 'microservices' +PROBE_TIMEOUT = 20 + +logger = logging.getLogger(__name__) + + +class ModeDetectionError(Exception): + """Raised when the installation flavour cannot be determined.""" + + def __init__(self, message, auth_error=False): + super().__init__(message) + self.auth_error = auth_error + + +def tls_verify_enabled(): + """TLS verification is on unless explicitly disabled via env. + + Secure by default (the CLI carries an org-level bearer token): the operator + must opt OUT with MDAST_TLS_VERIFY in {0,false,no,off} for self-signed stands. + An unset OR empty value counts as "not configured" and stays secure, so a + docker `-e MDAST_TLS_VERIFY` passthrough with no value can't silently + disable verification. + """ + raw = os.environ.get(TLS_VERIFY_ENV_VAR) + if raw is None: + return True + raw = raw.strip().lower() + if raw == '': + return True + return raw not in ('0', 'false', 'no', 'off') + + +def _looks_microservices(payload): + """True if an /architectures/ payload is scanyon-native (microservices).""" + if not isinstance(payload, list) or not payload: + return False + item = payload[0] + if not isinstance(item, dict): + return False + # scanyon-native: type is a string ANDROID/IOS and os_version is present + type_value = item.get('type') + return isinstance(type_value, str) and type_value.upper() in ('ANDROID', 'IOS') + + +def _looks_monolith(payload): + """True if an /architectures/ payload is monolith-shaped (int type code).""" + if not isinstance(payload, list) or not payload: + return False + item = payload[0] + return isinstance(item, dict) and isinstance(item.get('type'), int) + + +def _probe(base_url, path, scheme, ci_token, verify): + """GET a probe endpoint; return (status_code | None, parsed_json | None).""" + try: + resp = requests.get(f'{base_url}{path}', + headers={'Authorization': f'{scheme} {ci_token}'}, + verify=verify, + timeout=PROBE_TIMEOUT) + except requests.RequestException as ex: + logger.debug(f'Probe {scheme} {path} failed: {type(ex).__name__}: {ex}') + return None, None + payload = None + if resp.status_code == 200: + try: + payload = resp.json() + except ValueError: + payload = None + return resp.status_code, payload + + +def resolve_installation_mode(base_url, ci_token, company_id, mode=None, verify=None): + """Return MODE_MONOLITH or MODE_MICROSERVICES. + + ``base_url`` is the normalized base ending with ``/rest`` (both installations + serve the CLI routes under this prefix). + """ + if verify is None: + verify = tls_verify_enabled() + mode = (mode or os.environ.get(MODE_ENV_VAR) or MODE_AUTO).strip().lower() + if mode == MODE_MONOLITH or mode == MODE_MICROSERVICES: + logger.info(f'Installation mode forced via {MODE_ENV_VAR}: {mode}') + return mode + if mode != MODE_AUTO: + raise ModeDetectionError( + f'Unknown {MODE_ENV_VAR} value: {mode!r} ' + f'(expected {MODE_AUTO}/{MODE_MONOLITH}/{MODE_MICROSERVICES})') + + # Microservices probe: Bearer + architectures, classify by payload shape. + ms_status, ms_payload = _probe(base_url, '/architectures/', 'Bearer', ci_token, verify) + if ms_status == 200 and _looks_microservices(ms_payload): + logger.info('Detected microservices installation (Clark facade)') + return MODE_MICROSERVICES + + # Monolith probe: Token + architectures, classify by payload shape. + mono_status, mono_payload = _probe(base_url, '/architectures/', 'Token', ci_token, verify) + if mono_status == 200 and _looks_monolith(mono_payload): + logger.info('Detected monolith installation') + return MODE_MONOLITH + + # Ambiguous 200 (payload matched neither shape) — do not guess. + if ms_status == 200 or mono_status == 200: + empty_list = ms_payload == [] or mono_payload == [] + hint = ('The /architectures/ list is empty, so the installation flavour cannot be ' + 'inferred from it. ' if empty_list else + 'The payload matched neither the microservices (string type + os_version) ' + 'nor the monolith (int type) shape. ') + raise ModeDetectionError( + f'Cannot detect installation mode: /architectures/ returned 200 but {hint}' + f'Force the mode via {MODE_ENV_VAR}=monolith|microservices.') + + auth_error = 401 in (ms_status, mono_status) or 403 in (ms_status, mono_status) + raise ModeDetectionError( + 'Cannot detect installation mode via GET /architectures/ ' + f'(Bearer -> {ms_status}, Token -> {mono_status}). ' + f'Check --url/--token, or force the mode via {MODE_ENV_VAR}.', + auth_error=auth_error) diff --git a/mdast_cli_core/microservices.py b/mdast_cli_core/microservices.py new file mode 100644 index 0000000..ec3299e --- /dev/null +++ b/mdast_cli_core/microservices.py @@ -0,0 +1,240 @@ +"""Client for the microservices installation (Clark facade behind Envoy ExtAuth -> OPA). + +Public method names match the monolith client (mDastToken) per ASM-671-01; +routes and payloads follow the Clark facade contract (STGRST-670/671/694/698): + +- auth: `Authorization: Bearer ` (org-level opaque token issued in UI); +- all routes live under the same `/rest` prefix as the monolith (base_url ends + with `/rest` after mdast_scan.py normalization), so the base URL handling is + shared between both installations; +- tenant comes from the `X-Organization-Id` header injected by the platform, + the `/organizations/{id}/` URL segment is gone, `company_id` is ignored; +- scans are keyed by application md5 (not application_id) and are created with + the two-phase model (`fsm_locked=true` + explicit `/scans/{id}/start/`). +""" +import hashlib +import json +import os + +import requests + +from .base import mDastBase + +HTTP_REQUEST_TIMEOUT = 30 +HTTP_DOWNLOAD_TIMEOUT = 300 +UPLOAD_EXTRA_TIMEOUT = 120 + + +def file_md5(file_path): + """Lower-case hex md5 of a file (Clark multipart contract requires lower-case).""" + file_hash = hashlib.md5() + with open(file_path, 'rb') as f: + while chunk := f.read(8192): + file_hash.update(chunk) + return file_hash.hexdigest().lower() + + +def extract_error_message(resp): + """Best-effort human-readable message from Clark error responses. + + /rest routes answer with the monolith envelope {error_code, message}; + the report route answers with the FastAPI envelope {"detail": ...}; + scanyon 429 passes through {"detail": ..., "reasons": [...]}. + """ + try: + data = resp.json() + except ValueError: + return resp.text + if isinstance(data, dict): + if data.get('message'): + prefix = data.get('error_code') + message = data['message'] + return f'{prefix}: {message}' if prefix else message + if data.get('detail') is not None: + detail = data['detail'] + if not isinstance(detail, str): + detail = json.dumps(detail, ensure_ascii=False) + reasons = data.get('reasons') + if reasons: + detail = f'{detail} (reasons: {json.dumps(reasons, ensure_ascii=False)})' + return detail + return resp.text + + +class mDastMicroservices(mDastBase): + """API client for the Clark facade of the microservices installation.""" + + def __init__(self, base_url, ci_token, company_id=None, user_agent=None, verify=True): + super().__init__(base_url) + # F14: strip trailing slash so f'{self.url}/architectures/' can't produce + # a double slash (a strict OPA/Envoy path match may 403/404 on '//'). + self.url = base_url.rstrip('/') + # company_id is accepted for interface parity with mDastToken and ignored: + # the organization is resolved server-side from the token (X-Organization-Id). + self.company_id = company_id + self.current_context = {'company': company_id} + # F4: TLS verification on by default (org bearer token is a credential); + # opt out only via MDAST_TLS_VERIFY for self-signed stands. + self.verify = verify + self.timeout = HTTP_REQUEST_TIMEOUT + self.headers = {'Authorization': 'Bearer {0}'.format(ci_token), + 'Content-Type': 'application/json'} + if user_agent: + self.headers['User-Agent'] = user_agent + + def set_headers(self, ci_token): + self.headers['Authorization'] = 'Bearer {0}'.format(ci_token) + + def _request_headers(self, json_body=True): + headers = {'Authorization': self.headers['Authorization']} + if 'User-Agent' in self.headers: + headers['User-Agent'] = self.headers['User-Agent'] + if json_body: + headers['Content-Type'] = 'application/json' + return headers + + # --- preflight --- + + def get_architectures(self): + return requests.get(f'{self.url}/architectures/', + headers=self.headers, + verify=self.verify, + timeout=self.timeout) + + def get_engines(self): + return requests.get(f'{self.url}/engines/', + headers=self.headers, + verify=self.verify, + timeout=self.timeout) + + def get_testcase(self, testcase_id): + return requests.get(f'{self.url}/testcases/{testcase_id}/', + headers=self.headers, + verify=self.verify, + timeout=self.timeout) + + # --- application upload / dedup (STG-4451) --- + + def check_app_md5(self, org_id, md5): + # org_id is ignored: no organization segment in facade URLs (DEC-694-05) + return requests.get(f'{self.url}/applications/?md5={md5}', + headers=self.headers, + verify=self.verify, + timeout=self.timeout) + + def upload_application(self, path, architecture_type=None, upload_timeout=None): + """Upload via POST /rest/applications/upload_info/. + + Contract (STG-4451 / STG-4450): multipart field `md5` (lower-case hex) + strictly before `file`; `X-File-Size` header with the exact binary size; + Content-Length is set by requests (body is fully built, no chunked TE). + `architecture_type` is accepted for interface parity and not sent: + the platform parses the binary itself. + """ + md5 = file_md5(path) + file_size = os.path.getsize(path) + headers = self._request_headers(json_body=False) + headers['X-File-Size'] = str(file_size) + params = {} + request_timeout = HTTP_DOWNLOAD_TIMEOUT + if upload_timeout is not None: + params['upload_timeout'] = int(upload_timeout) + request_timeout = int(upload_timeout) + UPLOAD_EXTRA_TIMEOUT + with open(path, 'rb') as f: + # ordered list keeps `md5` as the first multipart part (facade requirement) + multipart = [ + ('md5', (None, md5)), + ('file', (os.path.split(path)[-1], f, 'application/octet-stream')), + ] + return requests.post(f'{self.url}/applications/upload_info/', + headers=headers, + files=multipart, + params=params or None, + verify=self.verify, + timeout=request_timeout) + + # --- scan flow (STG-4475) --- + + def precheck_scan(self, md5, profile_id, testcase_id, scan_type): + """Gate pre-check before scan creation (DEC-671-05). + + Field is `application_md5` in the scanyon contract (not `md5`). + scan_type mirrors the create call: MANUAL for a scan without a test case, + AUTO for a test-case replay. + """ + data = { + 'application_md5': md5, + 'profile_id': profile_id, + 'type': scan_type, + 'testcase_id': testcase_id, + } + return requests.post(f'{self.url}/scans/start/precheck/', + headers=self.headers, + data=json.dumps(data), + verify=self.verify, + timeout=self.timeout) + + def _create_scan(self, project_id, profile_id, app_md5, test_case_id, scan_type): + # scanyon: AUTO requires testcase_id; a scan without a test case is MANUAL + # (install/launch/wait/stop semantics, wait=True on the server side). + # testcase_id key is mandatory in ScanIn even when null. + data = { + 'md5': app_md5, + 'type': scan_type, + 'testcase_id': test_case_id, + 'fsm_locked': True, + } + if project_id: + data['project_id'] = project_id + if profile_id: + data['profile_id'] = profile_id + return requests.post(f'{self.url}/scans/start/', + headers=self.headers, + data=json.dumps(data), + verify=self.verify, + timeout=self.timeout) + + def create_manual_scan(self, project_id, profile_id, app_md5, arch_id=None): + # arch_id accepted and ignored (microservices resolve platform from app metadata) + return self._create_scan(project_id, profile_id, app_md5, None, 'MANUAL') + + def create_auto_scan(self, project_id, profile_id, app_md5, arch_id, test_case_id): + return self._create_scan(project_id, profile_id, app_md5, test_case_id, 'AUTO') + + def create_appium_scan(self, project_id, profile_id, app_id, arch_id, appium_script_path): + raise NotImplementedError( + 'Appium scans are not supported on the microservices installation (OOS-671-06)') + + def start_scan(self, dast_id): + return requests.post(f'{self.url}/scans/{dast_id}/start/', + headers=self.headers, + verify=self.verify, + timeout=self.timeout) + + def stop_scan(self, scan_id): + return requests.post(f'{self.url}/scans/{scan_id}/stop/', + headers=self.headers, + verify=self.verify, + timeout=self.timeout) + + def get_scan_info(self, scan_id): + return requests.get(f'{self.url}/scans/{scan_id}/', + headers=self.headers, + verify=self.verify, + timeout=self.timeout) + + # --- reports (STG-4478) --- + + def _download_report_output(self, dast_id, output): + return requests.get(f'{self.url}/scans/{dast_id}/report', + params={'output': output}, + allow_redirects=True, + headers=self.headers, + verify=self.verify, + timeout=HTTP_DOWNLOAD_TIMEOUT) + + def download_report(self, dast_id): + return self._download_report_output(dast_id, 'pdf') + + def download_scan_json_result(self, dast_id): + return self._download_report_output(dast_id, 'json') diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..f731fd7 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,10 @@ +[pytest] +testpaths = tests +markers = + unit: fast isolated tests of a single module + contract: tests pinning the HTTP contract with the Clark facade / monolith + smoke: end-to-end CLI flows against a mocked server + e2e: negative end-to-end CLI flows asserting exit codes + security: security-focused checks (secrets handling, header trust, TLS) +filterwarnings = + ignore::DeprecationWarning diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..71ff239 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +pytest>=8.0 +responses>=0.25 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9c44fb3 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,101 @@ +import hashlib +import sys + +import pytest +import responses as responses_lib + +BASE_URL = 'https://stand.example' +REST_URL = f'{BASE_URL}/rest' +TOKEN = 'very-secret-org-token-value-1234567890' +COMPANY_ID = '1' + + +@pytest.fixture +def tmp_apk(tmp_path): + path = tmp_path / 'app.apk' + path.write_bytes(b'PK\x03\x04' + b'binary-payload' * 128) + return str(path) + + +@pytest.fixture +def apk_md5(tmp_apk): + with open(tmp_apk, 'rb') as f: + return hashlib.md5(f.read()).hexdigest().lower() + + +@pytest.fixture +def mocked_responses(): + with responses_lib.RequestsMock(assert_all_requests_are_fired=False) as rsps: + yield rsps + + +@pytest.fixture +def no_sleep(monkeypatch): + monkeypatch.setattr('time.sleep', lambda *_args, **_kwargs: None) + + +@pytest.fixture +def ms_mode(monkeypatch): + monkeypatch.setenv('MDAST_CLI_MODE', 'microservices') + + +@pytest.fixture +def monolith_mode(monkeypatch): + monkeypatch.setenv('MDAST_CLI_MODE', 'monolith') + + +def run_main(monkeypatch, argv): + """Run mdast_scan.main() with the given CLI args, return the exit code. + + The monolith success path returns from main() without sys.exit - that is + process exit code 0. + """ + from mdast_cli import mdast_scan + monkeypatch.setattr(sys, 'argv', ['mdast_cli'] + argv) + try: + mdast_scan.main() + except SystemExit as excinfo: + return int(excinfo.code or 0) + return 0 + + +def scan_json(scan_id=77, stage='WORKING', status='PROCESSING', message=None, **extra): + payload = { + 'id': scan_id, + 'stage': stage, + 'status': status, + 'message': message, + 'md5': 'a' * 32, + 'type': 'AUTO', + 'testcase_id': None, + 'storage_id': None, + 'project': {'id': 1, 'name': 'proj'}, + 'profile': {'id': 2, 'name': 'prof'}, + 'fsm_locked': False, + } + payload.update(extra) + return payload + + +def application_json(md5, app_id=10): + return { + 'id': app_id, + 'architecture_type': 'Android', + 'file_url': 'https://s3.example/presigned', + 'icon_url': None, + 'name': 'Example App', + 'package_name': 'com.example.app', + 'md5': md5, + 'file_size': 1234, + 'version_code': 1, + 'version_name': '1.0', + 'min_sdk_version': '21', + 'max_sdk_version': None, + 'target_sdk_version': '33', + 'main_activity': 'com.example.Main', + 'main_activities_objects': [], + 'is_debuggable': False, + 'signature': 'abc', + 'is_parked': False, + 'distribution_type': 0, + } diff --git a/tests/test_client_microservices.py b/tests/test_client_microservices.py new file mode 100644 index 0000000..23845f2 --- /dev/null +++ b/tests/test_client_microservices.py @@ -0,0 +1,136 @@ +"""Unit/contract tests for the microservices client (Clark facade URLs, bodies, headers).""" +import json + +import pytest +import responses + +from mdast_cli_core.microservices import file_md5, mDastMicroservices +from tests.conftest import REST_URL, TOKEN + +pytestmark = [pytest.mark.unit, pytest.mark.contract] + + +@pytest.fixture +def client(): + return mDastMicroservices(REST_URL, TOKEN, company_id=1, user_agent='mdast_cli/test') + + +def last_request(rsps): + return rsps.calls[-1].request + + +def test_bearer_auth_scheme(client, mocked_responses): + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', json=[]) + client.get_architectures() + assert last_request(mocked_responses).headers['Authorization'] == f'Bearer {TOKEN}' + + +def test_user_agent_is_sent(client, mocked_responses): + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', json=[]) + client.get_architectures() + assert last_request(mocked_responses).headers['User-Agent'] == 'mdast_cli/test' + + +def test_engines_url_has_no_organization_segment(client, mocked_responses): + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', json=[]) + client.get_engines() + assert '/organizations/' not in last_request(mocked_responses).url + + +def test_dedup_url(client, mocked_responses): + mocked_responses.add(responses.GET, f'{REST_URL}/applications/', json=[]) + client.check_app_md5(None, 'a' * 32) + request = last_request(mocked_responses) + assert request.url == f'{REST_URL}/applications/?md5={"a" * 32}' + assert '/organizations/' not in request.url + + +def test_testcase_url(client, mocked_responses): + mocked_responses.add(responses.GET, f'{REST_URL}/testcases/5/', json={'os': 'ANDROID'}) + client.get_testcase(5) + assert last_request(mocked_responses).url == f'{REST_URL}/testcases/5/' + + +def test_create_scan_body_auto(client, mocked_responses): + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', json={'id': 1}, status=200) + client.create_auto_scan(project_id=3, profile_id=4, app_md5='b' * 32, + arch_id=999, test_case_id=5) + body = json.loads(last_request(mocked_responses).body) + assert body == {'md5': 'b' * 32, 'type': 'AUTO', 'testcase_id': 5, + 'fsm_locked': True, 'project_id': 3, 'profile_id': 4} + + +def test_create_scan_body_manual_is_manual_type(client, mocked_responses): + """Scan without a test case is MANUAL (scanyon rejects AUTO without testcase_id); + testcase_id key stays present (null).""" + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', json={'id': 1}, status=200) + client.create_manual_scan(project_id=None, profile_id=None, app_md5='b' * 32) + body = json.loads(last_request(mocked_responses).body) + assert body == {'md5': 'b' * 32, 'type': 'MANUAL', 'testcase_id': None, 'fsm_locked': True} + assert 'architecture_id' not in body + assert 'application_id' not in body + + +def test_precheck_body_uses_application_md5_field(client, mocked_responses): + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/precheck/', + json={'warnings': []}) + client.precheck_scan('c' * 32, profile_id=7, testcase_id=None, scan_type='MANUAL') + body = json.loads(last_request(mocked_responses).body) + assert body == {'application_md5': 'c' * 32, 'profile_id': 7, + 'type': 'MANUAL', 'testcase_id': None} + + +def test_scan_lifecycle_urls(client, mocked_responses): + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/start/', json={'id': 77}) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/stop/', json={'id': 77}) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', json={'id': 77}) + client.start_scan(77) + client.stop_scan(77) + client.get_scan_info(77) + urls = [call.request.url for call in mocked_responses.calls] + assert urls == [f'{REST_URL}/scans/77/start/', f'{REST_URL}/scans/77/stop/', + f'{REST_URL}/scans/77/'] + + +def test_report_urls_and_output_param(client, mocked_responses): + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', body=b'%PDF') + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', json={'summary': {}}) + client.download_report(77) + client.download_scan_json_result(77) + assert mocked_responses.calls[0].request.url == f'{REST_URL}/scans/77/report?output=pdf' + assert mocked_responses.calls[1].request.url == f'{REST_URL}/scans/77/report?output=json' + + +def test_upload_multipart_md5_first_and_headers(client, mocked_responses, tmp_apk, apk_md5): + mocked_responses.add(responses.POST, f'{REST_URL}/applications/upload_info/', + json={'id': 10}, status=201) + client.upload_application(tmp_apk) + request = last_request(mocked_responses) + body = request.body if isinstance(request.body, bytes) else request.body.read() + md5_pos = body.index(b'name="md5"') + file_pos = body.index(b'name="file"') + assert md5_pos < file_pos, 'md5 multipart field must precede file (facade contract)' + assert apk_md5.encode() in body + import os + assert request.headers['X-File-Size'] == str(os.path.getsize(tmp_apk)) + assert int(request.headers['Content-Length']) >= os.path.getsize(tmp_apk) + assert 'chunked' not in request.headers.get('Transfer-Encoding', '') + + +def test_upload_timeout_param(client, mocked_responses, tmp_apk): + mocked_responses.add(responses.POST, f'{REST_URL}/applications/upload_info/', + json={'id': 10}, status=201) + client.upload_application(tmp_apk, upload_timeout=120) + assert 'upload_timeout=120' in last_request(mocked_responses).url + + +def test_file_md5_is_lower_case_hex(tmp_apk): + digest = file_md5(tmp_apk) + assert digest == digest.lower() + assert len(digest) == 32 + int(digest, 16) + + +def test_appium_scan_not_supported(client): + with pytest.raises(NotImplementedError): + client.create_appium_scan(1, 2, 3, 4, '/tmp/script.py') diff --git a/tests/test_download_scan_positive.py b/tests/test_download_scan_positive.py new file mode 100644 index 0000000..6cada16 --- /dev/null +++ b/tests/test_download_scan_positive.py @@ -0,0 +1,224 @@ +"""Positive e2e: download a build from a store, then scan it in Sting (microservices). + +Each test mocks the store's HTTP endpoints to deliver a valid APK, then lets the +CLI run the full microservices flow (upload -> precheck -> create -> start -> poll +-> report) against the mocked Sting facade. This is the happy path of the +"download from store -> scan" integration, one store per test. + +Store-side FAILURE modes are intentionally NOT here (per scope: negatives cover +work with Sting, positives cover download-from-store + scan). See test_negative_e2e.py. +""" +import hashlib +import io +import json +import os +import zipfile + +import pytest +import responses +from responses import matchers + +from tests.conftest import (BASE_URL, REST_URL, TOKEN, application_json, run_main, + scan_json) +from tests.test_smoke_flows import register_ms_happy_path + +pytestmark = pytest.mark.e2e + +APK_BYTES = b'PK\x03\x04' + b'fake-apk-payload' * 64 + + +def _valid_zip_apk(): + """A real (minimal) zip — APK is a zip; some downloaders (rustore) validate the container.""" + buf = io.BytesIO() + with zipfile.ZipFile(buf, 'w') as z: + z.writestr('AndroidManifest.xml', b'\x00') + return buf.getvalue() + + +APK_ZIP = _valid_zip_apk() + + +def _write_app(download_path, name): + """Write a real app file to download_path and return its path (for monkeypatched downloaders).""" + os.makedirs(str(download_path), exist_ok=True) + p = os.path.join(str(download_path), name) + with open(p, 'wb') as f: + f.write(APK_ZIP) + return p + + +def scan_argv(distribution_argv, download_path): + """Common scan flags appended to a store's download flags.""" + return distribution_argv + [ + '--download_path', str(download_path), + '--url', BASE_URL, '--company_id', '1', '--token', TOKEN, + '--profile_id', '2', '--testcase_id', '5', '--report_format', 'json', + ] + + +def _assert_scanned_ok(exit_code, mocked_responses): + assert exit_code == 0 + # the scan was actually created against the facade with the downloaded build + creates = [c for c in mocked_responses.calls if c.request.url == f'{REST_URL}/scans/start/'] + assert creates, 'a scan must be created from the downloaded store build' + + +# --- RuMarket --------------------------------------------------------------- + +def test_rumarket_download_then_scan(mocked_responses, monkeypatch, tmp_path, apk_md5, + no_sleep, ms_mode): + # store: app info + apk download + mocked_responses.add(responses.GET, + 'https://store-api.ruplay.market/api/v1/app/getApp/com.example.app', + json={'data': { + 'packageName': 'com.example.app', + 'author': {'name': 'Acme'}, + 'iconUrl': 'https://x/icon.png', + 'latestApk': {'name': 'app.apk', 'versionName': '1.0', + 'versionCode': 1, 'minSdkVersion': 21, + 'targetSdkVersion': 33, 'size': len(APK_BYTES)}, + }}) + mocked_responses.add(responses.GET, 'https://cdn.ruplay.market/data/apks/app.apk', + body=APK_BYTES, content_type='application/vnd.android.package-archive') + # Sting happy path + register_ms_happy_path(mocked_responses, apk_md5) + exit_code = run_main(monkeypatch, scan_argv( + ['--distribution_system', 'rumarket', '--rumarket_package_name', 'com.example.app'], + tmp_path)) + _assert_scanned_ok(exit_code, mocked_responses) + + +# --- Huawei AppGallery ------------------------------------------------------ + +def test_appgallery_download_then_scan(mocked_responses, monkeypatch, tmp_path, apk_md5, + no_sleep, ms_mode): + mocked_responses.add(responses.POST, + 'https://web-drru.hispace.dbankcloud.ru/webedge/getInterfaceCode', + json='ifc-code') + mocked_responses.add(responses.GET, + 'https://web-drru.hispace.dbankcloud.ru/uowap/index', + json={'layoutData': [{'dataList': [{ + 'package': 'com.example.app', 'appid': 'C123', 'name': 'App', + 'versionName': '1.0', 'versionCode': 1, 'targetSDK': 33, + 'size': len(APK_BYTES), 'md5': 'abc', 'icon': 'https://x/i.png'}]}]}) + mocked_responses.add(responses.GET, + 'https://appgallery.cloud.huawei.com/appdl/C123', + body=APK_BYTES, content_type='application/vnd.android.package-archive') + register_ms_happy_path(mocked_responses, apk_md5) + exit_code = run_main(monkeypatch, scan_argv( + ['--distribution_system', 'appgallery', '--appgallery_app_id', 'C123'], + tmp_path)) + _assert_scanned_ok(exit_code, mocked_responses) + + +# --- Nexus 3 (session + search + download) --- +def test_nexus_download_then_scan(mocked_responses, monkeypatch, tmp_path, apk_md5, no_sleep, ms_mode): + nx = 'http://nexus.example' + mocked_responses.add(responses.POST, f'{nx}/service/rapture/session', json={}, status=200) + mocked_responses.add(responses.GET, f'{nx}/service/rest/v1/search', json={'items': [{'assets': [{ + 'contentType': 'application/vnd.android.package-archive', 'downloadUrl': f'{nx}/repo/app.apk'}]}]}) + mocked_responses.add(responses.GET, f'{nx}/repo/app.apk', body=APK_BYTES) + register_ms_happy_path(mocked_responses, apk_md5) + argv = ['--distribution_system', 'nexus', '--nexus_url', nx, '--nexus_login', 'u', + '--nexus_password', 'p', '--nexus_repo_name', 'releases', '--nexus_group_id', 'com.example', + '--nexus_artifact_id', 'app', '--nexus_version', '1.0'] + _assert_scanned_ok(run_main(monkeypatch, scan_argv(argv, tmp_path)), mocked_responses) + + +# --- Nexus 2 (maven content GET) --- +def test_nexus2_download_then_scan(mocked_responses, monkeypatch, tmp_path, apk_md5, no_sleep, ms_mode): + nx = 'http://nexus2.example' + mocked_responses.add(responses.GET, f'{nx}/service/local/artifact/maven/content', body=APK_BYTES) + register_ms_happy_path(mocked_responses, apk_md5) + argv = ['--distribution_system', 'nexus2', '--nexus2_url', nx, '--nexus2_login', 'u', + '--nexus2_password', 'p', '--nexus2_repo_name', 'releases', '--nexus2_group_id', 'com.example', + '--nexus2_artifact_id', 'app', '--nexus2_version', '1.0', '--nexus2_extension', 'apk'] + _assert_scanned_ok(run_main(monkeypatch, scan_argv(argv, tmp_path)), mocked_responses) + + +# --- RuStore (overallInfo + download-link + apk; validates zip container) --- +def test_rustore_download_then_scan(mocked_responses, monkeypatch, tmp_path, apk_md5, no_sleep, ms_mode): + mocked_responses.add(responses.GET, + 'https://backapi.rustore.ru/applicationData/overallInfo/com.example.app', + json={'body': {'appId': 42, 'packageName': 'com.example.app', 'versionName': '1.0', + 'versionCode': 1, 'minSdkVersion': 21, 'maxSdkVersion': 33, 'targetSdkVersion': 33, + 'fileSize': len(APK_ZIP), 'iconUrl': 'https://x/i.png', 'companyName': 'Acme'}}) + mocked_responses.add(responses.POST, 'https://backapi.rustore.ru/applicationData/download-link', + json={'body': {'apkUrl': 'https://cdn.rustore.example/app.apk'}}) + mocked_responses.add(responses.GET, 'https://cdn.rustore.example/app.apk', body=APK_ZIP, + content_type='application/vnd.android.package-archive') + register_ms_happy_path(mocked_responses, apk_md5) + argv = ['--distribution_system', 'rustore', '--rustore_package_name', 'com.example.app'] + _assert_scanned_ok(run_main(monkeypatch, scan_argv(argv, tmp_path)), mocked_responses) + + +# --- Firebase (downloader monkeypatched: google-auth/service-account internals out of scope) --- +def test_firebase_download_then_scan(mocked_responses, monkeypatch, tmp_path, apk_md5, no_sleep, ms_mode): + monkeypatch.setattr('mdast_cli.mdast_scan.firebase_download_app', + lambda download_path, *a, **k: _write_app(download_path, 'fb_app-1.0.apk')) + register_ms_happy_path(mocked_responses, apk_md5) + argv = ['--distribution_system', 'firebase', '--firebase_project_number', '123', + '--firebase_app_id', '1:123:android:abc', '--firebase_account_json_path', '/tmp/fake.json', + '--firebase_file_extension', 'apk'] + _assert_scanned_ok(run_main(monkeypatch, scan_argv(argv, tmp_path)), mocked_responses) + + +# --- Google Play (apkeep subprocess monkeypatched) --- +def test_google_play_download_then_scan(mocked_responses, monkeypatch, tmp_path, apk_md5, no_sleep, ms_mode): + monkeypatch.setattr('mdast_cli.mdast_scan.GooglePlay.login', lambda self: None) + monkeypatch.setattr('mdast_cli.mdast_scan.GooglePlay.download_app', + lambda self, download_path, *a, **k: _write_app(download_path, 'gp_app-1.0.apk')) + register_ms_happy_path(mocked_responses, apk_md5) + argv = ['--distribution_system', 'google_play', '--google_play_package_name', 'com.example.app', + '--google_play_email', 'u@example.com', '--google_play_aas_token', 'aas_et/xxx'] + _assert_scanned_ok(run_main(monkeypatch, scan_argv(argv, tmp_path)), mocked_responses) + + +# --- Apple App Store (iOS; StoreClient monkeypatched). iOS -> manual scan on an iOS engine. --- +def _register_ms_ios_manual(rsps, apk_md5): + rsps.add(responses.GET, f'{REST_URL}/architectures/', + json=[{'id': 3, 'type': 'IOS', 'os_version': '16', 'name': 'iOS 16'}]) + rsps.add(responses.GET, f'{REST_URL}/engines/', json=[{'type': 'IOS', 'status': 'STARTED'}]) + rsps.add(responses.GET, f'{REST_URL}/applications/', json=[]) + rsps.add(responses.POST, f'{REST_URL}/applications/upload_info/', + json=application_json(apk_md5), status=201) + rsps.add(responses.POST, f'{REST_URL}/scans/start/precheck/', json={'warnings': []}) + rsps.add(responses.POST, f'{REST_URL}/scans/start/', + json=scan_json(stage='CREATED', status='INITIAL', type='MANUAL')) + rsps.add(responses.POST, f'{REST_URL}/scans/77/start/', + json=scan_json(stage='WORKING', status='PROCESSING')) + rsps.add(responses.POST, f'{REST_URL}/scans/77/stop/', + json=scan_json(stage='STOP', status='PROCESSING')) + rsps.add(responses.GET, f'{REST_URL}/scans/77/', json=scan_json(stage='WORKING', status='PROCESSING')) + rsps.add(responses.GET, f'{REST_URL}/scans/77/', json=scan_json(stage='SUCCESS', status='COMPLETE')) + rsps.add(responses.GET, f'{REST_URL}/scans/77/report', body=b'%PDF-1.4 x', + match=[matchers.query_param_matcher({'output': 'pdf'})]) + + +def test_appstore_download_then_scan(mocked_responses, monkeypatch, tmp_path, apk_md5, no_sleep, ms_mode): + # The AppStore downloader returns its own md5 ('d'*32) as the 2nd tuple element. + # F1: on the microservices installation that md5 is DELIBERATELY ignored - the + # Apple store md5 differs from the file actually uploaded (post-download the IPA + # is re-signed/re-zipped), so dedup/precheck/create must key off the LOCAL file md5. + appstore_md5 = 'd' * 32 + monkeypatch.setattr('mdast_cli.mdast_scan.AppStore.download_app', + lambda self, download_path, *a, **k: (_write_app(download_path, 'as_app-1.0.ipa'), appstore_md5)) + _register_ms_ios_manual(mocked_responses, apk_md5) + argv = ['--distribution_system', 'appstore', '--appstore_app_id', '123', + '--appstore_apple_id', 'u@example.com', '--appstore_password', 'pw', '--appstore_2FA', '123456', + '--download_path', str(tmp_path), '--url', BASE_URL, '--company_id', '1', '--token', TOKEN, + '--profile_id', '2'] + _assert_scanned_ok(run_main(monkeypatch, argv), mocked_responses) + + # The local file md5 (of what was actually written/uploaded), NOT the Apple md5. + local_md5 = hashlib.md5(APK_ZIP).hexdigest().lower() + assert local_md5 != appstore_md5 + # create-scan (POST /scans/start/) must carry the local file md5, not 'd'*32. + create = next(c for c in mocked_responses.calls + if c.request.url == f'{REST_URL}/scans/start/' and c.request.method == 'POST') + assert json.loads(create.request.body)['md5'] == local_md5 + # ...and the Apple store md5 must appear nowhere in the create/precheck payloads. + for c in mocked_responses.calls: + if c.request.method == 'POST' and c.request.body and isinstance(c.request.body, (str, bytes)): + body = c.request.body if isinstance(c.request.body, str) else c.request.body.decode('utf-8', 'ignore') + assert appstore_md5 not in body diff --git a/tests/test_error_envelopes.py b/tests/test_error_envelopes.py new file mode 100644 index 0000000..9e9f7fc --- /dev/null +++ b/tests/test_error_envelopes.py @@ -0,0 +1,53 @@ +"""Contract tests for Clark error envelope parsing. + +/rest routes answer with the monolith envelope {error_code, message}; +the report route answers with FastAPI {"detail": ...}; +scanyon 429 body carries {"detail", "reasons"}. +""" +import pytest +import responses +import requests + +from mdast_cli_core.microservices import extract_error_message + +pytestmark = pytest.mark.contract + +URL = 'https://stand.example/x' + + +def _resp(mocked_responses, **kwargs): + mocked_responses.add(responses.GET, URL, **kwargs) + return requests.get(URL) + + +def test_monolith_envelope(mocked_responses): + message = extract_error_message(_resp( + mocked_responses, status=413, + json={'error_code': 'file_too_large', 'message': 'File exceeds 500 MB limit'})) + assert message == 'file_too_large: File exceeds 500 MB limit' + + +def test_fastapi_detail_string(mocked_responses): + message = extract_error_message(_resp( + mocked_responses, status=409, json={'detail': 'scan is not finished'})) + assert message == 'scan is not finished' + + +def test_fastapi_detail_validation_list(mocked_responses): + message = extract_error_message(_resp( + mocked_responses, status=422, + json={'detail': [{'loc': ['body', 'md5'], 'msg': 'field required'}]})) + assert 'field required' in message + + +def test_429_reasons_preserved(mocked_responses): + message = extract_error_message(_resp( + mocked_responses, status=429, + json={'detail': 'Scan limit exceeded', 'reasons': [{'limit': 'scans_per_day'}]})) + assert 'Scan limit exceeded' in message + assert 'scans_per_day' in message + + +def test_non_json_body(mocked_responses): + message = extract_error_message(_resp(mocked_responses, status=502, body='Bad Gateway')) + assert message == 'Bad Gateway' diff --git a/tests/test_mode_detection.py b/tests/test_mode_detection.py new file mode 100644 index 0000000..832ff81 --- /dev/null +++ b/tests/test_mode_detection.py @@ -0,0 +1,84 @@ +"""Unit tests for installation mode auto-detection and env override. + +Auto-detection probes GET {base}/architectures/ and classifies by payload shape: +microservices = string type (ANDROID/IOS), monolith = int type code. +""" +import pytest +import responses + +from mdast_cli_core.factory import (MODE_MICROSERVICES, MODE_MONOLITH, ModeDetectionError, + resolve_installation_mode) +from tests.conftest import COMPANY_ID, REST_URL, TOKEN + +pytestmark = pytest.mark.unit + +ARCH_URL = f'{REST_URL}/architectures/' +MS_ARCH = [{'id': 1, 'type': 'ANDROID', 'os_version': '11', 'name': 'Android 11'}] +MONO_ARCH = [{'id': 1, 'type': 1, 'name': 'Android 11'}] + + +def test_env_override_microservices(monkeypatch): + monkeypatch.setenv('MDAST_CLI_MODE', 'microservices') + assert resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) == MODE_MICROSERVICES + + +def test_env_override_monolith(monkeypatch): + monkeypatch.setenv('MDAST_CLI_MODE', 'monolith') + assert resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) == MODE_MONOLITH + + +def test_env_invalid_value(monkeypatch): + monkeypatch.setenv('MDAST_CLI_MODE', 'quantum') + with pytest.raises(ModeDetectionError): + resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) + + +def test_autodetect_microservices_by_payload(mocked_responses, monkeypatch): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, ARCH_URL, json=MS_ARCH) # Bearer probe + assert resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) == MODE_MICROSERVICES + assert mocked_responses.calls[0].request.headers['Authorization'] == f'Bearer {TOKEN}' + + +def test_autodetect_monolith_by_payload(mocked_responses, monkeypatch): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + # Bearer probe -> monolith answers 401 (Token-only), then Token probe -> int type + mocked_responses.add(responses.GET, ARCH_URL, status=401) + mocked_responses.add(responses.GET, ARCH_URL, json=MONO_ARCH) + assert resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) == MODE_MONOLITH + assert mocked_responses.calls[1].request.headers['Authorization'] == f'Token {TOKEN}' + + +def test_monolith_200_on_bearer_not_misclassified(mocked_responses, monkeypatch): + """A monolith (int-type payload) answering 200 to the Bearer probe must NOT be + read as microservices - classification is by payload shape, not status.""" + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, ARCH_URL, json=MONO_ARCH) # Bearer -> monolith shape + mocked_responses.add(responses.GET, ARCH_URL, json=MONO_ARCH) # Token -> monolith shape + assert resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) == MODE_MONOLITH + + +def test_ambiguous_200_payload_raises(mocked_responses, monkeypatch): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, ARCH_URL, json={'unexpected': 'shape'}) + mocked_responses.add(responses.GET, ARCH_URL, json={'unexpected': 'shape'}) + with pytest.raises(ModeDetectionError): + resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) + + +def test_autodetect_failure_reports_auth_error(mocked_responses, monkeypatch): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, ARCH_URL, status=401) + mocked_responses.add(responses.GET, ARCH_URL, status=401) + with pytest.raises(ModeDetectionError) as excinfo: + resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) + assert excinfo.value.auth_error is True + + +def test_autodetect_failure_network(mocked_responses, monkeypatch): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, ARCH_URL, status=404) + mocked_responses.add(responses.GET, ARCH_URL, status=500) + with pytest.raises(ModeDetectionError) as excinfo: + resolve_installation_mode(REST_URL, TOKEN, COMPANY_ID) + assert excinfo.value.auth_error is False diff --git a/tests/test_negative_e2e.py b/tests/test_negative_e2e.py new file mode 100644 index 0000000..246f3a7 --- /dev/null +++ b/tests/test_negative_e2e.py @@ -0,0 +1,470 @@ +"""Negative end-to-end flows: the CLI must fail with the RIGHT exit code. + +These exercise main() end to end (arg parse -> mode -> flow) against a mocked +server and assert the process exit code, which is the contract CI depends on: + + 2 INVALID_ARGS | 4 DOWNLOAD_FAILED | 5 SCAN_FAILED | + 6 NETWORK_ERROR | 7 AUTH_ERROR | 8 PRECHECK_BLOCKED + +Exit codes must be identical in meaning on both installations (F12). +""" +import json + +import pytest +import responses +from responses import matchers + +from tests.conftest import BASE_URL, REST_URL, TOKEN, application_json, run_main, scan_json + +pytestmark = pytest.mark.e2e + +ARCH = f'{REST_URL}/architectures/' + + +def base_argv(tmp_apk, *extra): + return ['--distribution_system', 'file', '--file_path', tmp_apk, + '--url', BASE_URL, '--company_id', '1', '--token', TOKEN, *extra] + + +def _register_ms_preamble(rsps, apk_md5, engine_status='STARTED', engine_type='ANDROID', + apps=None): + """Everything up to (not including) upload/create, microservices shaped.""" + rsps.add(responses.GET, ARCH, json=[ + {'id': 1, 'type': 'ANDROID', 'os_version': '11', 'name': 'Android 11'}]) + rsps.add(responses.GET, f'{REST_URL}/engines/', + json=[{'engine_id': 'e-1', 'type': engine_type, 'status': engine_status}]) + rsps.add(responses.GET, f'{REST_URL}/applications/', + json=apps if apps is not None else []) + + +# --- mode detection --------------------------------------------------------- + +def test_mode_detection_auth_failure_exit_7(mocked_responses, monkeypatch, tmp_apk): + """Both probes answer 401 -> AUTH_ERROR, and the token never leaks.""" + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, ARCH, status=401) # Bearer probe + mocked_responses.add(responses.GET, ARCH, status=401) # Token probe + assert run_main(monkeypatch, base_argv(tmp_apk)) == 7 + + +def test_mode_detection_ambiguous_payload_exit_6(mocked_responses, monkeypatch, tmp_apk): + """200 with a payload matching neither shape is a gateway/config problem (network).""" + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, ARCH, json={'unexpected': 'shape'}) + mocked_responses.add(responses.GET, ARCH, json={'unexpected': 'shape'}) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def test_mode_detection_unreachable_exit_6(mocked_responses, monkeypatch, tmp_apk): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + mocked_responses.add(responses.GET, ARCH, status=500) + mocked_responses.add(responses.GET, ARCH, status=503) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +# --- upload ----------------------------------------------------------------- + +def test_ms_upload_gateway_502_exhausted_exit_6(mocked_responses, monkeypatch, tmp_apk, + apk_md5, no_sleep, ms_mode): + """A persistent 502 on upload is retryable infra -> NETWORK_ERROR, not SCAN_FAILED.""" + _register_ms_preamble(mocked_responses, apk_md5) + mocked_responses.add(responses.POST, f'{REST_URL}/applications/upload_info/', status=502) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def test_ms_upload_504_exit_6(mocked_responses, monkeypatch, tmp_apk, apk_md5, no_sleep, ms_mode): + """504 = server still parsing; re-run dedupes. Retryable -> NETWORK_ERROR.""" + _register_ms_preamble(mocked_responses, apk_md5) + mocked_responses.add(responses.POST, f'{REST_URL}/applications/upload_info/', status=504) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def test_ms_upload_auth_401_exit_7(mocked_responses, monkeypatch, tmp_apk, apk_md5, + no_sleep, ms_mode): + _register_ms_preamble(mocked_responses, apk_md5) + mocked_responses.add(responses.POST, f'{REST_URL}/applications/upload_info/', status=401) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 7 + + +# --- preflight rejections --------------------------------------------------- + +def test_ms_no_active_engine_exit_5(mocked_responses, monkeypatch, tmp_apk, apk_md5, + no_sleep, ms_mode): + """Android app but the only engine is iOS -> cannot scan (SCAN_FAILED).""" + _register_ms_preamble(mocked_responses, apk_md5, engine_type='IOS') + assert run_main(monkeypatch, base_argv(tmp_apk)) == 5 + + +def test_ms_engine_present_but_down_exit_5(mocked_responses, monkeypatch, tmp_apk, apk_md5, + no_sleep, ms_mode): + _register_ms_preamble(mocked_responses, apk_md5, engine_status='STOPPED_SIGTERM') + assert run_main(monkeypatch, base_argv(tmp_apk)) == 5 + + +def test_ms_testcase_platform_mismatch_exit_2(mocked_responses, monkeypatch, tmp_apk, apk_md5, + no_sleep, ms_mode): + """Android file + iOS-recorded test case is a user mistake (INVALID_ARGS).""" + mocked_responses.add(responses.GET, ARCH, + json=[{'id': 1, 'type': 'ANDROID', 'os_version': '11'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/testcases/5/', json={'id': 5, 'os': 'IOS'}) + assert run_main(monkeypatch, base_argv(tmp_apk, '--testcase_id', '5')) == 2 + + +# --- scan lifecycle --------------------------------------------------------- + +def _register_ms_through_start(rsps, apk_md5): + rsps.add(responses.GET, f'{REST_URL}/testcases/5/', json={'id': 5, 'os': 'ANDROID'}) + _register_ms_preamble(rsps, apk_md5, apps=[application_json(apk_md5)]) + rsps.add(responses.POST, f'{REST_URL}/scans/start/precheck/', json={'warnings': []}) + rsps.add(responses.POST, f'{REST_URL}/scans/start/', + json=scan_json(stage='CREATED', status='INITIAL')) + rsps.add(responses.POST, f'{REST_URL}/scans/77/start/', + json=scan_json(stage='START', status='INITIAL')) + + +def test_ms_scan_never_terminal_times_out_exit_5(mocked_responses, monkeypatch, tmp_apk, + apk_md5, no_sleep, ms_mode): + """A scan that never reaches a terminal (stage, status) must FAIL, not hang or pass.""" + monkeypatch.setattr('mdast_cli.ms_flow.TRY', 2) + _register_ms_through_start(mocked_responses, apk_md5) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='WORKING', status='PROCESSING')) + assert run_main(monkeypatch, base_argv(tmp_apk, '--testcase_id', '5')) == 5 + + +def test_ms_scan_fail_terminal_exit_5(mocked_responses, monkeypatch, tmp_apk, apk_md5, + no_sleep, ms_mode): + _register_ms_through_start(mocked_responses, apk_md5) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='FAIL', status='FAIL', message='boom')) + assert run_main(monkeypatch, base_argv(tmp_apk, '--testcase_id', '5')) == 5 + + +# --- manual scan without a profile (F10) ------------------------------------ + +def test_ms_manual_without_profile_skips_precheck_and_stops(mocked_responses, monkeypatch, + tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode): + """F10: no --profile_id -> profile is auto-created, so pre-check (422, profile does + not exist yet) is skipped rather than blocking; the manual scan is stopped by the + CLI after the wait and is still expected to finish SUCCESS.""" + monkeypatch.chdir(tmp_path) + _register_ms_preamble(mocked_responses, apk_md5, apps=[application_json(apk_md5)]) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/precheck/', + json={'detail': 'profile not found'}, status=422) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', + json=scan_json(stage='CREATED', status='INITIAL', type='MANUAL')) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/start/', + json=scan_json(stage='WORKING', status='PROCESSING')) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/stop/', + json=scan_json(stage='STOP', status='PROCESSING')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='WORKING', status='PROCESSING')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='SUCCESS', status='COMPLETE')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', body=b'%PDF-1.4 x', + match=[matchers.query_param_matcher({'output': 'pdf'})]) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 0 + create = [c for c in mocked_responses.calls if c.request.url == f'{REST_URL}/scans/start/'] + body = json.loads(create[0].request.body) + assert body['type'] == 'MANUAL' + assert 'profile_id' not in body, 'no --profile_id -> profile_id must be omitted (auto-create)' + assert [c for c in mocked_responses.calls if c.request.url == f'{REST_URL}/scans/77/stop/'], \ + 'manual scan must be stopped by the CLI after the wait' + + +# --- reports (soft-fail) ---------------------------------------------------- + +def test_ms_report_soft_fail_keeps_scan_green(mocked_responses, monkeypatch, tmp_path, + tmp_apk, apk_md5, no_sleep, ms_mode, caplog): + """PDF render fails (Pepper 502) but the scan finished SUCCESS: exit 0, JSON still saved.""" + monkeypatch.chdir(tmp_path) + _register_ms_through_start(mocked_responses, apk_md5) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='SUCCESS', status='COMPLETE')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', status=502, + match=[matchers.query_param_matcher({'output': 'pdf'})]) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', + json={'summary': {'scan_id': 77}}, + match=[matchers.query_param_matcher({'output': 'json'})]) + exit_code = run_main(monkeypatch, base_argv(tmp_apk, '--testcase_id', '5', + '--report_format', 'all')) + assert exit_code == 0 + assert not (tmp_path / 'scan_report_77.pdf').exists() + assert json.loads((tmp_path / 'scan_report_77.json').read_text())['summary']['scan_id'] == 77 + + +def test_ms_default_report_is_pdf(mocked_responses, monkeypatch, tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode): + """No report flags at all -> a PDF is produced by default, named by scan id.""" + monkeypatch.chdir(tmp_path) + _register_ms_through_start(mocked_responses, apk_md5) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='SUCCESS', status='COMPLETE')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', body=b'%PDF-1.4 x', + match=[matchers.query_param_matcher({'output': 'pdf'})]) + assert run_main(monkeypatch, base_argv(tmp_apk, '--testcase_id', '5')) == 0 + assert (tmp_path / 'scan_report_77.pdf').read_bytes().startswith(b'%PDF') + assert not (tmp_path / 'scan_report_77.json').exists() + + +def test_ms_report_format_none_writes_nothing(mocked_responses, monkeypatch, tmp_path, + tmp_apk, apk_md5, no_sleep, ms_mode): + monkeypatch.chdir(tmp_path) + _register_ms_through_start(mocked_responses, apk_md5) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='SUCCESS', status='COMPLETE')) + assert run_main(monkeypatch, base_argv(tmp_apk, '--testcase_id', '5', + '--report_format', 'none')) == 0 + assert not list(tmp_path.glob('scan_report_*')) + + +# --- arg / input validation ------------------------------------------------- + +def test_missing_file_path_exit_2(monkeypatch, tmp_path): + missing = str(tmp_path / 'does_not_exist.apk') + assert run_main(monkeypatch, ['--distribution_system', 'file', '--file_path', missing, + '--url', BASE_URL, '--company_id', '1', '--token', TOKEN]) == 2 + + +def test_scan_without_credentials_exit_2(monkeypatch, tmp_apk): + """No --url/--company_id/--token and not --download_only -> argparse error (2).""" + assert run_main(monkeypatch, ['--distribution_system', 'file', '--file_path', tmp_apk]) == 2 + + +def test_cr_report_without_credentials_exit_2(monkeypatch, tmp_apk): + assert run_main(monkeypatch, base_argv(tmp_apk, '--cr_report')) == 2 + + +# --- URL normalization robustness ------------------------------------------- + +def test_url_already_ending_in_rest_slash_still_works(mocked_responses, monkeypatch, tmp_path, + tmp_apk, apk_md5, no_sleep, ms_mode): + """--url '.../rest/' must not produce '//' paths (which a strict facade 404s). + + All endpoints are registered at the single-slash canonical form; if + normalization regressed to leaving a trailing slash, the requests would miss + and the flow would fail with NETWORK_ERROR instead of 0. + """ + monkeypatch.chdir(tmp_path) + _register_ms_through_start(mocked_responses, apk_md5) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='SUCCESS', status='COMPLETE')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', body=b'%PDF-1.4 x', + match=[matchers.query_param_matcher({'output': 'pdf'})]) + argv = ['--distribution_system', 'file', '--file_path', tmp_apk, + '--url', f'{BASE_URL}/rest/', '--company_id', '1', '--token', TOKEN, + '--testcase_id', '5'] + assert run_main(monkeypatch, argv) == 0 + assert (tmp_path / 'scan_report_77.pdf').exists() + + +# --- monolith parity -------------------------------------------------------- + +def _register_mono_through_start(rsps): + """Monolith manual flow up to (and including) start; dedup returns an existing + app so the upload step is skipped. Caller registers the GET /dasts/77/ polls.""" + rest = REST_URL + rsps.add(responses.GET, f'{rest}/architectures/', + json=[{'id': 1, 'name': 'Android 11', 'type': 1}]) + rsps.add(responses.GET, f'{rest}/organizations/1/engines/', + json=[{'architecture': 1, 'state': 3}]) + rsps.add(responses.GET, f'{rest}/organizations/1/applications/', + json=[{'id': 10, 'package_name': 'com.example.app', 'version_name': '1.0', 'md5': 'x'}]) + rsps.add(responses.POST, f'{rest}/organizations/1/dasts/', + json={'id': 77, 'project': {'id': 1}, 'profile': {'id': 2}}, status=201) + rsps.add(responses.POST, f'{rest}/dasts/77/start/', json={}, status=200) + + +def _register_mono_through_success(rsps): + _register_mono_through_start(rsps) + rest = REST_URL + rsps.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 2}) + rsps.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 2}) + rsps.add(responses.POST, f'{rest}/dasts/77/stop/', json={}, status=200) + rsps.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 4}) + + +def test_autodetect_resolves_monolith_and_runs(mocked_responses, monkeypatch, tmp_apk): + """Auto mode must classify a monolith (int-type payload) and run the monolith flow, + even though the Bearer probe answers 200 - classification is by payload shape.""" + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + _register_mono_through_start(mocked_responses) + assert run_main(monkeypatch, base_argv(tmp_apk, '--nowait')) == 0 + # the Token-scheme /organizations/1/ URLs prove the monolith flow ran + assert any('/organizations/1/' in c.request.url for c in mocked_responses.calls) + + +def test_monolith_mid_poll_401_is_auth_error_exit_7(mocked_responses, monkeypatch, tmp_apk, + no_sleep, monolith_mode): + """F12: a token expiring mid-scan (poll 401) must exit AUTH_ERROR(7), like microservices.""" + _register_mono_through_start(mocked_responses) + mocked_responses.add(responses.GET, f'{REST_URL}/dasts/77/', status=401, + json={'detail': 'token expired'}) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 7 + + +def test_monolith_network_exception_is_network_error_exit_6(monkeypatch, mocked_responses, + tmp_apk, monolith_mode): + """F12: a bare connection error in the monolith flow -> NETWORK_ERROR(6), not a traceback+1.""" + # no endpoints registered -> the first monolith call raises ConnectionError + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def test_monolith_invalid_architecture_id_exit_2(mocked_responses, monkeypatch, tmp_apk, + monolith_mode): + """An --architecture_id the server doesn't know is a user error -> INVALID_ARGS(2), not a crash.""" + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', + json=[{'id': 1, 'name': 'Android 11', 'type': 1}]) + assert run_main(monkeypatch, base_argv(tmp_apk, '--architecture_id', '999')) == 2 + + +def test_monolith_report_pdf_hardfail_exit_5(mocked_responses, monkeypatch, tmp_path, tmp_apk, + no_sleep, monolith_mode): + """Monolith keeps report download as a HARD fail (unlike microservices soft-fail).""" + monkeypatch.chdir(tmp_path) + _register_mono_through_success(mocked_responses) + mocked_responses.add(responses.GET, f'{REST_URL}/dasts/77/report/', status=502) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 5 + + +def test_monolith_json_report_non_json_body_exit_5(mocked_responses, monkeypatch, tmp_path, + tmp_apk, no_sleep, monolith_mode): + """A 200 with a non-JSON body must exit SCAN_FAILED(5), not traceback, and leave no file.""" + monkeypatch.chdir(tmp_path) + _register_mono_through_success(mocked_responses) + mocked_responses.add(responses.GET, f'{REST_URL}/dasts/77/report/', body=b'oops') + assert run_main(monkeypatch, base_argv(tmp_apk, '--report_format', 'json')) == 5 + assert not list(tmp_path.glob('scan_report_*')) + + +def test_monolith_scan_failed_state_exit_5(mocked_responses, monkeypatch, tmp_path, tmp_apk, + no_sleep, monolith_mode): + """Monolith must map a non-SUCCESS terminal state to SCAN_FAILED, same as microservices.""" + monkeypatch.chdir(tmp_path) + rest = REST_URL + mocked_responses.add(responses.GET, f'{rest}/architectures/', + json=[{'id': 1, 'name': 'Android 11', 'type': 1}]) + mocked_responses.add(responses.GET, f'{rest}/organizations/1/engines/', + json=[{'architecture': 1, 'state': 3}]) + mocked_responses.add(responses.GET, f'{rest}/organizations/1/applications/', + json=[{'id': 10, 'package_name': 'com.example.app', + 'version_name': '1.0', 'md5': 'x'}]) + mocked_responses.add(responses.POST, f'{rest}/organizations/1/dasts/', + json={'id': 77, 'project': {'id': 1}, 'profile': {'id': 2}}, status=201) + mocked_responses.add(responses.POST, f'{rest}/dasts/77/start/', json={}, status=200) + mocked_responses.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 2}) + mocked_responses.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 2}) + mocked_responses.add(responses.POST, f'{rest}/dasts/77/stop/', json={}, status=200) + mocked_responses.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 5}) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 5 + + +# ===== more Sting-side negatives (microservices) — expanded coverage ===== + +def test_ms_architectures_5xx_is_network_6(mocked_responses, monkeypatch, tmp_apk, no_sleep, ms_mode): + mocked_responses.add(responses.GET, ARCH, status=500) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def test_ms_architectures_non_json_is_network_6(mocked_responses, monkeypatch, tmp_apk, no_sleep, ms_mode): + """A 200 with a non-JSON body (nginx/Envoy HTML stub) is a gateway problem, not a crash.""" + mocked_responses.add(responses.GET, ARCH, body='gw', status=200, content_type='text/html') + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def test_ms_engines_5xx_is_network_6(mocked_responses, monkeypatch, tmp_apk, no_sleep, ms_mode): + mocked_responses.add(responses.GET, ARCH, json=[{'id': 1, 'type': 'ANDROID', 'os_version': '11'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', status=500) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def test_ms_engines_empty_no_active_engine_5(mocked_responses, monkeypatch, tmp_apk, no_sleep, ms_mode): + mocked_responses.add(responses.GET, ARCH, json=[{'id': 1, 'type': 'ANDROID', 'os_version': '11'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', json=[]) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 5 + + +def test_ms_dedup_5xx_is_network_6(mocked_responses, monkeypatch, tmp_apk, no_sleep, ms_mode): + mocked_responses.add(responses.GET, ARCH, json=[{'id': 1, 'type': 'ANDROID', 'os_version': '11'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', + json=[{'type': 'ANDROID', 'status': 'STARTED'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/applications/', status=500) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def _ms_through_precheck(rsps, apk_md5): + """architectures + testcase + engines + dedup(existing) + precheck OK; caller adds create/start.""" + rsps.add(responses.GET, ARCH, json=[{'id': 1, 'type': 'ANDROID', 'os_version': '11'}]) + rsps.add(responses.GET, f'{REST_URL}/testcases/5/', json={'id': 5, 'os': 'ANDROID'}) + rsps.add(responses.GET, f'{REST_URL}/engines/', json=[{'type': 'ANDROID', 'status': 'STARTED'}]) + rsps.add(responses.GET, f'{REST_URL}/applications/', json=[application_json(apk_md5)]) + rsps.add(responses.POST, f'{REST_URL}/scans/start/precheck/', json={'warnings': []}) + + +def test_ms_create_scan_4xx_is_scan_failed_5(mocked_responses, monkeypatch, tmp_apk, apk_md5, + no_sleep, ms_mode): + _ms_through_precheck(mocked_responses, apk_md5) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', + json={'error_code': 'bad_request', 'message': 'nope'}, status=400) + assert run_main(monkeypatch, base_argv(tmp_apk, '--profile_id', '2', '--testcase_id', '5')) == 5 + + +def test_ms_create_scan_no_id_is_scan_failed_5(mocked_responses, monkeypatch, tmp_apk, apk_md5, + no_sleep, ms_mode): + _ms_through_precheck(mocked_responses, apk_md5) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', json={}, status=200) + assert run_main(monkeypatch, base_argv(tmp_apk, '--profile_id', '2', '--testcase_id', '5')) == 5 + + +def test_ms_start_5xx_is_scan_failed_5(mocked_responses, monkeypatch, tmp_apk, apk_md5, + no_sleep, ms_mode): + _ms_through_precheck(mocked_responses, apk_md5) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', + json=scan_json(stage='CREATED', status='INITIAL')) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/start/', + json={'error_code': 'x', 'message': 'boom'}, status=500) + assert run_main(monkeypatch, base_argv(tmp_apk, '--profile_id', '2', '--testcase_id', '5')) == 5 + + +def test_ms_partial_complete_succeeds_0(mocked_responses, monkeypatch, tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode): + """PARTIAL_COMPLETE is a green terminal (some modules skipped): exit 0 with a warning.""" + monkeypatch.chdir(tmp_path) + _register_ms_through_start(mocked_responses, apk_md5) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='SUCCESS', status='PARTIAL_COMPLETE')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', body=b'%PDF-1.4 x', + match=[matchers.query_param_matcher({'output': 'pdf'})]) + assert run_main(monkeypatch, base_argv(tmp_apk, '--testcase_id', '5')) == 0 + + +def test_ms_both_reports_fail_still_green_0(mocked_responses, monkeypatch, tmp_path, tmp_apk, + apk_md5, no_sleep, ms_mode): + """Scan SUCCESS but BOTH reports 502 -> soft-fail: exit 0 (report service is separate).""" + monkeypatch.chdir(tmp_path) + _register_ms_through_start(mocked_responses, apk_md5) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='SUCCESS', status='COMPLETE')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', status=502, + match=[matchers.query_param_matcher({'output': 'pdf'})]) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/report', status=502, + match=[matchers.query_param_matcher({'output': 'json'})]) + assert run_main(monkeypatch, base_argv(tmp_apk, '--testcase_id', '5', '--report_format', 'all')) == 0 + + +# ===== more Sting-side negatives (monolith) ===== + +def test_monolith_architectures_5xx_is_network_6(mocked_responses, monkeypatch, tmp_apk, + no_sleep, monolith_mode): + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', status=500) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 6 + + +def test_monolith_no_active_engine_5(mocked_responses, monkeypatch, tmp_apk, no_sleep, monolith_mode): + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', + json=[{'id': 1, 'name': 'Android 11', 'type': 1}]) + # engine present but state != 3 (not active) + mocked_responses.add(responses.GET, f'{REST_URL}/organizations/1/engines/', + json=[{'architecture': 1, 'state': 2}]) + assert run_main(monkeypatch, base_argv(tmp_apk)) == 5 diff --git a/tests/test_poll_resilience.py b/tests/test_poll_resilience.py new file mode 100644 index 0000000..cbd394b --- /dev/null +++ b/tests/test_poll_resilience.py @@ -0,0 +1,90 @@ +"""Poll resilience: transient downstream 5xx during scan polling must be retried, +not fatal (a long CI poll should survive flaky blips while the scan keeps running).""" +from unittest import mock + +import pytest +import requests +import responses + +from mdast_cli.helpers.exit_codes import ExitCode +from mdast_cli import ms_flow +from mdast_cli_core.microservices import mDastMicroservices +from tests.conftest import REST_URL, TOKEN + +pytestmark = pytest.mark.unit + +SCAN_URL = f'{REST_URL}/scans/55/' + + +@pytest.fixture +def client(): + return mDastMicroservices(REST_URL, TOKEN) + + +def test_transient_502_then_success(client, mocked_responses, no_sleep): + mocked_responses.add(responses.GET, SCAN_URL, status=502, + json={'error_code': 'downstream_unavailable', 'message': 'x'}) + mocked_responses.add(responses.GET, SCAN_URL, status=503, json={'error_code': 'busy'}) + mocked_responses.add(responses.GET, SCAN_URL, + json={'id': 55, 'stage': 'WORKING', 'status': 'PROCESSING'}) + scan = ms_flow._get_scan(client, 55) + assert scan['stage'] == 'WORKING' + + +def test_persistent_502_exits_network_error(client, mocked_responses, no_sleep): + for _ in range(ms_flow.POLL_TRANSIENT_RETRIES + 1): + mocked_responses.add(responses.GET, SCAN_URL, status=502, + json={'error_code': 'downstream_unavailable', 'message': 'x'}) + with pytest.raises(SystemExit) as excinfo: + ms_flow._get_scan(client, 55) + assert excinfo.value.code == ExitCode.NETWORK_ERROR + + +def test_404_is_not_retried(client, mocked_responses, no_sleep): + mocked_responses.add(responses.GET, SCAN_URL, status=404, json={'error_code': 'not_found'}) + with pytest.raises(SystemExit): + ms_flow._get_scan(client, 55) + # only one call - 404 is terminal, not retried + assert len([c for c in mocked_responses.calls if c.request.url == SCAN_URL]) == 1 + + +def test_network_exception_retried_then_success(client, no_sleep): + resp_ok = mock.Mock(status_code=200) + resp_ok.json.return_value = {'id': 55, 'stage': 'SUCCESS', 'status': 'COMPLETE'} + seq = [requests.ConnectionError('boom'), resp_ok] + with mock.patch.object(client, 'get_scan_info', side_effect=seq): + scan = ms_flow._get_scan(client, 55) + assert scan['status'] == 'COMPLETE' + + +# --- report download resilience (soft-fail contract, STG-4478) --- + +REPORT_URL = f'{REST_URL}/scans/55/report' + + +def test_report_transient_502_then_success(client, mocked_responses, no_sleep): + """A transient 502 on the report service is retried, then the PDF is returned.""" + mocked_responses.add(responses.GET, REPORT_URL, status=502, json={'error_code': 'busy'}) + mocked_responses.add(responses.GET, REPORT_URL, body=b'%PDF-1.4 x', + content_type='application/pdf') + resp = ms_flow._download_report(client.download_report, 55, 'PDF report') + assert resp is not None + assert resp.content == b'%PDF-1.4 x' + + +def test_report_persistent_failure_soft_fails_to_none(client, mocked_responses, no_sleep): + """A report that never renders returns None (soft-fail) - it must NOT raise: + the scan already succeeded, so a report-service outage cannot turn it red.""" + for _ in range(ms_flow.POLL_TRANSIENT_RETRIES + 1): + mocked_responses.add(responses.GET, REPORT_URL, status=502, json={'error_code': 'busy'}) + resp = ms_flow._download_report(client.download_report, 55, 'PDF report') + assert resp is None + + +def test_report_network_exception_then_success(client, no_sleep): + """A raw connection drop on report fetch is retried, not fatal.""" + resp_ok = mock.Mock(status_code=200, content=b'%PDF-1.4 x') + seq = [requests.ConnectionError('boom'), resp_ok] + fetch = mock.Mock(side_effect=seq) + resp = ms_flow._download_report(fetch, 55, 'PDF report') + assert resp is resp_ok diff --git a/tests/test_precheck_gate.py b/tests/test_precheck_gate.py new file mode 100644 index 0000000..3e8166c --- /dev/null +++ b/tests/test_precheck_gate.py @@ -0,0 +1,107 @@ +"""Unit/contract tests for the pre-check gate (DEC-671-05) and warning rendering.""" +import pytest +import responses + +from mdast_cli.helpers.exit_codes import ExitCode +from mdast_cli.ms_flow import render_precheck_warning, run_precheck_gate +from mdast_cli_core.microservices import mDastMicroservices +from tests.conftest import REST_URL, TOKEN + +pytestmark = [pytest.mark.unit, pytest.mark.contract] + +PRECHECK_URL = f'{REST_URL}/scans/start/precheck/' + + +@pytest.fixture +def client(): + return mDastMicroservices(REST_URL, TOKEN) + + +def test_empty_warnings_pass(client, mocked_responses): + mocked_responses.add(responses.POST, PRECHECK_URL, json={'warnings': []}) + run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None, scan_type='MANUAL') + + +def test_warnings_block_with_exit_code_8(client, mocked_responses, capsys): + mocked_responses.add(responses.POST, PRECHECK_URL, json={'warnings': [ + {'type': 'empty_profile', 'payload': {}}, + {'type': 'architecture_unsupported', + 'payload': {'platform': 'Android', 'os_version': '99'}}, + ]}) + with pytest.raises(SystemExit) as excinfo: + run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None, scan_type='MANUAL') + assert excinfo.value.code == ExitCode.PRECHECK_BLOCKED == 8 + stderr = capsys.readouterr().err + assert '[empty_profile]' in stderr + assert '[architecture_unsupported]' in stderr + assert '"os_version": "99"' in stderr + + +def test_precheck_unavailable_blocks(client, mocked_responses): + """precheck_unavailable is a blocking warning too (STG-4475).""" + mocked_responses.add(responses.POST, PRECHECK_URL, json={'warnings': [ + {'type': 'precheck_unavailable', + 'payload': {'unavailable_dependencies': ['application_lookup']}}, + ]}) + with pytest.raises(SystemExit) as excinfo: + run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None, scan_type='MANUAL') + assert excinfo.value.code == ExitCode.PRECHECK_BLOCKED + + +def test_precheck_gateway_5xx_is_network_error_not_block(client, mocked_responses, no_sleep): + """A transient gateway 5xx is retryable infra, not a policy block -> NETWORK_ERROR(6).""" + mocked_responses.add(responses.POST, PRECHECK_URL, status=502, + json={'error_code': 'downstream_unavailable', 'message': 'Scanyon down'}) + with pytest.raises(SystemExit) as excinfo: + run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None, scan_type='MANUAL') + assert excinfo.value.code == ExitCode.NETWORK_ERROR + + +def test_precheck_network_exception_is_network_error(client, mocked_responses, no_sleep): + """A raw connection error during pre-check -> NETWORK_ERROR(6), still fail-closed.""" + # no mock registered for PRECHECK_URL -> responses raises ConnectionError + with pytest.raises(SystemExit) as excinfo: + run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None, scan_type='MANUAL') + assert excinfo.value.code == ExitCode.NETWORK_ERROR + + +def test_precheck_non_transient_non_200_blocks(client, mocked_responses): + """A genuine non-transient rejection (e.g. 400) is a policy block -> exit 8.""" + mocked_responses.add(responses.POST, PRECHECK_URL, status=400, + json={'error_code': 'bad_request', 'message': 'nope'}) + with pytest.raises(SystemExit) as excinfo: + run_precheck_gate(client, 'a' * 32, profile_id=1, testcase_id=None, scan_type='MANUAL') + assert excinfo.value.code == ExitCode.PRECHECK_BLOCKED + + +def test_profile_not_found_fails(client, mocked_responses): + mocked_responses.add(responses.POST, PRECHECK_URL, status=404, + json={'error_code': 'not_found', 'message': 'profile not found'}) + with pytest.raises(SystemExit) as excinfo: + run_precheck_gate(client, 'a' * 32, profile_id=42, testcase_id=None, scan_type='MANUAL') + assert excinfo.value.code == ExitCode.SCAN_FAILED + + +def test_missing_profile_id_skips_on_422(client, mocked_responses, caplog): + """Interim behavior until scanyon makes profile_id optional: 422 + no profile -> skip.""" + mocked_responses.add(responses.POST, PRECHECK_URL, status=422, + json={'error_code': 'validation_error', 'message': 'profile_id required'}) + run_precheck_gate(client, 'a' * 32, profile_id=None, testcase_id=None, scan_type='MANUAL') + assert any('pre-check skipped' in record.message.lower() for record in caplog.records) + + +def test_unknown_warning_type_rendered_generically(): + """Contract extension is backward-compatible: unknown types must not crash (DEC-666-04).""" + line = render_precheck_warning({'type': 'brand_new_rule', 'payload': {'x': 1}}) + assert '[brand_new_rule]' in line + assert '"x": 1' in line + + +def test_warning_without_payload_rendered(): + line = render_precheck_warning({'type': 'empty_profile'}) + assert '[empty_profile]' in line + + +def test_malformed_warning_rendered(): + line = render_precheck_warning({}) + assert '[unknown]' in line diff --git a/tests/test_report_targets.py b/tests/test_report_targets.py new file mode 100644 index 0000000..94468ee --- /dev/null +++ b/tests/test_report_targets.py @@ -0,0 +1,55 @@ +"""Unit tests for the shared report-target resolver (used by both installations).""" +from types import SimpleNamespace + +import pytest + +from mdast_cli.helpers.helpers import resolve_report_targets + +pytestmark = pytest.mark.unit + + +def args(report_format=None, pdf=None, json=None): + return SimpleNamespace(report_format=report_format, + pdf_report_file_name=pdf, + summary_report_json_file_name=json) + + +def test_default_is_pdf_named_by_scan_id(): + assert resolve_report_targets(args(), 77) == {'pdf': 'scan_report_77.pdf'} + + +def test_format_json_only(): + assert resolve_report_targets(args(report_format='json'), 77) == {'json': 'scan_report_77.json'} + + +def test_format_all_produces_both(): + assert resolve_report_targets(args(report_format='all'), 77) == { + 'pdf': 'scan_report_77.pdf', 'json': 'scan_report_77.json'} + + +def test_format_none_produces_nothing(): + assert resolve_report_targets(args(report_format='none'), 77) == {} + + +def test_explicit_pdf_name_only(): + assert resolve_report_targets(args(pdf='r'), 77) == {'pdf': 'r.pdf'} + + +def test_explicit_json_name_does_not_pull_in_default_pdf(): + # regression: a defaulted 'pdf' format must not sneak a PDF in when the user + # asked only for a JSON file by name. + assert resolve_report_targets(args(json='r'), 77) == {'json': 'r.json'} + + +def test_extension_added_only_when_missing(): + assert resolve_report_targets(args(pdf='r.pdf', json='r.json'), 77) == { + 'pdf': 'r.pdf', 'json': 'r.json'} + + +def test_format_flag_adds_other_format_to_explicit_name(): + assert resolve_report_targets(args(report_format='all', pdf='p'), 77) == { + 'pdf': 'p.pdf', 'json': 'scan_report_77.json'} + + +def test_explicit_name_wins_over_none(): + assert resolve_report_targets(args(report_format='none', pdf='p'), 77) == {'pdf': 'p.pdf'} diff --git a/tests/test_security.py b/tests/test_security.py new file mode 100644 index 0000000..b305407 --- /dev/null +++ b/tests/test_security.py @@ -0,0 +1,125 @@ +"""Security-focused tests: secrets handling, server-controlled inputs, TLS.""" +import logging +import os +from unittest import mock + +import pytest +import responses + +from mdast_cli_core import factory +from mdast_cli_core.microservices import mDastMicroservices +from tests.conftest import BASE_URL, REST_URL, TOKEN, application_json, run_main, scan_json +from tests.test_smoke_flows import ms_argv, register_ms_happy_path + +pytestmark = pytest.mark.security + + +def test_token_never_appears_in_logs(mocked_responses, monkeypatch, tmp_path, tmp_apk, + apk_md5, no_sleep, ms_mode, caplog, capsys): + """The org token is a credential: it must not leak to logs, stdout or stderr.""" + monkeypatch.chdir(tmp_path) + register_ms_happy_path(mocked_responses, apk_md5) + with caplog.at_level(logging.DEBUG): + exit_code = run_main(monkeypatch, ms_argv(tmp_apk) + ['--verbose']) + assert exit_code == 0 + for record in caplog.records: + assert TOKEN not in record.getMessage() + captured = capsys.readouterr() + assert TOKEN not in captured.out + assert TOKEN not in captured.err + + +def test_token_never_in_urls(mocked_responses, monkeypatch, tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode): + """The token travels only in the Authorization header, never in a URL/query.""" + monkeypatch.chdir(tmp_path) + register_ms_happy_path(mocked_responses, apk_md5) + run_main(monkeypatch, ms_argv(tmp_apk)) + for call in mocked_responses.calls: + assert TOKEN not in call.request.url + assert call.request.headers['Authorization'] == f'Bearer {TOKEN}' + + +def test_content_disposition_cannot_control_local_path(mocked_responses, monkeypatch, + tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode): + """A hostile/compromised server must not choose where the CLI writes files. + + Report file names come from CLI arguments only; Content-Disposition + (path traversal attempt here) is ignored. + """ + monkeypatch.chdir(tmp_path) + register_ms_happy_path(mocked_responses, apk_md5) + mocked_responses.replace( + responses.GET, f'{REST_URL}/scans/77/report', body=b'%PDF-1.4 fake', + headers={'Content-Disposition': 'attachment; filename="../../../../tmp/evil.pdf"'}) + exit_code = run_main(monkeypatch, ms_argv(tmp_apk)) + assert exit_code == 0 + written = sorted(p.name for p in tmp_path.iterdir()) + assert 'scan_report_pdf.pdf' in written + assert not any('evil' in name for name in written) + assert not os.path.exists('/tmp/evil.pdf') + + +def test_x_file_size_matches_actual_content(mocked_responses, tmp_apk): + """Anti request smuggling consistency: X-File-Size equals the real binary size.""" + mocked_responses.add(responses.POST, f'{REST_URL}/applications/upload_info/', + json={'id': 1}, status=201) + client = mDastMicroservices(REST_URL, TOKEN) + client.upload_application(tmp_apk) + request = mocked_responses.calls[-1].request + declared = int(request.headers['X-File-Size']) + assert declared == os.path.getsize(tmp_apk) + assert declared <= int(request.headers['Content-Length']) + + +def test_tls_verify_enabled_by_default(): + """F4: secure by default - client verifies TLS unless explicitly told not to.""" + client = mDastMicroservices(REST_URL, TOKEN) + with mock.patch('mdast_cli_core.microservices.requests.get') as mocked_get: + client.get_architectures() + assert mocked_get.call_args.kwargs['verify'] is True + + +def test_tls_verify_can_be_disabled_via_client_flag(): + client = mDastMicroservices(REST_URL, TOKEN, verify=False) + with mock.patch('mdast_cli_core.microservices.requests.get') as mocked_get: + client.get_architectures() + assert mocked_get.call_args.kwargs['verify'] is False + + +@pytest.mark.parametrize('raw,expected', [ + # secure by default: unset/empty -> True; only explicit off-values disable + (None, True), ('1', True), ('true', True), ('YES', True), ('on', True), ('', True), + ('0', False), ('false', False), ('no', False), ('off', False), +]) +def test_tls_verify_env_parsing(monkeypatch, raw, expected): + if raw is None: + monkeypatch.delenv(factory.TLS_VERIFY_ENV_VAR, raising=False) + else: + monkeypatch.setenv(factory.TLS_VERIFY_ENV_VAR, raw) + assert factory.tls_verify_enabled() is expected + + +def test_precheck_payload_with_control_characters_is_safe(): + """Warning payload is server-controlled: rendering must be json-escaped, one line.""" + from mdast_cli.ms_flow import render_precheck_warning + line = render_precheck_warning({ + 'type': 'empty_profile', + 'payload': {'note': 'line1\nline2\x1b[31mred'}, + }) + assert '\n' not in line + assert '\x1b' not in line + + +def test_mode_probe_does_not_leak_token_on_error(mocked_responses, monkeypatch): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + # The factory probes GET /architectures/ (Bearer, then Token) - both 401 here. + # One registration serves both probes (responses reuses it for repeat calls). + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', status=401) + with pytest.raises(factory.ModeDetectionError) as excinfo: + factory.resolve_installation_mode(REST_URL, TOKEN, '1') + # An auth failure must be classified as such, and the token must never appear + # in the surfaced error (it is logged/raised near the Authorization header). + assert excinfo.value.auth_error is True + assert TOKEN not in str(excinfo.value) diff --git a/tests/test_smoke_flows.py b/tests/test_smoke_flows.py new file mode 100644 index 0000000..bf6060e --- /dev/null +++ b/tests/test_smoke_flows.py @@ -0,0 +1,244 @@ +"""Smoke tests: full CLI flows against a mocked server. + +The microservices flow follows the acceptance scenario of STG-4475: +architectures -> engines -> dedup/upload -> pre-check gate -> create (fsm_locked) +-> start -> polling to a terminal (stage, status) -> reports. +The monolith flow is pinned as a regression guard (dual-mode must not break it). +""" +import json +import os + +import pytest +import responses + +from tests.conftest import BASE_URL, REST_URL, TOKEN, application_json, run_main, scan_json + +pytestmark = pytest.mark.smoke + + +def register_ms_happy_path(rsps, apk_md5, with_testcase=True): + rsps.add(responses.GET, f'{REST_URL}/architectures/', json=[ + {'id': 1, 'type': 'ANDROID', 'os_version': '11', 'name': 'Android 11', 'description': 'API 30'}, + ]) + if with_testcase: + rsps.add(responses.GET, f'{REST_URL}/testcases/5/', json={'id': 5, 'os': 'ANDROID'}) + rsps.add(responses.GET, f'{REST_URL}/engines/', json=[ + {'engine_id': 'e-1', 'type': 'ANDROID', 'status': 'STARTED'}, + {'engine_id': 'e-2', 'type': 'IOS', 'status': 'STOPPED_SIGTERM'}, + ]) + rsps.add(responses.GET, f'{REST_URL}/applications/', json=[]) + rsps.add(responses.POST, f'{REST_URL}/applications/upload_info/', + json=application_json(apk_md5), status=201) + rsps.add(responses.POST, f'{REST_URL}/scans/start/precheck/', json={'warnings': []}) + rsps.add(responses.POST, f'{REST_URL}/scans/start/', + json=scan_json(stage='CREATED', status='INITIAL'), status=200) + rsps.add(responses.POST, f'{REST_URL}/scans/77/start/', + json=scan_json(stage='START', status='INITIAL')) + rsps.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='WORKING', status='PROCESSING')) + rsps.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='SUCCESS', status='COMPLETE')) + rsps.add(responses.GET, f'{REST_URL}/scans/77/report', body=b'%PDF-1.4 fake', + content_type='application/pdf', + headers={'Content-Disposition': 'attachment; filename="dast_77.pdf"'}) + rsps.add(responses.GET, f'{REST_URL}/scans/77/report', + json={'summary': {'scan_id': 77}, 'defect_summary': {}, 'requirements': [], + 'defects': []}, + headers={'Content-Disposition': 'attachment; filename="dast_77.json"'}) + + +def ms_argv(tmp_apk, with_testcase=True): + argv = ['--distribution_system', 'file', '--file_path', tmp_apk, + '--url', BASE_URL, '--company_id', '1', '--token', TOKEN, + '--profile_id', '2', + '--pdf_report_file_name', 'scan_report_pdf', + '--summary_report_json_file_name', 'scan_report_json'] + if with_testcase: + argv += ['--testcase_id', '5'] + return argv + + +def test_ms_full_flow_success(mocked_responses, monkeypatch, tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode): + monkeypatch.chdir(tmp_path) + register_ms_happy_path(mocked_responses, apk_md5) + exit_code = run_main(monkeypatch, ms_argv(tmp_apk)) + assert exit_code == 0 + assert (tmp_path / 'scan_report_pdf.pdf').read_bytes().startswith(b'%PDF') + report = json.loads((tmp_path / 'scan_report_json.json').read_text()) + assert report['summary']['scan_id'] == 77 + # create scan body follows the scanyon-native contract + create_calls = [c for c in mocked_responses.calls + if c.request.url == f'{REST_URL}/scans/start/'] + body = json.loads(create_calls[0].request.body) + assert body['md5'] == apk_md5 + assert body['type'] == 'AUTO' + assert body['fsm_locked'] is True + + +def test_ms_full_flow_with_autodetect(mocked_responses, monkeypatch, tmp_path, tmp_apk, + apk_md5, no_sleep): + monkeypatch.delenv('MDAST_CLI_MODE', raising=False) + monkeypatch.chdir(tmp_path) + register_ms_happy_path(mocked_responses, apk_md5) + exit_code = run_main(monkeypatch, ms_argv(tmp_apk)) + assert exit_code == 0 + + +def test_ms_manual_flow_stops_scan(mocked_responses, monkeypatch, tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode): + """Scan without a test case keeps the manual semantics: wait, stop, expect SUCCESS.""" + monkeypatch.chdir(tmp_path) + register_ms_happy_path(mocked_responses, apk_md5, with_testcase=False) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/stop/', + json=scan_json(stage='STOP', status='PROCESSING')) + exit_code = run_main(monkeypatch, ms_argv(tmp_apk, with_testcase=False)) + assert exit_code == 0 + stop_calls = [c for c in mocked_responses.calls + if c.request.url == f'{REST_URL}/scans/77/stop/'] + assert stop_calls, 'manual flow must request scan stop' + + +def test_ms_precheck_gate_blocks(mocked_responses, monkeypatch, tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode, capsys): + monkeypatch.chdir(tmp_path) + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', json=[]) + mocked_responses.add(responses.GET, f'{REST_URL}/testcases/5/', json={'os': 'ANDROID'}) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', + json=[{'type': 'ANDROID', 'status': 'STARTED'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/applications/', + json=[application_json(apk_md5)]) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/precheck/', json={ + 'warnings': [{'type': 'empty_profile', 'payload': {}}]}) + exit_code = run_main(monkeypatch, ms_argv(tmp_apk)) + assert exit_code == 8 + assert '[empty_profile]' in capsys.readouterr().err + scan_creates = [c for c in mocked_responses.calls + if c.request.url == f'{REST_URL}/scans/start/'] + assert not scan_creates, 'scan must not be created when pre-check blocks' + + +def test_ms_scan_fail_state(mocked_responses, monkeypatch, tmp_path, tmp_apk, apk_md5, + no_sleep, ms_mode): + monkeypatch.chdir(tmp_path) + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', json=[]) + mocked_responses.add(responses.GET, f'{REST_URL}/testcases/5/', json={'os': 'ANDROID'}) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', + json=[{'type': 'ANDROID', 'status': 'STARTED'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/applications/', + json=[application_json(apk_md5)]) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/precheck/', + json={'warnings': []}) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', + json=scan_json(stage='CREATED', status='INITIAL')) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/start/', + json=scan_json(stage='START')) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='FAIL', status='FAIL', + message='All modules failed')) + exit_code = run_main(monkeypatch, ms_argv(tmp_apk)) + assert exit_code == 5 + + +def test_ms_start_409_facade_retry_is_tolerated(mocked_responses, monkeypatch, tmp_path, + tmp_apk, apk_md5, no_sleep, ms_mode): + """POST /scans/{id}/start/ is not idempotent: a facade retry after the first + (successful) call returns 409, but the scan did start. CLI must confirm by state + and continue, not fail.""" + monkeypatch.chdir(tmp_path) + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', json=[]) + mocked_responses.add(responses.GET, f'{REST_URL}/testcases/5/', json={'os': 'ANDROID'}) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', + json=[{'type': 'ANDROID', 'status': 'STARTED'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/applications/', + json=[application_json(apk_md5)]) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/precheck/', + json={'warnings': []}) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', + json=scan_json(stage='CREATED', status='INITIAL')) + # start returns 409 (retry after first unlock succeeded) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/start/', + json={'error_code': 'bad_request', 'message': 'недоступны'}, status=409) + # but the scan is already past initial - 409 must be tolerated + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='START', status='INITIAL')) + exit_code = run_main(monkeypatch, ms_argv(tmp_apk) + ['--nowait']) + assert exit_code == 0 + + +def test_ms_start_409_real_conflict_fails(mocked_responses, monkeypatch, tmp_path, + tmp_apk, apk_md5, no_sleep, ms_mode): + """A genuine 409 (scan still in CREATED) must NOT be masked - CLI fails.""" + monkeypatch.chdir(tmp_path) + mocked_responses.add(responses.GET, f'{REST_URL}/architectures/', json=[]) + mocked_responses.add(responses.GET, f'{REST_URL}/testcases/5/', json={'os': 'ANDROID'}) + mocked_responses.add(responses.GET, f'{REST_URL}/engines/', + json=[{'type': 'ANDROID', 'status': 'STARTED'}]) + mocked_responses.add(responses.GET, f'{REST_URL}/applications/', + json=[application_json(apk_md5)]) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/precheck/', + json={'warnings': []}) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/start/', + json=scan_json(stage='CREATED', status='INITIAL')) + mocked_responses.add(responses.POST, f'{REST_URL}/scans/77/start/', + json={'error_code': 'bad_request', 'message': 'x'}, status=409) + mocked_responses.add(responses.GET, f'{REST_URL}/scans/77/', + json=scan_json(stage='CREATED', status='INITIAL')) + exit_code = run_main(monkeypatch, ms_argv(tmp_apk) + ['--nowait']) + assert exit_code == 5 + + +def test_ms_appium_rejected(monkeypatch, tmp_apk, ms_mode): + exit_code = run_main(monkeypatch, [ + '--distribution_system', 'file', '--file_path', tmp_apk, + '--url', BASE_URL, '--company_id', '1', '--token', TOKEN, + '--appium_script_path', '/tmp/script.py']) + assert exit_code == 2 + + +def test_ms_cr_report_rejected(monkeypatch, tmp_apk, ms_mode): + exit_code = run_main(monkeypatch, [ + '--distribution_system', 'file', '--file_path', tmp_apk, + '--url', BASE_URL, '--company_id', '1', '--token', TOKEN, + '--cr_report', '--stingray_login', 'x', '--stingray_password', 'y']) + assert exit_code == 2 + + +def test_monolith_regression_flow(mocked_responses, monkeypatch, tmp_path, tmp_apk, + no_sleep, monolith_mode): + """The pre-dual-mode monolith flow must keep working byte-for-byte.""" + monkeypatch.chdir(tmp_path) + rest = REST_URL + mocked_responses.add(responses.GET, f'{rest}/architectures/', json=[ + {'id': 1, 'name': 'Android 11', 'type': 1}, + {'id': 3, 'name': 'iOS 14', 'type': 2}, + ]) + mocked_responses.add(responses.GET, f'{rest}/organizations/1/engines/', json=[ + {'architecture': 1, 'state': 3}, + ]) + mocked_responses.add(responses.GET, f'{rest}/organizations/1/applications/', json=[]) + mocked_responses.add(responses.POST, f'{rest}/organizations/1/applications/', + json={'id': 10}, status=201) + mocked_responses.add(responses.POST, f'{rest}/organizations/1/dasts/', + json={'id': 77, 'project': {'id': 1}, 'profile': {'id': 2}}, + status=201) + mocked_responses.add(responses.POST, f'{rest}/dasts/77/start/', json={}, status=200) + mocked_responses.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 2}) + mocked_responses.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 2}) + mocked_responses.add(responses.POST, f'{rest}/dasts/77/stop/', json={}, status=200) + mocked_responses.add(responses.GET, f'{rest}/dasts/77/', json={'id': 77, 'state': 4}) + mocked_responses.add(responses.GET, f'{rest}/dasts/77/report/', body=b'%PDF-1.4 mono') + mocked_responses.add(responses.GET, f'{rest}/dasts/77/report/', json={'summary': {}}) + exit_code = run_main(monkeypatch, [ + '--distribution_system', 'file', '--file_path', tmp_apk, + '--url', BASE_URL, '--company_id', '1', '--token', TOKEN, + '--pdf_report_file_name', 'mono_pdf', + '--summary_report_json_file_name', 'mono_json']) + assert exit_code == 0 + assert (tmp_path / 'mono_pdf.pdf').read_bytes().startswith(b'%PDF') + assert (tmp_path / 'mono_json.json').exists() + # monolith URL shapes preserved: organization segment and Token scheme + engine_calls = [c for c in mocked_responses.calls + if '/organizations/1/engines/' in c.request.url] + assert engine_calls + assert engine_calls[0].request.headers['Authorization'] == f'Token {TOKEN}' diff --git a/tests/test_state_machine.py b/tests/test_state_machine.py new file mode 100644 index 0000000..affda77 --- /dev/null +++ b/tests/test_state_machine.py @@ -0,0 +1,57 @@ +"""Unit tests for the scanyon-native (stage, status) state machine helpers.""" +import pytest + +from mdast_cli.helpers.const import TERMINAL_SCAN_PAIRS +from mdast_cli.ms_flow import is_success, is_terminal, resolve_platform, scan_pair + +pytestmark = pytest.mark.unit + + +@pytest.mark.parametrize('stage,status', sorted(TERMINAL_SCAN_PAIRS)) +def test_terminal_pairs(stage, status): + assert is_terminal({'stage': stage, 'status': status}) + + +@pytest.mark.parametrize('stage,status', [ + ('CREATED', 'INITIAL'), + ('START', 'PROCESSING'), + ('WORKING', 'PROCESSING'), + ('WORKING', 'WAITING'), + ('STOP', 'INITIAL'), + ('STOP', 'PROCESSING'), + # STOP is transitional even with COMPLETE status: the FSM proceeds to SUCCESS + ('STOP', 'COMPLETE'), + ('SUCCESS', 'PROCESSING'), + ('FAIL', 'PROCESSING'), +]) +def test_non_terminal_pairs(stage, status): + assert not is_terminal({'stage': stage, 'status': status}) + + +def test_success_complete(): + assert is_success({'stage': 'SUCCESS', 'status': 'COMPLETE'}) + + +def test_success_partial_complete(): + assert is_success({'stage': 'SUCCESS', 'status': 'PARTIAL_COMPLETE'}) + + +def test_fail_is_not_success(): + assert not is_success({'stage': 'FAIL', 'status': 'FAIL'}) + + +def test_scan_pair_missing_fields(): + assert scan_pair({}) == (None, None) + assert not is_terminal({}) + + +@pytest.mark.parametrize('file_name,expected', [ + ('app.apk', 'ANDROID'), + ('app.aab', 'ANDROID'), + ('app.apks', 'ANDROID'), + ('app.zip', 'ANDROID'), + ('app.ipa', 'IOS'), + ('app.exe', None), +]) +def test_resolve_platform(file_name, expected): + assert resolve_platform(f'/tmp/{file_name}') == expected