diff --git a/.github/workflows/sklearn.dispatch-release.yml b/.github/workflows/sklearn.dispatch-release.yml index 6746155e66cf..d310cb28c1c2 100644 --- a/.github/workflows/sklearn.dispatch-release.yml +++ b/.github/workflows/sklearn.dispatch-release.yml @@ -4,12 +4,13 @@ on: workflow_dispatch: inputs: config-file: - description: "Path to image config YAML (e.g., .github/config/image/sklearn/sagemaker-py312.yml)" + description: "Path to image config YAML" required: true type: string + default: ".github/config/image/sklearn/sagemaker-py312.yml" concurrency: - group: ${{ github.workflow }} + group: ${{ github.workflow }}-${{ inputs.config-file }} cancel-in-progress: false permissions: diff --git a/.github/workflows/sklearn.pipeline.yml b/.github/workflows/sklearn.pipeline.yml index a2576848679d..5c99c579f045 100644 --- a/.github/workflows/sklearn.pipeline.yml +++ b/.github/workflows/sklearn.pipeline.yml @@ -41,6 +41,11 @@ on: required: false type: boolean default: true + run-sagemaker-test: + description: "Run SageMaker tests (real training jobs + endpoints)" + required: false + type: boolean + default: true release: description: "Run release job after tests pass" required: false @@ -131,15 +136,27 @@ jobs: config-file: ${{ inputs.config-file }} image-uri: ${{ needs.build.outputs.image-uri || '' }} + # SageMaker tests — real training jobs + endpoints + sagemaker-test: + if: ${{ always() && !failure() && !cancelled() && inputs.run-sagemaker-test }} + needs: [build] + concurrency: + group: ${{ github.workflow }}-sagemaker-test-${{ inputs.config-file }}-${{ github.ref }} + cancel-in-progress: true + uses: ./.github/workflows/sklearn.tests-sagemaker.yml + with: + config-file: ${{ inputs.config-file }} + image-uri: ${{ needs.build.outputs.image-uri || '' }} + release-gate: if: >- github.ref == 'refs/heads/main' && - (contains(github.workflow_ref, '.autorelease') || contains(github.workflow_ref, '.dispatch-release')) && + contains(github.workflow_ref, '.dispatch-release') && !contains(github.workflow_ref, '.pr') && inputs.release && !failure() && !cancelled() && needs.build.result == 'success' - needs: [build, sanity-test, security-test, unit-test, integ-test] + needs: [build, sanity-test, security-test, unit-test, integ-test, sagemaker-test] runs-on: ubuntu-latest outputs: environment: ${{ steps.check.outputs.environment }} diff --git a/.github/workflows/sklearn.pr-1.4-py312.yml b/.github/workflows/sklearn.pr-1.4-py312.yml index 599c58fe19d7..fa619187cebb 100644 --- a/.github/workflows/sklearn.pr-1.4-py312.yml +++ b/.github/workflows/sklearn.pr-1.4-py312.yml @@ -11,6 +11,7 @@ on: - ".github/workflows/sklearn.pr-1.4-py312.yml" - ".github/workflows/sklearn.tests-unit.yml" - ".github/workflows/sklearn.tests-integ-local.yml" + - ".github/workflows/sklearn.tests-sagemaker.yml" - "docker/sklearn/1.4-2-py312/**" - "docker/sklearn/resources/**" - "scripts/ci/build/sklearn/**" @@ -18,6 +19,7 @@ on: - "scripts/docker/telemetry/**" - "test/sanity/**" - "test/security/data/ecr_scan_allowlist/sklearn/**" + - "test/sklearn/**" - "test/telemetry/**" - "!docs/**" @@ -41,6 +43,7 @@ jobs: outputs: build-change: ${{ steps.changes.outputs.build-change }} integ-test-change: ${{ steps.changes.outputs.integ-test-change }} + sagemaker-test-change: ${{ steps.changes.outputs.sagemaker-test-change }} sanity-test-change: ${{ steps.changes.outputs.sanity-test-change }} telemetry-test-change: ${{ steps.changes.outputs.telemetry-test-change }} unit-test-change: ${{ steps.changes.outputs.unit-test-change }} @@ -60,6 +63,9 @@ jobs: - "test/security/data/ecr_scan_allowlist/sklearn/**" integ-test-change: - "test/sklearn/**" + sagemaker-test-change: + - "test/sklearn/sagemaker/**" + - ".github/workflows/sklearn.tests-sagemaker.yml" sanity-test-change: - "test/sanity/**" telemetry-test-change: @@ -91,6 +97,7 @@ jobs: tag-suffix: pr-${{ github.event.pull_request.number }} build: ${{ needs.check-changes.outputs.build-change == 'true' }} run-integ-test: ${{ needs.check-changes.outputs.build-change == 'true' || needs.check-changes.outputs.integ-test-change == 'true' }} + run-sagemaker-test: ${{ needs.check-changes.outputs.build-change == 'true' || needs.check-changes.outputs.sagemaker-test-change == 'true' }} run-sanity-test: ${{ needs.check-changes.outputs.build-change == 'true' || needs.check-changes.outputs.sanity-test-change == 'true' }} run-security-test: ${{ needs.check-changes.outputs.build-change == 'true' }} run-telemetry-test: false diff --git a/.github/workflows/sklearn.tests-integ-local.yml b/.github/workflows/sklearn.tests-integ-local.yml index b783ee6ff1df..0947a75a86f5 100644 --- a/.github/workflows/sklearn.tests-integ-local.yml +++ b/.github/workflows/sklearn.tests-integ-local.yml @@ -35,7 +35,7 @@ jobs: with: image-uri: ${{ inputs.image-uri }} ci-aws-account-id: ${{ vars.CI_AWS_ACCOUNT_ID }} - prod-aws-account-id: ${{ vars.PROD_AWS_ACCOUNT_ID }} + prod-aws-account-id: ${{ vars.PROD_AWS_ACCOUNT_ID_SAGEMAKER }} aws-region: ${{ vars.AWS_REGION }} config-file: ${{ inputs.config-file }} @@ -61,6 +61,25 @@ jobs: echo "framework-version=$(yq '.metadata.framework_version' "$CONFIG_FILE")" >> $GITHUB_OUTPUT echo "sklearn-branch=$(yq '.build.sklearn_container_branch // "master"' "$CONFIG_FILE")" >> $GITHUB_OUTPUT echo "python-version=$(yq '.build.python_version' "$CONFIG_FILE")" >> $GITHUB_OUTPUT + echo "dockerfile=$(yq '.build.dockerfile' "$CONFIG_FILE")" >> $GITHUB_OUTPUT + + - name: Verify pinned package versions match declared requirements.txt + if: steps.check.outputs.exists == 'true' + env: + IMAGE_URI: ${{ steps.image.outputs.image-uri }} + DOCKERFILE: ${{ steps.config.outputs.dockerfile }} + run: | + REQS_FILE="$(dirname "$DOCKERFILE")/requirements.txt" + if [[ ! -f "$REQS_FILE" ]]; then + echo "No requirements.txt at $REQS_FILE — skipping version check" + exit 0 + fi + docker run --rm \ + -v "$PWD/$REQS_FILE:/tmp/requirements.txt:ro" \ + -v "$PWD/test/sklearn/scripts/check_versions.py:/tmp/check_versions.py:ro" \ + --entrypoint python3 \ + "$IMAGE_URI" \ + /tmp/check_versions.py /tmp/requirements.txt - name: Clone sagemaker-scikit-learn-container (tests + source) if: steps.check.outputs.exists == 'true' diff --git a/.github/workflows/sklearn.tests-sagemaker.yml b/.github/workflows/sklearn.tests-sagemaker.yml new file mode 100644 index 000000000000..d1679f825eca --- /dev/null +++ b/.github/workflows/sklearn.tests-sagemaker.yml @@ -0,0 +1,90 @@ +name: Reusable Scikit-learn SageMaker Tests + +on: + workflow_call: + inputs: + config-file: + description: "Path to image config YAML" + required: true + type: string + image-uri: + description: "Image URI to test. If empty, derives prod URI from config." + required: false + type: string + default: "" + +permissions: + contents: read + +jobs: + preflight: + runs-on: + - codebuild-runner-${{ github.run_id }}-${{ github.run_attempt }} + fleet:default-runner + buildspec-override:true + outputs: + exists: ${{ steps.check.outputs.exists }} + image-uri: ${{ steps.image.outputs.image-uri }} + aws-account-id: ${{ steps.image.outputs.aws-account-id }} + framework-version: ${{ steps.config.outputs.framework-version }} + steps: + - uses: actions/checkout@v6 + + - name: Resolve image URI + id: image + uses: ./.github/actions/resolve-image-uri + with: + image-uri: ${{ inputs.image-uri }} + ci-aws-account-id: ${{ vars.CI_AWS_ACCOUNT_ID }} + prod-aws-account-id: ${{ vars.PROD_AWS_ACCOUNT_ID_SAGEMAKER }} + aws-region: ${{ vars.AWS_REGION }} + config-file: ${{ inputs.config-file }} + + - name: Check image exists + id: check + uses: ./.github/actions/check-image-exists + with: + image-uri: ${{ steps.image.outputs.image-uri }} + + - name: Parse config + id: config + run: | + CONFIG_FILE="${{ inputs.config-file }}" + echo "framework-version=$(yq '.metadata.framework_version' "$CONFIG_FILE")" >> $GITHUB_OUTPUT + + # SageMaker tests — real training jobs + endpoints + sagemaker-test: + if: ${{ needs.preflight.outputs.exists == 'true' }} + needs: [preflight] + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + test-module: + - test_inference + - test_inference_mme + - test_network_isolation + - test_scoring_pipelines + - test_script_mode + runs-on: + - codebuild-runner-${{ github.run_id }}-${{ github.run_attempt }} + fleet:default-runner + buildspec-override:true + steps: + - uses: actions/checkout@v6 + + - name: Install test dependencies + run: | + uv venv --python 3.12 + source .venv/bin/activate + uv pip install -r test/requirements.txt + uv pip install -r test/sklearn/requirements.txt + + - name: Run ${{ matrix.test-module }} + run: | + source .venv/bin/activate + cd test/ + python3 -m pytest -v --tb=short -rA --log-cli-level=INFO \ + --image-uri ${{ needs.preflight.outputs.image-uri }} \ + --sklearn-version ${{ needs.preflight.outputs.framework-version }} \ + sklearn/sagemaker/${{ matrix.test-module }}.py diff --git a/.github/workflows/sklearn.tests-unit.yml b/.github/workflows/sklearn.tests-unit.yml index 4dab855018c5..f06fea714f5d 100644 --- a/.github/workflows/sklearn.tests-unit.yml +++ b/.github/workflows/sklearn.tests-unit.yml @@ -32,7 +32,7 @@ jobs: with: image-uri: ${{ inputs.image-uri }} ci-aws-account-id: ${{ vars.CI_AWS_ACCOUNT_ID }} - prod-aws-account-id: ${{ vars.PROD_AWS_ACCOUNT_ID }} + prod-aws-account-id: ${{ vars.PROD_AWS_ACCOUNT_ID_SAGEMAKER }} aws-region: ${{ vars.AWS_REGION }} config-file: ${{ inputs.config-file }} diff --git a/test/sklearn/__init__.py b/test/sklearn/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/sklearn/conftest.py b/test/sklearn/conftest.py new file mode 100644 index 000000000000..055e2f2e670e --- /dev/null +++ b/test/sklearn/conftest.py @@ -0,0 +1,22 @@ +"""Shared pytest configuration for Scikit-learn tests.""" + +import pytest +from test_utils.constants import SAGEMAKER_ROLE + + +def pytest_addoption(parser): + parser.addoption( + "--sklearn-version", + default="1.4.2", + help="Scikit-learn version under test (e.g. 1.4.2)", + ) + + +@pytest.fixture(scope="session") +def sklearn_version(request): + return request.config.getoption("--sklearn-version") + + +@pytest.fixture(scope="session") +def role(): + return SAGEMAKER_ROLE diff --git a/test/sklearn/requirements.txt b/test/sklearn/requirements.txt new file mode 100644 index 000000000000..d371ab0d94a9 --- /dev/null +++ b/test/sklearn/requirements.txt @@ -0,0 +1 @@ +sagemaker>=2,<3 diff --git a/test/sklearn/sagemaker/__init__.py b/test/sklearn/sagemaker/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/sklearn/sagemaker/conftest.py b/test/sklearn/sagemaker/conftest.py new file mode 100644 index 000000000000..11ee7a84eae2 --- /dev/null +++ b/test/sklearn/sagemaker/conftest.py @@ -0,0 +1,296 @@ +"""Shared fixtures and helpers for Scikit-learn SageMaker tests.""" + +import logging +import time + +import boto3 +import pytest +from sagemaker.estimator import Estimator +from sagemaker.inputs import TrainingInput +from sagemaker.model import Model +from sagemaker.multidatamodel import MultiDataModel +from sagemaker.pipeline import PipelineModel +from test_utils import random_suffix_name + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.INFO) + +E2E_TEST_BUCKET = "amazonai-algorithms-integration-tests" +E2E_DATA_PREFIX = "input/scikit-learn" + +_created_models = [] +_created_endpoints = [] + + +def s3_uri(bucket, key): + return f"s3://{bucket}/{key}" + + +def data_uri(key): + return s3_uri(E2E_TEST_BUCKET, f"{E2E_DATA_PREFIX}/{key}") + + +def cleanup_resources(): + """Delete all SageMaker resources created during the test session.""" + sm = boto3.client("sagemaker") + LOGGER.info( + f"Session cleanup: {len(_created_endpoints)} endpoints, {len(_created_models)} models" + ) + for ep in _created_endpoints: + try: + sm.delete_endpoint(EndpointName=ep) + LOGGER.info(f"Deleted endpoint {ep}") + except Exception as e: + LOGGER.warning(f"Failed to delete endpoint {ep}: {e}") + try: + sm.delete_endpoint_config(EndpointConfigName=ep) + LOGGER.info(f"Deleted endpoint-config {ep}") + except Exception as e: + LOGGER.warning(f"Failed to delete endpoint-config {ep}: {e}") + for model_name in _created_models: + try: + sm.delete_model(ModelName=model_name) + LOGGER.info(f"Deleted model {model_name}") + except Exception as e: + LOGGER.warning(f"Failed to delete model {model_name}: {e}") + _created_endpoints.clear() + _created_models.clear() + + +@pytest.fixture(autouse=True, scope="session") +def _cleanup_after_session(): + """Automatically clean up all SageMaker resources after the test session.""" + yield + cleanup_resources() + + +def run_training_job( + image_uri, + role, + hyperparameters, + train_s3_key, + content_type=None, + test_name="train", + instance_type="ml.m5.xlarge", + instance_count=1, + volume_size=10, + max_run=1800, + input_mode="File", + train_distribution="ShardedByS3Key", + enable_network_isolation=False, + extra_channels=None, +): + """Launch a SageMaker training job and return (job_name, duration, description).""" + job_name = random_suffix_name(f"skl-{test_name}", 32) + output_path = s3_uri(E2E_TEST_BUCKET, f"e2e-output/{job_name}") + + estimator = Estimator( + image_uri=image_uri, + role=role, + instance_count=instance_count, + instance_type=instance_type, + output_path=output_path, + hyperparameters=hyperparameters, + volume_size=volume_size, + max_run=max_run, + input_mode=input_mode, + enable_network_isolation=enable_network_isolation, + ) + + train_input_kwargs = {"distribution": train_distribution} + if content_type is not None: + train_input_kwargs["content_type"] = content_type + channels = { + "train": TrainingInput( + s3_data=data_uri(train_s3_key), + **train_input_kwargs, + ), + } + + if extra_channels: + for name, value in extra_channels.items(): + if isinstance(value, tuple): + uri, ch_content_type = value + channels[name] = TrainingInput(s3_data=uri, content_type=ch_content_type) + else: + channels[name] = TrainingInput(s3_data=value) + + LOGGER.info(f"Starting job: {job_name} ({instance_count}x {instance_type})") + sm = boto3.client("sagemaker") + start = time.time() + try: + estimator.fit(channels, job_name=job_name) + except Exception: + try: + sm.stop_training_job(TrainingJobName=job_name) + except Exception: + pass + raise + duration = time.time() - start + + desc = sm.describe_training_job(TrainingJobName=job_name) + LOGGER.info( + f"Job {job_name} completed in {duration:.0f}s — status: {desc['TrainingJobStatus']}" + ) + return job_name, duration, desc + + +def deploy_endpoint( + image_uri, role, model_data, test_name="ep", instance_type="ml.m5.xlarge", env=None +): + """Deploy a real-time endpoint and return (predictor, endpoint_name).""" + from sagemaker.predictor import Predictor + + endpoint_name = random_suffix_name(f"skl-{test_name}", 32) + model = Model( + image_uri=image_uri, + model_data=model_data, + role=role, + env=env, + ) + LOGGER.info(f"Deploying endpoint {endpoint_name} on {instance_type} (model_data={model_data})") + start = time.time() + try: + model.deploy( + initial_instance_count=1, + instance_type=instance_type, + endpoint_name=endpoint_name, + ) + except Exception: + if model.name: + _created_models.append(model.name) + raise + LOGGER.info(f"Endpoint {endpoint_name} InService in {time.time() - start:.0f}s") + _created_models.append(model.name) + _created_endpoints.append(endpoint_name) + predictor = Predictor(endpoint_name=endpoint_name) + return predictor, endpoint_name + + +def deploy_multi_model_endpoint( + image_uri, + role, + model_data_prefix, + test_name="mme", + instance_type="ml.m5.xlarge", + env=None, +): + """Deploy a SageMaker Multi-Model Endpoint. + + `model_data_prefix` is the S3 prefix where model tarballs live + (each tarball becomes a target model addressable by name). + Returns (predictor, endpoint_name, mm_model). + """ + from sagemaker.predictor import Predictor + + endpoint_name = random_suffix_name(f"skl-{test_name}", 32) + mm_model = MultiDataModel( + name=random_suffix_name(f"skl-{test_name}-model", 32), + model_data_prefix=model_data_prefix, + image_uri=image_uri, + role=role, + env=env, + ) + LOGGER.info(f"Deploying MME {endpoint_name} on {instance_type} (prefix={model_data_prefix})") + start = time.time() + try: + mm_model.deploy( + initial_instance_count=1, + instance_type=instance_type, + endpoint_name=endpoint_name, + ) + except Exception: + if mm_model.name: + _created_models.append(mm_model.name) + raise + LOGGER.info(f"MME {endpoint_name} InService in {time.time() - start:.0f}s") + _created_models.append(mm_model.name) + _created_endpoints.append(endpoint_name) + predictor = Predictor(endpoint_name=endpoint_name) + return predictor, endpoint_name, mm_model + + +def deploy_inference_pipeline(models, role, test_name="pipe", instance_type="ml.m5.xlarge"): + """Deploy a multi-container inference pipeline endpoint. + + `models` is an ordered list of (image_uri, model_data, env) tuples. + Returns (predictor, endpoint_name). + """ + from sagemaker.predictor import Predictor + + endpoint_name = random_suffix_name(f"skl-{test_name}", 32) + pipeline_models = [ + Model(image_uri=img, model_data=data, role=role, env=env) for img, data, env in models + ] + pipeline = PipelineModel( + name=random_suffix_name(f"skl-{test_name}-model", 32), + role=role, + models=pipeline_models, + ) + LOGGER.info( + f"Deploying pipeline endpoint {endpoint_name} on {instance_type} " + f"({len(pipeline_models)} containers)" + ) + start = time.time() + try: + pipeline.deploy( + initial_instance_count=1, + instance_type=instance_type, + endpoint_name=endpoint_name, + ) + except Exception: + for m in pipeline_models: + if m.name: + _created_models.append(m.name) + raise + LOGGER.info(f"Pipeline endpoint {endpoint_name} InService in {time.time() - start:.0f}s") + for m in pipeline_models: + if m.name: + _created_models.append(m.name) + _created_endpoints.append(endpoint_name) + predictor = Predictor(endpoint_name=endpoint_name) + return predictor, endpoint_name + + +def predict_and_log(predictor, payload, **kwargs): + """Invoke an endpoint and log latency + first 200 chars of the response.""" + payload_len = len(payload) if isinstance(payload, (str, bytes)) else len(str(payload)) + LOGGER.info(f"POST {predictor.endpoint_name} payload_len={payload_len}") + start = time.time() + response = predictor.predict(payload, **kwargs) + LOGGER.info( + f"{predictor.endpoint_name} responded in {(time.time() - start) * 1000:.0f}ms: " + f"{str(response)[:200]}" + ) + return response + + +def delete_endpoint(endpoint_name): + """Delete endpoint, endpoint config, and associated model(s).""" + sm = boto3.client("sagemaker") + try: + ep_config = sm.describe_endpoint_config(EndpointConfigName=endpoint_name) + for variant in ep_config.get("ProductionVariants", []): + model_name = variant.get("ModelName") + if model_name: + try: + sm.delete_model(ModelName=model_name) + LOGGER.info(f"Deleted model {model_name}") + except Exception as e: + LOGGER.warning(f"Failed to delete model {model_name}: {e}") + if model_name in _created_models: + _created_models.remove(model_name) + except Exception as e: + LOGGER.warning(f"Failed to describe endpoint-config {endpoint_name}: {e}") + try: + sm.delete_endpoint(EndpointName=endpoint_name) + LOGGER.info(f"Deleted endpoint {endpoint_name}") + except Exception as e: + LOGGER.warning(f"Failed to delete endpoint {endpoint_name}: {e}") + try: + sm.delete_endpoint_config(EndpointConfigName=endpoint_name) + LOGGER.info(f"Deleted endpoint-config {endpoint_name}") + except Exception as e: + LOGGER.warning(f"Failed to delete endpoint-config {endpoint_name}: {e}") + if endpoint_name in _created_endpoints: + _created_endpoints.remove(endpoint_name) diff --git a/test/sklearn/sagemaker/test_inference.py b/test/sklearn/sagemaker/test_inference.py new file mode 100644 index 000000000000..798c3391e28d --- /dev/null +++ b/test/sklearn/sagemaker/test_inference.py @@ -0,0 +1,44 @@ +"""Online inference test — deploy an endpoint with a no-op echo model, +send a JSON payload, verify a response comes back. +""" + +import json + +import pytest + +from .conftest import data_uri, delete_endpoint, deploy_endpoint, predict_and_log + +ECHO_MODEL_KEY = "model/empty.tar.gz" +ECHO_CODE_KEY = "code/echo-2.4.10.tar.gz" + + +def _payload(feature_dim=100): + return json.dumps({"instances": [{"data": {"features": {"values": list(range(feature_dim))}}}]}) + + +@pytest.fixture(scope="module") +def echo_env(): + return { + "SAGEMAKER_PROGRAM": "echo.py", + "SAGEMAKER_SUBMIT_DIRECTORY": data_uri(ECHO_CODE_KEY), + } + + +class TestInference: + def test_json_inference(self, image_uri, role, echo_env): + endpoint_name = None + try: + predictor, endpoint_name = deploy_endpoint( + image_uri=image_uri, + role=role, + model_data=data_uri(ECHO_MODEL_KEY), + env=echo_env, + test_name="inference-solo", + ) + predictor.content_type = "application/json" + predictor.accept = "application/json" + response = predict_and_log(predictor, _payload()) + assert response is not None + finally: + if endpoint_name: + delete_endpoint(endpoint_name) diff --git a/test/sklearn/sagemaker/test_inference_mme.py b/test/sklearn/sagemaker/test_inference_mme.py new file mode 100644 index 000000000000..b3fc87a70bed --- /dev/null +++ b/test/sklearn/sagemaker/test_inference_mme.py @@ -0,0 +1,75 @@ +"""Inference tests — both single-model and multi-model endpoints against the +sklearn MME model tarballs. Verifies both endpoint modes round-trip a +CSV payload. +""" + +import pytest + +from .conftest import ( + data_uri, + delete_endpoint, + deploy_endpoint, + deploy_multi_model_endpoint, + predict_and_log, +) + +# MME lazy-loads only the tarball named in `target_model`, so the sibling +# `code/user_code.tar.gz` under this prefix is never invoked as a model. +MME_MODEL_PREFIX = "mme_models/" +MME_TARGET_MODELS = ["sklearn_1_model_0.tar.gz", "sklearn_1_model_1.tar.gz"] +MME_CODE_KEY = "mme_models/code/user_code.tar.gz" + +# Fitted models have n_features_in_ = 6. Multi-row CSV exercises batch predict. +SAMPLE_PAYLOAD = "\n".join( + [ + "0.0, 0.0, 0.0, 0.0, 0.0, 0.0", + "0.0, 2.0, 4.0, 6.0, 9.0, 3.0", + ] +) + + +@pytest.fixture(scope="module") +def mme_env(): + return { + "SAGEMAKER_PROGRAM": "script.py", + "SAGEMAKER_SUBMIT_DIRECTORY": data_uri(MME_CODE_KEY), + } + + +class TestInferenceMME: + def test_csv_single_model(self, image_uri, role, mme_env): + endpoint_name = None + try: + predictor, endpoint_name = deploy_endpoint( + image_uri=image_uri, + role=role, + model_data=data_uri(f"{MME_MODEL_PREFIX}{MME_TARGET_MODELS[0]}"), + env=mme_env, + test_name="inference-single", + ) + predictor.content_type = "text/csv" + predictor.accept = "text/csv" + response = predict_and_log(predictor, SAMPLE_PAYLOAD) + assert response is not None + finally: + if endpoint_name: + delete_endpoint(endpoint_name) + + def test_csv_multimodel(self, image_uri, role, mme_env): + endpoint_name = None + try: + predictor, endpoint_name, _ = deploy_multi_model_endpoint( + image_uri=image_uri, + role=role, + model_data_prefix=data_uri(MME_MODEL_PREFIX), + env=mme_env, + test_name="inference-mme", + ) + predictor.content_type = "text/csv" + predictor.accept = "text/csv" + for target_model in MME_TARGET_MODELS: + response = predict_and_log(predictor, SAMPLE_PAYLOAD, target_model=target_model) + assert response is not None + finally: + if endpoint_name: + delete_endpoint(endpoint_name) diff --git a/test/sklearn/sagemaker/test_network_isolation.py b/test/sklearn/sagemaker/test_network_isolation.py new file mode 100644 index 000000000000..9cce1e22b681 --- /dev/null +++ b/test/sklearn/sagemaker/test_network_isolation.py @@ -0,0 +1,29 @@ +"""Network isolation training test — parquet training via script mode +with `enable_network_isolation=True`. Verifies the container completes training +without needing external network access. +""" + +from .conftest import data_uri, run_training_job + +CODE_TARBALL_KEY = "code/pandas-parquet-file-1.4-2.tar.gz" +CODE_TARBALL_LOCAL = "/opt/ml/input/data/code/pandas-parquet-file-1.4-2.tar.gz" + + +class TestNetworkIsolation: + def test_script_mode(self, image_uri, role): + hp = { + "sagemaker_program": "train.py", + "sagemaker_submit_directory": CODE_TARBALL_LOCAL, + } + _, _, desc = run_training_job( + image_uri=image_uri, + role=role, + hyperparameters=hp, + train_s3_key="data/train.parquet", + content_type="application/x-parquet", + test_name="netiso", + volume_size=20, + enable_network_isolation=True, + extra_channels={"code": (data_uri(CODE_TARBALL_KEY), "text/plain")}, + ) + assert desc["TrainingJobStatus"] == "Completed" diff --git a/test/sklearn/sagemaker/test_scoring_pipelines.py b/test/sklearn/sagemaker/test_scoring_pipelines.py new file mode 100644 index 000000000000..72f752a8be33 --- /dev/null +++ b/test/sklearn/sagemaker/test_scoring_pipelines.py @@ -0,0 +1,79 @@ +"""Inference-pipeline endpoint tests — deploy sklearn + linear-learner in an +inference pipeline (both directions), verify a JSON payload round-trips. +""" + +import json + +import pytest + +from .conftest import ( + E2E_TEST_BUCKET, + data_uri, + delete_endpoint, + deploy_inference_pipeline, + predict_and_log, + s3_uri, +) + +# us-west-2 SageMaker 1P algo ECR account for linear-learner +LINEAR_LEARNER_IMAGE = "174872318107.dkr.ecr.us-west-2.amazonaws.com/linear-learner:latest" + +SKLEARN_MODEL_KEY = "model/empty.tar.gz" +SKLEARN_CODE_KEY = "code/echo-2.4.10.tar.gz" +LINEAR_LEARNER_MODEL = s3_uri(E2E_TEST_BUCKET, "input/linearlearner/a9a/model.tar.gz") + + +def _linear_learner_payload(): + """JSON payload matching linear-learner's expected 123-feature input dim.""" + return json.dumps({"instances": [{"data": {"features": {"values": list(range(123))}}}]}) + + +@pytest.fixture(scope="module") +def sklearn_container(image_uri): + return ( + image_uri, + data_uri(SKLEARN_MODEL_KEY), + { + "SAGEMAKER_PROGRAM": "echo.py", + "SAGEMAKER_SUBMIT_DIRECTORY": data_uri(SKLEARN_CODE_KEY), + }, + ) + + +@pytest.fixture(scope="module") +def algo_container(): + return (LINEAR_LEARNER_IMAGE, LINEAR_LEARNER_MODEL, {}) + + +class TestScoringPipelines: + def test_sklearn_then_algo(self, role, sklearn_container, algo_container): + endpoint_name = None + try: + predictor, endpoint_name = deploy_inference_pipeline( + models=[sklearn_container, algo_container], + role=role, + test_name="pipe-skl-algo", + ) + predictor.content_type = "application/json" + predictor.accept = "application/json" + response = predict_and_log(predictor, _linear_learner_payload()) + assert response is not None + finally: + if endpoint_name: + delete_endpoint(endpoint_name) + + def test_algo_then_sklearn(self, role, sklearn_container, algo_container): + endpoint_name = None + try: + predictor, endpoint_name = deploy_inference_pipeline( + models=[algo_container, sklearn_container], + role=role, + test_name="pipe-algo-skl", + ) + predictor.content_type = "application/json" + predictor.accept = "application/json" + response = predict_and_log(predictor, _linear_learner_payload()) + assert response is not None + finally: + if endpoint_name: + delete_endpoint(endpoint_name) diff --git a/test/sklearn/sagemaker/test_script_mode.py b/test/sklearn/sagemaker/test_script_mode.py new file mode 100644 index 000000000000..a75692c9f694 --- /dev/null +++ b/test/sklearn/sagemaker/test_script_mode.py @@ -0,0 +1,25 @@ +"""Script mode training with `requirements.txt` — verifies the container +installs extra Python dependencies from a user-provided requirements.txt +alongside the training script. +""" + +from .conftest import data_uri, run_training_job + +CODE_TARBALL_KEY = "code/requirements.tar.gz" + + +class TestScriptMode: + def test_requirements_install(self, image_uri, role): + hp = { + "sagemaker_program": "train.py", + "sagemaker_submit_directory": data_uri(CODE_TARBALL_KEY), + } + _, _, desc = run_training_job( + image_uri=image_uri, + role=role, + hyperparameters=hp, + train_s3_key="data/train.parquet", + test_name="script-reqs", + volume_size=20, + ) + assert desc["TrainingJobStatus"] == "Completed" diff --git a/test/sklearn/scripts/check_versions.py b/test/sklearn/scripts/check_versions.py new file mode 100644 index 000000000000..4541be4bc475 --- /dev/null +++ b/test/sklearn/scripts/check_versions.py @@ -0,0 +1,94 @@ +"""Verify installed package versions match declared specifiers in requirements.txt. + +Runs inside the built sklearn container. Reads a requirements.txt (copied in by +the caller), parses each entry as a PEP 508 requirement, and asserts the +installed version satisfies the declared specifier (==, >=, <, ~=, etc.). + +Bare entries with no specifier (e.g. `certifi`) can't be verified — logged and +skipped. Unparsable lines are logged and skipped. + +Usage: python3 check_versions.py + +Exits non-zero on any drift, with a summary of what changed. +""" + +import importlib.metadata +import sys + +from packaging.requirements import InvalidRequirement, Requirement +from packaging.version import Version + + +def parse_reqs(path): + reqs = [] + unparsable = [] + with open(path) as f: + for raw in f: + line = raw.split("#", 1)[0].strip() + if not line: + continue + try: + reqs.append((Requirement(line), line)) + except InvalidRequirement: + unparsable.append(line) + return reqs, unparsable + + +def installed_version(name): + for candidate in (name, name.replace("-", "_"), name.replace("_", "-")): + try: + return importlib.metadata.version(candidate) + except importlib.metadata.PackageNotFoundError: + continue + return None + + +def main(path): + reqs, unparsable = parse_reqs(path) + if not reqs: + print(f"No requirements found in {path}", file=sys.stderr) + sys.exit(1) + + drift = [] + missing = [] + unconstrained = [] + checked = 0 + + for req, raw in reqs: + actual = installed_version(req.name) + if actual is None: + missing.append(req.name) + continue + if not req.specifier: + unconstrained.append(raw) + continue + checked += 1 + if Version(actual) not in req.specifier: + drift.append((req.name, str(req.specifier), actual)) + + print(f"Checked {checked} constrained packages against declared specifiers.") + if unconstrained: + print(f"Skipped {len(unconstrained)} unconstrained entries (no version specifier):") + for line in unconstrained: + print(f" {line}") + if unparsable: + print(f"Skipped {len(unparsable)} unparsable lines:", file=sys.stderr) + for line in unparsable: + print(f" {line}", file=sys.stderr) + + if not drift and not missing: + print(f"All {checked} constrained packages satisfy declared specifiers.") + return + + for name, spec, act in drift: + print(f"DRIFT: {name} declared{spec} but installed=={act}", file=sys.stderr) + for name in missing: + print(f"MISSING: {name} declared but not installed", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("usage: check_versions.py ", file=sys.stderr) + sys.exit(2) + main(sys.argv[1])