diff --git a/.github/workflows/ai-pr-review.yml b/.github/workflows/ai-pr-review.yml index a9977c01d..5143690f9 100644 --- a/.github/workflows/ai-pr-review.yml +++ b/.github/workflows/ai-pr-review.yml @@ -10,18 +10,24 @@ permissions: jobs: ai-review: + if: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 + ref: ${{ github.event.pull_request.base.ref }} - name: Install dependencies run: | pip install requests jq + - name: Fetch PR head + run: | + git fetch origin ${{ github.event.pull_request.head.sha }} + - name: Get incremental diff id: diff run: | diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 6e0d10267..cf456e2c4 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -24,14 +24,14 @@ jobs: fail-fast: false steps: # Step 1: Checkout the code - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 2 ssh-key: ${{ secrets.DEPLOY_KEY }} ref: ${{ github.ref_name }} # Step 2: Set up Python - - uses: actions/setup-python@v3 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 # Step 3: Check if VERSION file has changed in this push - name: Check if VERSION file has changed @@ -41,13 +41,13 @@ jobs: git config --global user.email "mlcommons-bot@users.noreply.github.com" if git diff --name-only $(git merge-base HEAD HEAD~1) | grep -q "VERSION"; then echo "VERSION file has been modified" - echo "::set-output name=version_changed::true" + echo "version_changed=true" >> "$GITHUB_OUTPUT" new_version=$(cat VERSION) else echo "VERSION file has NOT been modified" - echo "::set-output name=version_changed::false" + echo "version_changed=false" >> "$GITHUB_OUTPUT" fi - echo "::set-output name=new_version::$new_version" + echo "new_version=$new_version" >> "$GITHUB_OUTPUT" # Step 4: Increment version if VERSION was not changed - name: Increment version if necessary @@ -65,7 +65,7 @@ jobs: new_version="$major.$minor.$patch" echo $new_version > VERSION echo "New version: $new_version" - echo "::set-output name=new_version::$new_version" + echo "new_version=$new_version" >> "$GITHUB_OUTPUT" # Step 5: Commit the updated version to the repository - name: Commit updated version @@ -87,7 +87,7 @@ jobs: # Step 8: Publish to PyPI - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 with: verify-metadata: true skip-existing: true diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index fedf10051..d2af70416 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -17,7 +17,7 @@ jobs: - name: "MLCommons CLA bot check" if: (github.event.comment.body == 'recheck') || github.event_name == 'pull_request_target' # Alpha Release - uses: mlcommons/cla-bot@master + uses: mlcommons/cla-bot@7facd19917cdb24be27bd8d266b41094506a0fb2 # master 2026-03-24 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # the below token should have repo scope and must be manually added by you in the repository's secret @@ -26,7 +26,7 @@ jobs: path-to-signatures: 'cla-bot/v1/cla.json' # branch should not be protected branch: 'main' - allowlist: user1,bot* + allowlist: user1,mlcommons-bot,mlc-automations,bot*,copilot-swe-agent[bot],Copilot* remote-organization-name: mlcommons remote-repository-name: systems diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ad586317f..1f40bfa7b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -19,6 +19,8 @@ on: schedule: - cron: '43 6 * * 0' +permissions: {} + jobs: analyze: name: Analyze (${{ matrix.language }}) @@ -57,7 +59,7 @@ jobs: # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 # Add any setup steps before running the `github/codeql-action/init` action. # This includes steps like installing compilers or runtimes (`actions/setup-node` @@ -67,7 +69,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@5c8a8a642e79153f5d047b10ec1cba1d1cc65699 # v3.35.1 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -95,6 +97,6 @@ jobs: exit 1 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@5c8a8a642e79153f5d047b10ec1cba1d1cc65699 # v3.35.1 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml new file mode 100644 index 000000000..681587d93 --- /dev/null +++ b/.github/workflows/format.yml @@ -0,0 +1,60 @@ +# Automatic code formatting +name: "Code formatting" +on: + push: + branches: + - "**" + +env: + python_version: "3.9" + +jobs: + format-code: + if: github.actor != 'mlc-automations' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.MLC_AUTOMATIONS_APP_ID }} + private-key: ${{ secrets.MLC_AUTOMATIONS_PRIVATE_KEY }} + + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + + - name: Set up Python ${{ env.python_version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.python_version }} + + - name: Format modified Python files + env: + filter: ${{ github.event.before }} + run: | + python3 -m pip install autopep8 + for FILE in $(git diff --name-only $filter | grep -E '.*\.py$') + do + # Check if the file still exists in the working tree + if [ -f "$FILE" ]; then + autopep8 --in-place -a "$FILE" + git add "$FILE" + fi + done + + - name: Commit and push changes + run: | + HAS_CHANGES=$(git diff --staged --name-only) + if [ ${#HAS_CHANGES} -gt 0 ]; then + # Use the GitHub actor's name and email + git config --global user.name mlc-automations + git config --global user.email "3246381+mlc-automations@users.noreply.github.com" + # Commit changes + git commit -m '[Automated Commit] Format Codebase' + git push + fi diff --git a/.github/workflows/mlperf-inference-bert.yml b/.github/workflows/mlperf-inference-bert.yml index 55474b067..3c47a5363 100644 --- a/.github/workflows/mlperf-inference-bert.yml +++ b/.github/workflows/mlperf-inference-bert.yml @@ -28,9 +28,9 @@ jobs: - os: windows-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} - name: Install mlcflow diff --git a/.github/workflows/mlperf-inference-resnet50.yml b/.github/workflows/mlperf-inference-resnet50.yml index 94b3e6ba5..5f2d3507f 100644 --- a/.github/workflows/mlperf-inference-resnet50.yml +++ b/.github/workflows/mlperf-inference-resnet50.yml @@ -28,11 +28,11 @@ jobs: python-version: 3.13 runs-on: "${{ matrix.on }}" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bed75398c..8553e71f5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,10 +17,10 @@ jobs: steps: - name: Checkout repository normally - uses: actions/checkout@v3 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" diff --git a/.github/workflows/reset-fork.yml b/.github/workflows/reset-fork.yml index 78ea9f86f..9d5ce4e59 100644 --- a/.github/workflows/reset-fork.yml +++ b/.github/workflows/reset-fork.yml @@ -8,13 +8,16 @@ on: required: false default: '' +permissions: + contents: write + jobs: reset-branch: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 @@ -24,7 +27,9 @@ jobs: - name: Use Input Branch if: ${{ inputs.branch != '' }} - run: echo "branch=${{ inputs.branch }}" >> $GITHUB_ENV + env: + BRANCH_INPUT: ${{ inputs.branch }} + run: echo "branch=${BRANCH_INPUT}" >> $GITHUB_ENV - name: Add Upstream Remote run: | diff --git a/.github/workflows/test-installer-curl.yml b/.github/workflows/test-installer-curl.yml new file mode 100644 index 000000000..e97ae7e7a --- /dev/null +++ b/.github/workflows/test-installer-curl.yml @@ -0,0 +1,527 @@ +name: Test MLCFlow Installer + +on: + pull_request: + branches: + - main + - dev + paths: + - 'docs/install/mlcflow_linux.sh' + - '.github/workflows/test-installer-curl.yml' + workflow_dispatch: + +permissions: + contents: read + +# Only allow one workflow run per PR to conserve CI resources +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true # Might want to discuss on this - this saves the CI minutes but the information about what caused error in previous commit might be lost if multiple commits are pushed in quick succession. We can consider changing this to "false" if we want to preserve all test results for all commits. + +jobs: + # =========================================================================== + # Test Matrix: Native GitHub Actions Runners + # =========================================================================== + # These tests run on native GitHub Actions runners for Ubuntu and macOS. + # The installer is downloaded via curl and piped to bash, exactly as users + # will execute it in production. + test-native-runners: + name: Test on ${{ matrix.os }} (${{ matrix.scenario }}) + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + # =================================================================== + # Ubuntu 22.04 LTS Tests + # =================================================================== + # Test basic installation on Ubuntu 22.04 + - os: Ubuntu 22.04 + runner: ubuntu-22.04 + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + # Test quiet mode with minimal output + - os: Ubuntu 22.04 + runner: ubuntu-22.04 + scenario: quiet-mode + test-type: success + extra-flags: "--yes --quiet" + description: "Quiet mode with minimal output" + + # Test upgrade mode on existing installation + - os: Ubuntu 22.04 + runner: ubuntu-22.04 + scenario: upgrade-mode + test-type: success + extra-flags: "--yes --upgrade" + description: "Upgrade existing installation" + + # =================================================================== + # Ubuntu 24.04 LTS Tests + # =================================================================== + # Test basic installation on latest Ubuntu LTS + - os: Ubuntu 24.04 + runner: ubuntu-latest + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + # =================================================================== + # macOS Tests + # =================================================================== + # Test basic installation on macOS with Homebrew + - os: macos-latest + runner: macos-latest + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + # Test custom venv directory on macOS + - os: macos-latest + runner: macos-latest + scenario: custom-venv + test-type: success + extra-flags: "--yes --venv-dir /tmp/custom_mlcflow_venv" + description: "Custom virtual environment directory" + + steps: + # Determine the source URL based on the event type + # For pull_request: use the PR head branch + # For workflow_dispatch: use dev branch from mlcommons/mlcflow + - name: Determine Installer Script URL + env: + PR_OWNER: ${{ github.event.pull_request.head.repo.owner.login }} + PR_REPO: ${{ github.event.pull_request.head.repo.name }} + PR_BRANCH: ${{ github.event.pull_request.head.ref }} + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + # For PRs, use the head branch from the PR + OWNER="${PR_OWNER}" + REPO="${PR_REPO}" + BRANCH="${PR_BRANCH}" + else + # For workflow_dispatch and other events, use dev branch + OWNER="mlcommons" + REPO="mlcflow" + BRANCH="anandhu-eng-patch-1" #"dev" + fi + + INSTALLER_URL="https://raw.githubusercontent.com/${OWNER}/${REPO}/refs/heads/${BRANCH}/docs/install/mlcflow_linux.sh" + echo "INSTALLER_URL=$INSTALLER_URL" >> $GITHUB_ENV + echo "✅ Installer URL: $INSTALLER_URL" + + # ===================================================================== + # Test Case: Install via Curl-Pipe Method + # ===================================================================== + # This is the primary test that validates the installer using the exact + # method that users will use: curl /mlcflow_linux.sh | bash -s -- + - name: "Test: ${{ matrix.description }}" + run: | + echo "==========================================" + echo "Test: ${{ matrix.description }}" + echo "OS: ${{ matrix.os }}" + echo "Scenario: ${{ matrix.scenario }}" + echo "Extra Flags: ${{ matrix.extra-flags }}" + echo "Installer URL: $INSTALLER_URL" + echo "==========================================" + + # Download and execute the installer via curl-pipe method from GitHub + # This is the EXACT method that users will use in production + echo "Downloading and executing installer via curl-pipe method..." + if curl -sSL "$INSTALLER_URL" | bash -s -- ${{ matrix.extra-flags }}; then + echo "✅ Installer completed successfully" + INSTALL_SUCCESS=true + else + EXIT_CODE=$? + echo "❌ Installer failed with exit code: $EXIT_CODE" + INSTALL_SUCCESS=false + fi + + # Store result for validation step + echo "INSTALL_SUCCESS=$INSTALL_SUCCESS" >> $GITHUB_ENV + + # ===================================================================== + # Validate Installation Success + # ===================================================================== + # After installation completes, verify that expected artifacts exist + - name: Validate Installation Artifacts + if: env.INSTALL_SUCCESS == 'true' + run: | + echo "==========================================" + echo "Validating Installation Artifacts" + echo "==========================================" + + # Determine the venv directory based on test scenario + if [[ "${{ matrix.scenario }}" == "custom-venv" ]]; then + VENV_DIR="/tmp/custom_mlcflow_venv" + else + VENV_DIR="$HOME/mlcflow" + fi + + echo "Expected venv directory: $VENV_DIR" + + # Validate 1: Virtual environment directory exists + if [ -d "$VENV_DIR" ]; then + echo "✅ Virtual environment directory exists" + else + echo "❌ Virtual environment directory not found" + exit 1 + fi + + # Validate 2: Activation script exists + if [ -f "$VENV_DIR/bin/activate" ]; then + echo "✅ Virtual environment activation script exists" + else + echo "❌ Activation script not found" + exit 1 + fi + + # Validate 3: Python executable exists in venv + if [ -f "$VENV_DIR/bin/python3" ] || [ -f "$VENV_DIR/bin/python" ]; then + echo "✅ Python executable found in virtual environment" + else + echo "❌ Python executable not found in virtual environment" + exit 1 + fi + + # Validate 4: MLCFlow package is installed + if source "$VENV_DIR/bin/activate" && python3 -c "import mlcflow" 2>/dev/null; then + echo "✅ MLCFlow package is importable" + else + echo "⚠️ MLCFlow package may not be fully installed (repo cloning may have failed)" + fi + + # Validate 5: mlc CLI command is available after activation + if source "$VENV_DIR/bin/activate" && command -v mlc >/dev/null 2>&1; then + echo "✅ mlc CLI command is available" + + # Try to execute a harmless command to verify CLI works + if mlc help >/dev/null 2>&1; then + echo "✅ mlc help command executed successfully" + else + echo "⚠️ mlc command exists but may not be fully functional" + fi + else + echo "⚠️ mlc CLI command not found (may be due to repo cloning issues)" + fi + + # Validate 6: Check if automation repository was cloned + if [ -d "$HOME/MLC/repos" ]; then + echo "✅ MLC repos directory exists" + echo "Contents:" + ls -la "$HOME/MLC/repos" || true + else + echo "⚠️ MLC repos directory not found (repo cloning may have failed)" + fi + + # ===================================================================== + # Verify Expected Test Outcome + # ===================================================================== + # Confirm that the test result matches the expected outcome + - name: Verify Test Outcome + run: | + echo "==========================================" + echo "Verifying Test Outcome" + echo "==========================================" + + EXPECTED_RESULT="${{ matrix.test-type }}" + ACTUAL_SUCCESS="${{ env.INSTALL_SUCCESS }}" + + echo "Expected: $EXPECTED_RESULT" + echo "Actual Success: $ACTUAL_SUCCESS" + + if [[ "$EXPECTED_RESULT" == "success" && "$ACTUAL_SUCCESS" == "true" ]]; then + echo "✅ Test passed: Installation succeeded as expected" + exit 0 + elif [[ "$EXPECTED_RESULT" == "failure" && "$ACTUAL_SUCCESS" == "false" ]]; then + echo "✅ Test passed: Installation failed as expected" + exit 0 + else + echo "❌ Test failed: Unexpected outcome" + exit 1 + fi + + # =========================================================================== + # Test Matrix: Docker Containers for Non-Native Distributions + # =========================================================================== + # These tests run inside Docker containers for Linux distributions that + # don't have native GitHub Actions runners (Debian, RHEL-family). + test-docker-containers: + name: Test on ${{ matrix.os }} (${{ matrix.scenario }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # =================================================================== + # Ubuntu 20.04 LTS Tests + # =================================================================== + # Test basic non-interactive installation with all default settings + - os: Ubuntu 20.04 + container: ubuntu:20.04 + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + # Test installation with custom venv directory + - os: Ubuntu 20.04 + container: ubuntu:20.04 + scenario: custom-venv + test-type: success + extra-flags: "--yes --venv-dir /tmp/custom_mlcflow_venv" + description: "Custom virtual environment directory" + + # =================================================================== + # Debian 11 Tests + # =================================================================== + # Test basic installation on Debian 11 (Bullseye) + - os: Debian 11 + container: debian:11 + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + # Test custom venv directory on Debian + - os: Debian 11 + container: debian:11 + scenario: custom-venv + test-type: success + extra-flags: "--yes --venv-dir /tmp/custom_mlcflow_venv" + description: "Custom virtual environment directory" + + # =================================================================== + # Debian 12 Tests + # =================================================================== + # Test basic installation on Debian 12 (Bookworm) + - os: Debian 12 + container: debian:12 + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + # =================================================================== + # Rocky Linux 9 Tests + # =================================================================== + # Test basic installation on Rocky Linux 9 (RHEL-compatible) + - os: Rocky Linux 9 + container: rockylinux:9 + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + # Test verbose mode on Rocky Linux + - os: Rocky Linux 9 + container: rockylinux:9 + scenario: verbose-mode + test-type: success + extra-flags: "--yes --verbose" + description: "Verbose logging mode" + + # =================================================================== + # AlmaLinux 9 Tests + # =================================================================== + # Test basic installation on AlmaLinux 9 (RHEL-compatible) + - os: AlmaLinux 9 + container: almalinux:9 + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + # =================================================================== + # CentOS Stream 9 Tests + # =================================================================== + # Test basic installation on CentOS Stream 9 (RHEL-compatible) + - os: CentOS Stream 9 + container: quay.io/centos/centos:stream9 + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + steps: + # Determine the source URL based on the event type + - name: Determine Installer Script URL + env: + PR_OWNER: ${{ github.event.pull_request.head.repo.owner.login }} + PR_REPO: ${{ github.event.pull_request.head.repo.name }} + PR_BRANCH: ${{ github.event.pull_request.head.ref }} + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + OWNER="${PR_OWNER}" + REPO="${PR_REPO}" + BRANCH="${PR_BRANCH}" + else + OWNER="mlcommons" + REPO="mlcflow" + BRANCH="anandhu-eng-patch-1" #"dev" + fi + + INSTALLER_URL="https://raw.githubusercontent.com/${OWNER}/${REPO}/refs/heads/${BRANCH}/docs/install/mlcflow_linux.sh" + echo "INSTALLER_URL=$INSTALLER_URL" >> $GITHUB_ENV + echo "✅ Installer URL: $INSTALLER_URL" + + # ===================================================================== + # Test Case: Install via Curl-Pipe Method in Docker Container + # ===================================================================== + # This test runs the installer inside a Docker container that represents + # the target Linux distribution. The installer is downloaded via curl + # from GitHub and piped to bash, exactly as users will execute it. + - name: "Test: ${{ matrix.description }} in ${{ matrix.os }}" + run: | + echo "==========================================" + echo "Test: ${{ matrix.description }}" + echo "OS: ${{ matrix.os }}" + echo "Container: ${{ matrix.container }}" + echo "Scenario: ${{ matrix.scenario }}" + echo "Extra Flags: ${{ matrix.extra-flags }}" + echo "Installer URL: $INSTALLER_URL" + echo "==========================================" + + # Run the installer inside the Docker container via curl-pipe method + # Downloads directly from GitHub, testing the real distribution method + docker run --rm \ + ${{ matrix.container }} \ + bash -c " + # Install curl if not present (required for downloading the script) + if ! command -v curl >/dev/null 2>&1; then + if command -v apt-get >/dev/null 2>&1; then + apt-get update -qq && apt-get install -y -qq curl + elif command -v dnf >/dev/null 2>&1; then + dnf install -y -q curl-minimal + elif command -v yum >/dev/null 2>&1; then + yum install -y -q curl-minimal + fi + fi + + # Download and execute installer via curl-pipe method from GitHub + echo '=== Downloading and executing installer via curl-pipe method ===' + curl -sSL '$INSTALLER_URL' | bash -s -- ${{ matrix.extra-flags }} + " && DOCKER_EXIT_CODE=0 || DOCKER_EXIT_CODE=$? + + # Evaluate the result + if [ $DOCKER_EXIT_CODE -eq 0 ]; then + echo "✅ Installer completed successfully in container" + echo "INSTALL_SUCCESS=true" >> $GITHUB_ENV + else + echo "❌ Installer failed in container with exit code: $DOCKER_EXIT_CODE" + echo "INSTALL_SUCCESS=false" >> $GITHUB_ENV + fi + + # ===================================================================== + # Validate Installation Inside Container + # ===================================================================== + # After installation completes, run the container again to verify artifacts + - name: Validate Installation Artifacts in Container + if: env.INSTALL_SUCCESS == 'true' + run: | + echo "==========================================" + echo "Validating Installation Artifacts" + echo "==========================================" + + # Determine the venv directory based on test scenario + if [[ "${{ matrix.scenario }}" == "custom-venv" ]]; then + VENV_DIR="/tmp/custom_mlcflow_venv" + else + VENV_DIR="/root/mlcflow" + fi + + echo "Expected venv directory: $VENV_DIR" + + # Run validation commands inside a new container instance + # Note: The previous container is ephemeral, so we validate the + # installation success based on exit code rather than persistent state + docker run --rm ${{ matrix.container }} bash -c " + echo '=== Validation Complete ===' + echo 'Note: Container installations are ephemeral in CI.' + echo 'Success is determined by installer exit code.' + " + + echo "✅ Container installation validation complete" + + # ===================================================================== + # Verify Expected Test Outcome + # ===================================================================== + - name: Verify Test Outcome + run: | + echo "==========================================" + echo "Verifying Test Outcome" + echo "==========================================" + + EXPECTED_RESULT="${{ matrix.test-type }}" + ACTUAL_SUCCESS="${{ env.INSTALL_SUCCESS }}" + + echo "Expected: $EXPECTED_RESULT" + echo "Actual Success: $ACTUAL_SUCCESS" + + if [[ "$EXPECTED_RESULT" == "success" && "$ACTUAL_SUCCESS" == "true" ]]; then + echo "✅ Test passed: Installation succeeded as expected" + exit 0 + elif [[ "$EXPECTED_RESULT" == "failure" && "$ACTUAL_SUCCESS" == "false" ]]; then + echo "✅ Test passed: Installation failed as expected" + exit 0 + else + echo "❌ Test failed: Unexpected outcome" + exit 1 + fi + + # =========================================================================== + # Final Test Summary + # =========================================================================== + test-summary: + name: Test Summary + runs-on: ubuntu-latest + needs: + - test-native-runners + - test-docker-containers + if: always() + steps: + - name: Generate Test Summary + run: | + echo "==========================================" + echo " MLCFlow Installer CI Test Summary" + echo "==========================================" + echo "" + echo "This CI workflow validates the MLCFlow installer by downloading" + echo "directly from GitHub and using the exact distribution method that" + echo "users will use in production:" + echo " curl -sSL /mlcflow_linux.sh | bash" + echo "" + echo "✅ Test Coverage:" + echo " - Native GitHub runners (Ubuntu, macOS)" + echo " - Docker containers (Debian, Rocky, Alma, CentOS)" + echo " - Multiple installation modes (basic, custom, verbose, quiet)" + echo "" + echo "⚠️ Known Limitation:" + echo " - Only non-interactive mode is tested in CI" + echo " - Interactive prompts are not covered by automated tests" + echo "" + echo "==========================================" + echo "" + + # Check if all required jobs succeeded + NATIVE_STATUS="${{ needs.test-native-runners.result }}" + DOCKER_STATUS="${{ needs.test-docker-containers.result }}" + + echo "Job Results:" + echo " Native Runners: $NATIVE_STATUS" + echo " Docker Containers: $DOCKER_STATUS" + echo "" + + if [[ "$NATIVE_STATUS" == "success" && \ + "$DOCKER_STATUS" == "success" ]]; then + echo "Result: ✅ ALL TESTS PASSED" + exit 0 + else + echo "Result: ❌ SOME TESTS FAILED" + exit 1 + fi diff --git a/.github/workflows/test-mlc-core-actions.yaml b/.github/workflows/test-mlc-core-actions.yaml index 5896cd324..661447cbf 100644 --- a/.github/workflows/test-mlc-core-actions.yaml +++ b/.github/workflows/test-mlc-core-actions.yaml @@ -24,9 +24,9 @@ jobs: - os: macos-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} @@ -220,6 +220,116 @@ jobs: mlc rm repo mlcommons@mlperf-automations -f mlcr detect,cpu -j + - name: Test 26 - Test reindex command and verify index files are updated + run: | + INDEX_SCRIPT="${HOME}/MLC/repos/index_script.json" + INDEX_CACHE="${HOME}/MLC/repos/index_cache.json" + INDEX_EXPERIMENT="${HOME}/MLC/repos/index_experiment.json" + + # Store initial modification times + if [ -f "$INDEX_SCRIPT" ]; then + BEFORE_SCRIPT=$(stat -c %Y "$INDEX_SCRIPT" 2>/dev/null || stat -f %m "$INDEX_SCRIPT" 2>/dev/null) + fi + if [ -f "$INDEX_CACHE" ]; then + BEFORE_CACHE=$(stat -c %Y "$INDEX_CACHE" 2>/dev/null || stat -f %m "$INDEX_CACHE" 2>/dev/null) + fi + if [ -f "$INDEX_EXPERIMENT" ]; then + BEFORE_EXPERIMENT=$(stat -c %Y "$INDEX_EXPERIMENT" 2>/dev/null || stat -f %m "$INDEX_EXPERIMENT" 2>/dev/null) + fi + + sleep 1 + + # Test reindex all + mlc reindex + + # Verify all index files were updated + if [ -f "$INDEX_SCRIPT" ]; then + AFTER_SCRIPT=$(stat -c %Y "$INDEX_SCRIPT" 2>/dev/null || stat -f %m "$INDEX_SCRIPT" 2>/dev/null) + if [ "$BEFORE_SCRIPT" = "$AFTER_SCRIPT" ]; then + echo "index_script.json was not updated after 'mlc reindex'. Exiting with failure." + exit 1 + fi + fi + + sleep 1 + BEFORE_SCRIPT=$(stat -c %Y "$INDEX_SCRIPT" 2>/dev/null || stat -f %m "$INDEX_SCRIPT" 2>/dev/null) + + # Test reindex specific target + mlc reindex script + + AFTER_SCRIPT=$(stat -c %Y "$INDEX_SCRIPT" 2>/dev/null || stat -f %m "$INDEX_SCRIPT" 2>/dev/null) + if [ "$BEFORE_SCRIPT" = "$AFTER_SCRIPT" ]; then + echo "index_script.json was not updated after 'mlc reindex script'. Exiting with failure." + exit 1 + fi + + # Test other reindex commands + mlc reindex all + mlc reindex cache + mlc reindex experiment + + - name: Test 27 - Test index handling when script/cache/repo is manually deleted + run: | + # Add a test script + mlc add script test-delete-script --tags=test,delete,temp + + # Get the actual script path from find command + SCRIPT_PATH=$(mlc find script test-delete-script -p 2>&1) + echo "Script path: $SCRIPT_PATH" + + if [ -z "$SCRIPT_PATH" ]; then + echo "Script was not found after adding. Exiting with failure." + exit 1 + fi + + # Manually delete the script + if [ -d "$SCRIPT_PATH" ]; then + rm -rf "$SCRIPT_PATH" + echo "Manually deleted script at $SCRIPT_PATH" + else + echo "Script directory not found at $SCRIPT_PATH. Exiting with failure." + exit 1 + fi + + # Verify the deleted script is no longer in the index + # The find command will automatically trigger index rebuild and detect the deletion + FIND_RESULT=$(mlc find script test-delete-script -p 2>/dev/null) + if [ -n "$FIND_RESULT" ]; then + echo "ERROR: Deleted script still found in index. Found: $FIND_RESULT" + exit 1 + fi + + echo "Script deletion test passed successfully" + + # Test with cache: run a script to create cache, then delete it manually + mlc run script --tags=detect,os --quiet + + # Find and delete a cache entry + CACHE_PATH=$(mlc find cache --tags=detect,os -p 2>/dev/null | head -1) + if [ -n "$CACHE_PATH" ] && [ -d "$CACHE_PATH" ]; then + echo "Found cache at: $CACHE_PATH" + + # Get the cache directory name for later verification + CACHE_DIR_NAME=$(basename "$CACHE_PATH") + + rm -rf "$CACHE_PATH" + echo "Manually deleted cache at $CACHE_PATH" + + # Verify the deleted cache is no longer in the index + # Check if the cache directory name appears in any found caches + if mlc find cache --tags=detect,os -p 2>/dev/null | grep -q "$CACHE_DIR_NAME"; then + echo "ERROR: Deleted cache directory still found in index." + echo "Deleted cache: $CACHE_PATH" + echo "Found caches:" + mlc find cache --tags=detect,os -p 2>/dev/null + exit 1 + fi + + echo "Cache deletion test passed successfully" + else + echo "No cache found to test deletion" + fi + test_mlc_access_core_actions: runs-on: ${{ matrix.os }} @@ -233,9 +343,9 @@ jobs: - os: macos-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} @@ -284,9 +394,9 @@ jobs: os: ["ubuntu-latest", "windows-latest", "macos-latest"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} @@ -371,9 +481,9 @@ jobs: action: ["mlcr", "mlce", "mlct"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/test-mlc-docker-core.yml b/.github/workflows/test-mlc-docker-core.yml index 30812ff19..c36684942 100644 --- a/.github/workflows/test-mlc-docker-core.yml +++ b/.github/workflows/test-mlc-docker-core.yml @@ -23,9 +23,9 @@ jobs: - os: macos-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/test-mlc-podman.yml b/.github/workflows/test-mlc-podman.yml index c7aec199a..6242ab6e0 100644 --- a/.github/workflows/test-mlc-podman.yml +++ b/.github/workflows/test-mlc-podman.yml @@ -25,9 +25,9 @@ jobs: - os: macos-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} diff --git a/VERSION b/VERSION index 852ed67cf..23aa83906 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.18 +1.2.2 diff --git a/docs/install/README.md b/docs/install/README.md new file mode 100644 index 000000000..1d01f4dd8 --- /dev/null +++ b/docs/install/README.md @@ -0,0 +1,375 @@ +# MLCFlow Unix Installer + +A Bootstrap installer for MLCFlow that automatically detects your Unix-based operating system (Linux/macOS), installs required dependencies, sets up a Python virtual environment, and configures the MLCFLow automation framework. + +> **Platform Note**: This installer is designed for **Unix-based systems only** (Linux distributions and macOS). It does not support Windows. Windows users should use WSL2 (Windows Subsystem for Linux) or a Linux virtual machine. We plan to release an installer script for Windows soon. + +## Purpose + +This installer provides a **one-command setup** for the MLCFlow package and the MLPerf automation repository. It handles all the complexity of: +- Detecting your Linux distribution or macOS +- Automatically detecting and using sudo when needed +- Installing missing system packages +- Validating Python installation and version +- Setting up isolated Python environments +- Installing MLCFlow and its dependencies +- Cloning the automation repository + +**After installation, activate the virtual environment** to use MLCFlow commands: +```bash +source ~/mlcflow/bin/activate +``` + +## Supported Platforms + +### Linux Distributions +- **Ubuntu**: 20.04, 22.04, 24.04 LTS +- **Debian**: 10+ (any version with Python 3.7+) +- **Rocky Linux**: 9.x +- **AlmaLinux**: 9.x +- **CentOS Stream**: 9 +- **RHEL**: 9+ (Red Hat Enterprise Linux) + +### macOS +- **macOS**: 11+ (Big Sur and later) +- **Requirement**: Homebrew must be installed + +### Architecture +- **x86_64** (amd64) +- **aarch64** (arm64) + +### Python Version +- **Minimum**: Python 3.7 +- **Recommended**: Python 3.8+ + +> **Note**: RHEL/Rocky Linux 8 ships with Python 3.6 by default, which is below the minimum requirement. Users should install Python 3.8+ from AppStream modules or alternative sources. + +## Installation Flow + +``` +┌─────────────────────────────────────────────────────────────┐ +│ 1. Detect Operating System & Package Manager │ +│ ├─ Ubuntu/Debian → apt │ +│ ├─ RHEL/Rocky/Alma → dnf/yum │ +│ └─ macOS → brew │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 2. Check System Dependencies │ +│ - git, curl/curl-minimal, wget, unzip │ +│ - python3, python3-pip, python3-venv │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 3. Install Missing Dependencies │ +│ - Uses detected package manager │ +│ - Requests sudo/root privileges if needed │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 4. Validate Python Environment │ +│ - Check Python version (≥ 3.7) │ +│ - Verify pip module availability │ +│ - Verify venv module availability │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 5. Create Virtual Environment │ +│ - Location: ~/mlcflow (or custom) │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 6. Install MLCFlow Package │ +│ - Install mlcflow via pip │ +│ - Install all dependencies │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 7. Prompt for Repository Details (if interactive) │ +│ - Repo name (default: mlcommons@mlperf-automations) │ +│ - Branch (default: dev) │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 8. Clone Automation Repository │ +│ - Uses 'mlc pull repo' command │ +│ - Stored in ~/MLC/repos/ (mlc default location) │ +└─────────────────────────────────────────────────────────────┘ + ↓ + ✅ Installation Complete +``` + +## Usage + +### Basic Installation (Interactive) +```bash +bash mlcflow_linux.sh +``` +Prompts for repository name and branch, then proceeds with installation. + +### Automated Installation (Non-Interactive) +```bash +bash mlcflow_linux.sh --yes +``` +Uses all default values without prompting. + +### Custom Virtual Environment Location +```bash +bash mlcflow_linux.sh --yes --venv-dir /opt/mlcflow_env +``` + +### Custom Repository and Branch +```bash +bash mlcflow_linux.sh \ + --yes \ + --mlc-repo myorganization@my-mlperf-fork \ + --mlc-repo-branch feature-branch +``` + +### Upgrade Existing Installation +```bash +bash mlcflow_linux.sh --yes --upgrade +``` + +### Verbose Mode (Debugging) +```bash +bash mlcflow_linux.sh --yes --verbose +``` + +### Quiet Mode (Minimal Output) +```bash +bash mlcflow_linux.sh --yes --quiet +``` + +## Command-Line Arguments + +| Argument | Description | Default | +|----------|-------------|---------| +| `--yes` | Auto-confirm all prompts (non-interactive mode) | Interactive | +| `--upgrade` | Upgrade mlcflow if already installed | Skip if present | +| `--venv-dir ` | Custom virtual environment directory | `~/mlcflow` | +| `--mlc-repo ` | Repository in format `owner@repo` | `mlcommons@mlperf-automations` | +| `--mlc-repo-branch ` | Git branch to clone | `dev` | +| `--install-python` | Auto-install Python if incompatible | Prompt user | +| `--verbose` | Enable debug logging | Normal logging | +| `--quiet` | Minimal output (errors/warnings only) | Normal logging | +| `--help` | Display help message and exit | - | + +## SUDO and Privilege Handling + +The installer automatically detects your privilege level and handles system package installation accordingly. **You do not need to explicitly specify sudo** - the script handles it internally. + +### How the Script Detects Privileges + +When you run the script, it automatically checks: +1. **Are you running as root?** (EUID == 0) +2. **Is the `sudo` command available?** +3. **What package manager is being used?** + +Based on this detection, it chooses the appropriate execution method. + + +### Privilege Detection Logic +```bash +# The installer detects privileges in this order: +1. Check if running as root (EUID == 0) +2. Check if 'sudo' command is available +3. For package managers: + - apt/yum/dnf: Require root or sudo + - brew (macOS): No sudo needed +4. For Python operations: No privileges required (user-space) +``` + +### Summary: What Requires Privileges? + +| Operation | Requires Root/Sudo | +|-----------|-------------------| +| Install system packages | Yes | +| Install system packages | Yes | +| Install system packages | +| Create Python venv | No | +| Install pip packages | No | +| Clone git repositories | No | + +## What Gets Installed + +### System Packages +**Ubuntu/Debian**: +- `python3`, `python3-pip`, `python3-venv` +- `git`, `curl`, `wget`, `unzip` + +**RHEL/Rocky/Alma/CentOS**: +- `python3`, `python3-pip`, `python3-venv` +- `git`, `curl-minimal`, `wget`, `unzip` + +> **Note**: Uses `curl-minimal` on RHEL systems to avoid package conflicts with pre-installed `curl-minimal`. + +**macOS (via Homebrew)**: +- `python`, `git`, `curl`, `wget`, `unzip` + +### Python Packages (in virtual environment) +- `mlcflow` - Main automation CLI +- `requests` - HTTP library +- `pyyaml` - YAML parser +- `giturlparse` - Git URL utilities +- `colorama` - Cross-platform colored terminal output + +### Automation Repository +- Cloned via `mlc pull repo` command +- Default location: `~/MLC/repos/mlcommons@mlperf-automations/` +- Contains MLPerf automation scripts and configurations + +## Post-Installation + +### Activate Virtual Environment (Required) + +**Important**: To use MLCFlow commands, you must activate the virtual environment: + +```bash +# Activate the virtual environment +source ~/mlcflow/bin/activate + +# Your prompt should change to show (mlcflow) or similar +``` + +### Verify Installation +```bash +# After activating the virtual environment: +mlc --help + +# Verify version +mlc version + +# List available scripts +mlc list scripts +``` + +### Deactivate Virtual Environment +```bash +# When you're done working with MLCFlow +deactivate +``` + +### File Locations +- **Virtual Environment**: `~/mlcflow` (or custom path) +- **Automation Repository**: `~/MLC/repos/mlcommons@mlperf-automations/` +- **MLC Cache**: `~/MLC/repos/` + +## Troubleshooting + +### "Python version < 3.7" +**Problem**: System Python is too old (e.g., Python 3.6 on RHEL 8) + +**Solution**: +```bash +# RHEL 8 / Rocky 8 +sudo dnf module install python38 +# or +sudo dnf install python39 + +# Then rerun installer +bash mlcflow_linux.sh --yes +``` + +### "mlc: command not found" after installation +**Problem**: Virtual environment is not activated + +**Solution**: +```bash +# Activate the virtual environment +source ~/mlcflow/bin/activate + +# Now mlc command should be available +mlc --help +``` + +### Package conflicts on RHEL/Rocky +**Problem**: `curl` and `curl-minimal` conflict + +**Solution**: The installer now uses `curl-minimal` automatically. If you encounter issues: +```bash +# Remove conflicting package +sudo dnf remove curl + +# Rerun installer +bash mlcflow_linux.sh --yes +``` + +### "Homebrew not found" on macOS +**Problem**: Homebrew is not installed + +**Solution**: +```bash +# Install Homebrew +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + +# Rerun installer +bash mlcflow_linux.sh --yes +``` + +### Permission denied errors +**Problem**: Missing sudo privileges and required packages are not installed + +**Solution**: +```bash +# If you don't have sudo AND packages are missing +# Ask your system administrator to install these packages: +# - Ubuntu/Debian: python3 python3-pip python3-venv git curl wget unzip +# - RHEL/Rocky: python3 python3-pip python3-venv git curl-minimal wget unzip +# Then run: +bash mlcflow_linux.sh --yes +``` + +### Virtual environment creation fails +**Problem**: `python3-venv` module missing + +**Solution**: The installer should detect and install it, but if manual installation is needed: +```bash +# Ubuntu/Debian +sudo apt install python3-venv + +# RHEL/Rocky +sudo dnf install python3-venv + +# Rerun installer +bash mlcflow_linux.sh --yes +``` + +## Testing and CI + +### Automated Testing + +The installer is continuously tested via GitHub Actions across all supported platforms using the exact distribution method that users will use in production (`curl /mlcflow_linux.sh | bash`). The CI workflow validates: + +- **Operating Systems**: Ubuntu 20.04/22.04/24.04, macOS 13, Debian 11/12, Rocky Linux 9, AlmaLinux 9, CentOS Stream 9 + +### Known Testing Limitations + +**TODO**: The CI workflow currently only tests **non-interactive mode** (with `--yes` flag). The interactive installation path, which prompts users for repository name and branch, is not covered by automated tests. + +If you discover issues with interactive mode, please report them via GitHub Issues [here](https://github.com/mlcommons/mlcflow/issues). + +## Support + +For issues, feature requests, or contributions: +- **Repository**: https://github.com/mlcommons/mlperf-automations +- **Issues**: https://github.com/mlcommons/mlperf-automations/issues +- **Documentation**: https://mlcommons.github.io/mlperf-automations/ + +## License + +This installer script is part of the MLPerf Automations project and is distributed under the same license as the main project. + +## Changelog + +### v1.0 (Current) +- Initial release with comprehensive OS support (Linux/macOS) +- Automatic dependency detection and installation +- pip/venv module validation +- Automatic sudo detection and privilege handling +- macOS support with Homebrew +- Interactive and non-interactive modes +- RHEL package conflict resolution (curl-minimal) +- Manual virtual environment activation required + +--- diff --git a/docs/install/mlcflow_linux.sh b/docs/install/mlcflow_linux.sh new file mode 100755 index 000000000..123fc72aa --- /dev/null +++ b/docs/install/mlcflow_linux.sh @@ -0,0 +1,419 @@ +#!/usr/bin/env bash +# ============================================================================== +# MLCFlow Generic Installer (v1) +# Supports: +# - Ubuntu 20.04+ +# - RHEL family (RHEL, Alma, CentOS Stream) +# - macOS (with Homebrew) +# - x86_64 and aarch64 + +# Exit if a command fails +# Treats unset variables as errors +# Makes pipeline fails if any command fails +set -euo pipefail + +# ------------------------------------------------------------------------------ +# Default Configuration +# ------------------------------------------------------------------------------ + +MIN_PYTHON_VERSION="3.7" +DEFAULT_VENV_DIR="$HOME/mlcflow" +DEFAULT_REPO="mlcommons@mlperf-automations" +DEFAULT_BRANCH="dev" + +UPGRADE=false +ASSUME_YES=false +INSTALL_PYTHON=false +VERBOSE=false +QUIET=false +VENV_DIR="$DEFAULT_VENV_DIR" +MLC_REPO="$DEFAULT_REPO" +MLC_BRANCH="$DEFAULT_BRANCH" + +# ------------------------------------------------------------------------------ +# Logging System +# ------------------------------------------------------------------------------ + +INTERACTIVE=false +# checks if the stdout connected to a terminal +if [ -t 1 ]; then + INTERACTIVE=true +fi + +if $INTERACTIVE; then + COLOR_RED="\033[0;31m" + COLOR_GREEN="\033[0;32m" + COLOR_YELLOW="\033[1;33m" + COLOR_BLUE="\033[0;34m" + COLOR_RESET="\033[0m" +else + COLOR_RED="" + COLOR_GREEN="" + COLOR_YELLOW="" + COLOR_BLUE="" + COLOR_RESET="" +fi + +log_info() { + $QUIET && return + echo -e "${COLOR_GREEN}[INFO]${COLOR_RESET} $1" +} + +log_warn() { + echo -e "${COLOR_YELLOW}[WARN]${COLOR_RESET} $1" +} + +log_error() { + echo -e "${COLOR_RED}[ERROR]${COLOR_RESET} $1" +} + +log_debug() { + $VERBOSE || return + echo -e "${COLOR_BLUE}[DEBUG]${COLOR_RESET} $1" +} + +# ------------------------------------------------------------------------------ +# Usage +# ------------------------------------------------------------------------------ + +usage() { +cat < Custom virtual environment path + --mlc-repo Override automation repo + --mlc-repo-branch Override repo branch + --install-python Auto-install Python if incompatible + --verbose Enable debug logs + --quiet Minimal output + --help Show this help + +EOF +exit 0 +} + +# ------------------------------------------------------------------------------ +# Argument Parsing +# ------------------------------------------------------------------------------ + +# Loops through the arguments provided +# If an option expects a value after it, reads it and skips for next iteration +while [[ $# -gt 0 ]]; do + case "$1" in + --yes) ASSUME_YES=true ;; + --upgrade) UPGRADE=true ;; + --venv-dir) VENV_DIR="$2"; shift ;; + --mlc-repo) MLC_REPO="$2"; shift ;; + --mlc-repo-branch) MLC_BRANCH="$2"; shift ;; + --install-python) INSTALL_PYTHON=true ;; + --verbose) VERBOSE=true ;; + --quiet) QUIET=true ;; + --help) usage ;; + *) log_error "Unknown argument: $1"; exit 1 ;; + esac + shift +done + +# ------------------------------------------------------------------------------ +# Detect OS and Package Manager +# ------------------------------------------------------------------------------ + +detect_os() { + if [ "$(uname)" = "Darwin" ]; then + OS_ID="macos" + OS_VERSION="$(sw_vers -productVersion 2>/dev/null || echo unknown)" + else + if [ ! -f /etc/os-release ]; then + log_error "Cannot detect operating system." + exit 1 + fi + # loads the content from os-release as variables + source /etc/os-release + OS_ID="$ID" + OS_VERSION="$VERSION_ID" + fi + + case "$OS_ID" in + ubuntu|debian) + PKG_MANAGER="apt" + ;; + rhel|rocky|almalinux|centos) + if command -v dnf >/dev/null 2>&1; then + PKG_MANAGER="dnf" + else + PKG_MANAGER="yum" + fi + ;; + macos) + PKG_MANAGER="brew" + ;; + *) + log_error "Unsupported OS: $OS_ID" + exit 1 + ;; + esac + + log_info "Detected OS: $OS_ID $OS_VERSION" + log_info "Using package manager: $PKG_MANAGER" +} + +# ------------------------------------------------------------------------------ +# Privilege Detection +# ------------------------------------------------------------------------------ + +# Handles the following cases: +# 1. If the script is already running as root (EUID=0), commands can be executed directly. +# 2. If the script is not running as root but the `sudo` command is available, +# privileged commands will be executed using sudo. +# 3. If neither root privileges nor sudo are available, the script will fail +# when attempting to run commands that require elevated permissions. +if [ "$EUID" -eq 0 ]; then + USE_SUDO=false +elif command -v sudo >/dev/null 2>&1; then + USE_SUDO=true +else + USE_SUDO=false +fi + +run_root() { + if $USE_SUDO; then + sudo "$@" + elif [ "$EUID" -eq 0 ]; then + "$@" + else + log_error "Root or sudo required to install system dependencies." + exit 1 + fi +} + +# ------------------------------------------------------------------------------ +# System Dependencies +# ------------------------------------------------------------------------------ + +require_root_if_needed() { + if [ "$PKG_MANAGER" = "brew" ]; then + return + fi + + if [ "$EUID" -ne 0 ] && ! $USE_SUDO; then + log_error "Root or sudo required to install missing dependencies." + exit 1 + fi +} + +have_pip_module() { + python3 -m pip --version >/dev/null 2>&1 +} + +have_venv_module() { + python3 -c 'import venv' >/dev/null 2>&1 +} + +check_missing_dependencies() { + MISSING_DEPS=() + + command -v git >/dev/null 2>&1 || MISSING_DEPS+=("git") + command -v curl >/dev/null 2>&1 || MISSING_DEPS+=("curl") + command -v wget >/dev/null 2>&1 || MISSING_DEPS+=("wget") + command -v unzip >/dev/null 2>&1 || MISSING_DEPS+=("unzip") + + if ! command -v python3 >/dev/null 2>&1; then + MISSING_DEPS+=("python3") + else + have_pip_module || MISSING_DEPS+=("python3-pip") + have_venv_module || MISSING_DEPS+=("python3-venv") + fi +} + +install_packages() { + log_info "Installing system dependencies..." + + case "$PKG_MANAGER" in + apt) + require_root_if_needed + run_root apt update + run_root apt install -y python3 python3-pip python3-venv git curl wget unzip + ;; + yum|dnf) + require_root_if_needed + # RHEL-family images may ship curl-minimal and conflict with curl package. + run_root "$PKG_MANAGER" install -y python3 python3-pip git curl-minimal wget unzip + # Some RHEL-family variants package venv separately + run_root "$PKG_MANAGER" install -y python3-venv >/dev/null 2>&1 || true + ;; + brew) + if ! command -v brew >/dev/null 2>&1; then + log_error "Homebrew is required on macOS. Please install it from https://brew.sh" + exit 1 + fi + brew update + brew install python git curl wget unzip + ;; + esac +} + +# ------------------------------------------------------------------------------ +# Python Validation +# ------------------------------------------------------------------------------ + +version_ge() { + [ "$(printf '%s\n' "$2" "$1" | sort -V | head -n1)" = "$2" ] +} + +ensure_python() { + if ! command -v python3 >/dev/null 2>&1; then + log_warn "Python3 not found." + handle_python_install + fi + + if ! command -v python3 >/dev/null 2>&1; then + log_error "Python3 is still unavailable after attempted installation." + exit 1 + fi + + PY_VERSION=$(python3 -c 'import sys; print(".".join(map(str, sys.version_info[:3])))') + log_info "Detected Python version: $PY_VERSION" + + if version_ge "$PY_VERSION" "$MIN_PYTHON_VERSION"; then + log_info "Python version is compatible." + else + log_warn "Python version < $MIN_PYTHON_VERSION" + handle_python_install + fi + + if ! have_pip_module; then + log_warn "python3 pip module is missing. Installing..." + install_packages + fi + + if ! have_venv_module; then + log_warn "python3 venv module is missing. Installing..." + install_packages + fi + + if ! have_pip_module || ! have_venv_module; then + log_error "pip/venv modules are still missing after attempted installation." + exit 1 + fi +} + +handle_python_install() { + if $INSTALL_PYTHON || $ASSUME_YES; then + install_packages + return + fi + + if ! $INTERACTIVE; then + log_error "Incompatible Python and non-interactive mode. Run with --install-python to automatically install." + exit 1 + fi + + read -p "Install compatible Python? [y/N]: " response + if [[ "$response" =~ ^[Yy]$ ]]; then + install_packages + else + log_error "Cannot proceed without compatible Python." + exit 1 + fi +} + +# ------------------------------------------------------------------------------ +# Virtual Environment +# ------------------------------------------------------------------------------ + +setup_venv() { + log_info "Setting up virtual environment at: $VENV_DIR" + + if [ -d "$VENV_DIR" ]; then + log_info "Reusing existing virtual environment." + else + python3 -m venv "$VENV_DIR" + fi + + # Activate venv + # shellcheck disable=SC1090 + source "$VENV_DIR/bin/activate" +} + +# ------------------------------------------------------------------------------ +# Install / Upgrade MLCFlow +# ------------------------------------------------------------------------------ + +install_mlcflow() { + if python3 -m pip show mlcflow >/dev/null 2>&1; then + if $UPGRADE; then + log_info "Upgrading mlcflow..." + python3 -m pip install --upgrade mlcflow + else + log_info "mlcflow already installed. Skipping." + fi + else + log_info "Installing mlcflow..." + python3 -m pip install mlcflow + fi +} + +# ------------------------------------------------------------------------------ +# Pull Automation Repo +# ------------------------------------------------------------------------------ + +prompt_repo_details() { + if ! $INTERACTIVE || $ASSUME_YES; then + return + fi + + read -r -p "Automation repo [${MLC_REPO}]: " repo_input + if [ -n "${repo_input}" ]; then + MLC_REPO="${repo_input}" + fi + + read -r -p "Automation branch [${MLC_BRANCH}]: " branch_input + if [ -n "${branch_input}" ]; then + MLC_BRANCH="${branch_input}" + fi +} + +pull_repo() { + log_info "Pulling automation repo:" + log_info " Repo : ${MLC_REPO}" + log_info " Branch : ${MLC_BRANCH}" + + mlc pull repo "${MLC_REPO}" --checkout="${MLC_BRANCH}" +} + +# ------------------------------------------------------------------------------ +# Main Execution +# ------------------------------------------------------------------------------ + +main() { + detect_os + check_missing_dependencies + if [ "${#MISSING_DEPS[@]}" -gt 0 ]; then + log_warn "Missing dependencies: ${MISSING_DEPS[*]}" + install_packages + else + log_info "All base dependencies are present." + fi + + ensure_python + setup_venv + install_mlcflow + prompt_repo_details + pull_repo + + log_info "Installation completed successfully." + echo "" + echo "Virtual environment:" + echo " $VENV_DIR" + echo "" + echo "Activate with:" + echo " source $VENV_DIR/bin/activate" + echo "" + echo "Verify:" + echo " mlc --help" +} + +main diff --git a/mlc/__init__.py b/mlc/__init__.py index 968168c9b..8320a86a4 100644 --- a/mlc/__init__.py +++ b/mlc/__init__.py @@ -1,5 +1,44 @@ -__version__ = "0.1.0" - from .action import access +import os +import subprocess + + +def _get_version(): + """Read version from VERSION file or package metadata, and append git commit hash if available.""" + pkg_dir = os.path.dirname(os.path.abspath(__file__)) + root_dir = os.path.dirname(pkg_dir) + + # Read VERSION file (works in dev/source tree) + version = None + for vpath in [os.path.join(root_dir, "VERSION"), + os.path.join(pkg_dir, "VERSION")]: + if os.path.isfile(vpath): + with open(vpath) as f: + version = f.read().strip() + break + + # Fall back to installed package metadata + if not version: + try: + from importlib.metadata import version as pkg_version + version = pkg_version("mlcflow") + except Exception: + version = "0.0.0" + + # Append git short commit hash if in a git repo + try: + commit = subprocess.check_output( + ["git", "-C", root_dir, "rev-parse", "--short", "HEAD"], + stderr=subprocess.DEVNULL, text=True + ).strip() + if commit: + version = f"{version}+{commit}" + except Exception: + pass + + return version + + +__version__ = _get_version() __all__ = ['access'] diff --git a/mlc/action.py b/mlc/action.py index 665727a0e..28df4b373 100644 --- a/mlc/action.py +++ b/mlc/action.py @@ -16,6 +16,8 @@ from .error_codes import WarningCode # Base class for actions + + class Action: repos_path = None cfg = None @@ -23,7 +25,7 @@ class Action: logger = None local_repo = None current_repo_path = None - repos = [] #list of Repo objects + repos = [] # list of Repo objects # Main access function to simulate a Python interface for CLI def access(self, options): @@ -35,42 +37,47 @@ def access(self, options): """ from .action_factory import get_action - #logger.info(f"options in access = {options}") - + # logger.info(f"options in access = {options}") + action_name = options.get('action') if not action_name: return {'return': 1, 'error': "'action' key is required in options"} - #logger.info(f"options = {options}") + # logger.info(f"options = {options}") action_name = action_name.replace("-", "_") action_target = options.get('target') if not action_target: - action_target = options.get('automation', 'script') # Default to script if not provided + # Default to script if not provided + action_target = options.get('automation', 'script') action_target_split = action_target.split(",") action_target = action_target_split[0] - action = get_action(action_target, self.parent if self.parent else self) + action = get_action(action_target, + self.parent if self.parent else self) if action and hasattr(action, action_name): # Find the method and call it with the options method = getattr(action, action_name) result = method(options) - #logger.info(f"result ={result}") + # logger.info(f"result ={result}") return result else: - return {'return': 1, 'error': f"'{action_name}' action is not supported for {action_target}."} + return { + 'return': 1, 'error': f"'{action_name}' action is not supported for {action_target}."} return {'return': 0} def find_target_folder(self, target): - # Traverse through each repo to find the first 'target' folder inside an 'automation' folder + # Traverse through each repo to find the first 'target' folder inside + # an 'automation' folder for repo in self.repos: repo_path = repo.path if os.path.isdir(repo_path): automation_folder = os.path.join(repo_path, 'automation') - + if os.path.isdir(automation_folder): - # Check if there's a 'script' folder inside the 'automation' folder + # Check if there's a 'script' folder inside the + # 'automation' folder target_folder = os.path.join(automation_folder, target) if os.path.isdir(target_folder): return target_folder @@ -82,7 +89,8 @@ def load_repos_and_meta(self): # Read the JSON file line by line try: - # Load and parse the JSON file containing the list of repository paths + # Load and parse the JSON file containing the list of repository + # paths with open(repos_file_path, 'r') as file: repo_paths = json.load(file) # Load the JSON file into a list except json.JSONDecodeError as e: @@ -94,7 +102,7 @@ def load_repos_and_meta(self): except Exception as e: logger.error(f"Error reading file: {e}") return [] - + def is_curdir_inside_path(base_path): # Convert to absolute paths base_path = Path(base_path).resolve() @@ -106,9 +114,15 @@ def is_curdir_inside_path(base_path): # Iterate through the list of repository paths for repo_path in repo_paths: if not os.path.exists(repo_path): - logger.warning(f"""Warning: {repo_path} not found. Considering it as a corrupt entry and deleting from repos.json...""") + logger.warning( + f"""Warning: {repo_path} not found. Considering it as a corrupt entry and deleting from repos.json...""") from .repo_action import rm_repo - res = rm_repo(repo_path, os.path.join(self.repos_path, 'repos.json'), True) + res = rm_repo( + repo_path, + os.path.join( + self.repos_path, + 'repos.json'), + True) if res["return"] > 0: return res @@ -126,7 +140,8 @@ def is_curdir_inside_path(base_path): # Check if meta.yaml exists if not os.path.isfile(meta_yaml_path): - logger.warning(f"{meta_yaml_path} not found. Could be due to accidental deletion of meta.yaml. Try to stash the changes or reclone by doing `rm repo` and `pull repo`. Skipping...") + logger.warning( + f"{meta_yaml_path} not found. Could be due to accidental deletion of meta.yaml. Try to stash the changes or reclone by doing `rm repo` and `pull repo`. Skipping...") continue # Load the YAML file @@ -164,49 +179,54 @@ def load_repos(self): except Exception as e: logger.error(f"Error reading file: {e}") return None - + def get_index(self): if self._index is None: self._index = Index(self.repos_path, self.repos) return self._index - def __init__(self): + def __init__(self): setup_logging(log_path=os.getcwd(), log_file='.mlc-log.txt') self.logger = logger - temp_repo = os.environ.get('MLC_REPOS','').strip() + temp_repo = os.environ.get('MLC_REPOS', '').strip() if temp_repo == '': - self.repos_path = os.path.join(os.path.expanduser("~"), "MLC", "repos") + self.repos_path = os.path.join( + os.path.expanduser("~"), "MLC", "repos") else: self.repos_path = temp_repo mlc_local_repo_path = os.path.join(self.repos_path, 'local') - - mlc_local_repo_path_expanded = Path(mlc_local_repo_path).expanduser().resolve() + + mlc_local_repo_path_expanded = Path( + mlc_local_repo_path).expanduser().resolve() if not os.path.exists(mlc_local_repo_path): os.makedirs(mlc_local_repo_path, exist_ok=True) - + if not os.path.isfile(os.path.join(mlc_local_repo_path, "meta.yaml")): - local_repo_meta = {"alias": "local", "name": "MLC local repository", "uid": utils.get_new_uid()['uid']} + local_repo_meta = { + "alias": "local", + "name": "MLC local repository", + "uid": utils.get_new_uid()['uid']} with open(os.path.join(mlc_local_repo_path, "meta.yaml"), "w") as json_file: json.dump(local_repo_meta, json_file, indent=4) - + repo_json_path = os.path.join(self.repos_path, "repos.json") if not os.path.exists(repo_json_path): with open(repo_json_path, 'w') as f: json.dump([str(mlc_local_repo_path_expanded)], f, indent=2) - logger.info(f"Created repos.json in {os.path.dirname(self.repos_path)} and initialised with local cache folder path: {mlc_local_repo_path}") + logger.info( + f"Created repos.json in {os.path.dirname(self.repos_path)} and initialised with local cache folder path: {mlc_local_repo_path}") self.local_cache_path = os.path.join(mlc_local_repo_path, "cache") if not os.path.exists(self.local_cache_path): os.makedirs(self.local_cache_path, exist_ok=True) self.repos = self.load_repos_and_meta() - #logger.info(f"In Action class: {self.repos_path}") + # logger.info(f"In Action class: {self.repos_path}") self._index = None - def add(self, i): """ Adds a new item to the repository. @@ -226,7 +246,6 @@ def add(self, i): item_repo = i.get("item_repo") if not item_repo: item_repo = self.local_repo - # Parse item details item = i.get("item") @@ -257,21 +276,23 @@ def add(self, i): return res if len(res["list"]) == 0: - return {'return': 1, 'error': f"""The given repo {item_repo} is not registered in MLC"""} + return { + 'return': 1, 'error': f"""The given repo {item_repo} is not registered in MLC"""} # Determine paths and metadata format repo = res["list"][0] repo_path = repo.path - + target_name = i.get('target_name', self.action_type) target_path = os.path.join(repo_path, target_name) if target_name in ["cache", "experiment"]: - extra_tags_suffix=i.get('extra_tags', '').replace(",", "-")[:15] + extra_tags_suffix = i.get('extra_tags', '').replace(",", "-")[:15] if extra_tags_suffix != '': suffix = f"_{extra_tags_suffix}" else: suffix = '' - folder_name = f"""{i["script_alias"]}{suffix}_{item_name or item_id[:8]}""" if i.get("script_alias") else item_name or item_id + folder_name = f"""{i["script_alias"]}{suffix}_{item_name or item_id[:8]}""" if i.get( + "script_alias") else item_name or item_id else: folder_name = item_name or item_id @@ -283,7 +304,13 @@ def add(self, i): # Create item directory if it does not exist os.makedirs(item_path) - res = self.save_new_meta(i, item_id, item_name, target_name, item_path, repo) + res = self.save_new_meta( + i, + item_id, + item_name, + target_name, + item_path, + repo) if res['return'] > 0: return res @@ -293,7 +320,7 @@ def add(self, i): "path": item_path, "repo": repo } - + def rm(self, i): """ Removes an item from the repository. @@ -311,7 +338,7 @@ def rm(self, i): inp = {} # Parse item details - item = i.get("item",i.get('artifact', i.get('details'))) + item = i.get("item", i.get('artifact', i.get('details'))) item_name, item_id, item_tags = (None, None, None) if item: item_parts = item.split(",") @@ -327,12 +354,15 @@ def rm(self, i): inp['fetch_all'] = True # Check force remove is set to True - # Setting force remove to true would lead to removal of assets without user prompt + # Setting force remove to true would lead to removal of assets without + # user prompt force_remove = True if i.get('f') else False if item_name: inp['alias'] = item_name - inp['folder_name'] = item_name #we dont know if the user gave the alias or the folder name, we first check for alias and then the folder name + # we dont know if the user gave the alias or the folder name, we + # first check for alias and then the folder name + inp['folder_name'] = item_name if utils.is_uid(item_name): inp['uid'] = item_name elif item_id: @@ -349,11 +379,14 @@ def rm(self, i): if len(res['list']) == 0: # Do not error out if fetch_all is used if inp.get("fetch_all", False) == True: - logger.warning(f"{target_name} is empty! nothing to be cleared!") - return {"return": 0, "warnings": [{"code": WarningCode.EMPTY_TARGET.code, "description": f"{target_name} is empty! nothing to be cleared!"}]} + logger.warning( + f"{target_name} is empty! nothing to be cleared!") + return {"return": 0, "warnings": [ + {"code": WarningCode.EMPTY_TARGET.code, "description": f"{target_name} is empty! nothing to be cleared!"}]} else: logger.warning(f"No {target_name} found for {inp}") - return {'return': 0, "warnings": [{"code": WarningCode.EMPTY_TARGET.code, "description": f"No {target_name} found for {inp}"}]} + return {'return': 0, "warnings": [ + {"code": WarningCode.EMPTY_TARGET.code, "description": f"No {target_name} found for {inp}"}]} elif len(res['list']) > 1: logger.info(f"More than 1 {target_name} found for {inp}:") if not i.get('all'): @@ -361,37 +394,40 @@ def rm(self, i): logger.info(f"{idx}. Path: {item.path}, Meta: {item.meta}") if not force_remove: - user_choice = input("Would you like to proceed with all items? (yes/no): ").strip().lower() + user_choice = input( + "Would you like to proceed with all items? (yes/no): ").strip().lower() if user_choice in ['yes', 'y']: force_remove = True - + results = res['list'] - + for result in results: item_path = result.path item_meta = result.meta - - + if os.path.exists(item_path): if force_remove == True: shutil.rmtree(item_path) else: - user_choice = input(f"Confirm to delete {target_name} item: {item_path}? (yes/no): ").strip().lower() + user_choice = input( + f"Confirm to delete {target_name} item: {item_path}? (yes/no): ").strip().lower() if user_choice not in ['yes', 'y']: continue else: shutil.rmtree(item_path) - logger.info(f"{target_name} item: {item_path} has been successfully removed") + logger.info( + f"{target_name} item: {item_path} has been successfully removed") self.get_index().rm(item_meta, target_name, item_path) - + return { "return": 0, "message": f"Item {item_path} successfully removed", } - def save_new_meta(self, i, item_id, item_name, target_name, item_path, repo): + def save_new_meta(self, i, item_id, item_name, + target_name, item_path, repo): # Prepare metadata item_meta = i.get('meta', {}) item_meta.update({ @@ -400,8 +436,10 @@ def save_new_meta(self, i, item_id, item_name, target_name, item_path, repo): }) # Process tags - tags = i.get("tags", "").split(",") if i.get("tags") else item_meta.get("tags", []) - new_tags = i.get("new_tags", "").split(",") if i.get("new_tags") else [] + tags = i.get("tags", "").split(",") if i.get( + "tags") else item_meta.get("tags", []) + new_tags = i.get("new_tags", "").split( + ",") if i.get("new_tags") else [] item_meta["tags"] = list(set(tags + new_tags)) # Ensure unique tags @@ -416,7 +454,7 @@ def save_new_meta(self, i, item_id, item_name, target_name, item_path, repo): if save_result["return"] > 0: return save_result - + self.get_index().add(item_meta, target_name, item_path, repo) return {'return': 0} @@ -446,10 +484,11 @@ def update(self, i): found_items = search_result['list'] if not found_items: res = self.add(i) - if res['return'] > 0 : + if res['return'] > 0: return res found_items.append(Item(res['path'], res['repo'])) - #return {'return': 0, 'message': 'No items found for the given tags.'} + # return {'return': 0, 'message': 'No items found for the given + # tags.'} # Step 2: Prepare to update tags search_tags = i.get("search_tags", []) @@ -457,9 +496,11 @@ def update(self, i): new_tags = set(search_tags) if len(found_items) > 1: # Step 3: Ask user for confirmation if multiple items are found - user_input = input(f"{len(found_items)} items found. Do you want to update all? (yes/no): ").strip().lower() + user_input = input( + f"{len(found_items)} items found. Do you want to update all? (yes/no): ").strip().lower() if user_input not in ['yes', 'y']: - return {'return': 0, 'message': 'Update operation canceled by the user.'} + return {'return': 0, + 'message': 'Update operation canceled by the user.'} new_meta = i.get('meta') if new_meta.get('tags'): @@ -472,7 +513,7 @@ def update(self, i): item_meta_path = os.path.join(item.path, "meta.json") if os.path.exists(item_meta_path): res = utils.load_json(item_meta_path) - if res['return']> 0: + if res['return'] > 0: return res meta = res['meta'] if i.get('replace_lists') and i.get("tags"): @@ -481,30 +522,33 @@ def update(self, i): current_tags = set(meta.get("tags", [])) updated_tags = current_tags.union(new_tags) meta["tags"] = list(updated_tags) - utils.merge_dicts({"dict1": meta, "dict2": new_meta, "append_lists": True, "append_unique":True}) - + utils.merge_dicts({"dict1": meta, + "dict2": new_meta, + "append_lists": True, + "append_unique": True}) + # Save the updated meta back to the item item.meta = meta save_result = utils.save_json(item_meta_path, meta=meta) self.get_index().update(meta, target_name, item.path, item.repo) - return {'return': 0, 'message': f"Tags updated successfully for {len(found_items)} item(s).", 'list': found_items } - - + return { + 'return': 0, 'message': f"Tags updated successfully for {len(found_items)} item(s).", 'list': found_items} def cp(self, run_args): action_target = run_args['target'] if action_target != "script": - return {"return": 1, "error": f"The {action_target} target is not currently supported for mv/cp actions"} + return { + "return": 1, "error": f"The {action_target} target is not currently supported for mv/cp actions"} inp = {} src_item = run_args.get('src') src_tags = None - + if src_item: # remove backslash if there in src item if src_item.endswith('/'): src_item = src_item[:-1] - + src_split = src_item.split(":") if len(src_split) > 1: src_repo = src_split[0].strip() @@ -513,15 +557,18 @@ def cp(self, run_args): src_item = src_split[0].strip() inp['alias'] = src_item - inp['folder_name'] = src_item #we dont know if the user gave the alias or the folder name, we first check for alias and then the folder name - + # we dont know if the user gave the alias or the folder name, we + # first check for alias and then the folder name + inp['folder_name'] = src_item + if utils.is_uid(src_item): inp['uid'] = src_item src_id = src_item else: - #src_tags must be there + # src_tags must be there if not run_args.get("src_tags"): - return {'return': 1, 'error': 'Either "src" or "src_tags" must be provided as an input for cp method'} + return { + 'return': 1, 'error': 'Either "src" or "src_tags" must be provided as an input for cp method'} src_tags = run_args['src_tags'] inp['tags'] = src_tags src_id = src_tags @@ -542,7 +589,8 @@ def cp(self, run_args): # Ask user to choose an item while True: - choice = input("Select the correct one (enter number, default=1): ").strip() + choice = input( + "Select the correct one (enter number, default=1): ").strip() if choice == "": choice = 1 try: @@ -550,7 +598,8 @@ def cp(self, run_args): if 0 <= choice < len(res['list']): break else: - print("Invalid selection. Please enter a number from the list.") + print( + "Invalid selection. Please enter a number from the list.") except ValueError: print("Invalid input. Please enter a number.") @@ -565,21 +614,25 @@ def cp(self, run_args): target_repo_name = target_split[0].strip() if target_repo_name == ".": if not self.current_repo_path: - return {'return': 1, 'error': f"""Current directory is not inside a registered MLC repo and so using ".:" is not valid"""} + return { + 'return': 1, 'error': f"""Current directory is not inside a registered MLC repo and so using ".:" is not valid"""} target_repo_name = os.path.basename(self.current_repo_path) else: - if not any(os.path.basename(repodata.path) == target_repo_name for repodata in self.repos): + if not any(os.path.basename(repodata.path) == + target_repo_name for repodata in self.repos): return {'return': 1, 'error': f"""The target repo {target_repo} is not registered in MLC. Either register in MLC by cloning from Git through command `mlc pull repo` or create repo using `mlc add repo` command and try to rerun the command again"""} target_repo_path = os.path.join(self.repos_path, target_repo_name) - target_repo = next((k for k in self.repos if os.path.basename(k.path) == target_repo_name), None) + target_repo = next( + (k for k in self.repos if os.path.basename( + k.path) == target_repo_name), None) target_item_name = target_split[1].strip() else: target_repo = result.repo target_repo_path = result.repo.path target_item_name = target_split[0].strip() - - target_item_path = os.path.join(target_repo_path, action_target, target_item_name) + target_item_path = os.path.join( + target_repo_path, action_target, target_item_name) res = self.copy_item(src_item_path, target_item_path) if res['return'] > 0: return res @@ -602,13 +655,20 @@ def cp(self, run_args): return res item_id = res['uid'] - res = self.save_new_meta(ii, item_id, target_item_name, action_target, target_item_path, target_repo) + res = self.save_new_meta( + ii, + item_id, + target_item_name, + action_target, + target_item_path, + target_repo) dest_item = Item(target_item_path, target_repo) - + if res['return'] > 0: return res - logger.info(f"{action_target} {src_item_path} copied to {target_item_path}") + logger.info( + f"{action_target} {src_item_path} copied to {target_item_path}") return {'return': 0, 'src': result, 'dest': dest_item} @@ -616,9 +676,11 @@ def copy_item(self, source_path, destination_path): try: # Copy the source folder to the destination shutil.copytree(source_path, destination_path) - logger.info(f"Folder successfully copied from {source_path} to {destination_path}") + logger.info( + f"Folder successfully copied from {source_path} to {destination_path}") except FileExistsError: - return {'return': 1, 'error': f"Destination folder {destination_path} already exists."} + return { + 'return': 1, 'error': f"Destination folder {destination_path} already exists."} except FileNotFoundError: return {'return': 1, 'error': f"Source folder {source_path} not found"} except Exception as e: @@ -629,7 +691,8 @@ def copy_item(self, source_path, destination_path): def mv(self, run_args): target_name = run_args['target'] if target_name != "script": - return {"return": 1, "error": f"The {target_name} target is not currently supported for mv/cp actions"} + return { + "return": 1, "error": f"The {target_name} target is not currently supported for mv/cp actions"} res = self.cp(run_args) if res['return'] > 0: return res @@ -641,12 +704,13 @@ def mv(self, run_args): res = self.rm(ii) if res['return'] > 0: return res - - #Put the src uid to the destination path + + # Put the src uid to the destination path dest.meta['uid'] = src.meta['uid'] dest._save_meta() self.get_index().update(dest.meta, target_name, dest.path, dest.repo) - logger.info(f"""Item with uid {dest.meta['uid']} successfully moved from {src.path} to {dest.path}""") + logger.info( + f"""Item with uid {dest.meta['uid']} successfully moved from {src.path} to {dest.path}""") return {'return': 0, 'src': src, 'dest': dest} @@ -672,8 +736,13 @@ def search(self, i): details = i['details'] details_split = details.split(",") if len(details_split) > 1: - alias = details_split[0] - uid = details_split[1] + # Only treat as alias,uid if the second part is actually a + # valid UID + if utils.is_uid(details_split[1]): + alias = details_split[0] + uid = details_split[1] + # Otherwise, don't parse as alias,uid - let it be treated as + # tags else: if utils.is_uid(details_split[0]): uid = details_split[0] @@ -692,7 +761,7 @@ def search(self, i): { "action": "find", "target": "repo", - "repo": f"{item_repo}" + "repo": f"{item_repo}" } ) if res["return"] > 0: @@ -704,7 +773,8 @@ def search(self, i): if target_index: if uid or alias: for res in target_index: - if (res["uid"] == uid or (alias and res["alias"] == alias)) and (not item_repo or item_repo == res['repo']): + if (res["uid"] == uid or (alias and res["alias"] == alias)) and ( + not item_repo or item_repo == res['repo']): it = Item(res['path'], res['repo']) result.append(it) found = True @@ -718,38 +788,93 @@ def search(self, i): if tags: tags_split = tags.split(",") else: - return {"return":1, "error": f"Tags are not specified for completing the requested action"} + return { + "return": 1, "error": f"Tags are not specified for completing the requested action"} if target == "script": - non_variation_tags = [t for t in tags_split if not t.startswith("_")] + non_variation_tags = [ + t for t in tags_split if not t.startswith("_")] tags_to_match = non_variation_tags elif target in ["cache", "experiment"]: tags_to_match = tags_split else: - return {'return': 1, 'error': f"""Target {target} not handled in mlc yet"""} + return { + 'return': 1, 'error': f"""Target {target} not handled in mlc yet"""} n_tags_ = [p for p in tags_to_match if p.startswith("-")] n_tags = [p[1:] for p in n_tags_] p_tags = list(set(tags_to_match) - set(n_tags_)) for res in target_index: c_tags = res["tags"] - if (exact_tags_match and set(p_tags) == set(c_tags)) or (not exact_tags_match and set(p_tags).issubset(set(c_tags)) and set(n_tags).isdisjoint(set(c_tags))): + if (exact_tags_match and set(p_tags) == set(c_tags)) or (not exact_tags_match and set( + p_tags).issubset(set(c_tags)) and set(n_tags).isdisjoint(set(c_tags))): it = Item(res['path'], res['repo']) result.append(it) return {'return': 0, 'list': result} find = search + def reindex(self, i): + """ + Reindex the specified target or all targets if none specified. + + Args: + i (dict): Input dictionary with the following keys: + - reindex_target (str, optional): Target to reindex ('script', 'cache', 'repo', 'all', or None). + If not provided or 'all', reindexes all targets. + + Returns: + dict: Result of the operation with 'return' code 0 on success. + + Example: + mlc reindex # Reindex all targets + mlc reindex script # Reindex only script target + mlc reindex cache # Reindex only cache target + """ + reindex_target = i.get('reindex_target') + + if not reindex_target or reindex_target == 'all' or reindex_target == 'repos' or reindex_target == 'repo': + # Reindex all targets + logger.info( + "Reindexing all targets (script, cache, experiment)...") + index = self.get_index() + index.build_index(force_rebuild=True) + + logger.info("Successfully reindexed all targets.") + return {'return': 0, 'message': 'All targets reindexed successfully'} + else: + + logger.info(f"Reindexing {reindex_target} target...") + index = self.get_index() + + # Clear the specific index + ''' + if reindex_target in index.indices: + index.indices[reindex_target] = [] + # Clear modified times for this target type only + keys_to_remove = [k for k in index.modified_times.keys() if reindex_target in k] + for key in keys_to_remove: + del index.modified_times[key] + ''' + + # Rebuild the index (we are rebuilding for all targets here as the + # individual target rebuild is not implemented and not very + # critical) + index.build_index(force_rebuild=True) + + logger.info(f"Successfully reindexed {reindex_target} target.") + return { + 'return': 0, 'message': f'{reindex_target} target reindexed successfully'} + + default_parent = None if not default_parent: default_parent = Action() + def access(i): from .action_factory import get_action - + action = i['action'] target = i.get('target', i.get('automation')) action_class = get_action(target, default_parent) r = action_class.access(i) return r - - - diff --git a/mlc/index.py b/mlc/index.py index 4a18e3eea..13858fddc 100644 --- a/mlc/index.py +++ b/mlc/index.py @@ -4,6 +4,10 @@ import yaml from .repo import Repo from datetime import datetime +from .meta_schema import validate_meta +from contextlib import contextmanager +from filelock import FileLock, Timeout + class CustomJSONEncoder(json.JSONEncoder): def default(self, obj): @@ -21,13 +25,12 @@ class Index: def __init__(self, repos_path, repos): """ Initialize the Index class. - + Args: repos_path (str): Path to the base folder containing repositories. """ self.repos_path = repos_path self.repos = repos - #logger.info(repos) logger.debug(f"Repos path for Index: {self.repos_path}") self.index_files = { @@ -36,49 +39,113 @@ def __init__(self, repos_path, repos): "experiment": os.path.join(repos_path, "index_experiment.json") } self.indices = {key: [] for key in self.index_files.keys()} - self.modified_times_file = os.path.join(repos_path, "modified_times.json") + self.modified_times_file = os.path.join( + repos_path, "modified_times.json") self.modified_times = self._load_modified_times() self._load_existing_index() self.build_index() + def _get_stored_mtime(self, key): + """ + Helper method to safely extract mtime from stored data. + Handles both old format (direct mtime) and new format (dict with mtime key). + """ + old = self.modified_times.get(key) + if old is None: + return None + return old["mtime"] if isinstance(old, dict) else old + def _load_modified_times(self): """ Load stored mtimes to check for changes in scripts. """ - if os.path.exists(self.modified_times_file): - try: - # logger.info(f"Loading modified times from {self.modified_times_file}") - with open(self.modified_times_file, "r") as f: - return json.load(f) - except Exception: - return {} - return {} + lock_file = self.modified_times_file + ".lock" + try: + with self._file_lock_with_incremental_timeout(lock_file): + # logger.debug(f"Lock acquired at {lock_file} for Loading Modified Times") + + if os.path.exists(self.modified_times_file): + # logger.info(f"Loading modified times from {self.modified_times_file}") + with open(self.modified_times_file, "r") as f: + return json.load(f) + else: + return {} + + except Timeout: + logger.warning(f"Timeout acquiring lock {lock_file}") + return {} + + except Exception as e: + logger.error(f"Error acquiring lock {lock_file}: {e}") + return {} + + @contextmanager + def _file_lock_with_incremental_timeout( + self, lock_file, timeout_seconds=60): + """ + Acquire a file lock by waiting up to a minute, then retrying once if it times out. + """ + try: + with FileLock(lock_file, timeout=timeout_seconds): + yield # Control goes to the caller's 'with' block while the file lock is held + return + except Timeout: + logger.warning( + f"Timeout acquiring lock {lock_file} after {int(timeout_seconds)}s. " + f"Retrying once for another {int(timeout_seconds)}s..." + ) + + with FileLock(lock_file, timeout=timeout_seconds): + yield def _save_modified_times(self): """ Save updated mtimes in modified_times json file. """ - #logger.debug(f"Saving modified times to {self.modified_times_file}") - with open(self.modified_times_file, "w") as f: - json.dump(self.modified_times, f, indent=4) + lock_file = self.modified_times_file + ".lock" + try: + with self._file_lock_with_incremental_timeout(lock_file): + # logger.debug(f"Lock acquired at {lock_file} for Saving Modified Times") + + # logger.debug(f"Saving modified times to {self.modified_times_file}") + with open(self.modified_times_file, "w") as f: + json.dump(self.modified_times, f, indent=4) + + except Timeout: + logger.warning( + f"Timeout acquiring lock {lock_file}, skipping modified times save") + + except Exception as e: + logger.error(f"Error saving modified times: {e}") def _load_existing_index(self): """ Load previously saved index to allow incremental updates. """ for folder_type, file_path in self.index_files.items(): - if os.path.exists(file_path): - try: - # logger.info(f"Loading existing index for {folder_type}") - with open(file_path, "r") as f: - self.indices[folder_type] = json.load(f) - # Convert repo dicts back into Repo objects - for item in self.indices[folder_type]: - if isinstance(item.get("repo"), dict): - item["repo"] = Repo(**item["repo"]) - - except Exception: - pass # fall back to empty index + lock_file = file_path + ".lock" + try: + with self._file_lock_with_incremental_timeout(lock_file): + # logger.debug(f"Lock acquired at {lock_file} for Loading Index for {folder_type}") + + if os.path.exists(file_path): + # logger.info(f"Loading existing index for {folder_type}") + with open(file_path, "r") as f: + self.indices[folder_type] = json.load(f) + # Convert repo dicts back into Repo objects + for item in self.indices[folder_type]: + if isinstance(item.get("repo"), dict): + item["repo"] = Repo(**item["repo"]) + else: + self.indices[folder_type] = [] + + except Timeout: + logger.error(f"Timeout acquiring lock {lock_file}") + self.indices[folder_type] = [] + + except (json.JSONDecodeError, IOError, KeyError, TypeError) as e: + logger.warning(f"Failed to load index for {folder_type}: {e}") + self.indices[folder_type] = [] # fall back to empty index def add(self, meta, folder_type, path, repo): if not repo: @@ -88,17 +155,17 @@ def add(self, meta, folder_type, path, repo): unique_id = meta['uid'] alias = meta['alias'] tags = meta['tags'] - + index = self.get_index(folder_type, unique_id) if index == -1: self.indices[folder_type].append({ - "uid": unique_id, - "tags": tags, - "alias": alias, - "path": path, - "repo": repo - }) + "uid": unique_id, + "tags": tags, + "alias": alias, + "path": path, + "repo": repo + }) self._save_indices() def get_index(self, folder_type, uid): @@ -112,39 +179,109 @@ def update(self, meta, folder_type, path, repo): alias = meta['alias'] tags = meta['tags'] index = self.get_index(folder_type, uid) - if index == -1: #add it + if index == -1: # add it self.add(meta, folder_type, path, repo) logger.debug(f"Index update failed, new index created for {uid}") else: self.indices[folder_type][index] = { - "uid": uid, - "tags": tags, - "alias": alias, - "path": path, - "repo": repo - } + "uid": uid, + "tags": tags, + "alias": alias, + "path": path, + "repo": repo + } self._save_indices() def rm(self, meta, folder_type, path): uid = meta['uid'] index = self.get_index(folder_type, uid) - if index == -1: - logger.warning(f"Index is not having the {folder_type} item {path}") + if index == -1: + logger.warning( + f"Index is not having the {folder_type} item {path}") else: - del(self.indices[folder_type][index]) + del (self.indices[folder_type][index]) self._save_indices() - def get_item_mtime(self,file): + def get_item_mtime(self, file): latest = 0 t = os.path.getmtime(file) if t > latest: latest = t return latest - - def build_index(self): + + def _index_single_repo(self, repo, repos_changed=False, + current_item_keys=None): + repo_path = repo.path + if not os.path.isdir(repo_path): + return False + + changed = False + + for folder_type in ["script", "cache", "experiment"]: + folder_path = os.path.join(repo_path, folder_type) + if not os.path.isdir(folder_path): + continue + + for automation_dir in os.listdir(folder_path): + automation_path = os.path.join(folder_path, automation_dir) + if not os.path.isdir(automation_path): + continue + + yaml_path = os.path.join(automation_path, "meta.yaml") + json_path = os.path.join(automation_path, "meta.json") + + if os.path.isfile(yaml_path): + config_path = yaml_path + elif os.path.isfile(json_path): + config_path = json_path + else: + # No config file found, remove from index if exists + delete_flag = False + + # Check and remove both possible config paths from + # modified_times + for config_name in ["meta.yaml", "meta.json"]: + config_key = os.path.join(automation_path, config_name) + if config_key in self.modified_times: + del self.modified_times[config_key] + delete_flag = True + + # Use exact path matching instead of substring + if any( + item["path"] == automation_path for item in self.indices[folder_type]): + logger.debug( + f"Removed index entry (if it exists) for {folder_type} : {automation_dir}") + delete_flag = True + self._remove_index_entry(automation_path) + + if delete_flag: + changed = True + continue + if current_item_keys is not None: + current_item_keys.add(config_path) + mtime = self.get_item_mtime(config_path) + old_mtime = self._get_stored_mtime(config_path) + + # skip if unchanged + if old_mtime == mtime and not repos_changed: + continue + + self.modified_times[config_path] = { + "mtime": mtime, + "date_time": datetime.fromtimestamp(mtime).strftime("%Y-%m-%d %H:%M:%S") + } + + # meta file changed, so reindex + self._process_config_file( + config_path, folder_type, automation_path, repo) + changed = True + + return changed + + def build_index(self, force_rebuild=False): """ Build shared indices for script, cache, and experiment folders across all repositories. - + Returns: None """ @@ -152,164 +289,79 @@ def build_index(self): # track all currently detected item paths current_item_keys = set() changed = False - repos_changed = False - - # load existing modified times - self.modified_times = self._load_modified_times() - index_json_path = os.path.join(self.repos_path, "index_script.json") + # load modified times + self.modified_times = self._load_modified_times() - rebuild_index = False + # if any index file is missing, force full rebuild + missing_indices = [] + for index_type, index_path in self.index_files.items(): + if not os.path.exists(index_path): + missing_indices.append(index_type) - #file does not exist, rebuild - if not os.path.exists(index_json_path): - logger.warning("index_script.json missing. Forcing full index rebuild...") - #logger.debug("Resetting modified_times...") + if missing_indices: + logger.warning( + f"Missing index files: {', '.join(missing_indices)}. Forcing full index rebuild...") self.modified_times = {} - self._save_modified_times() - #else: - # logger.debug("index_script.json exists. Skipping forced rebuild.") - - #check repos.json mtime - repos_json_path = os.path.join(self.repos_path, "repos.json") - repos_mtime = os.path.getmtime(repos_json_path) - - key = f"{repos_json_path}" - old = self.modified_times.get(key) - repo_old_mtime = old["mtime"] if isinstance(old, dict) else old - - #logger.debug(f"Current repos.json mtime: {repos_mtime}") - #logger.debug(f"Old repos.json mtime: {repo_old_mtime}") - current_item_keys.add(key) - - # if changed, reset indexes - if repo_old_mtime is None or repo_old_mtime != repos_mtime: - logger.debug("repos.json modified. Clearing index ........") - # reset indices - self.indices = {key: [] for key in self.index_files.keys()} - # record repo mtime - self.modified_times[key] = { - "mtime": repos_mtime, - "date_time": datetime.fromtimestamp(repos_mtime).strftime("%Y-%m-%d %H:%M:%S") - } - # clear modified times except for repos.json - self.modified_times = {key: self.modified_times[key]} - self._save_indices() - self._save_modified_times() - repos_changed = True - #else: - # logger.debug("Repos.json not modified") + self.indices = {k: [] for k in self.index_files.keys()} + force_rebuild = True + # index each repo for repo in self.repos: - repo_path = repo.path #os.path.join(self.repos_path, repo) - if not os.path.isdir(repo_path): - continue - #logger.debug(f"------------Checking repository: {repo_path}---------------") - # Filter for relevant directories in the repo - for folder_type in ["script", "cache", "experiment"]: - #logger.debug(f"Checking folder type: {folder_type}") - folder_path = os.path.join(repo_path, folder_type) - if not os.path.isdir(folder_path): - continue - - # Process each automation directory - for automation_dir in os.listdir(folder_path): - # logger.debug(f"Checking automation directory: {automation_dir}") - automation_path = os.path.join(folder_path, automation_dir) - if not os.path.isdir(automation_path): - #logger.debug(f"Skipping non-directory automation path: {automation_path}") - continue - - yaml_path = os.path.join(automation_path, "meta.yaml") - json_path = os.path.join(automation_path, "meta.json") - - if os.path.isfile(yaml_path): - # logger.debug(f"Found YAML config file: {yaml_path}") - config_path = yaml_path - elif os.path.isfile(json_path): - # logger.debug(f"Found JSON config file: {json_path}") - config_path = json_path - else: - #logger.debug(f"No config file found in {automation_path}, skipping") - delete_flag = False - if automation_dir in self.modified_times: - del self.modified_times[automation_dir] - if any(automation_dir in item["path"] for item in self.indices[folder_type]): - logger.debug(f"Removed index entry (if it exists) for {folder_type} : {automation_dir}") - delete_flag = True - self._remove_index_entry(automation_path) - if delete_flag: - self._save_indices() - continue - current_item_keys.add(config_path) - mtime = self.get_item_mtime(config_path) - - old = self.modified_times.get(config_path) - old_mtime = old["mtime"] if isinstance(old, dict) else old - - # skip if unchanged - if old_mtime == mtime and repos_changed != 1: - # logger.debug(f"No changes detected for {config_path}, skipping reindexing.") - continue - #if(old_mtime is None): - # logger.debug(f"New meta.yaml file detected: {config_path}. Adding to index.") - - # update mtime - #logger.debug(f"{config_path} is modified, index getting updated") - #if config_path not in self.modified_times: - # logger.debug(f"*************{config_path} not found in modified_times; creating new entry***************") - - self.modified_times[config_path] = { - "mtime": mtime, - "date_time": datetime.fromtimestamp(mtime).strftime("%Y-%m-%d %H:%M:%S") - } - #logger.debug(f"Modified time for {config_path} updated to {mtime}") - changed = True - # meta file changed, so reindex - self._process_config_file(config_path, folder_type, automation_path, repo) + repo_changed = self._index_single_repo( + repo, force_rebuild, current_item_keys) + if repo_changed: + changed = True # remove deleted scripts - old_keys = set(self.modified_times.keys()) - deleted_keys = old_keys - current_item_keys + deleted_keys = set(self.modified_times) - current_item_keys for key in deleted_keys: - logger.warning(f"Detected deleted item, removing entry from modified times: {key}") - del self.modified_times[key] folder_key = os.path.dirname(key) - #logger.warning(f"Removing index entry for folder: {folder_key}") + logger.warning(f"Detected deleted item: {key}") + logger.debug(f"Removing index entry for folder: {folder_key}") + del self.modified_times[key] self._remove_index_entry(folder_key) changed = True if deleted_keys: - logger.debug(f"Deleted keys removed from modified times and indices: {deleted_keys}") + logger.debug( + f"Deleted keys removed from modified times and indices: {deleted_keys}") - if changed: - logger.debug("Changes detected, saving updated index and modified times.") + if force_rebuild or changed: + logger.debug( + "Changes detected, saving updated index and modified times.") self._save_modified_times() self._save_indices() - #logger.debug("**************Index updated (changes detected).*************************") - #else: - #logger.debug("**************Index unchanged (no changes detected).********************") def _remove_index_entry(self, key): - logger.debug(f"Removing index entry for {key}") + logger.debug(f"Removing index entry for path: {key}") + # Normalize paths for comparison + normalized_key = os.path.normpath(key) for ft in self.indices: + original_count = len(self.indices[ft]) self.indices[ft] = [ item for item in self.indices[ft] - if key not in item["path"] + if os.path.normpath(item["path"]) != normalized_key ] + removed_count = original_count - len(self.indices[ft]) + if removed_count > 0: + logger.debug( + f"Removed {removed_count} item(s) from {ft} index") - def _delete_by_uid(self, folder_type, uid, alias): + def _delete_index_entries(self, folder_type, key, value): """ - Delete old index entry using UID (prevents duplicates). + Remove index entries matching for the same path or same UID. """ - #logger.debug(f"Deleting and updating index entry for the script {alias} with UID {uid}") + # logger.debug(f"Deleting index entries in {folder_type} where {key} == {value}") self.indices[folder_type] = [ item for item in self.indices[folder_type] - if item["uid"] != uid + if item.get(key) != value ] - def _process_config_file(self, config_file, folder_type, folder_path, repo): + def _process_config_file( + self, config_file, folder_type, folder_path, repo): """ - Process a single configuration file (meta.json or meta.yaml) and add its data to the corresponding index. + Process a single configuration file (meta.json or meta.yaml) and + add its data to the corresponding index when the configuration file appears to be changed. Args: config_file (str): Path to the configuration file. @@ -332,11 +384,13 @@ def _process_config_file(self, config_file, folder_type, folder_path, repo): with open(config_file, "r") as f: data = json.load(f) or {} else: - logger.warning(f"Skipping {config_file}: Unsupported file format.") + logger.warning( + f"Skipping {config_file}: Unsupported file format.") return - + if not isinstance(data, dict): - logger.warning(f"Skipping {config_file}: Invalid or empty meta") + logger.warning( + f"Skipping {config_file}: Invalid or empty meta") return # Extract necessary fields unique_id = data.get("uid") @@ -346,36 +400,101 @@ def _process_config_file(self, config_file, folder_type, folder_path, repo): tags = data.get("tags", []) alias = data.get("alias", None) - # Validate and add to indices - if unique_id: - self._delete_by_uid(folder_type, unique_id, alias) - self.indices[folder_type].append({ - "uid": unique_id, - "tags": tags, - "alias": alias, - "path": folder_path, - "repo": repo - }) - else: - logger.warning(f"Skipping {config_file}: Missing 'uid' field.") + # Validate script meta against schema during indexing + if folder_type == "script": + errors, warnings = validate_meta(data, config_file) + for e in errors: + logger.error(f"Meta validation error: {e}") + for w in warnings: + logger.debug(f"Meta validation warning: {w}") + if errors: + raise ValueError( + f"Meta validation failed for {config_file}. Fix the above error(s) and try again.") + + # Remove stale entry for the same meta file path if exists + self._delete_index_entries(folder_type, "path", folder_path) + + # Remove index entry with the same UID for other meta file if + # exists + self._delete_index_entries(folder_type, "uid", unique_id) + + self.indices[folder_type].append({ + "uid": unique_id, + "tags": tags, + "alias": alias, + "path": folder_path, + "repo": repo + }) except Exception as e: logger.error(f"Error processing {config_file}: {e}") - def _save_indices(self): """ Save the indices to JSON files. - + Returns: None """ - #logger.info(self.indices) + # logger.info(self.indices) for folder_type, index_data in self.indices.items(): output_file = self.index_files[folder_type] + lock_file = output_file + ".lock" try: - with open(output_file, "w") as f: - json.dump(index_data, f, indent=4, cls=CustomJSONEncoder) - #logger.debug(f"Shared index for {folder_type} saved to {output_file}.") + with self._file_lock_with_incremental_timeout(lock_file): + # logger.debug(f"Lock acquired at {lock_file} for Saving Index for {folder_type}") + + with open(output_file, "w") as f: + json.dump( + index_data, f, indent=4, cls=CustomJSONEncoder) + # logger.debug(f"Shared index for {folder_type} saved to {output_file}.") + + except Timeout: + logger.error(f"Timeout acquiring lock {lock_file}") + except Exception as e: - logger.error(f"Error saving shared index for {folder_type}: {e}") + logger.error( + f"Error saving shared index for {folder_type}: {e}") + + def add_repo(self, repo): + """ + Incrementally index a newly registered repository. + """ + changed = self._index_single_repo(repo, repos_changed=True) + + if changed: + self._save_indices() + self._save_modified_times() + + def remove_repo_from_index(self, repo_path): + """ + Remove all index entries and modified times belonging to a repo. + Called when a repo is unregistered from repos.json. + """ + + logger.info(f"Removing repo from index: {repo_path}") + changed = False + + # remove index entries + for folder_type in self.indices: + before = len(self.indices[folder_type]) + self.indices[folder_type] = [ + item for item in self.indices[folder_type] + if not item["path"].startswith(repo_path) + ] + if len(self.indices[folder_type]) != before: + changed = True + + # remove modified times + keys_to_delete = [ + k for k in self.modified_times + if k.startswith(repo_path) + ] + + for k in keys_to_delete: + del self.modified_times[k] + changed = True + + if changed: + self._save_indices() + self._save_modified_times() diff --git a/mlc/logger.py b/mlc/logger.py index b1f4d6178..e30dcd1e2 100644 --- a/mlc/logger.py +++ b/mlc/logger.py @@ -5,6 +5,7 @@ # Initialize colorama for Windows support colorama_init(autoreset=True) + class ColoredFormatter(logging.Formatter): """Custom formatter class to add colors to log levels""" COLORS = { @@ -15,30 +16,46 @@ class ColoredFormatter(logging.Formatter): } def format(self, record): - # Add color to the levelname + # Pad filename and line number for alignment + # Left-align filename with 15 char width + record.filename = f"{record.filename:<15}" + # Right-align line number with 4 char width + record.lineno = f"{record.lineno:>4}" + + # Trim WARNING to WARN + levelname = "WARN" if record.levelname == "WARNING" else record.levelname + + # Pad and add color to the levelname + # Left-align levelname with 5 char width + levelname_padded = f"{levelname:<5}" if record.levelname in self.COLORS: - record.levelname = f"{self.COLORS[record.levelname]}{record.levelname}{Style.RESET_ALL}" + record.levelname = f"{self.COLORS[record.levelname]}{levelname_padded}{Style.RESET_ALL}" + else: + record.levelname = levelname_padded return super().format(record) # Set up logging configuration -def setup_logging(log_path = os.getcwd(), log_file = '.mlc-log.txt'): - +def setup_logging(log_path=os.getcwd(), log_file='.mlc-log.txt'): + if not logger.hasHandlers(): - logFormatter = ColoredFormatter('[%(asctime)s %(filename)s:%(lineno)d %(levelname)s] - %(message)s') + logFormatter = ColoredFormatter( + '[%(asctime)s %(filename)s:%(lineno)s %(levelname)s] - %(message)s') # by default logging level is set to INFO is being set logger.setLevel(logging.INFO) - # File hander for logging in file in the specified path - file_handler = logging.FileHandler("{0}/{1}".format(log_path, log_file)) - file_handler.setFormatter(logging.Formatter('[%(asctime)s %(filename)s:%(lineno)d %(levelname)s] - %(message)s')) + file_handler = logging.FileHandler( + "{0}/{1}".format(log_path, log_file)) + file_handler.setFormatter(logging.Formatter( + '[%(asctime)s %(filename)s:%(lineno)d %(levelname)s] - %(message)s')) logger.addHandler(file_handler) - + # Console handler for logging on console consoleHandler = logging.StreamHandler() consoleHandler.setFormatter(logFormatter) logger.addHandler(consoleHandler) logger.propagate = False + logger = logging.getLogger(__name__) diff --git a/mlc/main.py b/mlc/main.py index 7cb040ac9..63fc6348f 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -26,7 +26,7 @@ class Automation: path = None def __init__(self, action, automation_type, automation_file): - #logger.info(f"action = {action}") + # logger.info(f"action = {action}") self.action_object = action self.automation_type = automation_type self.path = os.path.dirname(automation_file) @@ -68,16 +68,140 @@ def search(self, i): if tags or uid or i.get('all'): for res in target_index: c_tags = res["tags"] - if set(p_tags).issubset(set(c_tags)) and set(n_tags).isdisjoint(set(c_tags)) and (not uid or uid == res['uid']) and (not alias or alias == res['alias']): + if set(p_tags).issubset(set(c_tags)) and set(n_tags).isdisjoint(set(c_tags)) and ( + not uid or uid == res['uid']) and (not alias or alias == res['alias']): it = Item(res['path'], res['repo']) result.append(it) - #logger.info(result) + # logger.info(result) return {'return': 0, 'list': result} - #indices + # indices -mlc_run_cmd = None -def mlc_expand_short(action, target = "script"): +mlc_run_cmd = None +_current_target = None + + +def get_version_info(): + """Return mlcflow version string with commit hash.""" + try: + from . import __version__ + return f"mlcflow {__version__}" + except ImportError: + pass + try: + from importlib.metadata import version + return f"mlcflow {version('mlcflow')}" + except Exception: + return "mlcflow (unknown version)" + + +def _get_repo_hashes(): + """Get git info for all repos. Returns list of (alias, branch, hash, has_local_changes).""" + import subprocess + if default_parent is None: + return [] + results = [] + for repo in default_parent.repos: + alias = os.path.basename(repo.path) + git_dir = os.path.join(repo.path, '.git') + if not os.path.isdir(git_dir): + continue + try: + commit = subprocess.check_output( + ["git", "-C", repo.path, "rev-parse", "--short", "HEAD"], + stderr=subprocess.DEVNULL, text=True + ).strip() + branch = subprocess.check_output( + ["git", "-C", repo.path, "rev-parse", "--abbrev-ref", "HEAD"], + stderr=subprocess.DEVNULL, text=True + ).strip() + # Only tracked file changes (ignore untracked files) + dirty = subprocess.check_output( + ["git", "-C", repo.path, "status", "--porcelain", "-uno"], + stderr=subprocess.DEVNULL, text=True + ).strip() + results.append((alias, branch, commit, bool(dirty))) + except Exception: + pass + return results + + +def _report_error(e): + import traceback + from .script_action import ScriptExecutionError + + # Log the error with target context + etype = type(e).__name__ if not isinstance(e, ScriptExecutionError) else '' + prefix = f'{etype}: ' if etype else '' + if _current_target: + logger.error(f"Error during '{_current_target}' action: {prefix}{e}") + else: + logger.error(f"{prefix}{e}") + + # Show the last traceback frame from outside the mlcflow package + tb = traceback.extract_tb(e.__traceback__) + if tb: + _mlc_pkg_dir = os.path.dirname(os.path.abspath(__file__)) + # Prefer the last frame outside the mlcflow package + last = tb[-1] + for frame in reversed(tb): + if not os.path.abspath(frame.filename).startswith(_mlc_pkg_dir): + last = frame + break + logger.error(f" at {last.filename}:{last.lineno} in {last.name}") + + # For script execution errors, show actionable info + if isinstance(e, ScriptExecutionError): + script_name = e.script_name + repo_alias = e.repo_alias + run_args = e.run_args + + if script_name: + # Build rerun command with user-facing inputs only + rerun_parts = ["mlcr", script_name] + _skip_keys = { + 'mlc_run_cmd', 'tags', 'details', 'path_only', 'p', + 'action', 'target', 'rebuild', 'env', 'script_tags', + 'run_cmd', 'run_final_cmds', 'skip_run_cmd', 'run_cmd_prefix', + 'add_deps_recursive', 'add_deps', 'file_path', + 'quiet', 'real_run', 'fake_run_deps', 'keep_detached', + 'pass_user_group', 'use_host_group_id', 'use_host_user_id', + 'extra_run_args', 'port_maps', 'mounts', 'pre_run_cmds', + 'docker_run_deps', + } + for k, v in run_args.items(): + if k in _skip_keys: + continue + if isinstance(v, (dict, list)): + continue + if k.startswith('MLC_') or k.startswith('mlc_'): + continue + rerun_parts.append(f"--{k}={v}") + rerun_cmd = " ".join(rerun_parts) + logger.error(f"Failed script: {script_name}") + logger.error(f"To rerun just the failed part: {rerun_cmd}") + + if e.version_info_file: + logger.error(f"Dependency versions: {e.version_info_file}") + + # Derive issues URL from repo alias + issues_url = 'https://github.com/mlcommons/mlperf-automations/issues' + if repo_alias and '@' in repo_alias: + issues_url = 'https://github.com/' + \ + repo_alias.replace('@', '/') + '/issues' + logger.error( + f"Please file an issue at {issues_url} with the full console log.") + + # Show version and repo commit hashes for debugging + logger.error(f"{get_version_info()}") + repo_hashes = _get_repo_hashes() + if repo_hashes: + for alias, branch, commit, dirty in repo_hashes: + marker = " (local changes)" if dirty else "" + logger.error(f" {alias}: {branch} {commit}{marker}") + + +def mlc_expand_short(action, target="script"): global mlc_run_cmd mlc_run_cmd = shlex.join(sys.argv) # Insert the positional argument into sys.argv for the main function @@ -85,21 +209,50 @@ def mlc_expand_short(action, target = "script"): sys.argv.insert(2, target) # Call the main function - main() + try: + main() + except KeyboardInterrupt: + print("\nInterrupted.") + sys.exit(1) + except SystemExit: + raise + except Exception as e: + _report_error(e) + sys.exit(1) + def mlcr(): mlc_expand_short("run") + + def mlcd(): mlc_expand_short("docker") -def mlcdr(): - mlc_expand_short("docker") + + +def mlca(): + mlc_expand_short("apptainer") + + def mlcrr(): mlc_expand_short("remote-run") + + +def mlcre(): + mlc_expand_short("remote-experiment") + + +def mlcrd(): + mlc_expand_short("remote-docker") + + def mlce(): mlc_expand_short("experiment") + + def mlct(): mlc_expand_short("test") + def mlcp(): mlc_expand_short("pull", "repo") @@ -110,14 +263,35 @@ def process_console_output(res, target, action, run_args): logger.error("'list' entry not found in find result") return # Exit function if there's an error if len(res['list']) == 0: - logger.warning(f"""No {target} entry found for the specified input: {run_args}!""") + # Only show warning if not in path-only mode + if not run_args.get('path_only'): + logger.warning( + f"""No {target} entry found for the specified input: {run_args}!""") + logger.info( + "Tip: Run 'mlc pull repo' to fetch the latest upstream changes.") + repo_hashes = _get_repo_hashes() + for alias, branch, commit, dirty in repo_hashes: + if dirty: + logger.warning( + f"Repo '{alias}' ({branch}) has local changes - 'mlc pull repo' may fail. Commit or stash changes first.") else: for item in res['list']: - logger.info(f"""Item path: {item.path}""") + if run_args.get('path_only'): + # Print only the path without logger prefix for + # script-friendly output + print(item.path) + else: + logger.info(f"""Item path: {item.path}""") + if action == "reindex": + if "message" in res: + logger.info(res['message']) if "warnings" in res: - logger.warning(f"{len(res['warnings'])} warning(s) found during the execution of the mlc command.") + logger.warning( + f"{len(res['warnings'])} warning(s) found during the execution of the mlc command.") for warning in res["warnings"]: - logger.warning(f"Warning code: {warning['code']}, Discription: {warning['description']}") + logger.warning( + f"Warning code: {warning['code']}, Discription: {warning['description']}") + if default_parent is None: default_parent = Action() @@ -126,6 +300,7 @@ def process_console_output(res, target, action, run_args): log_flag_aliases = {'-v': '--verbose', '-s': '--silent'} log_levels = {'--verbose': logging.DEBUG, '--silent': logging.WARNING} + def convert_hyphen_to_underscore_in_args(): for i, arg in enumerate(sys.argv): if arg.startswith("--"): @@ -141,31 +316,76 @@ def convert_hyphen_to_underscore_in_args(): sys.argv[i] = f"--{new_name}" - def build_pre_parser(): pre_parser = argparse.ArgumentParser(add_help=False) - pre_parser.add_argument("action", nargs="?", help="Top-level action (run, build, help, etc.)") - pre_parser.add_argument("target", choices=['run', 'script', 'cache', 'repo', 'repos'], nargs="?", help="Target (repo, script, cache, ...)") + pre_parser.add_argument( + "action", + nargs="?", + help="Top-level action (run, build, help, etc.)") + pre_parser.add_argument( + "target", + choices=[ + 'run', + 'script', + 'cache', + 'repo', + 'repos', + 'experiment', + 'all'], + nargs="?", + help="Target (repo, script, cache, ...)") pre_parser.add_argument("-h", "--help", action="store_true") return pre_parser def build_parser(pre_args): - parser = argparse.ArgumentParser(prog="mlc", description="Manage repos, scripts, and caches.", add_help=False) - subparsers = parser.add_subparsers(dest="command", required=not pre_args.help) + parser = argparse.ArgumentParser( + prog="mlc", + description="Manage repos, scripts, and caches.", + add_help=False) + subparsers = parser.add_subparsers( + dest="command", required=not pre_args.help) # General commands - for action in ['run', 'pull', 'test', 'add', 'show', 'list', 'find', 'search', 'rm', 'cp', 'mv', 'help', 'prune']: + for action in ['run', 'pull', 'test', 'add', 'show', 'list', + 'find', 'search', 'rm', 'cp', 'mv', 'help', 'prune']: p = subparsers.add_parser(action, add_help=False) p.add_argument('target', choices=['repo', 'repos', 'script', 'cache']) - p.add_argument('details', nargs='?', help='Details or identifier (optional)') + p.add_argument( + 'details', + nargs='?', + help='Details or identifier (optional)') p.add_argument('extra', nargs=argparse.REMAINDER) + # Reindex command (target is optional) + reindex_parser = subparsers.add_parser('reindex', add_help=False) + reindex_parser.add_argument( + 'target', + nargs='?', + choices=[ + 'repo', + 'repos', + 'script', + 'cache', + 'experiment', + 'all'], + help='Target to reindex (optional, defaults to all)') + reindex_parser.add_argument( + 'details', + nargs='?', + help='Details or identifier (optional)') + reindex_parser.add_argument('extra', nargs=argparse.REMAINDER) + # Script-only - for action in ['docker', 'docker-run', 'experiment', 'remote-run', 'doc', 'lint']: + for action in ['docker', 'docker-run', 'apptainer', + 'experiment', 'remote-run', 'remote-experiment', + 'remote-docker', 'doc', 'lint']: p = subparsers.add_parser(action, add_help=False) p.add_argument('target', choices=['script', 'run']) - p.add_argument('details', nargs='?', help='Details or identifier (optional)') + p.add_argument( + 'details', + nargs='?', + help='Details or identifier (optional)') p.add_argument('extra', nargs=argparse.REMAINDER) # Load cfg @@ -177,7 +397,7 @@ def build_parser(pre_args): def configure_logging(args): if hasattr(args, 'extra') and args.extra: args.extra[:] = [log_flag_aliases.get(a, a) for a in args.extra] - + for flag, level in log_levels.items(): if flag in args.extra: logger.setLevel(level) @@ -192,17 +412,22 @@ def build_run_args(args): run_args = res['args_dict'] if not mlc_run_cmd: - mlc_run_cmd = shlex.join([os.path.basename(sys.argv[0]), *sys.argv[1:]]) + mlc_run_cmd = shlex.join( + [os.path.basename(sys.argv[0]), *sys.argv[1:]]) run_args['mlc_run_cmd'] = mlc_run_cmd if args.command in ['pull', 'rm', 'add', 'find'] and args.target == "repo": run_args['repo'] = args.details - if args.command in ['docker', 'docker-run', 'experiment', 'remote-run', 'doc', 'lint'] and args.target == "run": - #run_args['target'] = 'script' #dont modify this as script might have target as in input + if args.command in ['docker', 'docker-run', 'apptainer', 'experiment', + 'remote-run', 'remote-experiment', + 'remote-docker', 'doc', 'lint'] and args.target == "run": + # run_args['target'] = 'script' #dont modify this as script might have + # target as in input args.target = "script" - if args.details and not utils.is_uid(args.details) and not run_args.get("tags") and args.target in ["script", "cache"]: + if args.details and not utils.is_uid(args.details) and not run_args.get( + "tags") and args.target in ["script", "cache"]: run_args['tags'] = args.details if not run_args.get('details') and args.details: @@ -215,8 +440,17 @@ def build_run_args(args): if args.extra: run_args['dest'] = args.extra[0] + if hasattr(args, 'command') and args.command == "reindex": + if hasattr(args, 'target') and args.target: + run_args['reindex_target'] = args.target + + # Check for path-only flag (for script-friendly output) + if run_args.get('path_only') or run_args.get('p'): + run_args['path_only'] = True + return run_args + def is_quoted(arg): return (arg.startswith("'") and arg.endswith("'")) or \ (arg.startswith('"') and arg.endswith('"')) @@ -243,6 +477,7 @@ def check_raw_arguments_for_non_ascii(): "Please retype the arguments using plain ASCII.\n") sys.exit(1) + def main(): """ MLCFlow is a CLI tool for managing repos, scripts, and caches. @@ -250,47 +485,56 @@ def main(): You can also use this tool for any of your workflow automation tasks. MLCFlow CLI operates using actions and targets. It enables users to perform actions on specified targets using the following syntax: - + mlc [options] Here, actions represent the operations to be performed, and the target is the object on which the action is executed. Each target has a specific set of actions to tailor automation workflows, as shown below: - + | Target | Actions | |---------|-----------------------------------------------------------| | script | run, find/search, rm, mv, cp, add, test, docker-run, show | | cache | find/search, rm, show | | repo | pull, search, rm, list, find/search | - + Example: mlc run script detect-os - - For help related to a particular target, run: - + + For help related to a particular target, run: + mlc --help/-h Examples: mlc script --help mlc repo -h - - For help related to a specific action for a target, run: - + + For help related to a specific action for a target, run: + mlc --help/-h Examples: mlc run script --help mlc pull repo -h """ - + check_raw_arguments_for_non_ascii() convert_hyphen_to_underscore_in_args() + # Handle version before argparse to avoid --version conflicting with + # script arguments like --version=3.4 + if len(sys.argv) >= 2 and sys.argv[1] in ('--version', '-V', 'version'): + print(get_version_info()) + sys.exit(0) + pre_parser = build_pre_parser() pre_args, remaining_args = pre_parser.parse_known_args() parser = build_parser(pre_args) - args = parser.parse_args() if remaining_args or pre_args.target else pre_args - + # Force full parsing for reindex command even without target, or if there + # are remaining args or target + args = parser.parse_args() if ( + remaining_args or pre_args.target or pre_args.action == 'reindex') else pre_args + if hasattr(args, 'command') and args.command: args.command = args.command.replace("-", "_") @@ -300,11 +544,14 @@ def main(): if pre_args.help and not "tags" in run_args: help_text = "" if pre_args.target == "run": - if pre_args.action.startswith("docker"): + if pre_args.action.startswith( + "docker") or pre_args.action == "apptainer": pre_args.target = "script" else: - logger.error(f"Invalid action-target {pre_args.action} - {pre_args.target} combination") - raise Exception(f"Invalid action-target {pre_args.action} - {pre_args.target} combination") + logger.error( + f"Invalid action-target {pre_args.action} - {pre_args.target} combination") + raise Exception( + f"Invalid action-target {pre_args.action} - {pre_args.target} combination") if not pre_args.action and not pre_args.target: help_text += main.__doc__ elif pre_args.action and not pre_args.target: @@ -316,7 +563,8 @@ def main(): actions = get_action(pre_args.target, default_parent) help_text += actions.__doc__ # iterate through every method - for method_name, method in inspect.getmembers(actions.__class__, inspect.isfunction): + for method_name, method in inspect.getmembers( + actions.__class__, inspect.isfunction): method = getattr(actions, method_name) if method.__doc__ and not method.__doc__.startswith("_"): help_text += method.__doc__ @@ -326,29 +574,58 @@ def main(): method = getattr(actions, pre_args.action) help_text += actions.__doc__ help_text += method.__doc__ - except: - logger.error(f"Error: '{pre_args.action}' is not supported for {pre_args.target}.") + except BaseException: + logger.error( + f"Error: '{pre_args.action}' is not supported for {pre_args.target}.") if help_text != "": print(help_text) sys.exit(0) - if args.target == "repos": + if hasattr(args, 'target') and args.target == "repos": args.target = "repo" - + + # Handle reindex command specially - it can work without a target or with + # 'all' + if hasattr(args, 'command') and args.command == "reindex": + if not hasattr( + args, 'target') or not args.target or args.target == "all": + # Reindex all targets by using the base Action class + args.target = "script" # Use script as default to get access to the action + + # Check if command attribute exists + if not hasattr(args, 'command'): + logging.error("Error: No command specified.") + sys.exit(1) + + global _current_target + _current_target = args.target + action = get_action(args.target, default_parent) if not action or not hasattr(action, args.command): - logging.error("Error: '%s' is not supported for %s.", args.command, args.target) + logging.error( + "Error: '%s' is not supported for %s.", + args.command, + args.target) sys.exit(1) method = getattr(action, args.command) res = method(run_args) if res['return'] > 0: logging.error(res.get('error', f"Error in {action}")) - raise Exception(f"An error occurred {res}") + sys.exit(1) process_console_output(res, args.target, args.command, run_args) -if __name__ == '__main__': - main() +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print("\nInterrupted.") + sys.exit(1) + except SystemExit: + raise + except Exception as e: + _report_error(e) + sys.exit(1) diff --git a/mlc/meta_schema.py b/mlc/meta_schema.py new file mode 100644 index 000000000..ed7760110 --- /dev/null +++ b/mlc/meta_schema.py @@ -0,0 +1,458 @@ +""" +Schema specification for script meta.yaml files. + +Defines all valid keys, their types, and nesting rules for script meta.yaml. +Used by validate_meta() to check meta files during indexing and linting. +""" + +# ─── Type aliases ─────────────────────────────────────────────── +# Each value is a set of allowed Python type names (from type().__name__). +# "optional" means the key may be absent; all keys are optional unless in REQUIRED_KEYS. + +STR = {"str"} +BOOL = {"bool"} +INT = {"int"} +LIST = {"list"} +DICT = {"dict"} +STR_OR_BOOL = {"str", "bool"} +INT_OR_FLOAT = {"int", "float"} +STR_OR_FLOAT = {"str", "float"} +STR_OR_LIST = {"str", "list"} + +# ─── Required top-level keys ─────────────────────────────────── +REQUIRED_KEYS = {"alias", "uid", "automation_alias", "automation_uid"} + +# ─── Top-level key specification ─────────────────────────────── +# key -> set of allowed type names +TOP_LEVEL_SCHEMA = { + # Identity (required) + "alias": STR, + "uid": STR, + "automation_alias": STR, + "automation_uid": STR, + + # Metadata + "name": STR, + "category": STR, + "tags": LIST, # list[str] + "tags_help": STR, + "developers": STR, + "sort": INT, + "category_sort": INT, + "private": BOOL, + "min_mlc_version": STR, + + # Environment + "env": DICT, # dict[str, str] + "default_env": DICT, # dict[str, str] + "new_env_keys": LIST, # list[str] + "new_state_keys": LIST, # list[str] + "local_env_keys": LIST, # list[str] + "file_path_env_keys": LIST, # list[str] + "folder_path_env_keys": LIST, # list[str] + + # Cache + "cache": STR_OR_BOOL, + "can_force_cache": BOOL, + "cache_expiration": STR, + "extra_cache_tags_from_env": LIST, # list[str] + "clean_files": LIST, # list[str] + "clean_output_files": LIST, # list[str] + + # Input mapping + # dict[str, str] input_name -> ENV_KEY (or null) + "input_mapping": {*DICT, "NoneType"}, + # dict[str, dict] input_name -> {desc, choices, ...} (or null) + "input_description": {*DICT, "NoneType"}, + "env_key_mappings": DICT, # dict[str, str] + + # Dependencies + "deps": LIST, # list[dep_entry] + "prehook_deps": LIST, # list[dep_entry] + "posthook_deps": LIST, # list[dep_entry] + "post_deps": LIST, # list[dep_entry] + "predeps": BOOL, + + # Variations + "variations": DICT, # dict[str, variation_entry] + "variation_groups_order": LIST, # list[str] + "default_variation": STR, + "default_variations": DICT, # dict[str, str] + "invalid_variation_combinations": LIST, # list[list[str]] + "valid_variation_combinations": LIST, # list[list[str]] + + # Versions + "versions": DICT, # dict[str, version_entry] + "default_version": STR, + + # Docker + "docker": DICT, # dict - see DOCKER_SCHEMA + + # Output / debugging + "print_env_at_the_end": DICT, # dict[str, list[str]] + "print_files_if_script_error": LIST, # list[str] + "warnings": LIST, # list[str] + "sudo_install": BOOL, + + # Conditional meta update + "update_meta_if_env": LIST, # list[dict] + "remote_run": DICT, + + # Tests + "tests": DICT, # dict - see TESTS_SCHEMA +} + +# ─── Dependency entry keys ────────────────────────────────────── +DEP_ENTRY_SCHEMA = { + "tags": STR, + "names": STR_OR_LIST, + "env": DICT, + "enable_if_env": DICT, + "skip_if_env": DICT, + "skip_if_any_env": DICT, + "enable_if_any_env": DICT, + "extra_cache_tags": STR, + "update_tags_from_env_with_prefix": DICT, + "update_tags_from_env": LIST, + "force_env_keys": LIST, + "force_cache": BOOL, + "reuse_version": BOOL, + "inherit_variation_tags": STR_OR_BOOL, + "skip_inherit_variation_groups": LIST, + "version": STR, + "version_min": STR, + "version_max": STR_OR_FLOAT, + "version_max_usable": STR_OR_FLOAT, + "dynamic": BOOL, + "ignore_missing": BOOL, + "skip_if_fake_run": BOOL, + "verify": BOOL, + "md5sum": STR, + "revision": STR, + "model_filename": STR, + "full_subfolder": STR, + "env_key": STR, + "continue_on_error": BOOL, + "ignore_script_error": BOOL, + "inherit_cache_expiration": BOOL, + "update_tags_if_env": DICT, + "update_meta_if_env": LIST, +} + +# ─── Variation entry keys ─────────────────────────────────────── +VARIATION_ENTRY_SCHEMA = { + "env": {*DICT, "NoneType"}, + "group": STR, + "default": STR_OR_BOOL, + "default_variations": DICT, + "deps": LIST, + "prehook_deps": LIST, + "posthook_deps": LIST, + "post_deps": LIST, + "add_deps": DICT, + "add_deps_recursive": DICT, + "add_deps_tags": DICT, + "new_env_keys": LIST, + "new_state_keys": LIST, + "base": LIST, + "adr": DICT, + "ad": DICT, + "default_env": DICT, + "state": DICT, + "const": DICT, + "docker": DICT, + "alias": STR, + "default_version": STR_OR_FLOAT, + "required_disk_space": INT, + "cache_expiration": {*STR, *INT}, + "cache": BOOL, + "force_cache": BOOL, + "update_meta_if_env": LIST, + "warning": STR, + "warnings": LIST, + "names": LIST, + "default_variation": DICT, +} + +# ─── Docker section keys ──────────────────────────────────────── +DOCKER_SCHEMA = { + "real_run": BOOL, + "run": BOOL, + "skip_run_cmd": STR_OR_BOOL, + "interactive": BOOL, + "pre_run_cmds": LIST, + "deps": LIST, + "mounts": LIST, + "input_mapping": DICT, + "input_paths": LIST, + "skip_input_for_fake_run": LIST, + "os": STR, + "os_version": STR, + "base_image": STR, + "mlc_repo": STR, + "mlc_repo_branch": STR, + "mlc_repo_flags": STR, + "extra_run_args": STR, + "all_gpus": STR, + "user": STR, + "use_host_user_id": BOOL, + "use_host_group_id": STR_OR_BOOL, + "skip_mlc_sys_upgrade": STR, + "shm_size": STR, + "port_maps": LIST, + "image_tag_extra": STR, + "fake_run_deps": BOOL, + "pass_docker_to_script": BOOL, + "mount_current_dir": STR, + "use_google_dns": BOOL, + "add_quotes_to_keys": LIST, + "device": STR, + "run_cmd_prefix": STR, + "pass_user_group": BOOL, + "default_env": DICT, + "env": DICT, +} + +# ─── Tests section keys ───────────────────────────────────────── +TESTS_SCHEMA = { + "run_inputs": LIST, # list[dict] - each has variations_list, env, etc. + "needs_pat": BOOL, +} + +# ─── Tests run_inputs entry keys ──────────────────────────────── +TESTS_RUN_INPUT_SCHEMA = { + "variations_list": LIST, # list[str] + "env": DICT, + "test_input_index": STR, + "disable_run_script": BOOL, +} + + +# ─── update_meta_if_env entry keys ────────────────────────────── +UPDATE_META_IF_ENV_SCHEMA = { + "enable_if_env": DICT, + "enable_if_any_env": DICT, + "skip_if_env": DICT, + "skip_if_any_env": DICT, + "env": DICT, + "default_env": DICT, + "default_variations": DICT, + "docker": DICT, + "adr": DICT, + "ad": DICT, +} + + +def validate_meta(data, file_path=""): + """ + Validate a script meta.yaml dict against the schema. + + Args: + data (dict): Parsed meta.yaml content. + file_path (str): Path to meta file (for error messages). + + Returns: + list[str]: List of warning/error messages. Empty if valid. + """ + errors = [] + warnings = [] + + if not isinstance(data, dict): + return ["Meta is not a dictionary"], [] + + prefix = f"{file_path}: " if file_path else "" + + # Check required keys + for key in REQUIRED_KEYS: + if key not in data: + errors.append(f"{prefix}Missing required key '{key}'") + + # Check top-level keys + for key, value in data.items(): + if key not in TOP_LEVEL_SCHEMA: + # Check if it looks like a misplaced env variable + if key.startswith("MLC_"): + warnings.append( + f"{prefix}Key '{key}' looks like an env variable - should it be under 'env' or 'default_env'?") + else: + warnings.append(f"{prefix}Unknown top-level key '{key}'") + continue + + actual_type = type(value).__name__ + allowed = TOP_LEVEL_SCHEMA[key] + if actual_type not in allowed: + errors.append( + f"{prefix}Key '{key}' has type '{actual_type}', expected {allowed}") + + # Validate dependency lists + for dep_list_key in ["deps", "prehook_deps", + "posthook_deps", "post_deps", "post_deps_off"]: + deps = data.get(dep_list_key) + if deps is None: + continue + if not isinstance(deps, list): + continue + for i, dep in enumerate(deps): + if not isinstance(dep, dict): + errors.append( + f"{prefix}{dep_list_key}[{i}] is not a dict") + continue + for dk, dv in dep.items(): + if dk not in DEP_ENTRY_SCHEMA: + warnings.append( + f"{prefix}{dep_list_key}[{i}]: unknown dep key '{dk}'") + continue + actual = type(dv).__name__ + allowed = DEP_ENTRY_SCHEMA[dk] + if actual not in allowed: + errors.append( + f"{prefix}{dep_list_key}[{i}].{dk} has type '{actual}', expected {allowed}") + + # Validate enable_if_env/skip_if_env values are single + # strings/lists, not nested dicts + for ck in ["enable_if_env", "skip_if_env", + "skip_if_any_env", "enable_if_any_env"]: + cv = dep.get(ck) + if isinstance(cv, dict): + for ek, ev in cv.items(): + if isinstance(ev, (dict, list) + ) and not isinstance(ev, str): + if isinstance(ev, list): + for item in ev: + if not isinstance( + item, (str, int, float, bool)): + errors.append( + f"{prefix}{dep_list_key}[{i}].{ck}.{ek} list contains non-scalar: {type(item).__name__}") + + # Validate update_meta_if_env entries + umie = data.get("update_meta_if_env") + if isinstance(umie, list): + for i, entry in enumerate(umie): + if not isinstance(entry, dict): + errors.append(f"{prefix}update_meta_if_env[{i}] is not a dict") + continue + for ek, ev in entry.items(): + if ek not in UPDATE_META_IF_ENV_SCHEMA: + warnings.append( + f"{prefix}update_meta_if_env[{i}]: unknown key '{ek}'") + continue + actual = type(ev).__name__ + allowed = UPDATE_META_IF_ENV_SCHEMA[ek] + if actual not in allowed: + errors.append( + f"{prefix}update_meta_if_env[{i}].{ek} has type '{actual}', expected {allowed}") + # Validate enable_if_env/skip_if_env values inside + # update_meta_if_env + for ck in ["enable_if_env", "skip_if_env", + "skip_if_any_env", "enable_if_any_env"]: + cv = entry.get(ck) + if isinstance(cv, dict): + for ek2, ev2 in cv.items(): + if isinstance(ev2, (dict, list) + ) and not isinstance(ev2, str): + if isinstance(ev2, list): + for item in ev2: + if not isinstance( + item, (str, int, float, bool)): + errors.append( + f"{prefix}update_meta_if_env[{i}].{ck}.{ek2} list contains non-scalar: {type(item).__name__}") + + # Validate variations + variations = data.get("variations") + if isinstance(variations, dict): + for vname, vattrs in variations.items(): + if vattrs is None: + continue # empty variation is ok + if not isinstance(vattrs, dict): + warnings.append( + f"{prefix}variations.{vname} is not a dict (type: {type(vattrs).__name__})") + continue + for vk, vv in vattrs.items(): + if vk not in VARIATION_ENTRY_SCHEMA: + if not vk.startswith("MLC_"): + warnings.append( + f"{prefix}variations.{vname}: unknown variation key '{vk}'") + continue + actual = type(vv).__name__ + allowed = VARIATION_ENTRY_SCHEMA[vk] + if actual not in allowed: + errors.append( + f"{prefix}variations.{vname}.{vk} has type '{actual}', expected {allowed}") + + # Validate docker section + docker = data.get("docker") + if isinstance(docker, dict): + for dk, dv in docker.items(): + if dk not in DOCKER_SCHEMA: + warnings.append( + f"{prefix}docker: unknown key '{dk}'") + continue + actual = type(dv).__name__ + allowed = DOCKER_SCHEMA[dk] + if actual not in allowed: + errors.append( + f"{prefix}docker.{dk} has type '{actual}', expected {allowed}") + + # Validate tests section + tests = data.get("tests") + if isinstance(tests, dict): + for tk, tv in tests.items(): + if tk not in TESTS_SCHEMA: + warnings.append( + f"{prefix}tests: unknown key '{tk}'") + continue + actual = type(tv).__name__ + allowed = TESTS_SCHEMA[tk] + if actual not in allowed: + errors.append( + f"{prefix}tests.{tk} has type '{actual}', expected {allowed}") + + run_inputs = tests.get("run_inputs") + if isinstance(run_inputs, list): + for i, entry in enumerate(run_inputs): + if entry is None: + continue + if not isinstance(entry, dict): + errors.append( + f"{prefix}tests.run_inputs[{i}] is not a dict") + + # Check for variation entry keys mistakenly used as variation names + # Exclude common short words that are legitimately used as both + _variation_name_allowlist = { + "default", + "base", + "env", + "ad", + "cache", + "state", + "alias", + "warning", + "const"} + if isinstance(variations, dict): + for vname in variations: + if vname in VARIATION_ENTRY_SCHEMA and vname not in _variation_name_allowlist: + warnings.append( + f"{prefix}variation '{vname}' looks like a variation property key used as a variation name") + # Also check if variation attrs contain keys that look like + # variation names + vattrs = variations[vname] + if isinstance(vattrs, dict): + for vk in vattrs: + if vk in VARIATION_ENTRY_SCHEMA: + continue # valid property + if vk.startswith("MLC_"): + continue # env override + # Check if this unknown key is actually a known variation + # name in this script + if vk in variations and vk != vname: + warnings.append( + f"{prefix}variations.{vname}: key '{vk}' matches another variation name - possible indentation error") + + # Cross-key validations + default_variation = data.get("default_variation") + if default_variation and isinstance(variations, dict): + if default_variation not in variations: + errors.append( + f"{prefix}default_variation '{default_variation}' not found in variations") + + return errors, warnings diff --git a/mlc/repo_action.py b/mlc/repo_action.py index 8d04cfd50..cf5dabc5d 100644 --- a/mlc/repo_action.py +++ b/mlc/repo_action.py @@ -8,13 +8,16 @@ from . import utils from .logger import logger from urllib.parse import urlparse +from .repo import Repo +from .index import Index + class RepoAction(Action): """ #################################################################################################################### Repo Action #################################################################################################################### - + Currently, the following actions are supported for Repos: 1. add 2. find @@ -37,11 +40,10 @@ class RepoAction(Action): """ def __init__(self, parent=None): - #super().__init__(parent) + # super().__init__(parent) self.parent = parent self.__dict__.update(vars(parent)) - def add(self, run_args): """ #################################################################################################################### @@ -49,8 +51,8 @@ def add(self, run_args): Action: Add #################################################################################################################### - The `add` action is used to create a new MLC repository and register it in MLCFlow. - The newly created repo folder will be stored inside the `repos` folder within the parent MLC directory. + The `add` action is used to create a new MLC repository and register it in MLCFlow. + The newly created repo folder will be stored inside the `repos` folder within the parent MLC directory. Example Command: @@ -70,24 +72,27 @@ def add(self, run_args): """ if not run_args['repo']: logger.error("The repository to be added is not specified") - return {"return": 1, "error": "The repository to be added is not specified"} + return {"return": 1, + "error": "The repository to be added is not specified"} - i_repo_path = run_args['repo'] #can be a path, forder_name or URL + i_repo_path = run_args['repo'] # can be a path, forder_name or URL repo_folder_name = os.path.basename(i_repo_path.rstrip('/')) repo_path = os.path.join(self.repos_path, repo_folder_name) r = self.find(run_args) - + if r['return'] == 0 and len(r['list']) > 0: - return {'return': 1, "error": f"""Repo already exists at {r['list'][0]}"""} + return {'return': 1, + "error": f"""Repo already exists at {r['list'][0]}"""} for repo in self.repos: if repo.path == i_repo_path: - return {'return': 1, "error": f"""Repo already exists at {repo.path}"""} + return {'return': 1, + "error": f"""Repo already exists at {repo.path}"""} if not os.path.exists(i_repo_path): - #check if its an URL + # check if its an URL if utils.is_valid_url(i_repo_path): parsed = urlparse(i_repo_path) if parsed.hostname == "github.com": @@ -101,7 +106,7 @@ def add(self, run_args): else: repo_path = os.path.abspath(i_repo_path) - #check if it has MLC meta + # check if it has MLC meta meta_file = os.path.join(repo_path, "meta.yaml") if not os.path.exists(meta_file): meta = {} @@ -111,7 +116,7 @@ def add(self, run_args): utils.save_yaml(meta_file, meta) else: meta = utils.read_yaml(meta_file) - + self.register_repo(repo_path, meta, run_args.get('ignore_on_conflict')) return {'return': 0} @@ -119,45 +124,58 @@ def add(self, run_args): def conflicting_repo(self, repo_meta): for repo_object in self.repos: if repo_object.meta.get('uid', '') == '': - return {"return": 1, "error": f"UID is not present in file 'meta.yaml' in the repo path {repo_object.path}"} + return { + "return": 1, "error": f"UID is not present in file 'meta.yaml' in the repo path {repo_object.path}"} if repo_meta["uid"] == repo_object.meta.get('uid', ''): if repo_meta.get('path', '') == repo_object.path: - return {"return": 1, "error": f"Same repo is already registered"} + return {"return": 1, + "error": f"Same repo is already registered"} else: - return {"return": 1, "error": f"Conflicting with repo in the path {repo_object.path}", "conflicting_path": repo_object.path} + return {"return": 1, "error": f"Conflicting with repo in the path {repo_object.path}", + "conflicting_path": repo_object.path} return {"return": 0} - + def register_repo(self, repo_path, repo_meta, ignore_on_conflict=False): - + # Check UID conflicts is_conflict = self.conflicting_repo(repo_meta) if is_conflict['return'] > 0: if "UID not present" in is_conflict['error']: - logger.warning(f"UID not found in meta.yaml at {repo_path}. Repo can not be registered in MLC repos. Skipping...") + logger.warning( + f"UID not found in meta.yaml at {repo_path}. Repo can not be registered in MLC repos. Skipping...") return {"return": 0} - elif "already registered" in is_conflict["error"]: #at same path - #logger.warning(is_conflict["error"]) + elif "already registered" in is_conflict["error"]: # at same path + # logger.warning(is_conflict["error"]) logger.debug("No changes made to repos.json.") return {"return": 0} else: - logger.warning(f"The repo to be registered has conflict with the repo already in the path: {is_conflict['conflicting_path']}") + logger.warning( + f"The repo to be registered has conflict with the repo already in the path: {is_conflict['conflicting_path']}") if ignore_on_conflict: - logger.warning(f"Ignoring register as ignore_on_conflict is set") + logger.warning( + f"Ignoring register as ignore_on_conflict is set") return {"return": 0, 'conflict': True} self.unregister_repo(is_conflict['conflicting_path']) - logger.warning(f"{is_conflict['conflicting_path']} is unregistered.") - + logger.warning( + f"{is_conflict['conflicting_path']} is unregistered.") + if repo_meta.get('deps'): for dep in repo_meta['deps']: - self.pull_repo(dep['url'], branch=dep.get('branch'), checkout=dep.get('checkout'), ignore_on_conflict=dep.get('is_alias_okay', True)) + self.pull_repo( + dep['url'], + branch=dep.get('branch'), + checkout=dep.get('checkout'), + ignore_on_conflict=dep.get( + 'is_alias_okay', + True)) # Get the path to the repos.json file in $HOME/MLC repos_file_path = os.path.join(self.repos_path, 'repos.json') with open(repos_file_path, 'r') as f: repos_list = json.load(f) - + if repo_path not in repos_list: repos_list.append(repo_path) logger.info(f"Added new repo path: {repo_path}") @@ -165,15 +183,24 @@ def register_repo(self, repo_path, repo_meta, ignore_on_conflict=False): with open(repos_file_path, 'w') as f: json.dump(repos_list, f, indent=2) logger.info(f"Updated repos.json at {repos_file_path}") - - + + self.repos = self.load_repos_and_meta() + repo_obj = next( + (r for r in self.repos if r.path == repo_path), + None + ) + + if repo_obj: + index = Action.get_index(self) + index.add_repo(repo_obj) + logger.debug("Index file has been updated") + return {'return': 0} def unregister_repo(self, repo_path): repos_file_path = os.path.join(self.repos_path, 'repos.json') - - return unregister_repo(repo_path, repos_file_path) + return unregister_repo(repo_path, repos_file_path) def find(self, run_args): """ @@ -196,25 +223,28 @@ def find(self, run_args): """ # Get repos_list using the existing method repos_list = self.load_repos_and_meta() - if(run_args.get('item', run_args.get('artifact'))): + if (run_args.get('item', run_args.get('artifact'))): repo = run_args.get('item', run_args.get('artifact')) else: - repo = run_args.get('repo', run_args.get('item', run_args.get('artifact'))) + repo = run_args.get( + 'repo', run_args.get( + 'item', run_args.get('artifact'))) # Check if repo is None or empty if not repo: return {"return": 1, "error": "Please enter a Repo Alias, Repo UID, or Repo URL in one of the following formats:\n" - "- @\n" - "- \n" - "- \n" - "- \n" - "- ,"} + "- @\n" + "- \n" + "- \n" + "- \n" + "- ,"} # Handle the different repo input formats repo_name = None repo_uid = None - # Check if the repo is in the format of a repo UID (alphanumeric string) + # Check if the repo is in the format of a repo UID (alphanumeric + # string) if utils.is_uid(repo): repo_uid = repo if "," in repo: @@ -230,7 +260,8 @@ def find(self, run_args): parsed = urlparse(repo) except Exception: parsed = None - if parsed and parsed.scheme in ("http", "https") and parsed.hostname == "github.com": + if parsed and parsed.scheme in ( + "http", "https") and parsed.hostname == "github.com": result = self.github_url_to_user_repo_format(repo) if result["return"] == 0: repo_name = result["value"] @@ -242,11 +273,10 @@ def find(self, run_args): # Check if repo_name exists in repos.json matched_repo_path = None for repo_obj in repos_list: - if repo_name and repo_name == os.path.basename(repo_obj.path) : + if repo_name and repo_name == os.path.basename(repo_obj.path): matched_repo_path = repo_obj break - # Search through self.repos for matching repos lst = [] for i in self.repos: @@ -255,22 +285,23 @@ def find(self, run_args): elif repo_name == i.meta['alias']: lst.append(i) - # After loop, check if any match was found if not lst and not matched_repo_path: # Determine error message based on input if utils.is_uid(repo): - return {"return": 1, "error": f"No repository with UID: '{repo_uid}' was found"} + return { + "return": 1, "error": f"No repository with UID: '{repo_uid}' was found"} elif "," in repo and not matched_repo_path: - return {"return": 1, "error": f"No repository with alias: '{repo_name}' and UID: '{repo_uid}' was found"} + return { + "return": 1, "error": f"No repository with alias: '{repo_name}' and UID: '{repo_uid}' was found"} else: - return {"return": 1, "error": f"No repository with alias: '{repo_name}' was found"} + return { + "return": 1, "error": f"No repository with alias: '{repo_name}' was found"} - # Append the matched repo path - if(len(lst)==0 and matched_repo_path): + if (len(lst) == 0 and matched_repo_path): lst.append(matched_repo_path) - + return {'return': 0, 'list': lst} def github_url_to_user_repo_format(self, url): @@ -282,19 +313,22 @@ def github_url_to_user_repo_format(self, url): # """ # Regex to match GitHub URLs pattern = r"(?:https?://)?(?:www\.)?github\.com/([^/]+)/([^/.]+)(?:\.git)?" - + match = re.match(pattern, url) if match: user, repo_name = match.groups() return {"return": 0, "value": f"{user}@{repo_name}"} else: - return {"return": 0, "value": os.path.basename(url).replace(".git", "")} + return {"return": 0, "value": os.path.basename( + url).replace(".git", "")} + + def pull_repo(self, repo_url, branch=None, checkout=None, tag=None, + pat=None, ssh=None, ignore_on_conflict=False, repo_path=None): - def pull_repo(self, repo_url, branch=None, checkout = None, tag = None, pat = None, ssh = None, ignore_on_conflict = False, repo_path = None): - # Determine the checkout path from environment or default - repo_base_path = self.repos_path # either the value will be from 'MLC_REPOS' - os.makedirs(repo_base_path, exist_ok=True) # Ensure the directory exists + repo_base_path = self.repos_path # either the value will be from 'MLC_REPOS' + # Ensure the directory exists + os.makedirs(repo_base_path, exist_ok=True) # Handle user@repo format (convert to standard GitHub URL) if re.match(r'^[\w-]+@[\w-]+$', repo_url): @@ -313,7 +347,6 @@ def pull_repo(self, repo_url, branch=None, checkout = None, tag = None, pat = No else: repo_url = res["url"] - # Extract the repo name from URL repo_name = repo_url.split('/')[-1].replace('.git', '') res = self.github_url_to_user_repo_format(repo_url) @@ -328,39 +361,58 @@ def pull_repo(self, repo_url, branch=None, checkout = None, tag = None, pat = No # If the directory doesn't exist, clone it if not os.path.exists(repo_path): logger.info(f"Cloning repository {repo_url} to {repo_path}...") - + # Build clone command without branch if not provided clone_command = ['git', 'clone', repo_url, repo_path] if branch: - clone_command = ['git', 'clone', '--branch', branch, repo_url, repo_path] - + clone_command = [ + 'git', + 'clone', + '--branch', + branch, + repo_url, + repo_path] + subprocess.run(clone_command, check=True) else: - logger.info(f"Repository {repo_name} already exists at {repo_path}. Checking for local changes...") - + logger.info( + f"Repository {repo_name} already exists at {repo_path}. Checking for local changes...") + # Check for local changes - status_command = ['git', '-C', repo_path, 'status', '--porcelain', '--untracked-files=no'] - local_changes = subprocess.run(status_command, capture_output=True, text=True) + status_command = [ + 'git', + '-C', + repo_path, + 'status', + '--porcelain', + '--untracked-files=no'] + local_changes = subprocess.run( + status_command, capture_output=True, text=True) if local_changes.stdout.strip(): - logger.warning("There are local changes in the repository. Please commit or stash them before checking out.") + logger.warning( + "There are local changes in the repository. Please commit or stash them before checking out.") print(local_changes.stdout.strip()) - return {"return": 0, "warning": f"Local changes detected in the already existing repository: {repo_path}, skipping the pull"} + return { + "return": 0, "warning": f"Local changes detected in the already existing repository: {repo_path}, skipping the pull"} else: - logger.info("No local changes detected. Pulling latest changes...") - subprocess.run(['git', '-C', repo_path, 'pull'], check=True) + logger.info( + "No local changes detected. Pulling latest changes...") + subprocess.run( + ['git', '-C', repo_path, 'pull'], check=True) logger.info("Repository successfully pulled.") if tag: - checkout = "tags/"+tag + checkout = "tags/" + tag # Checkout to a specific branch or commit if --checkout is provided if checkout or tag: logger.info(f"Checking out to {checkout} in {repo_path}...") - subprocess.run(['git', '-C', repo_path, 'checkout', checkout], check=True) - - #if not tag: + subprocess.run( + ['git', '-C', repo_path, 'checkout', checkout], check=True) + + # if not tag: # subprocess.run(['git', '-C', repo_path, 'pull'], check=True) # logger.info("Repository successfully pulled.") @@ -369,12 +421,18 @@ def pull_repo(self, repo_url, branch=None, checkout = None, tag = None, pat = No # check the meta file to obtain uids meta_file_path = os.path.join(repo_path, 'meta.yaml') if not os.path.exists(meta_file_path): - logger.warning(f"meta.yaml not found in {repo_path}. Repo pulled but not registered in MLC repos. Skipping...") + logger.warning( + f"meta.yaml not found in {repo_path}. Repo pulled but not registered in MLC repos. Skipping...") return {"return": 0} - - with open(meta_file_path, 'r') as meta_file: - meta_data = yaml.safe_load(meta_file) - meta_data["path"] = repo_path + + try: + with open(meta_file_path, 'r') as meta_file: + meta_data = yaml.safe_load(meta_file) + meta_data["path"] = repo_path + except yaml.YAMLError as e: + logger.error(f"Error loading YAML configuration: {e}") + return {"return": 1, + "error": f"Syntax error in {meta_file_path}: {e}"} r = self.register_repo(repo_path, meta_data, ignore_on_conflict) if r['return'] > 0: @@ -385,7 +443,8 @@ def pull_repo(self, repo_url, branch=None, checkout = None, tag = None, pat = No except subprocess.CalledProcessError as e: return {'return': 1, 'error': f"Git command failed: {e}"} except Exception as e: - return {'return': 1, 'error': f"Error pulling repository: {str(e)}"} + return {'return': 1, + 'error': f"Error pulling repository: {str(e)}"} def pull(self, run_args): """ @@ -396,8 +455,8 @@ def pull(self, run_args): The `pull` action clones an MLC repository and registers it in MLC. - If the repository already exists locally in the MLC repos directory, it fetches the latest changes only if there are no - uncommited modifications(excluding untracked files/folders). The `pull` action could be also used to checkout + If the repository already exists locally in the MLC repos directory, it fetches the latest changes only if there are no + uncommited modifications(excluding untracked files/folders). The `pull` action could be also used to checkout to a particular branch, commit or release tag using flags --checkout and --tag. Example Command: @@ -413,7 +472,7 @@ def pull(self, run_args): Example Output: anandhu@anandhu-VivoBook-ASUSLaptop-X515UA-M515UA:~$ mlc pull repo mlcommons@mlperf-automations - [2025-02-19 16:46:27,208 main.py:1260 INFO] - Cloning repository https://github.com/mlcommons/mlperf-automations.git + [2025-02-19 16:46:27,208 main.py:1260 INFO] - Cloning repository https://github.com/mlcommons/mlperf-automations.git to /home/anandhu/MLC/repos/mlcommons@mlperf-automations... Cloning into '/home/anandhu/MLC/repos/mlcommons@mlperf-automations'... remote: Enumerating objects: 77610, done. @@ -427,7 +486,7 @@ def pull(self, run_args): [2025-02-19 16:46:57,605 main.py:1126 INFO] - Added new repo path: /home/anandhu/MLC/repos/mlcommons@mlperf-automations [2025-02-19 16:46:57,606 main.py:1130 INFO] - Updated repos.json at /home/anandhu/MLC/repos/repos.json - Note: + Note: - repo_uid and repo_alias are not supported in the pull action for the repo target. - Only one of --checkout, --branch, or --tag should be specified at a time. @@ -435,9 +494,11 @@ def pull(self, run_args): repo_url = run_args.get('repo', run_args.get('url', 'repo')) if not repo_url or repo_url == "repo": for repo_object in self.repos: - if os.path.exists(os.path.join(repo_object.path, ".git")) and os.access(repo_object.path, os.W_OK): + if os.path.exists(os.path.join(repo_object.path, ".git")) and os.access( + repo_object.path, os.W_OK): repo_folder_name = os.path.basename(repo_object.path) - res = self.pull_repo(repo_folder_name, repo_path = repo_object.path) + res = self.pull_repo( + repo_folder_name, repo_path=repo_object.path) if res['return'] > 0: return res else: @@ -449,18 +510,18 @@ def pull(self, run_args): ssh = run_args.get('ssh') if sum(bool(var) for var in [branch, checkout, tag]) > 1: - return {"return": 1, "error": "Only one among the three flags(branch, checkout and tag) could be specified"} + return { + "return": 1, "error": "Only one among the three flags(branch, checkout and tag) could be specified"} res = self.pull_repo(repo_url, branch, checkout, tag, pat, ssh) if res['return'] > 0: return res - return {'return': 0} def show(self, run_args): return self.list(run_args) - + def list(self, run_args): """ #################################################################################################################### @@ -469,7 +530,7 @@ def list(self, run_args): #################################################################################################################### The `list` action displays all registered MLC repositories along with their aliases and paths. - + Example Command: mlc list repo @@ -499,7 +560,7 @@ def list(self, run_args): print("-------------") logger.info("Repository listing ended") return {"return": 0} - + def rm(self, run_args): """ #################################################################################################################### @@ -507,10 +568,11 @@ def rm(self, run_args): Action: rm #################################################################################################################### - The `rm` action removes a specified repository from MLCFlow, deleting both the repo folder and its registration. - If there are any modified local changes, the user will be prompted for confirmation unless the `-f` flag is used + The `rm` action removes a specified repository from MLCFlow, deleting the repository folder, its index entries, + and its registration. + If there are any modified local changes, the user will be prompted for confirmation unless the `-f` flag is used for force removal. - + Example Command: mlc rm repo mlcommons@mlperf-automations @@ -528,16 +590,17 @@ def rm(self, run_args): """ if not run_args['repo']: logger.error("The repository to be removed is not specified") - return {"return": 1, "error": "The repository to be removed is not specified"} + return {"return": 1, + "error": "The repository to be removed is not specified"} r = self.find(run_args) - if r['return'] == 0: list_repos = r['list'] if len(list_repos) > 1: - return {"return": 1, "error": "Please select a unique repo by repo alias or repo UID to remove"} + return { + "return": 1, "error": "Please select a unique repo by repo alias or repo UID to remove"} repo = list_repos[0] repo_path = repo.path @@ -552,60 +615,78 @@ def rm(self, run_args): return r repos_file_path = os.path.join(self.repos_path, 'repos.json') - + force_remove = True if run_args.get('f') else False - - return rm_repo(repo_path, repos_file_path, force_remove) - -def rm_repo(repo_path, repos_file_path, force_remove): - logger.info("rm command has been called for repo. This would delete the repo folder and unregister the repo from repos.json") - - repo_name = os.path.basename(repo_path) - mlc_repos_path = os.path.abspath(os.path.dirname(repos_file_path)) - repo_parent_path = os.path.abspath(os.path.dirname(repo_path)) - - if os.path.isdir(repo_path) and os.path.samefile(mlc_repos_path, repo_parent_path): - # Check for local changes - status_command = ['git', '-C', repo_path, 'status', '--porcelain', '--untracked-files=no'] - local_changes = subprocess.run(status_command, capture_output=True, text=True) - - if local_changes.stdout: - logger.warning("Local changes detected in repository. Changes are listed below:") - print(local_changes.stdout) - confirm_remove = True if force_remove or (input("Continue to remove repo?").lower()) in ["yes", "y"] else False - else: - logger.info("No local changes detected. Removing repo...") - confirm_remove = True - if confirm_remove: - if force_remove: - logger.info("Force remove is set.") - shutil.rmtree(repo_path) - logger.info(f"Repo {repo_name} residing in path {repo_path} has been successfully removed") - logger.info("Checking whether the repo was registered in repos.json") - unregister_repo(repo_path, repos_file_path) - else: - logger.info("rm repo ooperation cancelled by user!") - + index = Action.get_index(self) + index.remove_repo_from_index(repo_path) + return rm_repo(repo_path, repos_file_path, force_remove) + + +def rm_repo(repo_path, repos_file_path, force_remove): + logger.info( + "rm command has been called for repo. This would delete the repo folder and unregister the repo from repos.json") + + repo_name = os.path.basename(repo_path) + mlc_repos_path = os.path.abspath(os.path.dirname(repos_file_path)) + repo_parent_path = os.path.abspath(os.path.dirname(repo_path)) + + if os.path.isdir(repo_path) and os.path.samefile( + mlc_repos_path, repo_parent_path): + # Check for local changes + status_command = [ + 'git', + '-C', + repo_path, + 'status', + '--porcelain', + '--untracked-files=no'] + local_changes = subprocess.run( + status_command, capture_output=True, text=True) + + if local_changes.stdout: + logger.warning( + "Local changes detected in repository. Changes are listed below:") + print(local_changes.stdout) + confirm_remove = True if force_remove or ( + input("Continue to remove repo?").lower()) in [ + "yes", "y"] else False else: - logger.warning(f"Repo {repo_name} was not found in the repo folder. repos.json will be checked for external paths. If any, that will be removed.") + logger.info("No local changes detected. Removing repo...") + confirm_remove = True + if confirm_remove: + if force_remove: + logger.info("Force remove is set.") + shutil.rmtree(repo_path) + logger.info( + f"Repo {repo_name} residing in path {repo_path} has been successfully removed") + logger.info( + "Checking whether the repo was registered in repos.json") unregister_repo(repo_path, repos_file_path) + else: + logger.info("rm repo ooperation cancelled by user!") + + else: + logger.warning( + f"Repo {repo_name} was not found in the repo folder. repos.json will be checked for external paths. If any, that will be removed.") + unregister_repo(repo_path, repos_file_path) + + return {"return": 0} + - return {"return": 0} - def unregister_repo(repo_path, repos_file_path): - logger.info(f"Unregistering the repo in path {repo_path}") + logger.info(f"Unregistering the repo in path {repo_path}") - with open(repos_file_path, 'r') as f: - repos_list = json.load(f) - - if repo_path in repos_list: - repos_list.remove(repo_path) - with open(repos_file_path, 'w') as f: - json.dump(repos_list, f, indent=2) - logger.info(f"Path: {repo_path} has been removed.") - else: - logger.info(f"Path: {repo_path} not found in {repos_file_path}. Nothing to be unregistered!") - - return {'return': 0} + with open(repos_file_path, 'r') as f: + repos_list = json.load(f) + + if repo_path in repos_list: + repos_list.remove(repo_path) + with open(repos_file_path, 'w') as f: + json.dump(repos_list, f, indent=2) + logger.info(f"Path: {repo_path} has been removed.") + else: + logger.info( + f"Path: {repo_path} not found in {repos_file_path}. Nothing to be unregistered!") + return {'return': 0} diff --git a/mlc/script_action.py b/mlc/script_action.py index 7af6ec5ff..5ce3bd1fb 100644 --- a/mlc/script_action.py +++ b/mlc/script_action.py @@ -1,3 +1,4 @@ +import re from .action import Action import os import sys @@ -8,6 +9,7 @@ from . import utils from .logger import logger + class ScriptAction(Action): """ #################################################################################################################### @@ -35,6 +37,7 @@ class ScriptAction(Action): """ parent = None + def __init__(self, parent=None): self.parent = parent self.__dict__.update(vars(parent)) @@ -59,7 +62,7 @@ def search(self, i): return res find = search - + def rm(self, i): """ #################################################################################################################### @@ -67,7 +70,7 @@ def rm(self, i): Action: Remove(rm) #################################################################################################################### - The `remove` (`rm`) action deletes one or more scripts from MLC repositories. + The `remove` (`rm`) action deletes one or more scripts from MLC repositories. Example Command: @@ -86,14 +89,14 @@ def show(self, run_args): Action: Show #################################################################################################################### - The `show` action retrieves the path and metadata of the searched script in MLC repositories. + The `show` action retrieves the path and metadata of the searched script in MLC repositories. Example Command: mlc show script --tags=detect,os Example Output: - + arjun@intel-spr-i9:~$ mlc show script --tags=detect,os [2025-02-14 02:56:16,604 main.py:1404 INFO] - Showing script with tags: detect,os Location: /home/arjun/MLC/repos/gateoverflow@mlperf-automations/script/detect-os: @@ -102,7 +105,7 @@ def show(self, run_args): alias: detect-os description: Detects the operating system and platform information tags: ['detect-os', 'detect', 'os', 'info'] - new_env_keys: ['MLC_HOST_OS_*', '+MLC_HOST_OS_*', 'MLC_HOST_PLATFORM_*', 'MLC_HOST_PYTHON_*', 'MLC_HOST_SYSTEM_NAME', + new_env_keys: ['MLC_HOST_OS_*', '+MLC_HOST_OS_*', 'MLC_HOST_PLATFORM_*', 'MLC_HOST_PYTHON_*', 'MLC_HOST_SYSTEM_NAME', 'MLC_RUN_STATE_DOCKER', '+PATH'] new_state_keys: ['os_uname_*'] ...................................................... @@ -117,7 +120,14 @@ def show(self, run_args): if res['return'] > 0: return res logger.info(f"Showing script with tags: {run_args.get('tags')}") - script_meta_keys_to_show = ["uid", "alias", "description", "tags", "new_env_keys", "new_state_keys", "cache"] + script_meta_keys_to_show = [ + "uid", + "alias", + "description", + "tags", + "new_env_keys", + "new_state_keys", + "cache"] for item in res['list']: print(f"""Location: {item.path}: Main Script Meta:""") @@ -128,9 +138,10 @@ def show(self, run_args): print(" Input mapping:") utils.printd(item.meta["input_mapping"], begin_spaces=8) print("......................................................") - print(f"""For full script meta, see meta file at {os.path.join(item.path, "meta.yaml")}""") + print( + f"""For full script meta, see meta file at {os.path.join(item.path, "meta.yaml")}""") print("") - + return {'return': 0} def add(self, i): @@ -140,17 +151,17 @@ def add(self, i): Action: Add #################################################################################################################### - The `add` action creates a new script in a registered MLC repository. + The `add` action creates a new script in a registered MLC repository. Syntax: mlc add script :new_script --tags=benchmark Options: - --template_tags: A comma-separated list of tags to create a new MLC script based on existing templates. + --template_tags: A comma-separated list of tags to create a new MLC script based on existing templates. Example Output: - + arjun@intel-spr-i9:~$ mlc add script gateoverflow@mlperf-automations --tags=benchmark --template_tags=app,mlperf,inference More than one script found for None: 1. /home/arjun/MLC/repos/gateoverflow@mlperf-automations/script/app-mlperf-inference-mlcommons-python @@ -188,7 +199,7 @@ def add(self, i): ii['src_tags'] = i.get("template_tags", "template,generic") ii['dest'] = item ii['tags'] = i.get('tags', []) - res = self.cp(ii) + res = self.cp(ii) return res @@ -208,7 +219,8 @@ def dynamic_import_module(self, script_path): module_name = os.path.splitext(os.path.basename(script_path))[0] spec = importlib.util.spec_from_file_location(module_name, script_path) if spec is None or spec.loader is None: - raise ImportError(f"Cannot create a module spec for: {script_path}") + raise ImportError( + f"Cannot create a module spec for: {script_path}") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -219,11 +231,12 @@ def call_script_module_function(self, function_name, run_args): self.action_type = "script" repos_folder = self.repos_path - # Import script submodule + # Import script submodule script_path = self.find_target_folder("script") if not script_path: - logger.warning("Script automation not found. Automatically pulling mlcommons@mlperf-automations repository...") - + logger.warning( + "Script automation not found. Automatically pulling mlcommons@mlperf-automations repository...") + # Use the access method to pull the required repository result = self.access({ "automation": "repo", @@ -239,11 +252,15 @@ def call_script_module_function(self, function_name, run_args): # Try to find the script path again after pulling script_path = self.find_target_folder("script") if not script_path: - return {'return': 1, 'error': f"""Script automation still not found after pulling mlcommons@mlperf-automations --branch=dev."""} + return { + 'return': 1, 'error': f"""Script automation still not found after pulling mlcommons@mlperf-automations --branch=dev."""} else: - # If pull failed, return the original error with additional info - logger.error(f"Failed to pull mlcommons@mlperf-automations repository: {result.get('error', 'Unknown error')}") - return {'return': 1, 'error': f"""Script automation not found and failed to automatically pull mlcommons@mlperf-automations --branch=dev. Please run "mlc pull repo mlcommons@mlperf-automations --branch=dev" manually: {result.get('error', 'Unknown error')}"""} + # If pull failed, return the original error with additional + # info + logger.error( + f"Failed to pull mlcommons@mlperf-automations repository: {result.get('error', 'Unknown error')}") + return { + 'return': 1, 'error': f"""Script automation not found and failed to automatically pull mlcommons@mlperf-automations --branch=dev. Please run "mlc pull repo mlcommons@mlperf-automations --branch=dev" manually: {result.get('error', 'Unknown error')}"""} module_path = os.path.join(script_path, "module.py") module = self.dynamic_import_module(module_path) @@ -254,45 +271,89 @@ def call_script_module_function(self, function_name, run_args): params = inspect.signature(ctor).parameters if 'run_args' in params: - automation_instance = module.ScriptAutomation(self, module_path, run_args) - else: - automation_instance = module.ScriptAutomation(self, module_path) - - if function_name == "run": - result = automation_instance.run(run_args) # Pass args to the run method - elif function_name == "docker": - result = automation_instance.docker(run_args) # Pass args to the run method - elif function_name == "test": - result = automation_instance.test(run_args) # Pass args to the run method - elif function_name == "experiment": - result = automation_instance.experiment(run_args) # Pass args to the experiment method - elif function_name == "remote_run": - result = automation_instance.remote_run(run_args) # Pass args to the experiment method - elif function_name == "help": - result = automation_instance.help(run_args) # Pass args to the help method - elif function_name == "doc": - result = automation_instance.doc(run_args) # Pass args to the doc method - elif function_name == "lint": - result = automation_instance.lint(run_args) # Pass args to the lint method + automation_instance = module.ScriptAutomation( + self, module_path, run_args) else: - return {'return': 1, 'error': f'Function {function_name} is not supported'} - + automation_instance = module.ScriptAutomation( + self, module_path) + + try: + if function_name == "run": + result = automation_instance.run( + run_args) # Pass args to the run method + elif function_name == "docker": + result = automation_instance.docker( + run_args) # Pass args to the run method + elif function_name == "test": + result = automation_instance.test( + run_args) # Pass args to the run method + elif function_name == "experiment": + result = automation_instance.experiment( + run_args) # Pass args to the experiment method + elif function_name == "remote_run": + result = automation_instance.remote_run( + run_args) # Pass args to the experiment method + elif function_name == "help": + result = automation_instance.help( + run_args) # Pass args to the help method + elif function_name == "doc": + result = automation_instance.doc( + run_args) # Pass args to the doc method + elif function_name == "lint": + result = automation_instance.lint( + run_args) # Pass args to the lint method + else: + return { + 'return': 1, 'error': f'Function {function_name} is not supported'} + except ScriptExecutionError: + raise + except Exception as exc: + _repo_match = re.search(r'/repos/([^/]+)/', module_path) + _repo_alias = _repo_match.group(1) if _repo_match else None + _script_name = run_args.get('tags', run_args.get('details')) + raise ScriptExecutionError( + f"Script {function_name} execution failed in {module_path}." + + "\nError : " + f"{type(exc).__name__}: {exc}", + script_name=_script_name, repo_alias=_repo_alias, module_path=module_path, + run_args=run_args) from exc + if result['return'] > 0: error = result.get('error', "") - raise ScriptExecutionError(f"Script {function_name} execution failed in {module_path}. \nError : {error}") - - if str(run_args.get("mlc_output")).lower() in ["on", "true", "yes", "1"]: + _name_match = re.search(r'name\s*=\s*([^,)]+)', error) + _script_name = _name_match.group(1).strip() if _name_match else run_args.get( + 'tags', run_args.get('details')) + _repo_match = re.search(r'/repos/([^/]+)/', module_path) + _repo_alias = _repo_match.group(1) if _repo_match else None + # Dump dependency version info to file for debugging + _version_info_file = None + _version_info = result.get('version_info', []) + if _version_info: + _version_info_file = os.path.join( + os.getcwd(), 'mlc-error-version-info.json') + try: + with open(_version_info_file, 'w') as _vf: + json.dump(_version_info, _vf, indent=2) + except Exception: + _version_info_file = None + raise ScriptExecutionError( + f"Script {function_name} execution failed in {module_path}. \nError : {error}", + script_name=_script_name, repo_alias=_repo_alias, module_path=module_path, + run_args=run_args, version_info_file=_version_info_file) + + if str(run_args.get("mlc_output")).lower() in [ + "on", "true", "yes", "1"]: with open("tmp-state.json", "w") as f: json.dump(result['new_state'], f, indent=2) with open("tmp-run-env.out", "w") as f: - for key,val in result['new_env'].items(): + for key, val in result['new_env'].items(): f.write(f"""{key}="{val}"\n""") return result else: logger.info("ScriptAutomation class not found in the script.") - return {'return': 1, 'error': 'ScriptAutomation class not found in the script.'} + return {'return': 1, + 'error': 'ScriptAutomation class not found in the script.'} def docker(self, run_args): return self.docker_run(run_args) @@ -304,13 +365,13 @@ def docker_run(self, run_args): Action: Docker #################################################################################################################### - The `docker` action runs scripts inside a containerized environment. + The `docker` action runs scripts inside a containerized environment. An MLCFlow script can be executed inside a Docker container using either of the following syntaxes: - 1. Docker Run: mlc docker run --tags=