From 6e1e5664f255df15e6bf41eecfa38c2356b10ea2 Mon Sep 17 00:00:00 2001 From: Sujith Kanakkassery Date: Wed, 14 Jan 2026 13:39:51 +0530 Subject: [PATCH 01/27] Force index rebuild when a new repo is pulled (#216) * Update repo_action.py to force index rebuild when a new repo is pulled * Update AI PR review workflow by adding fetch step for PR head * Update index.py to incrementally add and remove index entries of a repo --- .github/workflows/ai-pr-review.yml | 5 + mlc/index.py | 230 ++++++++++++++--------------- mlc/repo_action.py | 17 ++- 3 files changed, 136 insertions(+), 116 deletions(-) diff --git a/.github/workflows/ai-pr-review.yml b/.github/workflows/ai-pr-review.yml index a9977c01d..d8c51806c 100644 --- a/.github/workflows/ai-pr-review.yml +++ b/.github/workflows/ai-pr-review.yml @@ -17,11 +17,16 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + ref: ${{ github.event.pull_request.base.ref }} - name: Install dependencies run: | pip install requests jq + - name: Fetch PR head + run: | + git fetch origin ${{ github.event.pull_request.head.sha }} + - name: Get incremental diff id: diff run: | diff --git a/mlc/index.py b/mlc/index.py index 4a18e3eea..d72b75067 100644 --- a/mlc/index.py +++ b/mlc/index.py @@ -27,7 +27,6 @@ def __init__(self, repos_path, repos): """ self.repos_path = repos_path self.repos = repos - #logger.info(repos) logger.debug(f"Repos path for Index: {self.repos_path}") self.index_files = { @@ -140,7 +139,64 @@ def get_item_mtime(self,file): if t > latest: latest = t return latest - + + def _index_single_repo(self, repo, repos_changed=False, current_item_keys=None): + repo_path = repo.path + if not os.path.isdir(repo_path): + return False + + changed = False + + for folder_type in ["script", "cache", "experiment"]: + folder_path = os.path.join(repo_path, folder_type) + if not os.path.isdir(folder_path): + continue + + for automation_dir in os.listdir(folder_path): + automation_path = os.path.join(folder_path, automation_dir) + if not os.path.isdir(automation_path): + continue + + yaml_path = os.path.join(automation_path, "meta.yaml") + json_path = os.path.join(automation_path, "meta.json") + + if os.path.isfile(yaml_path): + config_path = yaml_path + elif os.path.isfile(json_path): + config_path = json_path + else: + #logger.debug(f"No config file found in {automation_path}, skipping") + delete_flag = False + if automation_dir in self.modified_times: + del self.modified_times[automation_dir] + if any(automation_dir in item["path"] for item in self.indices[folder_type]): + logger.debug(f"Removed index entry (if it exists) for {folder_type} : {automation_dir}") + delete_flag = True + self._remove_index_entry(automation_path) + if delete_flag: + self._save_indices() + continue + if current_item_keys is not None: + current_item_keys.add(config_path) + mtime = self.get_item_mtime(config_path) + old = self.modified_times.get(config_path) + old_mtime = old["mtime"] if isinstance(old, dict) else old + + # skip if unchanged + if old_mtime == mtime and repos_changed != 1: + continue + + self.modified_times[config_path] = { + "mtime": mtime, + "date_time": datetime.fromtimestamp(mtime).strftime("%Y-%m-%d %H:%M:%S") + } + + # meta file changed, so reindex + self._process_config_file(config_path, folder_type, automation_path, repo) + changed = True + + return changed + def build_index(self): """ Build shared indices for script, cache, and experiment folders across all repositories. @@ -152,125 +208,27 @@ def build_index(self): # track all currently detected item paths current_item_keys = set() changed = False - repos_changed = False - - # load existing modified times + force_rebuild = False + + # load modified times self.modified_times = self._load_modified_times() + # if missing index file, then force full rebuild index_json_path = os.path.join(self.repos_path, "index_script.json") - - rebuild_index = False - - #file does not exist, rebuild if not os.path.exists(index_json_path): logger.warning("index_script.json missing. Forcing full index rebuild...") - #logger.debug("Resetting modified_times...") self.modified_times = {} - self._save_modified_times() - #else: - # logger.debug("index_script.json exists. Skipping forced rebuild.") - - #check repos.json mtime - repos_json_path = os.path.join(self.repos_path, "repos.json") - repos_mtime = os.path.getmtime(repos_json_path) - - key = f"{repos_json_path}" - old = self.modified_times.get(key) - repo_old_mtime = old["mtime"] if isinstance(old, dict) else old - - #logger.debug(f"Current repos.json mtime: {repos_mtime}") - #logger.debug(f"Old repos.json mtime: {repo_old_mtime}") - current_item_keys.add(key) - - # if changed, reset indexes - if repo_old_mtime is None or repo_old_mtime != repos_mtime: - logger.debug("repos.json modified. Clearing index ........") - # reset indices - self.indices = {key: [] for key in self.index_files.keys()} - # record repo mtime - self.modified_times[key] = { - "mtime": repos_mtime, - "date_time": datetime.fromtimestamp(repos_mtime).strftime("%Y-%m-%d %H:%M:%S") - } - # clear modified times except for repos.json - self.modified_times = {key: self.modified_times[key]} - self._save_indices() - self._save_modified_times() - repos_changed = True - #else: - # logger.debug("Repos.json not modified") + self.indices = {k: [] for k in self.index_files.keys()} + force_rebuild = True + # index each repo for repo in self.repos: - repo_path = repo.path #os.path.join(self.repos_path, repo) - if not os.path.isdir(repo_path): - continue - #logger.debug(f"------------Checking repository: {repo_path}---------------") - # Filter for relevant directories in the repo - for folder_type in ["script", "cache", "experiment"]: - #logger.debug(f"Checking folder type: {folder_type}") - folder_path = os.path.join(repo_path, folder_type) - if not os.path.isdir(folder_path): - continue - - # Process each automation directory - for automation_dir in os.listdir(folder_path): - # logger.debug(f"Checking automation directory: {automation_dir}") - automation_path = os.path.join(folder_path, automation_dir) - if not os.path.isdir(automation_path): - #logger.debug(f"Skipping non-directory automation path: {automation_path}") - continue - - yaml_path = os.path.join(automation_path, "meta.yaml") - json_path = os.path.join(automation_path, "meta.json") - - if os.path.isfile(yaml_path): - # logger.debug(f"Found YAML config file: {yaml_path}") - config_path = yaml_path - elif os.path.isfile(json_path): - # logger.debug(f"Found JSON config file: {json_path}") - config_path = json_path - else: - #logger.debug(f"No config file found in {automation_path}, skipping") - delete_flag = False - if automation_dir in self.modified_times: - del self.modified_times[automation_dir] - if any(automation_dir in item["path"] for item in self.indices[folder_type]): - logger.debug(f"Removed index entry (if it exists) for {folder_type} : {automation_dir}") - delete_flag = True - self._remove_index_entry(automation_path) - if delete_flag: - self._save_indices() - continue - current_item_keys.add(config_path) - mtime = self.get_item_mtime(config_path) - - old = self.modified_times.get(config_path) - old_mtime = old["mtime"] if isinstance(old, dict) else old - - # skip if unchanged - if old_mtime == mtime and repos_changed != 1: - # logger.debug(f"No changes detected for {config_path}, skipping reindexing.") - continue - #if(old_mtime is None): - # logger.debug(f"New meta.yaml file detected: {config_path}. Adding to index.") - - # update mtime - #logger.debug(f"{config_path} is modified, index getting updated") - #if config_path not in self.modified_times: - # logger.debug(f"*************{config_path} not found in modified_times; creating new entry***************") - - self.modified_times[config_path] = { - "mtime": mtime, - "date_time": datetime.fromtimestamp(mtime).strftime("%Y-%m-%d %H:%M:%S") - } - #logger.debug(f"Modified time for {config_path} updated to {mtime}") - changed = True - # meta file changed, so reindex - self._process_config_file(config_path, folder_type, automation_path, repo) + repo_changed = self._index_single_repo(repo, force_rebuild, current_item_keys) + if repo_changed: + changed = True # remove deleted scripts - old_keys = set(self.modified_times.keys()) - deleted_keys = old_keys - current_item_keys + deleted_keys = set(self.modified_times) - current_item_keys for key in deleted_keys: logger.warning(f"Detected deleted item, removing entry from modified times: {key}") del self.modified_times[key] @@ -281,13 +239,10 @@ def build_index(self): if deleted_keys: logger.debug(f"Deleted keys removed from modified times and indices: {deleted_keys}") - if changed: + if force_rebuild or changed: logger.debug("Changes detected, saving updated index and modified times.") self._save_modified_times() self._save_indices() - #logger.debug("**************Index updated (changes detected).*************************") - #else: - #logger.debug("**************Index unchanged (no changes detected).********************") def _remove_index_entry(self, key): logger.debug(f"Removing index entry for {key}") @@ -379,3 +334,48 @@ def _save_indices(self): #logger.debug(f"Shared index for {folder_type} saved to {output_file}.") except Exception as e: logger.error(f"Error saving shared index for {folder_type}: {e}") + + + def add_repo(self, repo): + """ + Incrementally index a newly registered repository. + """ + changed = self._index_single_repo(repo, repos_changed=True) + + if changed: + self._save_indices() + self._save_modified_times() + + + def remove_repo_from_index(self, repo_path): + """ + Remove all index entries and modified times belonging to a repo. + Called when a repo is unregistered from repos.json. + """ + + logger.info(f"Removing repo from index: {repo_path}") + changed = False + + # remove index entries + for folder_type in self.indices: + before = len(self.indices[folder_type]) + self.indices[folder_type] = [ + item for item in self.indices[folder_type] + if not item["path"].startswith(repo_path) + ] + if len(self.indices[folder_type]) != before: + changed = True + + # remove modified times + keys_to_delete = [ + k for k in self.modified_times + if k.startswith(repo_path) + ] + + for k in keys_to_delete: + del self.modified_times[k] + changed = True + + if changed: + self._save_indices() + self._save_modified_times() \ No newline at end of file diff --git a/mlc/repo_action.py b/mlc/repo_action.py index 8d04cfd50..4a262f886 100644 --- a/mlc/repo_action.py +++ b/mlc/repo_action.py @@ -8,6 +8,8 @@ from . import utils from .logger import logger from urllib.parse import urlparse +from .repo import Repo +from .index import Index class RepoAction(Action): """ @@ -166,6 +168,16 @@ def register_repo(self, repo_path, repo_meta, ignore_on_conflict=False): json.dump(repos_list, f, indent=2) logger.info(f"Updated repos.json at {repos_file_path}") + self.repos = self.load_repos_and_meta() + repo_obj = next( + (r for r in self.repos if r.path == repo_path), + None + ) + + if repo_obj: + index = Action.get_index(self) + index.add_repo(repo_obj) + logger.debug("Index file has been updated") return {'return': 0} @@ -507,7 +519,8 @@ def rm(self, run_args): Action: rm #################################################################################################################### - The `rm` action removes a specified repository from MLCFlow, deleting both the repo folder and its registration. + The `rm` action removes a specified repository from MLCFlow, deleting the repository folder, its index entries, + and its registration. If there are any modified local changes, the user will be prompted for confirmation unless the `-f` flag is used for force removal. @@ -554,6 +567,8 @@ def rm(self, run_args): repos_file_path = os.path.join(self.repos_path, 'repos.json') force_remove = True if run_args.get('f') else False + index = Action.get_index(self) + index.remove_repo_from_index(repo_path) return rm_repo(repo_path, repos_file_path, force_remove) From b15d6c9b9b2278c17667386a1b804d258a49fbe8 Mon Sep 17 00:00:00 2001 From: amd-arsuresh Date: Sat, 31 Jan 2026 23:56:01 +0000 Subject: [PATCH 02/27] Improve the log output, better path handling for Windows (#217) --- mlc/index.py | 61 +++++++++++++++++++++++++++++++++------------------ mlc/logger.py | 16 +++++++++++--- mlc/utils.py | 28 +++++++++++++---------- 3 files changed, 70 insertions(+), 35 deletions(-) diff --git a/mlc/index.py b/mlc/index.py index d72b75067..5928197c1 100644 --- a/mlc/index.py +++ b/mlc/index.py @@ -40,6 +40,16 @@ def __init__(self, repos_path, repos): self._load_existing_index() self.build_index() + def _get_stored_mtime(self, key): + """ + Helper method to safely extract mtime from stored data. + Handles both old format (direct mtime) and new format (dict with mtime key). + """ + old = self.modified_times.get(key) + if old is None: + return None + return old["mtime"] if isinstance(old, dict) else old + def _load_modified_times(self): """ Load stored mtimes to check for changes in scripts. @@ -49,7 +59,8 @@ def _load_modified_times(self): # logger.info(f"Loading modified times from {self.modified_times_file}") with open(self.modified_times_file, "r") as f: return json.load(f) - except Exception: + except (json.JSONDecodeError, IOError) as e: + logger.warning(f"Failed to load modified times: {e}") return {} return {} @@ -76,7 +87,8 @@ def _load_existing_index(self): if isinstance(item.get("repo"), dict): item["repo"] = Repo(**item["repo"]) - except Exception: + except (json.JSONDecodeError, IOError, KeyError, TypeError) as e: + logger.warning(f"Failed to load index for {folder_type}: {e}") pass # fall back to empty index def add(self, meta, folder_type, path, repo): @@ -165,25 +177,32 @@ def _index_single_repo(self, repo, repos_changed=False, current_item_keys=None): elif os.path.isfile(json_path): config_path = json_path else: - #logger.debug(f"No config file found in {automation_path}, skipping") + # No config file found, remove from index if exists delete_flag = False - if automation_dir in self.modified_times: - del self.modified_times[automation_dir] - if any(automation_dir in item["path"] for item in self.indices[folder_type]): + + # Check and remove both possible config paths from modified_times + for config_name in ["meta.yaml", "meta.json"]: + config_key = os.path.join(automation_path, config_name) + if config_key in self.modified_times: + del self.modified_times[config_key] + delete_flag = True + + # Use exact path matching instead of substring + if any(item["path"] == automation_path for item in self.indices[folder_type]): logger.debug(f"Removed index entry (if it exists) for {folder_type} : {automation_dir}") delete_flag = True self._remove_index_entry(automation_path) + if delete_flag: self._save_indices() continue if current_item_keys is not None: current_item_keys.add(config_path) mtime = self.get_item_mtime(config_path) - old = self.modified_times.get(config_path) - old_mtime = old["mtime"] if isinstance(old, dict) else old + old_mtime = self._get_stored_mtime(config_path) # skip if unchanged - if old_mtime == mtime and repos_changed != 1: + if old_mtime == mtime and not repos_changed: continue self.modified_times[config_path] = { @@ -220,6 +239,7 @@ def build_index(self): self.modified_times = {} self.indices = {k: [] for k in self.index_files.keys()} force_rebuild = True + # index each repo for repo in self.repos: @@ -246,10 +266,12 @@ def build_index(self): def _remove_index_entry(self, key): logger.debug(f"Removing index entry for {key}") + # Normalize paths for comparison + normalized_key = os.path.normpath(key) for ft in self.indices: self.indices[ft] = [ item for item in self.indices[ft] - if key not in item["path"] + if os.path.normpath(item["path"]) != normalized_key ] def _delete_by_uid(self, folder_type, uid, alias): @@ -302,17 +324,14 @@ def _process_config_file(self, config_file, folder_type, folder_path, repo): alias = data.get("alias", None) # Validate and add to indices - if unique_id: - self._delete_by_uid(folder_type, unique_id, alias) - self.indices[folder_type].append({ - "uid": unique_id, - "tags": tags, - "alias": alias, - "path": folder_path, - "repo": repo - }) - else: - logger.warning(f"Skipping {config_file}: Missing 'uid' field.") + self._delete_by_uid(folder_type, unique_id, alias) + self.indices[folder_type].append({ + "uid": unique_id, + "tags": tags, + "alias": alias, + "path": folder_path, + "repo": repo + }) except Exception as e: logger.error(f"Error processing {config_file}: {e}") diff --git a/mlc/logger.py b/mlc/logger.py index b1f4d6178..8ea79625b 100644 --- a/mlc/logger.py +++ b/mlc/logger.py @@ -15,9 +15,19 @@ class ColoredFormatter(logging.Formatter): } def format(self, record): - # Add color to the levelname + # Pad filename and line number for alignment + record.filename = f"{record.filename:<15}" # Left-align filename with 15 char width + record.lineno = f"{record.lineno:>4}" # Right-align line number with 4 char width + + # Trim WARNING to WARN + levelname = "WARN" if record.levelname == "WARNING" else record.levelname + + # Pad and add color to the levelname + levelname_padded = f"{levelname:<5}" # Left-align levelname with 5 char width if record.levelname in self.COLORS: - record.levelname = f"{self.COLORS[record.levelname]}{record.levelname}{Style.RESET_ALL}" + record.levelname = f"{self.COLORS[record.levelname]}{levelname_padded}{Style.RESET_ALL}" + else: + record.levelname = levelname_padded return super().format(record) @@ -25,7 +35,7 @@ def format(self, record): def setup_logging(log_path = os.getcwd(), log_file = '.mlc-log.txt'): if not logger.hasHandlers(): - logFormatter = ColoredFormatter('[%(asctime)s %(filename)s:%(lineno)d %(levelname)s] - %(message)s') + logFormatter = ColoredFormatter('[%(asctime)s %(filename)s:%(lineno)s %(levelname)s] - %(message)s') # by default logging level is set to INFO is being set logger.setLevel(logging.INFO) diff --git a/mlc/utils.py b/mlc/utils.py index 342568305..1cc4e8f19 100644 --- a/mlc/utils.py +++ b/mlc/utils.py @@ -694,24 +694,30 @@ def extract_file(options): with zipfile.ZipFile(filename, 'r') as archive: members = archive.namelist() for member in members: - # Strip folder levels - stripped_path = os.path.join( - extract_to, *member.split(os.sep)[strip_folders:] - ) - if member.endswith('/'): # Directory - os.makedirs(stripped_path, exist_ok=True) - else: # File - os.makedirs(os.path.dirname(stripped_path), exist_ok=True) - with archive.open(member) as source, open(stripped_path, 'wb') as target: - shutil.copyfileobj(source, target) + # Strip folder levels (zip files always use forward slashes internally) + parts = member.split('/') + if len(parts) > strip_folders: + stripped_parts = parts[strip_folders:] + stripped_path = os.path.join(extract_to, *stripped_parts) + stripped_path = os.path.normpath(stripped_path) + + if member.endswith('/'): # Directory + os.makedirs(stripped_path, exist_ok=True) + else: # File + os.makedirs(os.path.dirname(stripped_path), exist_ok=True) + with archive.open(member) as source, open(stripped_path, 'wb') as target: + shutil.copyfileobj(source, target) elif tarfile.is_tarfile(filename): with tarfile.open(filename, 'r') as archive: members = archive.getmembers() for member in members: if strip_folders: + # Tar files also use forward slashes internally parts = member.name.split('/') - member.name = '/'.join(parts[strip_folders:]) + if len(parts) > strip_folders: + # Join with OS-specific separator for extraction + member.name = os.path.join(*parts[strip_folders:]) archive.extract(member, path=extract_to) else: From e2a2d448bcb83c62bdc5635626a60a016cf8e332 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 1 Feb 2026 00:03:34 +0000 Subject: [PATCH 03/27] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f63988778..3f9e7fda8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mlcflow" -version = "1.1.18" +version = "1.1.19" description = "An automation interface tailored for CPU/GPU benchmarking" authors = [ From 2d7e0cea710e7d647e22bfbdd2b41704f88d9397 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 1 Feb 2026 00:04:07 +0000 Subject: [PATCH 04/27] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 852ed67cf..4e036596e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.18 +1.1.19 From 984670c4a99381129a025552fe275111ed0cf024 Mon Sep 17 00:00:00 2001 From: amd-arsuresh Date: Sun, 1 Feb 2026 16:17:14 +0000 Subject: [PATCH 05/27] Support reindex action (#218) --- .github/workflows/test-mlc-core-actions.yaml | 110 +++++++++++++++++++ VERSION | 2 +- mlc/action.py | 55 +++++++++- mlc/index.py | 32 +++--- mlc/main.py | 45 +++++++- pyproject.toml | 2 +- 6 files changed, 224 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test-mlc-core-actions.yaml b/.github/workflows/test-mlc-core-actions.yaml index 5896cd324..de3a95003 100644 --- a/.github/workflows/test-mlc-core-actions.yaml +++ b/.github/workflows/test-mlc-core-actions.yaml @@ -220,6 +220,116 @@ jobs: mlc rm repo mlcommons@mlperf-automations -f mlcr detect,cpu -j + - name: Test 26 - Test reindex command and verify index files are updated + run: | + INDEX_SCRIPT="${HOME}/MLC/repos/index_script.json" + INDEX_CACHE="${HOME}/MLC/repos/index_cache.json" + INDEX_EXPERIMENT="${HOME}/MLC/repos/index_experiment.json" + + # Store initial modification times + if [ -f "$INDEX_SCRIPT" ]; then + BEFORE_SCRIPT=$(stat -c %Y "$INDEX_SCRIPT" 2>/dev/null || stat -f %m "$INDEX_SCRIPT" 2>/dev/null) + fi + if [ -f "$INDEX_CACHE" ]; then + BEFORE_CACHE=$(stat -c %Y "$INDEX_CACHE" 2>/dev/null || stat -f %m "$INDEX_CACHE" 2>/dev/null) + fi + if [ -f "$INDEX_EXPERIMENT" ]; then + BEFORE_EXPERIMENT=$(stat -c %Y "$INDEX_EXPERIMENT" 2>/dev/null || stat -f %m "$INDEX_EXPERIMENT" 2>/dev/null) + fi + + sleep 1 + + # Test reindex all + mlc reindex + + # Verify all index files were updated + if [ -f "$INDEX_SCRIPT" ]; then + AFTER_SCRIPT=$(stat -c %Y "$INDEX_SCRIPT" 2>/dev/null || stat -f %m "$INDEX_SCRIPT" 2>/dev/null) + if [ "$BEFORE_SCRIPT" = "$AFTER_SCRIPT" ]; then + echo "index_script.json was not updated after 'mlc reindex'. Exiting with failure." + exit 1 + fi + fi + + sleep 1 + BEFORE_SCRIPT=$(stat -c %Y "$INDEX_SCRIPT" 2>/dev/null || stat -f %m "$INDEX_SCRIPT" 2>/dev/null) + + # Test reindex specific target + mlc reindex script + + AFTER_SCRIPT=$(stat -c %Y "$INDEX_SCRIPT" 2>/dev/null || stat -f %m "$INDEX_SCRIPT" 2>/dev/null) + if [ "$BEFORE_SCRIPT" = "$AFTER_SCRIPT" ]; then + echo "index_script.json was not updated after 'mlc reindex script'. Exiting with failure." + exit 1 + fi + + # Test other reindex commands + mlc reindex all + mlc reindex cache + mlc reindex experiment + + - name: Test 27 - Test index handling when script/cache/repo is manually deleted + run: | + # Add a test script + mlc add script test-delete-script --tags=test,delete,temp + + # Get the actual script path from find command + SCRIPT_PATH=$(mlc find script test-delete-script -p 2>&1) + echo "Script path: $SCRIPT_PATH" + + if [ -z "$SCRIPT_PATH" ]; then + echo "Script was not found after adding. Exiting with failure." + exit 1 + fi + + # Manually delete the script + if [ -d "$SCRIPT_PATH" ]; then + rm -rf "$SCRIPT_PATH" + echo "Manually deleted script at $SCRIPT_PATH" + else + echo "Script directory not found at $SCRIPT_PATH. Exiting with failure." + exit 1 + fi + + # Verify the deleted script is no longer in the index + # The find command will automatically trigger index rebuild and detect the deletion + FIND_RESULT=$(mlc find script test-delete-script -p 2>/dev/null) + if [ -n "$FIND_RESULT" ]; then + echo "ERROR: Deleted script still found in index. Found: $FIND_RESULT" + exit 1 + fi + + echo "Script deletion test passed successfully" + + # Test with cache: run a script to create cache, then delete it manually + mlc run script --tags=detect,os --quiet + + # Find and delete a cache entry + CACHE_PATH=$(mlc find cache --tags=detect,os -p 2>/dev/null | head -1) + if [ -n "$CACHE_PATH" ] && [ -d "$CACHE_PATH" ]; then + echo "Found cache at: $CACHE_PATH" + + # Get the cache directory name for later verification + CACHE_DIR_NAME=$(basename "$CACHE_PATH") + + rm -rf "$CACHE_PATH" + echo "Manually deleted cache at $CACHE_PATH" + + # Verify the deleted cache is no longer in the index + # Check if the cache directory name appears in any found caches + if mlc find cache --tags=detect,os -p 2>/dev/null | grep -q "$CACHE_DIR_NAME"; then + echo "ERROR: Deleted cache directory still found in index." + echo "Deleted cache: $CACHE_PATH" + echo "Found caches:" + mlc find cache --tags=detect,os -p 2>/dev/null + exit 1 + fi + + echo "Cache deletion test passed successfully" + else + echo "No cache found to test deletion" + fi + test_mlc_access_core_actions: runs-on: ${{ matrix.os }} diff --git a/VERSION b/VERSION index 4e036596e..be5b4c7bb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.19 +1.1.20 diff --git a/mlc/action.py b/mlc/action.py index 665727a0e..6e39bd5f4 100644 --- a/mlc/action.py +++ b/mlc/action.py @@ -672,8 +672,11 @@ def search(self, i): details = i['details'] details_split = details.split(",") if len(details_split) > 1: - alias = details_split[0] - uid = details_split[1] + # Only treat as alias,uid if the second part is actually a valid UID + if utils.is_uid(details_split[1]): + alias = details_split[0] + uid = details_split[1] + # Otherwise, don't parse as alias,uid - let it be treated as tags else: if utils.is_uid(details_split[0]): uid = details_split[0] @@ -738,6 +741,54 @@ def search(self, i): find = search + def reindex(self, i): + """ + Reindex the specified target or all targets if none specified. + + Args: + i (dict): Input dictionary with the following keys: + - reindex_target (str, optional): Target to reindex ('script', 'cache', 'repo', 'all', or None). + If not provided or 'all', reindexes all targets. + + Returns: + dict: Result of the operation with 'return' code 0 on success. + + Example: + mlc reindex # Reindex all targets + mlc reindex script # Reindex only script target + mlc reindex cache # Reindex only cache target + """ + reindex_target = i.get('reindex_target') + + if not reindex_target or reindex_target == 'all' or reindex_target == 'repos' or reindex_target == 'repo': + # Reindex all targets + logger.info("Reindexing all targets (script, cache, experiment)...") + index = self.get_index() + index.build_index(force_rebuild=True) + + logger.info("Successfully reindexed all targets.") + return {'return': 0, 'message': 'All targets reindexed successfully'} + else: + + logger.info(f"Reindexing {reindex_target} target...") + index = self.get_index() + + # Clear the specific index + ''' + if reindex_target in index.indices: + index.indices[reindex_target] = [] + # Clear modified times for this target type only + keys_to_remove = [k for k in index.modified_times.keys() if reindex_target in k] + for key in keys_to_remove: + del index.modified_times[key] + ''' + + # Rebuild the index (we are rebuilding for all targets here as the individual target rebuild is not implemented and not very critical) + index.build_index(force_rebuild=True) + + logger.info(f"Successfully reindexed {reindex_target} target.") + return {'return': 0, 'message': f'{reindex_target} target reindexed successfully'} + default_parent = None if not default_parent: default_parent = Action() diff --git a/mlc/index.py b/mlc/index.py index 5928197c1..3c9dae003 100644 --- a/mlc/index.py +++ b/mlc/index.py @@ -194,7 +194,7 @@ def _index_single_repo(self, repo, repos_changed=False, current_item_keys=None): self._remove_index_entry(automation_path) if delete_flag: - self._save_indices() + changed = True continue if current_item_keys is not None: current_item_keys.add(config_path) @@ -216,7 +216,7 @@ def _index_single_repo(self, repo, repos_changed=False, current_item_keys=None): return changed - def build_index(self): + def build_index(self, force_rebuild=False): """ Build shared indices for script, cache, and experiment folders across all repositories. @@ -227,20 +227,22 @@ def build_index(self): # track all currently detected item paths current_item_keys = set() changed = False - force_rebuild = False # load modified times self.modified_times = self._load_modified_times() - # if missing index file, then force full rebuild - index_json_path = os.path.join(self.repos_path, "index_script.json") - if not os.path.exists(index_json_path): - logger.warning("index_script.json missing. Forcing full index rebuild...") + # if any index file is missing, force full rebuild + missing_indices = [] + for index_type, index_path in self.index_files.items(): + if not os.path.exists(index_path): + missing_indices.append(index_type) + + if missing_indices: + logger.warning(f"Missing index files: {', '.join(missing_indices)}. Forcing full index rebuild...") self.modified_times = {} self.indices = {k: [] for k in self.index_files.keys()} force_rebuild = True - - + # index each repo for repo in self.repos: repo_changed = self._index_single_repo(repo, force_rebuild, current_item_keys) @@ -250,10 +252,10 @@ def build_index(self): # remove deleted scripts deleted_keys = set(self.modified_times) - current_item_keys for key in deleted_keys: - logger.warning(f"Detected deleted item, removing entry from modified times: {key}") - del self.modified_times[key] folder_key = os.path.dirname(key) - #logger.warning(f"Removing index entry for folder: {folder_key}") + logger.warning(f"Detected deleted item: {key}") + logger.debug(f"Removing index entry for folder: {folder_key}") + del self.modified_times[key] self._remove_index_entry(folder_key) changed = True if deleted_keys: @@ -265,14 +267,18 @@ def build_index(self): self._save_indices() def _remove_index_entry(self, key): - logger.debug(f"Removing index entry for {key}") + logger.debug(f"Removing index entry for path: {key}") # Normalize paths for comparison normalized_key = os.path.normpath(key) for ft in self.indices: + original_count = len(self.indices[ft]) self.indices[ft] = [ item for item in self.indices[ft] if os.path.normpath(item["path"]) != normalized_key ] + removed_count = original_count - len(self.indices[ft]) + if removed_count > 0: + logger.debug(f"Removed {removed_count} item(s) from {ft} index") def _delete_by_uid(self, folder_type, uid, alias): """ diff --git a/mlc/main.py b/mlc/main.py index 7cb040ac9..ecb4b395f 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -110,10 +110,19 @@ def process_console_output(res, target, action, run_args): logger.error("'list' entry not found in find result") return # Exit function if there's an error if len(res['list']) == 0: - logger.warning(f"""No {target} entry found for the specified input: {run_args}!""") + # Only show warning if not in path-only mode + if not run_args.get('path_only'): + logger.warning(f"""No {target} entry found for the specified input: {run_args}!""") else: for item in res['list']: - logger.info(f"""Item path: {item.path}""") + if run_args.get('path_only'): + # Print only the path without logger prefix for script-friendly output + print(item.path) + else: + logger.info(f"""Item path: {item.path}""") + if action == "reindex": + if "message" in res: + logger.info(res['message']) if "warnings" in res: logger.warning(f"{len(res['warnings'])} warning(s) found during the execution of the mlc command.") for warning in res["warnings"]: @@ -145,7 +154,7 @@ def convert_hyphen_to_underscore_in_args(): def build_pre_parser(): pre_parser = argparse.ArgumentParser(add_help=False) pre_parser.add_argument("action", nargs="?", help="Top-level action (run, build, help, etc.)") - pre_parser.add_argument("target", choices=['run', 'script', 'cache', 'repo', 'repos'], nargs="?", help="Target (repo, script, cache, ...)") + pre_parser.add_argument("target", choices=['run', 'script', 'cache', 'repo', 'repos', 'experiment', 'all'], nargs="?", help="Target (repo, script, cache, ...)") pre_parser.add_argument("-h", "--help", action="store_true") return pre_parser @@ -161,6 +170,12 @@ def build_parser(pre_args): p.add_argument('details', nargs='?', help='Details or identifier (optional)') p.add_argument('extra', nargs=argparse.REMAINDER) + # Reindex command (target is optional) + reindex_parser = subparsers.add_parser('reindex', add_help=False) + reindex_parser.add_argument('target', nargs='?', choices=['repo', 'repos', 'script', 'cache', 'experiment', 'all'], help='Target to reindex (optional, defaults to all)') + reindex_parser.add_argument('details', nargs='?', help='Details or identifier (optional)') + reindex_parser.add_argument('extra', nargs=argparse.REMAINDER) + # Script-only for action in ['docker', 'docker-run', 'experiment', 'remote-run', 'doc', 'lint']: p = subparsers.add_parser(action, add_help=False) @@ -215,6 +230,14 @@ def build_run_args(args): if args.extra: run_args['dest'] = args.extra[0] + if hasattr(args, 'command') and args.command == "reindex": + if hasattr(args, 'target') and args.target: + run_args['reindex_target'] = args.target + + # Check for path-only flag (for script-friendly output) + if run_args.get('path_only') or run_args.get('p'): + run_args['path_only'] = True + return run_args def is_quoted(arg): @@ -289,7 +312,8 @@ def main(): pre_args, remaining_args = pre_parser.parse_known_args() parser = build_parser(pre_args) - args = parser.parse_args() if remaining_args or pre_args.target else pre_args + # Force full parsing for reindex command even without target, or if there are remaining args or target + args = parser.parse_args() if (remaining_args or pre_args.target or pre_args.action == 'reindex') else pre_args if hasattr(args, 'command') and args.command: args.command = args.command.replace("-", "_") @@ -332,8 +356,19 @@ def main(): print(help_text) sys.exit(0) - if args.target == "repos": + if hasattr(args, 'target') and args.target == "repos": args.target = "repo" + + # Handle reindex command specially - it can work without a target or with 'all' + if hasattr(args, 'command') and args.command == "reindex": + if not hasattr(args, 'target') or not args.target or args.target == "all": + # Reindex all targets by using the base Action class + args.target = "script" # Use script as default to get access to the action + + # Check if command attribute exists + if not hasattr(args, 'command'): + logging.error("Error: No command specified.") + sys.exit(1) action = get_action(args.target, default_parent) diff --git a/pyproject.toml b/pyproject.toml index 3f9e7fda8..457c2f7e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mlcflow" -version = "1.1.19" +version = "1.1.20" description = "An automation interface tailored for CPU/GPU benchmarking" authors = [ From 7eadf8dd7e22a927512b4951b8566c22dc25574b Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sun, 22 Mar 2026 17:15:58 +0530 Subject: [PATCH 06/27] Add MLCFlow generic installer script for Linux (#220) --- .github/workflows/test-installer-curl.yml | 516 ++++++++++++++++++++++ docs/install/README.md | 375 ++++++++++++++++ docs/install/mlcflow_linux.sh | 419 ++++++++++++++++++ 3 files changed, 1310 insertions(+) create mode 100644 .github/workflows/test-installer-curl.yml create mode 100644 docs/install/README.md create mode 100755 docs/install/mlcflow_linux.sh diff --git a/.github/workflows/test-installer-curl.yml b/.github/workflows/test-installer-curl.yml new file mode 100644 index 000000000..ee9d625b7 --- /dev/null +++ b/.github/workflows/test-installer-curl.yml @@ -0,0 +1,516 @@ +name: Test MLCFlow Installer + +on: + pull_request: + branches: + - main + - dev + paths: + - 'docs/install/mlcflow_linux.sh' + - '.github/workflows/test-installer-curl.yml' + workflow_dispatch: + +# Only allow one workflow run per PR to conserve CI resources +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true # Might want to discuss on this - this saves the CI minutes but the information about what caused error in previous commit might be lost if multiple commits are pushed in quick succession. We can consider changing this to "false" if we want to preserve all test results for all commits. + +jobs: + # =========================================================================== + # Test Matrix: Native GitHub Actions Runners + # =========================================================================== + # These tests run on native GitHub Actions runners for Ubuntu and macOS. + # The installer is downloaded via curl and piped to bash, exactly as users + # will execute it in production. + test-native-runners: + name: Test on ${{ matrix.os }} (${{ matrix.scenario }}) + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + # =================================================================== + # Ubuntu 22.04 LTS Tests + # =================================================================== + # Test basic installation on Ubuntu 22.04 + - os: Ubuntu 22.04 + runner: ubuntu-22.04 + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + # Test quiet mode with minimal output + - os: Ubuntu 22.04 + runner: ubuntu-22.04 + scenario: quiet-mode + test-type: success + extra-flags: "--yes --quiet" + description: "Quiet mode with minimal output" + + # Test upgrade mode on existing installation + - os: Ubuntu 22.04 + runner: ubuntu-22.04 + scenario: upgrade-mode + test-type: success + extra-flags: "--yes --upgrade" + description: "Upgrade existing installation" + + # =================================================================== + # Ubuntu 24.04 LTS Tests + # =================================================================== + # Test basic installation on latest Ubuntu LTS + - os: Ubuntu 24.04 + runner: ubuntu-latest + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + # =================================================================== + # macOS Tests + # =================================================================== + # Test basic installation on macOS with Homebrew + - os: macos-latest + runner: macos-latest + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + # Test custom venv directory on macOS + - os: macos-latest + runner: macos-latest + scenario: custom-venv + test-type: success + extra-flags: "--yes --venv-dir /tmp/custom_mlcflow_venv" + description: "Custom virtual environment directory" + + steps: + # Determine the source URL based on the event type + # For pull_request: use the PR head branch + # For workflow_dispatch: use dev branch from mlcommons/mlcflow + - name: Determine Installer Script URL + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + # For PRs, use the head branch from the PR + OWNER="${{ github.event.pull_request.head.repo.owner.login }}" + REPO="${{ github.event.pull_request.head.repo.name }}" + BRANCH="${{ github.event.pull_request.head.ref }}" + else + # For workflow_dispatch and other events, use dev branch + OWNER="mlcommons" + REPO="mlcflow" + BRANCH="anandhu-eng-patch-1" #"dev" + fi + + INSTALLER_URL="https://raw.githubusercontent.com/${OWNER}/${REPO}/refs/heads/${BRANCH}/docs/install/mlcflow_linux.sh" + echo "INSTALLER_URL=$INSTALLER_URL" >> $GITHUB_ENV + echo "✅ Installer URL: $INSTALLER_URL" + + # ===================================================================== + # Test Case: Install via Curl-Pipe Method + # ===================================================================== + # This is the primary test that validates the installer using the exact + # method that users will use: curl /mlcflow_linux.sh | bash -s -- + - name: "Test: ${{ matrix.description }}" + run: | + echo "==========================================" + echo "Test: ${{ matrix.description }}" + echo "OS: ${{ matrix.os }}" + echo "Scenario: ${{ matrix.scenario }}" + echo "Extra Flags: ${{ matrix.extra-flags }}" + echo "Installer URL: $INSTALLER_URL" + echo "==========================================" + + # Download and execute the installer via curl-pipe method from GitHub + # This is the EXACT method that users will use in production + echo "Downloading and executing installer via curl-pipe method..." + if curl -sSL "$INSTALLER_URL" | bash -s -- ${{ matrix.extra-flags }}; then + echo "✅ Installer completed successfully" + INSTALL_SUCCESS=true + else + EXIT_CODE=$? + echo "❌ Installer failed with exit code: $EXIT_CODE" + INSTALL_SUCCESS=false + fi + + # Store result for validation step + echo "INSTALL_SUCCESS=$INSTALL_SUCCESS" >> $GITHUB_ENV + + # ===================================================================== + # Validate Installation Success + # ===================================================================== + # After installation completes, verify that expected artifacts exist + - name: Validate Installation Artifacts + if: env.INSTALL_SUCCESS == 'true' + run: | + echo "==========================================" + echo "Validating Installation Artifacts" + echo "==========================================" + + # Determine the venv directory based on test scenario + if [[ "${{ matrix.scenario }}" == "custom-venv" ]]; then + VENV_DIR="/tmp/custom_mlcflow_venv" + else + VENV_DIR="$HOME/.mlcflow_venv" + fi + + echo "Expected venv directory: $VENV_DIR" + + # Validate 1: Virtual environment directory exists + if [ -d "$VENV_DIR" ]; then + echo "✅ Virtual environment directory exists" + else + echo "❌ Virtual environment directory not found" + exit 1 + fi + + # Validate 2: Activation script exists + if [ -f "$VENV_DIR/bin/activate" ]; then + echo "✅ Virtual environment activation script exists" + else + echo "❌ Activation script not found" + exit 1 + fi + + # Validate 3: Python executable exists in venv + if [ -f "$VENV_DIR/bin/python3" ] || [ -f "$VENV_DIR/bin/python" ]; then + echo "✅ Python executable found in virtual environment" + else + echo "❌ Python executable not found in virtual environment" + exit 1 + fi + + # Validate 4: MLCFlow package is installed + if source "$VENV_DIR/bin/activate" && python3 -c "import mlcflow" 2>/dev/null; then + echo "✅ MLCFlow package is importable" + else + echo "⚠️ MLCFlow package may not be fully installed (repo cloning may have failed)" + fi + + # Validate 5: mlc CLI command is available after activation + if source "$VENV_DIR/bin/activate" && command -v mlc >/dev/null 2>&1; then + echo "✅ mlc CLI command is available" + + # Try to execute a harmless command to verify CLI works + if mlc help >/dev/null 2>&1; then + echo "✅ mlc help command executed successfully" + else + echo "⚠️ mlc command exists but may not be fully functional" + fi + else + echo "⚠️ mlc CLI command not found (may be due to repo cloning issues)" + fi + + # Validate 6: Check if automation repository was cloned + if [ -d "$HOME/MLC/repos" ]; then + echo "✅ MLC repos directory exists" + echo "Contents:" + ls -la "$HOME/MLC/repos" || true + else + echo "⚠️ MLC repos directory not found (repo cloning may have failed)" + fi + + # ===================================================================== + # Verify Expected Test Outcome + # ===================================================================== + # Confirm that the test result matches the expected outcome + - name: Verify Test Outcome + run: | + echo "==========================================" + echo "Verifying Test Outcome" + echo "==========================================" + + EXPECTED_RESULT="${{ matrix.test-type }}" + ACTUAL_SUCCESS="${{ env.INSTALL_SUCCESS }}" + + echo "Expected: $EXPECTED_RESULT" + echo "Actual Success: $ACTUAL_SUCCESS" + + if [[ "$EXPECTED_RESULT" == "success" && "$ACTUAL_SUCCESS" == "true" ]]; then + echo "✅ Test passed: Installation succeeded as expected" + exit 0 + elif [[ "$EXPECTED_RESULT" == "failure" && "$ACTUAL_SUCCESS" == "false" ]]; then + echo "✅ Test passed: Installation failed as expected" + exit 0 + else + echo "❌ Test failed: Unexpected outcome" + exit 1 + fi + + # =========================================================================== + # Test Matrix: Docker Containers for Non-Native Distributions + # =========================================================================== + # These tests run inside Docker containers for Linux distributions that + # don't have native GitHub Actions runners (Debian, RHEL-family). + test-docker-containers: + name: Test on ${{ matrix.os }} (${{ matrix.scenario }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # =================================================================== + # Ubuntu 20.04 LTS Tests + # =================================================================== + # Test basic non-interactive installation with all default settings + - os: Ubuntu 20.04 + container: ubuntu:20.04 + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + # Test installation with custom venv directory + - os: Ubuntu 20.04 + container: ubuntu:20.04 + scenario: custom-venv + test-type: success + extra-flags: "--yes --venv-dir /tmp/custom_mlcflow_venv" + description: "Custom virtual environment directory" + + # =================================================================== + # Debian 11 Tests + # =================================================================== + # Test basic installation on Debian 11 (Bullseye) + - os: Debian 11 + container: debian:11 + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + # Test custom venv directory on Debian + - os: Debian 11 + container: debian:11 + scenario: custom-venv + test-type: success + extra-flags: "--yes --venv-dir /tmp/custom_mlcflow_venv" + description: "Custom virtual environment directory" + + # =================================================================== + # Debian 12 Tests + # =================================================================== + # Test basic installation on Debian 12 (Bookworm) + - os: Debian 12 + container: debian:12 + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + # =================================================================== + # Rocky Linux 9 Tests + # =================================================================== + # Test basic installation on Rocky Linux 9 (RHEL-compatible) + - os: Rocky Linux 9 + container: rockylinux:9 + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + # Test verbose mode on Rocky Linux + - os: Rocky Linux 9 + container: rockylinux:9 + scenario: verbose-mode + test-type: success + extra-flags: "--yes --verbose" + description: "Verbose logging mode" + + # =================================================================== + # AlmaLinux 9 Tests + # =================================================================== + # Test basic installation on AlmaLinux 9 (RHEL-compatible) + - os: AlmaLinux 9 + container: almalinux:9 + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + # =================================================================== + # CentOS Stream 9 Tests + # =================================================================== + # Test basic installation on CentOS Stream 9 (RHEL-compatible) + - os: CentOS Stream 9 + container: quay.io/centos/centos:stream9 + scenario: basic-install + test-type: success + extra-flags: "--yes" + description: "Basic non-interactive installation" + + steps: + # Determine the source URL based on the event type + - name: Determine Installer Script URL + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + OWNER="${{ github.event.pull_request.head.repo.owner.login }}" + REPO="${{ github.event.pull_request.head.repo.name }}" + BRANCH="${{ github.event.pull_request.head.ref }}" + else + OWNER="mlcommons" + REPO="mlcflow" + BRANCH="anandhu-eng-patch-1" #"dev" + fi + + INSTALLER_URL="https://raw.githubusercontent.com/${OWNER}/${REPO}/refs/heads/${BRANCH}/docs/install/mlcflow_linux.sh" + echo "INSTALLER_URL=$INSTALLER_URL" >> $GITHUB_ENV + echo "✅ Installer URL: $INSTALLER_URL" + + # ===================================================================== + # Test Case: Install via Curl-Pipe Method in Docker Container + # ===================================================================== + # This test runs the installer inside a Docker container that represents + # the target Linux distribution. The installer is downloaded via curl + # from GitHub and piped to bash, exactly as users will execute it. + - name: "Test: ${{ matrix.description }} in ${{ matrix.os }}" + run: | + echo "==========================================" + echo "Test: ${{ matrix.description }}" + echo "OS: ${{ matrix.os }}" + echo "Container: ${{ matrix.container }}" + echo "Scenario: ${{ matrix.scenario }}" + echo "Extra Flags: ${{ matrix.extra-flags }}" + echo "Installer URL: $INSTALLER_URL" + echo "==========================================" + + # Run the installer inside the Docker container via curl-pipe method + # Downloads directly from GitHub, testing the real distribution method + docker run --rm \ + ${{ matrix.container }} \ + bash -c " + # Install curl if not present (required for downloading the script) + if ! command -v curl >/dev/null 2>&1; then + if command -v apt-get >/dev/null 2>&1; then + apt-get update -qq && apt-get install -y -qq curl + elif command -v dnf >/dev/null 2>&1; then + dnf install -y -q curl-minimal + elif command -v yum >/dev/null 2>&1; then + yum install -y -q curl-minimal + fi + fi + + # Download and execute installer via curl-pipe method from GitHub + echo '=== Downloading and executing installer via curl-pipe method ===' + curl -sSL '$INSTALLER_URL' | bash -s -- ${{ matrix.extra-flags }} + " && DOCKER_EXIT_CODE=0 || DOCKER_EXIT_CODE=$? + + # Evaluate the result + if [ $DOCKER_EXIT_CODE -eq 0 ]; then + echo "✅ Installer completed successfully in container" + echo "INSTALL_SUCCESS=true" >> $GITHUB_ENV + else + echo "❌ Installer failed in container with exit code: $DOCKER_EXIT_CODE" + echo "INSTALL_SUCCESS=false" >> $GITHUB_ENV + fi + + # ===================================================================== + # Validate Installation Inside Container + # ===================================================================== + # After installation completes, run the container again to verify artifacts + - name: Validate Installation Artifacts in Container + if: env.INSTALL_SUCCESS == 'true' + run: | + echo "==========================================" + echo "Validating Installation Artifacts" + echo "==========================================" + + # Determine the venv directory based on test scenario + if [[ "${{ matrix.scenario }}" == "custom-venv" ]]; then + VENV_DIR="/tmp/custom_mlcflow_venv" + else + VENV_DIR="/root/.mlcflow_venv" + fi + + echo "Expected venv directory: $VENV_DIR" + + # Run validation commands inside a new container instance + # Note: The previous container is ephemeral, so we validate the + # installation success based on exit code rather than persistent state + docker run --rm ${{ matrix.container }} bash -c " + echo '=== Validation Complete ===' + echo 'Note: Container installations are ephemeral in CI.' + echo 'Success is determined by installer exit code.' + " + + echo "✅ Container installation validation complete" + + # ===================================================================== + # Verify Expected Test Outcome + # ===================================================================== + - name: Verify Test Outcome + run: | + echo "==========================================" + echo "Verifying Test Outcome" + echo "==========================================" + + EXPECTED_RESULT="${{ matrix.test-type }}" + ACTUAL_SUCCESS="${{ env.INSTALL_SUCCESS }}" + + echo "Expected: $EXPECTED_RESULT" + echo "Actual Success: $ACTUAL_SUCCESS" + + if [[ "$EXPECTED_RESULT" == "success" && "$ACTUAL_SUCCESS" == "true" ]]; then + echo "✅ Test passed: Installation succeeded as expected" + exit 0 + elif [[ "$EXPECTED_RESULT" == "failure" && "$ACTUAL_SUCCESS" == "false" ]]; then + echo "✅ Test passed: Installation failed as expected" + exit 0 + else + echo "❌ Test failed: Unexpected outcome" + exit 1 + fi + + # =========================================================================== + # Final Test Summary + # =========================================================================== + test-summary: + name: Test Summary + runs-on: ubuntu-latest + needs: + - test-native-runners + - test-docker-containers + if: always() + steps: + - name: Generate Test Summary + run: | + echo "==========================================" + echo " MLCFlow Installer CI Test Summary" + echo "==========================================" + echo "" + echo "This CI workflow validates the MLCFlow installer by downloading" + echo "directly from GitHub and using the exact distribution method that" + echo "users will use in production:" + echo " curl -sSL /mlcflow_linux.sh | bash" + echo "" + echo "✅ Test Coverage:" + echo " - Native GitHub runners (Ubuntu, macOS)" + echo " - Docker containers (Debian, Rocky, Alma, CentOS)" + echo " - Multiple installation modes (basic, custom, verbose, quiet)" + echo "" + echo "⚠️ Known Limitation:" + echo " - Only non-interactive mode is tested in CI" + echo " - Interactive prompts are not covered by automated tests" + echo "" + echo "==========================================" + echo "" + + # Check if all required jobs succeeded + NATIVE_STATUS="${{ needs.test-native-runners.result }}" + DOCKER_STATUS="${{ needs.test-docker-containers.result }}" + + echo "Job Results:" + echo " Native Runners: $NATIVE_STATUS" + echo " Docker Containers: $DOCKER_STATUS" + echo "" + + if [[ "$NATIVE_STATUS" == "success" && \ + "$DOCKER_STATUS" == "success" ]]; then + echo "Result: ✅ ALL TESTS PASSED" + exit 0 + else + echo "Result: ❌ SOME TESTS FAILED" + exit 1 + fi diff --git a/docs/install/README.md b/docs/install/README.md new file mode 100644 index 000000000..2070132ee --- /dev/null +++ b/docs/install/README.md @@ -0,0 +1,375 @@ +# MLCFlow Unix Installer + +A Bootstrap installer for MLCFlow that automatically detects your Unix-based operating system (Linux/macOS), installs required dependencies, sets up a Python virtual environment, and configures the MLCFLow automation framework. + +> **Platform Note**: This installer is designed for **Unix-based systems only** (Linux distributions and macOS). It does not support Windows. Windows users should use WSL2 (Windows Subsystem for Linux) or a Linux virtual machine. We plan to release an installer script for Windows soon. + +## Purpose + +This installer provides a **one-command setup** for the MLCFlow package and the MLPerf automation repository. It handles all the complexity of: +- Detecting your Linux distribution or macOS +- Automatically detecting and using sudo when needed +- Installing missing system packages +- Validating Python installation and version +- Setting up isolated Python environments +- Installing MLCFlow and its dependencies +- Cloning the automation repository + +**After installation, activate the virtual environment** to use MLCFlow commands: +```bash +source ~/.mlcflow_venv/bin/activate +``` + +## Supported Platforms + +### Linux Distributions +- **Ubuntu**: 20.04, 22.04, 24.04 LTS +- **Debian**: 10+ (any version with Python 3.7+) +- **Rocky Linux**: 9.x +- **AlmaLinux**: 9.x +- **CentOS Stream**: 9 +- **RHEL**: 9+ (Red Hat Enterprise Linux) + +### macOS +- **macOS**: 11+ (Big Sur and later) +- **Requirement**: Homebrew must be installed + +### Architecture +- **x86_64** (amd64) +- **aarch64** (arm64) + +### Python Version +- **Minimum**: Python 3.7 +- **Recommended**: Python 3.8+ + +> **Note**: RHEL/Rocky Linux 8 ships with Python 3.6 by default, which is below the minimum requirement. Users should install Python 3.8+ from AppStream modules or alternative sources. + +## Installation Flow + +``` +┌─────────────────────────────────────────────────────────────┐ +│ 1. Detect Operating System & Package Manager │ +│ ├─ Ubuntu/Debian → apt │ +│ ├─ RHEL/Rocky/Alma → dnf/yum │ +│ └─ macOS → brew │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 2. Check System Dependencies │ +│ - git, curl/curl-minimal, wget, unzip │ +│ - python3, python3-pip, python3-venv │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 3. Install Missing Dependencies │ +│ - Uses detected package manager │ +│ - Requests sudo/root privileges if needed │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 4. Validate Python Environment │ +│ - Check Python version (≥ 3.7) │ +│ - Verify pip module availability │ +│ - Verify venv module availability │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 5. Create Virtual Environment │ +│ - Location: ~/.mlcflow_venv (or custom) │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 6. Install MLCFlow Package │ +│ - Install mlcflow via pip │ +│ - Install all dependencies │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 7. Prompt for Repository Details (if interactive) │ +│ - Repo name (default: mlcommons@mlperf-automations) │ +│ - Branch (default: dev) │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 8. Clone Automation Repository │ +│ - Uses 'mlc pull repo' command │ +│ - Stored in ~/MLC/repos/ (mlc default location) │ +└─────────────────────────────────────────────────────────────┘ + ↓ + ✅ Installation Complete +``` + +## Usage + +### Basic Installation (Interactive) +```bash +bash mlcflow_linux.sh +``` +Prompts for repository name and branch, then proceeds with installation. + +### Automated Installation (Non-Interactive) +```bash +bash mlcflow_linux.sh --yes +``` +Uses all default values without prompting. + +### Custom Virtual Environment Location +```bash +bash mlcflow_linux.sh --yes --venv-dir /opt/mlcflow_env +``` + +### Custom Repository and Branch +```bash +bash mlcflow_linux.sh \ + --yes \ + --mlc-repo myorganization@my-mlperf-fork \ + --mlc-repo-branch feature-branch +``` + +### Upgrade Existing Installation +```bash +bash mlcflow_linux.sh --yes --upgrade +``` + +### Verbose Mode (Debugging) +```bash +bash mlcflow_linux.sh --yes --verbose +``` + +### Quiet Mode (Minimal Output) +```bash +bash mlcflow_linux.sh --yes --quiet +``` + +## Command-Line Arguments + +| Argument | Description | Default | +|----------|-------------|---------| +| `--yes` | Auto-confirm all prompts (non-interactive mode) | Interactive | +| `--upgrade` | Upgrade mlcflow if already installed | Skip if present | +| `--venv-dir ` | Custom virtual environment directory | `~/.mlcflow_venv` | +| `--mlc-repo ` | Repository in format `owner@repo` | `mlcommons@mlperf-automations` | +| `--mlc-repo-branch ` | Git branch to clone | `dev` | +| `--install-python` | Auto-install Python if incompatible | Prompt user | +| `--verbose` | Enable debug logging | Normal logging | +| `--quiet` | Minimal output (errors/warnings only) | Normal logging | +| `--help` | Display help message and exit | - | + +## SUDO and Privilege Handling + +The installer automatically detects your privilege level and handles system package installation accordingly. **You do not need to explicitly specify sudo** - the script handles it internally. + +### How the Script Detects Privileges + +When you run the script, it automatically checks: +1. **Are you running as root?** (EUID == 0) +2. **Is the `sudo` command available?** +3. **What package manager is being used?** + +Based on this detection, it chooses the appropriate execution method. + + +### Privilege Detection Logic +```bash +# The installer detects privileges in this order: +1. Check if running as root (EUID == 0) +2. Check if 'sudo' command is available +3. For package managers: + - apt/yum/dnf: Require root or sudo + - brew (macOS): No sudo needed +4. For Python operations: No privileges required (user-space) +``` + +### Summary: What Requires Privileges? + +| Operation | Requires Root/Sudo | +|-----------|-------------------| +| Install system packages | Yes | +| Install system packages | Yes | +| Install system packages | +| Create Python venv | No | +| Install pip packages | No | +| Clone git repositories | No | + +## What Gets Installed + +### System Packages +**Ubuntu/Debian**: +- `python3`, `python3-pip`, `python3-venv` +- `git`, `curl`, `wget`, `unzip` + +**RHEL/Rocky/Alma/CentOS**: +- `python3`, `python3-pip`, `python3-venv` +- `git`, `curl-minimal`, `wget`, `unzip` + +> **Note**: Uses `curl-minimal` on RHEL systems to avoid package conflicts with pre-installed `curl-minimal`. + +**macOS (via Homebrew)**: +- `python`, `git`, `curl`, `wget`, `unzip` + +### Python Packages (in virtual environment) +- `mlcflow` - Main automation CLI +- `requests` - HTTP library +- `pyyaml` - YAML parser +- `giturlparse` - Git URL utilities +- `colorama` - Cross-platform colored terminal output + +### Automation Repository +- Cloned via `mlc pull repo` command +- Default location: `~/MLC/repos/mlcommons@mlperf-automations/` +- Contains MLPerf automation scripts and configurations + +## Post-Installation + +### Activate Virtual Environment (Required) + +**Important**: To use MLCFlow commands, you must activate the virtual environment: + +```bash +# Activate the virtual environment +source ~/.mlcflow_venv/bin/activate + +# Your prompt should change to show (mlcflow_venv) or similar +``` + +### Verify Installation +```bash +# After activating the virtual environment: +mlc --help + +# Verify version +mlc version + +# List available scripts +mlc list scripts +``` + +### Deactivate Virtual Environment +```bash +# When you're done working with MLCFlow +deactivate +``` + +### File Locations +- **Virtual Environment**: `~/.mlcflow_venv` (or custom path) +- **Automation Repository**: `~/MLC/repos/mlcommons@mlperf-automations/` +- **MLC Cache**: `~/MLC/repos/` + +## Troubleshooting + +### "Python version < 3.7" +**Problem**: System Python is too old (e.g., Python 3.6 on RHEL 8) + +**Solution**: +```bash +# RHEL 8 / Rocky 8 +sudo dnf module install python38 +# or +sudo dnf install python39 + +# Then rerun installer +bash mlcflow_linux.sh --yes +``` + +### "mlc: command not found" after installation +**Problem**: Virtual environment is not activated + +**Solution**: +```bash +# Activate the virtual environment +source ~/.mlcflow_venv/bin/activate + +# Now mlc command should be available +mlc --help +``` + +### Package conflicts on RHEL/Rocky +**Problem**: `curl` and `curl-minimal` conflict + +**Solution**: The installer now uses `curl-minimal` automatically. If you encounter issues: +```bash +# Remove conflicting package +sudo dnf remove curl + +# Rerun installer +bash mlcflow_linux.sh --yes +``` + +### "Homebrew not found" on macOS +**Problem**: Homebrew is not installed + +**Solution**: +```bash +# Install Homebrew +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + +# Rerun installer +bash mlcflow_linux.sh --yes +``` + +### Permission denied errors +**Problem**: Missing sudo privileges and required packages are not installed + +**Solution**: +```bash +# If you don't have sudo AND packages are missing +# Ask your system administrator to install these packages: +# - Ubuntu/Debian: python3 python3-pip python3-venv git curl wget unzip +# - RHEL/Rocky: python3 python3-pip python3-venv git curl-minimal wget unzip +# Then run: +bash mlcflow_linux.sh --yes +``` + +### Virtual environment creation fails +**Problem**: `python3-venv` module missing + +**Solution**: The installer should detect and install it, but if manual installation is needed: +```bash +# Ubuntu/Debian +sudo apt install python3-venv + +# RHEL/Rocky +sudo dnf install python3-venv + +# Rerun installer +bash mlcflow_linux.sh --yes +``` + +## Testing and CI + +### Automated Testing + +The installer is continuously tested via GitHub Actions across all supported platforms using the exact distribution method that users will use in production (`curl /mlcflow_linux.sh | bash`). The CI workflow validates: + +- **Operating Systems**: Ubuntu 20.04/22.04/24.04, macOS 13, Debian 11/12, Rocky Linux 9, AlmaLinux 9, CentOS Stream 9 + +### Known Testing Limitations + +**TODO**: The CI workflow currently only tests **non-interactive mode** (with `--yes` flag). The interactive installation path, which prompts users for repository name and branch, is not covered by automated tests. + +If you discover issues with interactive mode, please report them via GitHub Issues [here](https://github.com/mlcommons/mlcflow/issues). + +## Support + +For issues, feature requests, or contributions: +- **Repository**: https://github.com/mlcommons/mlperf-automations +- **Issues**: https://github.com/mlcommons/mlperf-automations/issues +- **Documentation**: https://mlcommons.github.io/mlperf-automations/ + +## License + +This installer script is part of the MLPerf Automations project and is distributed under the same license as the main project. + +## Changelog + +### v1.0 (Current) +- Initial release with comprehensive OS support (Linux/macOS) +- Automatic dependency detection and installation +- pip/venv module validation +- Automatic sudo detection and privilege handling +- macOS support with Homebrew +- Interactive and non-interactive modes +- RHEL package conflict resolution (curl-minimal) +- Manual virtual environment activation required + +--- diff --git a/docs/install/mlcflow_linux.sh b/docs/install/mlcflow_linux.sh new file mode 100755 index 000000000..59e3addbd --- /dev/null +++ b/docs/install/mlcflow_linux.sh @@ -0,0 +1,419 @@ +#!/usr/bin/env bash +# ============================================================================== +# MLCFlow Generic Installer (v1) +# Supports: +# - Ubuntu 20.04+ +# - RHEL family (RHEL, Alma, CentOS Stream) +# - macOS (with Homebrew) +# - x86_64 and aarch64 + +# Exit if a command fails +# Treats unset variables as errors +# Makes pipeline fails if any command fails +set -euo pipefail + +# ------------------------------------------------------------------------------ +# Default Configuration +# ------------------------------------------------------------------------------ + +MIN_PYTHON_VERSION="3.7" +DEFAULT_VENV_DIR="$HOME/.mlcflow_venv" +DEFAULT_REPO="mlcommons@mlperf-automations" +DEFAULT_BRANCH="dev" + +UPGRADE=false +ASSUME_YES=false +INSTALL_PYTHON=false +VERBOSE=false +QUIET=false +VENV_DIR="$DEFAULT_VENV_DIR" +MLC_REPO="$DEFAULT_REPO" +MLC_BRANCH="$DEFAULT_BRANCH" + +# ------------------------------------------------------------------------------ +# Logging System +# ------------------------------------------------------------------------------ + +INTERACTIVE=false +# checks if the stdout connected to a terminal +if [ -t 1 ]; then + INTERACTIVE=true +fi + +if $INTERACTIVE; then + COLOR_RED="\033[0;31m" + COLOR_GREEN="\033[0;32m" + COLOR_YELLOW="\033[1;33m" + COLOR_BLUE="\033[0;34m" + COLOR_RESET="\033[0m" +else + COLOR_RED="" + COLOR_GREEN="" + COLOR_YELLOW="" + COLOR_BLUE="" + COLOR_RESET="" +fi + +log_info() { + $QUIET && return + echo -e "${COLOR_GREEN}[INFO]${COLOR_RESET} $1" +} + +log_warn() { + echo -e "${COLOR_YELLOW}[WARN]${COLOR_RESET} $1" +} + +log_error() { + echo -e "${COLOR_RED}[ERROR]${COLOR_RESET} $1" +} + +log_debug() { + $VERBOSE || return + echo -e "${COLOR_BLUE}[DEBUG]${COLOR_RESET} $1" +} + +# ------------------------------------------------------------------------------ +# Usage +# ------------------------------------------------------------------------------ + +usage() { +cat < Custom virtual environment path + --mlc-repo Override automation repo + --mlc-repo-branch Override repo branch + --install-python Auto-install Python if incompatible + --verbose Enable debug logs + --quiet Minimal output + --help Show this help + +EOF +exit 0 +} + +# ------------------------------------------------------------------------------ +# Argument Parsing +# ------------------------------------------------------------------------------ + +# Loops through the arguments provided +# If an option expects a value after it, reads it and skips for next iteration +while [[ $# -gt 0 ]]; do + case "$1" in + --yes) ASSUME_YES=true ;; + --upgrade) UPGRADE=true ;; + --venv-dir) VENV_DIR="$2"; shift ;; + --mlc-repo) MLC_REPO="$2"; shift ;; + --mlc-repo-branch) MLC_BRANCH="$2"; shift ;; + --install-python) INSTALL_PYTHON=true ;; + --verbose) VERBOSE=true ;; + --quiet) QUIET=true ;; + --help) usage ;; + *) log_error "Unknown argument: $1"; exit 1 ;; + esac + shift +done + +# ------------------------------------------------------------------------------ +# Detect OS and Package Manager +# ------------------------------------------------------------------------------ + +detect_os() { + if [ "$(uname)" = "Darwin" ]; then + OS_ID="macos" + OS_VERSION="$(sw_vers -productVersion 2>/dev/null || echo unknown)" + else + if [ ! -f /etc/os-release ]; then + log_error "Cannot detect operating system." + exit 1 + fi + # loads the content from os-release as variables + source /etc/os-release + OS_ID="$ID" + OS_VERSION="$VERSION_ID" + fi + + case "$OS_ID" in + ubuntu|debian) + PKG_MANAGER="apt" + ;; + rhel|rocky|almalinux|centos) + if command -v dnf >/dev/null 2>&1; then + PKG_MANAGER="dnf" + else + PKG_MANAGER="yum" + fi + ;; + macos) + PKG_MANAGER="brew" + ;; + *) + log_error "Unsupported OS: $OS_ID" + exit 1 + ;; + esac + + log_info "Detected OS: $OS_ID $OS_VERSION" + log_info "Using package manager: $PKG_MANAGER" +} + +# ------------------------------------------------------------------------------ +# Privilege Detection +# ------------------------------------------------------------------------------ + +# Handles the following cases: +# 1. If the script is already running as root (EUID=0), commands can be executed directly. +# 2. If the script is not running as root but the `sudo` command is available, +# privileged commands will be executed using sudo. +# 3. If neither root privileges nor sudo are available, the script will fail +# when attempting to run commands that require elevated permissions. +if [ "$EUID" -eq 0 ]; then + USE_SUDO=false +elif command -v sudo >/dev/null 2>&1; then + USE_SUDO=true +else + USE_SUDO=false +fi + +run_root() { + if $USE_SUDO; then + sudo "$@" + elif [ "$EUID" -eq 0 ]; then + "$@" + else + log_error "Root or sudo required to install system dependencies." + exit 1 + fi +} + +# ------------------------------------------------------------------------------ +# System Dependencies +# ------------------------------------------------------------------------------ + +require_root_if_needed() { + if [ "$PKG_MANAGER" = "brew" ]; then + return + fi + + if [ "$EUID" -ne 0 ] && ! $USE_SUDO; then + log_error "Root or sudo required to install missing dependencies." + exit 1 + fi +} + +have_pip_module() { + python3 -m pip --version >/dev/null 2>&1 +} + +have_venv_module() { + python3 -c 'import venv' >/dev/null 2>&1 +} + +check_missing_dependencies() { + MISSING_DEPS=() + + command -v git >/dev/null 2>&1 || MISSING_DEPS+=("git") + command -v curl >/dev/null 2>&1 || MISSING_DEPS+=("curl") + command -v wget >/dev/null 2>&1 || MISSING_DEPS+=("wget") + command -v unzip >/dev/null 2>&1 || MISSING_DEPS+=("unzip") + + if ! command -v python3 >/dev/null 2>&1; then + MISSING_DEPS+=("python3") + else + have_pip_module || MISSING_DEPS+=("python3-pip") + have_venv_module || MISSING_DEPS+=("python3-venv") + fi +} + +install_packages() { + log_info "Installing system dependencies..." + + case "$PKG_MANAGER" in + apt) + require_root_if_needed + run_root apt update + run_root apt install -y python3 python3-pip python3-venv git curl wget unzip + ;; + yum|dnf) + require_root_if_needed + # RHEL-family images may ship curl-minimal and conflict with curl package. + run_root "$PKG_MANAGER" install -y python3 python3-pip git curl-minimal wget unzip + # Some RHEL-family variants package venv separately + run_root "$PKG_MANAGER" install -y python3-venv >/dev/null 2>&1 || true + ;; + brew) + if ! command -v brew >/dev/null 2>&1; then + log_error "Homebrew is required on macOS. Please install it from https://brew.sh" + exit 1 + fi + brew update + brew install python git curl wget unzip + ;; + esac +} + +# ------------------------------------------------------------------------------ +# Python Validation +# ------------------------------------------------------------------------------ + +version_ge() { + [ "$(printf '%s\n' "$2" "$1" | sort -V | head -n1)" = "$2" ] +} + +ensure_python() { + if ! command -v python3 >/dev/null 2>&1; then + log_warn "Python3 not found." + handle_python_install + fi + + if ! command -v python3 >/dev/null 2>&1; then + log_error "Python3 is still unavailable after attempted installation." + exit 1 + fi + + PY_VERSION=$(python3 -c 'import sys; print(".".join(map(str, sys.version_info[:3])))') + log_info "Detected Python version: $PY_VERSION" + + if version_ge "$PY_VERSION" "$MIN_PYTHON_VERSION"; then + log_info "Python version is compatible." + else + log_warn "Python version < $MIN_PYTHON_VERSION" + handle_python_install + fi + + if ! have_pip_module; then + log_warn "python3 pip module is missing. Installing..." + install_packages + fi + + if ! have_venv_module; then + log_warn "python3 venv module is missing. Installing..." + install_packages + fi + + if ! have_pip_module || ! have_venv_module; then + log_error "pip/venv modules are still missing after attempted installation." + exit 1 + fi +} + +handle_python_install() { + if $INSTALL_PYTHON || $ASSUME_YES; then + install_packages + return + fi + + if ! $INTERACTIVE; then + log_error "Incompatible Python and non-interactive mode. Run with --install-python to automatically install." + exit 1 + fi + + read -p "Install compatible Python? [y/N]: " response + if [[ "$response" =~ ^[Yy]$ ]]; then + install_packages + else + log_error "Cannot proceed without compatible Python." + exit 1 + fi +} + +# ------------------------------------------------------------------------------ +# Virtual Environment +# ------------------------------------------------------------------------------ + +setup_venv() { + log_info "Setting up virtual environment at: $VENV_DIR" + + if [ -d "$VENV_DIR" ]; then + log_info "Reusing existing virtual environment." + else + python3 -m venv "$VENV_DIR" + fi + + # Activate venv + # shellcheck disable=SC1090 + source "$VENV_DIR/bin/activate" +} + +# ------------------------------------------------------------------------------ +# Install / Upgrade MLCFlow +# ------------------------------------------------------------------------------ + +install_mlcflow() { + if python3 -m pip show mlcflow >/dev/null 2>&1; then + if $UPGRADE; then + log_info "Upgrading mlcflow..." + python3 -m pip install --upgrade mlcflow + else + log_info "mlcflow already installed. Skipping." + fi + else + log_info "Installing mlcflow..." + python3 -m pip install mlcflow + fi +} + +# ------------------------------------------------------------------------------ +# Pull Automation Repo +# ------------------------------------------------------------------------------ + +prompt_repo_details() { + if ! $INTERACTIVE || $ASSUME_YES; then + return + fi + + read -r -p "Automation repo [${MLC_REPO}]: " repo_input + if [ -n "${repo_input}" ]; then + MLC_REPO="${repo_input}" + fi + + read -r -p "Automation branch [${MLC_BRANCH}]: " branch_input + if [ -n "${branch_input}" ]; then + MLC_BRANCH="${branch_input}" + fi +} + +pull_repo() { + log_info "Pulling automation repo:" + log_info " Repo : ${MLC_REPO}" + log_info " Branch : ${MLC_BRANCH}" + + mlc pull repo "${MLC_REPO}" --branch="${MLC_BRANCH}" +} + +# ------------------------------------------------------------------------------ +# Main Execution +# ------------------------------------------------------------------------------ + +main() { + detect_os + check_missing_dependencies + if [ "${#MISSING_DEPS[@]}" -gt 0 ]; then + log_warn "Missing dependencies: ${MISSING_DEPS[*]}" + install_packages + else + log_info "All base dependencies are present." + fi + + ensure_python + setup_venv + install_mlcflow + prompt_repo_details + pull_repo + + log_info "Installation completed successfully." + echo "" + echo "Virtual environment:" + echo " $VENV_DIR" + echo "" + echo "Activate with:" + echo " source $VENV_DIR/bin/activate" + echo "" + echo "Verify:" + echo " mlc --help" +} + +main From ae22d755fd098e355d9d9a2b0b541618f3da4ce3 Mon Sep 17 00:00:00 2001 From: Sujith Kanakkassery Date: Sun, 22 Mar 2026 17:19:32 +0530 Subject: [PATCH 07/27] Add file locking for index operations (#221) --- mlc/index.py | 117 +++++++++++++++++++++++++++++++++++++------------ pyproject.toml | 3 +- 2 files changed, 90 insertions(+), 30 deletions(-) diff --git a/mlc/index.py b/mlc/index.py index 3c9dae003..d1cfb8a5d 100644 --- a/mlc/index.py +++ b/mlc/index.py @@ -4,6 +4,8 @@ import yaml from .repo import Repo from datetime import datetime +from contextlib import contextmanager +from filelock import FileLock, Timeout class CustomJSONEncoder(json.JSONEncoder): def default(self, obj): @@ -54,42 +56,91 @@ def _load_modified_times(self): """ Load stored mtimes to check for changes in scripts. """ - if os.path.exists(self.modified_times_file): - try: - # logger.info(f"Loading modified times from {self.modified_times_file}") - with open(self.modified_times_file, "r") as f: - return json.load(f) - except (json.JSONDecodeError, IOError) as e: - logger.warning(f"Failed to load modified times: {e}") - return {} - return {} + lock_file = self.modified_times_file + ".lock" + try: + with self._file_lock_with_incremental_timeout(lock_file): + # logger.debug(f"Lock acquired at {lock_file} for Loading Modified Times") + + if os.path.exists(self.modified_times_file): + # logger.info(f"Loading modified times from {self.modified_times_file}") + with open(self.modified_times_file, "r") as f: + return json.load(f) + else: + return {} + + except Timeout: + logger.warning(f"Timeout acquiring lock {lock_file}") + return {} + + except Exception as e: + logger.error(f"Error acquiring lock {lock_file}: {e}") + return {} + + @contextmanager + def _file_lock_with_incremental_timeout(self, lock_file, timeout_seconds=60): + """ + Acquire a file lock by waiting up to a minute, then retrying once if it times out. + """ + try: + with FileLock(lock_file, timeout=timeout_seconds): + yield # Control goes to the caller's 'with' block while the file lock is held + return + except Timeout: + logger.warning( + f"Timeout acquiring lock {lock_file} after {int(timeout_seconds)}s. " + f"Retrying once for another {int(timeout_seconds)}s..." + ) + + with FileLock(lock_file, timeout=timeout_seconds): + yield def _save_modified_times(self): """ Save updated mtimes in modified_times json file. """ - #logger.debug(f"Saving modified times to {self.modified_times_file}") - with open(self.modified_times_file, "w") as f: - json.dump(self.modified_times, f, indent=4) + lock_file = self.modified_times_file + ".lock" + try: + with self._file_lock_with_incremental_timeout(lock_file): + # logger.debug(f"Lock acquired at {lock_file} for Saving Modified Times") + + #logger.debug(f"Saving modified times to {self.modified_times_file}") + with open(self.modified_times_file, "w") as f: + json.dump(self.modified_times, f, indent=4) + + except Timeout: + logger.warning(f"Timeout acquiring lock {lock_file}, skipping modified times save") + + except Exception as e: + logger.error(f"Error saving modified times: {e}") def _load_existing_index(self): """ Load previously saved index to allow incremental updates. """ for folder_type, file_path in self.index_files.items(): - if os.path.exists(file_path): - try: - # logger.info(f"Loading existing index for {folder_type}") - with open(file_path, "r") as f: - self.indices[folder_type] = json.load(f) - # Convert repo dicts back into Repo objects - for item in self.indices[folder_type]: - if isinstance(item.get("repo"), dict): - item["repo"] = Repo(**item["repo"]) - - except (json.JSONDecodeError, IOError, KeyError, TypeError) as e: - logger.warning(f"Failed to load index for {folder_type}: {e}") - pass # fall back to empty index + lock_file = file_path + ".lock" + try: + with self._file_lock_with_incremental_timeout(lock_file): + # logger.debug(f"Lock acquired at {lock_file} for Loading Index for {folder_type}") + + if os.path.exists(file_path): + # logger.info(f"Loading existing index for {folder_type}") + with open(file_path, "r") as f: + self.indices[folder_type] = json.load(f) + # Convert repo dicts back into Repo objects + for item in self.indices[folder_type]: + if isinstance(item.get("repo"), dict): + item["repo"] = Repo(**item["repo"]) + else: + self.indices[folder_type] = [] + + except Timeout: + logger.error(f"Timeout acquiring lock {lock_file}") + self.indices[folder_type] = [] + + except (json.JSONDecodeError, IOError, KeyError, TypeError) as e: + logger.warning(f"Failed to load index for {folder_type}: {e}") + self.indices[folder_type] = [] # fall back to empty index def add(self, meta, folder_type, path, repo): if not repo: @@ -353,10 +404,18 @@ def _save_indices(self): #logger.info(self.indices) for folder_type, index_data in self.indices.items(): output_file = self.index_files[folder_type] + lock_file = output_file + ".lock" try: - with open(output_file, "w") as f: - json.dump(index_data, f, indent=4, cls=CustomJSONEncoder) - #logger.debug(f"Shared index for {folder_type} saved to {output_file}.") + with self._file_lock_with_incremental_timeout(lock_file): + #logger.debug(f"Lock acquired at {lock_file} for Saving Index for {folder_type}") + + with open(output_file, "w") as f: + json.dump(index_data, f, indent=4, cls=CustomJSONEncoder) + #logger.debug(f"Shared index for {folder_type} saved to {output_file}.") + + except Timeout: + logger.error(f"Timeout acquiring lock {lock_file}") + except Exception as e: logger.error(f"Error saving shared index for {folder_type}: {e}") @@ -403,4 +462,4 @@ def remove_repo_from_index(self, repo_path): if changed: self._save_indices() - self._save_modified_times() \ No newline at end of file + self._save_modified_times() diff --git a/pyproject.toml b/pyproject.toml index 457c2f7e2..2b8bc8eac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,8 @@ dependencies = [ "requests", "pyyaml", "giturlparse", - "colorama" + "colorama", + "filelock" ] [project.urls] From fd1139cd99592931b350996e206c170737995a73 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 22 Mar 2026 11:50:29 +0000 Subject: [PATCH 08/27] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index be5b4c7bb..6f1824254 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.20 +1.1.21 From 4782ebc619293d0543da26499c026855a8112321 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 22 Mar 2026 11:50:41 +0000 Subject: [PATCH 09/27] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2b8bc8eac..e32aac71c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mlcflow" -version = "1.1.20" +version = "1.1.21" description = "An automation interface tailored for CPU/GPU benchmarking" authors = [ From fc18cba422ed030007a9bf8f1501ef229588eeaa Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 22 Mar 2026 13:37:47 +0000 Subject: [PATCH 10/27] Update VERSION From 325dd1452008bd055d49aa6d85bf3fbab3051013 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 22 Mar 2026 14:24:07 +0000 Subject: [PATCH 11/27] Increment version to 1.1.22 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6f1824254..c442f5e77 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.21 +1.1.22 From 941fd51c0c5a6839a50c4f96fa764fcd935d2e1b Mon Sep 17 00:00:00 2001 From: amd-arsuresh Date: Sun, 12 Apr 2026 23:45:37 +0100 Subject: [PATCH 12/27] Better error handling (#224) * Added script meta validation support * Improve error handling * Update meta_schema.py --- mlc/index.py | 11 ++ mlc/main.py | 32 +++- mlc/meta_schema.py | 435 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 475 insertions(+), 3 deletions(-) create mode 100644 mlc/meta_schema.py diff --git a/mlc/index.py b/mlc/index.py index d1cfb8a5d..3fbb31283 100644 --- a/mlc/index.py +++ b/mlc/index.py @@ -4,6 +4,7 @@ import yaml from .repo import Repo from datetime import datetime +from .meta_schema import validate_meta from contextlib import contextmanager from filelock import FileLock, Timeout @@ -380,6 +381,16 @@ def _process_config_file(self, config_file, folder_type, folder_path, repo): tags = data.get("tags", []) alias = data.get("alias", None) + # Validate script meta against schema during indexing + if folder_type == "script": + errors, warnings = validate_meta(data, config_file) + for e in errors: + logger.error(f"Meta validation error: {e}") + for w in warnings: + logger.debug(f"Meta validation warning: {w}") + if errors: + raise ValueError(f"Meta validation failed for {config_file}. Fix the above error(s) and try again.") + # Validate and add to indices self._delete_by_uid(folder_type, unique_id, alias) self.indices[folder_type].append({ diff --git a/mlc/main.py b/mlc/main.py index ecb4b395f..9b8b66b10 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -85,7 +85,20 @@ def mlc_expand_short(action, target = "script"): sys.argv.insert(2, target) # Call the main function - main() + try: + main() + except SystemExit: + raise + except Exception as e: + import traceback + tb = traceback.extract_tb(e.__traceback__) + if tb: + last = tb[-1] + logger.error(f"{e}") + logger.error(f" at {last.filename}:{last.lineno} in {last.name}") + else: + logger.error(f"{e}") + sys.exit(1) def mlcr(): mlc_expand_short("run") @@ -380,10 +393,23 @@ def main(): res = method(run_args) if res['return'] > 0: logging.error(res.get('error', f"Error in {action}")) - raise Exception(f"An error occurred {res}") + sys.exit(1) process_console_output(res, args.target, args.command, run_args) if __name__ == '__main__': - main() + try: + main() + except SystemExit: + raise + except Exception as e: + import traceback + tb = traceback.extract_tb(e.__traceback__) + if tb: + last = tb[-1] + logger.error(f"{e}") + logger.error(f" at {last.filename}:{last.lineno} in {last.name}") + else: + logger.error(f"{e}") + sys.exit(1) diff --git a/mlc/meta_schema.py b/mlc/meta_schema.py new file mode 100644 index 000000000..567d1b85b --- /dev/null +++ b/mlc/meta_schema.py @@ -0,0 +1,435 @@ +""" +Schema specification for script meta.yaml files. + +Defines all valid keys, their types, and nesting rules for script meta.yaml. +Used by validate_meta() to check meta files during indexing and linting. +""" + +# ─── Type aliases ─────────────────────────────────────────────── +# Each value is a set of allowed Python type names (from type().__name__). +# "optional" means the key may be absent; all keys are optional unless in REQUIRED_KEYS. + +STR = {"str"} +BOOL = {"bool"} +INT = {"int"} +LIST = {"list"} +DICT = {"dict"} +STR_OR_BOOL = {"str", "bool"} +INT_OR_FLOAT = {"int", "float"} +STR_OR_FLOAT = {"str", "float"} +STR_OR_LIST = {"str", "list"} + +# ─── Required top-level keys ─────────────────────────────────── +REQUIRED_KEYS = {"alias", "uid", "automation_alias", "automation_uid"} + +# ─── Top-level key specification ─────────────────────────────── +# key -> set of allowed type names +TOP_LEVEL_SCHEMA = { + # Identity (required) + "alias": STR, + "uid": STR, + "automation_alias": STR, + "automation_uid": STR, + + # Metadata + "name": STR, + "category": STR, + "tags": LIST, # list[str] + "tags_help": STR, + "developers": STR, + "sort": INT, + "category_sort": INT, + "private": BOOL, + "min_mlc_version": STR, + + # Environment + "env": DICT, # dict[str, str] + "default_env": DICT, # dict[str, str] + "new_env_keys": LIST, # list[str] + "new_state_keys": LIST, # list[str] + "local_env_keys": LIST, # list[str] + "file_path_env_keys": LIST, # list[str] + "folder_path_env_keys": LIST, # list[str] + + # Cache + "cache": STR_OR_BOOL, + "can_force_cache": BOOL, + "cache_expiration": STR, + "extra_cache_tags_from_env": LIST, # list[str] + "clean_files": LIST, # list[str] + "clean_output_files": LIST, # list[str] + + # Input mapping + "input_mapping": {*DICT, "NoneType"}, # dict[str, str] input_name -> ENV_KEY (or null) + "input_description": {*DICT, "NoneType"}, # dict[str, dict] input_name -> {desc, choices, ...} (or null) + "env_key_mappings": DICT, # dict[str, str] + + # Dependencies + "deps": LIST, # list[dep_entry] + "prehook_deps": LIST, # list[dep_entry] + "posthook_deps": LIST, # list[dep_entry] + "post_deps": LIST, # list[dep_entry] + "predeps": BOOL, + + # Variations + "variations": DICT, # dict[str, variation_entry] + "variation_groups_order": LIST, # list[str] + "default_variation": STR, + "default_variations": DICT, # dict[str, str] + "invalid_variation_combinations": LIST, # list[list[str]] + "valid_variation_combinations": LIST, # list[list[str]] + + # Versions + "versions": DICT, # dict[str, version_entry] + "default_version": STR, + + # Docker + "docker": DICT, # dict - see DOCKER_SCHEMA + + # Output / debugging + "print_env_at_the_end": DICT, # dict[str, list[str]] + "print_files_if_script_error": LIST, # list[str] + "warnings": LIST, # list[str] + "sudo_install": BOOL, + + # Conditional meta update + "update_meta_if_env": LIST, # list[dict] + "remote_run": DICT, + + # Tests + "tests": DICT, # dict - see TESTS_SCHEMA +} + +# ─── Dependency entry keys ────────────────────────────────────── +DEP_ENTRY_SCHEMA = { + "tags": STR, + "names": STR_OR_LIST, + "env": DICT, + "enable_if_env": DICT, + "skip_if_env": DICT, + "skip_if_any_env": DICT, + "enable_if_any_env": DICT, + "extra_cache_tags": STR, + "update_tags_from_env_with_prefix": DICT, + "update_tags_from_env": LIST, + "force_env_keys": LIST, + "force_cache": BOOL, + "reuse_version": BOOL, + "inherit_variation_tags": STR_OR_BOOL, + "skip_inherit_variation_groups": LIST, + "version": STR, + "version_min": STR, + "version_max": STR_OR_FLOAT, + "version_max_usable": STR_OR_FLOAT, + "dynamic": BOOL, + "ignore_missing": BOOL, + "skip_if_fake_run": BOOL, + "verify": BOOL, + "md5sum": STR, + "revision": STR, + "model_filename": STR, + "full_subfolder": STR, + "env_key": STR, + "continue_on_error": BOOL, + "ignore_script_error": BOOL, + "inherit_cache_expiration": BOOL, + "update_tags_if_env": DICT, + "update_meta_if_env": LIST, +} + +# ─── Variation entry keys ─────────────────────────────────────── +VARIATION_ENTRY_SCHEMA = { + "env": {*DICT, "NoneType"}, + "group": STR, + "default": STR_OR_BOOL, + "default_variations": DICT, + "deps": LIST, + "prehook_deps": LIST, + "posthook_deps": LIST, + "post_deps": LIST, + "add_deps": DICT, + "add_deps_recursive": DICT, + "add_deps_tags": DICT, + "new_env_keys": LIST, + "new_state_keys": LIST, + "base": LIST, + "adr": DICT, + "ad": DICT, + "default_env": DICT, + "state": DICT, + "const": DICT, + "docker": DICT, + "alias": STR, + "default_version": STR_OR_FLOAT, + "required_disk_space": INT, + "cache_expiration": {*STR, *INT}, + "cache": BOOL, + "force_cache": BOOL, + "update_meta_if_env": LIST, + "warning": STR, + "warnings": LIST, + "names": LIST, + "default_variation": DICT, +} + +# ─── Docker section keys ──────────────────────────────────────── +DOCKER_SCHEMA = { + "real_run": BOOL, + "run": BOOL, + "skip_run_cmd": STR_OR_BOOL, + "interactive": BOOL, + "pre_run_cmds": LIST, + "deps": LIST, + "mounts": LIST, + "input_mapping": DICT, + "input_paths": LIST, + "skip_input_for_fake_run": LIST, + "os": STR, + "os_version": STR, + "base_image": STR, + "mlc_repo": STR, + "mlc_repo_branch": STR, + "mlc_repo_flags": STR, + "extra_run_args": STR, + "all_gpus": STR, + "user": STR, + "use_host_user_id": BOOL, + "use_host_group_id": STR_OR_BOOL, + "skip_mlc_sys_upgrade": STR, + "shm_size": STR, + "port_maps": LIST, + "image_tag_extra": STR, + "fake_run_deps": BOOL, + "pass_docker_to_script": BOOL, + "mount_current_dir": STR, + "use_google_dns": BOOL, + "add_quotes_to_keys": LIST, + "device": STR, + "run_cmd_prefix": STR, + "pass_user_group": BOOL, + "default_env": DICT, + "env": DICT, +} + +# ─── Tests section keys ───────────────────────────────────────── +TESTS_SCHEMA = { + "run_inputs": LIST, # list[dict] - each has variations_list, env, etc. + "needs_pat": BOOL, +} + +# ─── Tests run_inputs entry keys ──────────────────────────────── +TESTS_RUN_INPUT_SCHEMA = { + "variations_list": LIST, # list[str] + "env": DICT, + "test_input_index": STR, + "disable_run_script": BOOL, +} + + +# ─── update_meta_if_env entry keys ────────────────────────────── +UPDATE_META_IF_ENV_SCHEMA = { + "enable_if_env": DICT, + "enable_if_any_env": DICT, + "skip_if_env": DICT, + "skip_if_any_env": DICT, + "env": DICT, + "default_env": DICT, + "default_variations": DICT, + "docker": DICT, + "adr": DICT, + "ad": DICT, +} + +def validate_meta(data, file_path=""): + """ + Validate a script meta.yaml dict against the schema. + + Args: + data (dict): Parsed meta.yaml content. + file_path (str): Path to meta file (for error messages). + + Returns: + list[str]: List of warning/error messages. Empty if valid. + """ + errors = [] + warnings = [] + + if not isinstance(data, dict): + return ["Meta is not a dictionary"], [] + + prefix = f"{file_path}: " if file_path else "" + + # Check required keys + for key in REQUIRED_KEYS: + if key not in data: + errors.append(f"{prefix}Missing required key '{key}'") + + # Check top-level keys + for key, value in data.items(): + if key not in TOP_LEVEL_SCHEMA: + # Check if it looks like a misplaced env variable + if key.startswith("MLC_"): + warnings.append( + f"{prefix}Key '{key}' looks like an env variable - should it be under 'env' or 'default_env'?") + else: + warnings.append(f"{prefix}Unknown top-level key '{key}'") + continue + + actual_type = type(value).__name__ + allowed = TOP_LEVEL_SCHEMA[key] + if actual_type not in allowed: + errors.append( + f"{prefix}Key '{key}' has type '{actual_type}', expected {allowed}") + + # Validate dependency lists + for dep_list_key in ["deps", "prehook_deps", "posthook_deps", "post_deps", "post_deps_off"]: + deps = data.get(dep_list_key) + if deps is None: + continue + if not isinstance(deps, list): + continue + for i, dep in enumerate(deps): + if not isinstance(dep, dict): + errors.append( + f"{prefix}{dep_list_key}[{i}] is not a dict") + continue + for dk, dv in dep.items(): + if dk not in DEP_ENTRY_SCHEMA: + warnings.append( + f"{prefix}{dep_list_key}[{i}]: unknown dep key '{dk}'") + continue + actual = type(dv).__name__ + allowed = DEP_ENTRY_SCHEMA[dk] + if actual not in allowed: + errors.append( + f"{prefix}{dep_list_key}[{i}].{dk} has type '{actual}', expected {allowed}") + + # Validate enable_if_env/skip_if_env values are single strings/lists, not nested dicts + for ck in ["enable_if_env", "skip_if_env", "skip_if_any_env", "enable_if_any_env"]: + cv = dep.get(ck) + if isinstance(cv, dict): + for ek, ev in cv.items(): + if isinstance(ev, (dict, list)) and not isinstance(ev, str): + if isinstance(ev, list): + for item in ev: + if not isinstance(item, (str, int, float, bool)): + errors.append( + f"{prefix}{dep_list_key}[{i}].{ck}.{ek} list contains non-scalar: {type(item).__name__}") + + # Validate update_meta_if_env entries + umie = data.get("update_meta_if_env") + if isinstance(umie, list): + for i, entry in enumerate(umie): + if not isinstance(entry, dict): + errors.append(f"{prefix}update_meta_if_env[{i}] is not a dict") + continue + for ek, ev in entry.items(): + if ek not in UPDATE_META_IF_ENV_SCHEMA: + warnings.append(f"{prefix}update_meta_if_env[{i}]: unknown key '{ek}'") + continue + actual = type(ev).__name__ + allowed = UPDATE_META_IF_ENV_SCHEMA[ek] + if actual not in allowed: + errors.append( + f"{prefix}update_meta_if_env[{i}].{ek} has type '{actual}', expected {allowed}") + # Validate enable_if_env/skip_if_env values inside update_meta_if_env + for ck in ["enable_if_env", "skip_if_env", "skip_if_any_env", "enable_if_any_env"]: + cv = entry.get(ck) + if isinstance(cv, dict): + for ek2, ev2 in cv.items(): + if isinstance(ev2, (dict, list)) and not isinstance(ev2, str): + if isinstance(ev2, list): + for item in ev2: + if not isinstance(item, (str, int, float, bool)): + errors.append( + f"{prefix}update_meta_if_env[{i}].{ck}.{ek2} list contains non-scalar: {type(item).__name__}") + + # Validate variations + variations = data.get("variations") + if isinstance(variations, dict): + for vname, vattrs in variations.items(): + if vattrs is None: + continue # empty variation is ok + if not isinstance(vattrs, dict): + warnings.append( + f"{prefix}variations.{vname} is not a dict (type: {type(vattrs).__name__})") + continue + for vk, vv in vattrs.items(): + if vk not in VARIATION_ENTRY_SCHEMA: + if not vk.startswith("MLC_"): + warnings.append( + f"{prefix}variations.{vname}: unknown variation key '{vk}'") + continue + actual = type(vv).__name__ + allowed = VARIATION_ENTRY_SCHEMA[vk] + if actual not in allowed: + errors.append( + f"{prefix}variations.{vname}.{vk} has type '{actual}', expected {allowed}") + + + # Validate docker section + docker = data.get("docker") + if isinstance(docker, dict): + for dk, dv in docker.items(): + if dk not in DOCKER_SCHEMA: + warnings.append( + f"{prefix}docker: unknown key '{dk}'") + continue + actual = type(dv).__name__ + allowed = DOCKER_SCHEMA[dk] + if actual not in allowed: + errors.append( + f"{prefix}docker.{dk} has type '{actual}', expected {allowed}") + + # Validate tests section + tests = data.get("tests") + if isinstance(tests, dict): + for tk, tv in tests.items(): + if tk not in TESTS_SCHEMA: + warnings.append( + f"{prefix}tests: unknown key '{tk}'") + continue + actual = type(tv).__name__ + allowed = TESTS_SCHEMA[tk] + if actual not in allowed: + errors.append( + f"{prefix}tests.{tk} has type '{actual}', expected {allowed}") + + run_inputs = tests.get("run_inputs") + if isinstance(run_inputs, list): + for i, entry in enumerate(run_inputs): + if entry is None: + continue + if not isinstance(entry, dict): + errors.append( + f"{prefix}tests.run_inputs[{i}] is not a dict") + + # Check for variation entry keys mistakenly used as variation names + # Exclude common short words that are legitimately used as both + _variation_name_allowlist = {"default", "base", "env", "ad", "cache", "state", "alias", "warning", "const"} + if isinstance(variations, dict): + for vname in variations: + if vname in VARIATION_ENTRY_SCHEMA and vname not in _variation_name_allowlist: + warnings.append( + f"{prefix}variation '{vname}' looks like a variation property key used as a variation name") + # Also check if variation attrs contain keys that look like variation names + vattrs = variations[vname] + if isinstance(vattrs, dict): + for vk in vattrs: + if vk in VARIATION_ENTRY_SCHEMA: + continue # valid property + if vk.startswith("MLC_"): + continue # env override + # Check if this unknown key is actually a known variation name in this script + if vk in variations and vk != vname: + warnings.append( + f"{prefix}variations.{vname}: key '{vk}' matches another variation name - possible indentation error") + + # Cross-key validations + default_variation = data.get("default_variation") + if default_variation and isinstance(variations, dict): + if default_variation not in variations: + errors.append( + f"{prefix}default_variation '{default_variation}' not found in variations") + + return errors, warnings From 3f8a2b9ab986efaa99bc93735ec6f65a4161cdb4 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 14 Apr 2026 23:40:52 +0530 Subject: [PATCH 13/27] Create GH action for code formatting (#223) * Update VERSION --- .github/workflows/cla.yml | 2 +- .github/workflows/format.yml | 60 ++++++ VERSION | 2 +- mlc/action.py | 288 +++++++++++++++++---------- mlc/index.py | 140 +++++++------ mlc/logger.py | 31 +-- mlc/main.py | 178 ++++++++++++----- mlc/meta_schema.py | 373 +++++++++++++++++++---------------- mlc/repo_action.py | 341 +++++++++++++++++++------------- mlc/utils.py | 152 ++++++++------ pyproject.toml | 2 +- 11 files changed, 960 insertions(+), 609 deletions(-) create mode 100644 .github/workflows/format.yml diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index fedf10051..024e3ff4d 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -26,7 +26,7 @@ jobs: path-to-signatures: 'cla-bot/v1/cla.json' # branch should not be protected branch: 'main' - allowlist: user1,bot* + allowlist: user1,mlc-automations,bot* remote-organization-name: mlcommons remote-repository-name: systems diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml new file mode 100644 index 000000000..681587d93 --- /dev/null +++ b/.github/workflows/format.yml @@ -0,0 +1,60 @@ +# Automatic code formatting +name: "Code formatting" +on: + push: + branches: + - "**" + +env: + python_version: "3.9" + +jobs: + format-code: + if: github.actor != 'mlc-automations' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.MLC_AUTOMATIONS_APP_ID }} + private-key: ${{ secrets.MLC_AUTOMATIONS_PRIVATE_KEY }} + + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + + - name: Set up Python ${{ env.python_version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.python_version }} + + - name: Format modified Python files + env: + filter: ${{ github.event.before }} + run: | + python3 -m pip install autopep8 + for FILE in $(git diff --name-only $filter | grep -E '.*\.py$') + do + # Check if the file still exists in the working tree + if [ -f "$FILE" ]; then + autopep8 --in-place -a "$FILE" + git add "$FILE" + fi + done + + - name: Commit and push changes + run: | + HAS_CHANGES=$(git diff --staged --name-only) + if [ ${#HAS_CHANGES} -gt 0 ]; then + # Use the GitHub actor's name and email + git config --global user.name mlc-automations + git config --global user.email "3246381+mlc-automations@users.noreply.github.com" + # Commit changes + git commit -m '[Automated Commit] Format Codebase' + git push + fi diff --git a/VERSION b/VERSION index c442f5e77..26aaba0e8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.22 +1.2.0 diff --git a/mlc/action.py b/mlc/action.py index 6e39bd5f4..28df4b373 100644 --- a/mlc/action.py +++ b/mlc/action.py @@ -16,6 +16,8 @@ from .error_codes import WarningCode # Base class for actions + + class Action: repos_path = None cfg = None @@ -23,7 +25,7 @@ class Action: logger = None local_repo = None current_repo_path = None - repos = [] #list of Repo objects + repos = [] # list of Repo objects # Main access function to simulate a Python interface for CLI def access(self, options): @@ -35,42 +37,47 @@ def access(self, options): """ from .action_factory import get_action - #logger.info(f"options in access = {options}") - + # logger.info(f"options in access = {options}") + action_name = options.get('action') if not action_name: return {'return': 1, 'error': "'action' key is required in options"} - #logger.info(f"options = {options}") + # logger.info(f"options = {options}") action_name = action_name.replace("-", "_") action_target = options.get('target') if not action_target: - action_target = options.get('automation', 'script') # Default to script if not provided + # Default to script if not provided + action_target = options.get('automation', 'script') action_target_split = action_target.split(",") action_target = action_target_split[0] - action = get_action(action_target, self.parent if self.parent else self) + action = get_action(action_target, + self.parent if self.parent else self) if action and hasattr(action, action_name): # Find the method and call it with the options method = getattr(action, action_name) result = method(options) - #logger.info(f"result ={result}") + # logger.info(f"result ={result}") return result else: - return {'return': 1, 'error': f"'{action_name}' action is not supported for {action_target}."} + return { + 'return': 1, 'error': f"'{action_name}' action is not supported for {action_target}."} return {'return': 0} def find_target_folder(self, target): - # Traverse through each repo to find the first 'target' folder inside an 'automation' folder + # Traverse through each repo to find the first 'target' folder inside + # an 'automation' folder for repo in self.repos: repo_path = repo.path if os.path.isdir(repo_path): automation_folder = os.path.join(repo_path, 'automation') - + if os.path.isdir(automation_folder): - # Check if there's a 'script' folder inside the 'automation' folder + # Check if there's a 'script' folder inside the + # 'automation' folder target_folder = os.path.join(automation_folder, target) if os.path.isdir(target_folder): return target_folder @@ -82,7 +89,8 @@ def load_repos_and_meta(self): # Read the JSON file line by line try: - # Load and parse the JSON file containing the list of repository paths + # Load and parse the JSON file containing the list of repository + # paths with open(repos_file_path, 'r') as file: repo_paths = json.load(file) # Load the JSON file into a list except json.JSONDecodeError as e: @@ -94,7 +102,7 @@ def load_repos_and_meta(self): except Exception as e: logger.error(f"Error reading file: {e}") return [] - + def is_curdir_inside_path(base_path): # Convert to absolute paths base_path = Path(base_path).resolve() @@ -106,9 +114,15 @@ def is_curdir_inside_path(base_path): # Iterate through the list of repository paths for repo_path in repo_paths: if not os.path.exists(repo_path): - logger.warning(f"""Warning: {repo_path} not found. Considering it as a corrupt entry and deleting from repos.json...""") + logger.warning( + f"""Warning: {repo_path} not found. Considering it as a corrupt entry and deleting from repos.json...""") from .repo_action import rm_repo - res = rm_repo(repo_path, os.path.join(self.repos_path, 'repos.json'), True) + res = rm_repo( + repo_path, + os.path.join( + self.repos_path, + 'repos.json'), + True) if res["return"] > 0: return res @@ -126,7 +140,8 @@ def is_curdir_inside_path(base_path): # Check if meta.yaml exists if not os.path.isfile(meta_yaml_path): - logger.warning(f"{meta_yaml_path} not found. Could be due to accidental deletion of meta.yaml. Try to stash the changes or reclone by doing `rm repo` and `pull repo`. Skipping...") + logger.warning( + f"{meta_yaml_path} not found. Could be due to accidental deletion of meta.yaml. Try to stash the changes or reclone by doing `rm repo` and `pull repo`. Skipping...") continue # Load the YAML file @@ -164,49 +179,54 @@ def load_repos(self): except Exception as e: logger.error(f"Error reading file: {e}") return None - + def get_index(self): if self._index is None: self._index = Index(self.repos_path, self.repos) return self._index - def __init__(self): + def __init__(self): setup_logging(log_path=os.getcwd(), log_file='.mlc-log.txt') self.logger = logger - temp_repo = os.environ.get('MLC_REPOS','').strip() + temp_repo = os.environ.get('MLC_REPOS', '').strip() if temp_repo == '': - self.repos_path = os.path.join(os.path.expanduser("~"), "MLC", "repos") + self.repos_path = os.path.join( + os.path.expanduser("~"), "MLC", "repos") else: self.repos_path = temp_repo mlc_local_repo_path = os.path.join(self.repos_path, 'local') - - mlc_local_repo_path_expanded = Path(mlc_local_repo_path).expanduser().resolve() + + mlc_local_repo_path_expanded = Path( + mlc_local_repo_path).expanduser().resolve() if not os.path.exists(mlc_local_repo_path): os.makedirs(mlc_local_repo_path, exist_ok=True) - + if not os.path.isfile(os.path.join(mlc_local_repo_path, "meta.yaml")): - local_repo_meta = {"alias": "local", "name": "MLC local repository", "uid": utils.get_new_uid()['uid']} + local_repo_meta = { + "alias": "local", + "name": "MLC local repository", + "uid": utils.get_new_uid()['uid']} with open(os.path.join(mlc_local_repo_path, "meta.yaml"), "w") as json_file: json.dump(local_repo_meta, json_file, indent=4) - + repo_json_path = os.path.join(self.repos_path, "repos.json") if not os.path.exists(repo_json_path): with open(repo_json_path, 'w') as f: json.dump([str(mlc_local_repo_path_expanded)], f, indent=2) - logger.info(f"Created repos.json in {os.path.dirname(self.repos_path)} and initialised with local cache folder path: {mlc_local_repo_path}") + logger.info( + f"Created repos.json in {os.path.dirname(self.repos_path)} and initialised with local cache folder path: {mlc_local_repo_path}") self.local_cache_path = os.path.join(mlc_local_repo_path, "cache") if not os.path.exists(self.local_cache_path): os.makedirs(self.local_cache_path, exist_ok=True) self.repos = self.load_repos_and_meta() - #logger.info(f"In Action class: {self.repos_path}") + # logger.info(f"In Action class: {self.repos_path}") self._index = None - def add(self, i): """ Adds a new item to the repository. @@ -226,7 +246,6 @@ def add(self, i): item_repo = i.get("item_repo") if not item_repo: item_repo = self.local_repo - # Parse item details item = i.get("item") @@ -257,21 +276,23 @@ def add(self, i): return res if len(res["list"]) == 0: - return {'return': 1, 'error': f"""The given repo {item_repo} is not registered in MLC"""} + return { + 'return': 1, 'error': f"""The given repo {item_repo} is not registered in MLC"""} # Determine paths and metadata format repo = res["list"][0] repo_path = repo.path - + target_name = i.get('target_name', self.action_type) target_path = os.path.join(repo_path, target_name) if target_name in ["cache", "experiment"]: - extra_tags_suffix=i.get('extra_tags', '').replace(",", "-")[:15] + extra_tags_suffix = i.get('extra_tags', '').replace(",", "-")[:15] if extra_tags_suffix != '': suffix = f"_{extra_tags_suffix}" else: suffix = '' - folder_name = f"""{i["script_alias"]}{suffix}_{item_name or item_id[:8]}""" if i.get("script_alias") else item_name or item_id + folder_name = f"""{i["script_alias"]}{suffix}_{item_name or item_id[:8]}""" if i.get( + "script_alias") else item_name or item_id else: folder_name = item_name or item_id @@ -283,7 +304,13 @@ def add(self, i): # Create item directory if it does not exist os.makedirs(item_path) - res = self.save_new_meta(i, item_id, item_name, target_name, item_path, repo) + res = self.save_new_meta( + i, + item_id, + item_name, + target_name, + item_path, + repo) if res['return'] > 0: return res @@ -293,7 +320,7 @@ def add(self, i): "path": item_path, "repo": repo } - + def rm(self, i): """ Removes an item from the repository. @@ -311,7 +338,7 @@ def rm(self, i): inp = {} # Parse item details - item = i.get("item",i.get('artifact', i.get('details'))) + item = i.get("item", i.get('artifact', i.get('details'))) item_name, item_id, item_tags = (None, None, None) if item: item_parts = item.split(",") @@ -327,12 +354,15 @@ def rm(self, i): inp['fetch_all'] = True # Check force remove is set to True - # Setting force remove to true would lead to removal of assets without user prompt + # Setting force remove to true would lead to removal of assets without + # user prompt force_remove = True if i.get('f') else False if item_name: inp['alias'] = item_name - inp['folder_name'] = item_name #we dont know if the user gave the alias or the folder name, we first check for alias and then the folder name + # we dont know if the user gave the alias or the folder name, we + # first check for alias and then the folder name + inp['folder_name'] = item_name if utils.is_uid(item_name): inp['uid'] = item_name elif item_id: @@ -349,11 +379,14 @@ def rm(self, i): if len(res['list']) == 0: # Do not error out if fetch_all is used if inp.get("fetch_all", False) == True: - logger.warning(f"{target_name} is empty! nothing to be cleared!") - return {"return": 0, "warnings": [{"code": WarningCode.EMPTY_TARGET.code, "description": f"{target_name} is empty! nothing to be cleared!"}]} + logger.warning( + f"{target_name} is empty! nothing to be cleared!") + return {"return": 0, "warnings": [ + {"code": WarningCode.EMPTY_TARGET.code, "description": f"{target_name} is empty! nothing to be cleared!"}]} else: logger.warning(f"No {target_name} found for {inp}") - return {'return': 0, "warnings": [{"code": WarningCode.EMPTY_TARGET.code, "description": f"No {target_name} found for {inp}"}]} + return {'return': 0, "warnings": [ + {"code": WarningCode.EMPTY_TARGET.code, "description": f"No {target_name} found for {inp}"}]} elif len(res['list']) > 1: logger.info(f"More than 1 {target_name} found for {inp}:") if not i.get('all'): @@ -361,37 +394,40 @@ def rm(self, i): logger.info(f"{idx}. Path: {item.path}, Meta: {item.meta}") if not force_remove: - user_choice = input("Would you like to proceed with all items? (yes/no): ").strip().lower() + user_choice = input( + "Would you like to proceed with all items? (yes/no): ").strip().lower() if user_choice in ['yes', 'y']: force_remove = True - + results = res['list'] - + for result in results: item_path = result.path item_meta = result.meta - - + if os.path.exists(item_path): if force_remove == True: shutil.rmtree(item_path) else: - user_choice = input(f"Confirm to delete {target_name} item: {item_path}? (yes/no): ").strip().lower() + user_choice = input( + f"Confirm to delete {target_name} item: {item_path}? (yes/no): ").strip().lower() if user_choice not in ['yes', 'y']: continue else: shutil.rmtree(item_path) - logger.info(f"{target_name} item: {item_path} has been successfully removed") + logger.info( + f"{target_name} item: {item_path} has been successfully removed") self.get_index().rm(item_meta, target_name, item_path) - + return { "return": 0, "message": f"Item {item_path} successfully removed", } - def save_new_meta(self, i, item_id, item_name, target_name, item_path, repo): + def save_new_meta(self, i, item_id, item_name, + target_name, item_path, repo): # Prepare metadata item_meta = i.get('meta', {}) item_meta.update({ @@ -400,8 +436,10 @@ def save_new_meta(self, i, item_id, item_name, target_name, item_path, repo): }) # Process tags - tags = i.get("tags", "").split(",") if i.get("tags") else item_meta.get("tags", []) - new_tags = i.get("new_tags", "").split(",") if i.get("new_tags") else [] + tags = i.get("tags", "").split(",") if i.get( + "tags") else item_meta.get("tags", []) + new_tags = i.get("new_tags", "").split( + ",") if i.get("new_tags") else [] item_meta["tags"] = list(set(tags + new_tags)) # Ensure unique tags @@ -416,7 +454,7 @@ def save_new_meta(self, i, item_id, item_name, target_name, item_path, repo): if save_result["return"] > 0: return save_result - + self.get_index().add(item_meta, target_name, item_path, repo) return {'return': 0} @@ -446,10 +484,11 @@ def update(self, i): found_items = search_result['list'] if not found_items: res = self.add(i) - if res['return'] > 0 : + if res['return'] > 0: return res found_items.append(Item(res['path'], res['repo'])) - #return {'return': 0, 'message': 'No items found for the given tags.'} + # return {'return': 0, 'message': 'No items found for the given + # tags.'} # Step 2: Prepare to update tags search_tags = i.get("search_tags", []) @@ -457,9 +496,11 @@ def update(self, i): new_tags = set(search_tags) if len(found_items) > 1: # Step 3: Ask user for confirmation if multiple items are found - user_input = input(f"{len(found_items)} items found. Do you want to update all? (yes/no): ").strip().lower() + user_input = input( + f"{len(found_items)} items found. Do you want to update all? (yes/no): ").strip().lower() if user_input not in ['yes', 'y']: - return {'return': 0, 'message': 'Update operation canceled by the user.'} + return {'return': 0, + 'message': 'Update operation canceled by the user.'} new_meta = i.get('meta') if new_meta.get('tags'): @@ -472,7 +513,7 @@ def update(self, i): item_meta_path = os.path.join(item.path, "meta.json") if os.path.exists(item_meta_path): res = utils.load_json(item_meta_path) - if res['return']> 0: + if res['return'] > 0: return res meta = res['meta'] if i.get('replace_lists') and i.get("tags"): @@ -481,30 +522,33 @@ def update(self, i): current_tags = set(meta.get("tags", [])) updated_tags = current_tags.union(new_tags) meta["tags"] = list(updated_tags) - utils.merge_dicts({"dict1": meta, "dict2": new_meta, "append_lists": True, "append_unique":True}) - + utils.merge_dicts({"dict1": meta, + "dict2": new_meta, + "append_lists": True, + "append_unique": True}) + # Save the updated meta back to the item item.meta = meta save_result = utils.save_json(item_meta_path, meta=meta) self.get_index().update(meta, target_name, item.path, item.repo) - return {'return': 0, 'message': f"Tags updated successfully for {len(found_items)} item(s).", 'list': found_items } - - + return { + 'return': 0, 'message': f"Tags updated successfully for {len(found_items)} item(s).", 'list': found_items} def cp(self, run_args): action_target = run_args['target'] if action_target != "script": - return {"return": 1, "error": f"The {action_target} target is not currently supported for mv/cp actions"} + return { + "return": 1, "error": f"The {action_target} target is not currently supported for mv/cp actions"} inp = {} src_item = run_args.get('src') src_tags = None - + if src_item: # remove backslash if there in src item if src_item.endswith('/'): src_item = src_item[:-1] - + src_split = src_item.split(":") if len(src_split) > 1: src_repo = src_split[0].strip() @@ -513,15 +557,18 @@ def cp(self, run_args): src_item = src_split[0].strip() inp['alias'] = src_item - inp['folder_name'] = src_item #we dont know if the user gave the alias or the folder name, we first check for alias and then the folder name - + # we dont know if the user gave the alias or the folder name, we + # first check for alias and then the folder name + inp['folder_name'] = src_item + if utils.is_uid(src_item): inp['uid'] = src_item src_id = src_item else: - #src_tags must be there + # src_tags must be there if not run_args.get("src_tags"): - return {'return': 1, 'error': 'Either "src" or "src_tags" must be provided as an input for cp method'} + return { + 'return': 1, 'error': 'Either "src" or "src_tags" must be provided as an input for cp method'} src_tags = run_args['src_tags'] inp['tags'] = src_tags src_id = src_tags @@ -542,7 +589,8 @@ def cp(self, run_args): # Ask user to choose an item while True: - choice = input("Select the correct one (enter number, default=1): ").strip() + choice = input( + "Select the correct one (enter number, default=1): ").strip() if choice == "": choice = 1 try: @@ -550,7 +598,8 @@ def cp(self, run_args): if 0 <= choice < len(res['list']): break else: - print("Invalid selection. Please enter a number from the list.") + print( + "Invalid selection. Please enter a number from the list.") except ValueError: print("Invalid input. Please enter a number.") @@ -565,21 +614,25 @@ def cp(self, run_args): target_repo_name = target_split[0].strip() if target_repo_name == ".": if not self.current_repo_path: - return {'return': 1, 'error': f"""Current directory is not inside a registered MLC repo and so using ".:" is not valid"""} + return { + 'return': 1, 'error': f"""Current directory is not inside a registered MLC repo and so using ".:" is not valid"""} target_repo_name = os.path.basename(self.current_repo_path) else: - if not any(os.path.basename(repodata.path) == target_repo_name for repodata in self.repos): + if not any(os.path.basename(repodata.path) == + target_repo_name for repodata in self.repos): return {'return': 1, 'error': f"""The target repo {target_repo} is not registered in MLC. Either register in MLC by cloning from Git through command `mlc pull repo` or create repo using `mlc add repo` command and try to rerun the command again"""} target_repo_path = os.path.join(self.repos_path, target_repo_name) - target_repo = next((k for k in self.repos if os.path.basename(k.path) == target_repo_name), None) + target_repo = next( + (k for k in self.repos if os.path.basename( + k.path) == target_repo_name), None) target_item_name = target_split[1].strip() else: target_repo = result.repo target_repo_path = result.repo.path target_item_name = target_split[0].strip() - - target_item_path = os.path.join(target_repo_path, action_target, target_item_name) + target_item_path = os.path.join( + target_repo_path, action_target, target_item_name) res = self.copy_item(src_item_path, target_item_path) if res['return'] > 0: return res @@ -602,13 +655,20 @@ def cp(self, run_args): return res item_id = res['uid'] - res = self.save_new_meta(ii, item_id, target_item_name, action_target, target_item_path, target_repo) + res = self.save_new_meta( + ii, + item_id, + target_item_name, + action_target, + target_item_path, + target_repo) dest_item = Item(target_item_path, target_repo) - + if res['return'] > 0: return res - logger.info(f"{action_target} {src_item_path} copied to {target_item_path}") + logger.info( + f"{action_target} {src_item_path} copied to {target_item_path}") return {'return': 0, 'src': result, 'dest': dest_item} @@ -616,9 +676,11 @@ def copy_item(self, source_path, destination_path): try: # Copy the source folder to the destination shutil.copytree(source_path, destination_path) - logger.info(f"Folder successfully copied from {source_path} to {destination_path}") + logger.info( + f"Folder successfully copied from {source_path} to {destination_path}") except FileExistsError: - return {'return': 1, 'error': f"Destination folder {destination_path} already exists."} + return { + 'return': 1, 'error': f"Destination folder {destination_path} already exists."} except FileNotFoundError: return {'return': 1, 'error': f"Source folder {source_path} not found"} except Exception as e: @@ -629,7 +691,8 @@ def copy_item(self, source_path, destination_path): def mv(self, run_args): target_name = run_args['target'] if target_name != "script": - return {"return": 1, "error": f"The {target_name} target is not currently supported for mv/cp actions"} + return { + "return": 1, "error": f"The {target_name} target is not currently supported for mv/cp actions"} res = self.cp(run_args) if res['return'] > 0: return res @@ -641,12 +704,13 @@ def mv(self, run_args): res = self.rm(ii) if res['return'] > 0: return res - - #Put the src uid to the destination path + + # Put the src uid to the destination path dest.meta['uid'] = src.meta['uid'] dest._save_meta() self.get_index().update(dest.meta, target_name, dest.path, dest.repo) - logger.info(f"""Item with uid {dest.meta['uid']} successfully moved from {src.path} to {dest.path}""") + logger.info( + f"""Item with uid {dest.meta['uid']} successfully moved from {src.path} to {dest.path}""") return {'return': 0, 'src': src, 'dest': dest} @@ -672,11 +736,13 @@ def search(self, i): details = i['details'] details_split = details.split(",") if len(details_split) > 1: - # Only treat as alias,uid if the second part is actually a valid UID + # Only treat as alias,uid if the second part is actually a + # valid UID if utils.is_uid(details_split[1]): alias = details_split[0] uid = details_split[1] - # Otherwise, don't parse as alias,uid - let it be treated as tags + # Otherwise, don't parse as alias,uid - let it be treated as + # tags else: if utils.is_uid(details_split[0]): uid = details_split[0] @@ -695,7 +761,7 @@ def search(self, i): { "action": "find", "target": "repo", - "repo": f"{item_repo}" + "repo": f"{item_repo}" } ) if res["return"] > 0: @@ -707,7 +773,8 @@ def search(self, i): if target_index: if uid or alias: for res in target_index: - if (res["uid"] == uid or (alias and res["alias"] == alias)) and (not item_repo or item_repo == res['repo']): + if (res["uid"] == uid or (alias and res["alias"] == alias)) and ( + not item_repo or item_repo == res['repo']): it = Item(res['path'], res['repo']) result.append(it) found = True @@ -721,20 +788,24 @@ def search(self, i): if tags: tags_split = tags.split(",") else: - return {"return":1, "error": f"Tags are not specified for completing the requested action"} + return { + "return": 1, "error": f"Tags are not specified for completing the requested action"} if target == "script": - non_variation_tags = [t for t in tags_split if not t.startswith("_")] + non_variation_tags = [ + t for t in tags_split if not t.startswith("_")] tags_to_match = non_variation_tags elif target in ["cache", "experiment"]: tags_to_match = tags_split else: - return {'return': 1, 'error': f"""Target {target} not handled in mlc yet"""} + return { + 'return': 1, 'error': f"""Target {target} not handled in mlc yet"""} n_tags_ = [p for p in tags_to_match if p.startswith("-")] n_tags = [p[1:] for p in n_tags_] p_tags = list(set(tags_to_match) - set(n_tags_)) for res in target_index: c_tags = res["tags"] - if (exact_tags_match and set(p_tags) == set(c_tags)) or (not exact_tags_match and set(p_tags).issubset(set(c_tags)) and set(n_tags).isdisjoint(set(c_tags))): + if (exact_tags_match and set(p_tags) == set(c_tags)) or (not exact_tags_match and set( + p_tags).issubset(set(c_tags)) and set(n_tags).isdisjoint(set(c_tags))): it = Item(res['path'], res['repo']) result.append(it) return {'return': 0, 'list': result} @@ -759,20 +830,21 @@ def reindex(self, i): mlc reindex cache # Reindex only cache target """ reindex_target = i.get('reindex_target') - + if not reindex_target or reindex_target == 'all' or reindex_target == 'repos' or reindex_target == 'repo': # Reindex all targets - logger.info("Reindexing all targets (script, cache, experiment)...") + logger.info( + "Reindexing all targets (script, cache, experiment)...") index = self.get_index() index.build_index(force_rebuild=True) - + logger.info("Successfully reindexed all targets.") return {'return': 0, 'message': 'All targets reindexed successfully'} else: - + logger.info(f"Reindexing {reindex_target} target...") index = self.get_index() - + # Clear the specific index ''' if reindex_target in index.indices: @@ -782,25 +854,27 @@ def reindex(self, i): for key in keys_to_remove: del index.modified_times[key] ''' - - # Rebuild the index (we are rebuilding for all targets here as the individual target rebuild is not implemented and not very critical) + + # Rebuild the index (we are rebuilding for all targets here as the + # individual target rebuild is not implemented and not very + # critical) index.build_index(force_rebuild=True) - + logger.info(f"Successfully reindexed {reindex_target} target.") - return {'return': 0, 'message': f'{reindex_target} target reindexed successfully'} + return { + 'return': 0, 'message': f'{reindex_target} target reindexed successfully'} + default_parent = None if not default_parent: default_parent = Action() + def access(i): from .action_factory import get_action - + action = i['action'] target = i.get('target', i.get('automation')) action_class = get_action(target, default_parent) r = action_class.access(i) return r - - - diff --git a/mlc/index.py b/mlc/index.py index 3fbb31283..ce9768e00 100644 --- a/mlc/index.py +++ b/mlc/index.py @@ -8,6 +8,7 @@ from contextlib import contextmanager from filelock import FileLock, Timeout + class CustomJSONEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, Repo): @@ -24,7 +25,7 @@ class Index: def __init__(self, repos_path, repos): """ Initialize the Index class. - + Args: repos_path (str): Path to the base folder containing repositories. """ @@ -38,7 +39,8 @@ def __init__(self, repos_path, repos): "experiment": os.path.join(repos_path, "index_experiment.json") } self.indices = {key: [] for key in self.index_files.keys()} - self.modified_times_file = os.path.join(repos_path, "modified_times.json") + self.modified_times_file = os.path.join( + repos_path, "modified_times.json") self.modified_times = self._load_modified_times() self._load_existing_index() self.build_index() @@ -68,7 +70,7 @@ def _load_modified_times(self): return json.load(f) else: return {} - + except Timeout: logger.warning(f"Timeout acquiring lock {lock_file}") return {} @@ -78,13 +80,14 @@ def _load_modified_times(self): return {} @contextmanager - def _file_lock_with_incremental_timeout(self, lock_file, timeout_seconds=60): + def _file_lock_with_incremental_timeout( + self, lock_file, timeout_seconds=60): """ Acquire a file lock by waiting up to a minute, then retrying once if it times out. """ try: with FileLock(lock_file, timeout=timeout_seconds): - yield # Control goes to the caller's 'with' block while the file lock is held + yield # Control goes to the caller's 'with' block while the file lock is held return except Timeout: logger.warning( @@ -104,12 +107,13 @@ def _save_modified_times(self): with self._file_lock_with_incremental_timeout(lock_file): # logger.debug(f"Lock acquired at {lock_file} for Saving Modified Times") - #logger.debug(f"Saving modified times to {self.modified_times_file}") + # logger.debug(f"Saving modified times to {self.modified_times_file}") with open(self.modified_times_file, "w") as f: json.dump(self.modified_times, f, indent=4) - + except Timeout: - logger.warning(f"Timeout acquiring lock {lock_file}, skipping modified times save") + logger.warning( + f"Timeout acquiring lock {lock_file}, skipping modified times save") except Exception as e: logger.error(f"Error saving modified times: {e}") @@ -134,7 +138,7 @@ def _load_existing_index(self): item["repo"] = Repo(**item["repo"]) else: self.indices[folder_type] = [] - + except Timeout: logger.error(f"Timeout acquiring lock {lock_file}") self.indices[folder_type] = [] @@ -151,17 +155,17 @@ def add(self, meta, folder_type, path, repo): unique_id = meta['uid'] alias = meta['alias'] tags = meta['tags'] - + index = self.get_index(folder_type, unique_id) if index == -1: self.indices[folder_type].append({ - "uid": unique_id, - "tags": tags, - "alias": alias, - "path": path, - "repo": repo - }) + "uid": unique_id, + "tags": tags, + "alias": alias, + "path": path, + "repo": repo + }) self._save_indices() def get_index(self, folder_type, uid): @@ -175,36 +179,38 @@ def update(self, meta, folder_type, path, repo): alias = meta['alias'] tags = meta['tags'] index = self.get_index(folder_type, uid) - if index == -1: #add it + if index == -1: # add it self.add(meta, folder_type, path, repo) logger.debug(f"Index update failed, new index created for {uid}") else: self.indices[folder_type][index] = { - "uid": uid, - "tags": tags, - "alias": alias, - "path": path, - "repo": repo - } + "uid": uid, + "tags": tags, + "alias": alias, + "path": path, + "repo": repo + } self._save_indices() def rm(self, meta, folder_type, path): uid = meta['uid'] index = self.get_index(folder_type, uid) - if index == -1: - logger.warning(f"Index is not having the {folder_type} item {path}") + if index == -1: + logger.warning( + f"Index is not having the {folder_type} item {path}") else: - del(self.indices[folder_type][index]) + del (self.indices[folder_type][index]) self._save_indices() - def get_item_mtime(self,file): + def get_item_mtime(self, file): latest = 0 t = os.path.getmtime(file) if t > latest: latest = t return latest - def _index_single_repo(self, repo, repos_changed=False, current_item_keys=None): + def _index_single_repo(self, repo, repos_changed=False, + current_item_keys=None): repo_path = repo.path if not os.path.isdir(repo_path): return False @@ -231,20 +237,23 @@ def _index_single_repo(self, repo, repos_changed=False, current_item_keys=None): else: # No config file found, remove from index if exists delete_flag = False - - # Check and remove both possible config paths from modified_times + + # Check and remove both possible config paths from + # modified_times for config_name in ["meta.yaml", "meta.json"]: config_key = os.path.join(automation_path, config_name) if config_key in self.modified_times: del self.modified_times[config_key] delete_flag = True - + # Use exact path matching instead of substring - if any(item["path"] == automation_path for item in self.indices[folder_type]): - logger.debug(f"Removed index entry (if it exists) for {folder_type} : {automation_dir}") + if any( + item["path"] == automation_path for item in self.indices[folder_type]): + logger.debug( + f"Removed index entry (if it exists) for {folder_type} : {automation_dir}") delete_flag = True self._remove_index_entry(automation_path) - + if delete_flag: changed = True continue @@ -261,17 +270,18 @@ def _index_single_repo(self, repo, repos_changed=False, current_item_keys=None): "mtime": mtime, "date_time": datetime.fromtimestamp(mtime).strftime("%Y-%m-%d %H:%M:%S") } - + # meta file changed, so reindex - self._process_config_file(config_path, folder_type, automation_path, repo) + self._process_config_file( + config_path, folder_type, automation_path, repo) changed = True - + return changed def build_index(self, force_rebuild=False): """ Build shared indices for script, cache, and experiment folders across all repositories. - + Returns: None """ @@ -288,16 +298,18 @@ def build_index(self, force_rebuild=False): for index_type, index_path in self.index_files.items(): if not os.path.exists(index_path): missing_indices.append(index_type) - + if missing_indices: - logger.warning(f"Missing index files: {', '.join(missing_indices)}. Forcing full index rebuild...") + logger.warning( + f"Missing index files: {', '.join(missing_indices)}. Forcing full index rebuild...") self.modified_times = {} self.indices = {k: [] for k in self.index_files.keys()} force_rebuild = True - + # index each repo for repo in self.repos: - repo_changed = self._index_single_repo(repo, force_rebuild, current_item_keys) + repo_changed = self._index_single_repo( + repo, force_rebuild, current_item_keys) if repo_changed: changed = True @@ -311,10 +323,12 @@ def build_index(self, force_rebuild=False): self._remove_index_entry(folder_key) changed = True if deleted_keys: - logger.debug(f"Deleted keys removed from modified times and indices: {deleted_keys}") + logger.debug( + f"Deleted keys removed from modified times and indices: {deleted_keys}") if force_rebuild or changed: - logger.debug("Changes detected, saving updated index and modified times.") + logger.debug( + "Changes detected, saving updated index and modified times.") self._save_modified_times() self._save_indices() @@ -330,19 +344,21 @@ def _remove_index_entry(self, key): ] removed_count = original_count - len(self.indices[ft]) if removed_count > 0: - logger.debug(f"Removed {removed_count} item(s) from {ft} index") + logger.debug( + f"Removed {removed_count} item(s) from {ft} index") def _delete_by_uid(self, folder_type, uid, alias): """ Delete old index entry using UID (prevents duplicates). """ - #logger.debug(f"Deleting and updating index entry for the script {alias} with UID {uid}") + # logger.debug(f"Deleting and updating index entry for the script {alias} with UID {uid}") self.indices[folder_type] = [ item for item in self.indices[folder_type] if item["uid"] != uid ] - def _process_config_file(self, config_file, folder_type, folder_path, repo): + def _process_config_file( + self, config_file, folder_type, folder_path, repo): """ Process a single configuration file (meta.json or meta.yaml) and add its data to the corresponding index. @@ -367,11 +383,13 @@ def _process_config_file(self, config_file, folder_type, folder_path, repo): with open(config_file, "r") as f: data = json.load(f) or {} else: - logger.warning(f"Skipping {config_file}: Unsupported file format.") + logger.warning( + f"Skipping {config_file}: Unsupported file format.") return - + if not isinstance(data, dict): - logger.warning(f"Skipping {config_file}: Invalid or empty meta") + logger.warning( + f"Skipping {config_file}: Invalid or empty meta") return # Extract necessary fields unique_id = data.get("uid") @@ -389,7 +407,8 @@ def _process_config_file(self, config_file, folder_type, folder_path, repo): for w in warnings: logger.debug(f"Meta validation warning: {w}") if errors: - raise ValueError(f"Meta validation failed for {config_file}. Fix the above error(s) and try again.") + raise ValueError( + f"Meta validation failed for {config_file}. Fix the above error(s) and try again.") # Validate and add to indices self._delete_by_uid(folder_type, unique_id, alias) @@ -404,32 +423,32 @@ def _process_config_file(self, config_file, folder_type, folder_path, repo): except Exception as e: logger.error(f"Error processing {config_file}: {e}") - def _save_indices(self): """ Save the indices to JSON files. - + Returns: None """ - #logger.info(self.indices) + # logger.info(self.indices) for folder_type, index_data in self.indices.items(): output_file = self.index_files[folder_type] lock_file = output_file + ".lock" try: with self._file_lock_with_incremental_timeout(lock_file): - #logger.debug(f"Lock acquired at {lock_file} for Saving Index for {folder_type}") + # logger.debug(f"Lock acquired at {lock_file} for Saving Index for {folder_type}") with open(output_file, "w") as f: - json.dump(index_data, f, indent=4, cls=CustomJSONEncoder) - #logger.debug(f"Shared index for {folder_type} saved to {output_file}.") - + json.dump( + index_data, f, indent=4, cls=CustomJSONEncoder) + # logger.debug(f"Shared index for {folder_type} saved to {output_file}.") + except Timeout: logger.error(f"Timeout acquiring lock {lock_file}") except Exception as e: - logger.error(f"Error saving shared index for {folder_type}: {e}") - + logger.error( + f"Error saving shared index for {folder_type}: {e}") def add_repo(self, repo): """ @@ -441,7 +460,6 @@ def add_repo(self, repo): self._save_indices() self._save_modified_times() - def remove_repo_from_index(self, repo_path): """ Remove all index entries and modified times belonging to a repo. diff --git a/mlc/logger.py b/mlc/logger.py index 8ea79625b..e30dcd1e2 100644 --- a/mlc/logger.py +++ b/mlc/logger.py @@ -5,6 +5,7 @@ # Initialize colorama for Windows support colorama_init(autoreset=True) + class ColoredFormatter(logging.Formatter): """Custom formatter class to add colors to log levels""" COLORS = { @@ -16,14 +17,17 @@ class ColoredFormatter(logging.Formatter): def format(self, record): # Pad filename and line number for alignment - record.filename = f"{record.filename:<15}" # Left-align filename with 15 char width - record.lineno = f"{record.lineno:>4}" # Right-align line number with 4 char width - + # Left-align filename with 15 char width + record.filename = f"{record.filename:<15}" + # Right-align line number with 4 char width + record.lineno = f"{record.lineno:>4}" + # Trim WARNING to WARN levelname = "WARN" if record.levelname == "WARNING" else record.levelname - + # Pad and add color to the levelname - levelname_padded = f"{levelname:<5}" # Left-align levelname with 5 char width + # Left-align levelname with 5 char width + levelname_padded = f"{levelname:<5}" if record.levelname in self.COLORS: record.levelname = f"{self.COLORS[record.levelname]}{levelname_padded}{Style.RESET_ALL}" else: @@ -32,23 +36,26 @@ def format(self, record): # Set up logging configuration -def setup_logging(log_path = os.getcwd(), log_file = '.mlc-log.txt'): - +def setup_logging(log_path=os.getcwd(), log_file='.mlc-log.txt'): + if not logger.hasHandlers(): - logFormatter = ColoredFormatter('[%(asctime)s %(filename)s:%(lineno)s %(levelname)s] - %(message)s') + logFormatter = ColoredFormatter( + '[%(asctime)s %(filename)s:%(lineno)s %(levelname)s] - %(message)s') # by default logging level is set to INFO is being set logger.setLevel(logging.INFO) - # File hander for logging in file in the specified path - file_handler = logging.FileHandler("{0}/{1}".format(log_path, log_file)) - file_handler.setFormatter(logging.Formatter('[%(asctime)s %(filename)s:%(lineno)d %(levelname)s] - %(message)s')) + file_handler = logging.FileHandler( + "{0}/{1}".format(log_path, log_file)) + file_handler.setFormatter(logging.Formatter( + '[%(asctime)s %(filename)s:%(lineno)d %(levelname)s] - %(message)s')) logger.addHandler(file_handler) - + # Console handler for logging on console consoleHandler = logging.StreamHandler() consoleHandler.setFormatter(logFormatter) logger.addHandler(consoleHandler) logger.propagate = False + logger = logging.getLogger(__name__) diff --git a/mlc/main.py b/mlc/main.py index 9b8b66b10..c164f5b6d 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -26,7 +26,7 @@ class Automation: path = None def __init__(self, action, automation_type, automation_file): - #logger.info(f"action = {action}") + # logger.info(f"action = {action}") self.action_object = action self.automation_type = automation_type self.path = os.path.dirname(automation_file) @@ -68,16 +68,19 @@ def search(self, i): if tags or uid or i.get('all'): for res in target_index: c_tags = res["tags"] - if set(p_tags).issubset(set(c_tags)) and set(n_tags).isdisjoint(set(c_tags)) and (not uid or uid == res['uid']) and (not alias or alias == res['alias']): + if set(p_tags).issubset(set(c_tags)) and set(n_tags).isdisjoint(set(c_tags)) and ( + not uid or uid == res['uid']) and (not alias or alias == res['alias']): it = Item(res['path'], res['repo']) result.append(it) - #logger.info(result) + # logger.info(result) return {'return': 0, 'list': result} - #indices + # indices + mlc_run_cmd = None -def mlc_expand_short(action, target = "script"): + +def mlc_expand_short(action, target="script"): global mlc_run_cmd mlc_run_cmd = shlex.join(sys.argv) # Insert the positional argument into sys.argv for the main function @@ -100,19 +103,31 @@ def mlc_expand_short(action, target = "script"): logger.error(f"{e}") sys.exit(1) + def mlcr(): mlc_expand_short("run") + + def mlcd(): mlc_expand_short("docker") + + def mlcdr(): mlc_expand_short("docker") + + def mlcrr(): mlc_expand_short("remote-run") + + def mlce(): mlc_expand_short("experiment") + + def mlct(): mlc_expand_short("test") + def mlcp(): mlc_expand_short("pull", "repo") @@ -125,11 +140,13 @@ def process_console_output(res, target, action, run_args): if len(res['list']) == 0: # Only show warning if not in path-only mode if not run_args.get('path_only'): - logger.warning(f"""No {target} entry found for the specified input: {run_args}!""") + logger.warning( + f"""No {target} entry found for the specified input: {run_args}!""") else: for item in res['list']: if run_args.get('path_only'): - # Print only the path without logger prefix for script-friendly output + # Print only the path without logger prefix for + # script-friendly output print(item.path) else: logger.info(f"""Item path: {item.path}""") @@ -137,9 +154,12 @@ def process_console_output(res, target, action, run_args): if "message" in res: logger.info(res['message']) if "warnings" in res: - logger.warning(f"{len(res['warnings'])} warning(s) found during the execution of the mlc command.") + logger.warning( + f"{len(res['warnings'])} warning(s) found during the execution of the mlc command.") for warning in res["warnings"]: - logger.warning(f"Warning code: {warning['code']}, Discription: {warning['description']}") + logger.warning( + f"Warning code: {warning['code']}, Discription: {warning['description']}") + if default_parent is None: default_parent = Action() @@ -148,6 +168,7 @@ def process_console_output(res, target, action, run_args): log_flag_aliases = {'-v': '--verbose', '-s': '--silent'} log_levels = {'--verbose': logging.DEBUG, '--silent': logging.WARNING} + def convert_hyphen_to_underscore_in_args(): for i, arg in enumerate(sys.argv): if arg.startswith("--"): @@ -163,37 +184,75 @@ def convert_hyphen_to_underscore_in_args(): sys.argv[i] = f"--{new_name}" - def build_pre_parser(): pre_parser = argparse.ArgumentParser(add_help=False) - pre_parser.add_argument("action", nargs="?", help="Top-level action (run, build, help, etc.)") - pre_parser.add_argument("target", choices=['run', 'script', 'cache', 'repo', 'repos', 'experiment', 'all'], nargs="?", help="Target (repo, script, cache, ...)") + pre_parser.add_argument( + "action", + nargs="?", + help="Top-level action (run, build, help, etc.)") + pre_parser.add_argument( + "target", + choices=[ + 'run', + 'script', + 'cache', + 'repo', + 'repos', + 'experiment', + 'all'], + nargs="?", + help="Target (repo, script, cache, ...)") pre_parser.add_argument("-h", "--help", action="store_true") return pre_parser def build_parser(pre_args): - parser = argparse.ArgumentParser(prog="mlc", description="Manage repos, scripts, and caches.", add_help=False) - subparsers = parser.add_subparsers(dest="command", required=not pre_args.help) + parser = argparse.ArgumentParser( + prog="mlc", + description="Manage repos, scripts, and caches.", + add_help=False) + subparsers = parser.add_subparsers( + dest="command", required=not pre_args.help) # General commands - for action in ['run', 'pull', 'test', 'add', 'show', 'list', 'find', 'search', 'rm', 'cp', 'mv', 'help', 'prune']: + for action in ['run', 'pull', 'test', 'add', 'show', 'list', + 'find', 'search', 'rm', 'cp', 'mv', 'help', 'prune']: p = subparsers.add_parser(action, add_help=False) p.add_argument('target', choices=['repo', 'repos', 'script', 'cache']) - p.add_argument('details', nargs='?', help='Details or identifier (optional)') + p.add_argument( + 'details', + nargs='?', + help='Details or identifier (optional)') p.add_argument('extra', nargs=argparse.REMAINDER) # Reindex command (target is optional) reindex_parser = subparsers.add_parser('reindex', add_help=False) - reindex_parser.add_argument('target', nargs='?', choices=['repo', 'repos', 'script', 'cache', 'experiment', 'all'], help='Target to reindex (optional, defaults to all)') - reindex_parser.add_argument('details', nargs='?', help='Details or identifier (optional)') + reindex_parser.add_argument( + 'target', + nargs='?', + choices=[ + 'repo', + 'repos', + 'script', + 'cache', + 'experiment', + 'all'], + help='Target to reindex (optional, defaults to all)') + reindex_parser.add_argument( + 'details', + nargs='?', + help='Details or identifier (optional)') reindex_parser.add_argument('extra', nargs=argparse.REMAINDER) # Script-only - for action in ['docker', 'docker-run', 'experiment', 'remote-run', 'doc', 'lint']: + for action in ['docker', 'docker-run', + 'experiment', 'remote-run', 'doc', 'lint']: p = subparsers.add_parser(action, add_help=False) p.add_argument('target', choices=['script', 'run']) - p.add_argument('details', nargs='?', help='Details or identifier (optional)') + p.add_argument( + 'details', + nargs='?', + help='Details or identifier (optional)') p.add_argument('extra', nargs=argparse.REMAINDER) # Load cfg @@ -205,7 +264,7 @@ def build_parser(pre_args): def configure_logging(args): if hasattr(args, 'extra') and args.extra: args.extra[:] = [log_flag_aliases.get(a, a) for a in args.extra] - + for flag, level in log_levels.items(): if flag in args.extra: logger.setLevel(level) @@ -220,17 +279,21 @@ def build_run_args(args): run_args = res['args_dict'] if not mlc_run_cmd: - mlc_run_cmd = shlex.join([os.path.basename(sys.argv[0]), *sys.argv[1:]]) + mlc_run_cmd = shlex.join( + [os.path.basename(sys.argv[0]), *sys.argv[1:]]) run_args['mlc_run_cmd'] = mlc_run_cmd if args.command in ['pull', 'rm', 'add', 'find'] and args.target == "repo": run_args['repo'] = args.details - if args.command in ['docker', 'docker-run', 'experiment', 'remote-run', 'doc', 'lint'] and args.target == "run": - #run_args['target'] = 'script' #dont modify this as script might have target as in input + if args.command in ['docker', 'docker-run', 'experiment', + 'remote-run', 'doc', 'lint'] and args.target == "run": + # run_args['target'] = 'script' #dont modify this as script might have + # target as in input args.target = "script" - if args.details and not utils.is_uid(args.details) and not run_args.get("tags") and args.target in ["script", "cache"]: + if args.details and not utils.is_uid(args.details) and not run_args.get( + "tags") and args.target in ["script", "cache"]: run_args['tags'] = args.details if not run_args.get('details') and args.details: @@ -246,13 +309,14 @@ def build_run_args(args): if hasattr(args, 'command') and args.command == "reindex": if hasattr(args, 'target') and args.target: run_args['reindex_target'] = args.target - + # Check for path-only flag (for script-friendly output) if run_args.get('path_only') or run_args.get('p'): run_args['path_only'] = True return run_args + def is_quoted(arg): return (arg.startswith("'") and arg.endswith("'")) or \ (arg.startswith('"') and arg.endswith('"')) @@ -279,6 +343,7 @@ def check_raw_arguments_for_non_ascii(): "Please retype the arguments using plain ASCII.\n") sys.exit(1) + def main(): """ MLCFlow is a CLI tool for managing repos, scripts, and caches. @@ -286,38 +351,38 @@ def main(): You can also use this tool for any of your workflow automation tasks. MLCFlow CLI operates using actions and targets. It enables users to perform actions on specified targets using the following syntax: - + mlc [options] Here, actions represent the operations to be performed, and the target is the object on which the action is executed. Each target has a specific set of actions to tailor automation workflows, as shown below: - + | Target | Actions | |---------|-----------------------------------------------------------| | script | run, find/search, rm, mv, cp, add, test, docker-run, show | | cache | find/search, rm, show | | repo | pull, search, rm, list, find/search | - + Example: mlc run script detect-os - - For help related to a particular target, run: - + + For help related to a particular target, run: + mlc --help/-h Examples: mlc script --help mlc repo -h - - For help related to a specific action for a target, run: - + + For help related to a specific action for a target, run: + mlc --help/-h Examples: mlc run script --help mlc pull repo -h """ - + check_raw_arguments_for_non_ascii() convert_hyphen_to_underscore_in_args() @@ -325,9 +390,11 @@ def main(): pre_args, remaining_args = pre_parser.parse_known_args() parser = build_parser(pre_args) - # Force full parsing for reindex command even without target, or if there are remaining args or target - args = parser.parse_args() if (remaining_args or pre_args.target or pre_args.action == 'reindex') else pre_args - + # Force full parsing for reindex command even without target, or if there + # are remaining args or target + args = parser.parse_args() if ( + remaining_args or pre_args.target or pre_args.action == 'reindex') else pre_args + if hasattr(args, 'command') and args.command: args.command = args.command.replace("-", "_") @@ -340,8 +407,10 @@ def main(): if pre_args.action.startswith("docker"): pre_args.target = "script" else: - logger.error(f"Invalid action-target {pre_args.action} - {pre_args.target} combination") - raise Exception(f"Invalid action-target {pre_args.action} - {pre_args.target} combination") + logger.error( + f"Invalid action-target {pre_args.action} - {pre_args.target} combination") + raise Exception( + f"Invalid action-target {pre_args.action} - {pre_args.target} combination") if not pre_args.action and not pre_args.target: help_text += main.__doc__ elif pre_args.action and not pre_args.target: @@ -353,7 +422,8 @@ def main(): actions = get_action(pre_args.target, default_parent) help_text += actions.__doc__ # iterate through every method - for method_name, method in inspect.getmembers(actions.__class__, inspect.isfunction): + for method_name, method in inspect.getmembers( + actions.__class__, inspect.isfunction): method = getattr(actions, method_name) if method.__doc__ and not method.__doc__.startswith("_"): help_text += method.__doc__ @@ -363,30 +433,36 @@ def main(): method = getattr(actions, pre_args.action) help_text += actions.__doc__ help_text += method.__doc__ - except: - logger.error(f"Error: '{pre_args.action}' is not supported for {pre_args.target}.") + except BaseException: + logger.error( + f"Error: '{pre_args.action}' is not supported for {pre_args.target}.") if help_text != "": print(help_text) sys.exit(0) if hasattr(args, 'target') and args.target == "repos": args.target = "repo" - - # Handle reindex command specially - it can work without a target or with 'all' + + # Handle reindex command specially - it can work without a target or with + # 'all' if hasattr(args, 'command') and args.command == "reindex": - if not hasattr(args, 'target') or not args.target or args.target == "all": + if not hasattr( + args, 'target') or not args.target or args.target == "all": # Reindex all targets by using the base Action class args.target = "script" # Use script as default to get access to the action - + # Check if command attribute exists if not hasattr(args, 'command'): logging.error("Error: No command specified.") sys.exit(1) - + action = get_action(args.target, default_parent) if not action or not hasattr(action, args.command): - logging.error("Error: '%s' is not supported for %s.", args.command, args.target) + logging.error( + "Error: '%s' is not supported for %s.", + args.command, + args.target) sys.exit(1) method = getattr(action, args.command) @@ -397,6 +473,7 @@ def main(): process_console_output(res, args.target, args.command, run_args) + if __name__ == '__main__': try: main() @@ -412,4 +489,3 @@ def main(): else: logger.error(f"{e}") sys.exit(1) - diff --git a/mlc/meta_schema.py b/mlc/meta_schema.py index 567d1b85b..ed7760110 100644 --- a/mlc/meta_schema.py +++ b/mlc/meta_schema.py @@ -26,220 +26,223 @@ # key -> set of allowed type names TOP_LEVEL_SCHEMA = { # Identity (required) - "alias": STR, - "uid": STR, - "automation_alias": STR, - "automation_uid": STR, + "alias": STR, + "uid": STR, + "automation_alias": STR, + "automation_uid": STR, # Metadata - "name": STR, - "category": STR, - "tags": LIST, # list[str] - "tags_help": STR, - "developers": STR, - "sort": INT, - "category_sort": INT, - "private": BOOL, - "min_mlc_version": STR, + "name": STR, + "category": STR, + "tags": LIST, # list[str] + "tags_help": STR, + "developers": STR, + "sort": INT, + "category_sort": INT, + "private": BOOL, + "min_mlc_version": STR, # Environment - "env": DICT, # dict[str, str] - "default_env": DICT, # dict[str, str] - "new_env_keys": LIST, # list[str] - "new_state_keys": LIST, # list[str] - "local_env_keys": LIST, # list[str] - "file_path_env_keys": LIST, # list[str] - "folder_path_env_keys": LIST, # list[str] + "env": DICT, # dict[str, str] + "default_env": DICT, # dict[str, str] + "new_env_keys": LIST, # list[str] + "new_state_keys": LIST, # list[str] + "local_env_keys": LIST, # list[str] + "file_path_env_keys": LIST, # list[str] + "folder_path_env_keys": LIST, # list[str] # Cache - "cache": STR_OR_BOOL, - "can_force_cache": BOOL, - "cache_expiration": STR, + "cache": STR_OR_BOOL, + "can_force_cache": BOOL, + "cache_expiration": STR, "extra_cache_tags_from_env": LIST, # list[str] - "clean_files": LIST, # list[str] - "clean_output_files": LIST, # list[str] + "clean_files": LIST, # list[str] + "clean_output_files": LIST, # list[str] # Input mapping - "input_mapping": {*DICT, "NoneType"}, # dict[str, str] input_name -> ENV_KEY (or null) - "input_description": {*DICT, "NoneType"}, # dict[str, dict] input_name -> {desc, choices, ...} (or null) - "env_key_mappings": DICT, # dict[str, str] + # dict[str, str] input_name -> ENV_KEY (or null) + "input_mapping": {*DICT, "NoneType"}, + # dict[str, dict] input_name -> {desc, choices, ...} (or null) + "input_description": {*DICT, "NoneType"}, + "env_key_mappings": DICT, # dict[str, str] # Dependencies - "deps": LIST, # list[dep_entry] - "prehook_deps": LIST, # list[dep_entry] - "posthook_deps": LIST, # list[dep_entry] - "post_deps": LIST, # list[dep_entry] - "predeps": BOOL, + "deps": LIST, # list[dep_entry] + "prehook_deps": LIST, # list[dep_entry] + "posthook_deps": LIST, # list[dep_entry] + "post_deps": LIST, # list[dep_entry] + "predeps": BOOL, # Variations - "variations": DICT, # dict[str, variation_entry] - "variation_groups_order": LIST, # list[str] - "default_variation": STR, - "default_variations": DICT, # dict[str, str] + "variations": DICT, # dict[str, variation_entry] + "variation_groups_order": LIST, # list[str] + "default_variation": STR, + "default_variations": DICT, # dict[str, str] "invalid_variation_combinations": LIST, # list[list[str]] - "valid_variation_combinations": LIST, # list[list[str]] + "valid_variation_combinations": LIST, # list[list[str]] # Versions - "versions": DICT, # dict[str, version_entry] - "default_version": STR, + "versions": DICT, # dict[str, version_entry] + "default_version": STR, # Docker - "docker": DICT, # dict - see DOCKER_SCHEMA + "docker": DICT, # dict - see DOCKER_SCHEMA # Output / debugging - "print_env_at_the_end": DICT, # dict[str, list[str]] + "print_env_at_the_end": DICT, # dict[str, list[str]] "print_files_if_script_error": LIST, # list[str] - "warnings": LIST, # list[str] - "sudo_install": BOOL, + "warnings": LIST, # list[str] + "sudo_install": BOOL, # Conditional meta update - "update_meta_if_env": LIST, # list[dict] - "remote_run": DICT, + "update_meta_if_env": LIST, # list[dict] + "remote_run": DICT, # Tests - "tests": DICT, # dict - see TESTS_SCHEMA + "tests": DICT, # dict - see TESTS_SCHEMA } # ─── Dependency entry keys ────────────────────────────────────── DEP_ENTRY_SCHEMA = { - "tags": STR, - "names": STR_OR_LIST, - "env": DICT, - "enable_if_env": DICT, - "skip_if_env": DICT, - "skip_if_any_env": DICT, - "enable_if_any_env": DICT, - "extra_cache_tags": STR, + "tags": STR, + "names": STR_OR_LIST, + "env": DICT, + "enable_if_env": DICT, + "skip_if_env": DICT, + "skip_if_any_env": DICT, + "enable_if_any_env": DICT, + "extra_cache_tags": STR, "update_tags_from_env_with_prefix": DICT, - "update_tags_from_env": LIST, - "force_env_keys": LIST, - "force_cache": BOOL, - "reuse_version": BOOL, - "inherit_variation_tags": STR_OR_BOOL, - "skip_inherit_variation_groups": LIST, - "version": STR, - "version_min": STR, - "version_max": STR_OR_FLOAT, - "version_max_usable": STR_OR_FLOAT, - "dynamic": BOOL, - "ignore_missing": BOOL, - "skip_if_fake_run": BOOL, - "verify": BOOL, - "md5sum": STR, - "revision": STR, - "model_filename": STR, - "full_subfolder": STR, - "env_key": STR, - "continue_on_error": BOOL, - "ignore_script_error": BOOL, - "inherit_cache_expiration": BOOL, - "update_tags_if_env": DICT, - "update_meta_if_env": LIST, + "update_tags_from_env": LIST, + "force_env_keys": LIST, + "force_cache": BOOL, + "reuse_version": BOOL, + "inherit_variation_tags": STR_OR_BOOL, + "skip_inherit_variation_groups": LIST, + "version": STR, + "version_min": STR, + "version_max": STR_OR_FLOAT, + "version_max_usable": STR_OR_FLOAT, + "dynamic": BOOL, + "ignore_missing": BOOL, + "skip_if_fake_run": BOOL, + "verify": BOOL, + "md5sum": STR, + "revision": STR, + "model_filename": STR, + "full_subfolder": STR, + "env_key": STR, + "continue_on_error": BOOL, + "ignore_script_error": BOOL, + "inherit_cache_expiration": BOOL, + "update_tags_if_env": DICT, + "update_meta_if_env": LIST, } # ─── Variation entry keys ─────────────────────────────────────── VARIATION_ENTRY_SCHEMA = { - "env": {*DICT, "NoneType"}, - "group": STR, - "default": STR_OR_BOOL, - "default_variations": DICT, - "deps": LIST, - "prehook_deps": LIST, - "posthook_deps": LIST, - "post_deps": LIST, - "add_deps": DICT, - "add_deps_recursive": DICT, - "add_deps_tags": DICT, - "new_env_keys": LIST, - "new_state_keys": LIST, - "base": LIST, - "adr": DICT, - "ad": DICT, - "default_env": DICT, - "state": DICT, - "const": DICT, - "docker": DICT, - "alias": STR, - "default_version": STR_OR_FLOAT, - "required_disk_space": INT, - "cache_expiration": {*STR, *INT}, - "cache": BOOL, - "force_cache": BOOL, - "update_meta_if_env": LIST, - "warning": STR, - "warnings": LIST, - "names": LIST, - "default_variation": DICT, + "env": {*DICT, "NoneType"}, + "group": STR, + "default": STR_OR_BOOL, + "default_variations": DICT, + "deps": LIST, + "prehook_deps": LIST, + "posthook_deps": LIST, + "post_deps": LIST, + "add_deps": DICT, + "add_deps_recursive": DICT, + "add_deps_tags": DICT, + "new_env_keys": LIST, + "new_state_keys": LIST, + "base": LIST, + "adr": DICT, + "ad": DICT, + "default_env": DICT, + "state": DICT, + "const": DICT, + "docker": DICT, + "alias": STR, + "default_version": STR_OR_FLOAT, + "required_disk_space": INT, + "cache_expiration": {*STR, *INT}, + "cache": BOOL, + "force_cache": BOOL, + "update_meta_if_env": LIST, + "warning": STR, + "warnings": LIST, + "names": LIST, + "default_variation": DICT, } # ─── Docker section keys ──────────────────────────────────────── DOCKER_SCHEMA = { - "real_run": BOOL, - "run": BOOL, - "skip_run_cmd": STR_OR_BOOL, - "interactive": BOOL, - "pre_run_cmds": LIST, - "deps": LIST, - "mounts": LIST, - "input_mapping": DICT, - "input_paths": LIST, - "skip_input_for_fake_run": LIST, - "os": STR, - "os_version": STR, - "base_image": STR, - "mlc_repo": STR, - "mlc_repo_branch": STR, - "mlc_repo_flags": STR, - "extra_run_args": STR, - "all_gpus": STR, - "user": STR, - "use_host_user_id": BOOL, - "use_host_group_id": STR_OR_BOOL, - "skip_mlc_sys_upgrade": STR, - "shm_size": STR, - "port_maps": LIST, - "image_tag_extra": STR, - "fake_run_deps": BOOL, - "pass_docker_to_script": BOOL, - "mount_current_dir": STR, - "use_google_dns": BOOL, - "add_quotes_to_keys": LIST, - "device": STR, - "run_cmd_prefix": STR, - "pass_user_group": BOOL, - "default_env": DICT, - "env": DICT, + "real_run": BOOL, + "run": BOOL, + "skip_run_cmd": STR_OR_BOOL, + "interactive": BOOL, + "pre_run_cmds": LIST, + "deps": LIST, + "mounts": LIST, + "input_mapping": DICT, + "input_paths": LIST, + "skip_input_for_fake_run": LIST, + "os": STR, + "os_version": STR, + "base_image": STR, + "mlc_repo": STR, + "mlc_repo_branch": STR, + "mlc_repo_flags": STR, + "extra_run_args": STR, + "all_gpus": STR, + "user": STR, + "use_host_user_id": BOOL, + "use_host_group_id": STR_OR_BOOL, + "skip_mlc_sys_upgrade": STR, + "shm_size": STR, + "port_maps": LIST, + "image_tag_extra": STR, + "fake_run_deps": BOOL, + "pass_docker_to_script": BOOL, + "mount_current_dir": STR, + "use_google_dns": BOOL, + "add_quotes_to_keys": LIST, + "device": STR, + "run_cmd_prefix": STR, + "pass_user_group": BOOL, + "default_env": DICT, + "env": DICT, } # ─── Tests section keys ───────────────────────────────────────── TESTS_SCHEMA = { - "run_inputs": LIST, # list[dict] - each has variations_list, env, etc. - "needs_pat": BOOL, + "run_inputs": LIST, # list[dict] - each has variations_list, env, etc. + "needs_pat": BOOL, } # ─── Tests run_inputs entry keys ──────────────────────────────── TESTS_RUN_INPUT_SCHEMA = { - "variations_list": LIST, # list[str] - "env": DICT, - "test_input_index": STR, - "disable_run_script": BOOL, + "variations_list": LIST, # list[str] + "env": DICT, + "test_input_index": STR, + "disable_run_script": BOOL, } # ─── update_meta_if_env entry keys ────────────────────────────── UPDATE_META_IF_ENV_SCHEMA = { - "enable_if_env": DICT, - "enable_if_any_env": DICT, - "skip_if_env": DICT, - "skip_if_any_env": DICT, - "env": DICT, - "default_env": DICT, - "default_variations": DICT, - "docker": DICT, - "adr": DICT, - "ad": DICT, + "enable_if_env": DICT, + "enable_if_any_env": DICT, + "skip_if_env": DICT, + "skip_if_any_env": DICT, + "env": DICT, + "default_env": DICT, + "default_variations": DICT, + "docker": DICT, + "adr": DICT, + "ad": DICT, } + def validate_meta(data, file_path=""): """ Validate a script meta.yaml dict against the schema. @@ -282,7 +285,8 @@ def validate_meta(data, file_path=""): f"{prefix}Key '{key}' has type '{actual_type}', expected {allowed}") # Validate dependency lists - for dep_list_key in ["deps", "prehook_deps", "posthook_deps", "post_deps", "post_deps_off"]: + for dep_list_key in ["deps", "prehook_deps", + "posthook_deps", "post_deps", "post_deps_off"]: deps = data.get(dep_list_key) if deps is None: continue @@ -304,15 +308,19 @@ def validate_meta(data, file_path=""): errors.append( f"{prefix}{dep_list_key}[{i}].{dk} has type '{actual}', expected {allowed}") - # Validate enable_if_env/skip_if_env values are single strings/lists, not nested dicts - for ck in ["enable_if_env", "skip_if_env", "skip_if_any_env", "enable_if_any_env"]: + # Validate enable_if_env/skip_if_env values are single + # strings/lists, not nested dicts + for ck in ["enable_if_env", "skip_if_env", + "skip_if_any_env", "enable_if_any_env"]: cv = dep.get(ck) if isinstance(cv, dict): for ek, ev in cv.items(): - if isinstance(ev, (dict, list)) and not isinstance(ev, str): + if isinstance(ev, (dict, list) + ) and not isinstance(ev, str): if isinstance(ev, list): for item in ev: - if not isinstance(item, (str, int, float, bool)): + if not isinstance( + item, (str, int, float, bool)): errors.append( f"{prefix}{dep_list_key}[{i}].{ck}.{ek} list contains non-scalar: {type(item).__name__}") @@ -325,22 +333,27 @@ def validate_meta(data, file_path=""): continue for ek, ev in entry.items(): if ek not in UPDATE_META_IF_ENV_SCHEMA: - warnings.append(f"{prefix}update_meta_if_env[{i}]: unknown key '{ek}'") + warnings.append( + f"{prefix}update_meta_if_env[{i}]: unknown key '{ek}'") continue actual = type(ev).__name__ allowed = UPDATE_META_IF_ENV_SCHEMA[ek] if actual not in allowed: errors.append( f"{prefix}update_meta_if_env[{i}].{ek} has type '{actual}', expected {allowed}") - # Validate enable_if_env/skip_if_env values inside update_meta_if_env - for ck in ["enable_if_env", "skip_if_env", "skip_if_any_env", "enable_if_any_env"]: + # Validate enable_if_env/skip_if_env values inside + # update_meta_if_env + for ck in ["enable_if_env", "skip_if_env", + "skip_if_any_env", "enable_if_any_env"]: cv = entry.get(ck) if isinstance(cv, dict): for ek2, ev2 in cv.items(): - if isinstance(ev2, (dict, list)) and not isinstance(ev2, str): + if isinstance(ev2, (dict, list) + ) and not isinstance(ev2, str): if isinstance(ev2, list): for item in ev2: - if not isinstance(item, (str, int, float, bool)): + if not isinstance( + item, (str, int, float, bool)): errors.append( f"{prefix}update_meta_if_env[{i}].{ck}.{ek2} list contains non-scalar: {type(item).__name__}") @@ -366,7 +379,6 @@ def validate_meta(data, file_path=""): errors.append( f"{prefix}variations.{vname}.{vk} has type '{actual}', expected {allowed}") - # Validate docker section docker = data.get("docker") if isinstance(docker, dict): @@ -406,13 +418,23 @@ def validate_meta(data, file_path=""): # Check for variation entry keys mistakenly used as variation names # Exclude common short words that are legitimately used as both - _variation_name_allowlist = {"default", "base", "env", "ad", "cache", "state", "alias", "warning", "const"} + _variation_name_allowlist = { + "default", + "base", + "env", + "ad", + "cache", + "state", + "alias", + "warning", + "const"} if isinstance(variations, dict): for vname in variations: if vname in VARIATION_ENTRY_SCHEMA and vname not in _variation_name_allowlist: warnings.append( f"{prefix}variation '{vname}' looks like a variation property key used as a variation name") - # Also check if variation attrs contain keys that look like variation names + # Also check if variation attrs contain keys that look like + # variation names vattrs = variations[vname] if isinstance(vattrs, dict): for vk in vattrs: @@ -420,7 +442,8 @@ def validate_meta(data, file_path=""): continue # valid property if vk.startswith("MLC_"): continue # env override - # Check if this unknown key is actually a known variation name in this script + # Check if this unknown key is actually a known variation + # name in this script if vk in variations and vk != vname: warnings.append( f"{prefix}variations.{vname}: key '{vk}' matches another variation name - possible indentation error") diff --git a/mlc/repo_action.py b/mlc/repo_action.py index 4a262f886..78ec0ea2f 100644 --- a/mlc/repo_action.py +++ b/mlc/repo_action.py @@ -11,12 +11,13 @@ from .repo import Repo from .index import Index + class RepoAction(Action): """ #################################################################################################################### Repo Action #################################################################################################################### - + Currently, the following actions are supported for Repos: 1. add 2. find @@ -39,11 +40,10 @@ class RepoAction(Action): """ def __init__(self, parent=None): - #super().__init__(parent) + # super().__init__(parent) self.parent = parent self.__dict__.update(vars(parent)) - def add(self, run_args): """ #################################################################################################################### @@ -51,8 +51,8 @@ def add(self, run_args): Action: Add #################################################################################################################### - The `add` action is used to create a new MLC repository and register it in MLCFlow. - The newly created repo folder will be stored inside the `repos` folder within the parent MLC directory. + The `add` action is used to create a new MLC repository and register it in MLCFlow. + The newly created repo folder will be stored inside the `repos` folder within the parent MLC directory. Example Command: @@ -72,24 +72,27 @@ def add(self, run_args): """ if not run_args['repo']: logger.error("The repository to be added is not specified") - return {"return": 1, "error": "The repository to be added is not specified"} + return {"return": 1, + "error": "The repository to be added is not specified"} - i_repo_path = run_args['repo'] #can be a path, forder_name or URL + i_repo_path = run_args['repo'] # can be a path, forder_name or URL repo_folder_name = os.path.basename(i_repo_path.rstrip('/')) repo_path = os.path.join(self.repos_path, repo_folder_name) r = self.find(run_args) - + if r['return'] == 0 and len(r['list']) > 0: - return {'return': 1, "error": f"""Repo already exists at {r['list'][0]}"""} + return {'return': 1, + "error": f"""Repo already exists at {r['list'][0]}"""} for repo in self.repos: if repo.path == i_repo_path: - return {'return': 1, "error": f"""Repo already exists at {repo.path}"""} + return {'return': 1, + "error": f"""Repo already exists at {repo.path}"""} if not os.path.exists(i_repo_path): - #check if its an URL + # check if its an URL if utils.is_valid_url(i_repo_path): parsed = urlparse(i_repo_path) if parsed.hostname == "github.com": @@ -103,7 +106,7 @@ def add(self, run_args): else: repo_path = os.path.abspath(i_repo_path) - #check if it has MLC meta + # check if it has MLC meta meta_file = os.path.join(repo_path, "meta.yaml") if not os.path.exists(meta_file): meta = {} @@ -113,7 +116,7 @@ def add(self, run_args): utils.save_yaml(meta_file, meta) else: meta = utils.read_yaml(meta_file) - + self.register_repo(repo_path, meta, run_args.get('ignore_on_conflict')) return {'return': 0} @@ -121,45 +124,58 @@ def add(self, run_args): def conflicting_repo(self, repo_meta): for repo_object in self.repos: if repo_object.meta.get('uid', '') == '': - return {"return": 1, "error": f"UID is not present in file 'meta.yaml' in the repo path {repo_object.path}"} + return { + "return": 1, "error": f"UID is not present in file 'meta.yaml' in the repo path {repo_object.path}"} if repo_meta["uid"] == repo_object.meta.get('uid', ''): if repo_meta.get('path', '') == repo_object.path: - return {"return": 1, "error": f"Same repo is already registered"} + return {"return": 1, + "error": f"Same repo is already registered"} else: - return {"return": 1, "error": f"Conflicting with repo in the path {repo_object.path}", "conflicting_path": repo_object.path} + return {"return": 1, "error": f"Conflicting with repo in the path {repo_object.path}", + "conflicting_path": repo_object.path} return {"return": 0} - + def register_repo(self, repo_path, repo_meta, ignore_on_conflict=False): - + # Check UID conflicts is_conflict = self.conflicting_repo(repo_meta) if is_conflict['return'] > 0: if "UID not present" in is_conflict['error']: - logger.warning(f"UID not found in meta.yaml at {repo_path}. Repo can not be registered in MLC repos. Skipping...") + logger.warning( + f"UID not found in meta.yaml at {repo_path}. Repo can not be registered in MLC repos. Skipping...") return {"return": 0} - elif "already registered" in is_conflict["error"]: #at same path - #logger.warning(is_conflict["error"]) + elif "already registered" in is_conflict["error"]: # at same path + # logger.warning(is_conflict["error"]) logger.debug("No changes made to repos.json.") return {"return": 0} else: - logger.warning(f"The repo to be registered has conflict with the repo already in the path: {is_conflict['conflicting_path']}") + logger.warning( + f"The repo to be registered has conflict with the repo already in the path: {is_conflict['conflicting_path']}") if ignore_on_conflict: - logger.warning(f"Ignoring register as ignore_on_conflict is set") + logger.warning( + f"Ignoring register as ignore_on_conflict is set") return {"return": 0, 'conflict': True} self.unregister_repo(is_conflict['conflicting_path']) - logger.warning(f"{is_conflict['conflicting_path']} is unregistered.") - + logger.warning( + f"{is_conflict['conflicting_path']} is unregistered.") + if repo_meta.get('deps'): for dep in repo_meta['deps']: - self.pull_repo(dep['url'], branch=dep.get('branch'), checkout=dep.get('checkout'), ignore_on_conflict=dep.get('is_alias_okay', True)) + self.pull_repo( + dep['url'], + branch=dep.get('branch'), + checkout=dep.get('checkout'), + ignore_on_conflict=dep.get( + 'is_alias_okay', + True)) # Get the path to the repos.json file in $HOME/MLC repos_file_path = os.path.join(self.repos_path, 'repos.json') with open(repos_file_path, 'r') as f: repos_list = json.load(f) - + if repo_path not in repos_list: repos_list.append(repo_path) logger.info(f"Added new repo path: {repo_path}") @@ -167,7 +183,7 @@ def register_repo(self, repo_path, repo_meta, ignore_on_conflict=False): with open(repos_file_path, 'w') as f: json.dump(repos_list, f, indent=2) logger.info(f"Updated repos.json at {repos_file_path}") - + self.repos = self.load_repos_and_meta() repo_obj = next( (r for r in self.repos if r.path == repo_path), @@ -178,14 +194,13 @@ def register_repo(self, repo_path, repo_meta, ignore_on_conflict=False): index = Action.get_index(self) index.add_repo(repo_obj) logger.debug("Index file has been updated") - + return {'return': 0} def unregister_repo(self, repo_path): repos_file_path = os.path.join(self.repos_path, 'repos.json') - - return unregister_repo(repo_path, repos_file_path) + return unregister_repo(repo_path, repos_file_path) def find(self, run_args): """ @@ -208,25 +223,28 @@ def find(self, run_args): """ # Get repos_list using the existing method repos_list = self.load_repos_and_meta() - if(run_args.get('item', run_args.get('artifact'))): + if (run_args.get('item', run_args.get('artifact'))): repo = run_args.get('item', run_args.get('artifact')) else: - repo = run_args.get('repo', run_args.get('item', run_args.get('artifact'))) + repo = run_args.get( + 'repo', run_args.get( + 'item', run_args.get('artifact'))) # Check if repo is None or empty if not repo: return {"return": 1, "error": "Please enter a Repo Alias, Repo UID, or Repo URL in one of the following formats:\n" - "- @\n" - "- \n" - "- \n" - "- \n" - "- ,"} + "- @\n" + "- \n" + "- \n" + "- \n" + "- ,"} # Handle the different repo input formats repo_name = None repo_uid = None - # Check if the repo is in the format of a repo UID (alphanumeric string) + # Check if the repo is in the format of a repo UID (alphanumeric + # string) if utils.is_uid(repo): repo_uid = repo if "," in repo: @@ -242,7 +260,8 @@ def find(self, run_args): parsed = urlparse(repo) except Exception: parsed = None - if parsed and parsed.scheme in ("http", "https") and parsed.hostname == "github.com": + if parsed and parsed.scheme in ( + "http", "https") and parsed.hostname == "github.com": result = self.github_url_to_user_repo_format(repo) if result["return"] == 0: repo_name = result["value"] @@ -254,11 +273,10 @@ def find(self, run_args): # Check if repo_name exists in repos.json matched_repo_path = None for repo_obj in repos_list: - if repo_name and repo_name == os.path.basename(repo_obj.path) : + if repo_name and repo_name == os.path.basename(repo_obj.path): matched_repo_path = repo_obj break - # Search through self.repos for matching repos lst = [] for i in self.repos: @@ -267,22 +285,23 @@ def find(self, run_args): elif repo_name == i.meta['alias']: lst.append(i) - # After loop, check if any match was found if not lst and not matched_repo_path: # Determine error message based on input if utils.is_uid(repo): - return {"return": 1, "error": f"No repository with UID: '{repo_uid}' was found"} + return { + "return": 1, "error": f"No repository with UID: '{repo_uid}' was found"} elif "," in repo and not matched_repo_path: - return {"return": 1, "error": f"No repository with alias: '{repo_name}' and UID: '{repo_uid}' was found"} + return { + "return": 1, "error": f"No repository with alias: '{repo_name}' and UID: '{repo_uid}' was found"} else: - return {"return": 1, "error": f"No repository with alias: '{repo_name}' was found"} + return { + "return": 1, "error": f"No repository with alias: '{repo_name}' was found"} - # Append the matched repo path - if(len(lst)==0 and matched_repo_path): + if (len(lst) == 0 and matched_repo_path): lst.append(matched_repo_path) - + return {'return': 0, 'list': lst} def github_url_to_user_repo_format(self, url): @@ -294,19 +313,22 @@ def github_url_to_user_repo_format(self, url): # """ # Regex to match GitHub URLs pattern = r"(?:https?://)?(?:www\.)?github\.com/([^/]+)/([^/.]+)(?:\.git)?" - + match = re.match(pattern, url) if match: user, repo_name = match.groups() return {"return": 0, "value": f"{user}@{repo_name}"} else: - return {"return": 0, "value": os.path.basename(url).replace(".git", "")} + return {"return": 0, "value": os.path.basename( + url).replace(".git", "")} + + def pull_repo(self, repo_url, branch=None, checkout=None, tag=None, + pat=None, ssh=None, ignore_on_conflict=False, repo_path=None): - def pull_repo(self, repo_url, branch=None, checkout = None, tag = None, pat = None, ssh = None, ignore_on_conflict = False, repo_path = None): - # Determine the checkout path from environment or default - repo_base_path = self.repos_path # either the value will be from 'MLC_REPOS' - os.makedirs(repo_base_path, exist_ok=True) # Ensure the directory exists + repo_base_path = self.repos_path # either the value will be from 'MLC_REPOS' + # Ensure the directory exists + os.makedirs(repo_base_path, exist_ok=True) # Handle user@repo format (convert to standard GitHub URL) if re.match(r'^[\w-]+@[\w-]+$', repo_url): @@ -325,7 +347,6 @@ def pull_repo(self, repo_url, branch=None, checkout = None, tag = None, pat = No else: repo_url = res["url"] - # Extract the repo name from URL repo_name = repo_url.split('/')[-1].replace('.git', '') res = self.github_url_to_user_repo_format(repo_url) @@ -340,39 +361,58 @@ def pull_repo(self, repo_url, branch=None, checkout = None, tag = None, pat = No # If the directory doesn't exist, clone it if not os.path.exists(repo_path): logger.info(f"Cloning repository {repo_url} to {repo_path}...") - + # Build clone command without branch if not provided clone_command = ['git', 'clone', repo_url, repo_path] if branch: - clone_command = ['git', 'clone', '--branch', branch, repo_url, repo_path] - + clone_command = [ + 'git', + 'clone', + '--branch', + branch, + repo_url, + repo_path] + subprocess.run(clone_command, check=True) else: - logger.info(f"Repository {repo_name} already exists at {repo_path}. Checking for local changes...") - + logger.info( + f"Repository {repo_name} already exists at {repo_path}. Checking for local changes...") + # Check for local changes - status_command = ['git', '-C', repo_path, 'status', '--porcelain', '--untracked-files=no'] - local_changes = subprocess.run(status_command, capture_output=True, text=True) + status_command = [ + 'git', + '-C', + repo_path, + 'status', + '--porcelain', + '--untracked-files=no'] + local_changes = subprocess.run( + status_command, capture_output=True, text=True) if local_changes.stdout.strip(): - logger.warning("There are local changes in the repository. Please commit or stash them before checking out.") + logger.warning( + "There are local changes in the repository. Please commit or stash them before checking out.") print(local_changes.stdout.strip()) - return {"return": 0, "warning": f"Local changes detected in the already existing repository: {repo_path}, skipping the pull"} + return { + "return": 0, "warning": f"Local changes detected in the already existing repository: {repo_path}, skipping the pull"} else: - logger.info("No local changes detected. Pulling latest changes...") - subprocess.run(['git', '-C', repo_path, 'pull'], check=True) + logger.info( + "No local changes detected. Pulling latest changes...") + subprocess.run( + ['git', '-C', repo_path, 'pull'], check=True) logger.info("Repository successfully pulled.") if tag: - checkout = "tags/"+tag + checkout = "tags/" + tag # Checkout to a specific branch or commit if --checkout is provided if checkout or tag: logger.info(f"Checking out to {checkout} in {repo_path}...") - subprocess.run(['git', '-C', repo_path, 'checkout', checkout], check=True) - - #if not tag: + subprocess.run( + ['git', '-C', repo_path, 'checkout', checkout], check=True) + + # if not tag: # subprocess.run(['git', '-C', repo_path, 'pull'], check=True) # logger.info("Repository successfully pulled.") @@ -381,9 +421,10 @@ def pull_repo(self, repo_url, branch=None, checkout = None, tag = None, pat = No # check the meta file to obtain uids meta_file_path = os.path.join(repo_path, 'meta.yaml') if not os.path.exists(meta_file_path): - logger.warning(f"meta.yaml not found in {repo_path}. Repo pulled but not registered in MLC repos. Skipping...") + logger.warning( + f"meta.yaml not found in {repo_path}. Repo pulled but not registered in MLC repos. Skipping...") return {"return": 0} - + with open(meta_file_path, 'r') as meta_file: meta_data = yaml.safe_load(meta_file) meta_data["path"] = repo_path @@ -397,7 +438,8 @@ def pull_repo(self, repo_url, branch=None, checkout = None, tag = None, pat = No except subprocess.CalledProcessError as e: return {'return': 1, 'error': f"Git command failed: {e}"} except Exception as e: - return {'return': 1, 'error': f"Error pulling repository: {str(e)}"} + return {'return': 1, + 'error': f"Error pulling repository: {str(e)}"} def pull(self, run_args): """ @@ -408,8 +450,8 @@ def pull(self, run_args): The `pull` action clones an MLC repository and registers it in MLC. - If the repository already exists locally in the MLC repos directory, it fetches the latest changes only if there are no - uncommited modifications(excluding untracked files/folders). The `pull` action could be also used to checkout + If the repository already exists locally in the MLC repos directory, it fetches the latest changes only if there are no + uncommited modifications(excluding untracked files/folders). The `pull` action could be also used to checkout to a particular branch, commit or release tag using flags --checkout and --tag. Example Command: @@ -425,7 +467,7 @@ def pull(self, run_args): Example Output: anandhu@anandhu-VivoBook-ASUSLaptop-X515UA-M515UA:~$ mlc pull repo mlcommons@mlperf-automations - [2025-02-19 16:46:27,208 main.py:1260 INFO] - Cloning repository https://github.com/mlcommons/mlperf-automations.git + [2025-02-19 16:46:27,208 main.py:1260 INFO] - Cloning repository https://github.com/mlcommons/mlperf-automations.git to /home/anandhu/MLC/repos/mlcommons@mlperf-automations... Cloning into '/home/anandhu/MLC/repos/mlcommons@mlperf-automations'... remote: Enumerating objects: 77610, done. @@ -439,7 +481,7 @@ def pull(self, run_args): [2025-02-19 16:46:57,605 main.py:1126 INFO] - Added new repo path: /home/anandhu/MLC/repos/mlcommons@mlperf-automations [2025-02-19 16:46:57,606 main.py:1130 INFO] - Updated repos.json at /home/anandhu/MLC/repos/repos.json - Note: + Note: - repo_uid and repo_alias are not supported in the pull action for the repo target. - Only one of --checkout, --branch, or --tag should be specified at a time. @@ -447,9 +489,11 @@ def pull(self, run_args): repo_url = run_args.get('repo', run_args.get('url', 'repo')) if not repo_url or repo_url == "repo": for repo_object in self.repos: - if os.path.exists(os.path.join(repo_object.path, ".git")) and os.access(repo_object.path, os.W_OK): + if os.path.exists(os.path.join(repo_object.path, ".git")) and os.access( + repo_object.path, os.W_OK): repo_folder_name = os.path.basename(repo_object.path) - res = self.pull_repo(repo_folder_name, repo_path = repo_object.path) + res = self.pull_repo( + repo_folder_name, repo_path=repo_object.path) if res['return'] > 0: return res else: @@ -461,18 +505,18 @@ def pull(self, run_args): ssh = run_args.get('ssh') if sum(bool(var) for var in [branch, checkout, tag]) > 1: - return {"return": 1, "error": "Only one among the three flags(branch, checkout and tag) could be specified"} + return { + "return": 1, "error": "Only one among the three flags(branch, checkout and tag) could be specified"} res = self.pull_repo(repo_url, branch, checkout, tag, pat, ssh) if res['return'] > 0: return res - return {'return': 0} def show(self, run_args): return self.list(run_args) - + def list(self, run_args): """ #################################################################################################################### @@ -481,7 +525,7 @@ def list(self, run_args): #################################################################################################################### The `list` action displays all registered MLC repositories along with their aliases and paths. - + Example Command: mlc list repo @@ -511,7 +555,7 @@ def list(self, run_args): print("-------------") logger.info("Repository listing ended") return {"return": 0} - + def rm(self, run_args): """ #################################################################################################################### @@ -519,11 +563,11 @@ def rm(self, run_args): Action: rm #################################################################################################################### - The `rm` action removes a specified repository from MLCFlow, deleting the repository folder, its index entries, + The `rm` action removes a specified repository from MLCFlow, deleting the repository folder, its index entries, and its registration. - If there are any modified local changes, the user will be prompted for confirmation unless the `-f` flag is used + If there are any modified local changes, the user will be prompted for confirmation unless the `-f` flag is used for force removal. - + Example Command: mlc rm repo mlcommons@mlperf-automations @@ -541,16 +585,17 @@ def rm(self, run_args): """ if not run_args['repo']: logger.error("The repository to be removed is not specified") - return {"return": 1, "error": "The repository to be removed is not specified"} + return {"return": 1, + "error": "The repository to be removed is not specified"} r = self.find(run_args) - if r['return'] == 0: list_repos = r['list'] if len(list_repos) > 1: - return {"return": 1, "error": "Please select a unique repo by repo alias or repo UID to remove"} + return { + "return": 1, "error": "Please select a unique repo by repo alias or repo UID to remove"} repo = list_repos[0] repo_path = repo.path @@ -565,62 +610,78 @@ def rm(self, run_args): return r repos_file_path = os.path.join(self.repos_path, 'repos.json') - + force_remove = True if run_args.get('f') else False index = Action.get_index(self) index.remove_repo_from_index(repo_path) - - return rm_repo(repo_path, repos_file_path, force_remove) - -def rm_repo(repo_path, repos_file_path, force_remove): - logger.info("rm command has been called for repo. This would delete the repo folder and unregister the repo from repos.json") - - repo_name = os.path.basename(repo_path) - mlc_repos_path = os.path.abspath(os.path.dirname(repos_file_path)) - repo_parent_path = os.path.abspath(os.path.dirname(repo_path)) - - if os.path.isdir(repo_path) and os.path.samefile(mlc_repos_path, repo_parent_path): - # Check for local changes - status_command = ['git', '-C', repo_path, 'status', '--porcelain', '--untracked-files=no'] - local_changes = subprocess.run(status_command, capture_output=True, text=True) - - if local_changes.stdout: - logger.warning("Local changes detected in repository. Changes are listed below:") - print(local_changes.stdout) - confirm_remove = True if force_remove or (input("Continue to remove repo?").lower()) in ["yes", "y"] else False - else: - logger.info("No local changes detected. Removing repo...") - confirm_remove = True - if confirm_remove: - if force_remove: - logger.info("Force remove is set.") - shutil.rmtree(repo_path) - logger.info(f"Repo {repo_name} residing in path {repo_path} has been successfully removed") - logger.info("Checking whether the repo was registered in repos.json") - unregister_repo(repo_path, repos_file_path) - else: - logger.info("rm repo ooperation cancelled by user!") - + return rm_repo(repo_path, repos_file_path, force_remove) + + +def rm_repo(repo_path, repos_file_path, force_remove): + logger.info( + "rm command has been called for repo. This would delete the repo folder and unregister the repo from repos.json") + + repo_name = os.path.basename(repo_path) + mlc_repos_path = os.path.abspath(os.path.dirname(repos_file_path)) + repo_parent_path = os.path.abspath(os.path.dirname(repo_path)) + + if os.path.isdir(repo_path) and os.path.samefile( + mlc_repos_path, repo_parent_path): + # Check for local changes + status_command = [ + 'git', + '-C', + repo_path, + 'status', + '--porcelain', + '--untracked-files=no'] + local_changes = subprocess.run( + status_command, capture_output=True, text=True) + + if local_changes.stdout: + logger.warning( + "Local changes detected in repository. Changes are listed below:") + print(local_changes.stdout) + confirm_remove = True if force_remove or ( + input("Continue to remove repo?").lower()) in [ + "yes", "y"] else False else: - logger.warning(f"Repo {repo_name} was not found in the repo folder. repos.json will be checked for external paths. If any, that will be removed.") + logger.info("No local changes detected. Removing repo...") + confirm_remove = True + if confirm_remove: + if force_remove: + logger.info("Force remove is set.") + shutil.rmtree(repo_path) + logger.info( + f"Repo {repo_name} residing in path {repo_path} has been successfully removed") + logger.info( + "Checking whether the repo was registered in repos.json") unregister_repo(repo_path, repos_file_path) + else: + logger.info("rm repo ooperation cancelled by user!") + + else: + logger.warning( + f"Repo {repo_name} was not found in the repo folder. repos.json will be checked for external paths. If any, that will be removed.") + unregister_repo(repo_path, repos_file_path) + + return {"return": 0} + - return {"return": 0} - def unregister_repo(repo_path, repos_file_path): - logger.info(f"Unregistering the repo in path {repo_path}") + logger.info(f"Unregistering the repo in path {repo_path}") - with open(repos_file_path, 'r') as f: - repos_list = json.load(f) - - if repo_path in repos_list: - repos_list.remove(repo_path) - with open(repos_file_path, 'w') as f: - json.dump(repos_list, f, indent=2) - logger.info(f"Path: {repo_path} has been removed.") - else: - logger.info(f"Path: {repo_path} not found in {repos_file_path}. Nothing to be unregistered!") - - return {'return': 0} + with open(repos_file_path, 'r') as f: + repos_list = json.load(f) + + if repo_path in repos_list: + repos_list.remove(repo_path) + with open(repos_file_path, 'w') as f: + json.dump(repos_list, f, indent=2) + logger.info(f"Path: {repo_path} has been removed.") + else: + logger.info( + f"Path: {repo_path} not found in {repos_file_path}. Nothing to be unregistered!") + return {'return': 0} diff --git a/mlc/utils.py b/mlc/utils.py index 1cc4e8f19..f75506710 100644 --- a/mlc/utils.py +++ b/mlc/utils.py @@ -1,3 +1,4 @@ +import sys import argparse import subprocess import re @@ -11,6 +12,7 @@ import tarfile import zipfile + def generate_temp_file(i): """ Generate a temporary file and optionally clean up the directory. @@ -37,11 +39,11 @@ def generate_temp_file(i): # Generate a unique file name using uuid temp_file_name = f"{prefix}{uuid.uuid4().hex}{suffix}" - + # Optionally write the string content to the file with open(temp_file_name, 'w') as temp_file: temp_file.write(content) - + # If remove_dir is True, remove the directory where the file is created if remove_dir: dir_name = os.path.dirname(temp_file_name) @@ -52,7 +54,9 @@ def generate_temp_file(i): except Exception as e: return {'return': 1, 'error': str(e)} -def load_txt(file_name, check_if_exists=False, split=False, match_text=None, fail_if_no_match=None, remove_after_read=False): + +def load_txt(file_name, check_if_exists=False, split=False, + match_text=None, fail_if_no_match=None, remove_after_read=False): """ Load text from a file with optional checks, processing, and regex matching. @@ -65,7 +69,7 @@ def load_txt(file_name, check_if_exists=False, split=False, match_text=None, fai remove_after_read (bool): If True, deletes the file after reading. Returns: - dict: + dict: * return (int): 0 if successful, 1 if an error occurred. * error (str): Error message if return > 0. * string (str): Loaded text if split is False. @@ -73,11 +77,11 @@ def load_txt(file_name, check_if_exists=False, split=False, match_text=None, fai """ if check_if_exists and not os.path.isfile(file_name): return {'return': 1, 'error': f"File '{file_name}' does not exist."} - + try: with open(file_name, 'r') as f: content = f.read() - match_ = False + match_ = False # Check for match_text using regex if match_text: match_ = re.search(match_text, content) @@ -85,22 +89,23 @@ def load_txt(file_name, check_if_exists=False, split=False, match_text=None, fai return {'return': 1, 'error': fail_if_no_match} if remove_after_read: os.remove(file_name) - + result = {'return': 0} if split: result['list'] = content.splitlines() result['string'] = content else: result['string'] = content - + if match_: result['match'] = match_ - + return result except Exception as e: return {'return': 1, 'error': str(e)} + def compare_versions(current_version, min_version): """ Compare two semantic version strings. @@ -128,6 +133,7 @@ def compare_versions(current_version, min_version): except Exception as e: raise ValueError(f"Invalid version format: {e}") + def run_system_cmd(i): """ Execute a system command in a specified path. @@ -181,13 +187,21 @@ def run_system_cmd(i): def print_env(env, yaml=True, sort_keys=True, begin_spaces=None): printd(env, yaml=yaml, sort_keys=sort_keys, begin_spaces=begin_spaces) -def printd(mydict, yaml=True, sort_keys=True, begin_spaces = None): + +def printd(mydict, yaml=True, sort_keys=True, begin_spaces=None): if yaml: - print_formatted_yaml(mydict, sort_keys=sort_keys, begin_spaces=begin_spaces) + print_formatted_yaml( + mydict, + sort_keys=sort_keys, + begin_spaces=begin_spaces) else: - print_formatted_json(mydict, sort_keys = sort_keys, begin_spaces = begin_spaces) + print_formatted_json( + mydict, + sort_keys=sort_keys, + begin_spaces=begin_spaces) + -def print_formatted_yaml(data, sort_keys=True, begin_spaces = None): +def print_formatted_yaml(data, sort_keys=True, begin_spaces=None): """ Converts a Python dictionary (or other serializable object) to a YAML-formatted string and prints it in a human-readable format. @@ -201,20 +215,22 @@ def print_formatted_yaml(data, sort_keys=True, begin_spaces = None): """ try: yaml_string = yaml.dump( - data, - default_flow_style=False, - sort_keys=False, + data, + default_flow_style=False, + sort_keys=False, allow_unicode=True ) if not begin_spaces: print(yaml_string) else: - indented_yaml_str = "\n".join(" " * begin_spaces + line for line in yaml_string.splitlines()) + indented_yaml_str = "\n".join( + " " * begin_spaces + line for line in yaml_string.splitlines()) print(indented_yaml_str) except yaml.YAMLError as e: print(f"Error formatting YAML: {e}") -def print_formatted_json(data, sort_keys = True, begin_spaces = None): + +def print_formatted_json(data, sort_keys=True, begin_spaces=None): """ Prints a dictionary as a formatted JSON string. @@ -229,11 +245,13 @@ def print_formatted_json(data, sort_keys = True, begin_spaces = None): if not begin_spaces: print(formatted_json) else: - indented_json_str = "\n".join(" " * begin_spaces + line for line in formatted_json.splitlines()) + indented_json_str = "\n".join( + " " * begin_spaces + line for line in formatted_json.splitlines()) print(indented_json_str) except TypeError as e: print(f"Error formatting JSON: {e}") + def read_yaml(filepath): try: with open(filepath, "r") as f: @@ -241,24 +259,26 @@ def read_yaml(filepath): except Exception as e: logger.info(f"Error reading YAML file {filepath}: {e}") + def read_json(filepath): try: with open(filepath, "r") as f: return json.load(f) except Exception as e: logger.info(f"Error reading JSON file {filepath}: {e}") - + + def merge_dicts(params, in_place=True): """ Merges two dictionaries with optional handling for lists and unique values. - + Args: params (dict): A dictionary containing: - 'dict1': First dictionary to merge. - 'dict2': Second dictionary to merge. - 'append_lists' (bool): If True, lists in dict1 and dict2 will be merged. - 'append_unique' (bool): If True, lists will only contain unique values after merging. - + Returns: dict: A new dictionary resulting from the merge. """ @@ -286,11 +306,12 @@ def merge_dicts(params, in_place=True): for k in params: if k not in ["dict1", "dict2"]: ii[k] = params[k] - merge_dicts(ii, in_place) + merge_dicts(ii, in_place) elif isinstance(existing_value, list) and isinstance(value, list): if append_lists: if append_unique: - # Combine dictionaries uniquely based on their key-value pairs + # Combine dictionaries uniquely based on their + # key-value pairs seen = set() merged_list = [] for item in existing_value + value: @@ -305,7 +326,7 @@ def merge_dicts(params, in_place=True): seen.add(item_frozenset) merged_list.append(item) merged_dict[key] = merged_list - + else: # Simply append the values merged_dict[key] = existing_value + value @@ -313,13 +334,13 @@ def merge_dicts(params, in_place=True): # If lists shouldn't be appended, override the value merged_dict[key] = value else: - # If it's not a list, simply overwrite or merge the value as needed + # If it's not a list, simply overwrite or merge the value as + # needed merged_dict[key] = value else: # If key doesn't exist in dict1, add it directly merged_dict[key] = value - return {'return': 0, 'merged': merged_dict, 'dict1': merged_dict} @@ -364,6 +385,7 @@ def save_yaml(file_name, meta, sort_keys=True): except Exception as e: return {'return': 1, 'error': str(e)} + def save_txt(file_name, string): """ Saves the provided string to a text file. @@ -400,12 +422,14 @@ def convert_args_to_dictionary(inp): args_dict[list_key] = arg_value.split(",") continue - # Handle dictionaries: `--adr.compiler.tags=gcc` becomes `{"adr": {"compiler": {"tags": "gcc"}}}` + # Handle dictionaries: `--adr.compiler.tags=gcc` becomes `{"adr": + # {"compiler": {"tags": "gcc"}}}` elif "." in arg_key: keys = arg_key.split(".") current = args_dict for part in keys[:-1]: - if part not in current or not isinstance(current[part], dict): + if part not in current or not isinstance( + current[part], dict): current[part] = {} current = current[part] current[keys[-1]] = arg_value @@ -425,21 +449,22 @@ def convert_args_to_dictionary(inp): return {'return': 0, 'args_dict': args_dict} + def is_uid(name): - """ - Checks if the given name is a 16-digit hexadecimal UID. + """ + Checks if the given name is a 16-digit hexadecimal UID. - Args: - name (str): The string to check. + Args: + name (str): The string to check. - Returns: - bool: True if the name is a 16-digit hexadecimal UID, False otherwise. - """ - # Define a regex pattern for a 16-digit hexadecimal UID - hex_uid_pattern = r"^[0-9a-fA-F]{16}$" + Returns: + bool: True if the name is a 16-digit hexadecimal UID, False otherwise. + """ + # Define a regex pattern for a 16-digit hexadecimal UID + hex_uid_pattern = r"^[0-9a-fA-F]{16}$" - # Check if the name matches the pattern - return bool(re.fullmatch(hex_uid_pattern, name)) + # Check if the name matches the pattern + return bool(re.fullmatch(hex_uid_pattern, name)) def is_valid_url(url): @@ -481,6 +506,7 @@ def sub_input(i, keys, reverse=False): return {'return': 0, 'result': result} + def assemble_object(alias, uid): """ Assemble an object by concatenating the alias and uid. @@ -503,10 +529,6 @@ def assemble_object(alias, uid): return result -import importlib.util -import sys -import os - def load_python_module(params): """ @@ -536,27 +558,32 @@ def load_python_module(params): # Check if the file exists at the given path if not os.path.isfile(full_path): - return {'return': 1, 'error': f"Error: The file at '{full_path}' does not exist."} + return {'return': 1, + 'error': f"Error: The file at '{full_path}' does not exist."} # Load the module dynamically using importlib try: # Specify the module spec spec = importlib.util.spec_from_file_location(module_name, full_path) if spec is None: - return {'return': 1, 'error': f"Error: Could not load the module '{module_name}' from '{full_path}'."} + return { + 'return': 1, 'error': f"Error: Could not load the module '{module_name}' from '{full_path}'."} # Load the module module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) - # Add the module to sys.modules so it can be accessed like a normal module + # Add the module to sys.modules so it can be accessed like a normal + # module sys.modules[module_name] = module # Return success with loaded code and full path return {'return': 0, 'code': module, 'path': full_path} except Exception as e: - return {'return': 1, 'error': f"Error: Failed to load module '{module_name}' from '{full_path}'. Error: {str(e)}"} + return { + 'return': 1, 'error': f"Error: Failed to load module '{module_name}' from '{full_path}'. Error: {str(e)}"} + def convert_env_to_dict(env_text): """ @@ -572,7 +599,8 @@ def convert_env_to_dict(env_text): # Split the text into lines and process each line for line in env_text.splitlines(): - # Strip any leading/trailing whitespace and ensure the line is not empty + # Strip any leading/trailing whitespace and ensure the line is not + # empty line = line.strip() if line and '=' in line: key, value = line.split('=', 1) @@ -580,7 +608,8 @@ def convert_env_to_dict(env_text): return {'return': 0, 'dict': env_dict} -def load_json(file_name, encoding = None): + +def load_json(file_name, encoding=None): """ Load JSON data from a file and handle errors. @@ -610,8 +639,8 @@ def load_json(file_name, encoding = None): return {'return': 1, 'error': f"Error decoding JSON in file '{file_name}'."} except Exception as e: - return {'return': 1, 'error': f"An unexpected error occurred: {str(e)}"} - + return {'return': 1, + 'error': f"An unexpected error occurred: {str(e)}"} def get_new_uid(): @@ -628,7 +657,8 @@ def get_new_uid(): return {"return": 0, "uid": new_uid} except Exception as e: return {"return": 1, "error": f"Failed to generate UID: {str(e)}"} - + + def modify_git_url(get_type, url, params={}): """ Modify the GitHub url to support cloning of repo with either ssh or pat @@ -641,13 +671,15 @@ def modify_git_url(get_type, url, params={}): from giturlparse import parse p = parse(url) if get_type == "ssh": - return {"return": 0, "url": p.url2ssh } + return {"return": 0, "url": p.url2ssh} elif get_type == "pat": token = params['token'] - return {"return": 0, "url":"https://git:" + token + "@" + p.host + "/" + p.owner + "/" + p.repo } + return {"return": 0, "url": "https://git:" + token + + "@" + p.host + "/" + p.owner + "/" + p.repo} else: return {"return": 1, "error": f"Unsupported type: {get_type}"} + def convert_tags_to_list(tags_string): """ Convert a comma-separated string into a list of tags. @@ -666,7 +698,6 @@ def convert_tags_to_list(tags_string): return {'return': 0, 'tags': tags_list} - def extract_file(options): """ Extracts a compressed file, optionally stripping folder levels. @@ -694,17 +725,19 @@ def extract_file(options): with zipfile.ZipFile(filename, 'r') as archive: members = archive.namelist() for member in members: - # Strip folder levels (zip files always use forward slashes internally) + # Strip folder levels (zip files always use forward slashes + # internally) parts = member.split('/') if len(parts) > strip_folders: stripped_parts = parts[strip_folders:] stripped_path = os.path.join(extract_to, *stripped_parts) stripped_path = os.path.normpath(stripped_path) - + if member.endswith('/'): # Directory os.makedirs(stripped_path, exist_ok=True) else: # File - os.makedirs(os.path.dirname(stripped_path), exist_ok=True) + os.makedirs( + os.path.dirname(stripped_path), exist_ok=True) with archive.open(member) as source, open(stripped_path, 'wb') as target: shutil.copyfileobj(source, target) @@ -724,4 +757,3 @@ def extract_file(options): raise ValueError(f"Unsupported file format: {filename}") print(f"Extraction complete. Files extracted to: {extract_to}") - diff --git a/pyproject.toml b/pyproject.toml index e32aac71c..e2dae36f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mlcflow" -version = "1.1.21" +version = "1.2.0" description = "An automation interface tailored for CPU/GPU benchmarking" authors = [ From cbb95e786986f4d3697ffd4b7ec6973425a27dc1 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 15 Apr 2026 22:28:26 +0530 Subject: [PATCH 14/27] Use checkout instead of branch for repo cloning/pull (#225) --- docs/install/mlcflow_linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install/mlcflow_linux.sh b/docs/install/mlcflow_linux.sh index 59e3addbd..85705398e 100755 --- a/docs/install/mlcflow_linux.sh +++ b/docs/install/mlcflow_linux.sh @@ -381,7 +381,7 @@ pull_repo() { log_info " Repo : ${MLC_REPO}" log_info " Branch : ${MLC_BRANCH}" - mlc pull repo "${MLC_REPO}" --branch="${MLC_BRANCH}" + mlc pull repo "${MLC_REPO}" --checkout="${MLC_BRANCH}" } # ------------------------------------------------------------------------------ From 057de8a074f978616f1bf3baa5b2b7ba4cd221ca Mon Sep 17 00:00:00 2001 From: amd-arsuresh Date: Thu, 16 Apr 2026 15:14:21 +0100 Subject: [PATCH 15/27] Improvements to GH actions, support mlc apptainer script (#226) * Improvements to GH actions * Version upgrade * mlc version support added * Support mlc apptainer script --- .github/workflows/ai-pr-review.yml | 3 +- .github/workflows/build_wheels.yml | 14 +- .github/workflows/cla.yml | 2 +- .github/workflows/codeql.yml | 8 +- .github/workflows/mlperf-inference-bert.yml | 4 +- .../workflows/mlperf-inference-resnet50.yml | 4 +- .github/workflows/publish.yml | 4 +- .github/workflows/reset-fork.yml | 9 +- .github/workflows/test-installer-curl.yml | 23 ++- .github/workflows/test-mlc-core-actions.yaml | 16 +- .github/workflows/test-mlc-docker-core.yml | 4 +- .github/workflows/test-mlc-podman.yml | 4 +- VERSION | 2 +- mlc/__init__.py | 39 +++- mlc/main.py | 168 +++++++++++++++--- mlc/script_action.py | 76 +++++--- mlc/utils.py | 2 + pyproject.toml | 4 +- 18 files changed, 303 insertions(+), 83 deletions(-) diff --git a/.github/workflows/ai-pr-review.yml b/.github/workflows/ai-pr-review.yml index d8c51806c..5143690f9 100644 --- a/.github/workflows/ai-pr-review.yml +++ b/.github/workflows/ai-pr-review.yml @@ -10,11 +10,12 @@ permissions: jobs: ai-review: + if: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 ref: ${{ github.event.pull_request.base.ref }} diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 6e0d10267..cf456e2c4 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -24,14 +24,14 @@ jobs: fail-fast: false steps: # Step 1: Checkout the code - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 2 ssh-key: ${{ secrets.DEPLOY_KEY }} ref: ${{ github.ref_name }} # Step 2: Set up Python - - uses: actions/setup-python@v3 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 # Step 3: Check if VERSION file has changed in this push - name: Check if VERSION file has changed @@ -41,13 +41,13 @@ jobs: git config --global user.email "mlcommons-bot@users.noreply.github.com" if git diff --name-only $(git merge-base HEAD HEAD~1) | grep -q "VERSION"; then echo "VERSION file has been modified" - echo "::set-output name=version_changed::true" + echo "version_changed=true" >> "$GITHUB_OUTPUT" new_version=$(cat VERSION) else echo "VERSION file has NOT been modified" - echo "::set-output name=version_changed::false" + echo "version_changed=false" >> "$GITHUB_OUTPUT" fi - echo "::set-output name=new_version::$new_version" + echo "new_version=$new_version" >> "$GITHUB_OUTPUT" # Step 4: Increment version if VERSION was not changed - name: Increment version if necessary @@ -65,7 +65,7 @@ jobs: new_version="$major.$minor.$patch" echo $new_version > VERSION echo "New version: $new_version" - echo "::set-output name=new_version::$new_version" + echo "new_version=$new_version" >> "$GITHUB_OUTPUT" # Step 5: Commit the updated version to the repository - name: Commit updated version @@ -87,7 +87,7 @@ jobs: # Step 8: Publish to PyPI - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 with: verify-metadata: true skip-existing: true diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index 024e3ff4d..bab869873 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -17,7 +17,7 @@ jobs: - name: "MLCommons CLA bot check" if: (github.event.comment.body == 'recheck') || github.event_name == 'pull_request_target' # Alpha Release - uses: mlcommons/cla-bot@master + uses: mlcommons/cla-bot@7facd19917cdb24be27bd8d266b41094506a0fb2 # master 2026-03-24 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # the below token should have repo scope and must be manually added by you in the repository's secret diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ad586317f..1f40bfa7b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -19,6 +19,8 @@ on: schedule: - cron: '43 6 * * 0' +permissions: {} + jobs: analyze: name: Analyze (${{ matrix.language }}) @@ -57,7 +59,7 @@ jobs: # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 # Add any setup steps before running the `github/codeql-action/init` action. # This includes steps like installing compilers or runtimes (`actions/setup-node` @@ -67,7 +69,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@5c8a8a642e79153f5d047b10ec1cba1d1cc65699 # v3.35.1 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -95,6 +97,6 @@ jobs: exit 1 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@5c8a8a642e79153f5d047b10ec1cba1d1cc65699 # v3.35.1 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/mlperf-inference-bert.yml b/.github/workflows/mlperf-inference-bert.yml index 55474b067..3c47a5363 100644 --- a/.github/workflows/mlperf-inference-bert.yml +++ b/.github/workflows/mlperf-inference-bert.yml @@ -28,9 +28,9 @@ jobs: - os: windows-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} - name: Install mlcflow diff --git a/.github/workflows/mlperf-inference-resnet50.yml b/.github/workflows/mlperf-inference-resnet50.yml index 94b3e6ba5..5f2d3507f 100644 --- a/.github/workflows/mlperf-inference-resnet50.yml +++ b/.github/workflows/mlperf-inference-resnet50.yml @@ -28,11 +28,11 @@ jobs: python-version: 3.13 runs-on: "${{ matrix.on }}" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ github.event.pull_request.head.sha }} - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bed75398c..8553e71f5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,10 +17,10 @@ jobs: steps: - name: Checkout repository normally - uses: actions/checkout@v3 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" diff --git a/.github/workflows/reset-fork.yml b/.github/workflows/reset-fork.yml index 78ea9f86f..9d5ce4e59 100644 --- a/.github/workflows/reset-fork.yml +++ b/.github/workflows/reset-fork.yml @@ -8,13 +8,16 @@ on: required: false default: '' +permissions: + contents: write + jobs: reset-branch: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 @@ -24,7 +27,9 @@ jobs: - name: Use Input Branch if: ${{ inputs.branch != '' }} - run: echo "branch=${{ inputs.branch }}" >> $GITHUB_ENV + env: + BRANCH_INPUT: ${{ inputs.branch }} + run: echo "branch=${BRANCH_INPUT}" >> $GITHUB_ENV - name: Add Upstream Remote run: | diff --git a/.github/workflows/test-installer-curl.yml b/.github/workflows/test-installer-curl.yml index ee9d625b7..4bfc1f16b 100644 --- a/.github/workflows/test-installer-curl.yml +++ b/.github/workflows/test-installer-curl.yml @@ -10,6 +10,9 @@ on: - '.github/workflows/test-installer-curl.yml' workflow_dispatch: +permissions: + contents: read + # Only allow one workflow run per PR to conserve CI resources concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -91,12 +94,16 @@ jobs: # For pull_request: use the PR head branch # For workflow_dispatch: use dev branch from mlcommons/mlcflow - name: Determine Installer Script URL + env: + PR_OWNER: ${{ github.event.pull_request.head.repo.owner.login }} + PR_REPO: ${{ github.event.pull_request.head.repo.name }} + PR_BRANCH: ${{ github.event.pull_request.head.ref }} run: | if [[ "${{ github.event_name }}" == "pull_request" ]]; then # For PRs, use the head branch from the PR - OWNER="${{ github.event.pull_request.head.repo.owner.login }}" - REPO="${{ github.event.pull_request.head.repo.name }}" - BRANCH="${{ github.event.pull_request.head.ref }}" + OWNER="${PR_OWNER}" + REPO="${PR_REPO}" + BRANCH="${PR_BRANCH}" else # For workflow_dispatch and other events, use dev branch OWNER="mlcommons" @@ -344,11 +351,15 @@ jobs: steps: # Determine the source URL based on the event type - name: Determine Installer Script URL + env: + PR_OWNER: ${{ github.event.pull_request.head.repo.owner.login }} + PR_REPO: ${{ github.event.pull_request.head.repo.name }} + PR_BRANCH: ${{ github.event.pull_request.head.ref }} run: | if [[ "${{ github.event_name }}" == "pull_request" ]]; then - OWNER="${{ github.event.pull_request.head.repo.owner.login }}" - REPO="${{ github.event.pull_request.head.repo.name }}" - BRANCH="${{ github.event.pull_request.head.ref }}" + OWNER="${PR_OWNER}" + REPO="${PR_REPO}" + BRANCH="${PR_BRANCH}" else OWNER="mlcommons" REPO="mlcflow" diff --git a/.github/workflows/test-mlc-core-actions.yaml b/.github/workflows/test-mlc-core-actions.yaml index de3a95003..661447cbf 100644 --- a/.github/workflows/test-mlc-core-actions.yaml +++ b/.github/workflows/test-mlc-core-actions.yaml @@ -24,9 +24,9 @@ jobs: - os: macos-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} @@ -343,9 +343,9 @@ jobs: - os: macos-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} @@ -394,9 +394,9 @@ jobs: os: ["ubuntu-latest", "windows-latest", "macos-latest"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} @@ -481,9 +481,9 @@ jobs: action: ["mlcr", "mlce", "mlct"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/test-mlc-docker-core.yml b/.github/workflows/test-mlc-docker-core.yml index 30812ff19..c36684942 100644 --- a/.github/workflows/test-mlc-docker-core.yml +++ b/.github/workflows/test-mlc-docker-core.yml @@ -23,9 +23,9 @@ jobs: - os: macos-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/test-mlc-podman.yml b/.github/workflows/test-mlc-podman.yml index c7aec199a..6242ab6e0 100644 --- a/.github/workflows/test-mlc-podman.yml +++ b/.github/workflows/test-mlc-podman.yml @@ -25,9 +25,9 @@ jobs: - os: macos-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} diff --git a/VERSION b/VERSION index 26aaba0e8..6085e9465 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.0 +1.2.1 diff --git a/mlc/__init__.py b/mlc/__init__.py index 968168c9b..7e7c44881 100644 --- a/mlc/__init__.py +++ b/mlc/__init__.py @@ -1,4 +1,41 @@ -__version__ = "0.1.0" +import os +import subprocess + +def _get_version(): + """Read version from VERSION file or package metadata, and append git commit hash if available.""" + pkg_dir = os.path.dirname(os.path.abspath(__file__)) + root_dir = os.path.dirname(pkg_dir) + + # Read VERSION file (works in dev/source tree) + version = None + for vpath in [os.path.join(root_dir, "VERSION"), os.path.join(pkg_dir, "VERSION")]: + if os.path.isfile(vpath): + with open(vpath) as f: + version = f.read().strip() + break + + # Fall back to installed package metadata + if not version: + try: + from importlib.metadata import version as pkg_version + version = pkg_version("mlcflow") + except Exception: + version = "0.0.0" + + # Append git short commit hash if in a git repo + try: + commit = subprocess.check_output( + ["git", "-C", root_dir, "rev-parse", "--short", "HEAD"], + stderr=subprocess.DEVNULL, text=True + ).strip() + if commit: + version = f"{version}+{commit}" + except Exception: + pass + + return version + +__version__ = _get_version() from .action import access diff --git a/mlc/main.py b/mlc/main.py index c164f5b6d..449ef9f4f 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -78,6 +78,122 @@ def search(self, i): mlc_run_cmd = None +_current_target = None + +def get_version_info(): + """Return mlcflow version string with commit hash.""" + try: + from . import __version__ + return f"mlcflow {__version__}" + except ImportError: + pass + try: + from importlib.metadata import version + return f"mlcflow {version('mlcflow')}" + except Exception: + return "mlcflow (unknown version)" + +def _get_repo_hashes(): + """Get git info for all repos. Returns list of (alias, branch, hash, has_local_changes).""" + import subprocess + if default_parent is None: + return [] + results = [] + for repo in default_parent.repos: + alias = os.path.basename(repo.path) + git_dir = os.path.join(repo.path, '.git') + if not os.path.isdir(git_dir): + continue + try: + commit = subprocess.check_output( + ["git", "-C", repo.path, "rev-parse", "--short", "HEAD"], + stderr=subprocess.DEVNULL, text=True + ).strip() + branch = subprocess.check_output( + ["git", "-C", repo.path, "rev-parse", "--abbrev-ref", "HEAD"], + stderr=subprocess.DEVNULL, text=True + ).strip() + # Only tracked file changes (ignore untracked files) + dirty = subprocess.check_output( + ["git", "-C", repo.path, "status", "--porcelain", "-uno"], + stderr=subprocess.DEVNULL, text=True + ).strip() + results.append((alias, branch, commit, bool(dirty))) + except Exception: + pass + return results + +def _report_error(e): + import traceback + from .script_action import ScriptExecutionError + + # Log the error with target context + etype = type(e).__name__ if not isinstance(e, ScriptExecutionError) else '' + prefix = f'{etype}: ' if etype else '' + if _current_target: + logger.error(f"Error during '{_current_target}' action: {prefix}{e}") + else: + logger.error(f"{prefix}{e}") + + # Show the last traceback frame from outside the mlcflow package + tb = traceback.extract_tb(e.__traceback__) + if tb: + _mlc_pkg_dir = os.path.dirname(os.path.abspath(__file__)) + # Prefer the last frame outside the mlcflow package + last = tb[-1] + for frame in reversed(tb): + if not os.path.abspath(frame.filename).startswith(_mlc_pkg_dir): + last = frame + break + logger.error(f" at {last.filename}:{last.lineno} in {last.name}") + + # For script execution errors, show actionable info + if isinstance(e, ScriptExecutionError): + script_name = e.script_name + repo_alias = e.repo_alias + run_args = e.run_args + + if script_name: + # Build rerun command with user-facing inputs only + rerun_parts = ["mlcr", script_name] + _skip_keys = { + 'mlc_run_cmd', 'tags', 'details', 'path_only', 'p', + 'action', 'target', 'rebuild', 'env', 'script_tags', + 'run_cmd', 'run_final_cmds', 'skip_run_cmd', 'run_cmd_prefix', + 'add_deps_recursive', 'add_deps', 'file_path', + 'quiet', 'real_run', 'fake_run_deps', 'keep_detached', + 'pass_user_group', 'use_host_group_id', 'use_host_user_id', + 'extra_run_args', 'port_maps', 'mounts', 'pre_run_cmds', + 'docker_run_deps', + } + for k, v in run_args.items(): + if k in _skip_keys: + continue + if isinstance(v, (dict, list)): + continue + if k.startswith('MLC_') or k.startswith('mlc_'): + continue + rerun_parts.append(f"--{k}={v}") + rerun_cmd = " ".join(rerun_parts) + logger.error(f"Failed script: {script_name}") + logger.error(f"To rerun just the failed part: {rerun_cmd}") + + if e.version_info_file: + logger.error(f"Dependency versions: {e.version_info_file}") + + # Derive issues URL from repo alias + issues_url = 'https://github.com/mlcommons/mlperf-automations/issues' + if repo_alias and '@' in repo_alias: + issues_url = 'https://github.com/' + repo_alias.replace('@', '/') + '/issues' + logger.error(f"Please file an issue at {issues_url} with the full console log.") + + # Show version and repo commit hashes for debugging + logger.error(f"{get_version_info()}") + repo_hashes = _get_repo_hashes() + if repo_hashes: + for alias, branch, commit, dirty in repo_hashes: + marker = " (local changes)" if dirty else "" + logger.error(f" {alias}: {branch} {commit}{marker}") def mlc_expand_short(action, target="script"): @@ -90,17 +206,13 @@ def mlc_expand_short(action, target="script"): # Call the main function try: main() + except KeyboardInterrupt: + print("\nInterrupted.") + sys.exit(1) except SystemExit: raise except Exception as e: - import traceback - tb = traceback.extract_tb(e.__traceback__) - if tb: - last = tb[-1] - logger.error(f"{e}") - logger.error(f" at {last.filename}:{last.lineno} in {last.name}") - else: - logger.error(f"{e}") + _report_error(e) sys.exit(1) @@ -116,6 +228,10 @@ def mlcdr(): mlc_expand_short("docker") +def mlca(): + mlc_expand_short("apptainer") + + def mlcrr(): mlc_expand_short("remote-run") @@ -140,8 +256,12 @@ def process_console_output(res, target, action, run_args): if len(res['list']) == 0: # Only show warning if not in path-only mode if not run_args.get('path_only'): - logger.warning( - f"""No {target} entry found for the specified input: {run_args}!""") + logger.warning(f"""No {target} entry found for the specified input: {run_args}!""") + logger.info("Tip: Run 'mlc pull repo' to fetch the latest upstream changes.") + repo_hashes = _get_repo_hashes() + for alias, branch, commit, dirty in repo_hashes: + if dirty: + logger.warning(f"Repo '{alias}' ({branch}) has local changes - 'mlc pull repo' may fail. Commit or stash changes first.") else: for item in res['list']: if run_args.get('path_only'): @@ -245,7 +365,7 @@ def build_parser(pre_args): reindex_parser.add_argument('extra', nargs=argparse.REMAINDER) # Script-only - for action in ['docker', 'docker-run', + for action in ['docker', 'docker-run', 'apptainer', 'experiment', 'remote-run', 'doc', 'lint']: p = subparsers.add_parser(action, add_help=False) p.add_argument('target', choices=['script', 'run']) @@ -286,7 +406,7 @@ def build_run_args(args): if args.command in ['pull', 'rm', 'add', 'find'] and args.target == "repo": run_args['repo'] = args.details - if args.command in ['docker', 'docker-run', 'experiment', + if args.command in ['docker', 'docker-run', 'apptainer', 'experiment', 'remote-run', 'doc', 'lint'] and args.target == "run": # run_args['target'] = 'script' #dont modify this as script might have # target as in input @@ -386,6 +506,12 @@ def main(): check_raw_arguments_for_non_ascii() convert_hyphen_to_underscore_in_args() + # Handle version before argparse to avoid --version conflicting with + # script arguments like --version=3.4 + if len(sys.argv) >= 2 and sys.argv[1] in ('--version', '-V', 'version'): + print(get_version_info()) + sys.exit(0) + pre_parser = build_pre_parser() pre_args, remaining_args = pre_parser.parse_known_args() @@ -404,7 +530,7 @@ def main(): if pre_args.help and not "tags" in run_args: help_text = "" if pre_args.target == "run": - if pre_args.action.startswith("docker"): + if pre_args.action.startswith("docker") or pre_args.action == "apptainer": pre_args.target = "script" else: logger.error( @@ -455,6 +581,10 @@ def main(): if not hasattr(args, 'command'): logging.error("Error: No command specified.") sys.exit(1) + + global _current_target + _current_target = args.target + action = get_action(args.target, default_parent) @@ -477,15 +607,11 @@ def main(): if __name__ == '__main__': try: main() + except KeyboardInterrupt: + print("\nInterrupted.") + sys.exit(1) except SystemExit: raise except Exception as e: - import traceback - tb = traceback.extract_tb(e.__traceback__) - if tb: - last = tb[-1] - logger.error(f"{e}") - logger.error(f" at {last.filename}:{last.lineno} in {last.name}") - else: - logger.error(f"{e}") + _report_error(e) sys.exit(1) diff --git a/mlc/script_action.py b/mlc/script_action.py index 7af6ec5ff..1ebac9b89 100644 --- a/mlc/script_action.py +++ b/mlc/script_action.py @@ -1,3 +1,4 @@ +import re from .action import Action import os import sys @@ -258,28 +259,56 @@ def call_script_module_function(self, function_name, run_args): else: automation_instance = module.ScriptAutomation(self, module_path) - if function_name == "run": - result = automation_instance.run(run_args) # Pass args to the run method - elif function_name == "docker": - result = automation_instance.docker(run_args) # Pass args to the run method - elif function_name == "test": - result = automation_instance.test(run_args) # Pass args to the run method - elif function_name == "experiment": - result = automation_instance.experiment(run_args) # Pass args to the experiment method - elif function_name == "remote_run": - result = automation_instance.remote_run(run_args) # Pass args to the experiment method - elif function_name == "help": - result = automation_instance.help(run_args) # Pass args to the help method - elif function_name == "doc": - result = automation_instance.doc(run_args) # Pass args to the doc method - elif function_name == "lint": - result = automation_instance.lint(run_args) # Pass args to the lint method - else: - return {'return': 1, 'error': f'Function {function_name} is not supported'} + try: + if function_name == "run": + result = automation_instance.run(run_args) # Pass args to the run method + elif function_name == "docker": + result = automation_instance.docker(run_args) # Pass args to the run method + elif function_name == "test": + result = automation_instance.test(run_args) # Pass args to the run method + elif function_name == "experiment": + result = automation_instance.experiment(run_args) # Pass args to the experiment method + elif function_name == "remote_run": + result = automation_instance.remote_run(run_args) # Pass args to the experiment method + elif function_name == "help": + result = automation_instance.help(run_args) # Pass args to the help method + elif function_name == "doc": + result = automation_instance.doc(run_args) # Pass args to the doc method + elif function_name == "lint": + result = automation_instance.lint(run_args) # Pass args to the lint method + else: + return {'return': 1, 'error': f'Function {function_name} is not supported'} + except ScriptExecutionError: + raise + except Exception as exc: + _repo_match = re.search(r'/repos/([^/]+)/', module_path) + _repo_alias = _repo_match.group(1) if _repo_match else None + _script_name = run_args.get('tags', run_args.get('details')) + raise ScriptExecutionError( + f"Script {function_name} execution failed in {module_path}." + "\nError : " + f"{type(exc).__name__}: {exc}", + script_name=_script_name, repo_alias=_repo_alias, module_path=module_path, + run_args=run_args) from exc if result['return'] > 0: error = result.get('error', "") - raise ScriptExecutionError(f"Script {function_name} execution failed in {module_path}. \nError : {error}") + _name_match = re.search(r'name\s*=\s*([^,)]+)', error) + _script_name = _name_match.group(1).strip() if _name_match else run_args.get('tags', run_args.get('details')) + _repo_match = re.search(r'/repos/([^/]+)/', module_path) + _repo_alias = _repo_match.group(1) if _repo_match else None + # Dump dependency version info to file for debugging + _version_info_file = None + _version_info = result.get('version_info', []) + if _version_info: + _version_info_file = os.path.join(os.getcwd(), 'mlc-error-version-info.json') + try: + with open(_version_info_file, 'w') as _vf: + json.dump(_version_info, _vf, indent=2) + except Exception: + _version_info_file = None + raise ScriptExecutionError( + f"Script {function_name} execution failed in {module_path}. \nError : {error}", + script_name=_script_name, repo_alias=_repo_alias, module_path=module_path, + run_args=run_args, version_info_file=_version_info_file) if str(run_args.get("mlc_output")).lower() in ["on", "true", "yes", "1"]: with open("tmp-state.json", "w") as f: @@ -489,5 +518,10 @@ def experiment(self, run_args): return self.call_script_module_function("experiment", run_args) class ScriptExecutionError(Exception): - # """Custom error for configuration issues.""" - pass + def __init__(self, message, script_name=None, repo_alias=None, module_path=None, run_args=None, version_info_file=None): + super().__init__(message) + self.script_name = script_name + self.repo_alias = repo_alias + self.module_path = module_path + self.run_args = run_args or {} + self.version_info_file = version_info_file diff --git a/mlc/utils.py b/mlc/utils.py index f75506710..9fdd980d4 100644 --- a/mlc/utils.py +++ b/mlc/utils.py @@ -11,6 +11,8 @@ import shutil import tarfile import zipfile +import logging +logger = logging.getLogger("mlc") def generate_temp_file(i): diff --git a/pyproject.toml b/pyproject.toml index e2dae36f8..088b8be4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mlcflow" -version = "1.2.0" +version = "1.2.1" description = "An automation interface tailored for CPU/GPU benchmarking" authors = [ @@ -45,9 +45,11 @@ py-modules = [] [project.scripts] mlc = "mlc.main:main" +mlcflow = "mlc.main:main" mlcr = "mlc.main:mlcr" mlcrr = "mlc.main:mlcrr" mlcd = "mlc.main:mlcd" +mlca = "mlc.main:mlca" mlce = "mlc.main:mlce" mlct = "mlc.main:mlct" mlcp = "mlc.main:mlcp" From dab759c6b4c7b3c8025b4c94d5f0e294df56e583 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 16 Apr 2026 19:44:22 +0530 Subject: [PATCH 16/27] Change venv name to mlcflow --- docs/install/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/install/README.md b/docs/install/README.md index 2070132ee..31288bbad 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -17,7 +17,7 @@ This installer provides a **one-command setup** for the MLCFlow package and the **After installation, activate the virtual environment** to use MLCFlow commands: ```bash -source ~/.mlcflow_venv/bin/activate +source ~/.mlcflow/bin/activate ``` ## Supported Platforms @@ -75,7 +75,7 @@ source ~/.mlcflow_venv/bin/activate ↓ ┌─────────────────────────────────────────────────────────────┐ │ 5. Create Virtual Environment │ -│ - Location: ~/.mlcflow_venv (or custom) │ +│ - Location: ~/.mlcflow (or custom) │ └─────────────────────────────────────────────────────────────┘ ↓ ┌─────────────────────────────────────────────────────────────┐ @@ -147,7 +147,7 @@ bash mlcflow_linux.sh --yes --quiet |----------|-------------|---------| | `--yes` | Auto-confirm all prompts (non-interactive mode) | Interactive | | `--upgrade` | Upgrade mlcflow if already installed | Skip if present | -| `--venv-dir ` | Custom virtual environment directory | `~/.mlcflow_venv` | +| `--venv-dir ` | Custom virtual environment directory | `~/.mlcflow` | | `--mlc-repo ` | Repository in format `owner@repo` | `mlcommons@mlperf-automations` | | `--mlc-repo-branch ` | Git branch to clone | `dev` | | `--install-python` | Auto-install Python if incompatible | Prompt user | @@ -227,9 +227,9 @@ Based on this detection, it chooses the appropriate execution method. ```bash # Activate the virtual environment -source ~/.mlcflow_venv/bin/activate +source ~/.mlcflow/bin/activate -# Your prompt should change to show (mlcflow_venv) or similar +# Your prompt should change to show (mlcflow) or similar ``` ### Verify Installation @@ -251,7 +251,7 @@ deactivate ``` ### File Locations -- **Virtual Environment**: `~/.mlcflow_venv` (or custom path) +- **Virtual Environment**: `~/.mlcflow` (or custom path) - **Automation Repository**: `~/MLC/repos/mlcommons@mlperf-automations/` - **MLC Cache**: `~/MLC/repos/` @@ -277,7 +277,7 @@ bash mlcflow_linux.sh --yes **Solution**: ```bash # Activate the virtual environment -source ~/.mlcflow_venv/bin/activate +source ~/.mlcflow/bin/activate # Now mlc command should be available mlc --help From e8203d4d91263527ae11ffc7b9d98527e45f256d Mon Sep 17 00:00:00 2001 From: mlc-automations <3246381+mlc-automations@users.noreply.github.com> Date: Thu, 16 Apr 2026 14:14:42 +0000 Subject: [PATCH 17/27] [Automated Commit] Format Codebase --- mlc/__init__.py | 7 +- mlc/main.py | 24 ++++--- mlc/script_action.py | 154 ++++++++++++++++++++++++++----------------- 3 files changed, 115 insertions(+), 70 deletions(-) diff --git a/mlc/__init__.py b/mlc/__init__.py index 7e7c44881..decdaca7c 100644 --- a/mlc/__init__.py +++ b/mlc/__init__.py @@ -1,6 +1,8 @@ +from .action import access import os import subprocess + def _get_version(): """Read version from VERSION file or package metadata, and append git commit hash if available.""" pkg_dir = os.path.dirname(os.path.abspath(__file__)) @@ -8,7 +10,8 @@ def _get_version(): # Read VERSION file (works in dev/source tree) version = None - for vpath in [os.path.join(root_dir, "VERSION"), os.path.join(pkg_dir, "VERSION")]: + for vpath in [os.path.join(root_dir, "VERSION"), + os.path.join(pkg_dir, "VERSION")]: if os.path.isfile(vpath): with open(vpath) as f: version = f.read().strip() @@ -35,8 +38,8 @@ def _get_version(): return version + __version__ = _get_version() -from .action import access __all__ = ['access'] diff --git a/mlc/main.py b/mlc/main.py index 449ef9f4f..9ca19b7d4 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -80,6 +80,7 @@ def search(self, i): mlc_run_cmd = None _current_target = None + def get_version_info(): """Return mlcflow version string with commit hash.""" try: @@ -93,6 +94,7 @@ def get_version_info(): except Exception: return "mlcflow (unknown version)" + def _get_repo_hashes(): """Get git info for all repos. Returns list of (alias, branch, hash, has_local_changes).""" import subprocess @@ -123,6 +125,7 @@ def _get_repo_hashes(): pass return results + def _report_error(e): import traceback from .script_action import ScriptExecutionError @@ -184,8 +187,10 @@ def _report_error(e): # Derive issues URL from repo alias issues_url = 'https://github.com/mlcommons/mlperf-automations/issues' if repo_alias and '@' in repo_alias: - issues_url = 'https://github.com/' + repo_alias.replace('@', '/') + '/issues' - logger.error(f"Please file an issue at {issues_url} with the full console log.") + issues_url = 'https://github.com/' + \ + repo_alias.replace('@', '/') + '/issues' + logger.error( + f"Please file an issue at {issues_url} with the full console log.") # Show version and repo commit hashes for debugging logger.error(f"{get_version_info()}") @@ -256,12 +261,15 @@ def process_console_output(res, target, action, run_args): if len(res['list']) == 0: # Only show warning if not in path-only mode if not run_args.get('path_only'): - logger.warning(f"""No {target} entry found for the specified input: {run_args}!""") - logger.info("Tip: Run 'mlc pull repo' to fetch the latest upstream changes.") + logger.warning( + f"""No {target} entry found for the specified input: {run_args}!""") + logger.info( + "Tip: Run 'mlc pull repo' to fetch the latest upstream changes.") repo_hashes = _get_repo_hashes() for alias, branch, commit, dirty in repo_hashes: if dirty: - logger.warning(f"Repo '{alias}' ({branch}) has local changes - 'mlc pull repo' may fail. Commit or stash changes first.") + logger.warning( + f"Repo '{alias}' ({branch}) has local changes - 'mlc pull repo' may fail. Commit or stash changes first.") else: for item in res['list']: if run_args.get('path_only'): @@ -530,7 +538,8 @@ def main(): if pre_args.help and not "tags" in run_args: help_text = "" if pre_args.target == "run": - if pre_args.action.startswith("docker") or pre_args.action == "apptainer": + if pre_args.action.startswith( + "docker") or pre_args.action == "apptainer": pre_args.target = "script" else: logger.error( @@ -581,11 +590,10 @@ def main(): if not hasattr(args, 'command'): logging.error("Error: No command specified.") sys.exit(1) - + global _current_target _current_target = args.target - action = get_action(args.target, default_parent) if not action or not hasattr(action, args.command): diff --git a/mlc/script_action.py b/mlc/script_action.py index 1ebac9b89..16f545eb1 100644 --- a/mlc/script_action.py +++ b/mlc/script_action.py @@ -9,6 +9,7 @@ from . import utils from .logger import logger + class ScriptAction(Action): """ #################################################################################################################### @@ -36,6 +37,7 @@ class ScriptAction(Action): """ parent = None + def __init__(self, parent=None): self.parent = parent self.__dict__.update(vars(parent)) @@ -60,7 +62,7 @@ def search(self, i): return res find = search - + def rm(self, i): """ #################################################################################################################### @@ -68,7 +70,7 @@ def rm(self, i): Action: Remove(rm) #################################################################################################################### - The `remove` (`rm`) action deletes one or more scripts from MLC repositories. + The `remove` (`rm`) action deletes one or more scripts from MLC repositories. Example Command: @@ -87,14 +89,14 @@ def show(self, run_args): Action: Show #################################################################################################################### - The `show` action retrieves the path and metadata of the searched script in MLC repositories. + The `show` action retrieves the path and metadata of the searched script in MLC repositories. Example Command: mlc show script --tags=detect,os Example Output: - + arjun@intel-spr-i9:~$ mlc show script --tags=detect,os [2025-02-14 02:56:16,604 main.py:1404 INFO] - Showing script with tags: detect,os Location: /home/arjun/MLC/repos/gateoverflow@mlperf-automations/script/detect-os: @@ -103,7 +105,7 @@ def show(self, run_args): alias: detect-os description: Detects the operating system and platform information tags: ['detect-os', 'detect', 'os', 'info'] - new_env_keys: ['MLC_HOST_OS_*', '+MLC_HOST_OS_*', 'MLC_HOST_PLATFORM_*', 'MLC_HOST_PYTHON_*', 'MLC_HOST_SYSTEM_NAME', + new_env_keys: ['MLC_HOST_OS_*', '+MLC_HOST_OS_*', 'MLC_HOST_PLATFORM_*', 'MLC_HOST_PYTHON_*', 'MLC_HOST_SYSTEM_NAME', 'MLC_RUN_STATE_DOCKER', '+PATH'] new_state_keys: ['os_uname_*'] ...................................................... @@ -118,7 +120,14 @@ def show(self, run_args): if res['return'] > 0: return res logger.info(f"Showing script with tags: {run_args.get('tags')}") - script_meta_keys_to_show = ["uid", "alias", "description", "tags", "new_env_keys", "new_state_keys", "cache"] + script_meta_keys_to_show = [ + "uid", + "alias", + "description", + "tags", + "new_env_keys", + "new_state_keys", + "cache"] for item in res['list']: print(f"""Location: {item.path}: Main Script Meta:""") @@ -129,9 +138,10 @@ def show(self, run_args): print(" Input mapping:") utils.printd(item.meta["input_mapping"], begin_spaces=8) print("......................................................") - print(f"""For full script meta, see meta file at {os.path.join(item.path, "meta.yaml")}""") + print( + f"""For full script meta, see meta file at {os.path.join(item.path, "meta.yaml")}""") print("") - + return {'return': 0} def add(self, i): @@ -141,17 +151,17 @@ def add(self, i): Action: Add #################################################################################################################### - The `add` action creates a new script in a registered MLC repository. + The `add` action creates a new script in a registered MLC repository. Syntax: mlc add script :new_script --tags=benchmark Options: - --template_tags: A comma-separated list of tags to create a new MLC script based on existing templates. + --template_tags: A comma-separated list of tags to create a new MLC script based on existing templates. Example Output: - + arjun@intel-spr-i9:~$ mlc add script gateoverflow@mlperf-automations --tags=benchmark --template_tags=app,mlperf,inference More than one script found for None: 1. /home/arjun/MLC/repos/gateoverflow@mlperf-automations/script/app-mlperf-inference-mlcommons-python @@ -189,7 +199,7 @@ def add(self, i): ii['src_tags'] = i.get("template_tags", "template,generic") ii['dest'] = item ii['tags'] = i.get('tags', []) - res = self.cp(ii) + res = self.cp(ii) return res @@ -209,7 +219,8 @@ def dynamic_import_module(self, script_path): module_name = os.path.splitext(os.path.basename(script_path))[0] spec = importlib.util.spec_from_file_location(module_name, script_path) if spec is None or spec.loader is None: - raise ImportError(f"Cannot create a module spec for: {script_path}") + raise ImportError( + f"Cannot create a module spec for: {script_path}") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -220,11 +231,12 @@ def call_script_module_function(self, function_name, run_args): self.action_type = "script" repos_folder = self.repos_path - # Import script submodule + # Import script submodule script_path = self.find_target_folder("script") if not script_path: - logger.warning("Script automation not found. Automatically pulling mlcommons@mlperf-automations repository...") - + logger.warning( + "Script automation not found. Automatically pulling mlcommons@mlperf-automations repository...") + # Use the access method to pull the required repository result = self.access({ "automation": "repo", @@ -240,11 +252,15 @@ def call_script_module_function(self, function_name, run_args): # Try to find the script path again after pulling script_path = self.find_target_folder("script") if not script_path: - return {'return': 1, 'error': f"""Script automation still not found after pulling mlcommons@mlperf-automations --branch=dev."""} + return { + 'return': 1, 'error': f"""Script automation still not found after pulling mlcommons@mlperf-automations --branch=dev."""} else: - # If pull failed, return the original error with additional info - logger.error(f"Failed to pull mlcommons@mlperf-automations repository: {result.get('error', 'Unknown error')}") - return {'return': 1, 'error': f"""Script automation not found and failed to automatically pull mlcommons@mlperf-automations --branch=dev. Please run "mlc pull repo mlcommons@mlperf-automations --branch=dev" manually: {result.get('error', 'Unknown error')}"""} + # If pull failed, return the original error with additional + # info + logger.error( + f"Failed to pull mlcommons@mlperf-automations repository: {result.get('error', 'Unknown error')}") + return { + 'return': 1, 'error': f"""Script automation not found and failed to automatically pull mlcommons@mlperf-automations --branch=dev. Please run "mlc pull repo mlcommons@mlperf-automations --branch=dev" manually: {result.get('error', 'Unknown error')}"""} module_path = os.path.join(script_path, "module.py") module = self.dynamic_import_module(module_path) @@ -255,29 +271,40 @@ def call_script_module_function(self, function_name, run_args): params = inspect.signature(ctor).parameters if 'run_args' in params: - automation_instance = module.ScriptAutomation(self, module_path, run_args) + automation_instance = module.ScriptAutomation( + self, module_path, run_args) else: - automation_instance = module.ScriptAutomation(self, module_path) + automation_instance = module.ScriptAutomation( + self, module_path) try: if function_name == "run": - result = automation_instance.run(run_args) # Pass args to the run method + result = automation_instance.run( + run_args) # Pass args to the run method elif function_name == "docker": - result = automation_instance.docker(run_args) # Pass args to the run method + result = automation_instance.docker( + run_args) # Pass args to the run method elif function_name == "test": - result = automation_instance.test(run_args) # Pass args to the run method + result = automation_instance.test( + run_args) # Pass args to the run method elif function_name == "experiment": - result = automation_instance.experiment(run_args) # Pass args to the experiment method + result = automation_instance.experiment( + run_args) # Pass args to the experiment method elif function_name == "remote_run": - result = automation_instance.remote_run(run_args) # Pass args to the experiment method + result = automation_instance.remote_run( + run_args) # Pass args to the experiment method elif function_name == "help": - result = automation_instance.help(run_args) # Pass args to the help method + result = automation_instance.help( + run_args) # Pass args to the help method elif function_name == "doc": - result = automation_instance.doc(run_args) # Pass args to the doc method + result = automation_instance.doc( + run_args) # Pass args to the doc method elif function_name == "lint": - result = automation_instance.lint(run_args) # Pass args to the lint method + result = automation_instance.lint( + run_args) # Pass args to the lint method else: - return {'return': 1, 'error': f'Function {function_name} is not supported'} + return { + 'return': 1, 'error': f'Function {function_name} is not supported'} except ScriptExecutionError: raise except Exception as exc: @@ -285,21 +312,24 @@ def call_script_module_function(self, function_name, run_args): _repo_alias = _repo_match.group(1) if _repo_match else None _script_name = run_args.get('tags', run_args.get('details')) raise ScriptExecutionError( - f"Script {function_name} execution failed in {module_path}." + "\nError : " + f"{type(exc).__name__}: {exc}", + f"Script {function_name} execution failed in {module_path}." + + "\nError : " + f"{type(exc).__name__}: {exc}", script_name=_script_name, repo_alias=_repo_alias, module_path=module_path, run_args=run_args) from exc - + if result['return'] > 0: error = result.get('error', "") _name_match = re.search(r'name\s*=\s*([^,)]+)', error) - _script_name = _name_match.group(1).strip() if _name_match else run_args.get('tags', run_args.get('details')) + _script_name = _name_match.group(1).strip() if _name_match else run_args.get( + 'tags', run_args.get('details')) _repo_match = re.search(r'/repos/([^/]+)/', module_path) _repo_alias = _repo_match.group(1) if _repo_match else None # Dump dependency version info to file for debugging _version_info_file = None _version_info = result.get('version_info', []) if _version_info: - _version_info_file = os.path.join(os.getcwd(), 'mlc-error-version-info.json') + _version_info_file = os.path.join( + os.getcwd(), 'mlc-error-version-info.json') try: with open(_version_info_file, 'w') as _vf: json.dump(_version_info, _vf, indent=2) @@ -310,18 +340,20 @@ def call_script_module_function(self, function_name, run_args): script_name=_script_name, repo_alias=_repo_alias, module_path=module_path, run_args=run_args, version_info_file=_version_info_file) - if str(run_args.get("mlc_output")).lower() in ["on", "true", "yes", "1"]: + if str(run_args.get("mlc_output")).lower() in [ + "on", "true", "yes", "1"]: with open("tmp-state.json", "w") as f: json.dump(result['new_state'], f, indent=2) with open("tmp-run-env.out", "w") as f: - for key,val in result['new_env'].items(): + for key, val in result['new_env'].items(): f.write(f"""{key}="{val}"\n""") return result else: logger.info("ScriptAutomation class not found in the script.") - return {'return': 1, 'error': 'ScriptAutomation class not found in the script.'} + return {'return': 1, + 'error': 'ScriptAutomation class not found in the script.'} def docker(self, run_args): return self.docker_run(run_args) @@ -333,13 +365,13 @@ def docker_run(self, run_args): Action: Docker #################################################################################################################### - The `docker` action runs scripts inside a containerized environment. + The `docker` action runs scripts inside a containerized environment. An MLCFlow script can be executed inside a Docker container using either of the following syntaxes: - 1. Docker Run: mlc docker run --tags=