From 2517e69b0eb9c3727676e58d124a3d709984322f Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 7 Feb 2024 16:01:21 +0800 Subject: [PATCH 01/27] update --- scripts/ci/sync_extensions.py | 6 +++--- scripts/ci/update_ext_cmd_tree.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/ci/sync_extensions.py b/scripts/ci/sync_extensions.py index 519865ff339..2fb0ee938ee 100644 --- a/scripts/ci/sync_extensions.py +++ b/scripts/ci/sync_extensions.py @@ -71,7 +71,7 @@ def _sync_wheel(ext, updated_indexes, failed_urls, overwrite, temp_dir): f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--file', f'{os.path.abspath(whl_path)}', '--auth-mode', 'login'] result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print(json.loads(result.stdout)) + print(result.stdout) cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--auth-mode', 'login'] result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -150,7 +150,7 @@ def main(): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{backup_index_name}', '--file', f'{os.path.abspath(target_index_path)}', '--auth-mode', 'login'] result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print(json.loads(result.stdout)) + print(result.stdout) # start with an empty index.json to sync all extensions initial_index = {"extensions": {}, "formatVersion": "1"} open(target_index_path, 'w').write(json.dumps(initial_index, indent=4, sort_keys=True)) @@ -176,7 +176,7 @@ def main(): f'{STORAGE_ACCOUNT}', '--name', f'{index_name}', '--file', f'{os.path.abspath(target_index_path)}', '--auth-mode', 'login'] result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print(json.loads(result.stdout)) + print(result.stdout) print("\nSync finished.") if updated_indexes: print("New extensions available in:") diff --git a/scripts/ci/update_ext_cmd_tree.py b/scripts/ci/update_ext_cmd_tree.py index ae4e497b8fa..6afdf656567 100644 --- a/scripts/ci/update_ext_cmd_tree.py +++ b/scripts/ci/update_ext_cmd_tree.py @@ -83,7 +83,7 @@ def upload_cmd_tree(): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--file', f'{file_path}', '--auth-mode', 'login'] result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print(json.loads(result.stdout)) + print(result.stdout) cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--auth-mode', 'login'] From bc3aed8815f1b1ba6555abb241c815f598643868 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 7 Feb 2024 16:11:20 +0800 Subject: [PATCH 02/27] update --- scripts/ci/sync_extensions.py | 16 +++++++--------- scripts/ci/update_ext_cmd_tree.py | 6 +++--- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/scripts/ci/sync_extensions.py b/scripts/ci/sync_extensions.py index 2fb0ee938ee..00a20a4ebd8 100644 --- a/scripts/ci/sync_extensions.py +++ b/scripts/ci/sync_extensions.py @@ -69,9 +69,8 @@ def _sync_wheel(ext, updated_indexes, failed_urls, overwrite, temp_dir): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--file', f'{os.path.abspath(whl_path)}', - '--auth-mode', 'login'] - result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print(result.stdout) + '--auth-mode', 'login', '--overwrite'] + subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--auth-mode', 'login'] result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -148,9 +147,9 @@ def main(): # backup the old index.json backup_index_name = f'{BLOB_PREFIX}/index.json.sav' if BLOB_PREFIX else 'index.json.sav' cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', - f'{STORAGE_ACCOUNT}', '--name', f'{backup_index_name}', '--file', f'{os.path.abspath(target_index_path)}', '--auth-mode', 'login'] - result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print(result.stdout) + f'{STORAGE_ACCOUNT}', '--name', f'{backup_index_name}', + '--file', f'{os.path.abspath(target_index_path)}', '--auth-mode', 'login', '--overwrite'] + subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) # start with an empty index.json to sync all extensions initial_index = {"extensions": {}, "formatVersion": "1"} open(target_index_path, 'w').write(json.dumps(initial_index, indent=4, sort_keys=True)) @@ -174,9 +173,8 @@ def main(): index_name = f'{BLOB_PREFIX}/index.json' if BLOB_PREFIX else 'index.json' cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{index_name}', '--file', f'{os.path.abspath(target_index_path)}', - '--auth-mode', 'login'] - result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print(result.stdout) + '--auth-mode', 'login', '--overwrite'] + subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) print("\nSync finished.") if updated_indexes: print("New extensions available in:") diff --git a/scripts/ci/update_ext_cmd_tree.py b/scripts/ci/update_ext_cmd_tree.py index 6afdf656567..bedf03056c6 100644 --- a/scripts/ci/update_ext_cmd_tree.py +++ b/scripts/ci/update_ext_cmd_tree.py @@ -81,9 +81,9 @@ def upload_cmd_tree(): file_path = os.path.expanduser(os.path.join('~', '.azure', file_name)) cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', - f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--file', f'{file_path}', '--auth-mode', 'login'] - result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print(result.stdout) + f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--file', f'{file_path}', '--auth-mode', 'login', + '--overwrite'] + subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--auth-mode', 'login'] From f2b7babe4eb0c965e2a5da21f1f600c561cae78e Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 7 Feb 2024 16:32:34 +0800 Subject: [PATCH 03/27] update --- scripts/ci/sync_extensions.py | 3 ++- scripts/ci/update_ext_cmd_tree.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/ci/sync_extensions.py b/scripts/ci/sync_extensions.py index 00a20a4ebd8..92663d2c91d 100644 --- a/scripts/ci/sync_extensions.py +++ b/scripts/ci/sync_extensions.py @@ -74,7 +74,8 @@ def _sync_wheel(ext, updated_indexes, failed_urls, overwrite, temp_dir): cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--auth-mode', 'login'] result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - url = json.loads(result.stdout) + print(result) + url = result.stdout.decode('utf-8') updated_index = ext updated_index['downloadUrl'] = url updated_indexes.append(updated_index) diff --git a/scripts/ci/update_ext_cmd_tree.py b/scripts/ci/update_ext_cmd_tree.py index bedf03056c6..8f30a437a2c 100644 --- a/scripts/ci/update_ext_cmd_tree.py +++ b/scripts/ci/update_ext_cmd_tree.py @@ -88,7 +88,8 @@ def upload_cmd_tree(): cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--auth-mode', 'login'] result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - url = json.loads(result.stdout) + print(result) + url = result.stdout.decode('utf-8') download_file_path = os.path.expanduser(os.path.join('~', '.azure', downloaded_file_name)) download_file(url, download_file_path) From 87f519a5e48d9556efcc75abcce4599ba84041e4 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 7 Feb 2024 17:03:52 +0800 Subject: [PATCH 04/27] update --- scripts/ci/sync_extensions.py | 21 ++++++++++++++------- scripts/ci/update_ext_cmd_tree.py | 12 ++++++++---- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/scripts/ci/sync_extensions.py b/scripts/ci/sync_extensions.py index 92663d2c91d..a3cc9ef89be 100644 --- a/scripts/ci/sync_extensions.py +++ b/scripts/ci/sync_extensions.py @@ -62,20 +62,25 @@ def _sync_wheel(ext, updated_indexes, failed_urls, overwrite, temp_dir): if not overwrite: cmd = ['az', 'storage', 'blob', 'exists', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--auth-mode', 'login'] - result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - if json.loads(result.stdout)['exists']: + result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if result.stdout and json.loads(result.stdout)['exists']: print("Skipping '{}' as it already exists...".format(whl_file)) return cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--file', f'{os.path.abspath(whl_path)}', '--auth-mode', 'login', '--overwrite'] - subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(result) cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--auth-mode', 'login'] - result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) print(result) - url = result.stdout.decode('utf-8') + if result.stdout: + url = json.loads(result.stdout) + else: + print("Failed to get the URL for '{}'".format(whl_file)) + raise updated_index = ext updated_index['downloadUrl'] = url updated_indexes.append(updated_index) @@ -150,7 +155,8 @@ def main(): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{backup_index_name}', '--file', f'{os.path.abspath(target_index_path)}', '--auth-mode', 'login', '--overwrite'] - subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(result) # start with an empty index.json to sync all extensions initial_index = {"extensions": {}, "formatVersion": "1"} open(target_index_path, 'w').write(json.dumps(initial_index, indent=4, sort_keys=True)) @@ -175,7 +181,8 @@ def main(): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{index_name}', '--file', f'{os.path.abspath(target_index_path)}', '--auth-mode', 'login', '--overwrite'] - subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(result) print("\nSync finished.") if updated_indexes: print("New extensions available in:") diff --git a/scripts/ci/update_ext_cmd_tree.py b/scripts/ci/update_ext_cmd_tree.py index 8f30a437a2c..2b421eba007 100644 --- a/scripts/ci/update_ext_cmd_tree.py +++ b/scripts/ci/update_ext_cmd_tree.py @@ -83,13 +83,17 @@ def upload_cmd_tree(): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--file', f'{file_path}', '--auth-mode', 'login', '--overwrite'] - subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(result) cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--auth-mode', 'login'] - result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print(result) - url = result.stdout.decode('utf-8') + result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if result.stdout: + url = json.loads(result.stdout) + else: + print(f"Failed to get the URL for '{blob_file_name}'") + raise download_file_path = os.path.expanduser(os.path.join('~', '.azure', downloaded_file_name)) download_file(url, download_file_path) From 8906e0b056322d730f6f423e9e573c3a221b8a94 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 7 Feb 2024 17:31:04 +0800 Subject: [PATCH 05/27] update --- scripts/ci/sync_extensions.py | 24 +++++++++++++++--------- scripts/ci/update_ext_cmd_tree.py | 10 ++++++---- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/scripts/ci/sync_extensions.py b/scripts/ci/sync_extensions.py index a3cc9ef89be..beec0213455 100644 --- a/scripts/ci/sync_extensions.py +++ b/scripts/ci/sync_extensions.py @@ -62,7 +62,7 @@ def _sync_wheel(ext, updated_indexes, failed_urls, overwrite, temp_dir): if not overwrite: cmd = ['az', 'storage', 'blob', 'exists', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--auth-mode', 'login'] - result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + result = subprocess.run(cmd, capture_output=True) if result.stdout and json.loads(result.stdout)['exists']: print("Skipping '{}' as it already exists...".format(whl_file)) return @@ -70,13 +70,15 @@ def _sync_wheel(ext, updated_indexes, failed_urls, overwrite, temp_dir): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--file', f'{os.path.abspath(whl_path)}', '--auth-mode', 'login', '--overwrite'] - result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - print(result) + result = subprocess.run(cmd, capture_output=True) + if result.returncode != 0: + print(f"Failed to upload '{whl_file}' to the storage account") + raise cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--auth-mode', 'login'] - result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + result = subprocess.run(cmd, capture_output=True) print(result) - if result.stdout: + if result.stdout and result.returncode == 0: url = json.loads(result.stdout) else: print("Failed to get the URL for '{}'".format(whl_file)) @@ -155,8 +157,10 @@ def main(): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{backup_index_name}', '--file', f'{os.path.abspath(target_index_path)}', '--auth-mode', 'login', '--overwrite'] - result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - print(result) + result = subprocess.run(cmd, capture_output=True) + if result.returncode != 0: + print(f"Failed to upload '{target_index_path}' to the storage account") + raise # start with an empty index.json to sync all extensions initial_index = {"extensions": {}, "formatVersion": "1"} open(target_index_path, 'w').write(json.dumps(initial_index, indent=4, sort_keys=True)) @@ -181,8 +185,10 @@ def main(): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{index_name}', '--file', f'{os.path.abspath(target_index_path)}', '--auth-mode', 'login', '--overwrite'] - result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - print(result) + result = subprocess.run(cmd, capture_output=True) + if result.returncode != 0: + print(f"Failed to upload '{target_index_path}' to the storage account") + raise print("\nSync finished.") if updated_indexes: print("New extensions available in:") diff --git a/scripts/ci/update_ext_cmd_tree.py b/scripts/ci/update_ext_cmd_tree.py index 2b421eba007..6f515eb17dd 100644 --- a/scripts/ci/update_ext_cmd_tree.py +++ b/scripts/ci/update_ext_cmd_tree.py @@ -83,13 +83,15 @@ def upload_cmd_tree(): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--file', f'{file_path}', '--auth-mode', 'login', '--overwrite'] - result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - print(result) + result = subprocess.run(cmd, capture_output=True) + if result.returncode != 0: + print(f"Failed to upload '{file_name}' to the storage account") + raise cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--auth-mode', 'login'] - result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if result.stdout: + result = subprocess.run(cmd, capture_output=True) + if result.stdout and result.returncode == 0: url = json.loads(result.stdout) else: print(f"Failed to get the URL for '{blob_file_name}'") From b6017d426473db8e5ab920e7e4e04b7104bfac88 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 7 Feb 2024 18:09:49 +0800 Subject: [PATCH 06/27] Update update_ext_cmd_tree.py --- scripts/ci/update_ext_cmd_tree.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/update_ext_cmd_tree.py b/scripts/ci/update_ext_cmd_tree.py index 6f515eb17dd..f884af8f878 100644 --- a/scripts/ci/update_ext_cmd_tree.py +++ b/scripts/ci/update_ext_cmd_tree.py @@ -85,8 +85,8 @@ def upload_cmd_tree(): '--overwrite'] result = subprocess.run(cmd, capture_output=True) if result.returncode != 0: - print(f"Failed to upload '{file_name}' to the storage account") - raise + print(f"Failed to upload '{blob_file_name}' to the storage account") + print(result) cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--auth-mode', 'login'] From 2c3d7a320719c974fe62203055ad39efaf33ec4c Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 7 Feb 2024 22:20:31 +0800 Subject: [PATCH 07/27] Update update_ext_cmd_tree.py --- scripts/ci/update_ext_cmd_tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/update_ext_cmd_tree.py b/scripts/ci/update_ext_cmd_tree.py index f884af8f878..6480d073a62 100644 --- a/scripts/ci/update_ext_cmd_tree.py +++ b/scripts/ci/update_ext_cmd_tree.py @@ -46,7 +46,7 @@ def update_cmd_tree(ext_name): sys.path.append(ext_dir) extension_command_table, _ = _load_extension_command_loader(invoker.commands_loader, None, ext_mod) - EXT_CMD_TREE_TO_UPLOAD = Session() + EXT_CMD_TREE_TO_UPLOAD = Session(encoding='utf-8') EXT_CMD_TREE_TO_UPLOAD.load(os.path.expanduser(os.path.join('~', '.azure', file_name))) root = {} for cmd_name, ext_cmd in extension_command_table.items(): From 5de410e6544ba1cbae5cd9f9ae34fa67693e0025 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 8 Feb 2024 12:25:35 +0800 Subject: [PATCH 08/27] update --- scripts/ci/sync_extensions.py | 44 +++++++++++++++++-------------- scripts/ci/update_ext_cmd_tree.py | 17 ++++++------ 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/scripts/ci/sync_extensions.py b/scripts/ci/sync_extensions.py index beec0213455..af241e083a0 100644 --- a/scripts/ci/sync_extensions.py +++ b/scripts/ci/sync_extensions.py @@ -62,27 +62,29 @@ def _sync_wheel(ext, updated_indexes, failed_urls, overwrite, temp_dir): if not overwrite: cmd = ['az', 'storage', 'blob', 'exists', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--auth-mode', 'login'] - result = subprocess.run(cmd, capture_output=True) - if result.stdout and json.loads(result.stdout)['exists']: + try: + result = subprocess.run(cmd, capture_output=True, check=True) + except subprocess.CalledProcessError as ex: + raise Exception(f"Failed to check if '{blob_name}' exists in the storage") from ex + if json.loads(result.stdout)['exists']: print("Skipping '{}' as it already exists...".format(whl_file)) return cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--file', f'{os.path.abspath(whl_path)}', '--auth-mode', 'login', '--overwrite'] - result = subprocess.run(cmd, capture_output=True) - if result.returncode != 0: - print(f"Failed to upload '{whl_file}' to the storage account") - raise + try: + subprocess.run(cmd, capture_output=True, check=True) + except subprocess.CalledProcessError as ex: + raise Exception(f"Failed to upload '{blob_name}' to the storage") from ex cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--auth-mode', 'login'] - result = subprocess.run(cmd, capture_output=True) - print(result) - if result.stdout and result.returncode == 0: + try: + result = subprocess.run(cmd, capture_output=True, check=True) url = json.loads(result.stdout) - else: - print("Failed to get the URL for '{}'".format(whl_file)) - raise + except subprocess.CalledProcessError as ex: + raise Exception(f"Failed to get the URL for '{blob_name}'") from ex + updated_index = ext updated_index['downloadUrl'] = url updated_indexes.append(updated_index) @@ -157,10 +159,11 @@ def main(): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{backup_index_name}', '--file', f'{os.path.abspath(target_index_path)}', '--auth-mode', 'login', '--overwrite'] - result = subprocess.run(cmd, capture_output=True) - if result.returncode != 0: - print(f"Failed to upload '{target_index_path}' to the storage account") - raise + try: + subprocess.run(cmd, capture_output=True, check=True) + except subprocess.CalledProcessError as ex: + raise Exception(f"Failed to upload '{backup_index_name}' to the storage account") from ex + # start with an empty index.json to sync all extensions initial_index = {"extensions": {}, "formatVersion": "1"} open(target_index_path, 'w').write(json.dumps(initial_index, indent=4, sort_keys=True)) @@ -185,10 +188,11 @@ def main(): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{index_name}', '--file', f'{os.path.abspath(target_index_path)}', '--auth-mode', 'login', '--overwrite'] - result = subprocess.run(cmd, capture_output=True) - if result.returncode != 0: - print(f"Failed to upload '{target_index_path}' to the storage account") - raise + try: + subprocess.run(cmd, capture_output=True, check=True) + except subprocess.CalledProcessError as ex: + raise Exception(f"Failed to upload '{index_name}' to the storage account") from ex + print("\nSync finished.") if updated_indexes: print("New extensions available in:") diff --git a/scripts/ci/update_ext_cmd_tree.py b/scripts/ci/update_ext_cmd_tree.py index 6480d073a62..eb5f54180b5 100644 --- a/scripts/ci/update_ext_cmd_tree.py +++ b/scripts/ci/update_ext_cmd_tree.py @@ -83,19 +83,18 @@ def upload_cmd_tree(): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--file', f'{file_path}', '--auth-mode', 'login', '--overwrite'] - result = subprocess.run(cmd, capture_output=True) - if result.returncode != 0: - print(f"Failed to upload '{blob_file_name}' to the storage account") - print(result) + try: + subprocess.run(cmd, capture_output=True, check=True) + except subprocess.CalledProcessError as ex: + raise Exception(f"Failed to upload '{blob_file_name}' to the storage account") from ex cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--auth-mode', 'login'] - result = subprocess.run(cmd, capture_output=True) - if result.stdout and result.returncode == 0: + try: + result = subprocess.run(cmd, capture_output=True, check=True) url = json.loads(result.stdout) - else: - print(f"Failed to get the URL for '{blob_file_name}'") - raise + except subprocess.CalledProcessError as ex: + raise Exception(f"Failed to get the URL for '{blob_file_name}'") from ex download_file_path = os.path.expanduser(os.path.join('~', '.azure', downloaded_file_name)) download_file(url, download_file_path) From bfe5ed16adcada6ff4095cfa508a0480a0257dad Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 8 Feb 2024 15:14:35 +0800 Subject: [PATCH 09/27] update --- scripts/ci/sync_extensions.py | 34 +++++++++++-------------------- scripts/ci/update_ext_cmd_tree.py | 17 +++++++--------- scripts/ci/util.py | 29 +++++++++++++++++++++----- 3 files changed, 43 insertions(+), 37 deletions(-) diff --git a/scripts/ci/sync_extensions.py b/scripts/ci/sync_extensions.py index 97402477c52..15a6c5ddc44 100644 --- a/scripts/ci/sync_extensions.py +++ b/scripts/ci/sync_extensions.py @@ -10,6 +10,8 @@ import json import subprocess +from util import run_az_cmd + DEFAULT_TARGET_INDEX_URL = os.getenv('AZURE_EXTENSION_TARGET_INDEX_URL') STORAGE_ACCOUNT = os.getenv('AZURE_EXTENSION_TARGET_STORAGE_ACCOUNT') STORAGE_CONTAINER = os.getenv('AZURE_EXTENSION_TARGET_STORAGE_CONTAINER') @@ -48,7 +50,6 @@ def download_file(url, file_path): if chunk: # ignore keep-alive new chunks f.write(chunk) - def _sync_wheel(ext, updated_indexes, failed_urls, overwrite, temp_dir): download_url = ext['downloadUrl'] whl_file = download_url.split('/')[-1] @@ -62,10 +63,8 @@ def _sync_wheel(ext, updated_indexes, failed_urls, overwrite, temp_dir): if not overwrite: cmd = ['az', 'storage', 'blob', 'exists', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--auth-mode', 'login'] - try: - result = subprocess.run(cmd, capture_output=True, check=True) - except subprocess.CalledProcessError as ex: - raise Exception(f"Failed to check if '{blob_name}' exists in the storage") from ex + message = f"Checking if '{blob_name}' exists in the storage" + result = run_az_cmd(cmd, message=message, raise_error=True) if json.loads(result.stdout)['exists']: print("Skipping '{}' as it already exists...".format(whl_file)) return @@ -73,18 +72,13 @@ def _sync_wheel(ext, updated_indexes, failed_urls, overwrite, temp_dir): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--file', f'{os.path.abspath(whl_path)}', '--auth-mode', 'login', '--overwrite'] - try: - subprocess.run(cmd, capture_output=True, check=True) - except subprocess.CalledProcessError as ex: - raise Exception(f"Failed to upload '{blob_name}' to the storage") from ex + message = f"uploading '{blob_name}' to the storage" + run_az_cmd(cmd, message=message, raise_error=True) cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--auth-mode', 'login'] - try: - result = subprocess.run(cmd, capture_output=True, check=True) - url = json.loads(result.stdout) - except subprocess.CalledProcessError as ex: - raise Exception(f"Failed to get the URL for '{blob_name}'") from ex + message = f"Getting the URL for '{blob_name}'" + result = run_az_cmd(cmd, message=message, raise_error=True) updated_index = ext updated_index['downloadUrl'] = url @@ -160,10 +154,8 @@ def main(): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{backup_index_name}', '--file', f'{os.path.abspath(target_index_path)}', '--auth-mode', 'login', '--overwrite'] - try: - subprocess.run(cmd, capture_output=True, check=True) - except subprocess.CalledProcessError as ex: - raise Exception(f"Failed to upload '{backup_index_name}' to the storage account") from ex + message = f"Uploading '{backup_index_name}' to the storage" + run_az_cmd(cmd, message=message, raise_error=True) # start with an empty index.json to sync all extensions initial_index = {"extensions": {}, "formatVersion": "1"} @@ -189,10 +181,8 @@ def main(): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{index_name}', '--file', f'{os.path.abspath(target_index_path)}', '--auth-mode', 'login', '--overwrite'] - try: - subprocess.run(cmd, capture_output=True, check=True) - except subprocess.CalledProcessError as ex: - raise Exception(f"Failed to upload '{index_name}' to the storage account") from ex + message = f"Uploading '{index_name}' to the storage" + run_az_cmd(cmd, message=message, raise_error=True) print("\nSync finished.") if updated_indexes: diff --git a/scripts/ci/update_ext_cmd_tree.py b/scripts/ci/update_ext_cmd_tree.py index eb5f54180b5..03e8df215fa 100644 --- a/scripts/ci/update_ext_cmd_tree.py +++ b/scripts/ci/update_ext_cmd_tree.py @@ -6,13 +6,14 @@ import filecmp import json import os -import subprocess import sys + from azure.cli.core import get_default_cli from azure.cli.core._session import Session from azure.cli.core.commands import _load_extension_command_loader from azure.cli.core.extension import get_extension_modname, get_extension_path from sync_extensions import download_file +from util import run_az_cmd STORAGE_ACCOUNT = os.getenv('AZURE_EXTENSION_CMD_TREE_STORAGE_ACCOUNT') STORAGE_CONTAINER = os.getenv('AZURE_EXTENSION_CMD_TREE_STORAGE_CONTAINER') @@ -83,18 +84,14 @@ def upload_cmd_tree(): cmd = ['az', 'storage', 'blob', 'upload', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--file', f'{file_path}', '--auth-mode', 'login', '--overwrite'] - try: - subprocess.run(cmd, capture_output=True, check=True) - except subprocess.CalledProcessError as ex: - raise Exception(f"Failed to upload '{blob_file_name}' to the storage account") from ex + message = f"Uploading '{blob_file_name}' to the storage" + run_az_cmd(cmd, message=message, raise_error=True) cmd = ['az', 'storage', 'blob', 'url', '--container-name', f'{STORAGE_CONTAINER}', '--account-name', f'{STORAGE_ACCOUNT}', '--name', f'{blob_file_name}', '--auth-mode', 'login'] - try: - result = subprocess.run(cmd, capture_output=True, check=True) - url = json.loads(result.stdout) - except subprocess.CalledProcessError as ex: - raise Exception(f"Failed to get the URL for '{blob_file_name}'") from ex + message = f"Getting the URL for '{blob_file_name}'" + result = run_az_cmd(cmd, message=message, raise_error=True) + url = json.loads(result.stdout) download_file_path = os.path.expanduser(os.path.join('~', '.azure', downloaded_file_name)) download_file(url, download_file_path) diff --git a/scripts/ci/util.py b/scripts/ci/util.py index ffc7d54797b..7d6cb8b30c1 100644 --- a/scripts/ci/util.py +++ b/scripts/ci/util.py @@ -3,15 +3,14 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +import json import logging import os import re import shlex -import json +import subprocess import zipfile -from subprocess import check_output - logger = logging.getLogger(__name__) # copy from wheel==0.30.0 @@ -136,7 +135,7 @@ def diff_code(start, end): # If running in Travis CI, only run tests for edited extensions commit_range = os.environ.get('TRAVIS_COMMIT_RANGE') - if commit_range and not check_output( + if commit_range and not subprocess.check_output( ['git', '--no-pager', 'diff', '--name-only', commit_range, '--', src_d_full]): continue @@ -154,7 +153,7 @@ def diff_code(start, end): else: cmd = cmd_tpl.format(start=start, end=end, code_dir=src_d_full) - if not check_output(shlex.split(cmd)): + if not subprocess.check_output(shlex.split(cmd)): continue diff_ref.append((pkg_name, src_d_full)) @@ -163,3 +162,23 @@ def diff_code(start, end): f'end: {end}, ' f'diff_ref: {diff_ref}.') return diff_ref + +def run_az_cmd(cmd, message=False, raise_error=True): + """ + :param cmd: The entire command line to run. + :param message: A custom message to print, or True (bool) to use a default. + :param raise_error: Whether to raise an exception if the command fails. + """ + # use default message if custom not provided + if message is True: + print(f'Running: {cmd}') + + if message: + print(f'{message}') + + try: + result = subprocess.run(cmd, capture_output=True, check=True) + except subprocess.CalledProcessError as ex: + if raise_error: + raise Exception(f"Failed to run command: {cmd}") from ex + return result From 8a844b73abf676614394a9967af46a2ca330787e Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 8 Feb 2024 16:01:55 +0800 Subject: [PATCH 10/27] Update sync_extensions.py --- scripts/ci/sync_extensions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ci/sync_extensions.py b/scripts/ci/sync_extensions.py index 15a6c5ddc44..1261594f0f6 100644 --- a/scripts/ci/sync_extensions.py +++ b/scripts/ci/sync_extensions.py @@ -5,9 +5,9 @@ # pylint: disable=line-too-long # pylint: disable=broad-except +import json import os import re -import json import subprocess from util import run_az_cmd @@ -79,6 +79,7 @@ def _sync_wheel(ext, updated_indexes, failed_urls, overwrite, temp_dir): f'{STORAGE_ACCOUNT}', '--name', f'{blob_name}', '--auth-mode', 'login'] message = f"Getting the URL for '{blob_name}'" result = run_az_cmd(cmd, message=message, raise_error=True) + url = json.loads(result.stdout) updated_index = ext updated_index['downloadUrl'] = url From 29f00c68145b2a2aa0b76bfd6264a41fdb051172 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 8 Feb 2024 16:39:49 +0800 Subject: [PATCH 11/27] Update util.py --- scripts/ci/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/ci/util.py b/scripts/ci/util.py index 7d6cb8b30c1..5872689f2a6 100644 --- a/scripts/ci/util.py +++ b/scripts/ci/util.py @@ -178,6 +178,7 @@ def run_az_cmd(cmd, message=False, raise_error=True): try: result = subprocess.run(cmd, capture_output=True, check=True) + return result except subprocess.CalledProcessError as ex: if raise_error: raise Exception(f"Failed to run command: {cmd}") from ex From e8c767ae40bc0d0136fd2c6fe9ec3e31e7af2950 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Fri, 31 May 2024 15:27:19 +0800 Subject: [PATCH 12/27] Update util.py --- scripts/ci/util.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/ci/util.py b/scripts/ci/util.py index 5872689f2a6..cee0a53ef28 100644 --- a/scripts/ci/util.py +++ b/scripts/ci/util.py @@ -163,23 +163,25 @@ def diff_code(start, end): f'diff_ref: {diff_ref}.') return diff_ref + def run_az_cmd(cmd, message=False, raise_error=True): """ :param cmd: The entire command line to run. :param message: A custom message to print, or True (bool) to use a default. :param raise_error: Whether to raise an exception if the command fails. """ - # use default message if custom not provided + # use default message if custom not provided if message is True: print(f'Running: {cmd}') - + if message: print(f'{message}') - + try: result = subprocess.run(cmd, capture_output=True, check=True) return result except subprocess.CalledProcessError as ex: + print(result) if raise_error: raise Exception(f"Failed to run command: {cmd}") from ex return result From 212f0829ab081675cba38406e1c578fdbc59c2b0 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Fri, 31 May 2024 16:14:55 +0800 Subject: [PATCH 13/27] Update util.py --- scripts/ci/util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/ci/util.py b/scripts/ci/util.py index cee0a53ef28..27d9020b33c 100644 --- a/scripts/ci/util.py +++ b/scripts/ci/util.py @@ -181,7 +181,7 @@ def run_az_cmd(cmd, message=False, raise_error=True): result = subprocess.run(cmd, capture_output=True, check=True) return result except subprocess.CalledProcessError as ex: - print(result) + error_message = ex.stderr if ex.stderr else str(ex) if raise_error: - raise Exception(f"Failed to run command: {cmd}") from ex - return result + raise Exception(f"Failed to run command: {cmd}\nError: {error_message}") from ex + return ex From 08f3b0acad3e481a8337a3eecd18771f57706e33 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Fri, 31 May 2024 17:27:21 +0800 Subject: [PATCH 14/27] Create build_all_ext.sh --- scripts/ci/build_all_ext.sh | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 scripts/ci/build_all_ext.sh diff --git a/scripts/ci/build_all_ext.sh b/scripts/ci/build_all_ext.sh new file mode 100644 index 00000000000..1429315021c --- /dev/null +++ b/scripts/ci/build_all_ext.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +changed_content=$(git --no-pager diff --diff-filter=ACMRT HEAD~$AZURE_EXTENSION_COMMIT_NUM -- src/index.json) +if [[ -z "$changed_content" ]]; then + echo "index.json not modified. End task." + exit 0 +fi + +pip install azure-cli-core azure-cli requests +pip install azure-storage-blob==1.5.0 +echo "Listing Available Extensions:" +az extension list-available -otable + +# turn off telemetry as it crowds output +export AZURE_CORE_COLLECT_TELEMETRY=False + +# use index.json in master branch +export AZURE_EXTENSION_INDEX_URL=https://raw.githubusercontent.com/Azure/azure-cli-extensions/master/src/index.json + +output=$(az extension list-available --query [].name -otsv) +# azure-cli-ml is replaced by ml +# disable alias which relies on Jinja2 2.10 +blocklist=("azure-cli-ml" "alias") + +rm -f ~/.azure/extCmdTreeToUpload.json + +filter_exts="" +for ext in $output; do + ext=${ext%$'\r'} # Remove a trailing newline when running on Windows. + if [[ " ${blocklist[@]} " =~ " ${ext} " ]]; then + continue + fi + filter_exts="${filter_exts} ${ext}" + echo "Adding extension:" $ext + az extension add --upgrade -n $ext + if [ $? != 0 ] + then + echo "Failed to load:" $ext + exit 1 + fi +done From bb4a4e258e48420fc638ffbe8d604b2fec3e3eec Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Mon, 3 Jun 2024 13:43:19 +0800 Subject: [PATCH 15/27] Revert "Create build_all_ext.sh" This reverts commit 08f3b0acad3e481a8337a3eecd18771f57706e33. --- scripts/ci/build_all_ext.sh | 41 ------------------------------------- 1 file changed, 41 deletions(-) delete mode 100644 scripts/ci/build_all_ext.sh diff --git a/scripts/ci/build_all_ext.sh b/scripts/ci/build_all_ext.sh deleted file mode 100644 index 1429315021c..00000000000 --- a/scripts/ci/build_all_ext.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -changed_content=$(git --no-pager diff --diff-filter=ACMRT HEAD~$AZURE_EXTENSION_COMMIT_NUM -- src/index.json) -if [[ -z "$changed_content" ]]; then - echo "index.json not modified. End task." - exit 0 -fi - -pip install azure-cli-core azure-cli requests -pip install azure-storage-blob==1.5.0 -echo "Listing Available Extensions:" -az extension list-available -otable - -# turn off telemetry as it crowds output -export AZURE_CORE_COLLECT_TELEMETRY=False - -# use index.json in master branch -export AZURE_EXTENSION_INDEX_URL=https://raw.githubusercontent.com/Azure/azure-cli-extensions/master/src/index.json - -output=$(az extension list-available --query [].name -otsv) -# azure-cli-ml is replaced by ml -# disable alias which relies on Jinja2 2.10 -blocklist=("azure-cli-ml" "alias") - -rm -f ~/.azure/extCmdTreeToUpload.json - -filter_exts="" -for ext in $output; do - ext=${ext%$'\r'} # Remove a trailing newline when running on Windows. - if [[ " ${blocklist[@]} " =~ " ${ext} " ]]; then - continue - fi - filter_exts="${filter_exts} ${ext}" - echo "Adding extension:" $ext - az extension add --upgrade -n $ext - if [ $? != 0 ] - then - echo "Failed to load:" $ext - exit 1 - fi -done From 218ca95ba2e19a0f2b2da8657a77712cbf5a6a7c Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Tue, 15 Oct 2024 12:22:50 +0800 Subject: [PATCH 16/27] Update build_ext_cmd_tree.sh --- scripts/ci/build_ext_cmd_tree.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/build_ext_cmd_tree.sh b/scripts/ci/build_ext_cmd_tree.sh index 53bed033b85..8c058c8d4f7 100644 --- a/scripts/ci/build_ext_cmd_tree.sh +++ b/scripts/ci/build_ext_cmd_tree.sh @@ -6,8 +6,8 @@ if [[ -z "$changed_content" ]]; then exit 0 fi -pip install azure-cli-core azure-cli requests -pip install azure-storage-blob==1.5.0 +# pip install azure-cli-core azure-cli requests +# pip install azure-storage-blob==1.5.0 echo "Listing Available Extensions:" az extension list-available -otable From 690b7ce7d120036107dbf6adf35bfa8108917d70 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Tue, 15 Oct 2024 14:24:25 +0800 Subject: [PATCH 17/27] Update build_ext_cmd_tree.sh --- scripts/ci/build_ext_cmd_tree.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/build_ext_cmd_tree.sh b/scripts/ci/build_ext_cmd_tree.sh index 8c058c8d4f7..6cc429161d0 100644 --- a/scripts/ci/build_ext_cmd_tree.sh +++ b/scripts/ci/build_ext_cmd_tree.sh @@ -32,7 +32,7 @@ for ext in $output; do fi filter_exts="${filter_exts} ${ext}" echo "Adding extension:" $ext - az extension add --upgrade -n $ext + az extension add --upgrade -n $ext --debug if [ $? != 0 ] then echo "Failed to load:" $ext From e8572a7eecafb2f7def55636634a359308260555 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Tue, 15 Oct 2024 16:24:09 +0800 Subject: [PATCH 18/27] Update build_ext_cmd_tree.sh --- scripts/ci/build_ext_cmd_tree.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/build_ext_cmd_tree.sh b/scripts/ci/build_ext_cmd_tree.sh index 6cc429161d0..54cdf735493 100644 --- a/scripts/ci/build_ext_cmd_tree.sh +++ b/scripts/ci/build_ext_cmd_tree.sh @@ -6,8 +6,6 @@ if [[ -z "$changed_content" ]]; then exit 0 fi -# pip install azure-cli-core azure-cli requests -# pip install azure-storage-blob==1.5.0 echo "Listing Available Extensions:" az extension list-available -otable @@ -40,4 +38,6 @@ for ext in $output; do fi done +pip install azure-cli-core azure-cli requests +pip install azure-storage-blob==1.5.0 python $(cd $(dirname $0); pwd)/update_ext_cmd_tree.py $filter_exts From 7447487be805ef28fa4f0a82df75cf574a87bbe6 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Tue, 15 Oct 2024 17:31:33 +0800 Subject: [PATCH 19/27] Update build_ext_cmd_tree.sh --- scripts/ci/build_ext_cmd_tree.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/ci/build_ext_cmd_tree.sh b/scripts/ci/build_ext_cmd_tree.sh index 54cdf735493..359ffeb4202 100644 --- a/scripts/ci/build_ext_cmd_tree.sh +++ b/scripts/ci/build_ext_cmd_tree.sh @@ -38,6 +38,7 @@ for ext in $output; do fi done -pip install azure-cli-core azure-cli requests -pip install azure-storage-blob==1.5.0 +pip install azure-cli-core azure-cli requests azure-storage-blob==1.5.0 +# arcdata: ModuleNotFoundError: No module named 'regex._regex' +pip install regex python $(cd $(dirname $0); pwd)/update_ext_cmd_tree.py $filter_exts From 868a7b21e3505811525da8fda8bc3817d6522b16 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Tue, 15 Oct 2024 17:32:04 +0800 Subject: [PATCH 20/27] Update build_ext_cmd_tree.sh --- scripts/ci/build_ext_cmd_tree.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/build_ext_cmd_tree.sh b/scripts/ci/build_ext_cmd_tree.sh index 359ffeb4202..4be87ceb611 100644 --- a/scripts/ci/build_ext_cmd_tree.sh +++ b/scripts/ci/build_ext_cmd_tree.sh @@ -30,7 +30,7 @@ for ext in $output; do fi filter_exts="${filter_exts} ${ext}" echo "Adding extension:" $ext - az extension add --upgrade -n $ext --debug + az extension add --upgrade -n $ext if [ $? != 0 ] then echo "Failed to load:" $ext From 80a3c1c855b406b74c6e9aa0e9853c04086c94d2 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Wed, 16 Oct 2024 16:00:05 +0800 Subject: [PATCH 21/27] Update build_ext_cmd_tree.sh --- scripts/ci/build_ext_cmd_tree.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ci/build_ext_cmd_tree.sh b/scripts/ci/build_ext_cmd_tree.sh index 4be87ceb611..c5d5ca67877 100644 --- a/scripts/ci/build_ext_cmd_tree.sh +++ b/scripts/ci/build_ext_cmd_tree.sh @@ -38,7 +38,8 @@ for ext in $output; do fi done -pip install azure-cli-core azure-cli requests azure-storage-blob==1.5.0 +pip install azure-cli-core azure-cli requests +# azure-storage-blob==1.5.0 # arcdata: ModuleNotFoundError: No module named 'regex._regex' pip install regex python $(cd $(dirname $0); pwd)/update_ext_cmd_tree.py $filter_exts From 6ac446573e6b4a09fba5e0f0e77047b6d861356b Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 17 Oct 2024 16:19:59 +0800 Subject: [PATCH 22/27] Update build_ext_cmd_tree.sh --- scripts/ci/build_ext_cmd_tree.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/build_ext_cmd_tree.sh b/scripts/ci/build_ext_cmd_tree.sh index c5d5ca67877..df5ecb0d172 100644 --- a/scripts/ci/build_ext_cmd_tree.sh +++ b/scripts/ci/build_ext_cmd_tree.sh @@ -38,7 +38,7 @@ for ext in $output; do fi done -pip install azure-cli-core azure-cli requests +pip install azure-cli-core==2.64.0 azure-cli==2.64.0 requests # azure-storage-blob==1.5.0 # arcdata: ModuleNotFoundError: No module named 'regex._regex' pip install regex From 9beaa143124e17aba3bfd7846c8c70ba6d570cf3 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Thu, 17 Oct 2024 17:14:34 +0800 Subject: [PATCH 23/27] Update build_ext_cmd_tree.sh --- scripts/ci/build_ext_cmd_tree.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/ci/build_ext_cmd_tree.sh b/scripts/ci/build_ext_cmd_tree.sh index df5ecb0d172..4be87ceb611 100644 --- a/scripts/ci/build_ext_cmd_tree.sh +++ b/scripts/ci/build_ext_cmd_tree.sh @@ -38,8 +38,7 @@ for ext in $output; do fi done -pip install azure-cli-core==2.64.0 azure-cli==2.64.0 requests -# azure-storage-blob==1.5.0 +pip install azure-cli-core azure-cli requests azure-storage-blob==1.5.0 # arcdata: ModuleNotFoundError: No module named 'regex._regex' pip install regex python $(cd $(dirname $0); pwd)/update_ext_cmd_tree.py $filter_exts From ac63598cf0b5ad09e6357a0eee8091167ded535d Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Fri, 20 Dec 2024 12:27:25 +0800 Subject: [PATCH 24/27] generate command tree for ml --- scripts/ci/update_ext_cmd_tree.py | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/scripts/ci/update_ext_cmd_tree.py b/scripts/ci/update_ext_cmd_tree.py index 03e8df215fa..ed877ad133c 100644 --- a/scripts/ci/update_ext_cmd_tree.py +++ b/scripts/ci/update_ext_cmd_tree.py @@ -7,6 +7,7 @@ import json import os import sys +import subprocess from azure.cli.core import get_default_cli from azure.cli.core._session import Session @@ -23,6 +24,36 @@ file_name = 'extCmdTreeToUpload.json' +def execute_command(command): + """Execute a shell command and return the output.""" + try: + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if result.returncode == 0: + return result.stdout.strip() + else: + return f"Error: {result.stderr.strip()}" + except Exception as e: + return f"Exception: {str(e)}" + + +def get_package_version(package_name): + """Get the current version of a Python package.""" + command = ["pip", "show", package_name] + output = execute_command(command) + if "Version:" in output: + for line in output.splitlines(): + if line.startswith("Version:"): + version = line.split(":")[1].strip() + print(f"{package_name} current version: {version}") + + +def upgrade_package(package_name): + """Upgrade a Python package to the latest version.""" + command = ["pip", "install", "--upgrade", package_name] + print(f"{command}") + return execute_command(command) + + def merge(data, key, value): if isinstance(value, str): if key in data: @@ -36,6 +67,10 @@ def merge(data, key, value): def update_cmd_tree(ext_name): print(f"Processing {ext_name}") + if ext_name == 'ml': + get_package_version("azure-storage-blob") + upgrade_package("azure-storage-blob") + get_package_version("azure-storage-blob") ext_dir = get_extension_path(ext_name) ext_mod = get_extension_modname(ext_name, ext_dir=ext_dir) From 9c841416328212bd3d0c0e190a2eaf479a73d0da Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Mon, 23 Dec 2024 14:36:26 +0800 Subject: [PATCH 25/27] generate command tree for ml --- scripts/ci/update_ext_cmd_tree.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/ci/update_ext_cmd_tree.py b/scripts/ci/update_ext_cmd_tree.py index ed877ad133c..c5444e6663e 100644 --- a/scripts/ci/update_ext_cmd_tree.py +++ b/scripts/ci/update_ext_cmd_tree.py @@ -71,6 +71,9 @@ def update_cmd_tree(ext_name): get_package_version("azure-storage-blob") upgrade_package("azure-storage-blob") get_package_version("azure-storage-blob") + get_package_version("rpds") + upgrade_package("rpds") + get_package_version("rpds") ext_dir = get_extension_path(ext_name) ext_mod = get_extension_modname(ext_name, ext_dir=ext_dir) From eac3fb4d6fb9d934c90a84e875ea99918effdd28 Mon Sep 17 00:00:00 2001 From: ZelinWang Date: Tue, 24 Dec 2024 14:03:08 +0800 Subject: [PATCH 26/27] generate command tree for ml --- scripts/ci/update_ext_cmd_tree.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/ci/update_ext_cmd_tree.py b/scripts/ci/update_ext_cmd_tree.py index c5444e6663e..621e7709f08 100644 --- a/scripts/ci/update_ext_cmd_tree.py +++ b/scripts/ci/update_ext_cmd_tree.py @@ -74,6 +74,9 @@ def update_cmd_tree(ext_name): get_package_version("rpds") upgrade_package("rpds") get_package_version("rpds") + get_package_version("rpds-py") + upgrade_package("rpds-py") + get_package_version("rpds-py") ext_dir = get_extension_path(ext_name) ext_mod = get_extension_modname(ext_name, ext_dir=ext_dir) From 49d26d1036508307b7be5e39d49a22941ce0f85a Mon Sep 17 00:00:00 2001 From: Zelin Wang Date: Wed, 1 Apr 2026 16:57:45 +0800 Subject: [PATCH 27/27] fix: add rdbms-connect and deploy-to-azure to cmd tree blocklist --- scripts/ci/build_ext_cmd_tree.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ci/build_ext_cmd_tree.sh b/scripts/ci/build_ext_cmd_tree.sh index 4be87ceb611..08b59a9e878 100644 --- a/scripts/ci/build_ext_cmd_tree.sh +++ b/scripts/ci/build_ext_cmd_tree.sh @@ -18,7 +18,8 @@ export AZURE_EXTENSION_INDEX_URL=https://raw.githubusercontent.com/Azure/azure-c output=$(az extension list-available --query [].name -otsv) # azure-cli-ml is replaced by ml # disable alias which relies on Jinja2 2.10 -blocklist=("azure-cli-ml" "alias") +# disable rdbms-connect and deploy-to-azure which cause cmd tree build failures +blocklist=("azure-cli-ml" "alias" "rdbms-connect" "deploy-to-azure") rm -f ~/.azure/extCmdTreeToUpload.json