From ac3912a6656daab4243fc3a3558a5c048522479a Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Thu, 15 May 2025 15:24:29 +0100 Subject: [PATCH 1/4] Try filelock with nb-tester kernel creation --- .gitignore | 1 + scripts/nb-tester/pyproject.toml | 1 + .../qiskit_docs_notebook_tester/__init__.py | 8 +++++++- .../qiskit_docs_notebook_tester/execute.py | 18 ++++++++++-------- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 86f2d400d15..c6ad52a86c4 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ poetry.lock /dist/ /test-results/ Untitled*.ipynb +*.lock diff --git a/scripts/nb-tester/pyproject.toml b/scripts/nb-tester/pyproject.toml index eba7df209df..e922386f74e 100644 --- a/scripts/nb-tester/pyproject.toml +++ b/scripts/nb-tester/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "nbformat~=5.10.4", "ipykernel~=6.29.2", "squeaky==0.7.0", + "filelock~=3.18.0", ] [[project.authors]] diff --git a/scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py b/scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py index 05b9e946b6b..97da3e2697a 100644 --- a/scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py +++ b/scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py @@ -18,6 +18,9 @@ import platform from datetime import datetime +import nbclient +from filelock import FileLock + from .config import get_notebook_jobs, get_parser from .execute import execute_notebook, cancel_trailing_jobs @@ -32,7 +35,10 @@ async def _main() -> None: start_time = datetime.now() print("Executing notebooks:") - results = await asyncio.gather(*(execute_notebook(job) for job in jobs)) + setup_kernel_lock = FileLock(".nbclient-kernel-creation.lock") + results = await asyncio.gather( + *(execute_notebook(job, setup_kernel_lock) for job in jobs) + ) if not args.ignore_trailing_jobs: print("Checking for trailing jobs...") diff --git a/scripts/nb-tester/qiskit_docs_notebook_tester/execute.py b/scripts/nb-tester/qiskit_docs_notebook_tester/execute.py index c0d91948eb0..82dc0baef0c 100644 --- a/scripts/nb-tester/qiskit_docs_notebook_tester/execute.py +++ b/scripts/nb-tester/qiskit_docs_notebook_tester/execute.py @@ -20,6 +20,7 @@ import nbclient import nbformat +from filelock import FileLock from jupyter_client.manager import start_new_async_kernel, AsyncKernelClient from qiskit_ibm_runtime import QiskitRuntimeService @@ -65,7 +66,7 @@ def extract_warnings(notebook: nbformat.NotebookNode) -> list[NotebookWarning]: return notebook_warnings -async def execute_notebook(job: NotebookJob) -> Result: +async def execute_notebook(job: NotebookJob, kernel_setup_lock: FileLock) -> Result: """ Wrapper function for `_execute_notebook` to print status and write result """ @@ -80,7 +81,7 @@ async def execute_notebook(job: NotebookJob) -> Result: nbclient.exceptions.CellTimeoutError, ) try: - nb = await _execute_notebook(job, working_directory.name) + nb = await _execute_notebook(job, working_directory.name, kernel_setup_lock) except execution_exceptions as err: print(f"❌ Problem in {job.path}:\n{err}") return Result(False, reason="Exception in notebook") @@ -118,7 +119,7 @@ async def _execute_in_kernel(kernel: AsyncKernelClient, code: str) -> None: async def _execute_notebook( - job: NotebookJob, working_directory: str + job: NotebookJob, working_directory: str, kernel_setup_lock: FileLock ) -> nbformat.NotebookNode: """ Use nbclient to execute notebook. The steps are: @@ -130,11 +131,12 @@ async def _execute_notebook( """ nb = nbformat.read(job.path, as_version=4) - kernel_manager, kernel = await start_new_async_kernel( - kernel_name="python3", - extra_arguments=["--InlineBackend.figure_format='svg'"], - cwd=working_directory, - ) + with kernel_setup_lock: + kernel_manager, kernel = await start_new_async_kernel( + kernel_name="python3", + extra_arguments=["--InlineBackend.figure_format='svg'"], + cwd=working_directory, + ) await _execute_in_kernel(kernel, job.pre_execute_code) if job.backend_patch: From 2042b4efca61b31ea9386bd7a20dec84cf128d21 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Thu, 15 May 2025 16:18:08 +0100 Subject: [PATCH 2/4] Actually lock correctly Not sure why filelock wasn't working, but asyncio.Lock() does work correctly (tested with print statements) --- scripts/nb-tester/pyproject.toml | 1 - scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py | 3 +-- scripts/nb-tester/qiskit_docs_notebook_tester/execute.py | 8 ++++---- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/nb-tester/pyproject.toml b/scripts/nb-tester/pyproject.toml index e922386f74e..eba7df209df 100644 --- a/scripts/nb-tester/pyproject.toml +++ b/scripts/nb-tester/pyproject.toml @@ -14,7 +14,6 @@ dependencies = [ "nbformat~=5.10.4", "ipykernel~=6.29.2", "squeaky==0.7.0", - "filelock~=3.18.0", ] [[project.authors]] diff --git a/scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py b/scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py index 97da3e2697a..ecf6b744d96 100644 --- a/scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py +++ b/scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py @@ -19,7 +19,6 @@ from datetime import datetime import nbclient -from filelock import FileLock from .config import get_notebook_jobs, get_parser from .execute import execute_notebook, cancel_trailing_jobs @@ -35,7 +34,7 @@ async def _main() -> None: start_time = datetime.now() print("Executing notebooks:") - setup_kernel_lock = FileLock(".nbclient-kernel-creation.lock") + setup_kernel_lock = asyncio.Lock() results = await asyncio.gather( *(execute_notebook(job, setup_kernel_lock) for job in jobs) ) diff --git a/scripts/nb-tester/qiskit_docs_notebook_tester/execute.py b/scripts/nb-tester/qiskit_docs_notebook_tester/execute.py index 82dc0baef0c..4e547a9175d 100644 --- a/scripts/nb-tester/qiskit_docs_notebook_tester/execute.py +++ b/scripts/nb-tester/qiskit_docs_notebook_tester/execute.py @@ -13,6 +13,7 @@ from __future__ import annotations +import asyncio import tempfile import textwrap from dataclasses import dataclass @@ -20,7 +21,6 @@ import nbclient import nbformat -from filelock import FileLock from jupyter_client.manager import start_new_async_kernel, AsyncKernelClient from qiskit_ibm_runtime import QiskitRuntimeService @@ -66,7 +66,7 @@ def extract_warnings(notebook: nbformat.NotebookNode) -> list[NotebookWarning]: return notebook_warnings -async def execute_notebook(job: NotebookJob, kernel_setup_lock: FileLock) -> Result: +async def execute_notebook(job: NotebookJob, kernel_setup_lock: asyncio.Lock) -> Result: """ Wrapper function for `_execute_notebook` to print status and write result """ @@ -119,7 +119,7 @@ async def _execute_in_kernel(kernel: AsyncKernelClient, code: str) -> None: async def _execute_notebook( - job: NotebookJob, working_directory: str, kernel_setup_lock: FileLock + job: NotebookJob, working_directory: str, kernel_setup_lock: asyncio.Lock ) -> nbformat.NotebookNode: """ Use nbclient to execute notebook. The steps are: @@ -131,7 +131,7 @@ async def _execute_notebook( """ nb = nbformat.read(job.path, as_version=4) - with kernel_setup_lock: + async with kernel_setup_lock: kernel_manager, kernel = await start_new_async_kernel( kernel_name="python3", extra_arguments=["--InlineBackend.figure_format='svg'"], From 64dad8e1a1e8ec6aeffafda768d1e421e8e4016d Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Fri, 16 May 2025 11:03:05 +0100 Subject: [PATCH 3/4] Remove leftovers --- .gitignore | 1 - scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index c6ad52a86c4..86f2d400d15 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,3 @@ poetry.lock /dist/ /test-results/ Untitled*.ipynb -*.lock diff --git a/scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py b/scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py index ecf6b744d96..fd1ea39869d 100644 --- a/scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py +++ b/scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py @@ -18,8 +18,6 @@ import platform from datetime import datetime -import nbclient - from .config import get_notebook_jobs, get_parser from .execute import execute_notebook, cancel_trailing_jobs From 6ba01f28db1928acbe70ebc5d84d80c06a443766 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Fri, 16 May 2025 11:03:40 +0100 Subject: [PATCH 4/4] Add comment --- scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py | 8 ++++++-- scripts/nb-tester/qiskit_docs_notebook_tester/execute.py | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py b/scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py index fd1ea39869d..c639e494a31 100644 --- a/scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py +++ b/scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py @@ -32,9 +32,13 @@ async def _main() -> None: start_time = datetime.now() print("Executing notebooks:") - setup_kernel_lock = asyncio.Lock() + + # New kernels choose a port on creation. They usually detect if the port is + # in use and will choose another if so, but there can be race conditions + # when creating many kernels at once. We use a lock to avoid this. + kernel_setup_lock = asyncio.Lock() results = await asyncio.gather( - *(execute_notebook(job, setup_kernel_lock) for job in jobs) + *(execute_notebook(job, kernel_setup_lock) for job in jobs) ) if not args.ignore_trailing_jobs: diff --git a/scripts/nb-tester/qiskit_docs_notebook_tester/execute.py b/scripts/nb-tester/qiskit_docs_notebook_tester/execute.py index 4e547a9175d..a073811805d 100644 --- a/scripts/nb-tester/qiskit_docs_notebook_tester/execute.py +++ b/scripts/nb-tester/qiskit_docs_notebook_tester/execute.py @@ -132,6 +132,10 @@ async def _execute_notebook( nb = nbformat.read(job.path, as_version=4) async with kernel_setup_lock: + # New kernels choose a port on creation. They usually detect if the + # port is in use and will choose another if so, but there can be race + # conditions when creating many kernels at once.The lock avoids this. + # This might be fixed by https://github.com/jupyter/nbclient/pull/327 kernel_manager, kernel = await start_new_async_kernel( kernel_name="python3", extra_arguments=["--InlineBackend.figure_format='svg'"],