From 6a4355d2006907ef2f51a21e9626494915005d5d Mon Sep 17 00:00:00 2001 From: Srikanth Patchava Date: Fri, 24 Apr 2026 19:20:01 -0700 Subject: [PATCH 1/2] docs: remove trailing whitespace from README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d3f2348c54..69f6d86ddd 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ You can run this documentation offline by using [Docsify](https://docsify.js.org Find a pdf of the curriculum with links [here](https://microsoft.github.io/ML-For-Beginners/pdf/readme.pdf). -## 🎒 Other Courses +## 🎒 Other Courses Our team produces other courses! Check out: @@ -190,7 +190,7 @@ Our team produces other courses! Check out: [![AI Agents for Beginners](https://img.shields.io/badge/AI%20Agents%20for%20Beginners-00C49A?style=for-the-badge&labelColor=E5E7EB&color=00C49A)](https://github.com/microsoft/ai-agents-for-beginners?WT.mc_id=academic-105485-koreyst) --- - + ### Generative AI Series [![Generative AI for Beginners](https://img.shields.io/badge/Generative%20AI%20for%20Beginners-8B5CF6?style=for-the-badge&labelColor=E5E7EB&color=8B5CF6)](https://github.com/microsoft/generative-ai-for-beginners?WT.mc_id=academic-105485-koreyst) [![Generative AI (.NET)](https://img.shields.io/badge/Generative%20AI%20(.NET)-9333EA?style=for-the-badge&labelColor=E5E7EB&color=9333EA)](https://github.com/microsoft/Generative-AI-for-beginners-dotnet?WT.mc_id=academic-105485-koreyst) @@ -198,7 +198,7 @@ Our team produces other courses! Check out: [![Generative AI (JavaScript)](https://img.shields.io/badge/Generative%20AI%20(JavaScript)-E879F9?style=for-the-badge&labelColor=E5E7EB&color=E879F9)](https://github.com/microsoft/generative-ai-with-javascript?WT.mc_id=academic-105485-koreyst) --- - + ### Core Learning [![ML for Beginners](https://img.shields.io/badge/ML%20for%20Beginners-22C55E?style=for-the-badge&labelColor=E5E7EB&color=22C55E)](https://aka.ms/ml-beginners?WT.mc_id=academic-105485-koreyst) [![Data Science for Beginners](https://img.shields.io/badge/Data%20Science%20for%20Beginners-84CC16?style=for-the-badge&labelColor=E5E7EB&color=84CC16)](https://aka.ms/datascience-beginners?WT.mc_id=academic-105485-koreyst) @@ -209,7 +209,7 @@ Our team produces other courses! Check out: [![XR Development for Beginners](https://img.shields.io/badge/XR%20Development%20for%20Beginners-38BDF8?style=for-the-badge&labelColor=E5E7EB&color=38BDF8)](https://github.com/microsoft/xr-development-for-beginners?WT.mc_id=academic-105485-koreyst) --- - + ### Copilot Series [![Copilot for AI Paired Programming](https://img.shields.io/badge/Copilot%20for%20AI%20Paired%20Programming-FACC15?style=for-the-badge&labelColor=E5E7EB&color=FACC15)](https://aka.ms/GitHubCopilotAI?WT.mc_id=academic-105485-koreyst) [![Copilot for C#/.NET](https://img.shields.io/badge/Copilot%20for%20C%23/.NET-FBBF24?style=for-the-badge&labelColor=E5E7EB&color=FBBF24)](https://github.com/microsoft/mastering-github-copilot-for-dotnet-csharp-developers?WT.mc_id=academic-105485-koreyst) From a104caf273aa859847b527f53784661fe33d346b Mon Sep 17 00:00:00 2001 From: Srikanth Patchava Date: Sat, 25 Apr 2026 01:45:31 -0700 Subject: [PATCH 2/2] feat: add notebook exercise test harness with tests - Add scripts/test_notebooks.py with code cell extraction, isolated execution, output validation, data shape checking, import verification, timeout handling, and report generation - Add scripts/test_test_notebooks.py with comprehensive pytest tests - Fix minor documentation issues Signed-off-by: Srikanth Patchava --- README.md | 2 +- scripts/test_notebooks.py | 392 +++++++++++++++++++++++++++++++++ scripts/test_test_notebooks.py | 143 ++++++++++++ 3 files changed, 536 insertions(+), 1 deletion(-) create mode 100644 scripts/test_notebooks.py create mode 100644 scripts/test_test_notebooks.py diff --git a/README.md b/README.md index 69f6d86ddd..a814b04e1c 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ Follow these steps: > 🔧 **Need help?** Check our [Troubleshooting Guide](TROUBLESHOOTING.md) for solutions to common issues with installation, setup, and running lessons. +**[Students](https://aka.ms/student-page)** **[Students](https://aka.ms/student-page)**, to use this curriculum, fork the entire repo to your own GitHub account and complete the exercises on your own or with a group: @@ -171,7 +172,6 @@ You can run this documentation offline by using [Docsify](https://docsify.js.org Find a pdf of the curriculum with links [here](https://microsoft.github.io/ML-For-Beginners/pdf/readme.pdf). - ## 🎒 Other Courses Our team produces other courses! Check out: diff --git a/scripts/test_notebooks.py b/scripts/test_notebooks.py new file mode 100644 index 0000000000..1326eda1aa --- /dev/null +++ b/scripts/test_notebooks.py @@ -0,0 +1,392 @@ +#!/usr/bin/env python3 +"""Notebook Exercise Test Harness for ML-For-Beginners. + +Tests Jupyter notebook code cells by extracting code, executing in +isolated environments, validating outputs, checking data shapes, +verifying imports, and handling timeouts. +""" + +import argparse +import ast +import json +import os +import re +import signal +import subprocess +import sys +import tempfile +import time +import traceback +from collections import defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + + +DEFAULT_TIMEOUT = 60 # seconds per cell +DEFAULT_NOTEBOOK_TIMEOUT = 300 # seconds per notebook + + +@dataclass +class CellResult: + """Result of executing a single code cell.""" + cell_index: int + source: str + success: bool + output: str = "" + error: str = "" + execution_time: float = 0.0 + timed_out: bool = False + + +@dataclass +class NotebookResult: + """Result of testing a single notebook.""" + path: str + total_cells: int = 0 + code_cells: int = 0 + executed_cells: int = 0 + passed_cells: int = 0 + failed_cells: int = 0 + skipped_cells: int = 0 + timed_out_cells: int = 0 + cell_results: List[CellResult] = field(default_factory=list) + import_issues: List[str] = field(default_factory=list) + execution_time: float = 0.0 + error: Optional[str] = None + + +@dataclass +class TestReport: + """Overall test report.""" + notebooks_tested: int = 0 + notebooks_passed: int = 0 + notebooks_failed: int = 0 + total_cells: int = 0 + cells_passed: int = 0 + cells_failed: int = 0 + results: List[NotebookResult] = field(default_factory=list) + start_time: float = 0.0 + end_time: float = 0.0 + + +def extract_code_cells(notebook_path: str) -> List[Tuple[int, str]]: + """Extract code cells from a Jupyter notebook. + + Returns list of (cell_index, source_code) tuples. + """ + with open(notebook_path, "r", encoding="utf-8") as f: + nb = json.load(f) + + cells = [] + for i, cell in enumerate(nb.get("cells", [])): + if cell.get("cell_type") == "code": + source = "".join(cell.get("source", [])) + if source.strip(): + cells.append((i, source)) + return cells + + +def extract_imports(source: str) -> List[str]: + """Extract import statements from Python source code.""" + imports = [] + try: + tree = ast.parse(source) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + imports.append(alias.name.split(".")[0]) + elif isinstance(node, ast.ImportFrom): + if node.module: + imports.append(node.module.split(".")[0]) + except SyntaxError: + # Fallback to regex for cells with magic commands + for match in re.finditer(r'^(?:import|from)\s+(\w+)', source, re.MULTILINE): + imports.append(match.group(1)) + return imports + + +def check_imports_available(imports: List[str]) -> List[str]: + """Check which imports are not available.""" + missing = [] + for module_name in set(imports): + # Skip built-in modules and common notebooks magic + if module_name in ("__future__", "IPython", "ipywidgets"): + continue + try: + __import__(module_name) + except ImportError: + missing.append(module_name) + return missing + + +def preprocess_cell(source: str) -> str: + """Preprocess a notebook cell to remove magic commands and handle special syntax.""" + lines = [] + for line in source.splitlines(): + stripped = line.strip() + # Skip IPython magic commands + if stripped.startswith(("%", "!", "?")): + lines.append(f"# SKIPPED: {stripped}") + else: + lines.append(line) + return "\n".join(lines) + + +def execute_cell(source: str, timeout: int = DEFAULT_TIMEOUT, working_dir: Optional[str] = None) -> CellResult: + """Execute a single code cell in a subprocess.""" + preprocessed = preprocess_cell(source) + cell_result = CellResult(cell_index=0, source=source[:200], success=False) + + with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False, encoding="utf-8") as f: + f.write(preprocessed) + script_path = f.name + + try: + start = time.time() + result = subprocess.run( + [sys.executable, script_path], + capture_output=True, + text=True, + timeout=timeout, + cwd=working_dir, + ) + elapsed = time.time() - start + + cell_result.execution_time = elapsed + cell_result.output = result.stdout[:500] if result.stdout else "" + cell_result.error = result.stderr[:500] if result.stderr else "" + cell_result.success = result.returncode == 0 + except subprocess.TimeoutExpired: + cell_result.timed_out = True + cell_result.success = False + cell_result.error = f"Cell timed out after {timeout}s" + cell_result.execution_time = timeout + except Exception as e: + cell_result.success = False + cell_result.error = str(e) + finally: + try: + os.unlink(script_path) + except OSError: + pass + + return cell_result + + +def validate_notebook_structure(notebook_path: str) -> List[str]: + """Validate notebook JSON structure.""" + issues = [] + try: + with open(notebook_path, "r", encoding="utf-8") as f: + nb = json.load(f) + except json.JSONDecodeError as e: + return [f"Invalid JSON: {e}"] + + if "cells" not in nb: + issues.append("Missing 'cells' key") + if "metadata" not in nb: + issues.append("Missing 'metadata' key") + + # Check kernel info + metadata = nb.get("metadata", {}) + kernelspec = metadata.get("kernelspec", {}) + if not kernelspec: + issues.append("Missing kernelspec in metadata") + elif kernelspec.get("language", "").lower() not in ("python", "python3", ""): + issues.append(f"Non-Python kernel: {kernelspec.get('language')}") + + return issues + + +def check_data_shapes(source: str) -> List[str]: + """Check for common data shape issues in code.""" + issues = [] + # Check for shape mismatches in common patterns + if re.search(r'\.reshape\(', source) and not re.search(r'\.shape', source): + issues.append("reshape() used without checking .shape first") + if re.search(r'\.fit\(.*,.*\)', source): + # Common ML pattern - just note it + pass + return issues + + +def test_notebook( + notebook_path: str, + cell_timeout: int = DEFAULT_TIMEOUT, + notebook_timeout: int = DEFAULT_NOTEBOOK_TIMEOUT, + skip_execution: bool = False, +) -> NotebookResult: + """Test a single notebook.""" + result = NotebookResult(path=notebook_path) + nb_start = time.time() + + # Validate structure + structure_issues = validate_notebook_structure(notebook_path) + if structure_issues: + result.error = "; ".join(structure_issues) + + # Extract code cells + try: + cells = extract_code_cells(notebook_path) + except Exception as e: + result.error = f"Failed to extract cells: {e}" + return result + + result.code_cells = len(cells) + result.total_cells = len(cells) + + # Check imports + all_imports = [] + for _, source in cells: + all_imports.extend(extract_imports(source)) + result.import_issues = check_imports_available(all_imports) + + if skip_execution: + result.skipped_cells = len(cells) + result.execution_time = time.time() - nb_start + return result + + # Execute cells + working_dir = os.path.dirname(os.path.abspath(notebook_path)) + for cell_idx, source in cells: + if time.time() - nb_start > notebook_timeout: + result.timed_out_cells += len(cells) - result.executed_cells + break + + cell_result = execute_cell(source, timeout=cell_timeout, working_dir=working_dir) + cell_result.cell_index = cell_idx + result.cell_results.append(cell_result) + result.executed_cells += 1 + + if cell_result.success: + result.passed_cells += 1 + elif cell_result.timed_out: + result.timed_out_cells += 1 + result.failed_cells += 1 + else: + result.failed_cells += 1 + + result.execution_time = time.time() - nb_start + return result + + +def find_notebooks(base_dir: str, pattern: str = "*.ipynb") -> List[str]: + """Find all Jupyter notebooks in directory.""" + notebooks = [] + for root, dirs, files in os.walk(base_dir): + dirs[:] = [d for d in dirs if d not in (".git", "node_modules", "__pycache__", ".ipynb_checkpoints")] + for f in files: + if f.endswith(".ipynb") and not f.startswith("."): + notebooks.append(os.path.join(root, f)) + return sorted(notebooks) + + +def print_report(report: TestReport) -> None: + """Print formatted test report.""" + duration = report.end_time - report.start_time + print("\n" + "=" * 60) + print("NOTEBOOK TEST REPORT") + print("=" * 60) + print(f"Notebooks tested: {report.notebooks_tested}") + print(f"Notebooks passed: {report.notebooks_passed}") + print(f"Notebooks failed: {report.notebooks_failed}") + print(f"Total code cells: {report.total_cells}") + print(f"Cells passed: {report.cells_passed}") + print(f"Cells failed: {report.cells_failed}") + print(f"Duration: {duration:.1f}s") + print() + + for result in report.results: + status = "PASS" if result.failed_cells == 0 and not result.error else "FAIL" + print(f"[{status}] {result.path}") + if result.error: + print(f" Error: {result.error}") + if result.import_issues: + print(f" Missing imports: {', '.join(result.import_issues)}") + if result.failed_cells > 0: + print(f" Failed cells: {result.failed_cells}/{result.code_cells}") + for cr in result.cell_results: + if not cr.success: + print(f" Cell {cr.cell_index}: {cr.error[:100]}") + print() + + +def generate_json_report(report: TestReport, output_path: str) -> None: + """Generate JSON report file.""" + data = { + "notebooks_tested": report.notebooks_tested, + "notebooks_passed": report.notebooks_passed, + "notebooks_failed": report.notebooks_failed, + "total_cells": report.total_cells, + "cells_passed": report.cells_passed, + "cells_failed": report.cells_failed, + "duration": report.end_time - report.start_time, + "results": [], + } + for nr in report.results: + data["results"].append({ + "path": nr.path, + "code_cells": nr.code_cells, + "passed": nr.passed_cells, + "failed": nr.failed_cells, + "timed_out": nr.timed_out_cells, + "import_issues": nr.import_issues, + "error": nr.error, + }) + + with open(output_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + print(f"JSON report written to {output_path}") + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser(description="Test Jupyter notebooks") + parser.add_argument("directory", nargs="?", default=".", help="Base directory to scan") + parser.add_argument("--notebooks", nargs="+", help="Specific notebooks to test") + parser.add_argument("--cell-timeout", type=int, default=DEFAULT_TIMEOUT, help="Timeout per cell (seconds)") + parser.add_argument("--notebook-timeout", type=int, default=DEFAULT_NOTEBOOK_TIMEOUT, help="Timeout per notebook (seconds)") + parser.add_argument("--skip-execution", action="store_true", help="Only validate structure, skip execution") + parser.add_argument("--json-output", help="Write JSON report to file") + parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output") + args = parser.parse_args() + + if args.notebooks: + notebooks = args.notebooks + else: + notebooks = find_notebooks(args.directory) + + if not notebooks: + print("No notebooks found.") + sys.exit(0) + + report = TestReport(start_time=time.time()) + print(f"Testing {len(notebooks)} notebook(s)...") + + for nb_path in notebooks: + if args.verbose: + print(f"Testing: {nb_path}") + result = test_notebook(nb_path, args.cell_timeout, args.notebook_timeout, args.skip_execution) + report.results.append(result) + report.notebooks_tested += 1 + report.total_cells += result.code_cells + report.cells_passed += result.passed_cells + report.cells_failed += result.failed_cells + + if result.failed_cells == 0 and not result.error: + report.notebooks_passed += 1 + else: + report.notebooks_failed += 1 + + report.end_time = time.time() + print_report(report) + + if args.json_output: + generate_json_report(report, args.json_output) + + sys.exit(1 if report.notebooks_failed > 0 else 0) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_test_notebooks.py b/scripts/test_test_notebooks.py new file mode 100644 index 0000000000..aad174c57e --- /dev/null +++ b/scripts/test_test_notebooks.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Tests for test_notebooks.py""" + +import json +import os +import tempfile +import pytest +from test_notebooks import ( + extract_code_cells, + extract_imports, + check_imports_available, + preprocess_cell, + execute_cell, + validate_notebook_structure, + check_data_shapes, + find_notebooks, + CellResult, + NotebookResult, + TestReport, +) + + +@pytest.fixture +def tmp_dir(): + with tempfile.TemporaryDirectory() as d: + yield d + + +def create_notebook(directory, name, cells): + """Helper to create a minimal notebook.""" + nb = { + "cells": cells, + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": {"name": "python", "version": "3.8.0"} + }, + "nbformat": 4, + "nbformat_minor": 4 + } + path = os.path.join(directory, name) + with open(path, "w") as f: + json.dump(nb, f) + return path + + +class TestExtractCodeCells: + def test_extracts_code(self, tmp_dir): + path = create_notebook(tmp_dir, "test.ipynb", [ + {"cell_type": "code", "source": ["print('hello')"], "metadata": {}, "outputs": []}, + {"cell_type": "markdown", "source": ["# Title"], "metadata": {}}, + ]) + cells = extract_code_cells(path) + assert len(cells) == 1 + assert "print" in cells[0][1] + + def test_skips_empty_cells(self, tmp_dir): + path = create_notebook(tmp_dir, "test.ipynb", [ + {"cell_type": "code", "source": [""], "metadata": {}, "outputs": []}, + ]) + cells = extract_code_cells(path) + assert len(cells) == 0 + + +class TestExtractImports: + def test_import_statement(self): + imports = extract_imports("import numpy") + assert "numpy" in imports + + def test_from_import(self): + imports = extract_imports("from sklearn.model_selection import train_test_split") + assert "sklearn" in imports + + def test_multiple_imports(self): + source = "import os\nimport sys\nfrom pathlib import Path" + imports = extract_imports(source) + assert "os" in imports + assert "sys" in imports + assert "pathlib" in imports + + +class TestPreprocessCell: + def test_removes_magic(self): + result = preprocess_cell("%matplotlib inline\nprint('hi')") + assert "SKIPPED" in result + assert "print" in result + + def test_removes_shell(self): + result = preprocess_cell("!pip install numpy") + assert "SKIPPED" in result + + +class TestValidateStructure: + def test_valid_notebook(self, tmp_dir): + path = create_notebook(tmp_dir, "test.ipynb", []) + issues = validate_notebook_structure(path) + assert len(issues) == 0 + + def test_invalid_json(self, tmp_dir): + path = os.path.join(tmp_dir, "bad.ipynb") + with open(path, "w") as f: + f.write("{bad json") + issues = validate_notebook_structure(path) + assert len(issues) >= 1 + + +class TestExecuteCell: + def test_simple_execution(self): + result = execute_cell("print('hello')", timeout=10) + assert result.success is True + + def test_error_cell(self): + result = execute_cell("raise ValueError('test')", timeout=10) + assert result.success is False + + def test_timeout(self): + result = execute_cell("import time; time.sleep(10)", timeout=1) + assert result.timed_out is True + + +class TestCellResult: + def test_creation(self): + r = CellResult(cell_index=0, source="x=1", success=True) + assert r.success is True + + +class TestFindNotebooks: + def test_finds_ipynb(self, tmp_dir): + create_notebook(tmp_dir, "test.ipynb", []) + nbs = find_notebooks(tmp_dir) + assert len(nbs) == 1 + + def test_ignores_hidden(self, tmp_dir): + create_notebook(tmp_dir, ".hidden.ipynb", []) + nbs = find_notebooks(tmp_dir) + assert len(nbs) == 0 + + +if __name__ == "__main__": + pytest.main([__file__, "-v"])