diff --git a/.dagger/modules/e2e/dagger.json b/.dagger/modules/e2e/dagger.json index 0466464..a8bdbc1 100644 --- a/.dagger/modules/e2e/dagger.json +++ b/.dagger/modules/e2e/dagger.json @@ -1,6 +1,6 @@ { "name": "e2e", - "engineVersion": "v0.21.4", + "engineVersion": "v1.0.0-0", "sdk": { "source": "dang" }, diff --git a/.dagger/modules/e2e/fixtures/mixed-discovery/.dagger-python-sdk-skip-generate b/.dagger/modules/e2e/fixtures/mixed-discovery/.dagger-python-sdk-skip-generate new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/mixed-discovery/.dagger-python-sdk-skip-generate @@ -0,0 +1 @@ + diff --git a/.dagger/modules/e2e/fixtures/mixed-discovery/ancestor/dagger-module.toml b/.dagger/modules/e2e/fixtures/mixed-discovery/ancestor/dagger-module.toml new file mode 100644 index 0000000..036bad5 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/mixed-discovery/ancestor/dagger-module.toml @@ -0,0 +1,4 @@ +engineVersion = "v1.0.0-0" + +[runtime] +source = "python" diff --git a/.dagger/modules/e2e/fixtures/mixed-discovery/ancestor/work/app/dagger.json b/.dagger/modules/e2e/fixtures/mixed-discovery/ancestor/work/app/dagger.json new file mode 100644 index 0000000..02e020a --- /dev/null +++ b/.dagger/modules/e2e/fixtures/mixed-discovery/ancestor/work/app/dagger.json @@ -0,0 +1,7 @@ +{ + "name": "mixed-discovery-app", + "engineVersion": "v1.0.0-0", + "sdk": { + "source": "python" + } +} diff --git a/.dagger/modules/e2e/fixtures/mixed-discovery/ancestor/work/app/nested/deeper/.keep b/.dagger/modules/e2e/fixtures/mixed-discovery/ancestor/work/app/nested/deeper/.keep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/mixed-discovery/ancestor/work/app/nested/deeper/.keep @@ -0,0 +1 @@ + diff --git a/.dagger/modules/e2e/fixtures/mixed-discovery/ancestor/work/app/pyproject.toml b/.dagger/modules/e2e/fixtures/mixed-discovery/ancestor/work/app/pyproject.toml new file mode 100644 index 0000000..cb28635 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/mixed-discovery/ancestor/work/app/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "mixed-discovery-app" +version = "0.1.0" +requires-python = ">=3.14" +dependencies = ["dagger-io"] + +[build-system] +requires = ["uv_build>=0.8.4,<0.9.0"] +build-backend = "uv_build" + +[tool.uv.sources] +dagger-io = { path = "sdk", editable = true } diff --git a/.dagger/modules/e2e/fixtures/mixed-discovery/ancestor/work/app/src/mixed_discovery_app/__init__.py b/.dagger/modules/e2e/fixtures/mixed-discovery/ancestor/work/app/src/mixed_discovery_app/__init__.py new file mode 100644 index 0000000..e8b0dc4 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/mixed-discovery/ancestor/work/app/src/mixed_discovery_app/__init__.py @@ -0,0 +1,6 @@ +from dagger import object_type + + +@object_type +class MixedDiscoveryApp: + pass diff --git a/.dagger/modules/e2e/fixtures/toml/.dagger-python-sdk-skip-generate b/.dagger/modules/e2e/fixtures/toml/.dagger-python-sdk-skip-generate new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/toml/.dagger-python-sdk-skip-generate @@ -0,0 +1 @@ + diff --git a/.dagger/modules/e2e/fixtures/toml/app/dagger-module.toml b/.dagger/modules/e2e/fixtures/toml/app/dagger-module.toml new file mode 100644 index 0000000..036bad5 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/toml/app/dagger-module.toml @@ -0,0 +1,4 @@ +engineVersion = "v1.0.0-0" + +[runtime] +source = "python" diff --git a/.dagger/modules/e2e/fixtures/toml/app/nested/.keep b/.dagger/modules/e2e/fixtures/toml/app/nested/.keep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/.dagger/modules/e2e/fixtures/toml/app/nested/.keep @@ -0,0 +1 @@ + diff --git a/.dagger/modules/e2e/main.dang b/.dagger/modules/e2e/main.dang index a39ae7b..a1932eb 100644 --- a/.dagger/modules/e2e/main.dang +++ b/.dagger/modules/e2e/main.dang @@ -13,6 +13,9 @@ type E2e { let depsModulePath: String! = fixtureRoot + "/deps/app" let configModulePath: String! = fixtureRoot + "/config/app" let configuredModulePath: String! = fixtureRoot + "/config/configured" + let tomlModulePath: String! = fixtureRoot + "/toml/app" + let mixedDiscoveryModulePath: String! = fixtureRoot + "/mixed-discovery/ancestor/work/app" + let mixedDiscoveryNestedPath: String! = mixedDiscoveryModulePath + "/nested/deeper" let generatedMarkerPath: String! = "sdk/src/dagger/client/gen.py" let generatedMarkerContents: String! = "Code generated by dagger." @@ -73,12 +76,47 @@ type E2e { let nested = pythonSdk.mod(ws, path: lookupNestedPath) let exact = pythonSdk.mod(ws, path: lookupModulePath, findUp: false) - assert(nested.path == lookupModulePath, "find-up lookup returned the wrong module path") - assert(exact.path == lookupModulePath, "exact lookup returned the wrong module path") + assert(nested.rootPath == lookupModulePath, "find-up lookup returned the wrong module path") + assert(exact.rootPath == lookupModulePath, "exact lookup returned the wrong module path") null } + """ + Lookup must choose the nearest module config even when its filename differs + from an ancestor config, and modern TOML modules must be supported. + """ + pub mixedConfigLookupCheck(ws: Workspace!): Void @check { + let mod = pythonSdk.mod(ws, path: tomlModulePath + "/nested") + assert(mod.rootPath == tomlModulePath, "lookup chose an outer dagger.json instead of the nearer dagger-module.toml") + null + } + + """ + Managed-module discovery is cwd-aware and supports both config formats. + """ + pub moduleDiscoveryCheck(ws: Workspace!): Void @check { + let roots = pythonSdk.modules(ws).{{rootPath}}.map { mod => mod.rootPath } + assert(contains(roots, generateModulePath), "root discovery missed a legacy Python module") + assert(contains(roots, tomlModulePath), "root discovery missed a TOML Python module") + assert(contains(roots, nonPythonModulePath) == false, "discovery included a non-Python module") + + let nestedWs = ws.directory("/").asWorkspace(cwd: lookupNestedPath) + let scoped = pythonSdk.modules(nestedWs).{{rootPath, path}} + let scopedRoots = scoped.map { mod => mod.rootPath } + let scopedPaths = scoped.map { mod => mod.path } + assert(scoped.length == 1, "nested cwd should see only its nearest managed module") + assert(contains(scopedRoots, lookupModulePath), "nested cwd resolved the wrong managed module") + assert(contains(scopedPaths, ".."), "module path should be relative to the nested cwd") + + let mixedWs = ws.directory("/").asWorkspace(cwd: mixedDiscoveryNestedPath) + let mixed = pythonSdk.modules(mixedWs).{{rootPath, path}} + assert(mixed.length == 1, "mixed-config cwd should see only its nearest managed module") + assert(contains(mixed.map { mod => mod.rootPath }, mixedDiscoveryModulePath), "mixed-config discovery chose the farther dagger-module.toml") + assert(contains(mixed.map { mod => mod.path }, "../.."), "mixed-config module path should be relative to the nested cwd") + null + } + """ initModule should seed SDK-owned template files without running codegen and without writing engine-owned files (dagger.json / dagger-module.toml). @@ -153,18 +191,6 @@ type E2e { null } - """ - initClient should not materialize files itself. The engine records the client - and owns the generated context changeset. - """ - pub initClientCheck(ws: Workspace!): Void @check { - let changes = pythonSdk.initClient(ws, path: outputRoot + "/client", module: generateModulePath, dev: true) - - assert(changes.isEmpty, "initClient should return an empty SDK changeset") - - null - } - """ config.get should reflect pyproject.toml and report unset values as null rather than guessing, and config.set should edit only pyproject.toml. diff --git a/dagger.json b/dagger.json index 62c29c8..f92c506 100644 --- a/dagger.json +++ b/dagger.json @@ -7,8 +7,8 @@ "dependencies": [ { "name": "polyfill", - "source": "github.com/dagger/sdk-sdk/polyfill@main", - "pin": "a16466390cacd68bd72f7ae9910d990966fb2226" + "source": "github.com/dagger/polyfill@main", + "pin": "e90bbfc4843258a877a3a95b8db1571e7981e65f" } ] } diff --git a/dagger.toml b/dagger.toml index 724adf6..47265a4 100644 --- a/dagger.toml +++ b/dagger.toml @@ -27,3 +27,9 @@ path = ".dagger/modules/e2e/fixtures/deps/app" [[modules.python-sdk.as-sdk.modules]] path = ".dagger/modules/e2e/fixtures/skip/app" + +[[modules.python-sdk.as-sdk.modules]] +path = ".dagger/modules/e2e/fixtures/toml/app" + +[[modules.python-sdk.as-sdk.modules]] +path = ".dagger/modules/e2e/fixtures/mixed-discovery/ancestor/work/app" diff --git a/docs/cwd-aware-discovery.md b/docs/cwd-aware-discovery.md new file mode 100644 index 0000000..e342183 --- /dev/null +++ b/docs/cwd-aware-discovery.md @@ -0,0 +1,20 @@ +# CWD-aware module discovery + +The Python SDK delegates config discovery to `github.com/dagger/polyfill`, then +intersects the discovered directories with `currentModule.asSDK.modules`. The +engine's managed-module list remains authoritative while the caller's current +directory determines scope. + +Discovery returns modules at or below the cwd and, when the cwd has no module +config, its nearest enclosing module. Both `dagger-module.toml` and legacy +`dagger.json` are considered together, so the nearest config wins regardless of +filename. Virtual environments and installed packages are excluded. + +```console +dagger check -l +dagger call e-2-e mixed-config-lookup-check +dagger call e-2-e module-discovery-check +``` + +The fixtures cover mixed nested config formats, modern and legacy configs, +non-Python exclusion, root discovery, and discovery from inside a module. diff --git a/mod.dang b/mod.dang index 6d64737..9ff4f4b 100644 --- a/mod.dang +++ b/mod.dang @@ -3,9 +3,9 @@ A Dagger module that uses the Python SDK. """ type Mod { """ - Workspace-relative path of this module root. + Workspace-root-relative path of this module root. """ - pub path: String! + pub rootPath: String! """ The workspace this module belongs to. @@ -17,18 +17,37 @@ type Mod { """ let skipGenerateFilename: String! + """ + Module root relative to the client's cwd. + """ + pub path: String! { + let cwd = ws.cwd.trimPrefix("/").trimSuffix("/") + if (rootPath == cwd) { + "." + } else if (cwd == "") { + rootPath + } else if (rootPath.trimPrefix(cwd + "/") != rootPath) { + rootPath.trimPrefix(cwd + "/") + } else if (rootPath == "." or cwd.trimPrefix(rootPath + "/") != cwd) { + let depth = if (rootPath == ".") { 0 } else { rootPath.split("/").length } + cwd.split("/").dropFirst(depth).map { segment => ".." }.join("/") + } else { + rootPath + } + } + """ Whether this module or an ancestor contains the configured generate skip marker. """ pub skipGenerate: Boolean! { - ws.findUp(name: skipGenerateFilename, from: path) != null + ws.findUp(name: skipGenerateFilename, from: rootPath) != null } """ Manage this module's Python build configuration (pyproject.toml). """ pub config: ModConfig! { - ModConfig(path: path, ws: ws) + ModConfig(path: rootPath, ws: ws) } """ @@ -39,7 +58,7 @@ type Mod { if (skipGenerate) { polyfill.workspace(ws).fork.changes } else { - polyfill.workspace(ws).moduleSource("/" + path).generate.changes + polyfill.workspace(ws).moduleSource("/" + rootPath).generate.changes } } } diff --git a/python-sdk.dang b/python-sdk.dang index 730f61e..6f0ca99 100644 --- a/python-sdk.dang +++ b/python-sdk.dang @@ -2,6 +2,10 @@ Manage Dagger modules that use the Python SDK. """ type PythonSdk { + let legacyConfigPattern = "\"sdk\"\\s*:\\s*\\{[^}]*\"source\"\\s*:\\s*\"python\"" + let tomlConfigPattern = "\\[runtime\\][^\\[]*source\\s*=\\s*\"python\"" + let moduleConfigFilenames: [String!]! = ["dagger-module.toml", "dagger.json"] + """ Marker filename that skips generate when found at or above a Python SDK module root. """ @@ -13,29 +17,47 @@ type PythonSdk { pub targetRuntime: String! { "python" } """ - Return every legacy dagger.json Dagger module whose sdk.source is "python". - - This discovery path is obsolete for workspace-managed modules; the engine - owns the modules..as-sdk.modules source of truth. + Return every managed Python SDK module visible from the client's cwd: the + nearest enclosing module plus modules at or below the cwd. Discovery uses the + shared polyfill and intersects its results with the SDK list on the passed + workspace. """ pub modules(ws: Workspace!): [Mod!]! { - ws - .directory("/", include: ["**/dagger.json"]) - .search( - pattern: "\"sdk\"\\s*:\\s*\\{[^}]*\"source\"\\s*:\\s*\"python\"", - globs: ["**/dagger.json"], - filesOnly: true, - multiline: true, - dotall: true, - ) - .{{filePath}} - .map { configPath => - mod( - ws, - if (configPath.filePath == "dagger.json") { "." } else { configPath.filePath.trimSuffix("/dagger.json") }, - findUp: false, - ) + let managed = ws.sdk(name: currentModule.name).modules.{{source}} + let cwd = normalizePath(ws.cwd) + polyfill.workspace(ws) + .findConfigDirs(moduleConfigFilenames, exclude: ["**/.venv/**", "**/site-packages/**"]) + .map { dir => workspacePath(cwd, dir) } + .uniq + .filter { path => managed.filter { m => normalizePath(m.source) == path }.length > 0 } + .map { path => Mod(rootPath: path, ws: ws, skipGenerateFilename: skipGenerateFilename) } + } + + let workspacePath(cwd: String!, path: String!): String! { + let base = if (cwd == ".") { [] } else { cwd.split("/") } + let segments = path.split("/").reduce(base) { acc, segment => + if (segment == "..") { + acc.dropLast(1) + } else if (segment == "." or segment == "") { + acc + } else { + acc + [segment] } + } + if (segments.length == 0) { "." } else { segments.join("/") } + } + + let normalizePath(path: String!): String! { + let normalized = path.trimPrefix("./").trimPrefix("/").trimSuffix("/") + if (normalized == "") { "." } else { normalized } + } + + let pathDepth(path: String): Int! { + if (path == null) { -1 } else if (path == ".") { 0 } else { path.split("/").length } + } + + let configDir(path: String!, filename: String!): String! { + normalizePath(path.trimPrefix("/").trimSuffix(filename)) } """ @@ -71,42 +93,42 @@ type PythonSdk { """ pub mod(ws: Workspace!, path: String! = ".", findUp: Boolean! = true): Mod! { let modPath = if (findUp) { - let foundConfigPath = ws.findUp("dagger.json", path) - if (foundConfigPath == null) { + let foundToml = ws.findUp("dagger-module.toml", path) + let foundJson = ws.findUp("dagger.json", path) + let tomlBase = if (foundToml == null) { null } else { configDir(foundToml, "dagger-module.toml") } + let jsonBase = if (foundJson == null) { null } else { configDir(foundJson, "dagger.json") } + + if (tomlBase == null and jsonBase == null) { raise "no Dagger module found containing path: " + path + } else if (tomlBase != null and pathDepth(tomlBase) > pathDepth(jsonBase)) { + validateConfig(ws, tomlBase, "dagger-module.toml", tomlConfigPattern, path) + } else if (jsonBase != null) { + validateConfig(ws, jsonBase, "dagger.json", legacyConfigPattern, path) } else { - let configPath = foundConfigPath.trimPrefix("/") - if ( - ws - .directory("/", include: [configPath]) - .file(configPath) - .search( - pattern: "\"sdk\"\\s*:\\s*\\{[^}]*\"source\"\\s*:\\s*\"python\"", - multiline: true, - dotall: true, - limit: 1, - ) - .{{id}} - .length == 0 - ) { - raise "Dagger module does not use the Python SDK: " + path - } else if (configPath == "dagger.json") { - "." - } else { - configPath.trimSuffix("/dagger.json") - } + raise "no Dagger module found containing path: " + path } } else { - path.trimPrefix("/") + normalizePath(path) } Mod( - path: modPath, + rootPath: modPath, ws: ws, skipGenerateFilename: skipGenerateFilename, ) } + let validateConfig(ws: Workspace!, base: String!, filename: String!, pattern: String!, requestedPath: String!): String! { + let configPath = if (base == ".") { filename } else { base + "/" + filename } + let matches = ws + .directory("/", include: [configPath]) + .file(configPath) + .search(pattern: pattern, multiline: true, dotall: true, limit: 1) + .{{id}} + .length > 0 + if (matches) { base } else { raise "Dagger module does not use the Python SDK: " + requestedPath } + } + """ Initialize Python-owned files for a new Dagger module. @@ -191,22 +213,17 @@ type PythonSdk { } """ - Generate all discovered legacy dagger.json Python SDK modules. - - This discovery path is obsolete for workspace-managed modules; the engine - owns the modules..as-sdk.modules source of truth. + Generate every managed Python SDK module visible from the client's cwd. Modules with the generate skip marker are skipped. """ pub generateAll(ws: Workspace!): Changeset! @generate { let pws = polyfill.workspace(ws) - currentModule.asSDK.modules - .{{path}} - .map { m => Mod(path: m.path, ws: ws, skipGenerateFilename: skipGenerateFilename) } + modules(ws) .filter { mod => mod.skipGenerate == false } .reduce(pws.fork) { fork, mod => - fork.merge(pws.moduleSource("/" + mod.path).generate) + fork.merge(pws.moduleSource("/" + mod.rootPath).generate) } .changes }