diff --git a/.github/workflows/dev-containers.yml b/.github/workflows/dev-containers.yml index a3dc3b96a..3397f2d28 100644 --- a/.github/workflows/dev-containers.yml +++ b/.github/workflows/dev-containers.yml @@ -19,6 +19,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + with: + submodules: recursive - uses: actions/setup-node@v5 with: node-version: '20.x' @@ -69,6 +71,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + submodules: recursive - name: Setup Node.js uses: actions/setup-node@v5 with: @@ -117,6 +121,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + submodules: recursive - name: Setup Node.js uses: actions/setup-node@v5 with: @@ -151,6 +157,8 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v6 + with: + submodules: recursive - name: Run install.sh tests run: sh scripts/install.test.sh diff --git a/.github/workflows/standalone-prototype.yml b/.github/workflows/standalone-prototype.yml index 1b226ab01..876c5deef 100644 --- a/.github/workflows/standalone-prototype.yml +++ b/.github/workflows/standalone-prototype.yml @@ -14,6 +14,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v5 + with: + submodules: recursive - name: Setup Node.js uses: actions/setup-node@v5 diff --git a/.github/workflows/standalone-release.yml b/.github/workflows/standalone-release.yml index 997c24dcc..582f5a79e 100644 --- a/.github/workflows/standalone-release.yml +++ b/.github/workflows/standalone-release.yml @@ -17,6 +17,8 @@ jobs: NODE_VERSION: '20.19.1' steps: - uses: actions/checkout@v4 + with: + submodules: recursive - name: Setup Node uses: actions/setup-node@v4 diff --git a/.github/workflows/test-docker-v20.yml b/.github/workflows/test-docker-v20.yml index 597d6d315..198315ab7 100644 --- a/.github/workflows/test-docker-v20.yml +++ b/.github/workflows/test-docker-v20.yml @@ -17,6 +17,8 @@ jobs: steps: - uses: actions/checkout@v6 + with: + submodules: recursive - uses: actions/setup-node@v5 with: diff --git a/.github/workflows/test-docker-v29.yml b/.github/workflows/test-docker-v29.yml index 5d4a0cc70..818569630 100644 --- a/.github/workflows/test-docker-v29.yml +++ b/.github/workflows/test-docker-v29.yml @@ -17,6 +17,8 @@ jobs: steps: - uses: actions/checkout@v6 + with: + submodules: recursive - uses: actions/setup-node@v5 with: diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 3d2e988d2..13fa24c87 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -52,6 +52,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + submodules: recursive - name: Setup Node.js uses: actions/setup-node@v5 with: diff --git a/.gitmodules b/.gitmodules index c98ac6546..e861acad7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "upstream"] path = upstream url = https://github.com/devcontainers/cli + branch = main diff --git a/AGENTS.md b/AGENTS.md index afcf40881..34874502e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,3 +21,8 @@ When asked to update upstream: ## Pathing expectations - Tests, scripts, and docs that need upstream assets should reference paths under `upstream/...` explicitly. - Avoid hardcoded assumptions that upstream files exist at repository root. + +## Submodule bump checklist +- Use `git submodule update --init --recursive` before running migration/parity checks. +- Record the new pinned revision with `git rev-parse HEAD:upstream` in PR notes/tests when changing compatibility behavior. +- Keep submodule updates reviewable by separating the submodule pointer bump from project-owned compatibility fixes when practical. diff --git a/README.md b/README.md index 599cab02d..c560c1afe 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,12 @@ If you clone this repository without submodules, initialize them before building git submodule update --init --recursive ``` +To inspect the exact compatibility target currently pinned by this repo: + +```bash +git rev-parse HEAD:upstream +``` + ## Context A development container allows you to use a container as a full-featured development environment. It can be used to run an application, to separate tools, libraries, or runtimes needed for working with a codebase, and to aid in continuous integration and testing. Dev containers can be run locally or remotely, in a private or public cloud. diff --git a/TODO.md b/TODO.md index 143e18a4f..b358fa470 100644 --- a/TODO.md +++ b/TODO.md @@ -182,10 +182,14 @@ This balances near-term user value with long-term maintainability. Move all vendored upstream TypeScript CLI sources out of repo root and treat `upstream/` (git submodule) as the canonical upstream baseline we target for compatibility. ### 1) Repository layout and ownership -- [ ] Confirm `upstream/` is the only place where upstream devcontainers/cli code lives. -- [ ] Remove duplicated upstream-owned files currently checked in at repository root once replacements are wired. -- [ ] Keep only project-owned integration/porting assets at repository root (Rust code, migration docs, compatibility harness, and project-specific tests). -- [ ] Add/refresh `.gitmodules` and contributor guidance so updating upstream is intentional and reviewable. +- [x] Confirm `upstream/` is the only place where upstream devcontainers/cli code lives. + - Added `collectDuplicateUpstreamPaths(...)` + `evaluateUpstreamSubmoduleCutoverReadiness(...)` with tests so duplicate upstream-owned paths outside `upstream/` are detected from filesystem layout. +- [x] Remove duplicated upstream-owned files currently checked in at repository root once replacements are wired. + - Removed root-level duplicated TypeScript sources/tests that are now sourced exclusively from `upstream/` for upstream-owned logic. +- [x] Keep only project-owned integration/porting assets at repository root (Rust code, migration docs, compatibility harness, and project-specific tests). + - Root `src/` now contains only migration/readiness contract helpers and project-owned tests. +- [x] Add/refresh `.gitmodules` and contributor guidance so updating upstream is intentional and reviewable. + - `.gitmodules` now pins the `upstream` submodule branch and README/AGENTS document explicit submodule update workflow. ### 2) Build/test path migration - [ ] Audit all test fixtures, scripts, and build commands that currently reference root-level upstream paths. @@ -194,7 +198,8 @@ Move all vendored upstream TypeScript CLI sources out of repo root and treat `up - [ ] Ensure CI jobs execute against `upstream/` sources and fail fast when submodule is missing/uninitialized. ### 3) Compatibility target versioning -- [ ] Define the compatibility contract as: “this repo targets the exact commit pinned in `upstream/`.” +- [x] Define the compatibility contract as: “this repo targets the exact commit pinned in `upstream/`.” + - Added `resolvePinnedUpstreamCommit(...)` and `formatUpstreamCompatibilityContract(...)` helpers (with tests) to make the pinned-commit contract explicit and machine-resolvable. - [ ] Expose the pinned upstream commit in test output/logging for traceability. - [ ] Add a dedicated CI check that reports diffs/regressions when submodule commit changes. - [ ] Create an “update upstream” workflow (bump submodule -> run parity suite -> fix breakages -> merge). diff --git a/esbuild.js b/esbuild.js index 2e386ea74..447fdb7a8 100644 --- a/esbuild.js +++ b/esbuild.js @@ -77,10 +77,10 @@ const watch = process.argv.indexOf('--watch') !== -1; `.trimStart() }, entryPoints: [ - './src/spec-node/devContainersSpecCLI.ts', + './upstream/src/spec-node/devContainersSpecCLI.ts', ], tsconfig: 'tsconfig.json', - outbase: 'src', + outbase: 'upstream/src', }; if (watch) { diff --git a/src/spec-common/async.ts b/src/spec-common/async.ts deleted file mode 100644 index baff3891a..000000000 --- a/src/spec-common/async.ts +++ /dev/null @@ -1,8 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -export async function delay(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); -} diff --git a/src/spec-common/cliHost.ts b/src/spec-common/cliHost.ts deleted file mode 100644 index 294f8be4a..000000000 --- a/src/spec-common/cliHost.ts +++ /dev/null @@ -1,279 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as path from 'path'; -import * as net from 'net'; -import * as os from 'os'; - -import { readLocalFile, writeLocalFile, mkdirpLocal, isLocalFile, renameLocal, readLocalDir, isLocalFolder } from '../spec-utils/pfs'; -import { URI } from 'vscode-uri'; -import { ExecFunction, getLocalUsername, plainExec, plainPtyExec, PtyExecFunction } from './commonUtils'; -import { Abort, Duplex, Sink, Source, SourceCallback } from 'pull-stream'; - -const toPull = require('stream-to-pull-stream'); - - -export type CLIHostType = 'local' | 'wsl' | 'container' | 'ssh'; - -export interface CLIHost { - type: CLIHostType; - platform: NodeJS.Platform; - arch: NodeJS.Architecture; - exec: ExecFunction; - ptyExec: PtyExecFunction; - cwd: string; - env: NodeJS.ProcessEnv; - path: typeof path.posix | typeof path.win32; - homedir(): Promise; - tmpdir(): Promise; - isFile(filepath: string): Promise; - isFolder(filepath: string): Promise; - readFile(filepath: string): Promise; - writeFile(filepath: string, content: Buffer): Promise; - rename(oldPath: string, newPath: string): Promise; - mkdirp(dirpath: string): Promise; - readDir(dirpath: string): Promise; - readDirWithTypes?(dirpath: string): Promise<[string, FileTypeBitmask][]>; - getUsername(): Promise; - getuid?: () => Promise; - getgid?: () => Promise; - toCommonURI(filePath: string): Promise; - connect: ConnectFunction; - reconnect?(): Promise; - terminate?(): Promise; -} - -export type ConnectFunction = (socketPath: string) => Duplex; - -export enum FileTypeBitmask { - Unknown = 0, - File = 1, - Directory = 2, - SymbolicLink = 64 -} - -export async function getCLIHost(localCwd: string, loadNativeModule: (moduleName: string) => Promise, allowInheritTTY: boolean): Promise { - const exec = plainExec(localCwd); - const ptyExec = await plainPtyExec(localCwd, loadNativeModule, allowInheritTTY); - return createLocalCLIHostFromExecFunctions(localCwd, exec, ptyExec, connectLocal); -} - -function createLocalCLIHostFromExecFunctions(localCwd: string, exec: ExecFunction, ptyExec: PtyExecFunction, connect: ConnectFunction): CLIHost { - return { - type: 'local', - platform: process.platform, - arch: process.arch, - exec, - ptyExec, - cwd: localCwd, - env: process.env, - path: path, - homedir: async () => os.homedir(), - tmpdir: async () => os.tmpdir(), - isFile: isLocalFile, - isFolder: isLocalFolder, - readFile: readLocalFile, - writeFile: writeLocalFile, - rename: renameLocal, - mkdirp: async (dirpath) => { - await mkdirpLocal(dirpath); - }, - readDir: readLocalDir, - getUsername: getLocalUsername, - getuid: process.platform === 'linux' || process.platform === 'darwin' ? async () => process.getuid!() : undefined, - getgid: process.platform === 'linux' || process.platform === 'darwin' ? async () => process.getgid!() : undefined, - toCommonURI: async (filePath) => URI.file(filePath), - connect, - }; -} - -// Parse a Cygwin socket cookie string to a raw Buffer -function cygwinUnixSocketCookieToBuffer(cookie: string) { - let bytes: number[] = []; - - cookie.split('-').map((number: string) => { - const bytesInChar = number.match(/.{2}/g); - if (bytesInChar !== null) { - bytesInChar.reverse().map((byte) => { - bytes.push(parseInt(byte, 16)); - }); - } - }); - return Buffer.from(bytes); -} - -// The cygwin/git bash ssh-agent server will reply us with the cookie back (16 bytes) -// + identifiers (12 bytes), skip them while forwarding data from ssh-agent to the client -function skipHeader(headerSize: number, err: Abort, data?: Buffer) { - if (err || data === undefined) { - return { headerSize, err }; - } - - if (headerSize === 0) { - // Fast path avoiding data buffer manipulation - // We don't need to modify the received data (handshake header - // already removed) - return { headerSize, data }; - } else if (data.length > headerSize) { - // We need to remove part of the data to forward - data = data.slice(headerSize, data.length); - headerSize = 0; - return { headerSize, data }; - } else { - // We need to remove all forwarded data - headerSize = headerSize - data.length; - return { headerSize }; - } -} - -// Function to handle the Cygwin/Gpg4win socket filtering -// These sockets need an handshake before forwarding client and server data -function handleUnixSocketOnWindows(socket: net.Socket, socketPath: string): Duplex { - let headerSize = 0; - let pendingSourceCallbacks: { abort: Abort; cb: SourceCallback }[] = []; - let pendingSinkCalls: Source[] = []; - let connectionDuplex: Duplex | undefined = undefined; - - let handleError = (err: Abort) => { - if (err instanceof Error) { - console.error(err); - } - socket.destroy(); - - // Notify pending callbacks with the error - for (let callback of pendingSourceCallbacks) { - callback.cb(err, undefined); - } - pendingSourceCallbacks = []; - - for (let callback of pendingSinkCalls) { - callback(err, (_abort, _data) => { }); - } - pendingSinkCalls = []; - }; - - function doSource(abort: Abort, cb: SourceCallback) { - (connectionDuplex as Duplex).source(abort, function (err, data) { - const res = skipHeader(headerSize, err, data); - headerSize = res.headerSize; - if (res.err || res.data) { - cb(res.err || null, res.data); - } else { - doSource(abort, cb); - } - }); - } - - (async () => { - const buf = await readLocalFile(socketPath); - const str = buf.toString(); - - // Try to parse cygwin socket data - const cygwinSocketParameters = str.match(/!(\d+)( s)? ((([A-Fa-f0-9]{2}){4}-?){4})/); - - let port: number; - let handshake: Buffer; - - if (cygwinSocketParameters !== null) { - // Cygwin / MSYS / Git Bash unix socket on Windows - const portStr = cygwinSocketParameters[1]; - const guidStr = cygwinSocketParameters[3]; - port = parseInt(portStr, 10); - const guid = cygwinUnixSocketCookieToBuffer(guidStr); - - let identifierData = Buffer.alloc(12); - identifierData.writeUInt32LE(process.pid, 0); - - handshake = Buffer.concat([guid, identifierData]); - - // Recv header size = GUID (16 bytes) + identifiers (3 * 4 bytes) - headerSize = 16 + 3 * 4; - } else { - // Gpg4Win unix socket - const i = buf.indexOf(0xa); - port = parseInt(buf.slice(0, i).toString(), 10); - handshake = buf.slice(i + 1); - - // No header will be received from Gpg4Win agent - headerSize = 0; - } - - // Handle connection errors and resets - socket.on('error', err => { - handleError(err); - }); - - socket.connect(port, '127.0.0.1', () => { - // Write handshake data to the ssh-agent/gpg-agent server - socket.write(handshake, err => { - if (err) { - // Error will be handled via the 'error' event - return; - } - - connectionDuplex = toPull.duplex(socket); - - // Call pending source calls, if the pull-stream connection was - // pull-ed before we got connected to the ssh-agent/gpg-agent - // server. - // The received data from ssh-agent/gpg-agent server is filtered - // to skip the handshake header. - for (let callback of pendingSourceCallbacks) { - doSource(callback.abort, callback.cb); - } - pendingSourceCallbacks = []; - - // Call pending sink calls after the handshake is completed - // to send what the client sent to us - for (let callback of pendingSinkCalls) { - (connectionDuplex as Duplex).sink(callback); - } - pendingSinkCalls = []; - }); - }); - })() - .catch(err => { - handleError(err); - }); - - // pull-stream source that remove the first bytes - let source: Source = function (abort: Abort, cb: SourceCallback) { - if (connectionDuplex !== undefined) { - doSource(abort, cb); - } else { - pendingSourceCallbacks.push({ abort: abort, cb: cb }); - } - }; - - // pull-stream sink. No filtering done, but we need to store calls in case - // the connection to the upstram ssh-agent/gpg-agent is not yet connected - let sink: Sink = function (source: Source) { - if (connectionDuplex !== undefined) { - connectionDuplex.sink(source); - } else { - pendingSinkCalls.push(source); - } - }; - - return { - source: source, - sink: sink - }; -} - -// Connect to a ssh-agent or gpg-agent, supporting multiple platforms -function connectLocal(socketPath: string) { - if (process.platform !== 'win32' || socketPath.startsWith('\\\\.\\pipe\\')) { - // Simple case: direct forwarding - return toPull.duplex(net.connect(socketPath)); - } - - // More complex case: we need to do an handshake to support Cygwin / Git Bash - // sockets or Gpg4Win sockets - - const socket = new net.Socket(); - - return handleUnixSocketOnWindows(socket, socketPath); -} diff --git a/src/spec-common/commonUtils.ts b/src/spec-common/commonUtils.ts deleted file mode 100644 index c74b3c67f..000000000 --- a/src/spec-common/commonUtils.ts +++ /dev/null @@ -1,598 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Writable, Readable } from 'stream'; -import * as path from 'path'; -import * as os from 'os'; -import * as fs from 'fs'; -import * as cp from 'child_process'; -import * as ptyType from 'node-pty'; -import { StringDecoder } from 'string_decoder'; - -import { toErrorText } from './errors'; -import { Disposable, Event, NodeEventEmitter } from '../spec-utils/event'; -import { isLocalFile } from '../spec-utils/pfs'; -import { escapeRegExCharacters } from '../spec-utils/strings'; -import { Log, nullLog } from '../spec-utils/log'; -import { ShellServer } from './shellServer'; - -export { CLIHost, getCLIHost } from './cliHost'; - -export interface Exec { - stdin: Writable; - stdout: Readable; - stderr: Readable; - exit: Promise<{ code: number | null; signal: string | null }>; - terminate(): Promise; -} - -export interface ExecParameters { - env?: NodeJS.ProcessEnv; - cwd?: string; - cmd: string; - args?: string[]; - stdio?: [cp.StdioNull | cp.StdioPipe, cp.StdioNull | cp.StdioPipe, cp.StdioNull | cp.StdioPipe]; - output: Log; -} - -export interface ExecFunction { - (params: ExecParameters): Promise; -} - -export type GoOS = { [OS in NodeJS.Platform]: OS extends 'win32' ? 'windows' : OS; }[NodeJS.Platform]; -export type GoARCH = { [ARCH in NodeJS.Architecture]: ARCH extends 'x64' ? 'amd64' : ARCH; }[NodeJS.Architecture]; - -export interface PlatformInfo { - os: GoOS; - arch: GoARCH; - variant?: string; -} - -export interface PtyExec { - onData: Event; - write?(data: string): void; - resize(cols: number, rows: number): void; - exit: Promise<{ code: number | undefined; signal: number | undefined }>; - terminate(): Promise; -} - -export interface PtyExecParameters { - env?: NodeJS.ProcessEnv; - cwd?: string; - cmd: string; - args?: string[]; - cols?: number; - rows?: number; - output: Log; -} - -export interface PtyExecFunction { - (params: PtyExecParameters): Promise; -} - -export function equalPaths(platform: NodeJS.Platform, a: string, b: string) { - if (platform === 'linux') { - return a === b; - } - return a.toLowerCase() === b.toLowerCase(); -} - -export async function runCommandNoPty(options: { - exec: ExecFunction; - cmd: string; - args?: string[]; - cwd?: string; - env?: NodeJS.ProcessEnv; - stdin?: Buffer | fs.ReadStream | Event; - output: Log; - print?: boolean | 'continuous' | 'onerror'; -}) { - const { exec, cmd, args, cwd, env, stdin, output, print } = options; - - const p = await exec({ - cmd, - args, - cwd, - env, - output, - }); - - return new Promise<{ stdout: Buffer; stderr: Buffer }>((resolve, reject) => { - const stdout: Buffer[] = []; - const stderr: Buffer[] = []; - - const stdoutDecoder = print === 'continuous' ? new StringDecoder() : undefined; - p.stdout.on('data', (chunk: Buffer) => { - stdout.push(chunk); - if (print === 'continuous') { - output.write(stdoutDecoder!.write(chunk)); - } - }); - p.stdout.on('error', (err: any) => { - // ENOTCONN seen with missing executable in addition to ENOENT on child_process. - if (err?.code !== 'ENOTCONN') { - throw err; - } - }); - const stderrDecoder = print === 'continuous' ? new StringDecoder() : undefined; - p.stderr.on('data', (chunk: Buffer) => { - stderr.push(chunk); - if (print === 'continuous') { - output.write(toErrorText(stderrDecoder!.write(chunk))); - } - }); - p.stderr.on('error', (err: any) => { - // ENOTCONN seen with missing executable in addition to ENOENT on child_process. - if (err?.code !== 'ENOTCONN') { - throw err; - } - }); - const subs: Disposable[] = []; - p.exit.then(({ code, signal }) => { - try { - const failed = !!code || !!signal; - subs.forEach(sub => sub.dispose()); - const stdoutBuf = Buffer.concat(stdout); - const stderrBuf = Buffer.concat(stderr); - if (print === true || (failed && print === 'onerror')) { - output.write(stdoutBuf.toString().replace(/\r?\n/g, '\r\n')); - output.write(toErrorText(stderrBuf.toString())); - } - if (print && code) { - output.write(`Exit code ${code}`); - } - if (print && signal) { - output.write(`Process signal ${signal}`); - } - if (failed) { - reject({ - message: `Command failed: ${cmd} ${(args || []).join(' ')}`, - stdout: stdoutBuf, - stderr: stderrBuf, - code, - signal, - }); - } else { - resolve({ - stdout: stdoutBuf, - stderr: stderrBuf, - }); - } - } catch (e) { - reject(e); - } - }, reject); - if (stdin instanceof Buffer) { - p.stdin.write(stdin, err => { - if (err) { - reject(err); - } - }); - p.stdin.end(); - } else if (stdin instanceof fs.ReadStream) { - stdin.pipe(p.stdin); - } else if (typeof stdin === 'function') { - subs.push(stdin(buf => p.stdin.write(buf))); - } - }); -} - -export async function runCommand(options: { - ptyExec: PtyExecFunction; - cmd: string; - args?: string[]; - cwd?: string; - env?: NodeJS.ProcessEnv; - output: Log; - resolveOn?: RegExp; - onDidInput?: Event; - stdin?: string; - print?: 'off' | 'continuous' | 'end'; -}) { - const { ptyExec, cmd, args, cwd, env, output, resolveOn, onDidInput, stdin } = options; - const print = options.print || 'continuous'; - - const p = await ptyExec({ - cmd, - args, - cwd, - env, - output: output, - }); - - return new Promise<{ cmdOutput: string }>((resolve, reject) => { - let cmdOutput = ''; - - const subs: Disposable[] = []; - if (p.write) { - if (stdin) { - p.write(stdin); - } - if (onDidInput) { - subs.push(onDidInput(data => p.write!(data))); - } - } - - p.onData(chunk => { - cmdOutput += chunk; - if (print === 'continuous') { - output.raw(chunk); - } - if (resolveOn && resolveOn.exec(cmdOutput)) { - resolve({ cmdOutput }); - } - }); - p.exit.then(({ code, signal }) => { - try { - if (print === 'end') { - output.raw(cmdOutput); - } - subs.forEach(sub => sub?.dispose()); - if (code || signal) { - reject({ - message: `Command failed: ${cmd} ${(args || []).join(' ')}`, - cmdOutput, - code, - signal, - }); - } else { - resolve({ cmdOutput }); - } - } catch (e) { - reject(e); - } - }, e => { - subs.forEach(sub => sub?.dispose()); - reject(e); - }); - }); -} - -// From https://man7.org/linux/man-pages/man7/signal.7.html: -export const processSignals: Record = { - SIGHUP: 1, - SIGINT: 2, - SIGQUIT: 3, - SIGILL: 4, - SIGTRAP: 5, - SIGABRT: 6, - SIGIOT: 6, - SIGBUS: 7, - SIGEMT: undefined, - SIGFPE: 8, - SIGKILL: 9, - SIGUSR1: 10, - SIGSEGV: 11, - SIGUSR2: 12, - SIGPIPE: 13, - SIGALRM: 14, - SIGTERM: 15, - SIGSTKFLT: 16, - SIGCHLD: 17, - SIGCLD: undefined, - SIGCONT: 18, - SIGSTOP: 19, - SIGTSTP: 20, - SIGTTIN: 21, - SIGTTOU: 22, - SIGURG: 23, - SIGXCPU: 24, - SIGXFSZ: 25, - SIGVTALRM: 26, - SIGPROF: 27, - SIGWINCH: 28, - SIGIO: 29, - SIGPOLL: 29, - SIGPWR: 30, - SIGINFO: undefined, - SIGLOST: undefined, - SIGSYS: 31, - SIGUNUSED: 31, -}; - -export function plainExec(defaultCwd: string | undefined): ExecFunction { - return async function (params: ExecParameters): Promise { - const { cmd, args, stdio, output } = params; - - const text = `Run: ${cmd} ${(args || []).join(' ').replace(/\n.*/g, '')}`; - const start = output.start(text); - - const cwd = params.cwd || defaultCwd; - const env = params.env ? { ...process.env, ...params.env } : process.env; - const exec = await findLocalWindowsExecutable(cmd, cwd, env, output); - const p = cp.spawn(exec, args, { cwd, env, stdio: stdio as any, windowsHide: true }); - - return { - stdin: p.stdin, - stdout: p.stdout, - stderr: p.stderr, - exit: new Promise((resolve, reject) => { - p.once('error', err => { - output.stop(text, start); - reject(err); - }); - p.once('close', (code, signal) => { - output.stop(text, start); - resolve({ code, signal }); - }); - }), - async terminate() { - p.kill('SIGKILL'); - } - }; - }; -} - -export async function plainPtyExec(defaultCwd: string | undefined, loadNativeModule: (moduleName: string) => Promise, allowInheritTTY: boolean): Promise { - const pty = await loadNativeModule('node-pty'); - if (!pty) { - const plain = plainExec(defaultCwd); - return plainExecAsPtyExec(plain, allowInheritTTY); - } - - return async function (params: PtyExecParameters): Promise { - const { cmd, args, output } = params; - - const text = `Run: ${cmd} ${(args || []).join(' ').replace(/\n.*/g, '')}`; - const start = output.start(text); - - const useConpty = false; // TODO: Investigate using a shell with ConPTY. https://github.com/Microsoft/vscode-remote/issues/1234#issuecomment-485501275 - const cwd = params.cwd || defaultCwd; - const env = params.env ? { ...process.env, ...params.env } : process.env; - const exec = await findLocalWindowsExecutable(cmd, cwd, env, output); - const p = pty.spawn(exec, args || [], { - cwd, - env: env as any, - cols: output.dimensions?.columns, - rows: output.dimensions?.rows, - useConpty, - }); - const subs = [ - output.onDidChangeDimensions && output.onDidChangeDimensions(e => p.resize(e.columns, e.rows)) - ]; - - return { - onData: p.onData.bind(p), - write: p.write.bind(p), - resize: p.resize.bind(p), - exit: new Promise(resolve => { - p.onExit(({ exitCode, signal }) => { - subs.forEach(sub => sub?.dispose()); - output.stop(text, start); - resolve({ code: exitCode, signal }); - if (process.platform === 'win32') { - try { - // In some cases the process hasn't cleanly exited on Windows and the winpty-agent gets left around - // https://github.com/microsoft/node-pty/issues/333 - p.kill(); - } catch { - } - } - }); - }), - async terminate() { - p.kill('SIGKILL'); - } - }; - }; -} - -export function plainExecAsPtyExec(plain: ExecFunction, allowInheritTTY: boolean): PtyExecFunction { - return async function (params: PtyExecParameters): Promise { - const p = await plain({ - ...params, - stdio: allowInheritTTY && params.output !== nullLog ? [ - process.stdin.isTTY ? 'inherit' : 'pipe', - process.stdout.isTTY ? 'inherit' : 'pipe', - process.stderr.isTTY ? 'inherit' : 'pipe', - ] : undefined, - }); - const onDataEmitter = new NodeEventEmitter(); - if (p.stdout) { - const stdoutDecoder = new StringDecoder(); - p.stdout.on('data', data => onDataEmitter.fire(stdoutDecoder.write(data))); - p.stdout.on('close', () => { - const end = stdoutDecoder.end(); - if (end) { - onDataEmitter.fire(end); - } - }); - } - if (p.stderr) { - const stderrDecoder = new StringDecoder(); - p.stderr.on('data', data => onDataEmitter.fire(stderrDecoder.write(data))); - p.stderr.on('close', () => { - const end = stderrDecoder.end(); - if (end) { - onDataEmitter.fire(end); - } - }); - } - return { - onData: onDataEmitter.event, - write: p.stdin ? p.stdin.write.bind(p.stdin) : undefined, - resize: () => {}, - exit: p.exit.then(({ code, signal }) => ({ - code: typeof code === 'number' ? code : undefined, - signal: typeof signal === 'string' ? processSignals[signal] : undefined, - })), - terminate: p.terminate.bind(p), - }; - }; -} - -async function findLocalWindowsExecutable(command: string, cwd = process.cwd(), env: Record, output: Log): Promise { - if (process.platform !== 'win32') { - return command; - } - - // From terminalTaskSystem.ts. - - // If we have an absolute path then we take it. - if (path.isAbsolute(command)) { - return await findLocalWindowsExecutableWithExtension(command) || command; - } - if (/[/\\]/.test(command)) { - // We have a directory and the directory is relative (see above). Make the path absolute - // to the current working directory. - const fullPath = path.join(cwd, command); - return await findLocalWindowsExecutableWithExtension(fullPath) || fullPath; - } - let pathValue: string | undefined = undefined; - let paths: string[] | undefined = undefined; - // The options can override the PATH. So consider that PATH if present. - if (env) { - // Path can be named in many different ways and for the execution it doesn't matter - for (let key of Object.keys(env)) { - if (key.toLowerCase() === 'path') { - const value = env[key]; - if (typeof value === 'string') { - pathValue = value; - paths = value.split(path.delimiter) - .filter(Boolean); - paths.push(path.join(env.ProgramW6432 || 'C:\\Program Files', 'Docker\\Docker\\resources\\bin')); // Fall back when newly installed. - } - break; - } - } - } - // No PATH environment. Bail out. - if (paths === void 0 || paths.length === 0) { - output.write(`findLocalWindowsExecutable: No PATH to look up executable '${command}'.`); - const err = new Error(`No PATH to look up executable '${command}'.`); - (err as any).code = 'ENOENT'; - throw err; - } - // We have a simple file name. We get the path variable from the env - // and try to find the executable on the path. - for (let pathEntry of paths) { - // The path entry is absolute. - let fullPath: string; - if (path.isAbsolute(pathEntry)) { - fullPath = path.join(pathEntry, command); - } else { - fullPath = path.join(cwd, pathEntry, command); - } - const withExtension = await findLocalWindowsExecutableWithExtension(fullPath); - if (withExtension) { - return withExtension; - } - } - // Not found in PATH. Bail out. - output.write(`findLocalWindowsExecutable: Exectuable '${command}' not found on PATH '${pathValue}'.`); - const err = new Error(`Exectuable '${command}' not found on PATH '${pathValue}'.`); - (err as any).code = 'ENOENT'; - throw err; -} - -const pathext = process.env.PATHEXT; -const executableExtensions = pathext ? pathext.toLowerCase().split(';') : ['.com', '.exe', '.bat', '.cmd']; - -async function findLocalWindowsExecutableWithExtension(fullPath: string) { - if (executableExtensions.indexOf(path.extname(fullPath)) !== -1) { - return await isLocalFile(fullPath) ? fullPath : undefined; - } - for (const ext of executableExtensions) { - const withExtension = fullPath + ext; - if (await isLocalFile(withExtension)) { - return withExtension; - } - } - return undefined; -} - -export function parseVersion(str: string) { - const m = /^'?v?(\d+(\.\d+)*)/.exec(str); - if (!m) { - return undefined; - } - return m[1].split('.') - .map(i => parseInt(i, 10)); -} - -export function isEarlierVersion(left: number[], right: number[]) { - for (let i = 0, n = Math.max(left.length, right.length); i < n; i++) { - const l = left[i] || 0; - const r = right[i] || 0; - if (l !== r) { - return l < r; - } - } - return false; // Equal. -} - -export async function loadNativeModule(moduleName: string): Promise { - // Check NODE_PATH for Electron. Do this first to avoid loading a binary-incompatible version from the local node_modules during development. - if (process.env.NODE_PATH) { - for (const nodePath of process.env.NODE_PATH.split(path.delimiter)) { - if (nodePath) { - try { - return require(`${nodePath}/${moduleName}`); - } catch (err) { - // Not available. - } - } - } - } - try { - return require(moduleName); - } catch (err) { - // Not available. - } - return undefined; -} - -export type PlatformSwitch = T | { posix: T; win32: T }; - -export function platformDispatch(platform: NodeJS.Platform, platformSwitch: PlatformSwitch) { - if (platformSwitch && typeof platformSwitch === 'object' && 'win32' in platformSwitch) { - return platform === 'win32' ? platformSwitch.win32 : platformSwitch.posix; - } - return platformSwitch; -} - -export async function isFile(shellServer: ShellServer, location: string) { - return platformDispatch(shellServer.platform, { - posix: async () => { - try { - await shellServer.exec(`test -f '${location}'`); - return true; - } catch (err) { - return false; - } - }, - win32: async () => { - return (await shellServer.exec(`Test-Path '${location}' -PathType Leaf`)) - .stdout.trim() === 'True'; - } - })(); -} - -let localUsername: Promise; -export async function getLocalUsername() { - if (localUsername === undefined) { - localUsername = (async () => { - try { - return os.userInfo().username; - } catch (err) { - if (process.platform !== 'linux') { - throw err; - } - // os.userInfo() fails with VS Code snap install: https://github.com/microsoft/vscode-remote-release/issues/6913 - const result = await runCommandNoPty({ exec: plainExec(undefined), cmd: 'id', args: ['-u', '-n'], output: nullLog }); - return result.stdout.toString().trim(); - } - })(); - } - return localUsername; -} - -export function getEntPasswdShellCommand(userNameOrId: string) { - const escapedForShell = userNameOrId.replace(/['\\]/g, '\\$&'); - const escapedForRexExp = escapeRegExCharacters(userNameOrId) - .replaceAll('\'', '\\\''); - // Leading space makes sure we don't concatenate to arithmetic expansion (https://tldp.org/LDP/abs/html/dblparens.html). - return ` (command -v getent >/dev/null 2>&1 && getent passwd '${escapedForShell}' || grep -E '^${escapedForRexExp}|^[^:]*:[^:]*:${escapedForRexExp}:' /etc/passwd || true)`; -} diff --git a/src/spec-common/dotfiles.ts b/src/spec-common/dotfiles.ts deleted file mode 100644 index d055763db..000000000 --- a/src/spec-common/dotfiles.ts +++ /dev/null @@ -1,130 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as path from 'path'; -import { LogLevel } from '../spec-utils/log'; - -import { ResolverParameters, ContainerProperties, createFileCommand } from './injectHeadless'; - -const installCommands = [ - 'install.sh', - 'install', - 'bootstrap.sh', - 'bootstrap', - 'script/bootstrap', - 'setup.sh', - 'setup', - 'script/setup', -]; - -export async function installDotfiles(params: ResolverParameters, properties: ContainerProperties, dockerEnvP: Promise>, secretsP: Promise>) { - let { repository, installCommand, targetPath } = params.dotfilesConfiguration; - if (!repository) { - return; - } - if (repository.indexOf(':') === -1 && !/^\.{0,2}\//.test(repository)) { - repository = `https://github.com/${repository}.git`; - } - const shellServer = properties.shellServer; - const markerFile = getDotfilesMarkerFile(properties); - const dockerEnvAndSecrets = { ...await dockerEnvP, ...await secretsP }; - const allEnv = Object.keys(dockerEnvAndSecrets) - .filter(key => !(key.startsWith('BASH_FUNC_') && key.endsWith('%%'))) - .reduce((env, key) => `${env}${key}=${quoteValue(dockerEnvAndSecrets[key])} `, ''); - try { - params.output.event({ - type: 'progress', - name: 'Installing Dotfiles', - status: 'running', - }); - if (installCommand) { - await shellServer.exec(`# Clone & install dotfiles via '${installCommand}' -${createFileCommand(markerFile)} || (echo dotfiles marker found && exit 1) || exit 0 -command -v git >/dev/null 2>&1 || (echo git not found && exit 1) || exit 0 -[ -e ${targetPath} ] || ${allEnv}git clone --depth 1 ${repository} ${targetPath} || exit $? -echo Setting current directory to '${targetPath}' -cd ${targetPath} - -if [ -f "./${installCommand}" ] -then - if [ ! -x "./${installCommand}" ] - then - echo Setting './${installCommand}' as executable - chmod +x "./${installCommand}" - fi - echo Executing command './${installCommand}'..\n - ${allEnv}"./${installCommand}" -elif [ -f "${installCommand}" ] -then - if [ ! -x "${installCommand}" ] - then - echo Setting '${installCommand}' as executable - chmod +x "${installCommand}" - fi - echo Executing command '${installCommand}'...\n - ${allEnv}"${installCommand}" -else - echo Could not locate '${installCommand}'...\n - exit 126 -fi -`, { logOutput: 'continuous', logLevel: LogLevel.Info }); - } else { - await shellServer.exec(`# Clone & install dotfiles -${createFileCommand(markerFile)} || (echo dotfiles marker found && exit 1) || exit 0 -command -v git >/dev/null 2>&1 || (echo git not found && exit 1) || exit 0 -[ -e ${targetPath} ] || ${allEnv}git clone --depth 1 ${repository} ${targetPath} || exit $? -echo Setting current directory to ${targetPath} -cd ${targetPath} -for f in ${installCommands.join(' ')} -do - if [ -e $f ] - then - installCommand=$f - break - fi -done -if [ -z "$installCommand" ] -then - dotfiles=$(ls -d ${targetPath}/.* 2>/dev/null | grep -v -E '/(.|..|.git)$') - if [ ! -z "$dotfiles" ] - then - echo Linking dotfiles: $dotfiles - ln -sf $dotfiles ~ 2>/dev/null - else - echo No dotfiles found. - fi -else - if [ ! -x "$installCommand" ] - then - echo Setting '${targetPath}'/"$installCommand" as executable - chmod +x "$installCommand" - fi - - echo Executing command '${targetPath}'/"$installCommand"...\n - ${allEnv}./"$installCommand" -fi -`, { logOutput: 'continuous', logLevel: LogLevel.Info }); - } - params.output.event({ - type: 'progress', - name: 'Installing Dotfiles', - status: 'succeeded', - }); - } catch (err) { - params.output.event({ - type: 'progress', - name: 'Installing Dotfiles', - status: 'failed', - }); - } -} - -function quoteValue(value: string | undefined) { - return `'${(value || '').replace(/'+/g, '\'"$&"\'')}'`; -} - -function getDotfilesMarkerFile(properties: ContainerProperties) { - return path.posix.join(properties.userDataFolder, '.dotfilesMarker'); -} diff --git a/src/spec-common/errors.ts b/src/spec-common/errors.ts deleted file mode 100644 index 33ea26eae..000000000 --- a/src/spec-common/errors.ts +++ /dev/null @@ -1,63 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ContainerProperties, CommonDevContainerConfig, ResolverParameters } from './injectHeadless'; - -export { toErrorText, toWarningText } from '../spec-utils/log'; - -export interface ContainerErrorAction { - readonly id: string; - readonly title: string; - readonly isCloseAffordance?: boolean; - readonly isLastAction: boolean; - applicable: (err: ContainerError, primary: boolean) => boolean | Promise; - execute: (err: ContainerError) => Promise; -} - -interface ContainerErrorData { - reload?: boolean; - start?: boolean; - attach?: boolean; - fileWithError?: string; - disallowedFeatureId?: string; - didStopContainer?: boolean; - learnMoreUrl?: string; -} - -interface ContainerErrorInfo { - description: string; - originalError?: any; - manageContainer?: boolean; - params?: ResolverParameters; - containerId?: string; - dockerParams?: any; // TODO - containerProperties?: ContainerProperties; - actions?: ContainerErrorAction[]; - data?: ContainerErrorData; -} - -export class ContainerError extends Error implements ContainerErrorInfo { - description!: string; - originalError?: any; - manageContainer = false; - params?: ResolverParameters; - containerId?: string; // TODO - dockerParams?: any; // TODO - volumeName?: string; - repositoryPath?: string; - folderPath?: string; - containerProperties?: ContainerProperties; - config?: CommonDevContainerConfig; - actions: ContainerErrorAction[] = []; - data: ContainerErrorData = {}; - - constructor(info: ContainerErrorInfo) { - super(info.originalError && info.originalError.message || info.description); - Object.assign(this, info); - if (this.originalError?.stack) { - this.stack = this.originalError.stack; - } - } -} diff --git a/src/spec-common/git.ts b/src/spec-common/git.ts deleted file mode 100644 index 4bbf50730..000000000 --- a/src/spec-common/git.ts +++ /dev/null @@ -1,33 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { runCommandNoPty, CLIHost } from './commonUtils'; -import { Log } from '../spec-utils/log'; -import { FileHost } from '../spec-utils/pfs'; - -export async function findGitRootFolder(cliHost: FileHost | CLIHost, folderPath: string, output: Log) { - if (!('exec' in cliHost)) { - for (let current = folderPath, previous = ''; current !== previous; previous = current, current = cliHost.path.dirname(current)) { - if (await cliHost.isFile(cliHost.path.join(current, '.git', 'config'))) { - return current; - } - } - return undefined; - } - try { - // Preserves symlinked paths (unlike --show-toplevel). - const { stdout } = await runCommandNoPty({ - exec: cliHost.exec, - cmd: 'git', - args: ['rev-parse', '--show-cdup'], - cwd: folderPath, - output, - }); - const cdup = stdout.toString().trim(); - return cliHost.path.resolve(folderPath, cdup); - } catch { - return undefined; - } -} diff --git a/src/spec-common/injectHeadless.ts b/src/spec-common/injectHeadless.ts deleted file mode 100644 index e7770d8f0..000000000 --- a/src/spec-common/injectHeadless.ts +++ /dev/null @@ -1,963 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as path from 'path'; -import * as fs from 'fs'; -import { StringDecoder } from 'string_decoder'; -import * as crypto from 'crypto'; - -import { ContainerError, toErrorText, toWarningText } from './errors'; -import { launch, ShellServer } from './shellServer'; -import { ExecFunction, CLIHost, PtyExecFunction, isFile, Exec, PtyExec, getEntPasswdShellCommand } from './commonUtils'; -import { Disposable, Event, NodeEventEmitter } from '../spec-utils/event'; -import { PackageConfiguration } from '../spec-utils/product'; -import { URI } from 'vscode-uri'; -import { containerSubstitute } from './variableSubstitution'; -import { delay } from './async'; -import { Log, LogEvent, LogLevel, makeLog, nullLog } from '../spec-utils/log'; -import { buildProcessTrees, findProcesses, Process, processTreeToString } from './proc'; -import { installDotfiles } from './dotfiles'; - -export enum ResolverProgress { - Begin, - CloningRepository, - BuildingImage, - StartingContainer, - InstallingServer, - StartingServer, - End, -} - -export interface ResolverParameters { - prebuild?: boolean; - computeExtensionHostEnv: boolean; - package: PackageConfiguration; - containerDataFolder: string | undefined; - containerSystemDataFolder: string | undefined; - appRoot: string | undefined; - extensionPath: string; - sessionId: string; - sessionStart: Date; - cliHost: CLIHost; - env: NodeJS.ProcessEnv; - cwd: string; - isLocalContainer: boolean; - dotfilesConfiguration: DotfilesConfiguration; - progress: (current: ResolverProgress) => void; - output: Log; - allowSystemConfigChange: boolean; - defaultUserEnvProbe: UserEnvProbe; - lifecycleHook: LifecycleHook; - getLogLevel: () => LogLevel; - onDidChangeLogLevel: Event; - loadNativeModule: (moduleName: string) => Promise; - allowInheritTTY: boolean; - shutdowns: (() => Promise)[]; - backgroundTasks: (Promise | (() => Promise))[]; - persistedFolder: string; // A path where config can be persisted and restored at a later time. Should default to tmpdir() folder if not provided. - remoteEnv: Record; - buildxPlatform: string | undefined; - buildxPush: boolean; - buildxOutput: string | undefined; - buildxCacheTo: string | undefined; - skipFeatureAutoMapping: boolean; - skipPostAttach: boolean; - containerSessionDataFolder?: string; - skipPersistingCustomizationsFromFeatures: boolean; - omitConfigRemotEnvFromMetadata?: boolean; - secretsP?: Promise>; - omitSyntaxDirective?: boolean; -} - -export interface LifecycleHook { - enabled: boolean; - skipNonBlocking: boolean; - output: Log; - onDidInput: Event; - done: () => void; -} - -export type LifecycleHooksInstallMap = { - [lifecycleHook in DevContainerLifecycleHook]: { - command: LifecycleCommand; - origin: string; - }[]; // In installation order. -}; - -export function createNullLifecycleHook(enabled: boolean, skipNonBlocking: boolean, output: Log): LifecycleHook { - function listener(data: Buffer) { - emitter.fire(data.toString()); - } - const emitter = new NodeEventEmitter({ - on: () => { - if (process.stdin.isTTY) { - process.stdin.setRawMode(true); - } - process.stdin.on('data', listener); - }, - off: () => process.stdin.off('data', listener), - }); - return { - enabled, - skipNonBlocking, - output: makeLog({ - ...output, - get dimensions() { - return output.dimensions; - }, - event: e => output.event({ - ...e, - channel: 'postCreate', - }), - }), - onDidInput: emitter.event, - done: () => { }, - }; -} - -export interface PortAttributes { - label: string | undefined; - onAutoForward: string | undefined; - elevateIfNeeded: boolean | undefined; -} - -export type UserEnvProbe = 'none' | 'loginInteractiveShell' | 'interactiveShell' | 'loginShell'; - -export type DevContainerLifecycleHook = 'initializeCommand' | 'onCreateCommand' | 'updateContentCommand' | 'postCreateCommand' | 'postStartCommand' | 'postAttachCommand'; - -const defaultWaitFor: DevContainerLifecycleHook = 'updateContentCommand'; - -export type LifecycleCommand = string | string[] | { [key: string]: string | string[] }; - -export interface CommonDevContainerConfig { - configFilePath?: URI; - remoteEnv?: Record; - forwardPorts?: (number | string)[]; - portsAttributes?: Record; - otherPortsAttributes?: PortAttributes; - features?: Record>; - onCreateCommand?: LifecycleCommand | Record; - updateContentCommand?: LifecycleCommand | Record; - postCreateCommand?: LifecycleCommand | Record; - postStartCommand?: LifecycleCommand | Record; - postAttachCommand?: LifecycleCommand | Record; - waitFor?: DevContainerLifecycleHook; - userEnvProbe?: UserEnvProbe; -} - -export interface CommonContainerMetadata { - onCreateCommand?: string | string[]; - updateContentCommand?: string | string[]; - postCreateCommand?: string | string[]; - postStartCommand?: string | string[]; - postAttachCommand?: string | string[]; - waitFor?: DevContainerLifecycleHook; - remoteEnv?: Record; - userEnvProbe?: UserEnvProbe; -} - -export type CommonMergedDevContainerConfig = MergedConfig; - -type MergedConfig = Omit & UpdatedConfigProperties; - -const replaceProperties = [ - 'onCreateCommand', - 'updateContentCommand', - 'postCreateCommand', - 'postStartCommand', - 'postAttachCommand', -] as const; - -interface UpdatedConfigProperties { - onCreateCommands?: LifecycleCommand[]; - updateContentCommands?: LifecycleCommand[]; - postCreateCommands?: LifecycleCommand[]; - postStartCommands?: LifecycleCommand[]; - postAttachCommands?: LifecycleCommand[]; -} - -export interface OSRelease { - hardware: string; - id: string; - version: string; -} - -export interface ContainerProperties { - createdAt: string | undefined; - startedAt: string | undefined; - osRelease: OSRelease; - user: string; - gid: string | undefined; - env: NodeJS.ProcessEnv; - shell: string; - homeFolder: string; - userDataFolder: string; - remoteWorkspaceFolder?: string; - remoteExec: ExecFunction; - remotePtyExec: PtyExecFunction; - remoteExecAsRoot?: ExecFunction; - shellServer: ShellServer; - launchRootShellServer?: () => Promise; -} - -export interface DotfilesConfiguration { - repository: string | undefined; - installCommand: string | undefined; - targetPath: string; -} - -export async function getContainerProperties(options: { - params: ResolverParameters; - createdAt: string | undefined; - startedAt: string | undefined; - remoteWorkspaceFolder: string | undefined; - containerUser: string | undefined; - containerGroup: string | undefined; - containerEnv: NodeJS.ProcessEnv | undefined; - remoteExec: ExecFunction; - remotePtyExec: PtyExecFunction; - remoteExecAsRoot: ExecFunction | undefined; - rootShellServer: ShellServer | undefined; -}) { - let { params, createdAt, startedAt, remoteWorkspaceFolder, containerUser, containerGroup, containerEnv, remoteExec, remotePtyExec, remoteExecAsRoot, rootShellServer } = options; - let shellServer: ShellServer; - if (rootShellServer && containerUser === 'root') { - shellServer = rootShellServer; - } else { - shellServer = await launch(remoteExec, params.output, params.sessionId); - } - if (!containerEnv) { - const PATH = (await shellServer.exec('echo $PATH')).stdout.trim(); - containerEnv = PATH ? { PATH } : {}; - } - if (!containerUser) { - containerUser = await getUser(shellServer); - } - if (!remoteExecAsRoot && containerUser === 'root') { - remoteExecAsRoot = remoteExec; - } - const osRelease = await getOSRelease(shellServer); - const passwdUser = await getUserFromPasswdDB(shellServer, containerUser); - if (!passwdUser) { - params.output.write(toWarningText(`User ${containerUser} not found with 'getent passwd'.`)); - } - const shell = await getUserShell(containerEnv, passwdUser); - const homeFolder = await getHomeFolder(shellServer, containerEnv, passwdUser); - const userDataFolder = getUserDataFolder(homeFolder, params); - let rootShellServerP: Promise | undefined; - if (rootShellServer) { - rootShellServerP = Promise.resolve(rootShellServer); - } else if (containerUser === 'root') { - rootShellServerP = Promise.resolve(shellServer); - } - const containerProperties: ContainerProperties = { - createdAt, - startedAt, - osRelease, - user: containerUser, - gid: containerGroup || passwdUser?.gid, - env: containerEnv, - shell, - homeFolder, - userDataFolder, - remoteWorkspaceFolder, - remoteExec, - remotePtyExec, - remoteExecAsRoot, - shellServer, - }; - if (rootShellServerP || remoteExecAsRoot) { - containerProperties.launchRootShellServer = () => rootShellServerP || (rootShellServerP = launch(remoteExecAsRoot!, params.output)); - } - return containerProperties; -} - -export async function getUser(shellServer: ShellServer) { - return (await shellServer.exec('id -un')).stdout.trim(); -} - -export async function getHomeFolder(shellServer: ShellServer, containerEnv: NodeJS.ProcessEnv, passwdUser: PasswdUser | undefined) { - if (containerEnv.HOME) { - if (containerEnv.HOME === passwdUser?.home || passwdUser?.uid === '0') { - return containerEnv.HOME; - } - try { - await shellServer.exec(`[ ! -e '${containerEnv.HOME}' ] || [ -w '${containerEnv.HOME}' ]`); - return containerEnv.HOME; - } catch { - // Exists but not writable. - } - } - return passwdUser?.home || '/root'; -} - -async function getUserShell(containerEnv: NodeJS.ProcessEnv, passwdUser: PasswdUser | undefined) { - return containerEnv.SHELL || (passwdUser && passwdUser.shell) || '/bin/sh'; -} - -export async function getUserFromPasswdDB(shellServer: ShellServer, userNameOrId: string) { - const { stdout } = await shellServer.exec(getEntPasswdShellCommand(userNameOrId), { logOutput: false }); - if (!stdout.trim()) { - return undefined; - } - return parseUserInPasswdDB(stdout); -} - -export interface PasswdUser { - name: string; - uid: string; - gid: string; - home: string; - shell: string; -} - -function parseUserInPasswdDB(etcPasswdLine: string): PasswdUser | undefined { - const row = etcPasswdLine - .replace(/\n$/, '') - .split(':'); - return { - name: row[0], - uid: row[2], - gid: row[3], - home: row[5], - shell: row[6] - }; -} - -export function getUserDataFolder(homeFolder: string, params: ResolverParameters) { - return path.posix.resolve(homeFolder, params.containerDataFolder || '.devcontainer'); -} - -export function getSystemVarFolder(params: ResolverParameters): string { - return params.containerSystemDataFolder || '/var/devcontainer'; -} - -export async function setupInContainer(params: ResolverParameters, containerProperties: ContainerProperties, config: CommonDevContainerConfig, mergedConfig: CommonMergedDevContainerConfig, lifecycleCommandOriginMap: LifecycleHooksInstallMap) { - await patchEtcEnvironment(params, containerProperties); - await patchEtcProfile(params, containerProperties); - const computeRemoteEnv = params.computeExtensionHostEnv || params.lifecycleHook.enabled; - const updatedConfig = containerSubstitute(params.cliHost.platform, config.configFilePath, containerProperties.env, config); - const updatedMergedConfig = containerSubstitute(params.cliHost.platform, mergedConfig.configFilePath, containerProperties.env, mergedConfig); - const remoteEnv = computeRemoteEnv ? probeRemoteEnv(params, containerProperties, updatedMergedConfig) : Promise.resolve({}); - const secretsP = params.secretsP || Promise.resolve({}); - if (params.lifecycleHook.enabled) { - await runLifecycleHooks(params, lifecycleCommandOriginMap, containerProperties, updatedMergedConfig, remoteEnv, secretsP, false); - } - return { - remoteEnv: params.computeExtensionHostEnv ? await remoteEnv : {}, - updatedConfig, - updatedMergedConfig, - }; -} - -export function probeRemoteEnv(params: ResolverParameters, containerProperties: ContainerProperties, config: CommonMergedDevContainerConfig) { - return probeUserEnv(params, containerProperties, config) - .then>(shellEnv => ({ - ...shellEnv, - ...params.remoteEnv, - ...config.remoteEnv, - } as Record)); -} - -export async function runLifecycleHooks(params: ResolverParameters, lifecycleHooksInstallMap: LifecycleHooksInstallMap, containerProperties: ContainerProperties, config: CommonMergedDevContainerConfig, remoteEnv: Promise>, secrets: Promise>, stopForPersonalization: boolean): Promise<'skipNonBlocking' | 'prebuild' | 'stopForPersonalization' | 'done'> { - const skipNonBlocking = params.lifecycleHook.skipNonBlocking; - const waitFor = config.waitFor || defaultWaitFor; - if (skipNonBlocking && waitFor === 'initializeCommand') { - return 'skipNonBlocking'; - } - - params.output.write('LifecycleCommandExecutionMap: ' + JSON.stringify(lifecycleHooksInstallMap, undefined, 4), LogLevel.Trace); - - await runPostCreateCommand(params, lifecycleHooksInstallMap, containerProperties, 'onCreateCommand', remoteEnv, secrets, false); - if (skipNonBlocking && waitFor === 'onCreateCommand') { - return 'skipNonBlocking'; - } - - await runPostCreateCommand(params, lifecycleHooksInstallMap, containerProperties, 'updateContentCommand', remoteEnv, secrets, !!params.prebuild); - if (skipNonBlocking && waitFor === 'updateContentCommand') { - return 'skipNonBlocking'; - } - - if (params.prebuild) { - return 'prebuild'; - } - - await runPostCreateCommand(params, lifecycleHooksInstallMap, containerProperties, 'postCreateCommand', remoteEnv, secrets, false); - if (skipNonBlocking && waitFor === 'postCreateCommand') { - return 'skipNonBlocking'; - } - - if (params.dotfilesConfiguration) { - await installDotfiles(params, containerProperties, remoteEnv, secrets); - } - - if (stopForPersonalization) { - return 'stopForPersonalization'; - } - - await runPostStartCommand(params, lifecycleHooksInstallMap, containerProperties, remoteEnv, secrets); - if (skipNonBlocking && waitFor === 'postStartCommand') { - return 'skipNonBlocking'; - } - - if (!params.skipPostAttach) { - await runPostAttachCommand(params, lifecycleHooksInstallMap, containerProperties, remoteEnv, secrets); - } - return 'done'; -} - -export async function getOSRelease(shellServer: ShellServer) { - let hardware = 'unknown'; - let id = 'unknown'; - let version = 'unknown'; - try { - hardware = (await shellServer.exec('uname -m')).stdout.trim(); - const { stdout } = await shellServer.exec('(cat /etc/os-release || cat /usr/lib/os-release) 2>/dev/null'); - id = (stdout.match(/^ID=([^\u001b\r\n]*)/m) || [])[1] || 'notfound'; - version = (stdout.match(/^VERSION_ID=([^\u001b\r\n]*)/m) || [])[1] || 'notfound'; - } catch (err) { - console.error(err); - // Optimistically continue. - } - return { hardware, id, version }; -} - -async function runPostCreateCommand(params: ResolverParameters, lifecycleCommandOriginMap: LifecycleHooksInstallMap, containerProperties: ContainerProperties, postCommandName: 'onCreateCommand' | 'updateContentCommand' | 'postCreateCommand', remoteEnv: Promise>, secrets: Promise>, rerun: boolean) { - const markerFile = path.posix.join(containerProperties.userDataFolder, `.${postCommandName}Marker`); - const doRun = !!containerProperties.createdAt && await updateMarkerFile(containerProperties.shellServer, markerFile, containerProperties.createdAt) || rerun; - await runLifecycleCommands(params, lifecycleCommandOriginMap, containerProperties, postCommandName, remoteEnv, secrets, doRun); -} - -async function runPostStartCommand(params: ResolverParameters, lifecycleCommandOriginMap: LifecycleHooksInstallMap, containerProperties: ContainerProperties, remoteEnv: Promise>, secrets: Promise>) { - const markerFile = path.posix.join(containerProperties.userDataFolder, '.postStartCommandMarker'); - const doRun = !!containerProperties.startedAt && await updateMarkerFile(containerProperties.shellServer, markerFile, containerProperties.startedAt); - await runLifecycleCommands(params, lifecycleCommandOriginMap, containerProperties, 'postStartCommand', remoteEnv, secrets, doRun); -} - -async function updateMarkerFile(shellServer: ShellServer, location: string, content: string) { - try { - await shellServer.exec(`mkdir -p '${path.posix.dirname(location)}' && CONTENT="$(cat '${location}' 2>/dev/null || echo ENOENT)" && [ "\${CONTENT:-${content}}" != '${content}' ] && echo '${content}' > '${location}'`); - return true; - } catch (err) { - return false; - } -} - -async function runPostAttachCommand(params: ResolverParameters, lifecycleCommandOriginMap: LifecycleHooksInstallMap, containerProperties: ContainerProperties, remoteEnv: Promise>, secrets: Promise>) { - await runLifecycleCommands(params, lifecycleCommandOriginMap, containerProperties, 'postAttachCommand', remoteEnv, secrets, true); -} - - -async function runLifecycleCommands(params: ResolverParameters, lifecycleCommandOriginMap: LifecycleHooksInstallMap, containerProperties: ContainerProperties, lifecycleHookName: 'onCreateCommand' | 'updateContentCommand' | 'postCreateCommand' | 'postStartCommand' | 'postAttachCommand', remoteEnv: Promise>, secrets: Promise>, doRun: boolean) { - const commandsForHook = lifecycleCommandOriginMap[lifecycleHookName]; - if (commandsForHook.length === 0) { - return; - } - - for (const { command, origin } of commandsForHook) { - const displayOrigin = origin ? (origin === 'devcontainer.json' ? origin : `Feature '${origin}'`) : '???'; /// '???' should never happen. - await runLifecycleCommand(params, containerProperties, command, displayOrigin, lifecycleHookName, remoteEnv, secrets, doRun); - } -} - -async function runLifecycleCommand({ lifecycleHook }: ResolverParameters, containerProperties: ContainerProperties, userCommand: LifecycleCommand, userCommandOrigin: string, lifecycleHookName: 'onCreateCommand' | 'updateContentCommand' | 'postCreateCommand' | 'postStartCommand' | 'postAttachCommand', remoteEnv: Promise>, secrets: Promise>, doRun: boolean) { - let hasCommand = false; - if (typeof userCommand === 'string') { - hasCommand = userCommand.trim().length > 0; - } else if (Array.isArray(userCommand)) { - hasCommand = userCommand.length > 0; - } else if (typeof userCommand === 'object') { - hasCommand = Object.keys(userCommand).length > 0; - } - if (doRun && userCommand && hasCommand) { - const progressName = `Running ${lifecycleHookName}...`; - const infoOutput = makeLog({ - event(e: LogEvent) { - lifecycleHook.output.event(e); - if (e.type === 'raw' && e.text.includes('::endstep::')) { - lifecycleHook.output.event({ - type: 'progress', - name: progressName, - status: 'running', - stepDetail: '' - }); - } - if (e.type === 'raw' && e.text.includes('::step::')) { - lifecycleHook.output.event({ - type: 'progress', - name: progressName, - status: 'running', - stepDetail: `${e.text.split('::step::')[1].split('\r\n')[0]}` - }); - } - }, - get dimensions() { - return lifecycleHook.output.dimensions; - }, - onDidChangeDimensions: lifecycleHook.output.onDidChangeDimensions, - }, LogLevel.Info); - const remoteCwd = containerProperties.remoteWorkspaceFolder || containerProperties.homeFolder; - async function runSingleCommand(postCommand: string | string[], name?: string) { - const progressDetails = typeof postCommand === 'string' ? postCommand : postCommand.join(' '); - infoOutput.event({ - type: 'progress', - name: progressName, - status: 'running', - stepDetail: progressDetails - }); - // If we have a command name then the command is running in parallel and - // we need to hold output until the command is done so that the output - // doesn't get interleaved with the output of other commands. - const printMode = name ? 'off' : 'continuous'; - const env = { ...(await remoteEnv), ...(await secrets) }; - try { - const { cmdOutput } = await runRemoteCommand({ ...lifecycleHook, output: infoOutput }, containerProperties, typeof postCommand === 'string' ? ['/bin/sh', '-c', postCommand] : postCommand, remoteCwd, { remoteEnv: env, pty: true, print: printMode }); - - // 'name' is set when parallel execution syntax is used. - if (name) { - infoOutput.raw(`\x1b[1mRunning ${name} of ${lifecycleHookName} from ${userCommandOrigin}...\x1b[0m\r\n${cmdOutput}\r\n`); - } - } catch (err) { - if (printMode === 'off' && err?.cmdOutput) { - infoOutput.raw(`\r\n\x1b[1m${err.cmdOutput}\x1b[0m\r\n\r\n`); - } - if (err && (err.code === 130 || err.signal === 2)) { // SIGINT seen on darwin as code === 130, would also make sense as signal === 2. - infoOutput.raw(`\r\n\x1b[1m${name ? `${name} of ${lifecycleHookName}` : lifecycleHookName} from ${userCommandOrigin} interrupted.\x1b[0m\r\n\r\n`); - } else { - if (err?.code) { - infoOutput.write(toErrorText(`${name ? `${name} of ${lifecycleHookName}` : lifecycleHookName} from ${userCommandOrigin} failed with exit code ${err.code}. Skipping any further user-provided commands.`)); - } - throw new ContainerError({ - description: `${name ? `${name} of ${lifecycleHookName}` : lifecycleHookName} from ${userCommandOrigin} failed.`, - originalError: err - }); - } - } - } - - infoOutput.raw(`\x1b[1mRunning the ${lifecycleHookName} from ${userCommandOrigin}...\x1b[0m\r\n\r\n`); - - try { - let commands; - if (typeof userCommand === 'string' || Array.isArray(userCommand)) { - commands = [runSingleCommand(userCommand)]; - } else { - commands = Object.keys(userCommand).map(name => { - const command = userCommand[name]; - return runSingleCommand(command, name); - }); - } - - const results = await Promise.allSettled(commands); // Wait for all commands to finish (successfully or not) before continuing. - const rejection = results.find(p => p.status === 'rejected'); - if (rejection) { - throw (rejection as PromiseRejectedResult).reason; - } - infoOutput.event({ - type: 'progress', - name: progressName, - status: 'succeeded', - }); - } catch (err) { - infoOutput.event({ - type: 'progress', - name: progressName, - status: 'failed', - }); - throw err; - } - } -} - -async function createFile(shellServer: ShellServer, location: string) { - try { - await shellServer.exec(createFileCommand(location)); - return true; - } catch (err) { - return false; - } -} - -export function createFileCommand(location: string) { - return `test ! -f '${location}' && set -o noclobber && mkdir -p '${path.posix.dirname(location)}' && { > '${location}' ; } 2> /dev/null`; -} - -export async function runRemoteCommand(params: { output: Log; onDidInput?: Event; stdin?: NodeJS.ReadStream; stdout?: NodeJS.WriteStream; stderr?: NodeJS.WriteStream }, { remoteExec, remotePtyExec }: ContainerProperties, cmd: string[], cwd?: string, options: { remoteEnv?: NodeJS.ProcessEnv; pty?: boolean; print?: 'off' | 'continuous' | 'end' } = {}) { - const print = options.print || 'end'; - let sub: Disposable | undefined; - let pp: Exec | PtyExec; - let cmdOutput = ''; - if (options.pty) { - const p = pp = await remotePtyExec({ - env: options.remoteEnv, - cwd, - cmd: cmd[0], - args: cmd.slice(1), - output: params.output, - }); - p.onData(chunk => { - cmdOutput += chunk; - if (print === 'continuous') { - if (params.stdout) { - params.stdout.write(chunk); - } else { - params.output.raw(chunk); - } - } - }); - if (p.write && params.onDidInput) { - params.onDidInput(data => p.write!(data)); - } else if (p.write && params.stdin) { - const listener = (data: Buffer): void => p.write!(data.toString()); - const stdin = params.stdin; - if (stdin.isTTY) { - stdin.setRawMode(true); - } - stdin.on('data', listener); - sub = { dispose: () => stdin.off('data', listener) }; - } - } else { - const p = pp = await remoteExec({ - env: options.remoteEnv, - cwd, - cmd: cmd[0], - args: cmd.slice(1), - output: params.output, - }); - const stdout: Buffer[] = []; - if (print === 'continuous' && params.stdout) { - p.stdout.pipe(params.stdout); - } else { - p.stdout.on('data', chunk => { - stdout.push(chunk); - if (print === 'continuous') { - params.output.raw(chunk.toString()); - } - }); - } - const stderr: Buffer[] = []; - if (print === 'continuous' && params.stderr) { - p.stderr.pipe(params.stderr); - } else { - p.stderr.on('data', chunk => { - stderr.push(chunk); - if (print === 'continuous') { - params.output.raw(chunk.toString()); - } - }); - } - if (params.onDidInput) { - params.onDidInput(data => p.stdin.write(data)); - } else if (params.stdin) { - params.stdin.pipe(p.stdin); - } - await pp.exit; - cmdOutput = `${Buffer.concat(stdout)}\n${Buffer.concat(stderr)}`; - } - const exit = await pp.exit; - if (sub) { - sub.dispose(); - } - if (print === 'end') { - params.output.raw(cmdOutput); - } - if (exit.code || exit.signal) { - return Promise.reject({ - message: `Command failed: ${cmd.join(' ')}`, - cmdOutput, - code: exit.code, - signal: exit.signal, - }); - } - return { - cmdOutput, - }; -} - -async function runRemoteCommandNoPty(params: { output: Log }, { remoteExec }: { remoteExec: ExecFunction }, cmd: string[], cwd?: string, options: { remoteEnv?: NodeJS.ProcessEnv; stdin?: Buffer | fs.ReadStream; silent?: boolean; print?: 'off' | 'continuous' | 'end'; resolveOn?: RegExp } = {}) { - const print = options.print || (options.silent ? 'off' : 'end'); - const p = await remoteExec({ - env: options.remoteEnv, - cwd, - cmd: cmd[0], - args: cmd.slice(1), - output: options.silent ? nullLog : params.output, - }); - const stdout: Buffer[] = []; - const stderr: Buffer[] = []; - const stdoutDecoder = new StringDecoder(); - const stderrDecoder = new StringDecoder(); - let stdoutStr = ''; - let stderrStr = ''; - let doResolveEarly: () => void; - let doRejectEarly: (err: any) => void; - const resolveEarly = new Promise((resolve, reject) => { - doResolveEarly = resolve; - doRejectEarly = reject; - }); - p.stdout.on('data', (chunk: Buffer) => { - stdout.push(chunk); - const str = stdoutDecoder.write(chunk); - if (print === 'continuous') { - params.output.write(str.replace(/\r?\n/g, '\r\n')); - } - stdoutStr += str; - if (options.resolveOn && options.resolveOn.exec(stdoutStr)) { - doResolveEarly(); - } - }); - p.stderr.on('data', (chunk: Buffer) => { - stderr.push(chunk); - stderrStr += stderrDecoder.write(chunk); - }); - if (options.stdin instanceof Buffer) { - p.stdin.write(options.stdin, err => { - if (err) { - doRejectEarly(err); - } - }); - p.stdin.end(); - } else if (options.stdin instanceof fs.ReadStream) { - options.stdin.pipe(p.stdin); - } - const exit = await Promise.race([p.exit, resolveEarly]); - const stdoutBuf = Buffer.concat(stdout); - const stderrBuf = Buffer.concat(stderr); - if (print === 'end') { - params.output.write(stdoutStr.replace(/\r?\n/g, '\r\n')); - params.output.write(toErrorText(stderrStr)); - } - const cmdOutput = `${stdoutStr}\n${stderrStr}`; - if (exit && (exit.code || exit.signal)) { - return Promise.reject({ - message: `Command failed: ${cmd.join(' ')}`, - cmdOutput, - stdout: stdoutBuf, - stderr: stderrBuf, - code: exit.code, - signal: exit.signal, - }); - } - return { - cmdOutput, - stdout: stdoutBuf, - stderr: stderrBuf, - }; -} - -async function patchEtcEnvironment(params: ResolverParameters, containerProperties: ContainerProperties) { - const markerFile = path.posix.join(getSystemVarFolder(params), `.patchEtcEnvironmentMarker`); - if (params.allowSystemConfigChange && containerProperties.launchRootShellServer && !(await isFile(containerProperties.shellServer, markerFile))) { - const rootShellServer = await containerProperties.launchRootShellServer(); - if (await createFile(rootShellServer, markerFile)) { - await rootShellServer.exec(`cat >> /etc/environment <<'etcEnvironmentEOF' -${Object.keys(containerProperties.env).map(k => `\n${k}="${containerProperties.env[k]}"`).join('')} -etcEnvironmentEOF -`); - } - } -} - -async function patchEtcProfile(params: ResolverParameters, containerProperties: ContainerProperties) { - const markerFile = path.posix.join(getSystemVarFolder(params), `.patchEtcProfileMarker`); - if (params.allowSystemConfigChange && containerProperties.launchRootShellServer && !(await isFile(containerProperties.shellServer, markerFile))) { - const rootShellServer = await containerProperties.launchRootShellServer(); - if (await createFile(rootShellServer, markerFile)) { - await rootShellServer.exec(`sed -i -E 's/((^|\\s)PATH=)([^\\$]*)$/\\1\${PATH:-\\3}/g' /etc/profile || true`); - } - } -} - -async function probeUserEnv(params: { defaultUserEnvProbe: UserEnvProbe; allowSystemConfigChange: boolean; output: Log; containerSessionDataFolder?: string }, containerProperties: { shell: string; remoteExec: ExecFunction; installFolder?: string; env?: NodeJS.ProcessEnv; shellServer?: ShellServer; launchRootShellServer?: (() => Promise); user?: string }, config?: CommonMergedDevContainerConfig) { - let userEnvProbe = getUserEnvProb(config, params); - if (!userEnvProbe || userEnvProbe === 'none') { - return {}; - } - - let env = await readUserEnvFromCache(userEnvProbe, params, containerProperties.shellServer); - if (env) { - return env; - } - - params.output.write('userEnvProbe: not found in cache'); - env = await runUserEnvProbe(userEnvProbe, params, containerProperties, 'cat /proc/self/environ', '\0'); - if (!env) { - params.output.write('userEnvProbe: falling back to printenv'); - env = await runUserEnvProbe(userEnvProbe, params, containerProperties, 'printenv', '\n'); - } - - if (env) { - await updateUserEnvCache(env, userEnvProbe, params, containerProperties.shellServer); - } - - return env || {}; -} - -async function readUserEnvFromCache(userEnvProbe: UserEnvProbe, params: { output: Log; containerSessionDataFolder?: string }, shellServer?: ShellServer) { - if (!shellServer || !params.containerSessionDataFolder) { - return undefined; - } - - const cacheFile = getUserEnvCacheFilePath(userEnvProbe, params.containerSessionDataFolder); - try { - if (await isFile(shellServer, cacheFile)) { - const { stdout } = await shellServer.exec(`cat '${cacheFile}'`); - return JSON.parse(stdout); - } - } - catch (e) { - params.output.write(`Failed to read/parse user env cache: ${e}`, LogLevel.Error); - } - - return undefined; -} - -async function updateUserEnvCache(env: Record, userEnvProbe: UserEnvProbe, params: { output: Log; containerSessionDataFolder?: string }, shellServer?: ShellServer) { - if (!shellServer || !params.containerSessionDataFolder) { - return; - } - - const cacheFile = getUserEnvCacheFilePath(userEnvProbe, params.containerSessionDataFolder); - try { - await shellServer.exec(`mkdir -p '${path.posix.dirname(cacheFile)}' && cat > '${cacheFile}' << 'envJSON' -${JSON.stringify(env, null, '\t')} -envJSON -`); - } - catch (e) { - params.output.write(`Failed to cache user env: ${e}`, LogLevel.Error); - } -} - -function getUserEnvCacheFilePath(userEnvProbe: UserEnvProbe, cacheFolder: string): string { - return path.posix.join(cacheFolder, `env-${userEnvProbe}.json`); -} - -async function runUserEnvProbe(userEnvProbe: UserEnvProbe, params: { allowSystemConfigChange: boolean; output: Log }, containerProperties: { shell: string; remoteExec: ExecFunction; installFolder?: string; env?: NodeJS.ProcessEnv; shellServer?: ShellServer; launchRootShellServer?: (() => Promise); user?: string }, cmd: string, sep: string) { - if (userEnvProbe === 'none') { - return {}; - } - try { - // From VS Code's shellEnv.ts - - const mark = crypto.randomUUID(); - const regex = new RegExp(mark + '([^]*)' + mark); - const systemShellUnix = containerProperties.shell; - params.output.write(`userEnvProbe shell: ${systemShellUnix}`); - - // handle popular non-POSIX shells - const name = path.posix.basename(systemShellUnix); - const command = `echo -n ${mark}; ${cmd}; echo -n ${mark}`; - let shellArgs: string[]; - if (/^pwsh(-preview)?$/.test(name)) { - shellArgs = userEnvProbe === 'loginInteractiveShell' || userEnvProbe === 'loginShell' ? - ['-Login', '-Command'] : // -Login must be the first option. - ['-Command']; - } else { - shellArgs = [ - userEnvProbe === 'loginInteractiveShell' ? '-lic' : - userEnvProbe === 'loginShell' ? '-lc' : - userEnvProbe === 'interactiveShell' ? '-ic' : - '-c' - ]; - } - - const traceOutput = makeLog(params.output, LogLevel.Trace); - const resultP = runRemoteCommandNoPty({ output: traceOutput }, { remoteExec: containerProperties.remoteExec }, [systemShellUnix, ...shellArgs, command], containerProperties.installFolder); - Promise.race([resultP, delay(2000)]) - .then(async result => { - if (!result) { - let processes: Process[]; - const shellServer = containerProperties.shellServer || await launch(containerProperties.remoteExec, params.output); - try { - ({ processes } = await findProcesses(shellServer)); - } finally { - if (!containerProperties.shellServer) { - await shellServer.process.terminate(); - } - } - const shell = processes.find(p => p.cmd.startsWith(systemShellUnix) && p.cmd.indexOf(mark) !== -1); - if (shell) { - const index = buildProcessTrees(processes); - const tree = index[shell.pid]; - params.output.write(`userEnvProbe is taking longer than 2 seconds. Process tree: -${processTreeToString(tree)}`); - } else { - params.output.write(`userEnvProbe is taking longer than 2 seconds. Process not found.`); - } - } - }, () => undefined) - .catch(err => params.output.write(toErrorText(err && (err.stack || err.message) || 'Error reading process tree.'))); - const result = await Promise.race([resultP, delay(10000)]); - if (!result) { - params.output.write(toErrorText(`userEnvProbe is taking longer than 10 seconds. Avoid waiting for user input in your shell's startup scripts. Continuing.`)); - return {}; - } - const raw = result.stdout.toString(); - const match = regex.exec(raw); - const rawStripped = match ? match[1] : ''; - if (!rawStripped) { - return undefined; // assume error - } - const env = rawStripped.split(sep) - .reduce((env, e) => { - const i = e.indexOf('='); - if (i !== -1) { - env[e.substring(0, i)] = e.substring(i + 1); - } - return env; - }, {} as Record); - params.output.write(`userEnvProbe parsed: ${JSON.stringify(env, undefined, ' ')}`, LogLevel.Trace); - delete env.PWD; - - const shellPath = env.PATH; - const containerPath = containerProperties.env?.PATH; - const doMergePaths = !(params.allowSystemConfigChange && containerProperties.launchRootShellServer) && shellPath && containerPath; - if (doMergePaths) { - const user = containerProperties.user; - env.PATH = mergePaths(shellPath, containerPath!, user === 'root' || user === '0'); - } - params.output.write(`userEnvProbe PATHs: -Probe: ${typeof shellPath === 'string' ? `'${shellPath}'` : 'None'} -Container: ${typeof containerPath === 'string' ? `'${containerPath}'` : 'None'}${doMergePaths ? ` -Merged: ${typeof env.PATH === 'string' ? `'${env.PATH}'` : 'None'}` : ''}`); - - return env; - } catch (err) { - params.output.write(toErrorText(err && (err.stack || err.message) || 'Error reading shell environment.')); - return {}; - } -} - -function getUserEnvProb(config: CommonMergedDevContainerConfig | undefined, params: { defaultUserEnvProbe: UserEnvProbe; allowSystemConfigChange: boolean; output: Log }) { - let userEnvProbe = config?.userEnvProbe; - params.output.write(`userEnvProbe: ${userEnvProbe || params.defaultUserEnvProbe}${userEnvProbe ? '' : ' (default)'}`); - if (!userEnvProbe) { - userEnvProbe = params.defaultUserEnvProbe; - } - return userEnvProbe; -} - -function mergePaths(shellPath: string, containerPath: string, rootUser: boolean) { - const result = shellPath.split(':'); - let insertAt = 0; - for (const entry of containerPath.split(':')) { - const i = result.indexOf(entry); - if (i === -1) { - if (rootUser || !/\/sbin(\/|$)/.test(entry)) { - result.splice(insertAt++, 0, entry); - } - } else { - insertAt = i + 1; - } - } - return result.join(':'); -} - -export async function finishBackgroundTasks(tasks: (Promise | (() => Promise))[]) { - for (const task of tasks) { - await (typeof task === 'function' ? task() : task); - } -} diff --git a/src/spec-common/proc.ts b/src/spec-common/proc.ts deleted file mode 100644 index b6a14d52b..000000000 --- a/src/spec-common/proc.ts +++ /dev/null @@ -1,69 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ShellServer } from './shellServer'; - -export interface Process { - pid: string; - ppid: string | undefined; - pgrp: string | undefined; - cwd: string; - mntNS: string; - cmd: string; - env: Record; -} - -export async function findProcesses(shellServer: ShellServer) { - const ps = 'for pid in `cd /proc && ls -d [0-9]*`; do { echo $pid ; readlink /proc/$pid/cwd ; readlink /proc/$pid/ns/mnt ; cat /proc/$pid/stat | tr "\n" " " ; echo ; xargs -0 < /proc/$pid/environ ; xargs -0 < /proc/$pid/cmdline ; } ; echo --- ; done ; readlink /proc/self/ns/mnt 2>/dev/null'; - const { stdout } = await shellServer.exec(ps, { logOutput: false }); - - const n = 6; - const sections = stdout.split('\n---\n'); - const mntNS = sections.pop()!.trim(); - const processes: Process[] = sections - .map(line => line.split('\n')) - .filter(parts => parts.length >= n) - .map(([pid, cwd, mntNS, stat, env, cmd]) => { - const statM: (string | undefined)[] = /.*\) [^ ]* ([^ ]*) ([^ ]*)/.exec(stat) || []; - return { - pid, - ppid: statM[1], - pgrp: statM[2], - cwd, - mntNS, - cmd, - env: env.split(' ') - .reduce((env, current) => { - const i = current.indexOf('='); - if (i !== -1) { - env[current.substr(0, i)] = current.substr(i + 1); - } - return env; - }, {} as Record), - }; - }); - return { - processes, - mntNS, - }; -} - -export interface ProcessTree { - process: Process; - childProcesses: ProcessTree[]; -} - -export function buildProcessTrees(processes: Process[]) { - const index: Record = {}; - processes.forEach(process => index[process.pid] = { process, childProcesses: [] }); - processes.filter(p => p.ppid) - .forEach(p => index[p.ppid!]?.childProcesses.push(index[p.pid])); - return index; -} - -export function processTreeToString(tree: ProcessTree, singleIndent = ' ', currentIndent = ' '): string { - return `${currentIndent}${tree.process.pid}: ${tree.process.cmd} -${tree.childProcesses.map(p => processTreeToString(p, singleIndent, currentIndent + singleIndent))}`; -} diff --git a/src/spec-common/shellServer.ts b/src/spec-common/shellServer.ts deleted file mode 100644 index 8cc1f5aa0..000000000 --- a/src/spec-common/shellServer.ts +++ /dev/null @@ -1,199 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as path from 'path'; - -import { StringDecoder } from 'string_decoder'; -import { ExecFunction, Exec, PlatformSwitch, platformDispatch } from './commonUtils'; -import { Log, LogLevel } from '../spec-utils/log'; - -export interface ShellServer { - exec(cmd: PlatformSwitch, options?: { logLevel?: LogLevel; logOutput?: boolean | 'continuous' | 'silent'; stdin?: Buffer }): Promise<{ stdout: string; stderr: string }>; - process: Exec; - platform: NodeJS.Platform; - path: typeof path.posix | typeof path.win32; -} - -export const EOT = '\u2404'; - -export async function launch(remoteExec: ExecFunction | Exec, output: Log, agentSessionId?: string, platform: NodeJS.Platform = 'linux', hostName: 'Host' | 'Container' = 'Container'): Promise { - const isExecFunction = typeof remoteExec === 'function'; - const isWindows = platform === 'win32'; - const p = isExecFunction ? await remoteExec({ - env: agentSessionId ? { VSCODE_REMOTE_CONTAINERS_SESSION: agentSessionId } : {}, - cmd: isWindows ? 'powershell' : '/bin/sh', - args: isWindows ? ['-NoProfile', '-Command', '-'] : [], - output, - }) : remoteExec; - if (!isExecFunction) { - // TODO: Pass in agentSessionId. - const stdinText = isWindows - ? `powershell -NoProfile -Command "powershell -NoProfile -Command -"\n` // Nested PowerShell (for some reason) avoids the echo of stdin on stdout. - : `/bin/sh -c 'echo ${EOT}; /bin/sh'\n`; - p.stdin.write(stdinText); - const eot = new Promise(resolve => { - let stdout = ''; - const stdoutDecoder = new StringDecoder(); - p.stdout.on('data', function eotListener(chunk: Buffer) { - stdout += stdoutDecoder.write(chunk); - if (stdout.includes(stdinText)) { - p.stdout.off('data', eotListener); - resolve(); - } - }); - }); - await eot; - } - - const monitor = monitorProcess(p); - - let lastExec: Promise | undefined; - async function exec(cmd: PlatformSwitch, options?: { logLevel?: LogLevel; logOutput?: boolean | 'continuous' | 'silent'; stdin?: Buffer }) { - const currentExec = lastExec = (async () => { - try { - await lastExec; - } catch (err) { - // ignore - } - return _exec(platformDispatch(platform, cmd), options); - })(); - try { - return await Promise.race([currentExec, monitor.unexpectedExit]); - } finally { - monitor.disposeStdioListeners(); - if (lastExec === currentExec) { - lastExec = undefined; - } - } - } - - async function _exec(cmd: string, options?: { logLevel?: LogLevel; logOutput?: boolean | 'continuous' | 'silent'; stdin?: Buffer }) { - const text = `Run in ${hostName.toLowerCase()}: ${cmd.replace(/\n.*/g, '')}`; - let start: number; - if (options?.logOutput !== 'silent') { - start = output.start(text, options?.logLevel); - } - if (p.stdin.destroyed) { - output.write('Stdin closed!'); - const { code, signal } = await p.exit; - return Promise.reject({ message: `Shell server terminated (code: ${code}, signal: ${signal})`, code, signal }); - } - if (platform === 'win32') { - p.stdin.write(`[Console]::Write('${EOT}'); ( ${cmd} ); [Console]::Write("${EOT}$LastExitCode ${EOT}"); [Console]::Error.Write('${EOT}')\n`); - } else { - p.stdin.write(`echo -n ${EOT}; ( ${cmd} ); echo -n ${EOT}$?${EOT}; echo -n ${EOT} >&2\n`); - } - const [stdoutP0, stdoutP] = read(p.stdout, [1, 2], options?.logOutput === 'continuous' ? (str, i, j) => { - if (i === 1 && j === 0) { - output.write(str, options?.logLevel); - } - } : () => undefined); - const stderrP = read(p.stderr, [1], options?.logOutput === 'continuous' ? (str, i, j) => { - if (i === 0 && j === 0) { - output.write(str, options?.logLevel); // TODO - } - } : () => undefined)[0]; - if (options?.stdin) { - await stdoutP0; // Wait so `cmd` has its stdin set up. - p.stdin.write(options?.stdin); - } - const [stdout, codeStr] = await stdoutP; - const [stderr] = await stderrP; - const code = parseInt(codeStr, 10) || 0; - if (options?.logOutput === undefined || options?.logOutput === true) { - output.write(stdout, options?.logLevel); - output.write(stderr, options?.logLevel); // TODO - if (code) { - output.write(`Exit code ${code}`, options?.logLevel); - } - } - if (options?.logOutput === 'continuous' && code) { - output.write(`Exit code ${code}`, options?.logLevel); - } - if (options?.logOutput !== 'silent') { - output.stop(text, start!, options?.logLevel); - } - if (code) { - return Promise.reject({ message: `Command in ${hostName.toLowerCase()} failed: ${cmd}`, code, stdout, stderr }); - } - return { stdout, stderr }; - } - - return { exec, process: p, platform, path: platformDispatch(platform, path) }; -} - -function read(stream: NodeJS.ReadableStream, numberOfResults: number[], log: (str: string, i: number, j: number) => void) { - const promises = numberOfResults.map(() => { - let cbs: { resolve: (value: string[]) => void; reject: () => void }; - const promise = new Promise((resolve, reject) => cbs = { resolve, reject }); - return { promise, ...cbs! }; - }); - const decoder = new StringDecoder('utf8'); - const strings: string[] = []; - - let j = 0; - let results: string[] = []; - function data(chunk: Buffer) { - const str = decoder.write(chunk); - consume(str); - } - function consume(str: string) { - // console.log(`consume ${numberOfResults}: '${str}'`); - const i = str.indexOf(EOT); - if (i !== -1) { - const s = str.substr(0, i); - strings.push(s); - log(s, j, results.length); - // console.log(`result ${numberOfResults}: '${strings.join('')}'`); - results.push(strings.join('')); - strings.length = 0; - if (results.length === numberOfResults[j]) { - promises[j].resolve(results); - j++; - results = []; - if (j === numberOfResults.length) { - stream.off('data', data); - } - } - if (i + 1 < str.length) { - consume(str.substr(i + 1)); - } - } else { - strings.push(str); - log(str, j, results.length); - } - } - stream.on('data', data); - - return promises.map(p => p.promise); -} - -function monitorProcess(p: Exec) { - let processExited: (err: any) => void; - const unexpectedExit = new Promise((_resolve, reject) => processExited = reject); - const stdout: Buffer[] = []; - const stderr: Buffer[] = []; - const stdoutListener = (chunk: Buffer) => stdout.push(chunk); - const stderrListener = (chunk: Buffer) => stderr.push(chunk); - p.stdout.on('data', stdoutListener); - p.stderr.on('data', stderrListener); - p.exit.then(({ code, signal }) => { - processExited(`Shell server terminated (code: ${code}, signal: ${signal}) -${Buffer.concat(stdout).toString()} -${Buffer.concat(stderr).toString()}`); - }, err => { - processExited(`Shell server failed: ${err && (err.stack || err.message)}`); - }); - const disposeStdioListeners = () => { - p.stdout.off('data', stdoutListener); - p.stderr.off('data', stderrListener); - stdout.length = 0; - stderr.length = 0; - }; - return { - unexpectedExit, - disposeStdioListeners, - }; -} diff --git a/src/spec-common/tsconfig.json b/src/spec-common/tsconfig.json deleted file mode 100644 index eff319378..000000000 --- a/src/spec-common/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "references": [ - { - "path": "../spec-utils" - } - ] -} \ No newline at end of file diff --git a/src/spec-common/variableSubstitution.ts b/src/spec-common/variableSubstitution.ts deleted file mode 100644 index d973f0cd6..000000000 --- a/src/spec-common/variableSubstitution.ts +++ /dev/null @@ -1,171 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as path from 'path'; -import * as crypto from 'crypto'; - -import { ContainerError } from './errors'; -import { URI } from 'vscode-uri'; - -export interface SubstitutionContext { - platform: NodeJS.Platform; - configFile?: URI; - localWorkspaceFolder?: string; - containerWorkspaceFolder?: string; - env: NodeJS.ProcessEnv; -} - -export function substitute(context: SubstitutionContext, value: T): T { - let env: NodeJS.ProcessEnv | undefined; - const isWindows = context.platform === 'win32'; - const updatedContext = { - ...context, - get env() { - return env || (env = normalizeEnv(isWindows, context.env)); - } - }; - const replace = replaceWithContext.bind(undefined, isWindows, updatedContext); - if (context.containerWorkspaceFolder) { - updatedContext.containerWorkspaceFolder = resolveString(replace, context.containerWorkspaceFolder); - } - return substitute0(replace, value); -} - -export function beforeContainerSubstitute(idLabels: Record | undefined, value: T): T { - let devcontainerId: string | undefined; - return substitute0(replaceDevContainerId.bind(undefined, () => devcontainerId || (idLabels && (devcontainerId = devcontainerIdForLabels(idLabels)))), value); -} - -export function containerSubstitute(platform: NodeJS.Platform, configFile: URI | undefined, containerEnv: NodeJS.ProcessEnv, value: T): T { - const isWindows = platform === 'win32'; - return substitute0(replaceContainerEnv.bind(undefined, isWindows, configFile, normalizeEnv(isWindows, containerEnv)), value); -} - -type Replace = (match: string, variable: string, args: string[]) => string; - -function substitute0(replace: Replace, value: any): any { - if (typeof value === 'string') { - return resolveString(replace, value); - } else if (Array.isArray(value)) { - return value.map(s => substitute0(replace, s)); - } else if (value && typeof value === 'object' && !URI.isUri(value)) { - const result: any = Object.create(null); - Object.keys(value).forEach(key => { - result[key] = substitute0(replace, value[key]); - }); - return result; - } - return value; -} - -const VARIABLE_REGEXP = /\$\{(.*?)\}/g; - -function normalizeEnv(isWindows: boolean, originalEnv: NodeJS.ProcessEnv): NodeJS.ProcessEnv { - if (isWindows) { - const env = Object.create(null); - Object.keys(originalEnv).forEach(key => { - env[key.toLowerCase()] = originalEnv[key]; - }); - return env; - } - return originalEnv; -} - -function resolveString(replace: Replace, value: string): string { - // loop through all variables occurrences in 'value' - return value.replace(VARIABLE_REGEXP, evaluateSingleVariable.bind(undefined, replace)); -} - -function evaluateSingleVariable(replace: Replace, match: string, variable: string): string { - - // try to separate variable arguments from variable name - let args: string[] = []; - const parts = variable.split(':'); - if (parts.length > 1) { - variable = parts[0]; - args = parts.slice(1); - } - - return replace(match, variable, args); -} - -function replaceWithContext(isWindows: boolean, context: SubstitutionContext, match: string, variable: string, args: string[]) { - switch (variable) { - case 'env': - case 'localEnv': - return lookupValue(isWindows, context.env, args, match, context.configFile); - - case 'localWorkspaceFolder': - return context.localWorkspaceFolder !== undefined ? context.localWorkspaceFolder : match; - - case 'localWorkspaceFolderBasename': - return context.localWorkspaceFolder !== undefined ? (isWindows ? path.win32 : path.posix).basename(context.localWorkspaceFolder) : match; - - case 'containerWorkspaceFolder': - return context.containerWorkspaceFolder !== undefined ? context.containerWorkspaceFolder : match; - - case 'containerWorkspaceFolderBasename': - return context.containerWorkspaceFolder !== undefined ? path.posix.basename(context.containerWorkspaceFolder) : match; - - default: - return match; - } -} - -function replaceContainerEnv(isWindows: boolean, configFile: URI | undefined, containerEnvObj: NodeJS.ProcessEnv, match: string, variable: string, args: string[]) { - switch (variable) { - case 'containerEnv': - return lookupValue(isWindows, containerEnvObj, args, match, configFile); - - default: - return match; - } -} - -function replaceDevContainerId(getDevContainerId: () => string | undefined, match: string, variable: string) { - switch (variable) { - case 'devcontainerId': - return getDevContainerId() || match; - - default: - return match; - } -} - -function lookupValue(isWindows: boolean, envObj: NodeJS.ProcessEnv, args: string[], match: string, configFile: URI | undefined) { - if (args.length > 0) { - let envVariableName = args[0]; - if (isWindows) { - envVariableName = envVariableName.toLowerCase(); - } - const env = envObj[envVariableName]; - if (typeof env === 'string') { - return env; - } - - if (args.length > 1) { - const defaultValue = args[1]; - return defaultValue; - } - - // For `env` we should do the same as a normal shell does - evaluates missing envs to an empty string #46436 - return ''; - } - throw new ContainerError({ - description: `'${match}'${configFile ? ` in ${path.posix.basename(configFile.path)}` : ''} can not be resolved because no environment variable name is given.` - }); -} - -function devcontainerIdForLabels(idLabels: Record): string { - const stringInput = JSON.stringify(idLabels, Object.keys(idLabels).sort()); // sort properties - const bufferInput = Buffer.from(stringInput, 'utf-8'); - const hash = crypto.createHash('sha256') - .update(bufferInput) - .digest(); - const uniqueId = BigInt(`0x${hash.toString('hex')}`) - .toString(32) - .padStart(52, '0'); - return uniqueId; -} \ No newline at end of file diff --git a/src/spec-configuration/configuration.ts b/src/spec-configuration/configuration.ts deleted file mode 100644 index 5995e7e2b..000000000 --- a/src/spec-configuration/configuration.ts +++ /dev/null @@ -1,269 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as path from 'path'; -import { URI } from 'vscode-uri'; -import { FileHost, parentURI, uriToFsPath } from './configurationCommonUtils'; -import { Mount } from './containerFeaturesConfiguration'; -import { RemoteDocuments } from './editableFiles'; - -export type DevContainerConfig = DevContainerFromImageConfig | DevContainerFromDockerfileConfig | DevContainerFromDockerComposeConfig; - -export interface PortAttributes { - label: string | undefined; - onAutoForward: string | undefined; - elevateIfNeeded: boolean | undefined; -} - -export type UserEnvProbe = 'none' | 'loginInteractiveShell' | 'interactiveShell' | 'loginShell'; - -export type DevContainerConfigCommand = 'initializeCommand' | 'onCreateCommand' | 'updateContentCommand' | 'postCreateCommand' | 'postStartCommand' | 'postAttachCommand'; - -export interface HostGPURequirements { - cores?: number; - memory?: string; -} - -export interface HostRequirements { - cpus?: number; - memory?: string; - storage?: string; - gpu?: boolean | 'optional' | HostGPURequirements; -} - -export interface DevContainerFeature { - userFeatureId: string; - options: boolean | string | Record; -} - -export interface DevContainerFromImageConfig { - configFilePath?: URI; - image?: string; // Only optional when setting up an existing container as a dev container. - name?: string; - forwardPorts?: (number | string)[]; - appPort?: number | string | (number | string)[]; - portsAttributes?: Record; - otherPortsAttributes?: PortAttributes; - runArgs?: string[]; - shutdownAction?: 'none' | 'stopContainer'; - overrideCommand?: boolean; - initializeCommand?: string | string[]; - onCreateCommand?: string | string[]; - updateContentCommand?: string | string[]; - postCreateCommand?: string | string[]; - postStartCommand?: string | string[]; - postAttachCommand?: string | string[]; - waitFor?: DevContainerConfigCommand; - /** remote path to folder or workspace */ - workspaceFolder?: string; - workspaceMount?: string; - mounts?: (Mount | string)[]; - containerEnv?: Record; - containerUser?: string; - init?: boolean; - privileged?: boolean; - capAdd?: string[]; - securityOpt?: string[]; - remoteEnv?: Record; - remoteUser?: string; - updateRemoteUserUID?: boolean; - userEnvProbe?: UserEnvProbe; - features?: Record>; - overrideFeatureInstallOrder?: string[]; - hostRequirements?: HostRequirements; - customizations?: Record; -} - -export type DevContainerFromDockerfileConfig = { - configFilePath: URI; - name?: string; - forwardPorts?: (number | string)[]; - appPort?: number | string | (number | string)[]; - portsAttributes?: Record; - otherPortsAttributes?: PortAttributes; - runArgs?: string[]; - shutdownAction?: 'none' | 'stopContainer'; - overrideCommand?: boolean; - initializeCommand?: string | string[]; - onCreateCommand?: string | string[]; - updateContentCommand?: string | string[]; - postCreateCommand?: string | string[]; - postStartCommand?: string | string[]; - postAttachCommand?: string | string[]; - waitFor?: DevContainerConfigCommand; - /** remote path to folder or workspace */ - workspaceFolder?: string; - workspaceMount?: string; - mounts?: (Mount | string)[]; - containerEnv?: Record; - containerUser?: string; - init?: boolean; - privileged?: boolean; - capAdd?: string[]; - securityOpt?: string[]; - remoteEnv?: Record; - remoteUser?: string; - updateRemoteUserUID?: boolean; - userEnvProbe?: UserEnvProbe; - features?: Record>; - overrideFeatureInstallOrder?: string[]; - hostRequirements?: HostRequirements; - customizations?: Record; -} & ( - { - dockerFile: string; - context?: string; - build?: { - target?: string; - args?: Record; - cacheFrom?: string | string[]; - options?: string[]; - }; - } - | - { - build: { - dockerfile: string; - context?: string; - target?: string; - args?: Record; - cacheFrom?: string | string[]; - options?: string[]; - }; - } - ); - -export interface DevContainerFromDockerComposeConfig { - configFilePath: URI; - dockerComposeFile: string | string[]; - service: string; - workspaceFolder: string; - name?: string; - forwardPorts?: (number | string)[]; - portsAttributes?: Record; - otherPortsAttributes?: PortAttributes; - shutdownAction?: 'none' | 'stopCompose'; - overrideCommand?: boolean; - initializeCommand?: string | string[]; - onCreateCommand?: string | string[]; - updateContentCommand?: string | string[]; - postCreateCommand?: string | string[]; - postStartCommand?: string | string[]; - postAttachCommand?: string | string[]; - waitFor?: DevContainerConfigCommand; - runServices?: string[]; - mounts?: (Mount | string)[]; - containerEnv?: Record; - containerUser?: string; - init?: boolean; - privileged?: boolean; - capAdd?: string[]; - securityOpt?: string[]; - remoteEnv?: Record; - remoteUser?: string; - updateRemoteUserUID?: boolean; - userEnvProbe?: UserEnvProbe; - features?: Record>; - overrideFeatureInstallOrder?: string[]; - hostRequirements?: HostRequirements; - customizations?: Record; -} - -interface DevContainerVSCodeConfig { - extensions?: string[]; - settings?: object; - devPort?: number; -} - -export interface VSCodeCustomizations { - vscode?: DevContainerVSCodeConfig; -} - -export function updateFromOldProperties(original: T): T { - // https://github.com/microsoft/dev-container-spec/issues/1 - if (!(original.extensions || original.settings || original.devPort !== undefined)) { - return original; - } - const copy = { ...original }; - const customizations = copy.customizations || (copy.customizations = {}); - const vscode = customizations.vscode || (customizations.vscode = {}); - if (copy.extensions) { - vscode.extensions = (vscode.extensions || []).concat(copy.extensions); - delete copy.extensions; - } - if (copy.settings) { - vscode.settings = { - ...copy.settings, - ...(vscode.settings || {}), - }; - delete copy.settings; - } - if (copy.devPort !== undefined && vscode.devPort === undefined) { - vscode.devPort = copy.devPort; - delete copy.devPort; - } - return copy; -} - -export function getConfigFilePath(cliHost: { platform: NodeJS.Platform }, config: { configFilePath: URI }, relativeConfigFilePath: string) { - return resolveConfigFilePath(cliHost, config.configFilePath, relativeConfigFilePath); -} - -export function resolveConfigFilePath(cliHost: { platform: NodeJS.Platform }, configFilePath: URI, relativeConfigFilePath: string) { - const folder = parentURI(configFilePath); - return configFilePath.with({ - path: path.posix.resolve(folder.path, (cliHost.platform === 'win32' && configFilePath.scheme !== RemoteDocuments.scheme) ? (path.win32.isAbsolute(relativeConfigFilePath) ? '/' : '') + relativeConfigFilePath.replace(/\\/g, '/') : relativeConfigFilePath) - }); -} - -export function isDockerFileConfig(config: DevContainerConfig): config is DevContainerFromDockerfileConfig { - return 'dockerFile' in config || ('build' in config && 'dockerfile' in config.build); -} - -export function getDockerfilePath(cliHost: { platform: NodeJS.Platform }, config: DevContainerFromDockerfileConfig) { - return getConfigFilePath(cliHost, config, getDockerfile(config)); -} - -export function getDockerfile(config: DevContainerFromDockerfileConfig) { - return 'dockerFile' in config ? config.dockerFile : config.build.dockerfile; -} - -export async function getDockerComposeFilePaths(cliHost: FileHost, config: DevContainerFromDockerComposeConfig, envForComposeFile: NodeJS.ProcessEnv, cwdForDefaultFiles: string) { - if (Array.isArray(config.dockerComposeFile)) { - if (config.dockerComposeFile.length) { - return config.dockerComposeFile.map(composeFile => uriToFsPath(getConfigFilePath(cliHost, config, composeFile), cliHost.platform)); - } - } else if (typeof config.dockerComposeFile === 'string') { - return [uriToFsPath(getConfigFilePath(cliHost, config, config.dockerComposeFile), cliHost.platform)]; - } - - const envComposeFile = envForComposeFile?.COMPOSE_FILE; - if (envComposeFile) { - return envComposeFile.split(cliHost.path.delimiter) - .map(composeFile => cliHost.path.resolve(cwdForDefaultFiles, composeFile)); - } - - try { - const envPath = cliHost.path.join(cwdForDefaultFiles, '.env'); - const buffer = await cliHost.readFile(envPath); - const match = /^COMPOSE_FILE=(.+)$/m.exec(buffer.toString()); - const envFileComposeFile = match && match[1].trim(); - if (envFileComposeFile) { - return envFileComposeFile.split(cliHost.path.delimiter) - .map(composeFile => cliHost.path.resolve(cwdForDefaultFiles, composeFile)); - } - } catch (err) { - if (!(err && (err.code === 'ENOENT' || err.code === 'EISDIR'))) { - throw err; - } - } - - const defaultFiles = [cliHost.path.resolve(cwdForDefaultFiles, 'docker-compose.yml')]; - const override = cliHost.path.resolve(cwdForDefaultFiles, 'docker-compose.override.yml'); - if (await cliHost.isFile(override)) { - defaultFiles.push(override); - } - return defaultFiles; -} diff --git a/src/spec-configuration/configurationCommonUtils.ts b/src/spec-configuration/configurationCommonUtils.ts deleted file mode 100644 index 371125f52..000000000 --- a/src/spec-configuration/configurationCommonUtils.ts +++ /dev/null @@ -1,75 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as path from 'path'; - -import { URI } from 'vscode-uri'; - -import { CLIHostDocuments } from './editableFiles'; -import { FileHost } from '../spec-utils/pfs'; - -export { FileHost } from '../spec-utils/pfs'; - -const enum CharCode { - Slash = 47, - Colon = 58, - A = 65, - Z = 90, - a = 97, - z = 122, -} - -export function uriToFsPath(uri: URI, platform: NodeJS.Platform): string { - - let value: string; - if (uri.authority && uri.path.length > 1 && (uri.scheme === 'file' || uri.scheme === CLIHostDocuments.scheme)) { - // unc path: file://shares/c$/far/boo - value = `//${uri.authority}${uri.path}`; - } else if ( - uri.path.charCodeAt(0) === CharCode.Slash - && (uri.path.charCodeAt(1) >= CharCode.A && uri.path.charCodeAt(1) <= CharCode.Z || uri.path.charCodeAt(1) >= CharCode.a && uri.path.charCodeAt(1) <= CharCode.z) - && uri.path.charCodeAt(2) === CharCode.Colon - ) { - // windows drive letter: file:///c:/far/boo - value = uri.path[1].toLowerCase() + uri.path.substr(2); - } else { - // other path - value = uri.path; - } - if (platform === 'win32') { - value = value.replace(/\//g, '\\'); - } - return value; -} - -export function getWellKnownDevContainerPaths(path_: typeof path.posix | typeof path.win32, folderPath: string): string[] { - return [ - path_.join(folderPath, '.devcontainer', 'devcontainer.json'), - path_.join(folderPath, '.devcontainer.json'), - ]; -} - -export function getDefaultDevContainerConfigPath(fileHost: FileHost, configFolderPath: string) { - return URI.file(fileHost.path.join(configFolderPath, '.devcontainer', 'devcontainer.json')) - .with({ scheme: CLIHostDocuments.scheme }); -} - -export async function getDevContainerConfigPathIn(fileHost: FileHost, configFolderPath: string) { - const possiblePaths = getWellKnownDevContainerPaths(fileHost.path, configFolderPath); - - for (let possiblePath of possiblePaths) { - if (await fileHost.isFile(possiblePath)) { - return URI.file(possiblePath) - .with({ scheme: CLIHostDocuments.scheme }); - } - } - - return undefined; -} - -export function parentURI(uri: URI) { - const parent = path.posix.dirname(uri.path); - return uri.with({ path: parent }); -} diff --git a/src/spec-configuration/containerCollectionsOCI.ts b/src/spec-configuration/containerCollectionsOCI.ts deleted file mode 100644 index a2e4ad55f..000000000 --- a/src/spec-configuration/containerCollectionsOCI.ts +++ /dev/null @@ -1,617 +0,0 @@ -import path from 'path'; -import * as semver from 'semver'; -import * as tar from 'tar'; -import * as jsonc from 'jsonc-parser'; -import * as crypto from 'crypto'; - -import { Log, LogLevel } from '../spec-utils/log'; -import { isLocalFile, mkdirpLocal, readLocalFile, writeLocalFile } from '../spec-utils/pfs'; -import { requestEnsureAuthenticated } from './httpOCIRegistry'; -import { GoARCH, GoOS, PlatformInfo } from '../spec-common/commonUtils'; - -export const DEVCONTAINER_MANIFEST_MEDIATYPE = 'application/vnd.devcontainers'; -export const DEVCONTAINER_TAR_LAYER_MEDIATYPE = 'application/vnd.devcontainers.layer.v1+tar'; -export const DEVCONTAINER_COLLECTION_LAYER_MEDIATYPE = 'application/vnd.devcontainers.collection.layer.v1+json'; - - -export interface CommonParams { - env: NodeJS.ProcessEnv; - output: Log; - cachedAuthHeader?: Record; // -} - -// Represents the unique OCI identifier for a Feature or Template. -// eg: ghcr.io/devcontainers/features/go:1.0.0 -// eg: ghcr.io/devcontainers/features/go@sha256:fe73f123927bd9ed1abda190d3009c4d51d0e17499154423c5913cf344af15a3 -// Constructed by 'getRef()' -export interface OCIRef { - registry: string; // 'ghcr.io' - owner: string; // 'devcontainers' - namespace: string; // 'devcontainers/features' - path: string; // 'devcontainers/features/go' - resource: string; // 'ghcr.io/devcontainers/features/go' - id: string; // 'go' - - version: string; // (Either the contents of 'tag' or 'digest') - tag?: string; // '1.0.0' - digest?: string; // 'sha256:fe73f123927bd9ed1abda190d3009c4d51d0e17499154423c5913cf344af15a3' -} - -// Represents the unique OCI identifier for a Collection's Metadata artifact. -// eg: ghcr.io/devcontainers/features:latest -// Constructed by 'getCollectionRef()' -export interface OCICollectionRef { - registry: string; // 'ghcr.io' - path: string; // 'devcontainers/features' - resource: string; // 'ghcr.io/devcontainers/features' - tag: 'latest'; // 'latest' (always) - version: 'latest'; // 'latest' (always) -} - -export interface OCILayer { - mediaType: string; - digest: string; - size: number; - annotations: { - 'org.opencontainers.image.title': string; - }; -} - -export interface OCIManifest { - digest?: string; - schemaVersion: number; - mediaType: string; - config: { - digest: string; - mediaType: string; - size: number; - }; - layers: OCILayer[]; - annotations?: { - 'dev.containers.metadata'?: string; - 'com.github.package.type'?: string; - }; -} - -export interface ManifestContainer { - manifestObj: OCIManifest; - manifestBuffer: Buffer; - contentDigest: string; - canonicalId: string; -} - - -interface OCITagList { - name: string; - tags: string[]; -} - -interface OCIImageIndexEntry { - mediaType: string; - size: number; - digest: string; - platform: { - architecture: string; - variant?: string; - os: string; - }; -} - -// https://github.com/opencontainers/image-spec/blob/main/manifest.md#example-image-manifest -interface OCIImageIndex { - schemaVersion: number; - mediaType: string; - manifests: OCIImageIndexEntry[]; -} - -// Following Spec: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-manifests -// Alternative Spec: https://docs.docker.com/registry/spec/api/#overview -// -// The path: -// 'namespace' in spec terminology for the given repository -// (eg: devcontainers/features/go) -const regexForPath = /^[a-z0-9]+([._-][a-z0-9]+)*(\/[a-z0-9]+([._-][a-z0-9]+)*)*$/; -// The reference: -// MUST be either (a) the digest of the manifest or (b) a tag -// MUST be at most 128 characters in length and MUST match the following regular expression: -const regexForVersionOrDigest = /^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$/; - -// https://go.dev/doc/install/source#environment -// Expected by OCI Spec as seen here: https://github.com/opencontainers/image-spec/blob/main/image-index.md#image-index-property-descriptions -export function mapNodeArchitectureToGOARCH(arch: NodeJS.Architecture): GoARCH { - switch (arch) { - case 'x64': - return 'amd64'; - default: - return arch; - } -} - -// https://go.dev/doc/install/source#environment -// Expected by OCI Spec as seen here: https://github.com/opencontainers/image-spec/blob/main/image-index.md#image-index-property-descriptions -export function mapNodeOSToGOOS(os: NodeJS.Platform): GoOS { - switch (os) { - case 'win32': - return 'windows'; - default: - return os; - } -} - -// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pulling-manifests -// Attempts to parse the given string into an OCIRef -export function getRef(output: Log, input: string): OCIRef | undefined { - // Normalize input by downcasing entire string - input = input.toLowerCase(); - - // Invalid if first character is a dot - if (input.startsWith('.')) { - output.write(`Input '${input}' failed validation. Expected input to not start with '.'`, LogLevel.Error); - return; - } - - const indexOfLastColon = input.lastIndexOf(':'); - const indexOfLastAtCharacter = input.lastIndexOf('@'); - - let resource = ''; - let tag: string | undefined = undefined; - let digest: string | undefined = undefined; - - // -- Resolve version - if (indexOfLastAtCharacter !== -1) { - // The version is specified by digest - // eg: ghcr.io/codspace/features/ruby@sha256:abcdefgh - resource = input.substring(0, indexOfLastAtCharacter); - const digestWithHashingAlgorithm = input.substring(indexOfLastAtCharacter + 1); - const splitOnColon = digestWithHashingAlgorithm.split(':'); - if (splitOnColon.length !== 2) { - output.write(`Failed to parse digest '${digestWithHashingAlgorithm}'. Expected format: 'sha256:abcdefghijk'`, LogLevel.Error); - return; - } - - if (splitOnColon[0] !== 'sha256') { - output.write(`Digest algorithm for input '${input}' failed validation. Expected hashing algorithm to be 'sha256'.`, LogLevel.Error); - return; - } - - if (!regexForVersionOrDigest.test(splitOnColon[1])) { - output.write(`Digest for input '${input}' failed validation. Expected digest to match regex '${regexForVersionOrDigest}'.`, LogLevel.Error); - } - - digest = digestWithHashingAlgorithm; - } else if (indexOfLastColon !== -1 && indexOfLastColon > input.lastIndexOf('/')) { - // The version is specified by tag - // eg: ghcr.io/codspace/features/ruby:1.0.0 - - // 1. The last colon is before the first slash (a port) - // eg: ghcr.io:8081/codspace/features/ruby - // 2. There is no tag at all - // eg: ghcr.io/codspace/features/ruby - resource = input.substring(0, indexOfLastColon); - tag = input.substring(indexOfLastColon + 1); - } else { - // There is no tag or digest, so assume 'latest' - resource = input; - tag = 'latest'; - } - - - if (tag && !regexForVersionOrDigest.test(tag)) { - output.write(`Tag '${tag}' for input '${input}' failed validation. Expected digest to match regex '${regexForVersionOrDigest}'.`, LogLevel.Error); - return; - } - - const splitOnSlash = resource.split('/'); - - if (splitOnSlash[1] === 'devcontainers-contrib') { - output.write(`Redirecting 'devcontainers-contrib' to 'devcontainers-extra'.`); - splitOnSlash[1] = 'devcontainers-extra'; - } - - const id = splitOnSlash[splitOnSlash.length - 1]; // Aka 'featureName' - Eg: 'ruby' - const owner = splitOnSlash[1]; - const registry = splitOnSlash[0]; - const namespace = splitOnSlash.slice(1, -1).join('/'); - - const path = `${namespace}/${id}`; - - if (!regexForPath.exec(path)) { - output.write(`Path '${path}' for input '${input}' failed validation. Expected path to match regex '${regexForPath}'.`, LogLevel.Error); - return; - } - - const version = digest || tag || 'latest'; // The most specific version. - - output.write(`> input: ${input}`, LogLevel.Trace); - output.write(`>`, LogLevel.Trace); - output.write(`> resource: ${resource}`, LogLevel.Trace); - output.write(`> id: ${id}`, LogLevel.Trace); - output.write(`> owner: ${owner}`, LogLevel.Trace); - output.write(`> namespace: ${namespace}`, LogLevel.Trace); // TODO: We assume 'namespace' includes at least one slash (eg: 'devcontainers/features') - output.write(`> registry: ${registry}`, LogLevel.Trace); - output.write(`> path: ${path}`, LogLevel.Trace); - output.write(`>`, LogLevel.Trace); - output.write(`> version: ${version}`, LogLevel.Trace); - output.write(`> tag?: ${tag}`, LogLevel.Trace); - output.write(`> digest?: ${digest}`, LogLevel.Trace); - - return { - id, - owner, - namespace, - registry, - resource, - path, - version, - tag, - digest, - }; -} - -export function getCollectionRef(output: Log, registry: string, namespace: string): OCICollectionRef | undefined { - // Normalize input by downcasing entire string - registry = registry.toLowerCase(); - namespace = namespace.toLowerCase(); - - const path = namespace; - const resource = `${registry}/${path}`; - - output.write(`> Inputs: registry='${registry}' namespace='${namespace}'`, LogLevel.Trace); - output.write(`>`, LogLevel.Trace); - output.write(`> resource: ${resource}`, LogLevel.Trace); - - if (!regexForPath.exec(path)) { - output.write(`Parsed path '${path}' from input failed validation.`, LogLevel.Error); - return undefined; - } - - return { - registry, - path, - resource, - version: 'latest', - tag: 'latest', - }; -} - -// Validate if a manifest exists and is reachable about the declared feature/template. -// Specification: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-manifests -export async function fetchOCIManifestIfExists(params: CommonParams, ref: OCIRef | OCICollectionRef, manifestDigest?: string): Promise { - const { output } = params; - - // Simple mechanism to avoid making a DNS request for - // something that is not a domain name. - if (ref.registry.indexOf('.') < 0 && !ref.registry.startsWith('localhost')) { - return; - } - - // TODO: Always use the manifest digest (the canonical digest) - // instead of the `ref.version` by referencing some lock file (if available). - let reference = ref.version; - if (manifestDigest) { - reference = manifestDigest; - } - const manifestUrl = `https://${ref.registry}/v2/${ref.path}/manifests/${reference}`; - output.write(`manifest url: ${manifestUrl}`, LogLevel.Trace); - const expectedDigest = manifestDigest || ('digest' in ref ? ref.digest : undefined); - const manifestContainer = await getManifest(params, manifestUrl, ref, undefined, expectedDigest); - - if (!manifestContainer || !manifestContainer.manifestObj) { - return; - } - - const { manifestObj } = manifestContainer; - - if (manifestObj.config.mediaType !== DEVCONTAINER_MANIFEST_MEDIATYPE) { - output.write(`(!) Unexpected manifest media type: ${manifestObj.config.mediaType}`, LogLevel.Error); - return undefined; - } - - return manifestContainer; -} - -export async function getManifest(params: CommonParams, url: string, ref: OCIRef | OCICollectionRef, mimeType?: string, expectedDigest?: string): Promise { - const { output } = params; - const res = await getBufferWithMimeType(params, url, ref, mimeType || 'application/vnd.oci.image.manifest.v1+json'); - if (!res) { - return undefined; - } - - const { body, headers } = res; - - // Per the specification: - // https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-manifests - // The registry server SHOULD return the canonical content digest in a header, but it's not required to. - // That is useful to have, so if the server doesn't provide it, recalculate it outselves. - // Headers are always automatically downcased by node. - let contentDigest = headers['docker-content-digest']; - if (!contentDigest || expectedDigest) { - if (!contentDigest) { - output.write('Registry did not send a \'docker-content-digest\' header. Recalculating...', LogLevel.Trace); - } - contentDigest = `sha256:${crypto.createHash('sha256').update(body).digest('hex')}`; - } - - if (expectedDigest && contentDigest !== expectedDigest) { - throw new Error(`Digest did not match for ${ref.resource}.`); - } - - return { - contentDigest, - manifestObj: JSON.parse(body.toString()), - manifestBuffer: body, - canonicalId: `${ref.resource}@${contentDigest}`, - }; -} - -// https://github.com/opencontainers/image-spec/blob/main/manifest.md -export async function getImageIndexEntryForPlatform(params: CommonParams, url: string, ref: OCIRef | OCICollectionRef, platformInfo: PlatformInfo, mimeType?: string): Promise { - const { output } = params; - const response = await getJsonWithMimeType(params, url, ref, mimeType || 'application/vnd.oci.image.index.v1+json'); - if (!response) { - return undefined; - } - - const { body: imageIndex } = response; - if (!imageIndex) { - output.write(`Unwrapped response for image index is undefined.`, LogLevel.Error); - return undefined; - } - - // Find a manifest for the current architecture and OS. - return imageIndex.manifests.find(m => { - if (m.platform?.architecture === platformInfo.arch && m.platform?.os === platformInfo.os) { - if (!platformInfo.variant || m.platform?.variant === platformInfo.variant) { - return m; - } - } - return undefined; - }); -} - -async function getBufferWithMimeType(params: CommonParams, url: string, ref: OCIRef | OCICollectionRef, mimeType: string): Promise<{ body: Buffer; headers: Record } | undefined> { - const { output } = params; - const headers = { - 'user-agent': 'devcontainer', - 'accept': mimeType, - }; - - const httpOptions = { - type: 'GET', - url: url, - headers: headers - }; - - const res = await requestEnsureAuthenticated(params, httpOptions, ref); - if (!res) { - output.write(`Request '${url}' failed`, LogLevel.Error); - return; - } - - // NOTE: A 404 is expected here if the manifest does not exist on the remote. - if (res.statusCode > 299) { - // Get the error out. - const errorMsg = res?.resBody?.toString(); - output.write(`Did not fetch target with expected mimetype '${mimeType}': ${errorMsg}`, LogLevel.Trace); - return; - } - - return { - body: res.resBody, - headers: res.resHeaders, - }; -} - -async function getJsonWithMimeType(params: CommonParams, url: string, ref: OCIRef | OCICollectionRef, mimeType: string): Promise<{ body: T; headers: Record } | undefined> { - const { output } = params; - let body: string = ''; - try { - const headers = { - 'user-agent': 'devcontainer', - 'accept': mimeType, - }; - - const httpOptions = { - type: 'GET', - url: url, - headers: headers - }; - - const res = await requestEnsureAuthenticated(params, httpOptions, ref); - if (!res) { - output.write(`Request '${url}' failed`, LogLevel.Error); - return; - } - - const { resBody, statusCode, resHeaders } = res; - body = resBody.toString(); - - // NOTE: A 404 is expected here if the manifest does not exist on the remote. - if (statusCode > 299) { - output.write(`Did not fetch target with expected mimetype '${mimeType}': ${body}`, LogLevel.Trace); - return; - } - const parsedBody: T = JSON.parse(body); - output.write(`Fetched: ${JSON.stringify(parsedBody, undefined, 4)}`, LogLevel.Trace); - return { - body: parsedBody, - headers: resHeaders, - }; - } catch (e) { - output.write(`Failed to parse JSON with mimeType '${mimeType}': ${body}`, LogLevel.Error); - return; - } -} - -// Gets published tags and sorts them by ascending semantic version. -// Omits any tags (eg: 'latest', or major/minor tags '1','1.0') that are not semantic versions. -export async function getVersionsStrictSorted(params: CommonParams, ref: OCIRef): Promise { - const { output } = params; - - const publishedTags = await getPublishedTags(params, ref); - if (!publishedTags) { - return; - } - - const sortedVersions = publishedTags - .filter(f => semver.valid(f)) // Remove all major,minor,latest tags - .sort((a, b) => semver.compare(a, b)); - - output.write(`Published versions (sorted) for '${ref.id}': ${JSON.stringify(sortedVersions, undefined, 2)}`, LogLevel.Trace); - - return sortedVersions; -} - -// Lists published tags of a Feature/Template -// Specification: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#content-discovery -export async function getPublishedTags(params: CommonParams, ref: OCIRef): Promise { - const { output } = params; - try { - const url = `https://${ref.registry}/v2/${ref.namespace}/${ref.id}/tags/list`; - - const headers = { - 'Accept': 'application/json', - }; - - const httpOptions = { - type: 'GET', - url: url, - headers: headers - }; - - const res = await requestEnsureAuthenticated(params, httpOptions, ref); - if (!res) { - output.write('Request failed', LogLevel.Error); - return; - } - - const { statusCode, resBody } = res; - const body = resBody.toString(); - - // Expected when publishing for the first time - if (statusCode === 404) { - return []; - // Unexpected Error - } else if (statusCode > 299) { - output.write(`(!) ERR: Could not fetch published tags for '${ref.namespace}/${ref.id}' : ${resBody ?? ''} `, LogLevel.Error); - return; - } - - const publishedVersionsResponse: OCITagList = JSON.parse(body); - - // Return published tags from the registry as-is, meaning: - // - Not necessarily sorted - // - *Including* major/minor/latest tags - return publishedVersionsResponse.tags; - } catch (e) { - output.write(`Failed to parse published versions: ${e}`, LogLevel.Error); - return; - } -} - -export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, expectedDigest: string, omitDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> { - // TODO: Parallelize if multiple layers (not likely). - // TODO: Seeking might be needed if the size is too large. - - const { output } = params; - try { - await mkdirpLocal(ociCacheDir); - const tempTarballPath = path.join(ociCacheDir, 'blob.tar'); - - const headers = { - 'Accept': 'application/vnd.oci.image.manifest.v1+json', - }; - - const httpOptions = { - type: 'GET', - url: url, - headers: headers - }; - - const res = await requestEnsureAuthenticated(params, httpOptions, ociRef); - if (!res) { - output.write('Request failed', LogLevel.Error); - return; - } - - const { statusCode, resBody } = res; - if (statusCode > 299) { - output.write(`Failed to fetch blob (${url}): ${resBody}`, LogLevel.Error); - return; - } - - const actualDigest = `sha256:${crypto.createHash('sha256').update(resBody).digest('hex')}`; - if (actualDigest !== expectedDigest) { - throw new Error(`Digest did not match for ${ociRef.resource}.`); - } - - await mkdirpLocal(destCachePath); - await writeLocalFile(tempTarballPath, resBody); - - // https://github.com/devcontainers/spec/blob/main/docs/specs/devcontainer-templates.md#the-optionalpaths-property - const directoriesToOmit = omitDuringExtraction.filter(f => f.endsWith('/*')).map(f => f.slice(0, -1)); - const filesToOmit = omitDuringExtraction.filter(f => !f.endsWith('/*')); - - output.write(`omitDuringExtraction: '${omitDuringExtraction.join(', ')}`, LogLevel.Trace); - output.write(`Files to omit: '${filesToOmit.join(', ')}'`, LogLevel.Info); - if (directoriesToOmit.length) { - output.write(`Dirs to omit : '${directoriesToOmit.join(', ')}'`, LogLevel.Info); - } - - const files: string[] = []; - await tar.x( - { - file: tempTarballPath, - cwd: destCachePath, - filter: (tPath, stat) => { - const entryType = 'type' in stat ? stat.type : (stat.isFile() ? 'File' : stat.isDirectory() ? 'Directory' : 'Other'); - output.write(`Testing '${tPath}'(${entryType})`, LogLevel.Trace); - const cleanedPath = tPath - .replace(/\\/g, '/') - .replace(/^\.\//, ''); - - if (filesToOmit.includes(cleanedPath) || directoriesToOmit.some(d => cleanedPath.startsWith(d))) { - output.write(` Omitting '${tPath}'`, LogLevel.Trace); - return false; // Skip - } - - const isFile = 'type' in stat ? stat.type === 'File' : stat.isFile(); - if (isFile) { - files.push(tPath); - } - - return true; // Keep - } - } - ); - output.write('Files extracted from blob: ' + files.join(', '), LogLevel.Trace); - - // No 'metadataFile' to look for. - if (!metadataFile) { - return { files, metadata: undefined }; - } - - // Attempt to extract 'metadataFile' - await tar.x( - { - file: tempTarballPath, - cwd: ociCacheDir, - filter: (tPath, _) => { - return tPath === `./${metadataFile}`; - } - }); - const pathToMetadataFile = path.join(ociCacheDir, metadataFile); - let metadata = undefined; - if (await isLocalFile(pathToMetadataFile)) { - output.write(`Found metadata file '${metadataFile}' in blob`, LogLevel.Trace); - metadata = jsonc.parse((await readLocalFile(pathToMetadataFile)).toString()); - } - - return { - files, metadata - }; - } catch (e) { - output.write(`Error getting blob: ${e}`, LogLevel.Error); - return; - } -} diff --git a/src/spec-configuration/containerCollectionsOCIPush.ts b/src/spec-configuration/containerCollectionsOCIPush.ts deleted file mode 100644 index 24f811663..000000000 --- a/src/spec-configuration/containerCollectionsOCIPush.ts +++ /dev/null @@ -1,416 +0,0 @@ -import * as path from 'path'; -import * as fs from 'fs'; -import * as crypto from 'crypto'; -import { delay } from '../spec-common/async'; -import { Log, LogLevel } from '../spec-utils/log'; -import { isLocalFile } from '../spec-utils/pfs'; -import { DEVCONTAINER_COLLECTION_LAYER_MEDIATYPE, DEVCONTAINER_TAR_LAYER_MEDIATYPE, fetchOCIManifestIfExists, OCICollectionRef, OCILayer, OCIManifest, OCIRef, CommonParams, ManifestContainer } from './containerCollectionsOCI'; -import { requestEnsureAuthenticated } from './httpOCIRegistry'; - -// (!) Entrypoint function to push a single feature/template to a registry. -// Devcontainer Spec (features) : https://containers.dev/implementors/features-distribution/#oci-registry -// Devcontainer Spec (templates): https://github.com/devcontainers/spec/blob/main/proposals/devcontainer-templates-distribution.md#oci-registry -// OCI Spec : https://github.com/opencontainers/distribution-spec/blob/main/spec.md#push -export async function pushOCIFeatureOrTemplate(params: CommonParams, ociRef: OCIRef, pathToTgz: string, tags: string[], collectionType: string, annotations: { [key: string]: string } = {}): Promise { - const { output } = params; - - output.write(`-- Starting push of ${collectionType} '${ociRef.id}' to '${ociRef.resource}' with tags '${tags.join(', ')}'`); - output.write(`${JSON.stringify(ociRef, null, 2)}`, LogLevel.Trace); - - if (!(await isLocalFile(pathToTgz))) { - output.write(`Blob ${pathToTgz} does not exist.`, LogLevel.Error); - return; - } - - const dataBytes = fs.readFileSync(pathToTgz); - - // Generate Manifest for given feature/template artifact. - const manifest = await generateCompleteManifestForIndividualFeatureOrTemplate(output, dataBytes, pathToTgz, ociRef, collectionType, annotations); - if (!manifest) { - output.write(`Failed to generate manifest for ${ociRef.id}`, LogLevel.Error); - return; - } - - output.write(`Generated manifest: \n${JSON.stringify(manifest?.manifestObj, undefined, 4)}`, LogLevel.Trace); - - // If the exact manifest digest already exists in the registry, we don't need to push individual blobs (it's already there!) - const existingManifest = await fetchOCIManifestIfExists(params, ociRef, manifest.contentDigest); - if (manifest.contentDigest && existingManifest) { - output.write(`Not reuploading blobs, digest already exists.`, LogLevel.Trace); - return await putManifestWithTags(params, manifest, ociRef, tags); - } - - const blobsToPush = [ - { - name: 'configLayer', - digest: manifest.manifestObj.config.digest, - size: manifest.manifestObj.config.size, - contents: Buffer.from('{}'), - }, - { - name: 'tgzLayer', - digest: manifest.manifestObj.layers[0].digest, - size: manifest.manifestObj.layers[0].size, - contents: dataBytes, - } - ]; - - - for await (const blob of blobsToPush) { - const { name, digest } = blob; - const blobExistsConfigLayer = await checkIfBlobExists(params, ociRef, digest); - output.write(`blob: '${name}' ${blobExistsConfigLayer ? 'DOES exists' : 'DOES NOT exist'} in registry.`, LogLevel.Trace); - - // PUT blobs - if (!blobExistsConfigLayer) { - - // Obtain session ID with `/v2//blobs/uploads/` - const blobPutLocationUriPath = await postUploadSessionId(params, ociRef); - if (!blobPutLocationUriPath) { - output.write(`Failed to get upload session ID`, LogLevel.Error); - return; - } - - if (!(await putBlob(params, blobPutLocationUriPath, ociRef, blob))) { - output.write(`Failed to PUT blob '${name}' with digest '${digest}'`, LogLevel.Error); - return; - } - } - } - - // Send a final PUT to combine blobs and tag manifest properly. - return await putManifestWithTags(params, manifest, ociRef, tags); -} - -// (!) Entrypoint function to push a collection metadata/overview file for a set of features/templates to a registry. -// Devcontainer Spec (features) : https://containers.dev/implementors/features-distribution/#oci-registry (see 'devcontainer-collection.json') -// Devcontainer Spec (templates): https://github.com/devcontainers/spec/blob/main/proposals/devcontainer-templates-distribution.md#oci-registry (see 'devcontainer-collection.json') -// OCI Spec : https://github.com/opencontainers/distribution-spec/blob/main/spec.md#push -export async function pushCollectionMetadata(params: CommonParams, collectionRef: OCICollectionRef, pathToCollectionJson: string, collectionType: string): Promise { - const { output } = params; - - output.write(`Starting push of latest ${collectionType} collection for namespace '${collectionRef.path}' to '${collectionRef.registry}'`); - output.write(`${JSON.stringify(collectionRef, null, 2)}`, LogLevel.Trace); - - if (!(await isLocalFile(pathToCollectionJson))) { - output.write(`Collection Metadata was not found at expected location: ${pathToCollectionJson}`, LogLevel.Error); - return; - } - - const dataBytes = fs.readFileSync(pathToCollectionJson); - - // Generate Manifest for collection artifact. - const manifest = await generateCompleteManifestForCollectionFile(output, dataBytes, collectionRef); - if (!manifest) { - output.write(`Failed to generate manifest for ${collectionRef.path}`, LogLevel.Error); - return; - } - output.write(`Generated manifest: \n${JSON.stringify(manifest?.manifestObj, undefined, 4)}`, LogLevel.Trace); - - // If the exact manifest digest already exists in the registry, we don't need to push individual blobs (it's already there!) - const existingManifest = await fetchOCIManifestIfExists(params, collectionRef, manifest.contentDigest); - if (manifest.contentDigest && existingManifest) { - output.write(`Not reuploading blobs, digest already exists.`, LogLevel.Trace); - return await putManifestWithTags(params, manifest, collectionRef, ['latest']); - } - - const blobsToPush = [ - { - name: 'configLayer', - digest: manifest.manifestObj.config.digest, - size: manifest.manifestObj.config.size, - contents: Buffer.from('{}'), - }, - { - name: 'collectionLayer', - digest: manifest.manifestObj.layers[0].digest, - size: manifest.manifestObj.layers[0].size, - contents: dataBytes, - } - ]; - - for await (const blob of blobsToPush) { - const { name, digest } = blob; - const blobExistsConfigLayer = await checkIfBlobExists(params, collectionRef, digest); - output.write(`blob: '${name}' with digest '${digest}' ${blobExistsConfigLayer ? 'already exists' : 'does not exist'} in registry.`, LogLevel.Trace); - - // PUT blobs - if (!blobExistsConfigLayer) { - - // Obtain session ID with `/v2//blobs/uploads/` - const blobPutLocationUriPath = await postUploadSessionId(params, collectionRef); - if (!blobPutLocationUriPath) { - output.write(`Failed to get upload session ID`, LogLevel.Error); - return; - } - - if (!(await putBlob(params, blobPutLocationUriPath, collectionRef, blob))) { - output.write(`Failed to PUT blob '${name}' with digest '${digest}'`, LogLevel.Error); - return; - } - } - } - - // Send a final PUT to combine blobs and tag manifest properly. - // Collections are always tagged 'latest' - return await putManifestWithTags(params, manifest, collectionRef, ['latest']); -} - -// --- Helper Functions - -// Spec: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests (PUT /manifests/) -async function putManifestWithTags(params: CommonParams, manifest: ManifestContainer, ociRef: OCIRef | OCICollectionRef, tags: string[]): Promise { - const { output } = params; - - output.write(`Tagging manifest with tags: ${tags.join(', ')}`, LogLevel.Trace); - - const { manifestBuffer, contentDigest } = manifest; - - for await (const tag of tags) { - const url = `https://${ociRef.registry}/v2/${ociRef.path}/manifests/${tag}`; - output.write(`PUT -> '${url}'`, LogLevel.Trace); - - const httpOptions = { - type: 'PUT', - url, - headers: { - 'content-type': 'application/vnd.oci.image.manifest.v1+json', - }, - data: manifestBuffer, - }; - - let res = await requestEnsureAuthenticated(params, httpOptions, ociRef); - if (!res) { - output.write('Request failed', LogLevel.Error); - return; - } - - // Retry logic: when request fails with HTTP 429: too many requests - // TODO: Wrap into `requestEnsureAuthenticated`? - if (res.statusCode === 429) { - output.write(`Failed to PUT manifest for tag ${tag} due to too many requests. Retrying...`, LogLevel.Warning); - await delay(2000); - - res = await requestEnsureAuthenticated(params, httpOptions, ociRef); - if (!res) { - output.write('Request failed', LogLevel.Error); - return; - } - } - - const { statusCode, resBody, resHeaders } = res; - - if (statusCode !== 201) { - const parsed = JSON.parse(resBody?.toString() || '{}'); - output.write(`Failed to PUT manifest for tag ${tag}\n${JSON.stringify(parsed, undefined, 4)}`, LogLevel.Error); - return; - } - - const dockerContentDigestResponseHeader = resHeaders['docker-content-digest']; - const locationResponseHeader = resHeaders['location'] || resHeaders['Location']; - output.write(`Tagged: ${tag} -> ${locationResponseHeader}`, LogLevel.Info); - output.write(`Returned Content-Digest: ${dockerContentDigestResponseHeader}`, LogLevel.Trace); - } - return contentDigest; -} - -// Spec: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#post-then-put (PUT ?digest=) -async function putBlob(params: CommonParams, blobPutLocationUriPath: string, ociRef: OCIRef | OCICollectionRef, blob: { name: string; digest: string; size: number; contents: Buffer }): Promise { - - const { output } = params; - const { name, digest, size, contents } = blob; - - output.write(`Starting PUT of ${name} blob '${digest}' (size=${size})`, LogLevel.Info); - - const headers = { - 'content-type': 'application/octet-stream', - 'content-length': `${size}` - }; - - // OCI distribution spec is ambiguous on whether we get back an absolute or relative path. - let url = ''; - if (blobPutLocationUriPath.startsWith('https://') || blobPutLocationUriPath.startsWith('http://')) { - url = blobPutLocationUriPath; - } else { - url = `https://${ociRef.registry}${blobPutLocationUriPath}`; - } - - // The MAY contain critical query parameters. - // Additionally, it SHOULD match exactly the obtained from the POST request. - // It SHOULD NOT be assembled manually by clients except where absolute/relative conversion is necessary. - const queryParamsStart = url.indexOf('?'); - if (queryParamsStart === -1) { - // Just append digest to the end. - url += `?digest=${digest}`; - } else { - url = url.substring(0, queryParamsStart) + `?digest=${digest}` + '&' + url.substring(queryParamsStart + 1); - } - - output.write(`PUT blob to -> ${url}`, LogLevel.Trace); - - const res = await requestEnsureAuthenticated(params, { type: 'PUT', url, headers, data: contents }, ociRef); - if (!res) { - output.write('Request failed', LogLevel.Error); - return false; - } - - const { statusCode, resBody } = res; - - if (statusCode !== 201) { - const parsed = JSON.parse(resBody?.toString() || '{}'); - output.write(`${statusCode}: Failed to upload blob '${digest}' to '${url}' \n${JSON.stringify(parsed, undefined, 4)}`, LogLevel.Error); - return false; - } - - return true; -} - -// Generate a layer that follows the `application/vnd.devcontainers.layer.v1+tar` mediaType as defined in -// Devcontainer Spec (features) : https://containers.dev/implementors/features-distribution/#oci-registry -// Devcontainer Spec (templates): https://github.com/devcontainers/spec/blob/main/proposals/devcontainer-templates-distribution.md#oci-registry -async function generateCompleteManifestForIndividualFeatureOrTemplate(output: Log, dataBytes: Buffer, pathToTgz: string, ociRef: OCIRef, collectionType: string, annotations: { [key: string]: string } = {}): Promise { - const tgzLayer = await calculateDataLayer(output, dataBytes, path.basename(pathToTgz), DEVCONTAINER_TAR_LAYER_MEDIATYPE); - if (!tgzLayer) { - output.write(`Failed to calculate tgz layer.`, LogLevel.Error); - return undefined; - } - - // Specific registries look for certain optional metadata - // in the manifest, in this case for UI presentation. - if (ociRef.registry === 'ghcr.io') { - annotations = { - ...annotations, - 'com.github.package.type': `devcontainer_${collectionType}`, - }; - } - - return await calculateManifestAndContentDigest(output, ociRef, tgzLayer, annotations); -} - -// Generate a layer that follows the `application/vnd.devcontainers.collection.layer.v1+json` mediaType as defined in -// Devcontainer Spec (features) : https://containers.dev/implementors/features-distribution/#oci-registry -// Devcontainer Spec (templates): https://github.com/devcontainers/spec/blob/main/proposals/devcontainer-templates-distribution.md#oci-registry -async function generateCompleteManifestForCollectionFile(output: Log, dataBytes: Buffer, collectionRef: OCICollectionRef): Promise { - const collectionMetadataLayer = await calculateDataLayer(output, dataBytes, 'devcontainer-collection.json', DEVCONTAINER_COLLECTION_LAYER_MEDIATYPE); - if (!collectionMetadataLayer) { - output.write(`Failed to calculate collection file layer.`, LogLevel.Error); - return undefined; - } - - let annotations: { [key: string]: string } | undefined = undefined; - // Specific registries look for certain optional metadata - // in the manifest, in this case for UI presentation. - if (collectionRef.registry === 'ghcr.io') { - annotations = { - 'com.github.package.type': 'devcontainer_collection', - }; - } - return await calculateManifestAndContentDigest(output, collectionRef, collectionMetadataLayer, annotations); -} - -// Generic construction of a layer in the manifest and digest for the generated layer. -export async function calculateDataLayer(output: Log, data: Buffer, basename: string, mediaType: string): Promise { - output.write(`Creating manifest from data`, LogLevel.Trace); - - const algorithm = 'sha256'; - const tarSha256 = crypto.createHash(algorithm).update(data).digest('hex'); - const digest = `${algorithm}:${tarSha256}`; - output.write(`Data layer digest: ${digest} (archive size: ${data.byteLength})`, LogLevel.Info); - - return { - mediaType, - digest, - size: data.byteLength, - annotations: { - 'org.opencontainers.image.title': basename, - } - }; -} - -// Spec: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#checking-if-content-exists-in-the-registry -// Requires registry auth token. -export async function checkIfBlobExists(params: CommonParams, ociRef: OCIRef | OCICollectionRef, digest: string): Promise { - const { output } = params; - - const url = `https://${ociRef.registry}/v2/${ociRef.path}/blobs/${digest}`; - const res = await requestEnsureAuthenticated(params, { type: 'HEAD', url, headers: {} }, ociRef); - if (!res) { - output.write('Request failed', LogLevel.Error); - return false; - } - - const statusCode = res.statusCode; - output.write(`checkIfBlobExists: ${url}: ${statusCode}`, LogLevel.Trace); - return statusCode === 200; -} - -// Spec: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#post-then-put -// Requires registry auth token. -async function postUploadSessionId(params: CommonParams, ociRef: OCIRef | OCICollectionRef): Promise { - const { output } = params; - - const url = `https://${ociRef.registry}/v2/${ociRef.path}/blobs/uploads/`; - output.write(`Generating Upload URL -> ${url}`, LogLevel.Trace); - const res = await requestEnsureAuthenticated(params, { type: 'POST', url, headers: {} }, ociRef); - - if (!res) { - output.write('Request failed', LogLevel.Error); - return; - } - - const { statusCode, resBody, resHeaders } = res; - - output.write(`${url}: ${statusCode}`, LogLevel.Trace); - if (statusCode === 202) { - const locationHeader = resHeaders['location'] || resHeaders['Location']; - if (!locationHeader) { - output.write(`${url}: Got 202 status code, but no location header found.`, LogLevel.Error); - return undefined; - } - output.write(`Generated Upload URL: ${locationHeader}`, LogLevel.Trace); - return locationHeader; - } else { - // Any other statusCode besides 202 is unexpected - // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#error-codes - const parsed = JSON.parse(resBody?.toString() || '{}'); - output.write(`${url}: Unexpected status code '${statusCode}' \n${JSON.stringify(parsed, undefined, 4)}`, LogLevel.Error); - return undefined; - } -} - -export async function calculateManifestAndContentDigest(output: Log, ociRef: OCIRef | OCICollectionRef, dataLayer: OCILayer, annotations: { [key: string]: string } | undefined): Promise { - // A canonical manifest digest is the sha256 hash of the JSON representation of the manifest, without the signature content. - // See: https://docs.docker.com/registry/spec/api/#content-digests - // Below is an example of a serialized manifest that should resolve to 'dd328c25cc7382aaf4e9ee10104425d9a2561b47fe238407f6c0f77b3f8409fc' - // {"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.devcontainers","digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},"layers":[{"mediaType":"application/vnd.devcontainers.layer.v1+tar","digest":"sha256:0bb92d2da46d760c599d0a41ed88d52521209408b529761417090b62ee16dfd1","size":3584,"annotations":{"org.opencontainers.image.title":"devcontainer-feature-color.tgz"}}],"annotations":{"dev.containers.metadata":"{\"id\":\"color\",\"version\":\"1.0.0\",\"name\":\"A feature to remind you of your favorite color\",\"options\":{\"favorite\":{\"type\":\"string\",\"enum\":[\"red\",\"gold\",\"green\"],\"default\":\"red\",\"description\":\"Choose your favorite color.\"}}}","com.github.package.type":"devcontainer_feature"}} - - let manifest: OCIManifest = { - schemaVersion: 2, - mediaType: 'application/vnd.oci.image.manifest.v1+json', - config: { - mediaType: 'application/vnd.devcontainers', - digest: 'sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a', // A empty json byte digest for the devcontainer mediaType. - size: 2 - }, - layers: [ - dataLayer - ], - }; - - if (annotations) { - manifest.annotations = annotations; - } - - const manifestBuffer = Buffer.from(JSON.stringify(manifest)); - const algorithm = 'sha256'; - const manifestHash = crypto.createHash(algorithm).update(manifestBuffer).digest('hex'); - const contentDigest = `${algorithm}:${manifestHash}`; - output.write(`Computed content digest from manifest: ${contentDigest}`, LogLevel.Info); - - return { - manifestBuffer, - manifestObj: manifest, - contentDigest, - canonicalId: `${ociRef.resource}@sha256:${manifestHash}` - }; -} diff --git a/src/spec-configuration/containerFeaturesConfiguration.ts b/src/spec-configuration/containerFeaturesConfiguration.ts deleted file mode 100644 index 78d6c8018..000000000 --- a/src/spec-configuration/containerFeaturesConfiguration.ts +++ /dev/null @@ -1,1261 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as jsonc from 'jsonc-parser'; -import * as path from 'path'; -import * as URL from 'url'; -import * as tar from 'tar'; -import * as crypto from 'crypto'; -import * as semver from 'semver'; -import * as os from 'os'; -import * as fs from 'fs'; - -import { DevContainerConfig, DevContainerFeature, VSCodeCustomizations } from './configuration'; -import { mkdirpLocal, readLocalFile, rmLocal, writeLocalFile, cpDirectoryLocal, isLocalFile } from '../spec-utils/pfs'; -import { Log, LogLevel, nullLog } from '../spec-utils/log'; -import { request } from '../spec-utils/httpRequest'; -import { fetchOCIFeature, tryGetOCIFeatureSet, fetchOCIFeatureManifestIfExistsFromUserIdentifier } from './containerFeaturesOCI'; -import { uriToFsPath } from './configurationCommonUtils'; -import { CommonParams, ManifestContainer, OCIManifest, OCIRef, getRef, getVersionsStrictSorted } from './containerCollectionsOCI'; -import { Lockfile, generateLockfile, readLockfile, writeLockfile } from './lockfile'; -import { computeDependsOnInstallationOrder } from './containerFeaturesOrder'; -import { logFeatureAdvisories } from './featureAdvisories'; -import { getEntPasswdShellCommand } from '../spec-common/commonUtils'; -import { ContainerError } from '../spec-common/errors'; - -// v1 -const V1_ASSET_NAME = 'devcontainer-features.tgz'; -export const V1_DEVCONTAINER_FEATURES_FILE_NAME = 'devcontainer-features.json'; - -// v2 -export const DEVCONTAINER_FEATURE_FILE_NAME = 'devcontainer-feature.json'; - -export type Feature = SchemaFeatureBaseProperties & SchemaFeatureLifecycleHooks & DeprecatedSchemaFeatureProperties & InternalFeatureProperties; - -export const FEATURES_CONTAINER_TEMP_DEST_FOLDER = '/tmp/dev-container-features'; - -export interface SchemaFeatureLifecycleHooks { - onCreateCommand?: string | string[]; - updateContentCommand?: string | string[]; - postCreateCommand?: string | string[]; - postStartCommand?: string | string[]; - postAttachCommand?: string | string[]; -} - -// Properties who are members of the schema -export interface SchemaFeatureBaseProperties { - id: string; - version?: string; - name?: string; - description?: string; - documentationURL?: string; - licenseURL?: string; - options?: Record; - containerEnv?: Record; - mounts?: Mount[]; - init?: boolean; - privileged?: boolean; - capAdd?: string[]; - securityOpt?: string[]; - entrypoint?: string; - customizations?: VSCodeCustomizations; - installsAfter?: string[]; - deprecated?: boolean; - legacyIds?: string[]; - dependsOn?: Record>; -} - -// Properties that are set programmatically for book-keeping purposes -export interface InternalFeatureProperties { - cachePath?: string; - internalVersion?: string; - consecutiveId?: string; - value: boolean | string | Record; - currentId?: string; - included: boolean; -} - -// Old or deprecated properties maintained for backwards compatibility -export interface DeprecatedSchemaFeatureProperties { - buildArg?: string; - include?: string[]; - exclude?: string[]; -} - -export type FeatureOption = { - type: 'boolean'; - default?: boolean; - description?: string; -} | { - type: 'string'; - enum?: string[]; - default?: string; - description?: string; -} | { - type: 'string'; - proposals?: string[]; - default?: string; - description?: string; -}; -export interface Mount { - type: 'bind' | 'volume'; - source?: string; - target: string; - external?: boolean; -} - -const normalizedMountKeys: Record = { - src: 'source', - destination: 'target', - dst: 'target', -}; - -export function parseMount(str: string): Mount { - return str.split(',') - .map(s => s.split('=')) - .reduce((acc, [key, value]) => ({ ...acc, [(normalizedMountKeys[key] || key)]: value }), {}) as Mount; -} - -export type SourceInformation = GithubSourceInformation | DirectTarballSourceInformation | FilePathSourceInformation | OCISourceInformation; - -interface BaseSourceInformation { - type: string; - userFeatureId: string; // Dictates how a supporting tool will locate and download a given feature. See https://github.com/devcontainers/spec/blob/main/proposals/devcontainer-features.md#referencing-a-feature - userFeatureIdWithoutVersion?: string; -} - -export interface OCISourceInformation extends BaseSourceInformation { - type: 'oci'; - featureRef: OCIRef; - manifest: OCIManifest; - manifestDigest: string; - userFeatureIdWithoutVersion: string; -} - -export interface DirectTarballSourceInformation extends BaseSourceInformation { - type: 'direct-tarball'; - tarballUri: string; -} - -export interface FilePathSourceInformation extends BaseSourceInformation { - type: 'file-path'; - resolvedFilePath: string; // Resolved, absolute file path -} - -// deprecated -export interface GithubSourceInformation extends BaseSourceInformation { - type: 'github-repo'; - apiUri: string; - unauthenticatedUri: string; - owner: string; - repo: string; - isLatest: boolean; // 'true' indicates user didn't supply a version tag, thus we implicitly pull latest. - tag?: string; - ref?: string; - sha?: string; - userFeatureIdWithoutVersion: string; -} - -export interface FeatureSet { - features: Feature[]; - internalVersion?: string; - sourceInformation: SourceInformation; - computedDigest?: string; -} - -export interface FeaturesConfig { - featureSets: FeatureSet[]; - dstFolder?: string; // set programatically -} - -export interface GitHubApiReleaseInfo { - assets: GithubApiReleaseAsset[]; - name: string; - tag_name: string; -} - -export interface GithubApiReleaseAsset { - url: string; - name: string; - content_type: string; - size: number; - download_count: number; - updated_at: string; -} - -export interface ContainerFeatureInternalParams { - extensionPath: string; - cacheFolder: string; - cwd: string; - output: Log; - env: NodeJS.ProcessEnv; - skipFeatureAutoMapping: boolean; - platform: NodeJS.Platform; - experimentalLockfile?: boolean; - experimentalFrozenLockfile?: boolean; -} - -// TODO: Move to node layer. -export function getContainerFeaturesBaseDockerFile(contentSourceRootPath: string) { - return ` - -#{nonBuildKitFeatureContentFallback} - -FROM $_DEV_CONTAINERS_BASE_IMAGE AS dev_containers_feature_content_normalize -USER root -COPY --from=dev_containers_feature_content_source ${path.posix.join(contentSourceRootPath, 'devcontainer-features.builtin.env')} /tmp/build-features/ -RUN chmod -R 0755 /tmp/build-features/ - -FROM $_DEV_CONTAINERS_BASE_IMAGE AS dev_containers_target_stage - -USER root - -RUN mkdir -p ${FEATURES_CONTAINER_TEMP_DEST_FOLDER} -COPY --from=dev_containers_feature_content_normalize /tmp/build-features/ ${FEATURES_CONTAINER_TEMP_DEST_FOLDER} - -#{featureLayer} - -#{containerEnv} - -ARG _DEV_CONTAINERS_IMAGE_USER=root -USER $_DEV_CONTAINERS_IMAGE_USER - -#{devcontainerMetadata} - -#{containerEnvMetadata} -`; -} - -export function getFeatureInstallWrapperScript(feature: Feature, featureSet: FeatureSet, options: string[]): string { - const id = escapeQuotesForShell(featureSet.sourceInformation.userFeatureIdWithoutVersion ?? 'Unknown'); - const name = escapeQuotesForShell(feature.name ?? 'Unknown'); - const description = escapeQuotesForShell(feature.description ?? ''); - const version = escapeQuotesForShell(feature.version ?? ''); - const documentation = escapeQuotesForShell(feature.documentationURL ?? ''); - const optionsIndented = escapeQuotesForShell(options.map(x => ` ${x}`).join('\n')); - - let warningHeader = ''; - if (feature.deprecated) { - warningHeader += `(!) WARNING: Using the deprecated Feature "${escapeQuotesForShell(feature.id)}". This Feature will no longer receive any further updates/support.\n`; - } - - if (feature?.legacyIds && feature.legacyIds.length > 0 && feature.currentId && feature.id !== feature.currentId) { - warningHeader += `(!) WARNING: This feature has been renamed. Please update the reference in devcontainer.json to "${escapeQuotesForShell(feature.currentId)}".`; - } - - const echoWarning = warningHeader ? `echo '${warningHeader}'` : ''; - const errorMessage = `ERROR: Feature "${name}" (${id}) failed to install!`; - const troubleshootingMessage = documentation - ? ` Look at the documentation at ${documentation} for help troubleshooting this error.` - : ''; - - return `#!/bin/sh -set -e - -on_exit () { - [ $? -eq 0 ] && exit - echo '${errorMessage}${troubleshootingMessage}' -} - -trap on_exit EXIT - -echo =========================================================================== -${echoWarning} -echo 'Feature : ${name}' -echo 'Description : ${description}' -echo 'Id : ${id}' -echo 'Version : ${version}' -echo 'Documentation : ${documentation}' -echo 'Options :' -echo '${optionsIndented}' -echo =========================================================================== - -set -a -. ../devcontainer-features.builtin.env -. ./devcontainer-features.env -set +a - -chmod +x ./install.sh -./install.sh -`; -} - -function escapeQuotesForShell(input: string) { - // The `input` is expected to be a string which will be printed inside single quotes - // by the caller. This means we need to escape any nested single quotes within the string. - // We can do this by ending the first string with a single quote ('), printing an escaped - // single quote (\'), and then opening a new string ('). - return input.replace(new RegExp(`'`, 'g'), `'\\''`); -} - -export function getFeatureLayers(featuresConfig: FeaturesConfig, containerUser: string, remoteUser: string, useBuildKitBuildContexts = false, contentSourceRootPath = '/tmp/build-features') { - - const builtinsEnvFile = `${path.posix.join(FEATURES_CONTAINER_TEMP_DEST_FOLDER, 'devcontainer-features.builtin.env')}`; - let result = `RUN \\ -echo "_CONTAINER_USER_HOME=$(${getEntPasswdShellCommand(containerUser)} | cut -d: -f6)" >> ${builtinsEnvFile} && \\ -echo "_REMOTE_USER_HOME=$(${getEntPasswdShellCommand(remoteUser)} | cut -d: -f6)" >> ${builtinsEnvFile} - -`; - - // Features version 1 - const folders = (featuresConfig.featureSets || []).filter(y => y.internalVersion !== '2').map(x => x.features[0].consecutiveId); - folders.forEach(folder => { - const source = path.posix.join(contentSourceRootPath, folder!); - const dest = path.posix.join(FEATURES_CONTAINER_TEMP_DEST_FOLDER, folder!); - if (!useBuildKitBuildContexts) { - result += `COPY --chown=root:root --from=dev_containers_feature_content_source ${source} ${dest} -RUN chmod -R 0755 ${dest} \\ -&& cd ${dest} \\ -&& chmod +x ./install.sh \\ -&& ./install.sh - -`; - } else { - result += `RUN --mount=type=bind,from=dev_containers_feature_content_source,source=${source},target=/tmp/build-features-src/${folder} \\ - cp -ar /tmp/build-features-src/${folder} ${FEATURES_CONTAINER_TEMP_DEST_FOLDER} \\ - && chmod -R 0755 ${dest} \\ - && cd ${dest} \\ - && chmod +x ./install.sh \\ - && ./install.sh \\ - && rm -rf ${dest} - -`; - } - }); - // Features version 2 - featuresConfig.featureSets.filter(y => y.internalVersion === '2').forEach(featureSet => { - featureSet.features.forEach(feature => { - result += generateContainerEnvs(feature.containerEnv); - const source = path.posix.join(contentSourceRootPath, feature.consecutiveId!); - const dest = path.posix.join(FEATURES_CONTAINER_TEMP_DEST_FOLDER, feature.consecutiveId!); - if (!useBuildKitBuildContexts) { - result += ` -COPY --chown=root:root --from=dev_containers_feature_content_source ${source} ${dest} -RUN chmod -R 0755 ${dest} \\ -&& cd ${dest} \\ -&& chmod +x ./devcontainer-features-install.sh \\ -&& ./devcontainer-features-install.sh - -`; - } else { - result += ` -RUN --mount=type=bind,from=dev_containers_feature_content_source,source=${source},target=/tmp/build-features-src/${feature.consecutiveId} \\ - cp -ar /tmp/build-features-src/${feature.consecutiveId} ${FEATURES_CONTAINER_TEMP_DEST_FOLDER} \\ - && chmod -R 0755 ${dest} \\ - && cd ${dest} \\ - && chmod +x ./devcontainer-features-install.sh \\ - && ./devcontainer-features-install.sh \\ - && rm -rf ${dest} - -`; - } - }); - }); - return result; -} - -// Features version two export their environment variables as part of the Dockerfile to make them available to subsequent features. -export function generateContainerEnvs(containerEnv: Record | undefined, escapeDollar = false): string { - if (!containerEnv) { - return ''; - } - const keys = Object.keys(containerEnv); - // https://docs.docker.com/engine/reference/builder/#envs - const r = escapeDollar ? /(?=["\\$])/g : /(?=["\\])/g; // escape double quotes, back slash, and optionally dollar sign - return keys.map(k => `ENV ${k}="${containerEnv[k] - .replace(r, '\\') - }"`).join('\n'); -} - -const allowedFeatureIdRegex = new RegExp('^[a-zA-Z0-9_-]*$'); - -// Parses a declared feature in user's devcontainer file into -// a usable URI to download remote features. -// RETURNS -// { -// "id", <----- The ID of the feature in the feature set. -// sourceInformation <----- Source information (is this locally cached, a GitHub remote feature, etc..), including tarballUri if applicable. -// } -// - -const cleanupIterationFetchAndMerge = async (tempTarballPath: string, output: Log) => { - // Non-fatal, will just get overwritten if we don't do the cleaned up. - try { - await rmLocal(tempTarballPath, { force: true }); - } catch (e) { - output.write(`Didn't remove temporary tarball from disk with caught exception: ${e?.Message} `, LogLevel.Trace); - } -}; - -function getRequestHeaders(params: CommonParams, sourceInformation: SourceInformation) { - const { env, output } = params; - let headers: { 'user-agent': string; 'Authorization'?: string; 'Accept'?: string } = { - 'user-agent': 'devcontainer' - }; - - const isGitHubUri = (srcInfo: DirectTarballSourceInformation) => { - const uri = srcInfo.tarballUri; - return uri.startsWith('https://github.com') || uri.startsWith('https://api.github.com'); - }; - - if (sourceInformation.type === 'github-repo' || (sourceInformation.type === 'direct-tarball' && isGitHubUri(sourceInformation))) { - const githubToken = env['GITHUB_TOKEN']; - if (githubToken) { - output.write('Using environment GITHUB_TOKEN.'); - headers.Authorization = `Bearer ${githubToken}`; - } else { - output.write('No environment GITHUB_TOKEN available.'); - } - } - return headers; -} - -async function askGitHubApiForTarballUri(sourceInformation: GithubSourceInformation, feature: Feature, headers: { 'user-agent': string; 'Authorization'?: string; 'Accept'?: string }, output: Log) { - const options = { - type: 'GET', - url: sourceInformation.apiUri, - headers - }; - - const apiInfo: GitHubApiReleaseInfo = JSON.parse(((await request(options, output)).toString())); - if (apiInfo) { - const asset = - apiInfo.assets.find(a => a.name === `${feature.id}.tgz`) // v2 - || apiInfo.assets.find(a => a.name === V1_ASSET_NAME) // v1 - || undefined; - - if (asset && asset.url) { - output.write(`Found url to fetch release artifact '${asset.name}'. Asset of size ${asset.size} has been downloaded ${asset.download_count} times and was last updated at ${asset.updated_at}`); - return asset.url; - } else { - output.write('Unable to fetch release artifact URI from GitHub API', LogLevel.Error); - return undefined; - } - } - return undefined; -} - -function updateFromOldProperties(original: T): T { - // https://github.com/microsoft/dev-container-spec/issues/1 - if (!original.features.find(f => f.extensions || f.settings)) { - return original; - } - return { - ...original, - features: original.features.map(f => { - if (!(f.extensions || f.settings)) { - return f; - } - const copy = { ...f }; - const customizations = copy.customizations || (copy.customizations = {}); - const vscode = customizations.vscode || (customizations.vscode = {}); - if (copy.extensions) { - vscode.extensions = (vscode.extensions || []).concat(copy.extensions); - delete copy.extensions; - } - if (copy.settings) { - vscode.settings = { - ...copy.settings, - ...(vscode.settings || {}), - }; - delete copy.settings; - } - return copy; - }), - }; -} - -// Generate a base featuresConfig object with the set of locally-cached features, -// as well as downloading and merging in remote feature definitions. -export async function generateFeaturesConfig(params: ContainerFeatureInternalParams, dstFolder: string, config: DevContainerConfig, additionalFeatures: Record>) { - const { output } = params; - - const workspaceRoot = params.cwd; - output.write(`workspace root: ${workspaceRoot}`, LogLevel.Trace); - - const userFeatures = updateDeprecatedFeaturesIntoOptions(userFeaturesToArray(config, additionalFeatures), output); - if (!userFeatures) { - return undefined; - } - - let configPath = config.configFilePath && uriToFsPath(config.configFilePath, params.platform); - output.write(`configPath: ${configPath}`, LogLevel.Trace); - - const ociCacheDir = await prepareOCICache(dstFolder); - - const { lockfile, initLockfile } = await readLockfile(config); - - const processFeature = async (_userFeature: DevContainerFeature) => { - return await processFeatureIdentifier(params, configPath, workspaceRoot, _userFeature, lockfile); - }; - - output.write('--- Processing User Features ----', LogLevel.Trace); - const featureSets = await computeDependsOnInstallationOrder(params, processFeature, userFeatures, config, lockfile); - if (!featureSets) { - throw new Error('Failed to compute Feature installation order!'); - } - - // Create the featuresConfig object. - const featuresConfig: FeaturesConfig = { - featureSets, - dstFolder - }; - - // Fetch features, stage into the appropriate build folder, and read the feature's devcontainer-feature.json - output.write('--- Fetching User Features ----', LogLevel.Trace); - await fetchFeatures(params, featuresConfig, dstFolder, ociCacheDir, lockfile); - - await logFeatureAdvisories(params, featuresConfig); - await writeLockfile(params, config, await generateLockfile(featuresConfig), initLockfile); - return featuresConfig; -} - -export async function loadVersionInfo(params: ContainerFeatureInternalParams, config: DevContainerConfig) { - const userFeatures = userFeaturesToArray(config); - if (!userFeatures) { - return { features: {} }; - } - - const { lockfile } = await readLockfile(config); - - const resolved: Record = {}; - - await Promise.all(userFeatures.map(async userFeature => { - const userFeatureId = userFeature.userFeatureId; - const featureRef = getRef(nullLog, userFeatureId); // Filters out Feature identifiers that cannot be versioned (e.g. local paths, deprecated, etc..) - if (featureRef) { - const versions = (await getVersionsStrictSorted(params, featureRef)) - ?.reverse() || []; - if (versions) { - const lockfileVersion = lockfile?.features[userFeatureId]?.version; - let wanted = lockfileVersion; - const tag = featureRef.tag; - if (tag) { - if (tag === 'latest') { - wanted = versions[0]; - } else { - wanted = versions.find(version => semver.satisfies(version, tag)); - } - } else if (featureRef.digest && !wanted) { - const { type, manifest } = await getFeatureIdType(params, userFeatureId, undefined); - if (type === 'oci' && manifest) { - const wantedFeature = await findOCIFeatureMetadata(params, manifest); - wanted = wantedFeature?.version; - } - } - resolved[userFeatureId] = { - current: lockfileVersion || wanted, - wanted, - wantedMajor: wanted && semver.major(wanted)?.toString(), - latest: versions[0], - latestMajor: versions[0] && semver.major(versions[0])?.toString(), - }; - } - } - })); - - // Reorder Features to match the order in which they were specified in config - return { - features: userFeatures.reduce((acc, userFeature) => { - const r = resolved[userFeature.userFeatureId]; - if (r) { - acc[userFeature.userFeatureId] = r; - } - return acc; - }, {} as Record) - }; -} - -async function findOCIFeatureMetadata(params: ContainerFeatureInternalParams, manifest: ManifestContainer) { - const annotation = manifest.manifestObj.annotations?.['dev.containers.metadata']; - if (annotation) { - return jsonc.parse(annotation) as Feature; - } - - // Backwards compatibility. - const featureSet = tryGetOCIFeatureSet(params.output, manifest.canonicalId, {}, manifest, manifest.canonicalId); - if (!featureSet) { - return undefined; - } - - const tmp = path.join(os.tmpdir(), crypto.randomUUID()); - const f = await fetchOCIFeature(params, featureSet, tmp, tmp, DEVCONTAINER_FEATURE_FILE_NAME); - return f.metadata as Feature | undefined; -} - -async function prepareOCICache(dstFolder: string) { - const ociCacheDir = path.join(dstFolder, 'ociCache'); - await mkdirpLocal(ociCacheDir); - - return ociCacheDir; -} - -export function userFeaturesToArray(config: DevContainerConfig, additionalFeatures?: Record>): DevContainerFeature[] | undefined { - if (!Object.keys(config.features || {}).length && !Object.keys(additionalFeatures || {}).length) { - return undefined; - } - - const userFeatures: DevContainerFeature[] = []; - const userFeatureKeys = new Set(); - - if (config.features) { - for (const userFeatureKey of Object.keys(config.features)) { - const userFeatureValue = config.features[userFeatureKey]; - const feature: DevContainerFeature = { - userFeatureId: userFeatureKey, - options: userFeatureValue - }; - userFeatures.push(feature); - userFeatureKeys.add(userFeatureKey); - } - } - - if (additionalFeatures) { - for (const userFeatureKey of Object.keys(additionalFeatures)) { - // add the additional feature if it hasn't already been added from the config features - if (!userFeatureKeys.has(userFeatureKey)) { - const userFeatureValue = additionalFeatures[userFeatureKey]; - const feature: DevContainerFeature = { - userFeatureId: userFeatureKey, - options: userFeatureValue - }; - userFeatures.push(feature); - } - } - } - - return userFeatures; -} - -const deprecatedFeaturesIntoOptions: Record = { - gradle: { - mapTo: 'java', - withOptions: { - installGradle: true - } - }, - maven: { - mapTo: 'java', - withOptions: { - installMaven: true - } - }, - jupyterlab: { - mapTo: 'python', - withOptions: { - installJupyterlab: true - } - }, -}; - -export function updateDeprecatedFeaturesIntoOptions(userFeatures: DevContainerFeature[] | undefined, output: Log) { - if (!userFeatures) { - output.write('No user features to update', LogLevel.Trace); - return; - } - - const newFeaturePath = 'ghcr.io/devcontainers/features'; - const versionBackwardComp = '1'; - for (const update of userFeatures.filter(feature => deprecatedFeaturesIntoOptions[feature.userFeatureId])) { - const { mapTo, withOptions } = deprecatedFeaturesIntoOptions[update.userFeatureId]; - output.write(`(!) WARNING: Using the deprecated '${update.userFeatureId}' Feature. It is now part of the '${mapTo}' Feature. See https://github.com/devcontainers/features/tree/main/src/${mapTo}#options for the updated Feature.`, LogLevel.Warning); - const qualifiedMapToId = `${newFeaturePath}/${mapTo}`; - let userFeature = userFeatures.find(feature => feature.userFeatureId === mapTo || feature.userFeatureId === qualifiedMapToId || feature.userFeatureId.startsWith(`${qualifiedMapToId}:`)); - if (userFeature) { - userFeature.options = { - ...( - typeof userFeature.options === 'object' ? userFeature.options : - typeof userFeature.options === 'string' ? { version: userFeature.options } : - {} - ), - ...withOptions, - }; - } else { - userFeature = { - userFeatureId: `${qualifiedMapToId}:${versionBackwardComp}`, - options: withOptions - }; - userFeatures.push(userFeature); - } - } - const updatedUserFeatures = userFeatures.filter(feature => !deprecatedFeaturesIntoOptions[feature.userFeatureId]); - return updatedUserFeatures; -} - -export async function getFeatureIdType(params: CommonParams, userFeatureId: string, lockfile: Lockfile | undefined) { - const { output } = params; - // See the specification for valid feature identifiers: - // > https://github.com/devcontainers/spec/blob/main/proposals/devcontainer-features.md#referencing-a-feature - // - // Additionally, we support the following deprecated syntaxes for backwards compatibility: - // (0) A 'local feature' packaged with the CLI. - // Syntax: - // - // (1) A feature backed by a GitHub Release - // Syntax: //[@version] - - // Legacy feature-set ID - if (!userFeatureId.includes('/') && !userFeatureId.includes('\\')) { - const errorMessage = `Legacy feature '${userFeatureId}' not supported. Please check https://containers.dev/features for replacements. -If you were hoping to use local Features, remember to prepend your Feature name with "./". Please check https://containers.dev/implementors/features-distribution/#addendum-locally-referenced for more information.`; - output.write(errorMessage, LogLevel.Error); - throw new ContainerError({ - description: errorMessage - }); - } - - // Direct tarball reference - if (userFeatureId.startsWith('https://')) { - return { type: 'direct-tarball', manifest: undefined }; - } - - // Local feature on disk - // !! NOTE: The ability for paths outside the project file tree will soon be removed. - if (userFeatureId.startsWith('./') || userFeatureId.startsWith('../') || userFeatureId.startsWith('/')) { - return { type: 'file-path', manifest: undefined }; - } - - const manifest = await fetchOCIFeatureManifestIfExistsFromUserIdentifier(params, userFeatureId, lockfile?.features[userFeatureId]?.integrity); - if (manifest) { - return { type: 'oci', manifest: manifest }; - } else { - output.write(`Could not resolve Feature manifest for '${userFeatureId}'. If necessary, provide registry credentials with 'docker login '.`, LogLevel.Warning); - output.write(`Falling back to legacy GitHub Releases mode to acquire Feature.`, LogLevel.Trace); - - // DEPRECATED: This is a legacy feature-set ID - return { type: 'github-repo', manifest: undefined }; - } -} - -export function getBackwardCompatibleFeatureId(output: Log, id: string) { - const migratedfeatures = ['aws-cli', 'azure-cli', 'desktop-lite', 'docker-in-docker', 'docker-from-docker', 'dotnet', 'git', 'git-lfs', 'github-cli', 'java', 'kubectl-helm-minikube', 'node', 'powershell', 'python', 'ruby', 'rust', 'sshd', 'terraform']; - const renamedFeatures = new Map(); - renamedFeatures.set('golang', 'go'); - renamedFeatures.set('common', 'common-utils'); - - const deprecatedFeaturesIntoOptions = new Map(); - deprecatedFeaturesIntoOptions.set('gradle', 'java'); - deprecatedFeaturesIntoOptions.set('maven', 'java'); - deprecatedFeaturesIntoOptions.set('jupyterlab', 'python'); - - const newFeaturePath = 'ghcr.io/devcontainers/features'; - // Note: Pin the versionBackwardComp to '1' to avoid breaking changes. - const versionBackwardComp = '1'; - - // Mapping feature references (old shorthand syntax) from "microsoft/vscode-dev-containers" to "ghcr.io/devcontainers/features" - if (migratedfeatures.includes(id)) { - output.write(`(!) WARNING: Using the deprecated '${id}' Feature. See https://github.com/devcontainers/features/tree/main/src/${id}#example-usage for the updated Feature.`, LogLevel.Warning); - return `${newFeaturePath}/${id}:${versionBackwardComp}`; - } - - // Mapping feature references (renamed old shorthand syntax) from "microsoft/vscode-dev-containers" to "ghcr.io/devcontainers/features" - if (renamedFeatures.get(id) !== undefined) { - output.write(`(!) WARNING: Using the deprecated '${id}' Feature. See https://github.com/devcontainers/features/tree/main/src/${renamedFeatures.get(id)}#example-usage for the updated Feature.`, LogLevel.Warning); - return `${newFeaturePath}/${renamedFeatures.get(id)}:${versionBackwardComp}`; - } - - if (deprecatedFeaturesIntoOptions.get(id) !== undefined) { - output.write(`(!) WARNING: Falling back to the deprecated '${id}' Feature. It is now part of the '${deprecatedFeaturesIntoOptions.get(id)}' Feature. See https://github.com/devcontainers/features/tree/main/src/${deprecatedFeaturesIntoOptions.get(id)}#options for the updated Feature.`, LogLevel.Warning); - } - - // Deprecated and all other features references (eg. fish, ghcr.io/devcontainers/features/go, ghcr.io/owner/repo/id etc) - return id; -} - -// Strictly processes the user provided feature identifier to determine sourceInformation type. -// Returns a featureSet per feature. -export async function processFeatureIdentifier(params: CommonParams, configPath: string | undefined, _workspaceRoot: string, userFeature: DevContainerFeature, lockfile?: Lockfile, skipFeatureAutoMapping?: boolean): Promise { - const { output } = params; - - output.write(`* Processing feature: ${userFeature.userFeatureId}`); - - // id referenced by the user before the automapping from old shorthand syntax to "ghcr.io/devcontainers/features" - const originalUserFeatureId = userFeature.userFeatureId; - // Adding backward compatibility - if (!skipFeatureAutoMapping) { - userFeature.userFeatureId = getBackwardCompatibleFeatureId(output, userFeature.userFeatureId); - } - - const { type, manifest } = await getFeatureIdType(params, userFeature.userFeatureId, lockfile); - - // remote tar file - if (type === 'direct-tarball') { - output.write(`Remote tar file found.`); - const tarballUri = new URL.URL(userFeature.userFeatureId); - - const fullPath = tarballUri.pathname; - const tarballName = fullPath.substring(fullPath.lastIndexOf('/') + 1); - output.write(`tarballName = ${tarballName}`, LogLevel.Trace); - - const regex = new RegExp('devcontainer-feature-(.*).tgz'); - const matches = regex.exec(tarballName); - - if (!matches || matches.length !== 2) { - output.write(`Expected tarball name to follow 'devcontainer-feature-.tgz' format. Received '${tarballName}'`, LogLevel.Error); - return undefined; - } - const id = matches[1]; - - if (id === '' || !allowedFeatureIdRegex.test(id)) { - output.write(`Parse error. Specify a feature id with alphanumeric, dash, or underscore characters. Received ${id}.`, LogLevel.Error); - return undefined; - } - - let feat: Feature = { - id: id, - name: userFeature.userFeatureId, - value: userFeature.options, - included: true, - }; - - let newFeaturesSet: FeatureSet = { - sourceInformation: { - type: 'direct-tarball', - tarballUri: tarballUri.toString(), - userFeatureId: originalUserFeatureId - }, - features: [feat], - }; - - return newFeaturesSet; - } - - // Spec: https://containers.dev/implementors/features-distribution/#addendum-locally-referenced - if (type === 'file-path') { - output.write(`Local disk feature.`); - - const id = path.basename(userFeature.userFeatureId); - - // Fail on Absolute paths. - if (path.isAbsolute(userFeature.userFeatureId)) { - output.write('An Absolute path to a local feature is not allowed.', LogLevel.Error); - return undefined; - } - - // Local-path features are expected to be a sub-folder of the '$WORKSPACE_ROOT/.devcontainer' folder. - if (!configPath) { - output.write('A local feature requires a configuration path.', LogLevel.Error); - return undefined; - } - const featureFolderPath = path.join(path.dirname(configPath), userFeature.userFeatureId); - - // Ensure we aren't escaping .devcontainer folder - const parent = path.join(_workspaceRoot, '.devcontainer'); - const child = featureFolderPath; - const relative = path.relative(parent, child); - output.write(`${parent} -> ${child}: Relative Distance = '${relative}'`, LogLevel.Trace); - if (relative.indexOf('..') !== -1) { - output.write(`Local file path parse error. Resolved path must be a child of the .devcontainer/ folder. Parsed: ${featureFolderPath}`, LogLevel.Error); - return undefined; - } - - output.write(`Resolved: ${userFeature.userFeatureId} -> ${featureFolderPath}`, LogLevel.Trace); - - // -- All parsing and validation steps complete at this point. - - output.write(`Parsed feature id: ${id}`, LogLevel.Trace); - let feat: Feature = { - id, - name: userFeature.userFeatureId, - value: userFeature.options, - included: true, - }; - - let newFeaturesSet: FeatureSet = { - sourceInformation: { - type: 'file-path', - resolvedFilePath: featureFolderPath, - userFeatureId: originalUserFeatureId - }, - features: [feat], - }; - - return newFeaturesSet; - } - - // (6) Oci Identifier - if (type === 'oci' && manifest) { - return tryGetOCIFeatureSet(output, userFeature.userFeatureId, userFeature.options, manifest, originalUserFeatureId); - } - - output.write(`Github feature.`); - // Github repository source. - let version = 'latest'; - let splitOnAt = userFeature.userFeatureId.split('@'); - if (splitOnAt.length > 2) { - output.write(`Parse error. Use the '@' symbol only to designate a version tag.`, LogLevel.Error); - return undefined; - } - if (splitOnAt.length === 2) { - output.write(`[${userFeature.userFeatureId}] has version ${splitOnAt[1]}`, LogLevel.Trace); - version = splitOnAt[1]; - } - - // Remaining info must be in the first part of the split. - const featureBlob = splitOnAt[0]; - const splitOnSlash = featureBlob.split('/'); - // We expect all GitHub/registry features to follow the triple slash pattern at this point - // eg: // - if (splitOnSlash.length !== 3 || splitOnSlash.some(x => x === '') || !allowedFeatureIdRegex.test(splitOnSlash[2])) { - // This is the final fallback. If we end up here, we weren't able to resolve the Feature - output.write(`Could not resolve Feature '${userFeature.userFeatureId}'. Ensure the Feature is published and accessible from your current environment.`, LogLevel.Error); - return undefined; - } - const owner = splitOnSlash[0]; - const repo = splitOnSlash[1]; - const id = splitOnSlash[2]; - - let feat: Feature = { - id: id, - name: userFeature.userFeatureId, - value: userFeature.options, - included: true, - }; - - const userFeatureIdWithoutVersion = originalUserFeatureId.split('@')[0]; - if (version === 'latest') { - let newFeaturesSet: FeatureSet = { - sourceInformation: { - type: 'github-repo', - apiUri: `https://api.github.com/repos/${owner}/${repo}/releases/latest`, - unauthenticatedUri: `https://github.com/${owner}/${repo}/releases/latest/download`, // v1/v2 implementations append name of relevant asset - owner, - repo, - isLatest: true, - userFeatureId: originalUserFeatureId, - userFeatureIdWithoutVersion - }, - features: [feat], - }; - return newFeaturesSet; - } else { - // We must have a tag, return a tarball URI for the tagged version. - let newFeaturesSet: FeatureSet = { - sourceInformation: { - type: 'github-repo', - apiUri: `https://api.github.com/repos/${owner}/${repo}/releases/tags/${version}`, - unauthenticatedUri: `https://github.com/${owner}/${repo}/releases/download/${version}`, // v1/v2 implementations append name of relevant asset - owner, - repo, - tag: version, - isLatest: false, - userFeatureId: originalUserFeatureId, - userFeatureIdWithoutVersion - }, - features: [feat], - }; - return newFeaturesSet; - } - - // TODO: Handle invalid source types better by refactoring this function. - // throw new Error(`Unsupported feature source type: ${type}`); -} - -async function fetchFeatures(params: { extensionPath: string; cwd: string; output: Log; env: NodeJS.ProcessEnv }, featuresConfig: FeaturesConfig, dstFolder: string, ociCacheDir: string, lockfile: Lockfile | undefined) { - const featureSets = featuresConfig.featureSets; - for (let idx = 0; idx < featureSets.length; idx++) { // Index represents the previously computed installation order. - const featureSet = featureSets[idx]; - try { - if (!featureSet || !featureSet.features || !featureSet.sourceInformation) { - continue; - } - - const { output } = params; - - const feature = featureSet.features[0]; - const consecutiveId = `${feature.id}_${idx}`; - // Calculate some predictable caching paths. - const featCachePath = path.join(dstFolder, consecutiveId); - const sourceInfoType = featureSet.sourceInformation?.type; - - feature.cachePath = featCachePath; - feature.consecutiveId = consecutiveId; - - if (!feature.consecutiveId || !feature.id || !featureSet?.sourceInformation || !featureSet.sourceInformation.userFeatureId) { - const err = 'Internal Features error. Missing required attribute(s).'; - throw new Error(err); - } - - const featureDebugId = `${feature.consecutiveId}_${sourceInfoType}`; - output.write(`* Fetching feature: ${featureDebugId}`); - - if (sourceInfoType === 'oci') { - output.write(`Fetching from OCI`, LogLevel.Trace); - await mkdirpLocal(featCachePath); - const res = await fetchOCIFeature(params, featureSet, ociCacheDir, featCachePath); - if (!res) { - const err = `Could not download OCI feature: ${featureSet.sourceInformation.featureRef.id}`; - throw new Error(err); - } - - if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath, featureSet.sourceInformation.manifestDigest))) { - const err = `Failed to parse feature '${featureDebugId}'. Please check your devcontainer.json 'features' attribute.`; - throw new Error(err); - } - output.write(`* Fetched feature: ${featureDebugId} version ${feature.version}`); - - continue; - } - - if (sourceInfoType === 'file-path') { - output.write(`Detected local file path`, LogLevel.Trace); - await mkdirpLocal(featCachePath); - const executionPath = featureSet.sourceInformation.resolvedFilePath; - await cpDirectoryLocal(executionPath, featCachePath); - - if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath, undefined))) { - const err = `Failed to parse feature '${featureDebugId}'. Please check your devcontainer.json 'features' attribute.`; - throw new Error(err); - } - continue; - } - - output.write(`Detected tarball`, LogLevel.Trace); - const headers = getRequestHeaders(params, featureSet.sourceInformation); - - // Ordered list of tarballUris to attempt to fetch from. - let tarballUris: (string | { uri: string; digest?: string })[] = []; - - if (sourceInfoType === 'github-repo') { - output.write('Determining tarball URI for provided github repo.', LogLevel.Trace); - if (headers.Authorization && headers.Authorization !== '') { - output.write('GITHUB_TOKEN available. Attempting to fetch via GH API.', LogLevel.Info); - const authenticatedGithubTarballUri = await askGitHubApiForTarballUri(featureSet.sourceInformation, feature, headers, output); - - if (authenticatedGithubTarballUri) { - tarballUris.push(authenticatedGithubTarballUri); - } else { - output.write('Failed to generate autenticated tarball URI for provided feature, despite a GitHub token present', LogLevel.Warning); - } - headers.Accept = 'Accept: application/octet-stream'; - } - - // Always add the unauthenticated URIs as fallback options. - output.write('Appending unauthenticated URIs for v2 and then v1', LogLevel.Trace); - tarballUris.push(`${featureSet.sourceInformation.unauthenticatedUri}/${feature.id}.tgz`); - tarballUris.push(`${featureSet.sourceInformation.unauthenticatedUri}/${V1_ASSET_NAME}`); - - } else { - // We have a plain ol' tarball URI, since we aren't in the github-repo case. - const uri = featureSet.sourceInformation.tarballUri; - const digest = lockfile?.features[uri]?.integrity; - tarballUris.push({ uri, digest }); - } - - // Attempt to fetch from 'tarballUris' in order, until one succeeds. - let res: { computedDigest: string } | undefined; - for (const tarballUri of tarballUris) { - const uri = typeof tarballUri === 'string' ? tarballUri : tarballUri.uri; - const digest = typeof tarballUri === 'string' ? undefined : tarballUri.digest; - res = await fetchContentsAtTarballUri(params, uri, digest, featCachePath, headers, dstFolder); - - if (res) { - output.write(`Succeeded fetching ${uri}`, LogLevel.Trace); - if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath, res.computedDigest))) { - const err = `Failed to parse feature '${featureDebugId}'. Please check your devcontainer.json 'features' attribute.`; - throw new Error(err); - } - break; - } - } - - if (!res) { - const msg = `(!) Failed to fetch tarball for ${featureDebugId} after attempting ${tarballUris.length} possibilities.`; - throw new Error(msg); - } - } - catch (e) { - params.output.write(`(!) ERR: Failed to fetch feature: ${e?.message ?? ''} `, LogLevel.Error); - throw e; - } - } -} - -export async function fetchContentsAtTarballUri(params: { output: Log; env: NodeJS.ProcessEnv }, tarballUri: string, expectedDigest: string | undefined, featCachePath: string, headers: { 'user-agent': string; 'Authorization'?: string; 'Accept'?: string } | undefined, dstFolder: string, metadataFile?: string): Promise<{ computedDigest: string; metadata: {} | undefined } | undefined> { - const { output } = params; - const tempTarballPath = path.join(dstFolder, 'temp.tgz'); - try { - const options = { - type: 'GET', - url: tarballUri, - headers: headers ?? getRequestHeaders(params, { tarballUri, userFeatureId: tarballUri, type: 'direct-tarball' }) - }; - - output.write(`Fetching tarball at ${options.url}`); - output.write(`Headers: ${JSON.stringify(options)}`, LogLevel.Trace); - const tarball = await request(options, output); - - if (!tarball || tarball.length === 0) { - output.write(`Did not receive a response from tarball download URI: ${tarballUri}`, LogLevel.Trace); - return undefined; - } - - const computedDigest = `sha256:${crypto.createHash('sha256').update(tarball).digest('hex')}`; - if (expectedDigest && computedDigest !== expectedDigest) { - throw new Error(`Digest did not match for ${tarballUri}.`); - } - - // Filter what gets emitted from the tar.extract(). - const filter = (file: string, _: fs.Stats | tar.ReadEntry) => { - // Don't include .dotfiles or the archive itself. - if (file.startsWith('./.') || file === `./${V1_ASSET_NAME}` || file === './.') { - return false; - } - return true; - }; - - output.write(`Preparing to unarchive received tgz from ${tempTarballPath} -> ${featCachePath}.`, LogLevel.Trace); - // Create the directory to cache this feature-set in. - await mkdirpLocal(featCachePath); - await writeLocalFile(tempTarballPath, tarball); - await tar.x( - { - file: tempTarballPath, - cwd: featCachePath, - filter - } - ); - - // No 'metadataFile' to look for. - if (!metadataFile) { - await cleanupIterationFetchAndMerge(tempTarballPath, output); - return { computedDigest, metadata: undefined }; - } - - // Attempt to extract 'metadataFile' - await tar.x( - { - file: tempTarballPath, - cwd: featCachePath, - filter: (path, _) => { - return path === `./${metadataFile}`; - } - }); - const pathToMetadataFile = path.join(featCachePath, metadataFile); - let metadata = undefined; - if (await isLocalFile(pathToMetadataFile)) { - output.write(`Found metadata file '${metadataFile}' in tgz`, LogLevel.Trace); - metadata = jsonc.parse((await readLocalFile(pathToMetadataFile)).toString()); - } - - await cleanupIterationFetchAndMerge(tempTarballPath, output); - return { computedDigest, metadata }; - } catch (e) { - output.write(`Caught failure when fetching from URI '${tarballUri}': ${e}`, LogLevel.Trace); - await cleanupIterationFetchAndMerge(tempTarballPath, output); - return undefined; - } -} - -// Reads the feature's 'devcontainer-feature.json` and applies any attributes to the in-memory Feature object. -// NOTE: -// Implements the latest ('internalVersion' = '2') parsing logic, -// Falls back to earlier implementation(s) if requirements not present. -// Returns a boolean indicating whether the feature was successfully parsed. -async function applyFeatureConfigToFeature(output: Log, featureSet: FeatureSet, feature: Feature, featCachePath: string, computedDigest: string | undefined): Promise { - const innerJsonPath = path.join(featCachePath, DEVCONTAINER_FEATURE_FILE_NAME); - - if (!(await isLocalFile(innerJsonPath))) { - output.write(`Feature ${feature.id} is not a 'v2' feature. Attempting fallback to 'v1' implementation.`, LogLevel.Trace); - output.write(`For v2, expected devcontainer-feature.json at ${innerJsonPath}`, LogLevel.Trace); - return await parseDevContainerFeature_v1Impl(output, featureSet, feature, featCachePath); - } - - featureSet.internalVersion = '2'; - featureSet.computedDigest = computedDigest; - feature.cachePath = featCachePath; - const jsonString: Buffer = await readLocalFile(innerJsonPath); - const featureJson = jsonc.parse(jsonString.toString()); - - - feature = { - ...featureJson, - ...feature - }; - - featureSet.features[0] = updateFromOldProperties({ features: [feature] }).features[0]; - - return true; -} - -async function parseDevContainerFeature_v1Impl(output: Log, featureSet: FeatureSet, feature: Feature, featCachePath: string): Promise { - - const pathToV1DevContainerFeatureJson = path.join(featCachePath, V1_DEVCONTAINER_FEATURES_FILE_NAME); - - if (!(await isLocalFile(pathToV1DevContainerFeatureJson))) { - output.write(`Failed to find ${V1_DEVCONTAINER_FEATURES_FILE_NAME} metadata file (v1)`, LogLevel.Error); - return false; - } - featureSet.internalVersion = '1'; - feature.cachePath = featCachePath; - const jsonString: Buffer = await readLocalFile(pathToV1DevContainerFeatureJson); - const featureJson: FeatureSet = jsonc.parse(jsonString.toString()); - - const seekedFeature = featureJson?.features.find(f => f.id === feature.id); - if (!seekedFeature) { - output.write(`Failed to find feature '${feature.id}' in provided v1 metadata file`, LogLevel.Error); - return false; - } - - feature = { - ...seekedFeature, - ...feature - }; - - featureSet.features[0] = updateFromOldProperties({ features: [feature] }).features[0]; - - - return true; -} - -export function getFeatureMainProperty(feature: Feature) { - return feature.options?.version ? 'version' : undefined; -} - -export function getFeatureMainValue(feature: Feature) { - const defaultProperty = getFeatureMainProperty(feature); - if (!defaultProperty) { - return !!feature.value; - } - if (typeof feature.value === 'object') { - const value = feature.value[defaultProperty]; - if (value === undefined && feature.options) { - return feature.options[defaultProperty]?.default; - } - return value; - } - if (feature.value === undefined && feature.options) { - return feature.options[defaultProperty]?.default; - } - return feature.value; -} - -export function getFeatureValueObject(feature: Feature) { - if (typeof feature.value === 'object') { - return { - ...getFeatureValueDefaults(feature), - ...feature.value - }; - } - const mainProperty = getFeatureMainProperty(feature); - if (!mainProperty) { - return getFeatureValueDefaults(feature); - } - return { - ...getFeatureValueDefaults(feature), - [mainProperty]: feature.value, - }; -} - -function getFeatureValueDefaults(feature: Feature) { - const options = feature.options || {}; - return Object.keys(options) - .reduce((defaults, key) => { - if ('default' in options[key]) { - defaults[key] = options[key].default; - } - return defaults; - }, {} as Record); -} diff --git a/src/spec-configuration/containerFeaturesOCI.ts b/src/spec-configuration/containerFeaturesOCI.ts deleted file mode 100644 index 84fb869cf..000000000 --- a/src/spec-configuration/containerFeaturesOCI.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { Log, LogLevel } from '../spec-utils/log'; -import { Feature, FeatureSet } from './containerFeaturesConfiguration'; -import { CommonParams, fetchOCIManifestIfExists, getBlob, getRef, ManifestContainer } from './containerCollectionsOCI'; - -export function tryGetOCIFeatureSet(output: Log, identifier: string, options: boolean | string | Record, manifest: ManifestContainer, originalUserFeatureId: string): FeatureSet | undefined { - const featureRef = getRef(output, identifier); - if (!featureRef) { - output.write(`Unable to parse '${identifier}'`, LogLevel.Error); - return undefined; - } - - const feat: Feature = { - id: featureRef.id, - included: true, - value: options - }; - - const userFeatureIdWithoutVersion = getFeatureIdWithoutVersion(originalUserFeatureId); - let featureSet: FeatureSet = { - sourceInformation: { - type: 'oci', - manifest: manifest.manifestObj, - manifestDigest: manifest.contentDigest, - featureRef: featureRef, - userFeatureId: originalUserFeatureId, - userFeatureIdWithoutVersion - - }, - features: [feat], - }; - - return featureSet; -} - -const lastDelimiter = /[:@][^/]*$/; -export function getFeatureIdWithoutVersion(featureId: string) { - const m = lastDelimiter.exec(featureId); - return m ? featureId.substring(0, m.index) : featureId; -} - -export async function fetchOCIFeatureManifestIfExistsFromUserIdentifier(params: CommonParams, identifier: string, manifestDigest?: string): Promise { - const { output } = params; - - const featureRef = getRef(output, identifier); - if (!featureRef) { - return undefined; - } - return await fetchOCIManifestIfExists(params, featureRef, manifestDigest); -} - -// Download a feature from which a manifest was previously downloaded. -// Specification: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-blobs -export async function fetchOCIFeature(params: CommonParams, featureSet: FeatureSet, ociCacheDir: string, featCachePath: string, metadataFile?: string) { - const { output } = params; - - if (featureSet.sourceInformation.type !== 'oci') { - output.write(`FeatureSet is not an OCI featureSet.`, LogLevel.Error); - throw new Error('FeatureSet is not an OCI featureSet.'); - } - - const { featureRef } = featureSet.sourceInformation; - - const layerDigest = featureSet.sourceInformation.manifest?.layers[0].digest; - const blobUrl = `https://${featureSet.sourceInformation.featureRef.registry}/v2/${featureSet.sourceInformation.featureRef.path}/blobs/${layerDigest}`; - output.write(`blob url: ${blobUrl}`, LogLevel.Trace); - - const blobResult = await getBlob(params, blobUrl, ociCacheDir, featCachePath, featureRef, layerDigest, undefined, metadataFile); - - if (!blobResult) { - throw new Error(`Failed to download package for ${featureSet.sourceInformation.featureRef.resource}`); - } - - return blobResult; -} diff --git a/src/spec-configuration/containerFeaturesOrder.ts b/src/spec-configuration/containerFeaturesOrder.ts deleted file mode 100644 index 18f449a0a..000000000 --- a/src/spec-configuration/containerFeaturesOrder.ts +++ /dev/null @@ -1,706 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as path from 'path'; -import * as jsonc from 'jsonc-parser'; -import * as os from 'os'; -import * as crypto from 'crypto'; - -import { DEVCONTAINER_FEATURE_FILE_NAME, DirectTarballSourceInformation, Feature, FeatureSet, FilePathSourceInformation, OCISourceInformation, fetchContentsAtTarballUri } from '../spec-configuration/containerFeaturesConfiguration'; -import { LogLevel } from '../spec-utils/log'; -import { DevContainerFeature } from './configuration'; -import { CommonParams, OCIRef } from './containerCollectionsOCI'; -import { isLocalFile, readLocalFile } from '../spec-utils/pfs'; -import { fetchOCIFeature } from './containerFeaturesOCI'; -import { Lockfile } from './lockfile'; - -interface FNode { - type: 'user-provided' | 'override' | 'resolved'; - userFeatureId: string; - options: string | boolean | Record; - - // FeatureSet contains 'sourceInformation', useful for: - // Providing information on if Feature is an OCI Feature, Direct HTTPS Feature, or Local Feature. - // Additionally, contains 'ref' and 'manifestDigest' for OCI Features - useful for sorting. - // Property set programatically when discovering all the nodes in the graph. - featureSet?: FeatureSet; - - // Graph directed adjacency lists. - dependsOn: FNode[]; - installsAfter: FNode[]; - - // If a Feature was renamed, this property will contain: - // [, <...allPreviousIds>] - // See: https://containers.dev/implementors/features/#steps-to-rename-a-feature - // Eg: ['node', 'nodejs', 'nodejs-feature'] - featureIdAliases?: string[]; - - // Round Order Priority - // Effective value is always the max - roundPriority: number; -} - -interface DependencyGraph { - worklist: FNode[]; -} - -function equals(params: CommonParams, a: FNode, b: FNode): boolean { - const { output } = params; - - const aSourceInfo = a.featureSet?.sourceInformation; - let bSourceInfo = b.featureSet?.sourceInformation; // Mutable only for type-casting. - - if (!aSourceInfo || !bSourceInfo) { - output.write(`Missing sourceInfo: equals(${aSourceInfo?.userFeatureId}, ${bSourceInfo?.userFeatureId})`, LogLevel.Trace); - throw new Error('ERR: Failure resolving Features.'); - } - - if (aSourceInfo.type !== bSourceInfo.type) { - return false; - } - - return compareTo(params, a, b) === 0; -} - -function satisfiesSoftDependency(params: CommonParams, node: FNode, softDep: FNode): boolean { - const { output } = params; - - const nodeSourceInfo = node.featureSet?.sourceInformation; - let softDepSourceInfo = softDep.featureSet?.sourceInformation; // Mutable only for type-casting. - - if (!nodeSourceInfo || !softDepSourceInfo) { - output.write(`Missing sourceInfo: satisifiesSoftDependency(${nodeSourceInfo?.userFeatureId}, ${softDepSourceInfo?.userFeatureId})`, LogLevel.Trace); - throw new Error('ERR: Failure resolving Features.'); - } - - if (nodeSourceInfo.type !== softDepSourceInfo.type) { - return false; - } - - switch (nodeSourceInfo.type) { - case 'oci': - softDepSourceInfo = softDepSourceInfo as OCISourceInformation; - const nodeFeatureRef = nodeSourceInfo.featureRef; - const softDepFeatureRef = softDepSourceInfo.featureRef; - const softDepFeatureRefPrefix = `${softDepFeatureRef.registry}/${softDepFeatureRef.namespace}`; - - return nodeFeatureRef.resource === softDepFeatureRef.resource // Same resource - || softDep.featureIdAliases?.some(legacyId => `${softDepFeatureRefPrefix}/${legacyId}` === nodeFeatureRef.resource) // Handle 'legacyIds' - || false; - - case 'file-path': - softDepSourceInfo = softDepSourceInfo as FilePathSourceInformation; - return nodeSourceInfo.resolvedFilePath === softDepSourceInfo.resolvedFilePath; - - case 'direct-tarball': - softDepSourceInfo = softDepSourceInfo as DirectTarballSourceInformation; - return nodeSourceInfo.tarballUri === softDepSourceInfo.tarballUri; - - default: - // Legacy - const softDepId = softDepSourceInfo.userFeatureIdWithoutVersion || softDepSourceInfo.userFeatureId; - const nodeId = nodeSourceInfo.userFeatureIdWithoutVersion || nodeSourceInfo.userFeatureId; - return softDepId === nodeId; - - } -} - -function optionsCompareTo(a: string | boolean | Record, b: string | boolean | Record): number { - if (typeof a === 'string' && typeof b === 'string') { - return a.localeCompare(b); - } - - if (typeof a === 'boolean' && typeof b === 'boolean') { - return a === b ? 0 : a ? 1 : -1; - } - - if (typeof a === 'object' && typeof b === 'object') { - // Compare lengths - const aKeys = Object.keys(a); - const bKeys = Object.keys(b); - if (aKeys.length !== bKeys.length) { - return aKeys.length - bKeys.length; - } - - aKeys.sort(); - bKeys.sort(); - - for (let i = 0; i < aKeys.length; i++) { - // Compare keys - if (aKeys[i] !== bKeys[i]) { - return aKeys[i].localeCompare(bKeys[i]); - } - // Compare values - const aVal = a[aKeys[i]]; - const bVal = b[bKeys[i]]; - if (typeof aVal === 'string' && typeof bVal === 'string') { - const v = aVal.localeCompare(bVal); - if (v !== 0) { - return v; - } - } - if (typeof aVal === 'boolean' && typeof bVal === 'boolean') { - const v = aVal === bVal ? 0 : aVal ? 1 : -1; - if (v !== 0) { - return v; - } - } - if (typeof aVal === 'undefined' || typeof bVal === 'undefined') { - const v = aVal === bVal ? 0 : (aVal === undefined) ? 1 : -1; - if (v !== 0) { - return v; - } - } - } - // Object is piece-wise equal - return 0; - } - return (typeof a).localeCompare(typeof b); -} - -function ociResourceCompareTo(a: { featureRef: OCIRef; aliases?: string[] }, b: { featureRef: OCIRef; aliases?: string[] }): number { - - // Left Side - const aFeatureRef = a.featureRef; - const aRegistryAndNamespace = `${aFeatureRef.registry}/${aFeatureRef.namespace}`; - - // Right Side - const bFeatureRef = b.featureRef; - const bRegistryAndNamespace = `${bFeatureRef.registry}/${bFeatureRef.namespace}`; - - // If the registry+namespace are different, sort by them - if (aRegistryAndNamespace !== bRegistryAndNamespace) { - return aRegistryAndNamespace.localeCompare(bRegistryAndNamespace); - } - - let commonId: string | undefined = undefined; - // Determine if any permutation of the set of valid Ids are equal - // Prefer the the canonical/non-legacy Id. - // https://containers.dev/implementors/features/#steps-to-rename-a-feature - for (const aId of a.aliases || [aFeatureRef.id]) { - if (commonId) { - break; - } - for (const bId of b.aliases || [bFeatureRef.id]) { - if (aId === bId) { - commonId = aId; - break; - } - } - } - - if (!commonId) { - // Sort by canonical id - return aFeatureRef.id.localeCompare(bFeatureRef.id); - } - - // The (registry + namespace + id) are equal. - return 0; -} - -// If the two features are equal, return 0. -// If the sorting algorithm should place A _before_ B, return negative number. -// If the sorting algorithm should place A _after_ B, return positive number. -function compareTo(params: CommonParams, a: FNode, b: FNode): number { - const { output } = params; - - const aSourceInfo = a.featureSet?.sourceInformation; - let bSourceInfo = b.featureSet?.sourceInformation; // Mutable only for type-casting. - - if (!aSourceInfo || !bSourceInfo) { - output.write(`Missing sourceInfo: compareTo(${aSourceInfo?.userFeatureId}, ${bSourceInfo?.userFeatureId})`, LogLevel.Trace); - throw new Error('ERR: Failure resolving Features.'); - } - - if (aSourceInfo.type !== bSourceInfo.type) { - return aSourceInfo.userFeatureId.localeCompare(bSourceInfo.userFeatureId); - } - - switch (aSourceInfo.type) { - case 'oci': - bSourceInfo = bSourceInfo as OCISourceInformation; - - const aDigest = aSourceInfo.manifestDigest; - const bDigest = bSourceInfo.manifestDigest; - - // Short circuit if the digests and options are equal - if (aDigest === bDigest && optionsCompareTo(a.options, b.options) === 0) { - return 0; - } - - // Compare two OCI Features by their - // resource accounting for legacy id aliases - const ociResourceVal = ociResourceCompareTo( - { featureRef: aSourceInfo.featureRef, aliases: a.featureIdAliases }, - { featureRef: bSourceInfo.featureRef, aliases: b.featureIdAliases } - ); - - if (ociResourceVal !== 0) { - return ociResourceVal; - } - - const aTag = aSourceInfo.featureRef.tag; - const bTag = bSourceInfo.featureRef.tag; - // Sort by tags (if both have tags) - // Eg: 1.9.9, 2.0.0, 2.0.1, 3, latest - if ((aTag && bTag) && (aTag !== bTag)) { - return aTag.localeCompare(bTag); - } - - // Sort by options - const optionsVal = optionsCompareTo(a.options, b.options); - if (optionsVal !== 0) { - return optionsVal; - } - - // Sort by manifest digest hash - if (aDigest !== bDigest) { - return aDigest.localeCompare(bDigest); - } - - // Consider these two OCI Features equal. - return 0; - - case 'file-path': - bSourceInfo = bSourceInfo as FilePathSourceInformation; - const pathCompare = aSourceInfo.resolvedFilePath.localeCompare(bSourceInfo.resolvedFilePath); - if (pathCompare !== 0) { - return pathCompare; - } - return optionsCompareTo(a.options, b.options); - - case 'direct-tarball': - bSourceInfo = bSourceInfo as DirectTarballSourceInformation; - const urlCompare = aSourceInfo.tarballUri.localeCompare(bSourceInfo.tarballUri); - if (urlCompare !== 0) { - return urlCompare; - } - return optionsCompareTo(a.options, b.options); - - default: - // Legacy - const aId = aSourceInfo.userFeatureIdWithoutVersion || aSourceInfo.userFeatureId; - const bId = bSourceInfo.userFeatureIdWithoutVersion || bSourceInfo.userFeatureId; - const userIdCompare = aId.localeCompare(bId); - if (userIdCompare !== 0) { - return userIdCompare; - } - return optionsCompareTo(a.options, b.options); - } -} - -async function applyOverrideFeatureInstallOrder( - params: CommonParams, - processFeature: (userFeature: DevContainerFeature) => Promise, - worklist: FNode[], - config: { overrideFeatureInstallOrder?: string[] }, -) { - const { output } = params; - - if (!config.overrideFeatureInstallOrder) { - return worklist; - } - - // Create an override node for each Feature in the override property. - const originalLength = config.overrideFeatureInstallOrder.length; - for (let i = config.overrideFeatureInstallOrder.length - 1; i >= 0; i--) { - const overrideFeatureId = config.overrideFeatureInstallOrder[i]; - - // First element == N, last element == 1 - const roundPriority = originalLength - i; - - const tmpOverrideNode: FNode = { - type: 'override', - userFeatureId: overrideFeatureId, - options: {}, - roundPriority, - installsAfter: [], - dependsOn: [], - featureSet: undefined, - }; - - const processed = await processFeature(tmpOverrideNode); - if (!processed) { - throw new Error(`Feature '${tmpOverrideNode.userFeatureId}' in 'overrideFeatureInstallOrder' could not be processed.`); - } - - tmpOverrideNode.featureSet = processed; - - // Scan the worklist, incrementing the priority of each Feature that matches the override. - for (const node of worklist) { - if (satisfiesSoftDependency(params, node, tmpOverrideNode)) { - // Increase the priority of this node to install it sooner. - output.write(`[override]: '${node.userFeatureId}' has override priority of ${roundPriority}`, LogLevel.Trace); - node.roundPriority = Math.max(node.roundPriority, roundPriority); - } - } - } - - // Return the modified worklist. - return worklist; -} - -async function _buildDependencyGraph( - params: CommonParams, - processFeature: (userFeature: DevContainerFeature) => Promise, - worklist: FNode[], - acc: FNode[], - lockfile: Lockfile | undefined): Promise { - const { output } = params; - - while (worklist.length > 0) { - const current = worklist.shift()!; - - output.write(`Resolving Feature dependencies for '${current.userFeatureId}'...`, LogLevel.Info); - - const processedFeature = await processFeature(current); - if (!processedFeature) { - throw new Error(`ERR: Feature '${current.userFeatureId}' could not be processed. You may not have permission to access this Feature, or may not be logged in. If the issue persists, report this to the Feature author.`); - } - - // Set the processed FeatureSet object onto Node. - current.featureSet = processedFeature; - - // If the current Feature is already in the accumulator, skip it. - // This stops cycles but doesn't report them. - // Cycles/inconsistencies are thrown as errors in the next stage (rounds). - if (acc.some(f => equals(params, f, current))) { - continue; - } - - const type = processedFeature.sourceInformation.type; - let metadata: Feature | undefined; - // Switch on the source type of the provided Feature. - // Retrieving the metadata for the Feature (the contents of 'devcontainer-feature.json') - switch (type) { - case 'oci': - metadata = await getOCIFeatureMetadata(params, current); - break; - - case 'file-path': - const filePath = (current.featureSet.sourceInformation as FilePathSourceInformation).resolvedFilePath; - const metadataFilePath = path.join(filePath, DEVCONTAINER_FEATURE_FILE_NAME); - if (!isLocalFile(filePath)) { - throw new Error(`Metadata file '${metadataFilePath}' cannot be read for Feature '${current.userFeatureId}'.`); - } - const serialized = (await readLocalFile(metadataFilePath)).toString(); - if (serialized) { - metadata = jsonc.parse(serialized) as Feature; - } - break; - - case 'direct-tarball': - const tarballUri = (processedFeature.sourceInformation as DirectTarballSourceInformation).tarballUri; - const expectedDigest = lockfile?.features[tarballUri]?.integrity; - metadata = await getTgzFeatureMetadata(params, current, expectedDigest); - break; - - default: - // Legacy - // No dependency metadata to retrieve. - break; - } - - // Resolve dependencies given the current Feature's metadata. - if (metadata) { - current.featureSet.features[0] = { - ...current.featureSet.features[0], - ...metadata, - }; - - // Dependency-related properties - const dependsOn = metadata.dependsOn || {}; - const installsAfter = metadata.installsAfter || []; - - // Remember legacyIds - const legacyIds = (metadata.legacyIds || []); - const currentId = metadata.currentId || metadata.id; - current.featureIdAliases = [currentId, ...legacyIds]; - - // Add a new node for each 'dependsOn' dependency onto the 'current' node. - // **Add this new node to the worklist to process recursively** - for (const [userFeatureId, options] of Object.entries(dependsOn)) { - const dependency: FNode = { - type: 'resolved', - userFeatureId, - options, - featureSet: undefined, - dependsOn: [], - installsAfter: [], - roundPriority: 0, - }; - current.dependsOn.push(dependency); - worklist.push(dependency); - } - - // Add a new node for each 'installsAfter' soft-dependency onto the 'current' node. - // Soft-dependencies are NOT recursively processed - do *not* add to worklist. - for (const userFeatureId of installsAfter) { - const dependency: FNode = { - type: 'resolved', - userFeatureId, - options: {}, - featureSet: undefined, - dependsOn: [], - installsAfter: [], - roundPriority: 0, - }; - const processedFeatureSet = await processFeature(dependency); - if (!processedFeatureSet) { - throw new Error(`installsAfter dependency '${userFeatureId}' of Feature '${current.userFeatureId}' could not be processed.`); - } - - dependency.featureSet = processedFeatureSet; - - // Resolve and add all 'legacyIds' as aliases for the soft dependency relationship. - // https://containers.dev/implementors/features/#steps-to-rename-a-feature - const softDepMetadata = await getOCIFeatureMetadata(params, dependency); - if (softDepMetadata) { - const legacyIds = softDepMetadata.legacyIds || []; - const currentId = softDepMetadata.currentId || softDepMetadata.id; - dependency.featureIdAliases = [currentId, ...legacyIds]; - } - - current.installsAfter.push(dependency); - } - } - - acc.push(current); - } - - // Return the accumulated collection of dependencies. - return { - worklist: acc, - }; -} - -async function getOCIFeatureMetadata(params: CommonParams, node: FNode): Promise { - const { output } = params; - - // TODO: Implement a caching layer here! - // This can be optimized to share work done here - // with the 'fetchFeatures()` stage later on. - const srcInfo = node?.featureSet?.sourceInformation; - if (!node.featureSet || !srcInfo || srcInfo.type !== 'oci') { - return; - } - - const manifest = srcInfo.manifest; - const annotation = manifest?.annotations?.['dev.containers.metadata']; - - if (annotation) { - return jsonc.parse(annotation) as Feature; - } else { - // For backwards compatibility, - // If the metadata is not present on the manifest, we have to fetch the entire blob - // to extract the 'installsAfter' property. - // TODO: Cache this smarter to reuse later! - const tmp = path.join(os.tmpdir(), crypto.randomUUID()); - const f = await fetchOCIFeature(params, node.featureSet, tmp, tmp, DEVCONTAINER_FEATURE_FILE_NAME); - - if (f && f.metadata) { - return f.metadata as Feature; - } - } - output.write('No metadata found for Feature', LogLevel.Trace); - return; -} - -async function getTgzFeatureMetadata(params: CommonParams, node: FNode, expectedDigest: string | undefined): Promise { - const { output } = params; - - // TODO: Implement a caching layer here! - // This can be optimized to share work done here - // with the 'fetchFeatures()` stage later on. - const srcInfo = node?.featureSet?.sourceInformation; - if (!node.featureSet || !srcInfo || srcInfo.type !== 'direct-tarball') { - return; - } - - const tmp = path.join(os.tmpdir(), crypto.randomUUID()); - const result = await fetchContentsAtTarballUri(params, srcInfo.tarballUri, expectedDigest, tmp, undefined, tmp, DEVCONTAINER_FEATURE_FILE_NAME); - if (!result || !result.metadata) { - output.write(`No metadata for Feature '${node.userFeatureId}' from '${srcInfo.tarballUri}'`, LogLevel.Trace); - return; - } - - const metadata = result.metadata as Feature; - return metadata; - -} - -// Creates the directed acyclic graph (DAG) of Features and their dependencies. -export async function buildDependencyGraph( - params: CommonParams, - processFeature: (userFeature: DevContainerFeature) => Promise, - userFeatures: DevContainerFeature[], - config: { overrideFeatureInstallOrder?: string[] }, - lockfile: Lockfile | undefined): Promise { - - const { output } = params; - - const rootNodes = - userFeatures.map(f => { - return { - type: 'user-provided', // This Feature was provided by the user in the 'features' object of devcontainer.json. - userFeatureId: f.userFeatureId, - options: f.options, - dependsOn: [], - installsAfter: [], - roundPriority: 0, - }; - }); - - output.write(`[* user-provided] ${rootNodes.map(n => n.userFeatureId).join(', ')}`, LogLevel.Trace); - - const { worklist } = await _buildDependencyGraph(params, processFeature, rootNodes, [], lockfile); - - output.write(`[* resolved worklist] ${worklist.map(n => n.userFeatureId).join(', ')}`, LogLevel.Trace); - - // Apply the 'overrideFeatureInstallOrder' to the worklist. - if (config?.overrideFeatureInstallOrder) { - await applyOverrideFeatureInstallOrder(params, processFeature, worklist, config); - } - - return { worklist }; -} - -// Returns the ordered list of FeatureSets to fetch and install, or undefined on error. -export async function computeDependsOnInstallationOrder( - params: CommonParams, - processFeature: (userFeature: DevContainerFeature) => Promise, - userFeatures: DevContainerFeature[], - config: { overrideFeatureInstallOrder?: string[] }, - lockfile?: Lockfile, - precomputedGraph?: DependencyGraph): Promise { - - const { output } = params; - - // Build dependency graph and resolves all to FeatureSets. - const graph = precomputedGraph ?? await buildDependencyGraph(params, processFeature, userFeatures, config, lockfile); - if (!graph) { - return; - } - - const { worklist } = graph; - - if (worklist.length === 0) { - output.write('Zero length or undefined worklist.', LogLevel.Error); - return; - } - - output.write(`${JSON.stringify(worklist, null, 2)}`, LogLevel.Trace); - - // Sanity check - if (worklist.some(node => !node.featureSet)) { - output.write('Feature dependency worklist contains one or more undefined entries.', LogLevel.Error); - throw new Error(`ERR: Failure resolving Features.`); - } - - output.write(`[raw worklist]: ${worklist.map(n => n.userFeatureId).join(', ')}`, LogLevel.Trace); - - // For each node in the worklist, remove all 'soft-dependency' graph edges that are irrelevant - // i.e. the node is not a 'soft match' for any node in the worklist itself - for (let i = 0; i < worklist.length; i++) { - const node = worklist[i]; - // reverse iterate - for (let j = node.installsAfter.length - 1; j >= 0; j--) { - const softDep = node.installsAfter[j]; - if (!worklist.some(n => satisfiesSoftDependency(params, n, softDep))) { - output.write(`Soft-dependency '${softDep.userFeatureId}' is not required. Removing from installation order...`, LogLevel.Info); - // Delete that soft-dependency - node.installsAfter.splice(j, 1); - } - } - } - - output.write(`[worklist-without-dangling-soft-deps]: ${worklist.map(n => n.userFeatureId).join(', ')}`, LogLevel.Trace); - output.write('Starting round-based Feature install order calculation from worklist...', LogLevel.Trace); - - const installationOrder: FNode[] = []; - while (worklist.length > 0) { - const round = worklist.filter(node => - // If the node has no hard/soft dependencies, the node can always be installed. - (node.dependsOn.length === 0 && node.installsAfter.length === 0) - // OR, every hard-dependency (dependsOn) AND soft-dependency (installsAfter) has been satified in prior rounds - || node.dependsOn.every(dep => - installationOrder.some(installed => equals(params, installed, dep))) - && node.installsAfter.every(dep => - installationOrder.some(installed => satisfiesSoftDependency(params, installed, dep)))); - - output.write(`\n[round] ${round.map(r => r.userFeatureId).join(', ')}`, LogLevel.Trace); - if (round.length === 0) { - output.write('Circular dependency detected!', LogLevel.Error); - output.write(`Nodes remaining: ${worklist.map(n => n.userFeatureId!).join(', ')}`, LogLevel.Error); - return; - } - - output.write(`[round-candidates] ${round.map(r => `${r.userFeatureId} (${r.roundPriority})`).join(', ')}`, LogLevel.Trace); - - // Given the set of eligible nodes to install this round, - // determine the highest 'roundPriority' present of the nodes in this - // round, and exclude nodes from this round with a lower priority. - // This ensures that both: - // - The pre-computed graph derived from dependOn/installsAfter is honored - // - The overrideFeatureInstallOrder property (more generically, 'roundPriority') is honored - const maxRoundPriority = Math.max(...round.map(r => r.roundPriority)); - round.splice(0, round.length, ...round.filter(node => node.roundPriority === maxRoundPriority)); - output.write(`[round-after-filter-priority] (maxPriority=${maxRoundPriority}) ${round.map(r => `${r.userFeatureId} (${r.roundPriority})`).join(', ')}`, LogLevel.Trace); - - // Delete all nodes present in this round from the worklist. - worklist.splice(0, worklist.length, ...worklist.filter(node => !round.some(r => equals(params, r, node)))); - - // Sort rounds lexicographically by id. - round.sort((a, b) => compareTo(params, a, b)); - output.write(`[round-after-comparesTo] ${round.map(r => r.userFeatureId).join(', ')}`, LogLevel.Trace); - - // Commit round - installationOrder.push(...round); - } - - return installationOrder.map(n => n.featureSet!); -} - -// Pretty-print the calculated graph in the mermaid flowchart format. -// Viewable by copy-pasting the output string to a live editor, i.e: https://mermaid.live/ -export function generateMermaidDiagram(params: CommonParams, graph: FNode[]) { - // Output dependency graph in a mermaid flowchart format - const roots = graph?.filter(f => f.type === 'user-provided')!; - let str = 'flowchart\n'; - for (const root of roots) { - str += `${generateMermaidNode(root)}\n`; - str += `${generateMermaidSubtree(params, root, graph).reduce((p, c) => p + c + '\n', '')}`; - } - return str; -} - -function generateMermaidSubtree(params: CommonParams, current: FNode, worklist: FNode[], acc: string[] = []) { - for (const child of current.dependsOn) { - // For each corresponding member of the worklist that satisfies this hard-dependency - for (const w of worklist) { - if (equals(params, w, child)) { - acc.push(`${generateMermaidNode(current)} --> ${generateMermaidNode(w)}`); - } - } - generateMermaidSubtree(params, child, worklist, acc); - } - for (const softDep of current.installsAfter) { - // For each corresponding member of the worklist that satisfies this soft-dependency - for (const w of worklist) { - if (satisfiesSoftDependency(params, w, softDep)) { - acc.push(`${generateMermaidNode(current)} -.-> ${generateMermaidNode(w)}`); - } - } - generateMermaidSubtree(params, softDep, worklist, acc); - } - return acc; -} - -function generateMermaidNode(node: FNode) { - const hasher = crypto.createHash('sha256', { encoding: 'hex' }); - const hash = hasher.update(JSON.stringify(node)).digest('hex').slice(0, 6); - const aliases = node.featureIdAliases && node.featureIdAliases.length > 0 ? `
aliases: ${node.featureIdAliases.join(', ')}` : ''; - return `${hash}[${node.userFeatureId}
<${node.roundPriority}>${aliases}]`; -} \ No newline at end of file diff --git a/src/spec-configuration/containerTemplatesConfiguration.ts b/src/spec-configuration/containerTemplatesConfiguration.ts deleted file mode 100644 index 2020bac65..000000000 --- a/src/spec-configuration/containerTemplatesConfiguration.ts +++ /dev/null @@ -1,33 +0,0 @@ -export interface Template { - id: string; - version?: string; - name?: string; - description?: string; - documentationURL?: string; - licenseURL?: string; - type?: string; // Added programatically during packaging - fileCount?: number; // Added programatically during packaging - featureIds?: string[]; - options?: Record; - platforms?: string[]; - publisher?: string; - keywords?: string[]; - optionalPaths?: string[]; - files: string[]; // Added programatically during packaging -} - -export type TemplateOption = { - type: 'boolean'; - default?: boolean; - description?: string; -} | { - type: 'string'; - enum?: string[]; - default?: string; - description?: string; -} | { - type: 'string'; - default?: string; - proposals?: string[]; - description?: string; -}; diff --git a/src/spec-configuration/containerTemplatesOCI.ts b/src/spec-configuration/containerTemplatesOCI.ts deleted file mode 100644 index 4c5c27755..000000000 --- a/src/spec-configuration/containerTemplatesOCI.ts +++ /dev/null @@ -1,177 +0,0 @@ -import { Log, LogLevel } from '../spec-utils/log'; -import * as os from 'os'; -import * as path from 'path'; -import * as jsonc from 'jsonc-parser'; -import { CommonParams, fetchOCIManifestIfExists, getBlob, getRef, ManifestContainer } from './containerCollectionsOCI'; -import { isLocalFile, readLocalFile, writeLocalFile } from '../spec-utils/pfs'; -import { DevContainerConfig } from './configuration'; -import { Template } from './containerTemplatesConfiguration'; - -export interface TemplateOptions { - [name: string]: string; -} -export interface TemplateFeatureOption { - id: string; - options: Record; -} - -export interface SelectedTemplate { - id: string; - options: TemplateOptions; - features: TemplateFeatureOption[]; - omitPaths: string[]; -} - -export async function fetchTemplate(params: CommonParams, selectedTemplate: SelectedTemplate, templateDestPath: string, userProvidedTmpDir?: string): Promise { - const { output } = params; - - let { id: userSelectedId, options: userSelectedOptions, omitPaths } = selectedTemplate; - const templateRef = getRef(output, userSelectedId); - if (!templateRef) { - output.write(`Failed to parse template ref for ${userSelectedId}`, LogLevel.Error); - return; - } - - const ociManifest = await fetchOCITemplateManifestIfExistsFromUserIdentifier(params, userSelectedId); - if (!ociManifest) { - output.write(`Failed to fetch template manifest for ${userSelectedId}`, LogLevel.Error); - return; - } - const blobDigest = ociManifest?.manifestObj?.layers[0]?.digest; - if (!blobDigest) { - output.write(`Failed to fetch template manifest for ${userSelectedId}`, LogLevel.Error); - return; - } - - const blobUrl = `https://${templateRef.registry}/v2/${templateRef.path}/blobs/${blobDigest}`; - output.write(`blob url: ${blobUrl}`, LogLevel.Trace); - - const tmpDir = userProvidedTmpDir || path.join(os.tmpdir(), 'vsch-template-temp', `${Date.now()}`); - const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, blobDigest, [...omitPaths, 'devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json'); - - if (!blobResult) { - output.write(`Failed to download package for ${templateRef.resource}`, LogLevel.Error); - return; - } - - const { files, metadata } = blobResult; - - // Auto-replace default values for values not provided by user. - if (metadata) { - const templateMetadata = metadata as Template; - if (templateMetadata.options) { - const templateOptions = templateMetadata.options; - for (const templateOptionKey of Object.keys(templateOptions)) { - if (userSelectedOptions[templateOptionKey] === undefined) { - // If the user didn't provide a value for this option, use the default if there is one in the extracted metadata. - const templateOption = templateOptions[templateOptionKey]; - - if (templateOption.type === 'string') { - const _default = templateOption.default; - if (_default) { - output.write(`Using default value for ${templateOptionKey} --> ${_default}`, LogLevel.Trace); - userSelectedOptions[templateOptionKey] = _default; - } - } - else if (templateOption.type === 'boolean') { - const _default = templateOption.default; - if (_default) { - output.write(`Using default value for ${templateOptionKey} --> ${_default}`, LogLevel.Trace); - userSelectedOptions[templateOptionKey] = _default.toString(); - } - } - } - } - } - } - - // Scan all template files and replace any templated values. - for (const f of files) { - output.write(`Scanning file '${f}'`, LogLevel.Trace); - const filePath = path.join(templateDestPath, f); - if (await isLocalFile(filePath)) { - const fileContents = await readLocalFile(filePath); - const fileContentsReplaced = replaceTemplatedValues(output, fileContents.toString(), userSelectedOptions); - await writeLocalFile(filePath, Buffer.from(fileContentsReplaced)); - } else { - output.write(`Could not find templated file '${f}'.`, LogLevel.Error); - } - } - - // Get the config. A template should not have more than one devcontainer.json. - const config = async (files: string[]) => { - const p = files.find(f => f.endsWith('devcontainer.json')); - if (p) { - const configPath = path.join(templateDestPath, p); - if (await isLocalFile(configPath)) { - const configContents = await readLocalFile(configPath); - return { - configPath, - configText: configContents.toString(), - configObject: jsonc.parse(configContents.toString()) as DevContainerConfig, - }; - } - } - return undefined; - }; - - if (selectedTemplate.features.length !== 0) { - const configResult = await config(files); - if (configResult) { - await addFeatures(output, selectedTemplate.features, configResult); - } else { - output.write(`Could not find a devcontainer.json to apply selected Features onto.`, LogLevel.Error); - } - } - - return files; -} - - -async function fetchOCITemplateManifestIfExistsFromUserIdentifier(params: CommonParams, identifier: string, manifestDigest?: string): Promise { - const { output } = params; - - const templateRef = getRef(output, identifier); - if (!templateRef) { - return undefined; - } - return await fetchOCIManifestIfExists(params, templateRef, manifestDigest); -} - -function replaceTemplatedValues(output: Log, template: string, options: TemplateOptions) { - const pattern = /\${templateOption:\s*(\w+?)\s*}/g; // ${templateOption:XXXX} - return template.replace(pattern, (_, token) => { - output.write(`Replacing ${token} with ${options[token]}`); - return options[token] || ''; - }); -} - -async function addFeatures(output: Log, newFeatures: TemplateFeatureOption[], configResult: { configPath: string; configText: string; configObject: DevContainerConfig }) { - const { configPath, configText, configObject } = configResult; - if (newFeatures) { - let previousText = configText; - let updatedText = configText; - - // Add the features property if it doesn't exist. - if (!configObject.features) { - const edits = jsonc.modify(updatedText, ['features'], {}, { formattingOptions: {} }); - updatedText = jsonc.applyEdits(updatedText, edits); - } - - for (const newFeature of newFeatures) { - let edits: jsonc.Edit[] = []; - const propertyPath = ['features', newFeature.id]; - - edits = edits.concat( - jsonc.modify(updatedText, propertyPath, newFeature.options ?? {}, { formattingOptions: {} } - )); - - updatedText = jsonc.applyEdits(updatedText, edits); - } - - if (previousText !== updatedText) { - output.write(`Updating ${configPath} with ${newFeatures.length} Features`, LogLevel.Trace); - await writeLocalFile(configPath, Buffer.from(updatedText)); - } - } -} \ No newline at end of file diff --git a/src/spec-configuration/controlManifest.ts b/src/spec-configuration/controlManifest.ts deleted file mode 100644 index a75688e6d..000000000 --- a/src/spec-configuration/controlManifest.ts +++ /dev/null @@ -1,110 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - *--------------------------------------------------------------------------------------------*/ - -import { promises as fs } from 'fs'; -import * as path from 'path'; -import * as jsonc from 'jsonc-parser'; - -import { request } from '../spec-utils/httpRequest'; -import * as crypto from 'crypto'; -import { Log, LogLevel } from '../spec-utils/log'; - -export interface DisallowedFeature { - featureIdPrefix: string; - documentationURL?: string; -} - -export interface FeatureAdvisory { - featureId: string; - introducedInVersion: string; - fixedInVersion: string; - description: string; - documentationURL?: string; - -} - -export interface DevContainerControlManifest { - disallowedFeatures: DisallowedFeature[]; - featureAdvisories: FeatureAdvisory[]; -} - -const controlManifestFilename = 'control-manifest.json'; - -const emptyControlManifest: DevContainerControlManifest = { - disallowedFeatures: [], - featureAdvisories: [], -}; - -const cacheTimeoutMillis = 5 * 60 * 1000; // 5 minutes - -export async function getControlManifest(cacheFolder: string, output: Log): Promise { - const controlManifestPath = path.join(cacheFolder, controlManifestFilename); - const cacheStat = await fs.stat(controlManifestPath) - .catch(err => { - if (err?.code !== 'ENOENT') { - throw err; - } - }); - const cacheBuffer = cacheStat?.isFile() ? await fs.readFile(controlManifestPath) - .catch(err => { - if (err?.code !== 'ENOENT') { - throw err; - } - }) : undefined; - const cachedManifest = cacheBuffer ? sanitizeControlManifest(jsonc.parse(cacheBuffer.toString())) : undefined; - if (cacheStat && cachedManifest && cacheStat.mtimeMs + cacheTimeoutMillis > Date.now()) { - return cachedManifest; - } - return updateControlManifest(controlManifestPath, cachedManifest, output); -} - -async function updateControlManifest(controlManifestPath: string, oldManifest: DevContainerControlManifest | undefined, output: Log): Promise { - let manifestBuffer: Buffer; - try { - manifestBuffer = await fetchControlManifest(output); - } catch (error) { - output.write(`Failed to fetch control manifest: ${error.message}`, LogLevel.Error); - if (oldManifest) { - // Keep old manifest to not lose existing information and update timestamp to avoid flooding the server. - const now = new Date(); - await fs.utimes(controlManifestPath, now, now); - return oldManifest; - } - manifestBuffer = Buffer.from(JSON.stringify(emptyControlManifest, undefined, 2)); - } - - const controlManifestTmpPath = `${controlManifestPath}-${crypto.randomUUID()}`; - await fs.mkdir(path.dirname(controlManifestPath), { recursive: true }); - await fs.writeFile(controlManifestTmpPath, manifestBuffer); - await fs.rename(controlManifestTmpPath, controlManifestPath); - return sanitizeControlManifest(jsonc.parse(manifestBuffer.toString())); -} - -async function fetchControlManifest(output: Log) { - return request({ - type: 'GET', - url: 'https://containers.dev/static/devcontainer-control-manifest.json', - headers: { - 'user-agent': 'devcontainers-vscode', - 'accept': 'application/json', - }, - }, output); -} - -function sanitizeControlManifest(manifest: any): DevContainerControlManifest { - if (!manifest || typeof manifest !== 'object') { - return emptyControlManifest; - } - const disallowedFeatures = manifest.disallowedFeatures as DisallowedFeature[] | undefined; - const featureAdvisories = manifest.featureAdvisories as FeatureAdvisory[] | undefined; - return { - disallowedFeatures: Array.isArray(disallowedFeatures) ? disallowedFeatures.filter(f => typeof f.featureIdPrefix === 'string') : [], - featureAdvisories: Array.isArray(featureAdvisories) ? featureAdvisories.filter(f => - typeof f.featureId === 'string' && - typeof f.introducedInVersion === 'string' && - typeof f.fixedInVersion === 'string' && - typeof f.description === 'string' - ) : [], - }; -} diff --git a/src/spec-configuration/editableFiles.ts b/src/spec-configuration/editableFiles.ts deleted file mode 100644 index d233a15fc..000000000 --- a/src/spec-configuration/editableFiles.ts +++ /dev/null @@ -1,163 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as crypto from 'crypto'; -import * as jsonc from 'jsonc-parser'; -import { URI } from 'vscode-uri'; -import { uriToFsPath, FileHost } from './configurationCommonUtils'; -import { readLocalFile, writeLocalFile } from '../spec-utils/pfs'; - -export type Edit = jsonc.Edit; - -export interface Documents { - readDocument(uri: URI): Promise; - applyEdits(uri: URI, edits: Edit[], content: string): Promise; -} - -export const fileDocuments: Documents = { - - async readDocument(uri: URI) { - switch (uri.scheme) { - case 'file': - try { - const buffer = await readLocalFile(uri.fsPath); - return buffer.toString(); - } catch (err) { - if (err && err.code === 'ENOENT') { - return undefined; - } - throw err; - } - default: - throw new Error(`Unsupported scheme: ${uri.toString()}`); - } - }, - - async applyEdits(uri: URI, edits: Edit[], content: string) { - switch (uri.scheme) { - case 'file': - const result = jsonc.applyEdits(content, edits); - await writeLocalFile(uri.fsPath, result); - break; - default: - throw new Error(`Unsupported scheme: ${uri.toString()}`); - } - } -}; - -export class CLIHostDocuments implements Documents { - - static scheme = 'vscode-fileHost'; - - constructor(private fileHost: FileHost) { - } - - async readDocument(uri: URI) { - switch (uri.scheme) { - case CLIHostDocuments.scheme: - try { - return (await this.fileHost.readFile(uriToFsPath(uri, this.fileHost.platform))).toString(); - } catch (err) { - return undefined; - } - default: - throw new Error(`Unsupported scheme: ${uri.toString()}`); - } - } - - async applyEdits(uri: URI, edits: Edit[], content: string) { - switch (uri.scheme) { - case CLIHostDocuments.scheme: - const result = jsonc.applyEdits(content, edits); - await this.fileHost.writeFile(uriToFsPath(uri, this.fileHost.platform), Buffer.from(result)); - break; - default: - throw new Error(`Unsupported scheme: ${uri.toString()}`); - } - } -} - -export class RemoteDocuments implements Documents { - - static scheme = 'vscode-remote'; - - private static nonce: string | undefined; - - constructor(private shellServer: ShellServer) { - } - - async readDocument(uri: URI) { - switch (uri.scheme) { - case RemoteDocuments.scheme: - try { - const { stdout } = await this.shellServer.exec(`cat ${uri.path}`); - return stdout; - } catch (err) { - return undefined; - } - default: - throw new Error(`Unsupported scheme: ${uri.toString()}`); - } - } - - async applyEdits(uri: URI, edits: Edit[], content: string) { - switch (uri.scheme) { - case RemoteDocuments.scheme: - try { - if (!RemoteDocuments.nonce) { - RemoteDocuments.nonce = crypto.randomUUID(); - } - const result = jsonc.applyEdits(content, edits); - const eof = `EOF-${RemoteDocuments.nonce}`; - await this.shellServer.exec(`cat <<'${eof}' >${uri.path} -${result} -${eof} -`); - } catch (err) { - console.log(err); // XXX - } - break; - default: - throw new Error(`Unsupported scheme: ${uri.toString()}`); - } - } -} - -export class AllDocuments implements Documents { - - constructor(private documents: Record) { - } - - async readDocument(uri: URI) { - const documents = this.documents[uri.scheme]; - if (!documents) { - throw new Error(`Unsupported scheme: ${uri.toString()}`); - } - return documents.readDocument(uri); - } - - async applyEdits(uri: URI, edits: Edit[], content: string) { - const documents = this.documents[uri.scheme]; - if (!documents) { - throw new Error(`Unsupported scheme: ${uri.toString()}`); - } - return documents.applyEdits(uri, edits, content); - } -} - -export function createDocuments(fileHost: FileHost, shellServer?: ShellServer): Documents { - const documents: Record = { - file: fileDocuments, - [CLIHostDocuments.scheme]: new CLIHostDocuments(fileHost), - }; - if (shellServer) { - documents[RemoteDocuments.scheme] = new RemoteDocuments(shellServer); - } - return new AllDocuments(documents); -} - -export interface ShellServer { - exec(cmd: string, options?: { logOutput?: boolean; stdin?: Buffer }): Promise<{ stdout: string; stderr: string }>; -} diff --git a/src/spec-configuration/featureAdvisories.ts b/src/spec-configuration/featureAdvisories.ts deleted file mode 100644 index 3eafa984a..000000000 --- a/src/spec-configuration/featureAdvisories.ts +++ /dev/null @@ -1,93 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { FeatureSet, FeaturesConfig, OCISourceInformation } from './containerFeaturesConfiguration'; -import { FeatureAdvisory, getControlManifest } from './controlManifest'; -import { parseVersion, isEarlierVersion } from '../spec-common/commonUtils'; -import { Log, LogLevel } from '../spec-utils/log'; - -export async function fetchFeatureAdvisories(params: { cacheFolder: string; output: Log }, featuresConfig: FeaturesConfig) { - - const features = featuresConfig.featureSets - .map(f => [f, f.sourceInformation] as const) - .filter((tup): tup is [FeatureSet, OCISourceInformation] => tup[1].type === 'oci') - .map(([set, source]) => ({ - id: `${source.featureRef.registry}/${source.featureRef.path}`, - version: set.features[0].version!, - })) - .sort((a, b) => a.id.localeCompare(b.id)); - if (!features.length) { - return []; - } - - const controlManifest = await getControlManifest(params.cacheFolder, params.output); - if (!controlManifest.featureAdvisories.length) { - return []; - } - - const featureAdvisories = controlManifest.featureAdvisories.reduce((acc, cur) => { - const list = acc.get(cur.featureId); - if (list) { - list.push(cur); - } else { - acc.set(cur.featureId, [cur]); - } - return acc; - }, new Map()); - - const parsedVersions = new Map(); - function lookupParsedVersion(version: string) { - if (!parsedVersions.has(version)) { - parsedVersions.set(version, parseVersion(version)); - } - return parsedVersions.get(version); - } - const featuresWithAdvisories = features.map(feature => { - const advisories = featureAdvisories.get(feature.id); - const featureVersion = lookupParsedVersion(feature.version); - if (!featureVersion) { - params.output.write(`Unable to parse version for feature ${feature.id}: ${feature.version}`, LogLevel.Warning); - return { - feature, - advisories: [], - }; - } - return { - feature, - advisories: advisories?.filter(advisory => { - const introducedInVersion = lookupParsedVersion(advisory.introducedInVersion); - const fixedInVersion = lookupParsedVersion(advisory.fixedInVersion); - if (!introducedInVersion || !fixedInVersion) { - return false; - } - return !isEarlierVersion(featureVersion, introducedInVersion) && isEarlierVersion(featureVersion, fixedInVersion); - }) || [], - }; - }).filter(f => f.advisories.length); - - return featuresWithAdvisories; -} - -export async function logFeatureAdvisories(params: { cacheFolder: string; output: Log }, featuresConfig: FeaturesConfig) { - - const featuresWithAdvisories = await fetchFeatureAdvisories(params, featuresConfig); - if (!featuresWithAdvisories.length) { - return; - } - - params.output.write(` - ------------------------------------------------------------------------------------------------------------ -FEATURE ADVISORIES:${featuresWithAdvisories.map(f => ` -- ${f.feature.id}:${f.feature.version}:${f.advisories.map(a => ` - - ${a.description} (introduced in ${a.introducedInVersion}, fixed in ${a.fixedInVersion}${a.documentationURL ? `, see ${a.documentationURL}` : ''})`) - .join('')}`) -.join('')} - -It is recommended that you update your configuration to versions of these features with the fixes applied. ------------------------------------------------------------------------------------------------------------ - -`, LogLevel.Warning); -} diff --git a/src/spec-configuration/httpOCIRegistry.ts b/src/spec-configuration/httpOCIRegistry.ts deleted file mode 100644 index 2bebba82e..000000000 --- a/src/spec-configuration/httpOCIRegistry.ts +++ /dev/null @@ -1,431 +0,0 @@ -import * as os from 'os'; -import * as path from 'path'; -import * as jsonc from 'jsonc-parser'; - -import { runCommandNoPty, plainExec } from '../spec-common/commonUtils'; -import { requestResolveHeaders } from '../spec-utils/httpRequest'; -import { LogLevel } from '../spec-utils/log'; -import { isLocalFile, readLocalFile } from '../spec-utils/pfs'; -import { CommonParams, OCICollectionRef, OCIRef } from './containerCollectionsOCI'; - -export type HEADERS = { 'authorization'?: string; 'user-agent'?: string; 'content-type'?: string; 'Accept'?: string; 'content-length'?: string }; - -interface DockerConfigFile { - auths: { - [registry: string]: { - auth: string; - identitytoken?: string; // Used by Azure Container Registry - }; - }; - credHelpers: { - [registry: string]: string; - }; - credsStore: string; -} - -interface CredentialHelperResult { - Username: string; - Secret: string; -} - -// WWW-Authenticate Regex -// realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:samalba/my-app:pull,push" -// realm="https://ghcr.io/token",service="ghcr.io",scope="repository:devcontainers/features:pull" -const realmRegex = /realm="([^"]+)"/; -const serviceRegex = /service="([^"]+)"/; -const scopeRegex = /scope="([^"]+)"/; - -// https://docs.docker.com/registry/spec/auth/token/#how-to-authenticate -export async function requestEnsureAuthenticated(params: CommonParams, httpOptions: { type: string; url: string; headers: HEADERS; data?: Buffer }, ociRef: OCIRef | OCICollectionRef) { - // If needed, Initialize the Authorization header cache. - if (!params.cachedAuthHeader) { - params.cachedAuthHeader = {}; - } - const { output, cachedAuthHeader } = params; - - // -- Update headers - httpOptions.headers['user-agent'] = 'devcontainer'; - // If the user has a cached auth token, attempt to use that first. - const maybeCachedAuthHeader = cachedAuthHeader[ociRef.registry]; - if (maybeCachedAuthHeader) { - output.write(`[httpOci] Applying cachedAuthHeader for registry ${ociRef.registry}...`, LogLevel.Trace); - httpOptions.headers.authorization = maybeCachedAuthHeader; - } - - const initialAttemptRes = await requestResolveHeaders(httpOptions, output); - - // For anything except a 401 (invalid/no token) or 403 (insufficient scope) - // response simply return the original response to the caller. - if (initialAttemptRes.statusCode !== 401 && initialAttemptRes.statusCode !== 403) { - output.write(`[httpOci] ${initialAttemptRes.statusCode} (${maybeCachedAuthHeader ? 'Cached' : 'NoAuth'}): ${httpOptions.url}`, LogLevel.Trace); - return initialAttemptRes; - } - - // -- 'responseAttempt' status code was 401 or 403 at this point. - - // Attempt to authenticate via WWW-Authenticate Header. - const wwwAuthenticate = initialAttemptRes.resHeaders['WWW-Authenticate'] || initialAttemptRes.resHeaders['www-authenticate']; - if (!wwwAuthenticate) { - output.write(`[httpOci] ERR: Server did not provide instructions to authentiate! (Required: A 'WWW-Authenticate' Header)`, LogLevel.Error); - return; - } - - const authenticationMethod = wwwAuthenticate.split(' ')[0]; - switch (authenticationMethod.toLowerCase()) { - // Basic realm="localhost" - case 'basic': - - output.write(`[httpOci] Attempting to authenticate via 'Basic' auth.`, LogLevel.Trace); - - const credential = await getCredential(params, ociRef); - const basicAuthCredential = credential?.base64EncodedCredential; - if (!basicAuthCredential) { - output.write(`[httpOci] ERR: No basic auth credentials to send for registry service '${ociRef.registry}'`, LogLevel.Error); - return; - } - - httpOptions.headers.authorization = `Basic ${basicAuthCredential}`; - break; - - // Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:samalba/my-app:pull,push" - case 'bearer': - - output.write(`[httpOci] Attempting to authenticate via 'Bearer' auth.`, LogLevel.Trace); - - const realmGroup = realmRegex.exec(wwwAuthenticate); - const serviceGroup = serviceRegex.exec(wwwAuthenticate); - const scopeGroup = scopeRegex.exec(wwwAuthenticate); - - if (!realmGroup || !serviceGroup) { - output.write(`[httpOci] WWW-Authenticate header is not in expected format. Got: ${wwwAuthenticate}`, LogLevel.Trace); - return; - } - - const wwwAuthenticateData = { - realm: realmGroup[1], - service: serviceGroup[1], - scope: scopeGroup ? scopeGroup[1] : '', - }; - - const bearerToken = await fetchRegistryBearerToken(params, ociRef, wwwAuthenticateData); - if (!bearerToken) { - output.write(`[httpOci] ERR: Failed to fetch Bearer token from registry.`, LogLevel.Error); - return; - } - - httpOptions.headers.authorization = `Bearer ${bearerToken}`; - break; - - default: - output.write(`[httpOci] ERR: Unsupported authentication mode '${authenticationMethod}'`, LogLevel.Error); - return; - } - - // Retry the request with the updated authorization header. - const reattemptRes = await requestResolveHeaders(httpOptions, output); - output.write(`[httpOci] ${reattemptRes.statusCode} on reattempt after auth: ${httpOptions.url}`, LogLevel.Trace); - - // Cache the auth header if the request did not result in an unauthorized response. - if (reattemptRes.statusCode !== 401) { - params.cachedAuthHeader[ociRef.registry] = httpOptions.headers.authorization; - } - - return reattemptRes; -} - -// Attempts to get the Basic auth credentials for the provided registry. -// This credential is used to offer the registry in exchange for a Bearer token. -// These may be: -// - parsed out of a special DEVCONTAINERS_OCI_AUTH environment variable -// - Read from a docker credential helper (https://docs.docker.com/engine/reference/commandline/login/#credentials-store) -// - Read from a docker config file -// - Crafted from the GITHUB_TOKEN environment variable -// Returns: -// - undefined: No credential was found. -// - object: A credential was found. -// - based64EncodedCredential: The base64 encoded credential, if any. -// - refreshToken: The refresh token, if any. -async function getCredential(params: CommonParams, ociRef: OCIRef | OCICollectionRef): Promise<{ base64EncodedCredential: string | undefined; refreshToken: string | undefined } | undefined> { - const { output, env } = params; - const { registry } = ociRef; - - if (!!env['DEVCONTAINERS_OCI_AUTH']) { - // eg: DEVCONTAINERS_OCI_AUTH=service1|user1|token1,service2|user2|token2 - const authContexts = env['DEVCONTAINERS_OCI_AUTH'].split(','); - const authContext = authContexts.find(a => a.split('|')[0] === registry); - - if (authContext) { - output.write(`[httpOci] Using match from DEVCONTAINERS_OCI_AUTH for registry '${registry}'`, LogLevel.Trace); - const split = authContext.split('|'); - const userToken = `${split[1]}:${split[2]}`; - return { - base64EncodedCredential: Buffer.from(userToken).toString('base64'), - refreshToken: undefined, - }; - } - } - - // Attempt to use the docker config file or available credential helper(s). - const credentialFromDockerConfig = await getCredentialFromDockerConfigOrCredentialHelper(params, registry); - if (credentialFromDockerConfig) { - return credentialFromDockerConfig; - } - - const githubToken = env['GITHUB_TOKEN']; - const githubHost = env['GITHUB_HOST']; - if (githubHost) { - output.write(`[httpOci] Environment GITHUB_HOST is set to '${githubHost}'`, LogLevel.Trace); - } - if (registry === 'ghcr.io' && githubToken && (!githubHost || githubHost === 'github.com')) { - output.write('[httpOci] Using environment GITHUB_TOKEN for auth', LogLevel.Trace); - const userToken = `USERNAME:${env['GITHUB_TOKEN']}`; - return { - base64EncodedCredential: Buffer.from(userToken).toString('base64'), - refreshToken: undefined, - }; - } - - // Represents anonymous access. - output.write(`[httpOci] No authentication credentials found for registry '${registry}'. Accessing anonymously.`, LogLevel.Trace); - return; -} - -async function existsInPath(filename: string): Promise { - if (!process.env.PATH) { - return false; - } - const paths = process.env.PATH.split(':'); - for (const path of paths) { - const fullPath = `${path}/${filename}`; - if (await isLocalFile(fullPath)) { - return true; - } - } - return false; -} - -async function getCredentialFromDockerConfigOrCredentialHelper(params: CommonParams, registry: string) { - const { output } = params; - - let configContainsAuth = false; - try { - // https://docs.docker.com/engine/reference/commandline/cli/#change-the-docker-directory - const customDockerConfigPath = process.env.DOCKER_CONFIG; - if (customDockerConfigPath) { - output.write(`[httpOci] Environment DOCKER_CONFIG is set to '${customDockerConfigPath}'`, LogLevel.Trace); - } - const dockerConfigRootDir = customDockerConfigPath || path.join(os.homedir(), '.docker'); - const dockerConfigFilePath = path.join(dockerConfigRootDir, 'config.json'); - if (await isLocalFile(dockerConfigFilePath)) { - const dockerConfig: DockerConfigFile = jsonc.parse((await readLocalFile(dockerConfigFilePath)).toString()); - - configContainsAuth = Object.keys(dockerConfig.credHelpers || {}).length > 0 || !!dockerConfig.credsStore || Object.keys(dockerConfig.auths || {}).length > 0; - // https://docs.docker.com/engine/reference/commandline/login/#credential-helpers - if (dockerConfig.credHelpers && dockerConfig.credHelpers[registry]) { - const credHelper = dockerConfig.credHelpers[registry]; - output.write(`[httpOci] Found credential helper '${credHelper}' in '${dockerConfigFilePath}' registry '${registry}'`, LogLevel.Trace); - const auth = await getCredentialFromHelper(params, registry, credHelper); - if (auth) { - return auth; - } - // https://docs.docker.com/engine/reference/commandline/login/#credentials-store - } else if (dockerConfig.credsStore) { - output.write(`[httpOci] Invoking credsStore credential helper '${dockerConfig.credsStore}'`, LogLevel.Trace); - const auth = await getCredentialFromHelper(params, registry, dockerConfig.credsStore); - if (auth) { - return auth; - } - } - if (dockerConfig.auths && dockerConfig.auths[registry]) { - output.write(`[httpOci] Found auths entry in '${dockerConfigFilePath}' for registry '${registry}'`, LogLevel.Trace); - const auth = dockerConfig.auths[registry].auth; - const identityToken = dockerConfig.auths[registry].identitytoken; // Refresh token, seen when running: 'az acr login -n ' - - if (identityToken) { - return { - refreshToken: identityToken, - base64EncodedCredential: undefined, - }; - } - - // Without the presence of an `identityToken`, assume auth is a base64-encoded 'user:token'. - return { - base64EncodedCredential: auth, - refreshToken: undefined, - }; - } - } - } catch (err) { - output.write(`[httpOci] Failed to read docker config.json: ${err}`, LogLevel.Trace); - return; - } - - if (!configContainsAuth) { - let defaultCredHelper = ''; - // Try platform-specific default credential helper - if (process.platform === 'linux') { - if (await existsInPath('pass')) { - defaultCredHelper = 'pass'; - } else { - defaultCredHelper = 'secret'; - } - } else if (process.platform === 'win32') { - defaultCredHelper = 'wincred'; - } else if (process.platform === 'darwin') { - defaultCredHelper = 'osxkeychain'; - } - if (defaultCredHelper !== '') { - output.write(`[httpOci] Invoking platform default credential helper '${defaultCredHelper}'`, LogLevel.Trace); - const auth = await getCredentialFromHelper(params, registry, defaultCredHelper); - if (auth) { - output.write('[httpOci] Found auth from platform default credential helper', LogLevel.Trace); - return auth; - } - } - } - - // No auth found from docker config or credential helper. - output.write(`[httpOci] No authentication credentials found for registry '${registry}' via docker config or credential helper.`, LogLevel.Trace); - return; -} - -async function getCredentialFromHelper(params: CommonParams, registry: string, credHelperName: string): Promise<{ base64EncodedCredential: string | undefined; refreshToken: string | undefined } | undefined> { - const { output } = params; - - let helperOutput: Buffer; - try { - const { stdout } = await runCommandNoPty({ - exec: plainExec(undefined), - cmd: 'docker-credential-' + credHelperName, - args: ['get'], - stdin: Buffer.from(registry, 'utf-8'), - output, - }); - helperOutput = stdout; - } catch (err) { - output.write(`[httpOci] Failed to query for '${registry}' credential from 'docker-credential-${credHelperName}': ${err}`, LogLevel.Trace); - return undefined; - } - if (helperOutput.length === 0) { - return undefined; - } - - let errors: jsonc.ParseError[] = []; - const creds: CredentialHelperResult = jsonc.parse(helperOutput.toString(), errors); - if (errors.length !== 0) { - output.write(`[httpOci] Credential helper ${credHelperName} returned non-JSON response "${helperOutput.toString()}" for registry '${registry}'`, LogLevel.Warning); - return undefined; - } - - if (creds.Username === '') { - return { - refreshToken: creds.Secret, - base64EncodedCredential: undefined, - }; - } - const userToken = `${creds.Username}:${creds.Secret}`; - return { - base64EncodedCredential: Buffer.from(userToken).toString('base64'), - refreshToken: undefined, - }; -} - -// https://docs.docker.com/registry/spec/auth/token/#requesting-a-token -async function fetchRegistryBearerToken(params: CommonParams, ociRef: OCIRef | OCICollectionRef, wwwAuthenticateData: { realm: string; service: string; scope: string }): Promise { - const { output } = params; - const { realm, service, scope } = wwwAuthenticateData; - - // TODO: Remove this. - if (realm.includes('mcr.microsoft.com')) { - return undefined; - } - - const headers: HEADERS = { - 'user-agent': 'devcontainer' - }; - - // The token server should first attempt to authenticate the client using any authentication credentials provided with the request. - // From Docker 1.11 the Docker engine supports both Basic Authentication and OAuth2 for getting tokens. - // Docker 1.10 and before, the registry client in the Docker Engine only supports Basic Authentication. - // If an attempt to authenticate to the token server fails, the token server should return a 401 Unauthorized response - // indicating that the provided credentials are invalid. - // > https://docs.docker.com/registry/spec/auth/token/#requesting-a-token - const userCredential = await getCredential(params, ociRef); - const basicAuthCredential = userCredential?.base64EncodedCredential; - const refreshToken = userCredential?.refreshToken; - - let httpOptions: { type: string; url: string; headers: Record; data?: Buffer }; - - // There are several different ways registries expect to handle the oauth token exchange. - // Depending on the type of credential available, use the most reasonable method. - if (refreshToken) { - const form_url_encoded = new URLSearchParams(); - form_url_encoded.append('client_id', 'devcontainer'); - form_url_encoded.append('grant_type', 'refresh_token'); - form_url_encoded.append('service', service); - form_url_encoded.append('scope', scope); - form_url_encoded.append('refresh_token', refreshToken); - - headers['content-type'] = 'application/x-www-form-urlencoded'; - - const url = realm; - output.write(`[httpOci] Attempting to fetch bearer token from: ${url}`, LogLevel.Trace); - - httpOptions = { - type: 'POST', - url, - headers: headers, - data: Buffer.from(form_url_encoded.toString()) - }; - } else { - if (basicAuthCredential) { - headers['authorization'] = `Basic ${basicAuthCredential}`; - } - - // realm="https://auth.docker.io/token" - // service="registry.docker.io" - // scope="repository:samalba/my-app:pull,push" - // Example: - // https://auth.docker.io/token?service=registry.docker.io&scope=repository:samalba/my-app:pull,push - const url = `${realm}?service=${service}&scope=${scope}`; - output.write(`[httpOci] Attempting to fetch bearer token from: ${url}`, LogLevel.Trace); - - httpOptions = { - type: 'GET', - url: url, - headers: headers, - }; - } - - let res = await requestResolveHeaders(httpOptions, output); - if (res && res.statusCode === 401 || res.statusCode === 403) { - output.write(`[httpOci] ${res.statusCode}: Credentials for '${service}' may be expired. Attempting request anonymously.`, LogLevel.Info); - const body = res.resBody?.toString(); - if (body) { - output.write(`${res.resBody.toString()}.`, LogLevel.Info); - } - - // Try again without user credentials. If we're here, their creds are likely expired. - delete headers['authorization']; - res = await requestResolveHeaders(httpOptions, output); - } - - if (!res || res.statusCode > 299 || !res.resBody) { - output.write(`[httpOci] ${res.statusCode}: Failed to fetch bearer token for '${service}': ${res.resBody.toString()}`, LogLevel.Error); - return; - } - - let scopeToken: string | undefined; - try { - const json = JSON.parse(res.resBody.toString()); - scopeToken = json.token || json.access_token; // ghcr uses 'token', acr uses 'access_token' - } catch { - // not JSON - } - if (!scopeToken) { - output.write(`[httpOci] Unexpected bearer token response format for '${service}: ${res.resBody.toString()}'`, LogLevel.Error); - return; - } - - return scopeToken; -} \ No newline at end of file diff --git a/src/spec-configuration/lockfile.ts b/src/spec-configuration/lockfile.ts deleted file mode 100644 index 9de0ba0b2..000000000 --- a/src/spec-configuration/lockfile.ts +++ /dev/null @@ -1,89 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as path from 'path'; -import { DevContainerConfig } from './configuration'; -import { readLocalFile, writeLocalFile } from '../spec-utils/pfs'; -import { ContainerFeatureInternalParams, DirectTarballSourceInformation, FeatureSet, FeaturesConfig, OCISourceInformation } from './containerFeaturesConfiguration'; - - -export interface Lockfile { - features: Record; -} - -export async function generateLockfile(featuresConfig: FeaturesConfig): Promise { - return featuresConfig.featureSets - .map(f => [f, f.sourceInformation] as const) - .filter((tup): tup is [FeatureSet, OCISourceInformation | DirectTarballSourceInformation] => ['oci', 'direct-tarball'].indexOf(tup[1].type) !== -1) - .map(([set, source]) => { - const dependsOn = Object.keys(set.features[0].dependsOn || {}); - return { - id: source.userFeatureId, - version: set.features[0].version!, - resolved: source.type === 'oci' ? - `${source.featureRef.registry}/${source.featureRef.path}@${set.computedDigest}` : - source.tarballUri, - integrity: set.computedDigest!, - dependsOn: dependsOn.length ? dependsOn : undefined, - }; - }) - .sort((a, b) => a.id.localeCompare(b.id)) - .reduce((acc, cur) => { - const feature = { ...cur }; - delete (feature as any).id; - acc.features[cur.id] = feature; - return acc; - }, { - features: {} as Record, - }); -} - -export async function writeLockfile(params: ContainerFeatureInternalParams, config: DevContainerConfig, lockfile: Lockfile, forceInitLockfile?: boolean): Promise { - const lockfilePath = getLockfilePath(config); - const oldLockfileContent = await readLocalFile(lockfilePath) - .catch(err => { - if (err?.code !== 'ENOENT') { - throw err; - } - }); - - if (!forceInitLockfile && !oldLockfileContent && !params.experimentalLockfile && !params.experimentalFrozenLockfile) { - return; - } - - const newLockfileContentString = JSON.stringify(lockfile, null, 2); - const newLockfileContent = Buffer.from(newLockfileContentString); - if (params.experimentalFrozenLockfile && !oldLockfileContent) { - throw new Error('Lockfile does not exist.'); - } - if (!oldLockfileContent || !newLockfileContent.equals(oldLockfileContent)) { - if (params.experimentalFrozenLockfile) { - throw new Error('Lockfile does not match.'); - } - await writeLocalFile(lockfilePath, newLockfileContent); - } - return; -} - -export async function readLockfile(config: DevContainerConfig): Promise<{ lockfile?: Lockfile; initLockfile?: boolean }> { - try { - const content = await readLocalFile(getLockfilePath(config)); - // If empty file, use as marker to initialize lockfile when build completes. - if (content.toString().trim() === '') { - return { initLockfile: true }; - } - return { lockfile: JSON.parse(content.toString()) as Lockfile }; - } catch (err) { - if (err?.code === 'ENOENT') { - return {}; - } - throw err; - } -} - -export function getLockfilePath(configOrPath: DevContainerConfig | string) { - const configPath = typeof configOrPath === 'string' ? configOrPath : configOrPath.configFilePath!.fsPath; - return path.join(path.dirname(configPath), path.basename(configPath).startsWith('.') ? '.devcontainer-lock.json' : 'devcontainer-lock.json'); -} diff --git a/src/spec-configuration/tsconfig.json b/src/spec-configuration/tsconfig.json deleted file mode 100644 index f33845140..000000000 --- a/src/spec-configuration/tsconfig.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "references": [ - { - "path": "../spec-common" - }, - { - "path": "../spec-utils" - } - ] -} \ No newline at end of file diff --git a/src/spec-configuration/typings/zlib-zstd.d.ts b/src/spec-configuration/typings/zlib-zstd.d.ts deleted file mode 100644 index 6614b3eab..000000000 --- a/src/spec-configuration/typings/zlib-zstd.d.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Stub types for Zstd compression classes added in Node.js 23.8.0 -// Required for minizlib's type definitions which reference these types -declare module 'zlib' { - interface ZstdCompress extends NodeJS.ReadWriteStream {} - interface ZstdDecompress extends NodeJS.ReadWriteStream {} -} diff --git a/src/spec-node/collectionCommonUtils/generateDocsCommandImpl.ts b/src/spec-node/collectionCommonUtils/generateDocsCommandImpl.ts deleted file mode 100644 index 0a866f818..000000000 --- a/src/spec-node/collectionCommonUtils/generateDocsCommandImpl.ts +++ /dev/null @@ -1,198 +0,0 @@ -import * as fs from 'fs'; -import * as path from 'path'; -import * as jsonc from 'jsonc-parser'; -import { Log, LogLevel } from '../../spec-utils/log'; - -const FEATURES_README_TEMPLATE = ` -# #{Name} - -#{Description} - -## Example Usage - -\`\`\`json -"features": { - "#{Registry}/#{Namespace}/#{Id}:#{Version}": {} -} -\`\`\` - -#{OptionsTable} -#{Customizations} -#{Notes} - ---- - -_Note: This file was auto-generated from the [devcontainer-feature.json](#{RepoUrl}). Add additional notes to a \`NOTES.md\`._ -`; - -const TEMPLATE_README_TEMPLATE = ` -# #{Name} - -#{Description} - -#{OptionsTable} - -#{Notes} - ---- - -_Note: This file was auto-generated from the [devcontainer-template.json](#{RepoUrl}). Add additional notes to a \`NOTES.md\`._ -`; - -export async function generateFeaturesDocumentation( - basePath: string, - ociRegistry: string, - namespace: string, - gitHubOwner: string, - gitHubRepo: string, - output: Log -) { - await _generateDocumentation(output, basePath, FEATURES_README_TEMPLATE, - 'devcontainer-feature.json', ociRegistry, namespace, gitHubOwner, gitHubRepo); -} - -export async function generateTemplatesDocumentation( - basePath: string, - gitHubOwner: string, - gitHubRepo: string, - output: Log -) { - await _generateDocumentation(output, basePath, TEMPLATE_README_TEMPLATE, - 'devcontainer-template.json', '', '', gitHubOwner, gitHubRepo); -} - -async function _generateDocumentation( - output: Log, - basePath: string, - readmeTemplate: string, - metadataFile: string, - ociRegistry: string = '', - namespace: string = '', - gitHubOwner: string = '', - gitHubRepo: string = '' -) { - const directories = fs.readdirSync(basePath); - - await Promise.all( - directories.map(async (f: string) => { - if (!f.startsWith('.')) { - const readmePath = path.join(basePath, f, 'README.md'); - output.write(`Generating ${readmePath}...`, LogLevel.Info); - - const jsonPath = path.join(basePath, f, metadataFile); - - if (!fs.existsSync(jsonPath)) { - output.write(`(!) Warning: ${metadataFile} not found at path '${jsonPath}'. Skipping...`, LogLevel.Warning); - return; - } - - let parsedJson: any | undefined = undefined; - try { - parsedJson = jsonc.parse(fs.readFileSync(jsonPath, 'utf8')); - } catch (err) { - output.write(`Failed to parse ${jsonPath}: ${err}`, LogLevel.Error); - return; - } - - if (!parsedJson || !parsedJson?.id) { - output.write(`${metadataFile} for '${f}' does not contain an 'id'`, LogLevel.Error); - return; - } - - // Add version - let version = 'latest'; - const parsedVersion: string = parsedJson?.version; - if (parsedVersion) { - // example - 1.0.0 - const splitVersion = parsedVersion.split('.'); - version = splitVersion[0]; - } - - const generateOptionsMarkdown = () => { - const options = parsedJson?.options; - if (!options) { - return ''; - } - - const keys = Object.keys(options); - const contents = keys - .map(k => { - const val = options[k]; - - const desc = val.description || '-'; - const type = val.type || '-'; - const def = val.default !== '' ? val.default : '-'; - - return `| ${k} | ${desc} | ${type} | ${def} |`; - }) - .join('\n'); - - return '## Options\n\n' + '| Options Id | Description | Type | Default Value |\n' + '|-----|-----|-----|-----|\n' + contents; - }; - - const generateNotesMarkdown = () => { - const notesPath = path.join(basePath, f, 'NOTES.md'); - return fs.existsSync(notesPath) ? fs.readFileSync(path.join(notesPath), 'utf8') : ''; - }; - - let urlToConfig = `${metadataFile}`; - const basePathTrimmed = basePath.startsWith('./') ? basePath.substring(2) : basePath; - if (gitHubOwner !== '' && gitHubRepo !== '') { - urlToConfig = `https://github.com/${gitHubOwner}/${gitHubRepo}/blob/main/${basePathTrimmed}/${f}/${metadataFile}`; - } - - let header; - const isDeprecated = parsedJson?.deprecated; - const hasLegacyIds = parsedJson?.legacyIds && parsedJson?.legacyIds.length > 0; - - if (isDeprecated || hasLegacyIds) { - header = '### **IMPORTANT NOTE**\n'; - - if (isDeprecated) { - header += `- **This Feature is deprecated, and will no longer receive any further updates/support.**\n`; - } - - if (hasLegacyIds) { - const formattedLegacyIds = parsedJson.legacyIds.map((legacyId: string) => `'${legacyId}'`); - header += `- **Ids used to publish this Feature in the past - ${formattedLegacyIds.join(', ')}**\n`; - } - } - - let extensions = ''; - if (parsedJson?.customizations?.vscode?.extensions) { - const extensionsList = parsedJson.customizations.vscode.extensions; - if (extensionsList && extensionsList.length > 0) { - extensions = - '\n## Customizations\n\n### VS Code Extensions\n\n' + extensionsList.map((ext: string) => `- \`${ext}\``).join('\n') + '\n'; - } - } - - let newReadme = readmeTemplate - // Templates & Features - .replace('#{Id}', parsedJson.id) - .replace('#{Name}', parsedJson.name ? `${parsedJson.name} (${parsedJson.id})` : `${parsedJson.id}`) - .replace('#{Description}', parsedJson.description ?? '') - .replace('#{OptionsTable}', generateOptionsMarkdown()) - .replace('#{Notes}', generateNotesMarkdown()) - .replace('#{RepoUrl}', urlToConfig) - // Features Only - .replace('#{Registry}', ociRegistry) - .replace('#{Namespace}', namespace) - .replace('#{Version}', version) - .replace('#{Customizations}', extensions); - - if (header) { - newReadme = header + newReadme; - } - - // Remove previous readme - if (fs.existsSync(readmePath)) { - fs.unlinkSync(readmePath); - } - - // Write new readme - fs.writeFileSync(readmePath, newReadme); - } - }) - ); -} diff --git a/src/spec-node/collectionCommonUtils/package.ts b/src/spec-node/collectionCommonUtils/package.ts deleted file mode 100644 index 0204b9c5f..000000000 --- a/src/spec-node/collectionCommonUtils/package.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { Argv } from 'yargs'; -import { CLIHost } from '../../spec-common/cliHost'; -import { Log } from '../../spec-utils/log'; - -const targetPositionalDescription = (collectionType: string) => ` -Package ${collectionType}s at provided [target] (default is cwd), where [target] is either: - 1. A path to the src folder of the collection with [1..n] ${collectionType}s. - 2. A path to a single ${collectionType} that contains a devcontainer-${collectionType}.json. - - Additionally, a 'devcontainer-collection.json' will be generated in the output directory. -`; - -export function PackageOptions(y: Argv, collectionType: string) { - return y - .options({ - 'output-folder': { type: 'string', alias: 'o', default: './output', description: 'Path to output directory. Will create directories as needed.' }, - 'force-clean-output-folder': { type: 'boolean', alias: 'f', default: false, description: 'Automatically delete previous output directory before packaging' }, - 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' }, - }) - .positional('target', { type: 'string', default: '.', description: targetPositionalDescription(collectionType) }) - .check(_argv => { - return true; - }); -} - -export interface PackageCommandInput { - cliHost: CLIHost; - targetFolder: string; - outputDir: string; - output: Log; - disposables: (() => Promise | undefined)[]; - isSingle?: boolean; // Packaging a collection of many features/templates. Should autodetect. - forceCleanOutputDir?: boolean; -} diff --git a/src/spec-node/collectionCommonUtils/packageCommandImpl.ts b/src/spec-node/collectionCommonUtils/packageCommandImpl.ts deleted file mode 100644 index cd54533c3..000000000 --- a/src/spec-node/collectionCommonUtils/packageCommandImpl.ts +++ /dev/null @@ -1,267 +0,0 @@ -import * as tar from 'tar'; -import * as jsonc from 'jsonc-parser'; -import * as os from 'os'; -import * as recursiveDirReader from 'recursive-readdir'; -import { PackageCommandInput } from './package'; -import { cpDirectoryLocal, isLocalFile, isLocalFolder, mkdirpLocal, readLocalDir, readLocalFile, rmLocal, writeLocalFile } from '../../spec-utils/pfs'; -import { Log, LogLevel } from '../../spec-utils/log'; -import path from 'path'; -import { DevContainerConfig, isDockerFileConfig } from '../../spec-configuration/configuration'; -import { Template } from '../../spec-configuration/containerTemplatesConfiguration'; -import { Feature } from '../../spec-configuration/containerFeaturesConfiguration'; -import { getRef } from '../../spec-configuration/containerCollectionsOCI'; - -export interface SourceInformation { - source: string; - owner?: string; - repo?: string; - tag?: string; - ref?: string; - sha?: string; -} - -export const OCICollectionFileName = 'devcontainer-collection.json'; - -export async function prepPackageCommand(args: PackageCommandInput, collectionType: string): Promise { - const { cliHost, targetFolder, outputDir, forceCleanOutputDir, output, disposables } = args; - - const targetFolderResolved = cliHost.path.resolve(targetFolder); - if (!(await isLocalFolder(targetFolderResolved))) { - throw new Error(`Target folder '${targetFolderResolved}' does not exist`); - } - - const outputDirResolved = cliHost.path.resolve(outputDir); - if (await isLocalFolder(outputDirResolved)) { - // Output dir exists. Delete it automatically if '-f' is true - if (forceCleanOutputDir) { - await rmLocal(outputDirResolved, { recursive: true, force: true }); - } - else { - output.write(`(!) ERR: Output directory '${outputDirResolved}' already exists. Manually delete, or pass '-f' to continue.`, LogLevel.Error); - process.exit(1); - } - } - - // Detect if we're packaging a collection or a single feature/template - const isValidFolder = await isLocalFolder(cliHost.path.join(targetFolderResolved)); - const isSingle = await isLocalFile(cliHost.path.join(targetFolderResolved, `devcontainer-${collectionType}.json`)); - - if (!isValidFolder) { - throw new Error(`Target folder '${targetFolderResolved}' does not exist`); - } - - // Generate output folder. - await mkdirpLocal(outputDirResolved); - - return { - cliHost, - targetFolder: targetFolderResolved, - outputDir: outputDirResolved, - forceCleanOutputDir, - output, - disposables, - isSingle - }; -} - -async function tarDirectory(folder: string, archiveName: string, outputDir: string) { - return new Promise((resolve) => resolve(tar.create({ file: path.join(outputDir, archiveName), cwd: folder }, ['.']))); -} - -export const getArchiveName = (f: string, collectionType: string) => `devcontainer-${collectionType}-${f}.tgz`; - -export async function packageSingleFeatureOrTemplate(args: PackageCommandInput, collectionType: string) { - const { output, targetFolder, outputDir } = args; - let metadatas = []; - - const devcontainerJsonName = `devcontainer-${collectionType}.json`; - const tmpSrcDir = path.join(os.tmpdir(), `/templates-src-output-${Date.now()}`); - await cpDirectoryLocal(targetFolder, tmpSrcDir); - - const jsonPath = path.join(tmpSrcDir, devcontainerJsonName); - if (!(await isLocalFile(jsonPath))) { - output.write(`${collectionType} is missing a ${devcontainerJsonName}`, LogLevel.Error); - return; - } - - if (collectionType === 'template') { - if (!(await addsAdditionalTemplateProps(tmpSrcDir, jsonPath, output))) { - return; - } - } else if (collectionType === 'feature') { - await addsAdditionalFeatureProps(jsonPath, output); - } - - const metadata = jsonc.parse(await readLocalFile(jsonPath, 'utf-8')); - if (!metadata.id || !metadata.version || !metadata.name) { - output.write(`${collectionType} is missing one of the following required properties in its devcontainer-${collectionType}.json: 'id', 'version', 'name'.`, LogLevel.Error); - return; - } - - const archiveName = getArchiveName(metadata.id, collectionType); - - await tarDirectory(tmpSrcDir, archiveName, outputDir); - output.write(`Packaged ${collectionType} '${metadata.id}'`, LogLevel.Info); - - metadatas.push(metadata); - await rmLocal(tmpSrcDir, { recursive: true, force: true }); - return metadatas; -} - -async function addsAdditionalTemplateProps(srcFolder: string, devcontainerTemplateJsonPath: string, output: Log): Promise { - const devcontainerFilePath = await getDevcontainerFilePath(srcFolder); - - if (!devcontainerFilePath) { - output.write(`Template is missing a devcontainer.json`, LogLevel.Error); - return false; - } - - const devcontainerJsonString: Buffer = await readLocalFile(devcontainerFilePath); - const config: DevContainerConfig = jsonc.parse(devcontainerJsonString.toString()); - - let type = undefined; - const devcontainerTemplateJsonString: Buffer = await readLocalFile(devcontainerTemplateJsonPath); - let templateData: Template = jsonc.parse(devcontainerTemplateJsonString.toString()); - - if ('image' in config) { - type = 'image'; - } else if (isDockerFileConfig(config)) { - type = 'dockerfile'; - } else if ('dockerComposeFile' in config) { - type = 'dockerCompose'; - } else { - output.write(`Dev container config (${devcontainerFilePath}) is missing one of "image", "dockerFile" or "dockerComposeFile" properties.`, LogLevel.Error); - return false; - } - - const fileNames = (await recursiveDirReader.default(srcFolder))?.map((f) => path.relative(srcFolder, f)) ?? []; - - templateData.type = type; - templateData.files = fileNames; - templateData.fileCount = fileNames.length; - templateData.featureIds = - config.features - ? Object.keys(config.features) - .map((f) => getRef(output, f)?.resource) - .filter((f) => f !== undefined) as string[] - : []; - - // If the Template is omitting a folder and that folder contains just a single file, - // replace the entry in the metadata with the full file name, - // as that provides a better user experience when tools consume the metadata. - // Eg: If the template is omitting ".github/*" and the Template source contains just a single file - // "workflow.yml", replace ".github/*" with ".github/workflow.yml" - if (templateData.optionalPaths && templateData.optionalPaths?.length) { - const optionalPaths = templateData.optionalPaths; - for (const optPath of optionalPaths) { - // Skip if not a directory - if (!optPath.endsWith('/*') || optPath.length < 3) { - continue; - } - const dirPath = optPath.slice(0, -2); - const dirFiles = fileNames.filter((f) => f.startsWith(dirPath)); - output.write(`Given optionalPath starting with '${dirPath}' has ${dirFiles.length} files`, LogLevel.Trace); - if (dirFiles.length === 1) { - // If that one item is a file and not a directory - const f = dirFiles[0]; - output.write(`Checking if optionalPath '${optPath}' with lone contents '${f}' is a file `, LogLevel.Trace); - const localPath = path.join(srcFolder, f); - if (await isLocalFile(localPath)) { - output.write(`Checked path '${localPath}' on disk is a file. Replacing optionalPaths entry '${optPath}' with '${f}'`, LogLevel.Trace); - templateData.optionalPaths[optionalPaths.indexOf(optPath)] = f; - } - } - } - } - - await writeLocalFile(devcontainerTemplateJsonPath, JSON.stringify(templateData, null, 4)); - - return true; -} - -// Programmatically adds 'currentId' if 'legacyIds' exist. -async function addsAdditionalFeatureProps(devcontainerFeatureJsonPath: string, output: Log): Promise { - const devcontainerFeatureJsonString: Buffer = await readLocalFile(devcontainerFeatureJsonPath); - let featureData: Feature = jsonc.parse(devcontainerFeatureJsonString.toString()); - - if (featureData.legacyIds && featureData.legacyIds.length > 0) { - featureData.currentId = featureData.id; - output.write(`Programmatically adding currentId:${featureData.currentId}...`, LogLevel.Trace); - - await writeLocalFile(devcontainerFeatureJsonPath, JSON.stringify(featureData, null, 4)); - } -} - -async function getDevcontainerFilePath(srcFolder: string): Promise { - const devcontainerFile = path.join(srcFolder, '.devcontainer.json'); - const devcontainerFileWithinDevcontainerFolder = path.join(srcFolder, '.devcontainer/devcontainer.json'); - - if (await isLocalFile(devcontainerFile)) { - return devcontainerFile; - } else if (await isLocalFile(devcontainerFileWithinDevcontainerFolder)) { - return devcontainerFileWithinDevcontainerFolder; - } - - return undefined; -} - -// Packages collection of Features or Templates -export async function packageCollection(args: PackageCommandInput, collectionType: string) { - const { output, targetFolder: srcFolder, outputDir } = args; - - const collectionDirs = await readLocalDir(srcFolder); - let metadatas = []; - - for await (const c of collectionDirs) { - output.write(`Processing ${collectionType}: ${c}...`, LogLevel.Info); - if (!c.startsWith('.')) { - const folder = path.join(srcFolder, c); - - // Validate minimal folder structure - const devcontainerJsonName = `devcontainer-${collectionType}.json`; - - if (!(await isLocalFile(path.join(folder, devcontainerJsonName)))) { - output.write(`(!) WARNING: ${collectionType} '${c}' is missing a ${devcontainerJsonName}. Skipping... `, LogLevel.Warning); - continue; - } - - const tmpSrcDir = path.join(os.tmpdir(), `/templates-src-output-${Date.now()}`); - await cpDirectoryLocal(folder, tmpSrcDir); - - const archiveName = getArchiveName(c, collectionType); - - const jsonPath = path.join(tmpSrcDir, devcontainerJsonName); - - if (collectionType === 'feature') { - const installShPath = path.join(tmpSrcDir, 'install.sh'); - if (!(await isLocalFile(installShPath))) { - output.write(`Feature '${c}' is missing an install.sh`, LogLevel.Error); - return; - } - - await addsAdditionalFeatureProps(jsonPath, output); - } else if (collectionType === 'template') { - if (!(await addsAdditionalTemplateProps(tmpSrcDir, jsonPath, output))) { - return; - } - } - - await tarDirectory(tmpSrcDir, archiveName, outputDir); - - const metadata = jsonc.parse(await readLocalFile(jsonPath, 'utf-8')); - if (!metadata.id || !metadata.version || !metadata.name) { - output.write(`${collectionType} '${c}' is missing one of the following required properties in its ${devcontainerJsonName}: 'id', 'version', 'name'.`, LogLevel.Error); - return; - } - metadatas.push(metadata); - await rmLocal(tmpSrcDir, { recursive: true, force: true }); - } - } - - if (metadatas.length === 0) { - return; - } - - output.write(`Packaged ${metadatas.length} ${collectionType}s!`, LogLevel.Info); - return metadatas; -} diff --git a/src/spec-node/collectionCommonUtils/publish.ts b/src/spec-node/collectionCommonUtils/publish.ts deleted file mode 100644 index be749139e..000000000 --- a/src/spec-node/collectionCommonUtils/publish.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Argv } from 'yargs'; - -const targetPositionalDescription = (collectionType: string) => ` -Package and publish ${collectionType}s at provided [target] (default is cwd), where [target] is either: - 1. A path to the src folder of the collection with [1..n] ${collectionType}s. - 2. A path to a single ${collectionType} that contains a devcontainer-${collectionType}.json. -`; - -export function publishOptions(y: Argv, collectionType: string) { - return y - .options({ - 'registry': { type: 'string', alias: 'r', default: 'ghcr.io', description: 'Name of the OCI registry.' }, - 'namespace': { type: 'string', alias: 'n', require: true, description: `Unique indentifier for the collection of ${collectionType}s. Example: /` }, - 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' } - }) - .positional('target', { type: 'string', default: '.', description: targetPositionalDescription(collectionType) }) - .check(_argv => { - return true; - }); -} diff --git a/src/spec-node/collectionCommonUtils/publishCommandImpl.ts b/src/spec-node/collectionCommonUtils/publishCommandImpl.ts deleted file mode 100644 index ebdb433e8..000000000 --- a/src/spec-node/collectionCommonUtils/publishCommandImpl.ts +++ /dev/null @@ -1,83 +0,0 @@ -import path from 'path'; -import * as semver from 'semver'; -import { Log, LogLevel } from '../../spec-utils/log'; -import { CommonParams, getPublishedTags, OCICollectionRef, OCIRef } from '../../spec-configuration/containerCollectionsOCI'; -import { OCICollectionFileName } from './packageCommandImpl'; -import { pushCollectionMetadata, pushOCIFeatureOrTemplate } from '../../spec-configuration/containerCollectionsOCIPush'; - -let semanticVersions: string[] = []; -function updateSemanticTagsList(publishedTags: string[], version: string, range: string, publishVersion: string) { - // Reference: https://github.com/npm/node-semver#ranges-1 - const publishedMaxVersion = semver.maxSatisfying(publishedTags, range); - if (publishedMaxVersion === null || semver.compare(version, publishedMaxVersion) === 1) { - semanticVersions.push(publishVersion); - } - return; -} - -export function getSemanticTags(version: string, tags: string[], output: Log) { - if (tags.includes(version)) { - output.write(`(!) WARNING: Version ${version} already exists, skipping ${version}...`, LogLevel.Warning); - return undefined; - } - - const parsedVersion = semver.parse(version); - if (!parsedVersion) { - output.write(`(!) ERR: Version ${version} is not a valid semantic version, skipping ${version}...`, LogLevel.Error); - process.exit(1); - } - - semanticVersions = []; - - // Adds semantic versions depending upon the existings (published) versions - // eg. 1.2.3 --> [1, 1.2, 1.2.3, latest] - updateSemanticTagsList(tags, version, `${parsedVersion.major}.x.x`, `${parsedVersion.major}`); - updateSemanticTagsList(tags, version, `${parsedVersion.major}.${parsedVersion.minor}.x`, `${parsedVersion.major}.${parsedVersion.minor}`); - semanticVersions.push(version); - updateSemanticTagsList(tags, version, `x.x.x`, 'latest'); - - return semanticVersions; -} - -export async function doPublishCommand(params: CommonParams, version: string, ociRef: OCIRef, outputDir: string, collectionType: string, archiveName: string, annotations: { [key: string]: string } = {}) { - const { output } = params; - - output.write(`Fetching published versions...`, LogLevel.Info); - const publishedTags = await getPublishedTags(params, ociRef); - - if (!publishedTags) { - return; - } - - const semanticTags: string[] | undefined = getSemanticTags(version, publishedTags, output); - - if (!!semanticTags) { - output.write(`Publishing tags: ${semanticTags.toString()}...`, LogLevel.Info); - const pathToTgz = path.join(outputDir, archiveName); - const digest = await pushOCIFeatureOrTemplate(params, ociRef, pathToTgz, semanticTags, collectionType, annotations); - if (!digest) { - output.write(`(!) ERR: Failed to publish ${collectionType}: '${ociRef.resource}'`, LogLevel.Error); - return; - } - output.write(`Published ${collectionType}: '${ociRef.id}'`, LogLevel.Info); - return { publishedTags: semanticTags, digest }; - } - - return {}; // Not an error if no versions were published, likely they just already existed and were skipped. -} - -export async function doPublishMetadata(params: CommonParams, collectionRef: OCICollectionRef, outputDir: string, collectionType: string): Promise { - const { output } = params; - - // Publishing Feature/Template Collection Metadata - output.write('Publishing collection metadata...', LogLevel.Info); - - const pathToCollectionFile = path.join(outputDir, OCICollectionFileName); - const publishedDigest = await pushCollectionMetadata(params, collectionRef, pathToCollectionFile, collectionType); - if (!publishedDigest) { - output.write(`(!) ERR: Failed to publish collection metadata: ${OCICollectionFileName}`, LogLevel.Error); - return; - } - output.write('Published collection metadata.', LogLevel.Info); - return publishedDigest; -} diff --git a/src/spec-node/configContainer.ts b/src/spec-node/configContainer.ts deleted file mode 100644 index c0b21eb82..000000000 --- a/src/spec-node/configContainer.ts +++ /dev/null @@ -1,117 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as path from 'path'; - -import * as jsonc from 'jsonc-parser'; - -import { openDockerfileDevContainer } from './singleContainer'; -import { openDockerComposeDevContainer } from './dockerCompose'; -import { ResolverResult, DockerResolverParameters, isDockerFileConfig, runInitializeCommand, getWorkspaceConfiguration, BindMountConsistency, uriToFsPath, DevContainerAuthority, isDevContainerAuthority, SubstituteConfig, SubstitutedConfig, addSubstitution, envListToObj, findContainerAndIdLabels } from './utils'; -import { beforeContainerSubstitute, substitute } from '../spec-common/variableSubstitution'; -import { ContainerError } from '../spec-common/errors'; -import { Workspace, workspaceFromPath, isWorkspacePath } from '../spec-utils/workspaces'; -import { URI } from 'vscode-uri'; -import { CLIHost } from '../spec-common/commonUtils'; -import { Log } from '../spec-utils/log'; -import { getDefaultDevContainerConfigPath, getDevContainerConfigPathIn } from '../spec-configuration/configurationCommonUtils'; -import { DevContainerConfig, DevContainerFromDockerComposeConfig, DevContainerFromDockerfileConfig, DevContainerFromImageConfig, updateFromOldProperties } from '../spec-configuration/configuration'; -import { ensureNoDisallowedFeatures } from './disallowedFeatures'; -import { DockerCLIParameters } from '../spec-shutdown/dockerUtils'; -import { createDocuments } from '../spec-configuration/editableFiles'; - - -export async function resolve(params: DockerResolverParameters, configFile: URI | undefined, overrideConfigFile: URI | undefined, providedIdLabels: string[] | undefined, additionalFeatures: Record>): Promise { - if (configFile && !/\/\.?devcontainer\.json$/.test(configFile.path)) { - throw new Error(`Filename must be devcontainer.json or .devcontainer.json (${uriToFsPath(configFile, params.common.cliHost.platform)}).`); - } - const parsedAuthority = params.parsedAuthority; - if (!parsedAuthority || isDevContainerAuthority(parsedAuthority)) { - return resolveWithLocalFolder(params, parsedAuthority, configFile, overrideConfigFile, providedIdLabels, additionalFeatures); - } else { - throw new Error(`Unexpected authority: ${JSON.stringify(parsedAuthority)}`); - } -} - -async function resolveWithLocalFolder(params: DockerResolverParameters, parsedAuthority: DevContainerAuthority | undefined, configFile: URI | undefined, overrideConfigFile: URI | undefined, providedIdLabels: string[] | undefined, additionalFeatures: Record>): Promise { - const { common, workspaceMountConsistencyDefault } = params; - const { cliHost, output } = common; - - const cwd = cliHost.cwd; // Can be inside WSL. - const workspace = parsedAuthority && workspaceFromPath(cliHost.path, isWorkspacePath(parsedAuthority.hostPath) ? cliHost.path.join(cwd, path.basename(parsedAuthority.hostPath)) : cwd); - - const configPath = configFile ? configFile : workspace - ? (await getDevContainerConfigPathIn(cliHost, workspace.configFolderPath) - || (overrideConfigFile ? getDefaultDevContainerConfigPath(cliHost, workspace.configFolderPath) : undefined)) - : overrideConfigFile; - const configs = configPath && await readDevContainerConfigFile(cliHost, workspace, configPath, params.mountWorkspaceGitRoot, params.mountGitWorktreeCommonDir, output, workspaceMountConsistencyDefault, overrideConfigFile) || undefined; - if (!configs) { - if (configPath || workspace) { - throw new ContainerError({ description: `Dev container config (${uriToFsPath(configPath || getDefaultDevContainerConfigPath(cliHost, workspace!.configFolderPath), cliHost.platform)}) not found.` }); - } else { - throw new ContainerError({ description: `No dev container config and no workspace found.` }); - } - } - const idLabels = providedIdLabels || (await findContainerAndIdLabels(params, undefined, providedIdLabels, workspace?.rootFolderPath, configPath?.fsPath, params.removeOnStartup)).idLabels; - const configWithRaw = addSubstitution(configs.config, config => beforeContainerSubstitute(envListToObj(idLabels), config)); - const { config } = configWithRaw; - - const { dockerCLI, dockerComposeCLI } = params; - const { env } = common; - const cliParams: DockerCLIParameters = { cliHost, dockerCLI, dockerComposeCLI, env, output, buildPlatformInfo: params.buildPlatformInfo, targetPlatformInfo: params.targetPlatformInfo }; - await ensureNoDisallowedFeatures(cliParams, config, additionalFeatures, idLabels); - - await runInitializeCommand({ ...params, common: { ...common, output: common.lifecycleHook.output } }, config.initializeCommand, common.lifecycleHook.onDidInput); - - let result: ResolverResult; - if (isDockerFileConfig(config) || 'image' in config) { - result = await openDockerfileDevContainer(params, configWithRaw as SubstitutedConfig, configs.workspaceConfig, idLabels, additionalFeatures); - } else if ('dockerComposeFile' in config) { - if (!workspace) { - throw new ContainerError({ description: `A Dev Container using Docker Compose requires a workspace folder.` }); - } - result = await openDockerComposeDevContainer(params, workspace, configWithRaw as SubstitutedConfig, idLabels, additionalFeatures); - } else { - throw new ContainerError({ description: `Dev container config (${(config as DevContainerConfig).configFilePath}) is missing one of "image", "dockerFile" or "dockerComposeFile" properties.` }); - } - return result; -} - -export async function readDevContainerConfigFile(cliHost: CLIHost, workspace: Workspace | undefined, configFile: URI, mountWorkspaceGitRoot: boolean, mountGitWorktreeCommonDir: boolean, output: Log, consistency?: BindMountConsistency, overrideConfigFile?: URI) { - const documents = createDocuments(cliHost); - const content = await documents.readDocument(overrideConfigFile ?? configFile); - if (!content) { - return undefined; - } - const raw = jsonc.parse(content) as DevContainerConfig | undefined; - const updated = raw && updateFromOldProperties(raw); - if (!updated || typeof updated !== 'object' || Array.isArray(updated)) { - throw new ContainerError({ description: `Dev container config (${uriToFsPath(configFile, cliHost.platform)}) must contain a JSON object literal.` }); - } - const workspaceConfig = await getWorkspaceConfiguration(cliHost, workspace, updated, mountWorkspaceGitRoot, mountGitWorktreeCommonDir, output, consistency); - const substitute0: SubstituteConfig = value => substitute({ - platform: cliHost.platform, - localWorkspaceFolder: workspace?.rootFolderPath, - containerWorkspaceFolder: workspaceConfig.workspaceFolder, - configFile, - env: cliHost.env, - }, value); - const config: DevContainerConfig = substitute0(updated); - if (typeof config.workspaceFolder === 'string') { - workspaceConfig.workspaceFolder = config.workspaceFolder; - } - if ('workspaceMount' in config) { - workspaceConfig.workspaceMount = config.workspaceMount; - } - config.configFilePath = configFile; - return { - config: { - config, - raw: updated, - substitute: substitute0, - }, - workspaceConfig, - }; -} diff --git a/src/spec-node/containerFeatures.ts b/src/spec-node/containerFeatures.ts deleted file mode 100644 index e05822d1b..000000000 --- a/src/spec-node/containerFeatures.ts +++ /dev/null @@ -1,492 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as path from 'path'; - -import { DevContainerConfig } from '../spec-configuration/configuration'; -import { dockerCLI, dockerPtyCLI, ImageDetails, toExecParameters, toPtyExecParameters } from '../spec-shutdown/dockerUtils'; -import { LogLevel, makeLog } from '../spec-utils/log'; -import { FeaturesConfig, getContainerFeaturesBaseDockerFile, getFeatureInstallWrapperScript, getFeatureLayers, getFeatureMainValue, getFeatureValueObject, generateFeaturesConfig, Feature, generateContainerEnvs } from '../spec-configuration/containerFeaturesConfiguration'; -import { readLocalFile } from '../spec-utils/pfs'; -import { includeAllConfiguredFeatures } from '../spec-utils/product'; -import { createFeaturesTempFolder, DockerResolverParameters, getCacheFolder, getFolderImageName, getEmptyContextFolder, SubstitutedConfig, isBuildxCacheToInline } from './utils'; -import { isEarlierVersion, parseVersion, runCommandNoPty } from '../spec-common/commonUtils'; -import { getDevcontainerMetadata, getDevcontainerMetadataLabel, getImageBuildInfoFromImage, ImageBuildInfo, ImageMetadataEntry, imageMetadataLabel, MergedDevContainerConfig } from './imageMetadata'; -import { supportsBuildContexts } from './dockerfileUtils'; -import { ContainerError } from '../spec-common/errors'; - -// Escapes environment variable keys. -// -// Environment variables must contain: -// - alpha-numeric values, or -// - the '_' character, and -// - a number cannot be the first character -export const getSafeId = (str: string) => str - .replace(/[^\w_]/g, '_') - .replace(/^[\d_]+/g, '_') - .toUpperCase(); - -export async function extendImage(params: DockerResolverParameters, config: SubstitutedConfig, imageName: string, additionalImageNames: string[], additionalFeatures: Record>, canAddLabelsToContainer: boolean) { - const { common } = params; - const { cliHost, output } = common; - - const imageBuildInfo = await getImageBuildInfoFromImage(params, imageName, config.substitute); - const extendImageDetails = await getExtendImageBuildInfo(params, config, imageName, imageBuildInfo, undefined, additionalFeatures, canAddLabelsToContainer); - if (!extendImageDetails?.featureBuildInfo) { - // no feature extensions - return - if (additionalImageNames.length) { - if (params.isTTY) { - await Promise.all(additionalImageNames.map(name => dockerPtyCLI(params, 'tag', imageName, name))); - } else { - await Promise.all(additionalImageNames.map(name => dockerCLI(params, 'tag', imageName, name))); - } - } - return { - updatedImageName: [imageName], - imageMetadata: getDevcontainerMetadata(imageBuildInfo.metadata, config, extendImageDetails?.featuresConfig), - imageDetails: async () => imageBuildInfo.imageDetails, - labels: extendImageDetails?.labels, - }; - } - const { featureBuildInfo, featuresConfig } = extendImageDetails; - - // Got feature extensions -> build the image - const dockerfilePath = cliHost.path.join(featureBuildInfo.dstFolder, 'Dockerfile.extended'); - await cliHost.writeFile(dockerfilePath, Buffer.from(featureBuildInfo.dockerfilePrefixContent + featureBuildInfo.dockerfileContent)); - const folderImageName = getFolderImageName(common); - const updatedImageName = `${imageName.startsWith(folderImageName) ? imageName : folderImageName}-features`; - - const args: string[] = []; - if (!params.buildKitVersion && - (params.buildxPlatform || params.buildxPush)) { - throw new ContainerError({ description: '--platform or --push require BuildKit enabled.', data: { fileWithError: dockerfilePath } }); - } - if (params.buildKitVersion) { - args.push('buildx', 'build'); - - // --platform - if (params.buildxPlatform) { - output.write('Setting BuildKit platform(s): ' + params.buildxPlatform, LogLevel.Trace); - args.push('--platform', params.buildxPlatform); - } - - // --push/--output - if (params.buildxPush) { - args.push('--push'); - } else { - if (params.buildxOutput) { - args.push('--output', params.buildxOutput); - } else { - args.push('--load'); // (short for --output=docker, i.e. load into normal 'docker images' collection) - } - } - if (params.buildxCacheTo) { - args.push('--cache-to', params.buildxCacheTo); - } - if (!isBuildxCacheToInline(params.buildxCacheTo)) { - args.push('--build-arg', 'BUILDKIT_INLINE_CACHE=1'); - } - if (!params.buildNoCache) { - params.additionalCacheFroms.forEach(cacheFrom => args.push('--cache-from', cacheFrom)); - } - - for (const buildContext in featureBuildInfo.buildKitContexts) { - args.push('--build-context', `${buildContext}=${featureBuildInfo.buildKitContexts[buildContext]}`); - } - - for (const securityOpt of featureBuildInfo.securityOpts) { - args.push('--security-opt', securityOpt); - } - } else { - // Not using buildx - args.push( - 'build', - ); - } - if (params.buildNoCache) { - args.push('--no-cache'); - } - for (const buildArg in featureBuildInfo.buildArgs) { - args.push('--build-arg', `${buildArg}=${featureBuildInfo.buildArgs[buildArg]}`); - } - // Once this is step merged with the user Dockerfile (or working against the base image), - // the path will be the dev container context - // Set empty dir under temp path as the context for now to ensure we don't have dependencies on the features content - const emptyTempDir = getEmptyContextFolder(common); - cliHost.mkdirp(emptyTempDir); - args.push( - '--target', featureBuildInfo.overrideTarget, - '-f', dockerfilePath, - ...additionalImageNames.length > 0 ? additionalImageNames.map(name => ['-t', name]).flat() : ['-t', updatedImageName], - ...params.additionalLabels.length > 0 ? params.additionalLabels.map(label => ['--label', label]).flat() : [], - emptyTempDir - ); - - if (params.isTTY) { - const infoParams = { ...toPtyExecParameters(params), output: makeLog(output, LogLevel.Info) }; - await dockerPtyCLI(infoParams, ...args); - } else { - const infoParams = { ...toExecParameters(params), output: makeLog(output, LogLevel.Info), print: 'continuous' as 'continuous' }; - await dockerCLI(infoParams, ...args); - } - return { - updatedImageName: additionalImageNames.length > 0 ? additionalImageNames : [updatedImageName], - imageMetadata: getDevcontainerMetadata(imageBuildInfo.metadata, config, featuresConfig), - imageDetails: async () => imageBuildInfo.imageDetails, - }; -} - -export async function getExtendImageBuildInfo(params: DockerResolverParameters, config: SubstitutedConfig, baseName: string, imageBuildInfo: ImageBuildInfo, composeServiceUser: string | undefined, additionalFeatures: Record>, canAddLabelsToContainer: boolean): Promise<{ featureBuildInfo?: ImageBuildOptions; featuresConfig?: FeaturesConfig; labels?: Record } | undefined> { - - // Creates the folder where the working files will be setup. - const dstFolder = await createFeaturesTempFolder(params.common); - - // Processes the user's configuration. - const platform = params.common.cliHost.platform; - - const cacheFolder = await getCacheFolder(params.common.cliHost); - const { experimentalLockfile, experimentalFrozenLockfile } = params; - const featuresConfig = await generateFeaturesConfig({ ...params.common, platform, cacheFolder, experimentalLockfile, experimentalFrozenLockfile }, dstFolder, config.config, additionalFeatures); - if (!featuresConfig) { - if (canAddLabelsToContainer && !imageBuildInfo.dockerfile) { - return { - labels: { - [imageMetadataLabel]: JSON.stringify(getDevcontainerMetadata(imageBuildInfo.metadata, config, undefined, [], getOmitDevcontainerPropertyOverride(params.common)).raw), - } - }; - } - return { featureBuildInfo: await getImageBuildOptions(params, config, dstFolder, baseName, imageBuildInfo) }; - } - - // Generates the end configuration. - const featureBuildInfo = await getFeaturesBuildOptions(params, config, featuresConfig, baseName, imageBuildInfo, composeServiceUser); - if (!featureBuildInfo) { - return undefined; - } - return { featureBuildInfo, featuresConfig }; - -} - -// NOTE: only exported to enable testing. Not meant to be called outside file. -export function generateContainerEnvsV1(featuresConfig: FeaturesConfig) { - let result = ''; - for (const fSet of featuresConfig.featureSets) { - // We only need to generate this ENV references for the initial features specification. - if (fSet.internalVersion !== '2') - { - result += '\n'; - result += fSet.features - .filter(f => (includeAllConfiguredFeatures || f.included) && f.value) - .reduce((envs, f) => envs.concat(generateContainerEnvs(f.containerEnv)), [] as string[]) - .join('\n'); - } - } - return result; -} - -export interface ImageBuildOptions { - dstFolder: string; - dockerfileContent: string; - overrideTarget: string; - dockerfilePrefixContent: string; - buildArgs: Record; - buildKitContexts: Record; - securityOpts: string[]; -} - -async function getImageBuildOptions(params: DockerResolverParameters, config: SubstitutedConfig, dstFolder: string, baseName: string, imageBuildInfo: ImageBuildInfo): Promise { - const syntax = imageBuildInfo.dockerfile?.preamble.directives.syntax; - return { - dstFolder, - dockerfileContent: ` -FROM $_DEV_CONTAINERS_BASE_IMAGE AS dev_containers_target_stage -${getDevcontainerMetadataLabel(getDevcontainerMetadata(imageBuildInfo.metadata, config, { featureSets: [] }, [], getOmitDevcontainerPropertyOverride(params.common)))} -`, - overrideTarget: 'dev_containers_target_stage', - dockerfilePrefixContent: `${syntax ? `# syntax=${syntax}` : ''} - ARG _DEV_CONTAINERS_BASE_IMAGE=placeholder -`, - buildArgs: { - _DEV_CONTAINERS_BASE_IMAGE: baseName, - } as Record, - buildKitContexts: {} as Record, - securityOpts: [], - }; -} - -function getOmitDevcontainerPropertyOverride(resolverParams: { omitConfigRemotEnvFromMetadata?: boolean }): (keyof DevContainerConfig & keyof ImageMetadataEntry)[] { - if (resolverParams.omitConfigRemotEnvFromMetadata) { - return ['remoteEnv']; - } - - return []; -} - -async function getFeaturesBuildOptions(params: DockerResolverParameters, devContainerConfig: SubstitutedConfig, featuresConfig: FeaturesConfig, baseName: string, imageBuildInfo: ImageBuildInfo, composeServiceUser: string | undefined): Promise { - const { common } = params; - const { cliHost, output } = common; - const { dstFolder } = featuresConfig; - - if (!dstFolder || dstFolder === '') { - output.write('dstFolder is undefined or empty in addContainerFeatures', LogLevel.Error); - return undefined; - } - - // With Buildkit (0.8.0 or later), we can supply an additional build context to provide access to - // the container-features content. - // For non-Buildkit, we build a temporary image to hold the container-features content in a way - // that is accessible from the docker build for non-BuiltKit builds - // TODO generate an image name that is specific to this dev container? - const buildKitVersionParsed = params.buildKitVersion?.versionMatch ? parseVersion(params.buildKitVersion.versionMatch) : undefined; - const minRequiredVersion = [0, 8, 0]; - const useBuildKitBuildContexts = buildKitVersionParsed ? !isEarlierVersion(buildKitVersionParsed, minRequiredVersion) : false; - const buildContentImageName = 'dev_container_feature_content_temp'; - const disableSELinuxLabels = useBuildKitBuildContexts && await isUsingSELinuxLabels(params); - // Access Docker engine version - const dockerEngineVersionParsed = params.dockerEngineVersion?.versionMatch ? parseVersion(params.dockerEngineVersion.versionMatch) : undefined; - const minDockerEngineVersion = [23, 0, 0]; - const skipDefaultSyntax = dockerEngineVersionParsed ? !isEarlierVersion(dockerEngineVersionParsed, minDockerEngineVersion) : false; - const omitPropertyOverride = params.common.skipPersistingCustomizationsFromFeatures ? ['customizations'] : []; - const imageMetadata = getDevcontainerMetadata(imageBuildInfo.metadata, devContainerConfig, featuresConfig, omitPropertyOverride, getOmitDevcontainerPropertyOverride(params.common)); - const { containerUser, remoteUser } = findContainerUsers(imageMetadata, composeServiceUser, imageBuildInfo.user); - const builtinVariables = [ - `_CONTAINER_USER=${containerUser}`, - `_REMOTE_USER=${remoteUser}`, - ]; - const envPath = cliHost.path.join(dstFolder, 'devcontainer-features.builtin.env'); - await cliHost.writeFile(envPath, Buffer.from(builtinVariables.join('\n') + '\n')); - - // When copying via buildkit, the content is accessed via '.' (i.e. in the context root) - // When copying via temp image, the content is in '/tmp/build-features' - const contentSourceRootPath = useBuildKitBuildContexts ? '.' : '/tmp/build-features/'; - const dockerfile = getContainerFeaturesBaseDockerFile(contentSourceRootPath) - .replace('#{nonBuildKitFeatureContentFallback}', useBuildKitBuildContexts ? '' : `FROM ${buildContentImageName} as dev_containers_feature_content_source`) - .replace('#{featureLayer}', getFeatureLayers(featuresConfig, containerUser, remoteUser, useBuildKitBuildContexts, contentSourceRootPath)) - .replace('#{containerEnv}', generateContainerEnvsV1(featuresConfig)) - .replace('#{devcontainerMetadata}', getDevcontainerMetadataLabel(imageMetadata)) - .replace('#{containerEnvMetadata}', generateContainerEnvs(devContainerConfig.config.containerEnv, true)) - ; - const syntax = imageBuildInfo.dockerfile?.preamble.directives.syntax; - const omitSyntaxDirective = common.omitSyntaxDirective; // Can be removed when https://github.com/moby/buildkit/issues/4556 is fixed - const dockerfilePrefixContent = `${omitSyntaxDirective ? '' : - skipDefaultSyntax ? (syntax ? `# syntax=${syntax}` : '') : - useBuildKitBuildContexts && !(imageBuildInfo.dockerfile && supportsBuildContexts(imageBuildInfo.dockerfile)) ? '# syntax=docker/dockerfile:1.4' : - syntax ? `# syntax=${syntax}` : ''} -ARG _DEV_CONTAINERS_BASE_IMAGE=placeholder -`; - - // Build devcontainer-features.env and devcontainer-features-install.sh file(s) for each features source folder - for await (const fSet of featuresConfig.featureSets) { - if (fSet.internalVersion === '2') - { - for await (const fe of fSet.features) { - if (fe.cachePath) - { - fe.internalVersion = '2'; - const envPath = cliHost.path.join(fe.cachePath, 'devcontainer-features.env'); - const variables = getFeatureEnvVariables(fe); - await cliHost.writeFile(envPath, Buffer.from(variables.join('\n'))); - - const installWrapperPath = cliHost.path.join(fe.cachePath, 'devcontainer-features-install.sh'); - const installWrapperContent = getFeatureInstallWrapperScript(fe, fSet, variables); - await cliHost.writeFile(installWrapperPath, Buffer.from(installWrapperContent)); - } - } - } else { - const featuresEnv = ([] as string[]).concat( - ...fSet.features - .filter(f => (includeAllConfiguredFeatures || f.included) && f.value) - .map(getFeatureEnvVariables) - ).join('\n'); - const envPath = cliHost.path.join(fSet.features[0].cachePath!, 'devcontainer-features.env'); - await Promise.all([ - cliHost.writeFile(envPath, Buffer.from(featuresEnv)), - ...fSet.features - .filter(f => (includeAllConfiguredFeatures || f.included) && f.value) - .map(f => { - const consecutiveId = f.consecutiveId; - if (!consecutiveId) { - throw new Error('consecutiveId is undefined for Feature ' + f.id); - } - const featuresEnv = [ - ...getFeatureEnvVariables(f), - `_BUILD_ARG_${getSafeId(f.id)}_TARGETPATH=${path.posix.join('/usr/local/devcontainer-features', consecutiveId)}` - ] - .join('\n'); - const envPath = cliHost.path.join(dstFolder, consecutiveId, 'devcontainer-features.env'); // next to bin/acquire - return cliHost.writeFile(envPath, Buffer.from(featuresEnv)); - }) - ]); - } - } - - // For non-BuildKit, build the temporary image for the container-features content - if (!useBuildKitBuildContexts) { - const buildContentDockerfile = ` - FROM scratch - COPY . /tmp/build-features/ - `; - const buildContentDockerfilePath = cliHost.path.join(dstFolder, 'Dockerfile.buildContent'); - await cliHost.writeFile(buildContentDockerfilePath, Buffer.from(buildContentDockerfile)); - const buildContentArgs = [ - 'build', - '-t', buildContentImageName, - '-f', buildContentDockerfilePath, - ]; - buildContentArgs.push(dstFolder); - - if (params.isTTY) { - const buildContentInfoParams = { ...toPtyExecParameters(params), output: makeLog(output, LogLevel.Info) }; - await dockerPtyCLI(buildContentInfoParams, ...buildContentArgs); - } else { - const buildContentInfoParams = { ...toExecParameters(params), output: makeLog(output, LogLevel.Info), print: 'continuous' as 'continuous' }; - await dockerCLI(buildContentInfoParams, ...buildContentArgs); - } - } - return { - dstFolder, - dockerfileContent: dockerfile, - overrideTarget: 'dev_containers_target_stage', - dockerfilePrefixContent, - buildArgs: { - _DEV_CONTAINERS_BASE_IMAGE: baseName, - _DEV_CONTAINERS_IMAGE_USER: imageBuildInfo.user, - _DEV_CONTAINERS_FEATURE_CONTENT_SOURCE: buildContentImageName, - }, - buildKitContexts: useBuildKitBuildContexts ? { dev_containers_feature_content_source: dstFolder } : {}, - securityOpts: disableSELinuxLabels ? ['label=disable'] : [], - }; -} - -async function isUsingSELinuxLabels(params: DockerResolverParameters): Promise { - try { - const { common } = params; - const { cliHost, output } = common; - return params.isPodman && cliHost.platform === 'linux' - && (await runCommandNoPty({ - exec: cliHost.exec, - cmd: 'getenforce', - output, - print: true, - })).stdout.toString().trim() !== 'Disabled' - && (await dockerCLI({ - ...toExecParameters(params), - print: true, - }, 'info', '-f', '{{.Host.Security.SELinuxEnabled}}')).stdout.toString().trim() === 'true'; - } catch { - // If we can't run the commands, assume SELinux is not enabled. - return false; - - } -} - -export function findContainerUsers(imageMetadata: SubstitutedConfig, composeServiceUser: string | undefined, imageUser: string) { - const reversed = imageMetadata.config.slice().reverse(); - const containerUser = reversed.find(entry => entry.containerUser)?.containerUser || composeServiceUser || imageUser; - const remoteUser = reversed.find(entry => entry.remoteUser)?.remoteUser || containerUser; - return { containerUser, remoteUser }; -} - - -function getFeatureEnvVariables(f: Feature) { - const values = getFeatureValueObject(f); - const idSafe = getSafeId(f.id); - const variables = []; - - if(f.internalVersion !== '2') - { - if (values) { - variables.push(...Object.keys(values) - .map(name => `_BUILD_ARG_${idSafe}_${getSafeId(name)}="${values[name]}"`)); - variables.push(`_BUILD_ARG_${idSafe}=true`); - } - if (f.buildArg) { - variables.push(`${f.buildArg}=${getFeatureMainValue(f)}`); - } - return variables; - } else { - if (values) { - variables.push(...Object.keys(values) - .map(name => `${getSafeId(name)}="${values[name]}"`)); - } - if (f.buildArg) { - variables.push(`${f.buildArg}=${getFeatureMainValue(f)}`); - } - return variables; - } -} - -export async function getRemoteUserUIDUpdateDetails(params: DockerResolverParameters, mergedConfig: MergedDevContainerConfig, imageName: string, imageDetails: () => Promise, runArgsUser: string | undefined) { - const { common } = params; - const { cliHost } = common; - const { updateRemoteUserUID } = mergedConfig; - if (params.updateRemoteUserUIDDefault === 'never' || !(typeof updateRemoteUserUID === 'boolean' ? updateRemoteUserUID : params.updateRemoteUserUIDDefault === 'on') || !(cliHost.platform === 'linux' || params.updateRemoteUserUIDOnMacOS && cliHost.platform === 'darwin')) { - return null; - } - const details = await imageDetails(); - const imageUser = details.Config.User || 'root'; - const remoteUser = mergedConfig.remoteUser || runArgsUser || imageUser; - if (remoteUser === 'root' || /^\d+$/.test(remoteUser)) { - return null; - } - const folderImageName = getFolderImageName(common); - const fixedImageName = `${imageName.startsWith(folderImageName) ? imageName : folderImageName}-uid`; - - return { - imageName: fixedImageName, - remoteUser, - imageUser, - platform: [details.Os, details.Architecture, details.Variant].filter(Boolean).join('/') - }; -} - -export async function updateRemoteUserUID(params: DockerResolverParameters, mergedConfig: MergedDevContainerConfig, imageName: string, imageDetails: () => Promise, runArgsUser: string | undefined) { - const { common } = params; - const { cliHost } = common; - - const updateDetails = await getRemoteUserUIDUpdateDetails(params, mergedConfig, imageName, imageDetails, runArgsUser); - if (!updateDetails) { - return imageName; - } - const { imageName: fixedImageName, remoteUser, imageUser, platform } = updateDetails; - - const dockerfileName = 'updateUID.Dockerfile'; - const srcDockerfile = path.join(common.extensionPath, 'scripts', dockerfileName); - const version = common.package.version; - const destDockerfile = cliHost.path.join(await getCacheFolder(cliHost), `${dockerfileName}-${version}`); - const tmpDockerfile = `${destDockerfile}-${Date.now()}`; - await cliHost.mkdirp(cliHost.path.dirname(tmpDockerfile)); - await cliHost.writeFile(tmpDockerfile, await readLocalFile(srcDockerfile)); - await cliHost.rename(tmpDockerfile, destDockerfile); - const emptyFolder = getEmptyContextFolder(common); - await cliHost.mkdirp(emptyFolder); - const args = [ - 'build', - '-f', destDockerfile, - '-t', fixedImageName, - ...(platform ? ['--platform', platform] : []), - '--build-arg', `BASE_IMAGE=${params.isPodman && !hasRegistryHostname(imageName) ? 'localhost/' : ''}${imageName}`, // Podman: https://github.com/microsoft/vscode-remote-release/issues/9748 - '--build-arg', `REMOTE_USER=${remoteUser}`, - '--build-arg', `NEW_UID=${await cliHost.getuid!()}`, - '--build-arg', `NEW_GID=${await cliHost.getgid!()}`, - '--build-arg', `IMAGE_USER=${imageUser}`, - emptyFolder, - ]; - if (params.isTTY) { - await dockerPtyCLI(params, ...args); - } else { - await dockerCLI(params, ...args); - } - return fixedImageName; -} - -function hasRegistryHostname(imageName: string) { - if (imageName.startsWith('localhost/')) { - return true; - } - const dot = imageName.indexOf('.'); - const slash = imageName.indexOf('/'); - return dot !== -1 && slash !== -1 && dot < slash; -} diff --git a/src/spec-node/devContainers.ts b/src/spec-node/devContainers.ts deleted file mode 100644 index a856890c6..000000000 --- a/src/spec-node/devContainers.ts +++ /dev/null @@ -1,299 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as path from 'path'; -import * as crypto from 'crypto'; -import * as os from 'os'; - -import { mapNodeOSToGOOS, mapNodeArchitectureToGOARCH } from '../spec-configuration/containerCollectionsOCI'; -import { DockerResolverParameters, DevContainerAuthority, UpdateRemoteUserUIDDefault, BindMountConsistency, getCacheFolder, GPUAvailability } from './utils'; -import { createNullLifecycleHook, finishBackgroundTasks, ResolverParameters, UserEnvProbe } from '../spec-common/injectHeadless'; -import { GoARCH, GoOS, getCLIHost, loadNativeModule } from '../spec-common/commonUtils'; -import { resolve } from './configContainer'; -import { URI } from 'vscode-uri'; -import { LogLevel, LogDimensions, toErrorText, createCombinedLog, createTerminalLog, Log, makeLog, LogFormat, createJSONLog, createPlainLog, LogHandler, replaceAllLog } from '../spec-utils/log'; -import { dockerComposeCLIConfig } from './dockerCompose'; -import { Mount } from '../spec-configuration/containerFeaturesConfiguration'; -import { getPackageConfig, PackageConfiguration } from '../spec-utils/product'; -import { dockerBuildKitVersion, dockerEngineVersion, isPodman } from '../spec-shutdown/dockerUtils'; -import { Event } from '../spec-utils/event'; - - -export interface ProvisionOptions { - dockerPath: string | undefined; - dockerComposePath: string | undefined; - containerDataFolder: string | undefined; - containerSystemDataFolder: string | undefined; - workspaceFolder: string | undefined; - workspaceMountConsistency?: BindMountConsistency; - gpuAvailability?: GPUAvailability; - mountWorkspaceGitRoot: boolean; - mountGitWorktreeCommonDir: boolean; - configFile: URI | undefined; - overrideConfigFile: URI | undefined; - logLevel: LogLevel; - logFormat: LogFormat; - log: (text: string) => void; - terminalDimensions: LogDimensions | undefined; - onDidChangeTerminalDimensions?: Event; - defaultUserEnvProbe: UserEnvProbe; - removeExistingContainer: boolean; - buildNoCache: boolean; - expectExistingContainer: boolean; - postCreateEnabled: boolean; - skipNonBlocking: boolean; - prebuild: boolean; - persistedFolder: string | undefined; - additionalMounts: Mount[]; - updateRemoteUserUIDDefault: UpdateRemoteUserUIDDefault; - remoteEnv: Record; - additionalCacheFroms: string[]; - useBuildKit: 'auto' | 'never'; - omitLoggerHeader?: boolean | undefined; - buildxPlatform: string | undefined; - buildxPush: boolean; - additionalLabels: string[]; - buildxOutput: string | undefined; - buildxCacheTo: string | undefined; - additionalFeatures?: Record>; - skipFeatureAutoMapping: boolean; - skipPostAttach: boolean; - containerSessionDataFolder?: string; - skipPersistingCustomizationsFromFeatures: boolean; - omitConfigRemotEnvFromMetadata?: boolean; - dotfiles: { - repository?: string; - installCommand?: string; - targetPath?: string; - }; - experimentalLockfile?: boolean; - experimentalFrozenLockfile?: boolean; - secretsP?: Promise>; - omitSyntaxDirective?: boolean; - includeConfig?: boolean; - includeMergedConfig?: boolean; -} - -export async function launch(options: ProvisionOptions, providedIdLabels: string[] | undefined, disposables: (() => Promise | undefined)[]) { - const params = await createDockerParams(options, disposables); - const output = params.common.output; - const text = 'Resolving Remote'; - const start = output.start(text); - - const result = await resolve(params, options.configFile, options.overrideConfigFile, providedIdLabels, options.additionalFeatures ?? {}); - output.stop(text, start); - const { dockerContainerId, composeProjectName } = result; - return { - containerId: dockerContainerId, - composeProjectName, - remoteUser: result.properties.user, - remoteWorkspaceFolder: result.properties.remoteWorkspaceFolder, - configuration: options.includeConfig ? result.config : undefined, - mergedConfiguration: options.includeMergedConfig ? result.mergedConfig : undefined, - finishBackgroundTasks: async () => { - try { - await finishBackgroundTasks(result.params.backgroundTasks); - } catch (err) { - output.write(toErrorText(String(err && (err.stack || err.message) || err))); - } - }, - }; -} - -export async function createDockerParams(options: ProvisionOptions, disposables: (() => Promise | undefined)[]): Promise { - const { persistedFolder, additionalMounts, updateRemoteUserUIDDefault, containerDataFolder, containerSystemDataFolder, workspaceMountConsistency, gpuAvailability, mountWorkspaceGitRoot, mountGitWorktreeCommonDir, remoteEnv, experimentalLockfile, experimentalFrozenLockfile, omitLoggerHeader, secretsP } = options; - let parsedAuthority: DevContainerAuthority | undefined; - if (options.workspaceFolder) { - parsedAuthority = { hostPath: options.workspaceFolder } as DevContainerAuthority; - } - const extensionPath = path.join(__dirname, '..', '..'); - const sessionStart = new Date(); - const pkg = getPackageConfig(); - const output = createLog(options, pkg, sessionStart, disposables, omitLoggerHeader, secretsP ? await secretsP : undefined); - - const appRoot = undefined; - const cwd = options.workspaceFolder || process.cwd(); - const allowInheritTTY = options.logFormat === 'text'; - const cliHost = await getCLIHost(cwd, loadNativeModule, allowInheritTTY); - const sessionId = crypto.randomUUID(); - - const common: ResolverParameters = { - prebuild: options.prebuild, - computeExtensionHostEnv: false, - package: pkg, - containerDataFolder, - containerSystemDataFolder, - appRoot, - extensionPath, // TODO: rename to packagePath - sessionId, - sessionStart, - cliHost, - env: cliHost.env, - cwd, - isLocalContainer: false, - progress: () => { }, - output, - allowSystemConfigChange: true, - defaultUserEnvProbe: options.defaultUserEnvProbe, - lifecycleHook: createNullLifecycleHook(options.postCreateEnabled, options.skipNonBlocking, output), - getLogLevel: () => options.logLevel, - onDidChangeLogLevel: () => ({ dispose() { } }), - loadNativeModule, - allowInheritTTY, - shutdowns: [], - backgroundTasks: [], - persistedFolder: persistedFolder || await getCacheFolder(cliHost), // Fallback to tmp folder, even though that isn't 'persistent' - remoteEnv, - secretsP, - buildxPlatform: options.buildxPlatform, - buildxPush: options.buildxPush, - buildxOutput: options.buildxOutput, - buildxCacheTo: options.buildxCacheTo, - skipFeatureAutoMapping: options.skipFeatureAutoMapping, - skipPostAttach: options.skipPostAttach, - containerSessionDataFolder: options.containerSessionDataFolder, - skipPersistingCustomizationsFromFeatures: options.skipPersistingCustomizationsFromFeatures, - omitConfigRemotEnvFromMetadata: options.omitConfigRemotEnvFromMetadata, - dotfilesConfiguration: { - repository: options.dotfiles.repository, - installCommand: options.dotfiles.installCommand, - targetPath: options.dotfiles.targetPath || '~/dotfiles', - }, - omitSyntaxDirective: options.omitSyntaxDirective, - }; - - const dockerPath = options.dockerPath || 'docker'; - const dockerComposePath = options.dockerComposePath || 'docker-compose'; - const dockerComposeCLI = dockerComposeCLIConfig({ - exec: cliHost.exec, - env: cliHost.env, - output: common.output, - }, dockerPath, dockerComposePath); - - const buildPlatformInfo = { - os: mapNodeOSToGOOS(cliHost.platform), - arch: mapNodeArchitectureToGOARCH(cliHost.arch), - }; - - const targetPlatformInfo = (() => { - if (common.buildxPlatform) { - const slash1 = common.buildxPlatform.indexOf('/'); - const slash2 = common.buildxPlatform.indexOf('/', slash1 + 1); - // `--platform linux/amd64/v3` `--platform linux/arm64/v8` - if (slash2 !== -1) { - return { - os: common.buildxPlatform.slice(0, slash1), - arch: common.buildxPlatform.slice(slash1 + 1, slash2), - variant: common.buildxPlatform.slice(slash2 + 1), - }; - } - // `--platform linux/amd64` and `--platform linux/arm64` - return { - os: common.buildxPlatform.slice(0, slash1), - arch: common.buildxPlatform.slice(slash1 + 1), - }; - } else { - // `--platform` omitted - return { - os: mapNodeOSToGOOS(cliHost.platform), - arch: mapNodeArchitectureToGOARCH(cliHost.arch), - }; - } - })(); - - const buildKitVersion = options.useBuildKit === 'never' ? undefined : (await dockerBuildKitVersion({ - cliHost, - dockerCLI: dockerPath, - dockerComposeCLI, - env: cliHost.env, - output, - buildPlatformInfo, - targetPlatformInfo - })); - - const dockerEngineVer = await dockerEngineVersion({ - cliHost, - dockerCLI: dockerPath, - dockerComposeCLI, - env: cliHost.env, - output, - buildPlatformInfo, - targetPlatformInfo - }); - - return { - common, - parsedAuthority, - dockerCLI: dockerPath, - isPodman: await isPodman({ exec: cliHost.exec, cmd: dockerPath, env: cliHost.env, output }), - dockerComposeCLI: dockerComposeCLI, - dockerEnv: cliHost.env, - workspaceMountConsistencyDefault: workspaceMountConsistency, - gpuAvailability: gpuAvailability || 'detect', - mountWorkspaceGitRoot, - mountGitWorktreeCommonDir, - updateRemoteUserUIDOnMacOS: false, - cacheMount: 'bind', - removeOnStartup: options.removeExistingContainer, - buildNoCache: options.buildNoCache, - expectExistingContainer: options.expectExistingContainer, - additionalMounts, - userRepositoryConfigurationPaths: [], - updateRemoteUserUIDDefault, - additionalCacheFroms: options.additionalCacheFroms, - buildKitVersion, - dockerEngineVersion: dockerEngineVer, - isTTY: process.stdout.isTTY || options.logFormat === 'json', - experimentalLockfile, - experimentalFrozenLockfile, - buildxPlatform: common.buildxPlatform, - buildxPush: common.buildxPush, - additionalLabels: options.additionalLabels, - buildxOutput: common.buildxOutput, - buildxCacheTo: common.buildxCacheTo, - buildPlatformInfo, - targetPlatformInfo - }; -} - -export interface LogOptions { - logLevel: LogLevel; - logFormat: LogFormat; - log: (text: string) => void; - terminalDimensions: LogDimensions | undefined; - onDidChangeTerminalDimensions?: Event; -} - -export function createLog(options: LogOptions, pkg: PackageConfiguration, sessionStart: Date, disposables: (() => Promise | undefined)[], omitHeader?: boolean, secrets?: Record) { - const header = omitHeader ? undefined : `${pkg.name} ${pkg.version}. Node.js ${process.version}. ${os.platform()} ${os.release()} ${os.arch()}.`; - const output = createLogFrom(options, sessionStart, header, secrets); - output.dimensions = options.terminalDimensions; - output.onDidChangeDimensions = options.onDidChangeTerminalDimensions; - disposables.push(() => output.join()); - return output; -} - -function createLogFrom({ log: write, logLevel, logFormat }: LogOptions, sessionStart: Date, header: string | undefined = undefined, secrets?: Record): Log & { join(): Promise } { - const handler = logFormat === 'json' ? createJSONLog(write, () => logLevel, sessionStart) : - process.stdout.isTTY ? createTerminalLog(write, () => logLevel, sessionStart) : - createPlainLog(write, () => logLevel); - const log = { - ...makeLog(createCombinedLog([maskSecrets(handler, secrets)], header)), - join: async () => { - // TODO: wait for write() to finish. - }, - }; - return log; -} - -function maskSecrets(handler: LogHandler, secrets?: Record): LogHandler { - if (secrets) { - const mask = '********'; - const secretValues = Object.values(secrets); - return replaceAllLog(handler, secretValues, mask); - } - - return handler; -} diff --git a/src/spec-node/devContainersSpecCLI.ts b/src/spec-node/devContainersSpecCLI.ts deleted file mode 100644 index 18c44136b..000000000 --- a/src/spec-node/devContainersSpecCLI.ts +++ /dev/null @@ -1,1444 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as path from 'path'; -import yargs, { Argv } from 'yargs'; -import textTable from 'text-table'; - -import * as jsonc from 'jsonc-parser'; - -import { createDockerParams, createLog, launch, ProvisionOptions } from './devContainers'; -import { SubstitutedConfig, createContainerProperties, envListToObj, inspectDockerImage, isDockerFileConfig, SubstituteConfig, addSubstitution, findContainerAndIdLabels, getCacheFolder, runAsyncHandler } from './utils'; -import { URI } from 'vscode-uri'; -import { ContainerError } from '../spec-common/errors'; -import { Log, LogDimensions, LogLevel, makeLog, mapLogLevel } from '../spec-utils/log'; -import { probeRemoteEnv, runLifecycleHooks, runRemoteCommand, UserEnvProbe, setupInContainer } from '../spec-common/injectHeadless'; -import { extendImage } from './containerFeatures'; -import { dockerCLI, DockerCLIParameters, dockerPtyCLI, inspectContainer } from '../spec-shutdown/dockerUtils'; -import { buildAndExtendDockerCompose, dockerComposeCLIConfig, getDefaultImageName, getProjectName, readDockerComposeConfig, readVersionPrefix } from './dockerCompose'; -import { DevContainerFromDockerComposeConfig, DevContainerFromDockerfileConfig, getDockerComposeFilePaths } from '../spec-configuration/configuration'; -import { workspaceFromPath } from '../spec-utils/workspaces'; -import { readDevContainerConfigFile } from './configContainer'; -import { getDefaultDevContainerConfigPath, getDevContainerConfigPathIn, uriToFsPath } from '../spec-configuration/configurationCommonUtils'; -import { CLIHost, getCLIHost } from '../spec-common/cliHost'; -import { loadNativeModule, processSignals } from '../spec-common/commonUtils'; -import { loadVersionInfo } from '../spec-configuration/containerFeaturesConfiguration'; -import { featuresTestOptions, featuresTestHandler } from './featuresCLI/test'; -import { featuresPackageHandler, featuresPackageOptions } from './featuresCLI/package'; -import { featuresPublishHandler, featuresPublishOptions } from './featuresCLI/publish'; -import { beforeContainerSubstitute, containerSubstitute, substitute } from '../spec-common/variableSubstitution'; -import { getPackageConfig, } from '../spec-utils/product'; -import { getDevcontainerMetadata, getImageBuildInfo, getImageMetadataFromContainer, ImageMetadataEntry, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata'; -import { templatesPublishHandler, templatesPublishOptions } from './templatesCLI/publish'; -import { templateApplyHandler, templateApplyOptions } from './templatesCLI/apply'; -import { featuresInfoHandler as featuresInfoHandler, featuresInfoOptions } from './featuresCLI/info'; -import { bailOut, buildNamedImageAndExtend } from './singleContainer'; -import { Event, NodeEventEmitter } from '../spec-utils/event'; -import { ensureNoDisallowedFeatures } from './disallowedFeatures'; -import { featuresResolveDependenciesHandler, featuresResolveDependenciesOptions } from './featuresCLI/resolveDependencies'; -import { getFeatureIdWithoutVersion } from '../spec-configuration/containerFeaturesOCI'; -import { featuresUpgradeHandler, featuresUpgradeOptions } from './upgradeCommand'; -import { readFeaturesConfig } from './featureUtils'; -import { featuresGenerateDocsHandler, featuresGenerateDocsOptions } from './featuresCLI/generateDocs'; -import { templatesGenerateDocsHandler, templatesGenerateDocsOptions } from './templatesCLI/generateDocs'; -import { mapNodeOSToGOOS, mapNodeArchitectureToGOARCH } from '../spec-configuration/containerCollectionsOCI'; -import { templateMetadataHandler, templateMetadataOptions } from './templatesCLI/metadata'; - -const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell'; - -const mountRegex = /^type=(bind|volume),source=([^,]+),target=([^,]+)(?:,external=(true|false))?$/; - -(async () => { - - const packageFolder = path.join(__dirname, '..', '..'); - const version = getPackageConfig().version; - const argv = process.argv.slice(2); - const restArgs = argv[0] === 'exec' && argv[1] !== '--help'; // halt-at-non-option doesn't work in subcommands: https://github.com/yargs/yargs/issues/1417 - const y = yargs([]) - .parserConfiguration({ - // By default, yargs allows `--no-myoption` to set a boolean `--myoption` to false - // Disable this to allow `--no-cache` on the `build` command to align with `docker build` syntax - 'boolean-negation': false, - 'halt-at-non-option': restArgs, - }) - .scriptName('devcontainer') - .version(version) - .demandCommand() - .strict(); - y.wrap(Math.min(120, y.terminalWidth())); - y.command('up', 'Create and run dev container', provisionOptions, provisionHandler); - y.command('set-up', 'Set up an existing container as a dev container', setUpOptions, setUpHandler); - y.command('build [path]', 'Build a dev container image', buildOptions, buildHandler); - y.command('run-user-commands', 'Run user commands', runUserCommandsOptions, runUserCommandsHandler); - y.command('read-configuration', 'Read configuration', readConfigurationOptions, readConfigurationHandler); - y.command('outdated', 'Show current and available versions', outdatedOptions, outdatedHandler); - y.command('upgrade', 'Upgrade lockfile', featuresUpgradeOptions, featuresUpgradeHandler); - y.command('features', 'Features commands', (y: Argv) => { - y.command('test [target]', 'Test Features', featuresTestOptions, featuresTestHandler); - y.command('package ', 'Package Features', featuresPackageOptions, featuresPackageHandler); - y.command('publish ', 'Package and publish Features', featuresPublishOptions, featuresPublishHandler); - y.command('info ', 'Fetch metadata for a published Feature', featuresInfoOptions, featuresInfoHandler); - y.command('resolve-dependencies', 'Read and resolve dependency graph from a configuration', featuresResolveDependenciesOptions, featuresResolveDependenciesHandler); - y.command('generate-docs', 'Generate documentation', featuresGenerateDocsOptions, featuresGenerateDocsHandler); - }); - y.command('templates', 'Templates commands', (y: Argv) => { - y.command('apply', 'Apply a template to the project', templateApplyOptions, templateApplyHandler); - y.command('publish ', 'Package and publish templates', templatesPublishOptions, templatesPublishHandler); - y.command('metadata ', 'Fetch a published Template\'s metadata', templateMetadataOptions, templateMetadataHandler); - y.command('generate-docs', 'Generate documentation', templatesGenerateDocsOptions, templatesGenerateDocsHandler); - }); - y.command(restArgs ? ['exec', '*'] : ['exec [args..]'], 'Execute a command on a running dev container', execOptions, execHandler); - y.epilog(`devcontainer@${version} ${packageFolder}`); - y.parse(restArgs ? argv.slice(1) : argv); - -})().catch(console.error); - -export type UnpackArgv = T extends Argv ? U : T; - -function provisionOptions(y: Argv) { - return y.options({ - 'docker-path': { type: 'string', description: 'Docker CLI path.' }, - 'docker-compose-path': { type: 'string', description: 'Docker Compose CLI path.' }, - 'container-data-folder': { type: 'string', description: 'Container data folder where user data inside the container will be stored.' }, - 'container-system-data-folder': { type: 'string', description: 'Container system data folder where system data inside the container will be stored.' }, - 'workspace-folder': { type: 'string', description: 'Workspace folder path. The devcontainer.json will be looked up relative to this path. If --id-label, --override-config, and --workspace-folder are not provided, this defaults to the current directory.' }, - 'workspace-mount-consistency': { choices: ['consistent' as 'consistent', 'cached' as 'cached', 'delegated' as 'delegated'], default: 'cached' as 'cached', description: 'Workspace mount consistency.' }, - 'gpu-availability': { choices: ['all' as 'all', 'detect' as 'detect', 'none' as 'none'], default: 'detect' as 'detect', description: 'Availability of GPUs in case the dev container requires any. `all` expects a GPU to be available.' }, - 'mount-workspace-git-root': { type: 'boolean', default: true, description: 'Mount the workspace using its Git root.' }, - 'mount-git-worktree-common-dir': { type: 'boolean', default: false, description: 'Mount the Git worktree common dir for Git operations to work in the container. This requires the worktree to be created with relative paths (`git worktree add --relative-paths`).' }, - 'id-label': { type: 'string', description: 'Id label(s) of the format name=value. These will be set on the container and used to query for an existing container. If no --id-label is given, one will be inferred from the --workspace-folder path.' }, - 'config': { type: 'string', description: 'devcontainer.json path. The default is to use .devcontainer/devcontainer.json or, if that does not exist, .devcontainer.json in the workspace folder.' }, - 'override-config': { type: 'string', description: 'devcontainer.json path to override any devcontainer.json in the workspace folder (or built-in configuration). This is required when there is no devcontainer.json otherwise.' }, - 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level for the --terminal-log-file. When set to trace, the log level for --log-file will also be set to trace.' }, - 'log-format': { choices: ['text' as 'text', 'json' as 'json'], default: 'text' as 'text', description: 'Log format.' }, - 'terminal-columns': { type: 'number', implies: ['terminal-rows'], description: 'Number of columns to render the output for. This is required for some of the subprocesses to correctly render their output.' }, - 'terminal-rows': { type: 'number', implies: ['terminal-columns'], description: 'Number of rows to render the output for. This is required for some of the subprocesses to correctly render their output.' }, - 'default-user-env-probe': { choices: ['none' as 'none', 'loginInteractiveShell' as 'loginInteractiveShell', 'interactiveShell' as 'interactiveShell', 'loginShell' as 'loginShell'], default: defaultDefaultUserEnvProbe, description: 'Default value for the devcontainer.json\'s "userEnvProbe".' }, - 'update-remote-user-uid-default': { choices: ['never' as 'never', 'on' as 'on', 'off' as 'off'], default: 'on' as 'on', description: 'Default for updating the remote user\'s UID and GID to the local user\'s one.' }, - 'remove-existing-container': { type: 'boolean', default: false, description: 'Removes the dev container if it already exists.' }, - 'build-no-cache': { type: 'boolean', default: false, description: 'Builds the image with `--no-cache` if the container does not exist.' }, - 'expect-existing-container': { type: 'boolean', default: false, description: 'Fail if the container does not exist.' }, - 'skip-post-create': { type: 'boolean', default: false, description: 'Do not run onCreateCommand, updateContentCommand, postCreateCommand, postStartCommand or postAttachCommand and do not install dotfiles.' }, - 'skip-non-blocking-commands': { type: 'boolean', default: false, description: 'Stop running user commands after running the command configured with waitFor or the updateContentCommand by default.' }, - prebuild: { type: 'boolean', default: false, description: 'Stop after onCreateCommand and updateContentCommand, rerunning updateContentCommand if it has run before.' }, - 'user-data-folder': { type: 'string', description: 'Host path to a directory that is intended to be persisted and share state between sessions.' }, - 'mount': { type: 'string', description: 'Additional mount point(s). Format: type=,source=,target=[,external=]' }, - 'remote-env': { type: 'string', description: 'Remote environment variables of the format name=value. These will be added when executing the user commands.' }, - 'cache-from': { type: 'string', description: 'Additional image to use as potential layer cache during image building' }, - 'cache-to': { type: 'string', description: 'Additional image to use as potential layer cache during image building' }, - 'buildkit': { choices: ['auto' as 'auto', 'never' as 'never'], default: 'auto' as 'auto', description: 'Control whether BuildKit should be used' }, - 'additional-features': { type: 'string', description: 'Additional features to apply to the dev container (JSON as per "features" section in devcontainer.json)' }, - 'skip-feature-auto-mapping': { type: 'boolean', default: false, hidden: true, description: 'Temporary option for testing.' }, - 'skip-post-attach': { type: 'boolean', default: false, description: 'Do not run postAttachCommand.' }, - 'dotfiles-repository': { type: 'string', description: 'URL of a dotfiles Git repository (e.g., https://github.com/owner/repository.git)' }, - 'dotfiles-install-command': { type: 'string', description: 'The command to run after cloning the dotfiles repository. Defaults to run the first file of `install.sh`, `install`, `bootstrap.sh`, `bootstrap`, `setup.sh` and `setup` found in the dotfiles repository`s root folder.' }, - 'dotfiles-target-path': { type: 'string', default: '~/dotfiles', description: 'The path to clone the dotfiles repository to. Defaults to `~/dotfiles`.' }, - 'container-session-data-folder': { type: 'string', description: 'Folder to cache CLI data, for example userEnvProbe results' }, - 'omit-config-remote-env-from-metadata': { type: 'boolean', default: false, hidden: true, description: 'Omit remoteEnv from devcontainer.json for container metadata label' }, - 'secrets-file': { type: 'string', description: 'Path to a json file containing secret environment variables as key-value pairs.' }, - 'experimental-lockfile': { type: 'boolean', default: false, hidden: true, description: 'Write lockfile' }, - 'experimental-frozen-lockfile': { type: 'boolean', default: false, hidden: true, description: 'Ensure lockfile remains unchanged' }, - 'omit-syntax-directive': { type: 'boolean', default: false, hidden: true, description: 'Omit Dockerfile syntax directives' }, - 'include-configuration': { type: 'boolean', default: false, description: 'Include configuration in result.' }, - 'include-merged-configuration': { type: 'boolean', default: false, description: 'Include merged configuration in result.' }, - }) - .check(argv => { - const idLabels = (argv['id-label'] && (Array.isArray(argv['id-label']) ? argv['id-label'] : [argv['id-label']])) as string[] | undefined; - if (idLabels?.some(idLabel => !/.+=.+/.test(idLabel))) { - throw new Error('Unmatched argument format: id-label must match ='); - } - // Default workspace-folder to current directory if not provided and no id-label or override-config - if (!argv['workspace-folder'] && !argv['id-label'] && !argv['override-config']) { - argv['workspace-folder'] = process.cwd(); - } - const mounts = (argv.mount && (Array.isArray(argv.mount) ? argv.mount : [argv.mount])) as string[] | undefined; - if (mounts?.some(mount => !mountRegex.test(mount))) { - throw new Error('Unmatched argument format: mount must match type=,source=,target=[,external=]'); - } - const remoteEnvs = (argv['remote-env'] && (Array.isArray(argv['remote-env']) ? argv['remote-env'] : [argv['remote-env']])) as string[] | undefined; - if (remoteEnvs?.some(remoteEnv => !/.+=.*/.test(remoteEnv))) { - throw new Error('Unmatched argument format: remote-env must match ='); - } - return true; - }); -} - -type ProvisionArgs = UnpackArgv>; - -function provisionHandler(args: ProvisionArgs) { - runAsyncHandler(provision.bind(null, args)); -} - -async function provision({ - 'user-data-folder': persistedFolder, - 'docker-path': dockerPath, - 'docker-compose-path': dockerComposePath, - 'container-data-folder': containerDataFolder, - 'container-system-data-folder': containerSystemDataFolder, - 'workspace-folder': workspaceFolderArg, - 'workspace-mount-consistency': workspaceMountConsistency, - 'gpu-availability': gpuAvailability, - 'mount-workspace-git-root': mountWorkspaceGitRoot, - 'mount-git-worktree-common-dir': mountGitWorktreeCommonDir, - 'id-label': idLabel, - config, - 'override-config': overrideConfig, - 'log-level': logLevel, - 'log-format': logFormat, - 'terminal-rows': terminalRows, - 'terminal-columns': terminalColumns, - 'default-user-env-probe': defaultUserEnvProbe, - 'update-remote-user-uid-default': updateRemoteUserUIDDefault, - 'remove-existing-container': removeExistingContainer, - 'build-no-cache': buildNoCache, - 'expect-existing-container': expectExistingContainer, - 'skip-post-create': skipPostCreate, - 'skip-non-blocking-commands': skipNonBlocking, - prebuild, - mount, - 'remote-env': addRemoteEnv, - 'cache-from': addCacheFrom, - 'cache-to': addCacheTo, - 'buildkit': buildkit, - 'additional-features': additionalFeaturesJson, - 'skip-feature-auto-mapping': skipFeatureAutoMapping, - 'skip-post-attach': skipPostAttach, - 'dotfiles-repository': dotfilesRepository, - 'dotfiles-install-command': dotfilesInstallCommand, - 'dotfiles-target-path': dotfilesTargetPath, - 'container-session-data-folder': containerSessionDataFolder, - 'omit-config-remote-env-from-metadata': omitConfigRemotEnvFromMetadata, - 'secrets-file': secretsFile, - 'experimental-lockfile': experimentalLockfile, - 'experimental-frozen-lockfile': experimentalFrozenLockfile, - 'omit-syntax-directive': omitSyntaxDirective, - 'include-configuration': includeConfig, - 'include-merged-configuration': includeMergedConfig, -}: ProvisionArgs) { - - const workspaceFolder = workspaceFolderArg ? path.resolve(process.cwd(), workspaceFolderArg) : undefined; - const addRemoteEnvs = addRemoteEnv ? (Array.isArray(addRemoteEnv) ? addRemoteEnv as string[] : [addRemoteEnv]) : []; - const addCacheFroms = addCacheFrom ? (Array.isArray(addCacheFrom) ? addCacheFrom as string[] : [addCacheFrom]) : []; - const additionalFeatures = additionalFeaturesJson ? jsonc.parse(additionalFeaturesJson) as Record> : {}; - const providedIdLabels = idLabel ? Array.isArray(idLabel) ? idLabel as string[] : [idLabel] : undefined; - - const cwd = workspaceFolder || process.cwd(); - const cliHost = await getCLIHost(cwd, loadNativeModule, logFormat === 'text'); - const secretsP = readSecretsFromFile({ secretsFile, cliHost }); - - const options: ProvisionOptions = { - dockerPath, - dockerComposePath, - containerDataFolder, - containerSystemDataFolder, - workspaceFolder, - workspaceMountConsistency, - gpuAvailability, - mountWorkspaceGitRoot, - mountGitWorktreeCommonDir, - configFile: config ? URI.file(path.resolve(process.cwd(), config)) : undefined, - overrideConfigFile: overrideConfig ? URI.file(path.resolve(process.cwd(), overrideConfig)) : undefined, - logLevel: mapLogLevel(logLevel), - logFormat, - log: text => process.stderr.write(text), - terminalDimensions: terminalColumns && terminalRows ? { columns: terminalColumns, rows: terminalRows } : undefined, - defaultUserEnvProbe, - removeExistingContainer, - buildNoCache, - expectExistingContainer, - postCreateEnabled: !skipPostCreate, - skipNonBlocking, - prebuild, - persistedFolder, - additionalMounts: mount ? (Array.isArray(mount) ? mount : [mount]).map(mount => { - const [, type, source, target, external] = mountRegex.exec(mount)!; - return { - type: type as 'bind' | 'volume', - source, - target, - external: external === 'true' - }; - }) : [], - dotfiles: { - repository: dotfilesRepository, - installCommand: dotfilesInstallCommand, - targetPath: dotfilesTargetPath, - }, - updateRemoteUserUIDDefault, - remoteEnv: envListToObj(addRemoteEnvs), - secretsP, - additionalCacheFroms: addCacheFroms, - useBuildKit: buildkit, - buildxPlatform: undefined, - buildxPush: false, - additionalLabels: [], - buildxOutput: undefined, - buildxCacheTo: addCacheTo, - additionalFeatures, - skipFeatureAutoMapping, - skipPostAttach, - containerSessionDataFolder, - skipPersistingCustomizationsFromFeatures: false, - omitConfigRemotEnvFromMetadata, - experimentalLockfile, - experimentalFrozenLockfile, - omitSyntaxDirective, - includeConfig, - includeMergedConfig, - }; - - const result = await doProvision(options, providedIdLabels); - const exitCode = result.outcome === 'error' ? 1 : 0; - await new Promise((resolve, reject) => { - process.stdout.write(JSON.stringify(result) + '\n', err => err ? reject(err) : resolve()); - }); - if (result.outcome === 'success') { - await result.finishBackgroundTasks(); - } - await result.dispose(); - process.exit(exitCode); -} - -async function doProvision(options: ProvisionOptions, providedIdLabels: string[] | undefined) { - const disposables: (() => Promise | undefined)[] = []; - const dispose = async () => { - await Promise.all(disposables.map(d => d())); - }; - try { - const result = await launch(options, providedIdLabels, disposables); - return { - outcome: 'success' as 'success', - dispose, - ...result, - }; - } catch (originalError) { - const originalStack = originalError?.stack; - const err = originalError instanceof ContainerError ? originalError : new ContainerError({ - description: 'An error occurred setting up the container.', - originalError - }); - if (originalStack) { - console.error(originalStack); - } - return { - outcome: 'error' as 'error', - message: err.message, - description: err.description, - containerId: err.containerId, - disallowedFeatureId: err.data.disallowedFeatureId, - didStopContainer: err.data.didStopContainer, - learnMoreUrl: err.data.learnMoreUrl, - dispose, - }; - } -} - -function setUpOptions(y: Argv) { - return y.options({ - 'docker-path': { type: 'string', description: 'Docker CLI path.' }, - 'container-data-folder': { type: 'string', description: 'Container data folder where user data inside the container will be stored.' }, - 'container-system-data-folder': { type: 'string', description: 'Container system data folder where system data inside the container will be stored.' }, - 'container-id': { type: 'string', required: true, description: 'Id of the container.' }, - 'config': { type: 'string', description: 'devcontainer.json path.' }, - 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level for the --terminal-log-file. When set to trace, the log level for --log-file will also be set to trace.' }, - 'log-format': { choices: ['text' as 'text', 'json' as 'json'], default: 'text' as 'text', description: 'Log format.' }, - 'terminal-columns': { type: 'number', implies: ['terminal-rows'], description: 'Number of columns to render the output for. This is required for some of the subprocesses to correctly render their output.' }, - 'terminal-rows': { type: 'number', implies: ['terminal-columns'], description: 'Number of rows to render the output for. This is required for some of the subprocesses to correctly render their output.' }, - 'default-user-env-probe': { choices: ['none' as 'none', 'loginInteractiveShell' as 'loginInteractiveShell', 'interactiveShell' as 'interactiveShell', 'loginShell' as 'loginShell'], default: defaultDefaultUserEnvProbe, description: 'Default value for the devcontainer.json\'s "userEnvProbe".' }, - 'skip-post-create': { type: 'boolean', default: false, description: 'Do not run onCreateCommand, updateContentCommand, postCreateCommand, postStartCommand or postAttachCommand and do not install dotfiles.' }, - 'skip-non-blocking-commands': { type: 'boolean', default: false, description: 'Stop running user commands after running the command configured with waitFor or the updateContentCommand by default.' }, - 'user-data-folder': { type: 'string', description: 'Host path to a directory that is intended to be persisted and share state between sessions.' }, - 'remote-env': { type: 'string', description: 'Remote environment variables of the format name=value. These will be added when executing the user commands.' }, - 'dotfiles-repository': { type: 'string', description: 'URL of a dotfiles Git repository (e.g., https://github.com/owner/repository.git)' }, - 'dotfiles-install-command': { type: 'string', description: 'The command to run after cloning the dotfiles repository. Defaults to run the first file of `install.sh`, `install`, `bootstrap.sh`, `bootstrap`, `setup.sh` and `setup` found in the dotfiles repository`s root folder.' }, - 'dotfiles-target-path': { type: 'string', default: '~/dotfiles', description: 'The path to clone the dotfiles repository to. Defaults to `~/dotfiles`.' }, - 'container-session-data-folder': { type: 'string', description: 'Folder to cache CLI data, for example userEnvProbe results' }, - 'include-configuration': { type: 'boolean', default: false, description: 'Include configuration in result.' }, - 'include-merged-configuration': { type: 'boolean', default: false, description: 'Include merged configuration in result.' }, - }) - .check(argv => { - const remoteEnvs = (argv['remote-env'] && (Array.isArray(argv['remote-env']) ? argv['remote-env'] : [argv['remote-env']])) as string[] | undefined; - if (remoteEnvs?.some(remoteEnv => !/.+=.*/.test(remoteEnv))) { - throw new Error('Unmatched argument format: remote-env must match ='); - } - return true; - }); -} - -type SetUpArgs = UnpackArgv>; - -function setUpHandler(args: SetUpArgs) { - runAsyncHandler(setUp.bind(null, args)); -} - -async function setUp(args: SetUpArgs) { - const result = await doSetUp(args); - const exitCode = result.outcome === 'error' ? 1 : 0; - await new Promise((resolve, reject) => { - process.stdout.write(JSON.stringify(result) + '\n', err => err ? reject(err) : resolve()); - }); - await result.dispose(); - process.exit(exitCode); -} - -async function doSetUp({ - 'user-data-folder': persistedFolder, - 'docker-path': dockerPath, - 'container-data-folder': containerDataFolder, - 'container-system-data-folder': containerSystemDataFolder, - 'container-id': containerId, - config: configParam, - 'log-level': logLevel, - 'log-format': logFormat, - 'terminal-rows': terminalRows, - 'terminal-columns': terminalColumns, - 'default-user-env-probe': defaultUserEnvProbe, - 'skip-post-create': skipPostCreate, - 'skip-non-blocking-commands': skipNonBlocking, - 'remote-env': addRemoteEnv, - 'dotfiles-repository': dotfilesRepository, - 'dotfiles-install-command': dotfilesInstallCommand, - 'dotfiles-target-path': dotfilesTargetPath, - 'container-session-data-folder': containerSessionDataFolder, - 'include-configuration': includeConfig, - 'include-merged-configuration': includeMergedConfig, -}: SetUpArgs) { - - const disposables: (() => Promise | undefined)[] = []; - const dispose = async () => { - await Promise.all(disposables.map(d => d())); - }; - try { - const addRemoteEnvs = addRemoteEnv ? (Array.isArray(addRemoteEnv) ? addRemoteEnv as string[] : [addRemoteEnv]) : []; - const configFile = configParam ? URI.file(path.resolve(process.cwd(), configParam)) : undefined; - const params = await createDockerParams({ - dockerPath, - dockerComposePath: undefined, - containerSessionDataFolder, - containerDataFolder, - containerSystemDataFolder, - workspaceFolder: undefined, - mountWorkspaceGitRoot: false, - mountGitWorktreeCommonDir: false, - configFile, - overrideConfigFile: undefined, - logLevel: mapLogLevel(logLevel), - logFormat, - log: text => process.stderr.write(text), - terminalDimensions: terminalColumns && terminalRows ? { columns: terminalColumns, rows: terminalRows } : undefined, - defaultUserEnvProbe, - removeExistingContainer: false, - buildNoCache: false, - expectExistingContainer: false, - postCreateEnabled: !skipPostCreate, - skipNonBlocking, - prebuild: false, - persistedFolder, - additionalMounts: [], - updateRemoteUserUIDDefault: 'never', - remoteEnv: envListToObj(addRemoteEnvs), - additionalCacheFroms: [], - useBuildKit: 'auto', - buildxPlatform: undefined, - buildxPush: false, - additionalLabels: [], - buildxOutput: undefined, - buildxCacheTo: undefined, - skipFeatureAutoMapping: false, - skipPostAttach: false, - skipPersistingCustomizationsFromFeatures: false, - dotfiles: { - repository: dotfilesRepository, - installCommand: dotfilesInstallCommand, - targetPath: dotfilesTargetPath, - }, - }, disposables); - - const { common } = params; - const { cliHost, output } = common; - const configs = configFile && await readDevContainerConfigFile(cliHost, undefined, configFile, params.mountWorkspaceGitRoot, params.mountGitWorktreeCommonDir, output, undefined, undefined); - if (configFile && !configs) { - throw new ContainerError({ description: `Dev container config (${uriToFsPath(configFile, cliHost.platform)}) not found.` }); - } - - const config0 = configs?.config || { - raw: {}, - config: {}, - substitute: value => substitute({ platform: cliHost.platform, env: cliHost.env }, value) - }; - - const container = await inspectContainer(params, containerId); - if (!container) { - bailOut(common.output, 'Dev container not found.'); - } - - const config = addSubstitution(config0, config => beforeContainerSubstitute(undefined, config)); - - const imageMetadata = getImageMetadataFromContainer(container, config, undefined, undefined, output).config; - const mergedConfig = mergeConfiguration(config.config, imageMetadata); - const containerProperties = await createContainerProperties(params, container.Id, configs?.workspaceConfig.workspaceFolder, mergedConfig.remoteUser); - const res = await setupInContainer(common, containerProperties, config.config, mergedConfig, lifecycleCommandOriginMapFromMetadata(imageMetadata)); - return { - outcome: 'success' as 'success', - configuration: includeConfig ? res.updatedConfig : undefined, - mergedConfiguration: includeMergedConfig ? res.updatedMergedConfig : undefined, - dispose, - }; - } catch (originalError) { - const originalStack = originalError?.stack; - const err = originalError instanceof ContainerError ? originalError : new ContainerError({ - description: 'An error occurred running user commands in the container.', - originalError - }); - if (originalStack) { - console.error(originalStack); - } - return { - outcome: 'error' as 'error', - message: err.message, - description: err.description, - dispose, - }; - } -} - -function buildOptions(y: Argv) { - return y.options({ - 'user-data-folder': { type: 'string', description: 'Host path to a directory that is intended to be persisted and share state between sessions.' }, - 'docker-path': { type: 'string', description: 'Docker CLI path.' }, - 'docker-compose-path': { type: 'string', description: 'Docker Compose CLI path.' }, - 'workspace-folder': { type: 'string', description: 'Workspace folder path. The devcontainer.json will be looked up relative to this path. If not provided, defaults to the current directory.' }, - 'config': { type: 'string', description: 'devcontainer.json path. The default is to use .devcontainer/devcontainer.json or, if that does not exist, .devcontainer.json in the workspace folder.' }, - 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' }, - 'log-format': { choices: ['text' as 'text', 'json' as 'json'], default: 'text' as 'text', description: 'Log format.' }, - 'no-cache': { type: 'boolean', default: false, description: 'Builds the image with `--no-cache`.' }, - 'image-name': { type: 'string', description: 'Image name.' }, - 'cache-from': { type: 'string', description: 'Additional image to use as potential layer cache' }, - 'cache-to': { type: 'string', description: 'A destination of buildx cache' }, - 'buildkit': { choices: ['auto' as 'auto', 'never' as 'never'], default: 'auto' as 'auto', description: 'Control whether BuildKit should be used' }, - 'platform': { type: 'string', description: 'Set target platforms.' }, - 'push': { type: 'boolean', default: false, description: 'Push to a container registry.' }, - 'label': { type: 'string', description: 'Provide key and value configuration that adds metadata to an image' }, - 'output': { type: 'string', description: 'Overrides the default behavior to load built images into the local docker registry. Valid options are the same ones provided to the --output option of docker buildx build.' }, - 'additional-features': { type: 'string', description: 'Additional features to apply to the dev container (JSON as per "features" section in devcontainer.json)' }, - 'skip-feature-auto-mapping': { type: 'boolean', default: false, hidden: true, description: 'Temporary option for testing.' }, - 'skip-persisting-customizations-from-features': { type: 'boolean', default: false, hidden: true, description: 'Do not save customizations from referenced Features as image metadata' }, - 'experimental-lockfile': { type: 'boolean', default: false, hidden: true, description: 'Write lockfile' }, - 'experimental-frozen-lockfile': { type: 'boolean', default: false, hidden: true, description: 'Ensure lockfile remains unchanged' }, - 'omit-syntax-directive': { type: 'boolean', default: false, hidden: true, description: 'Omit Dockerfile syntax directives' }, - }); -} - -type BuildArgs = UnpackArgv>; - -function buildHandler(args: BuildArgs) { - runAsyncHandler(build.bind(null, args)); -} - -async function build(args: BuildArgs) { - const result = await doBuild(args); - const exitCode = result.outcome === 'error' ? 1 : 0; - await new Promise((resolve, reject) => { - process.stdout.write(JSON.stringify(result) + '\n', err => err ? reject(err) : resolve()); - }); - await result.dispose(); - process.exit(exitCode); -} - -async function doBuild({ - 'user-data-folder': persistedFolder, - 'docker-path': dockerPath, - 'docker-compose-path': dockerComposePath, - 'workspace-folder': workspaceFolderArg, - config: configParam, - 'log-level': logLevel, - 'log-format': logFormat, - 'no-cache': buildNoCache, - 'image-name': argImageName, - 'cache-from': addCacheFrom, - 'buildkit': buildkit, - 'platform': buildxPlatform, - 'push': buildxPush, - 'label': buildxLabel, - 'output': buildxOutput, - 'cache-to': buildxCacheTo, - 'additional-features': additionalFeaturesJson, - 'skip-feature-auto-mapping': skipFeatureAutoMapping, - 'skip-persisting-customizations-from-features': skipPersistingCustomizationsFromFeatures, - 'experimental-lockfile': experimentalLockfile, - 'experimental-frozen-lockfile': experimentalFrozenLockfile, - 'omit-syntax-directive': omitSyntaxDirective, -}: BuildArgs) { - const disposables: (() => Promise | undefined)[] = []; - const dispose = async () => { - await Promise.all(disposables.map(d => d())); - }; - try { - const workspaceFolder = workspaceFolderArg ? path.resolve(process.cwd(), workspaceFolderArg) : process.cwd(); - const configFile: URI | undefined = configParam ? URI.file(path.resolve(process.cwd(), configParam)) : undefined; - const overrideConfigFile: URI | undefined = /* overrideConfig ? URI.file(path.resolve(process.cwd(), overrideConfig)) : */ undefined; - const addCacheFroms = addCacheFrom ? (Array.isArray(addCacheFrom) ? addCacheFrom as string[] : [addCacheFrom]) : []; - const additionalFeatures = additionalFeaturesJson ? jsonc.parse(additionalFeaturesJson) as Record> : {}; - const params = await createDockerParams({ - dockerPath, - dockerComposePath, - containerDataFolder: undefined, - containerSystemDataFolder: undefined, - workspaceFolder, - mountWorkspaceGitRoot: false, - mountGitWorktreeCommonDir: false, - configFile, - overrideConfigFile, - logLevel: mapLogLevel(logLevel), - logFormat, - log: text => process.stderr.write(text), - terminalDimensions: /* terminalColumns && terminalRows ? { columns: terminalColumns, rows: terminalRows } : */ undefined, // TODO - defaultUserEnvProbe: 'loginInteractiveShell', - removeExistingContainer: false, - buildNoCache, - expectExistingContainer: false, - postCreateEnabled: false, - skipNonBlocking: false, - prebuild: false, - persistedFolder, - additionalMounts: [], - updateRemoteUserUIDDefault: 'never', - remoteEnv: {}, - additionalCacheFroms: addCacheFroms, - useBuildKit: buildkit, - buildxPlatform, - buildxPush, - additionalLabels: [], - buildxOutput, - buildxCacheTo, - skipFeatureAutoMapping, - skipPostAttach: true, - skipPersistingCustomizationsFromFeatures: skipPersistingCustomizationsFromFeatures, - dotfiles: {}, - experimentalLockfile, - experimentalFrozenLockfile, - omitSyntaxDirective, - }, disposables); - - const { common, dockerComposeCLI } = params; - const { cliHost, env, output } = common; - const workspace = workspaceFromPath(cliHost.path, workspaceFolder); - const configPath = configFile ? configFile : workspace - ? (await getDevContainerConfigPathIn(cliHost, workspace.configFolderPath) - || (overrideConfigFile ? getDefaultDevContainerConfigPath(cliHost, workspace.configFolderPath) : undefined)) - : overrideConfigFile; - const configs = configPath && await readDevContainerConfigFile(cliHost, workspace, configPath, params.mountWorkspaceGitRoot, params.mountGitWorktreeCommonDir, output, undefined, overrideConfigFile) || undefined; - if (!configs) { - throw new ContainerError({ description: `Dev container config (${uriToFsPath(configFile || getDefaultDevContainerConfigPath(cliHost, workspace!.configFolderPath), cliHost.platform)}) not found.` }); - } - const configWithRaw = configs.config; - const { config } = configWithRaw; - let imageNameResult: string[] = ['']; - - if (buildxOutput && buildxPush) { - throw new ContainerError({ description: '--push true cannot be used with --output.' }); - } - - const buildParams: DockerCLIParameters = { cliHost, dockerCLI: params.dockerCLI, dockerComposeCLI, env, output, buildPlatformInfo: params.buildPlatformInfo, targetPlatformInfo: params.targetPlatformInfo }; - await ensureNoDisallowedFeatures(buildParams, config, additionalFeatures, undefined); - - // Support multiple use of `--image-name` - const imageNames = (argImageName && (Array.isArray(argImageName) ? argImageName : [argImageName]) as string[]) || undefined; - - // Support multiple use of `--label` - params.additionalLabels = (buildxLabel && (Array.isArray(buildxLabel) ? buildxLabel : [buildxLabel]) as string[]) || []; - - if (isDockerFileConfig(config)) { - - // Build the base image and extend with features etc. - let { updatedImageName } = await buildNamedImageAndExtend(params, configWithRaw as SubstitutedConfig, additionalFeatures, false, imageNames); - - if (imageNames) { - imageNameResult = imageNames; - } else { - imageNameResult = updatedImageName; - } - } else if ('dockerComposeFile' in config) { - - if (buildxPlatform || buildxPush) { - throw new ContainerError({ description: '--platform or --push not supported.' }); - } - - if (buildxOutput) { - throw new ContainerError({ description: '--output not supported.' }); - } - - if (buildxCacheTo) { - throw new ContainerError({ description: '--cache-to not supported.' }); - } - - const cwdEnvFile = cliHost.path.join(cliHost.cwd, '.env'); - const envFile = Array.isArray(config.dockerComposeFile) && config.dockerComposeFile.length === 0 && await cliHost.isFile(cwdEnvFile) ? cwdEnvFile : undefined; - const composeFiles = await getDockerComposeFilePaths(cliHost, config, cliHost.env, workspaceFolder); - - // If dockerComposeFile is an array, add -f in order. https://docs.docker.com/compose/extends/#multiple-compose-files - const composeGlobalArgs = ([] as string[]).concat(...composeFiles.map(composeFile => ['-f', composeFile])); - if (envFile) { - composeGlobalArgs.push('--env-file', envFile); - } - - const composeConfig = await readDockerComposeConfig(buildParams, composeFiles, envFile); - const projectName = await getProjectName(params, workspace, composeFiles, composeConfig); - const services = Object.keys(composeConfig.services || {}); - if (services.indexOf(config.service) === -1) { - throw new Error(`Service '${config.service}' configured in devcontainer.json not found in Docker Compose configuration.`); - } - - const versionPrefix = await readVersionPrefix(cliHost, composeFiles); - const infoParams = { ...params, common: { ...params.common, output: makeLog(buildParams.output, LogLevel.Info) } }; - const { overrideImageName } = await buildAndExtendDockerCompose(configWithRaw as SubstitutedConfig, projectName, infoParams, composeFiles, envFile, composeGlobalArgs, [config.service], params.buildNoCache || false, params.common.persistedFolder, 'docker-compose.devcontainer.build', versionPrefix, additionalFeatures, false, addCacheFroms); - - const service = composeConfig.services[config.service]; - const originalImageName = overrideImageName || service.image || getDefaultImageName(await buildParams.dockerComposeCLI(), projectName, config.service); - - if (imageNames) { - // Future improvement: Compose 2.6.0 (released 2022-05-30) added `tags` to the compose file. - if (params.isTTY) { - await Promise.all(imageNames.map(imageName => dockerPtyCLI(params, 'tag', originalImageName, imageName))); - } else { - await Promise.all(imageNames.map(imageName => dockerCLI(params, 'tag', originalImageName, imageName))); - } - imageNameResult = imageNames; - } else { - imageNameResult = originalImageName; - } - } else { - - if (!config.image) { - throw new ContainerError({ description: 'No image information specified in devcontainer.json.' }); - } - - await inspectDockerImage(params, config.image, true); - const { updatedImageName } = await extendImage(params, configWithRaw, config.image, imageNames || [], additionalFeatures, false); - - if (imageNames) { - imageNameResult = imageNames; - } else { - imageNameResult = updatedImageName; - } - } - - return { - outcome: 'success' as 'success', - imageName: imageNameResult, - dispose, - }; - } catch (originalError) { - const originalStack = originalError?.stack; - const err = originalError instanceof ContainerError ? originalError : new ContainerError({ - description: 'An error occurred building the container.', - originalError - }); - if (originalStack) { - console.error(originalStack); - } - return { - outcome: 'error' as 'error', - message: err.message, - description: err.description, - dispose, - }; - } -} - -function runUserCommandsOptions(y: Argv) { - return y.options({ - 'user-data-folder': { type: 'string', description: 'Host path to a directory that is intended to be persisted and share state between sessions.' }, - 'docker-path': { type: 'string', description: 'Docker CLI path.' }, - 'docker-compose-path': { type: 'string', description: 'Docker Compose CLI path.' }, - 'container-data-folder': { type: 'string', description: 'Container data folder where user data inside the container will be stored.' }, - 'container-system-data-folder': { type: 'string', description: 'Container system data folder where system data inside the container will be stored.' }, - 'workspace-folder': { type: 'string', description: 'Workspace folder path.The devcontainer.json will be looked up relative to this path. If --container-id, --id-label, and --workspace-folder are not provided, this defaults to the current directory.' }, - 'mount-workspace-git-root': { type: 'boolean', default: true, description: 'Mount the workspace using its Git root.' }, - 'mount-git-worktree-common-dir': { type: 'boolean', default: false, description: 'Mount the Git worktree common dir for Git operations to work in the container. This requires the worktree to be created with relative paths (`git worktree add --relative-paths`).' }, - 'container-id': { type: 'string', description: 'Id of the container to run the user commands for.' }, - 'id-label': { type: 'string', description: 'Id label(s) of the format name=value. If no --container-id is given the id labels will be used to look up the container. If no --id-label is given, one will be inferred from the --workspace-folder path.' }, - 'config': { type: 'string', description: 'devcontainer.json path. The default is to use .devcontainer/devcontainer.json or, if that does not exist, .devcontainer.json in the workspace folder.' }, - 'override-config': { type: 'string', description: 'devcontainer.json path to override any devcontainer.json in the workspace folder (or built-in configuration). This is required when there is no devcontainer.json otherwise.' }, - 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level for the --terminal-log-file. When set to trace, the log level for --log-file will also be set to trace.' }, - 'log-format': { choices: ['text' as 'text', 'json' as 'json'], default: 'text' as 'text', description: 'Log format.' }, - 'terminal-columns': { type: 'number', implies: ['terminal-rows'], description: 'Number of columns to render the output for. This is required for some of the subprocesses to correctly render their output.' }, - 'terminal-rows': { type: 'number', implies: ['terminal-columns'], description: 'Number of rows to render the output for. This is required for some of the subprocesses to correctly render their output.' }, - 'default-user-env-probe': { choices: ['none' as 'none', 'loginInteractiveShell' as 'loginInteractiveShell', 'interactiveShell' as 'interactiveShell', 'loginShell' as 'loginShell'], default: defaultDefaultUserEnvProbe, description: 'Default value for the devcontainer.json\'s "userEnvProbe".' }, - 'skip-non-blocking-commands': { type: 'boolean', default: false, description: 'Stop running user commands after running the command configured with waitFor or the updateContentCommand by default.' }, - prebuild: { type: 'boolean', default: false, description: 'Stop after onCreateCommand and updateContentCommand, rerunning updateContentCommand if it has run before.' }, - 'stop-for-personalization': { type: 'boolean', default: false, description: 'Stop for personalization.' }, - 'remote-env': { type: 'string', description: 'Remote environment variables of the format name=value. These will be added when executing the user commands.' }, - 'skip-feature-auto-mapping': { type: 'boolean', default: false, hidden: true, description: 'Temporary option for testing.' }, - 'skip-post-attach': { type: 'boolean', default: false, description: 'Do not run postAttachCommand.' }, - 'dotfiles-repository': { type: 'string', description: 'URL of a dotfiles Git repository (e.g., https://github.com/owner/repository.git)' }, - 'dotfiles-install-command': { type: 'string', description: 'The command to run after cloning the dotfiles repository. Defaults to run the first file of `install.sh`, `install`, `bootstrap.sh`, `bootstrap`, `setup.sh` and `setup` found in the dotfiles repository`s root folder.' }, - 'dotfiles-target-path': { type: 'string', default: '~/dotfiles', description: 'The path to clone the dotfiles repository to. Defaults to `~/dotfiles`.' }, - 'container-session-data-folder': { type: 'string', description: 'Folder to cache CLI data, for example userEnvProbe results' }, - 'secrets-file': { type: 'string', description: 'Path to a json file containing secret environment variables as key-value pairs.' }, - }) - .check(argv => { - const idLabels = (argv['id-label'] && (Array.isArray(argv['id-label']) ? argv['id-label'] : [argv['id-label']])) as string[] | undefined; - if (idLabels?.some(idLabel => !/.+=.+/.test(idLabel))) { - throw new Error('Unmatched argument format: id-label must match ='); - } - const remoteEnvs = (argv['remote-env'] && (Array.isArray(argv['remote-env']) ? argv['remote-env'] : [argv['remote-env']])) as string[] | undefined; - if (remoteEnvs?.some(remoteEnv => !/.+=.*/.test(remoteEnv))) { - throw new Error('Unmatched argument format: remote-env must match ='); - } - if (!argv['container-id'] && !idLabels?.length && !argv['workspace-folder']) { - argv['workspace-folder'] = process.cwd(); - } - return true; - }); -} - -type RunUserCommandsArgs = UnpackArgv>; - -function runUserCommandsHandler(args: RunUserCommandsArgs) { - runAsyncHandler(runUserCommands.bind(null, args)); -} -async function runUserCommands(args: RunUserCommandsArgs) { - const result = await doRunUserCommands(args); - const exitCode = result.outcome === 'error' ? 1 : 0; - await new Promise((resolve, reject) => { - process.stdout.write(JSON.stringify(result) + '\n', err => err ? reject(err) : resolve()); - }); - await result.dispose(); - process.exit(exitCode); -} - -async function doRunUserCommands({ - 'user-data-folder': persistedFolder, - 'docker-path': dockerPath, - 'docker-compose-path': dockerComposePath, - 'container-data-folder': containerDataFolder, - 'container-system-data-folder': containerSystemDataFolder, - 'workspace-folder': workspaceFolderArg, - 'mount-workspace-git-root': mountWorkspaceGitRoot, - 'mount-git-worktree-common-dir': mountGitWorktreeCommonDir, - 'container-id': containerId, - 'id-label': idLabel, - config: configParam, - 'override-config': overrideConfig, - 'log-level': logLevel, - 'log-format': logFormat, - 'terminal-rows': terminalRows, - 'terminal-columns': terminalColumns, - 'default-user-env-probe': defaultUserEnvProbe, - 'skip-non-blocking-commands': skipNonBlocking, - prebuild, - 'stop-for-personalization': stopForPersonalization, - 'remote-env': addRemoteEnv, - 'skip-feature-auto-mapping': skipFeatureAutoMapping, - 'skip-post-attach': skipPostAttach, - 'dotfiles-repository': dotfilesRepository, - 'dotfiles-install-command': dotfilesInstallCommand, - 'dotfiles-target-path': dotfilesTargetPath, - 'container-session-data-folder': containerSessionDataFolder, - 'secrets-file': secretsFile, -}: RunUserCommandsArgs) { - const disposables: (() => Promise | undefined)[] = []; - const dispose = async () => { - await Promise.all(disposables.map(d => d())); - }; - try { - const workspaceFolder = workspaceFolderArg ? path.resolve(process.cwd(), workspaceFolderArg) : undefined; - const providedIdLabels = idLabel ? Array.isArray(idLabel) ? idLabel as string[] : [idLabel] : undefined; - const addRemoteEnvs = addRemoteEnv ? (Array.isArray(addRemoteEnv) ? addRemoteEnv as string[] : [addRemoteEnv]) : []; - const configFile = configParam ? URI.file(path.resolve(process.cwd(), configParam)) : undefined; - const overrideConfigFile = overrideConfig ? URI.file(path.resolve(process.cwd(), overrideConfig)) : undefined; - - const cwd = workspaceFolder || process.cwd(); - const cliHost = await getCLIHost(cwd, loadNativeModule, logFormat === 'text'); - const secretsP = readSecretsFromFile({ secretsFile, cliHost }); - - const params = await createDockerParams({ - dockerPath, - dockerComposePath, - containerDataFolder, - containerSystemDataFolder, - workspaceFolder, - mountWorkspaceGitRoot, - mountGitWorktreeCommonDir, - configFile, - overrideConfigFile, - logLevel: mapLogLevel(logLevel), - logFormat, - log: text => process.stderr.write(text), - terminalDimensions: terminalColumns && terminalRows ? { columns: terminalColumns, rows: terminalRows } : undefined, - defaultUserEnvProbe, - removeExistingContainer: false, - buildNoCache: false, - expectExistingContainer: false, - postCreateEnabled: true, - skipNonBlocking, - prebuild, - persistedFolder, - additionalMounts: [], - updateRemoteUserUIDDefault: 'never', - remoteEnv: envListToObj(addRemoteEnvs), - additionalCacheFroms: [], - useBuildKit: 'auto', - buildxPlatform: undefined, - buildxPush: false, - additionalLabels: [], - buildxOutput: undefined, - buildxCacheTo: undefined, - skipFeatureAutoMapping, - skipPostAttach, - skipPersistingCustomizationsFromFeatures: false, - dotfiles: { - repository: dotfilesRepository, - installCommand: dotfilesInstallCommand, - targetPath: dotfilesTargetPath, - }, - containerSessionDataFolder, - secretsP, - }, disposables); - - const { common } = params; - const { output } = common; - const workspace = workspaceFolder ? workspaceFromPath(cliHost.path, workspaceFolder) : undefined; - const configPath = configFile ? configFile : workspace - ? (await getDevContainerConfigPathIn(cliHost, workspace.configFolderPath) - || (overrideConfigFile ? getDefaultDevContainerConfigPath(cliHost, workspace.configFolderPath) : undefined)) - : overrideConfigFile; - const configs = configPath && await readDevContainerConfigFile(cliHost, workspace, configPath, params.mountWorkspaceGitRoot, params.mountGitWorktreeCommonDir, output, undefined, overrideConfigFile) || undefined; - if ((configFile || workspaceFolder || overrideConfigFile) && !configs) { - throw new ContainerError({ description: `Dev container config (${uriToFsPath(configFile || getDefaultDevContainerConfigPath(cliHost, workspace!.configFolderPath), cliHost.platform)}) not found.` }); - } - - const config0 = configs?.config || { - raw: {}, - config: {}, - substitute: value => substitute({ platform: cliHost.platform, env: cliHost.env }, value) - }; - - const { container, idLabels } = await findContainerAndIdLabels(params, containerId, providedIdLabels, workspaceFolder, configPath?.fsPath); - if (!container) { - bailOut(common.output, 'Dev container not found.'); - } - - const config1 = addSubstitution(config0, config => beforeContainerSubstitute(envListToObj(idLabels), config)); - const config = addSubstitution(config1, config => containerSubstitute(cliHost.platform, config1.config.configFilePath, envListToObj(container.Config.Env), config)); - - const imageMetadata = getImageMetadataFromContainer(container, config, undefined, idLabels, output).config; - const mergedConfig = mergeConfiguration(config.config, imageMetadata); - const containerProperties = await createContainerProperties(params, container.Id, configs?.workspaceConfig.workspaceFolder, mergedConfig.remoteUser); - const updatedConfig = containerSubstitute(cliHost.platform, config.config.configFilePath, containerProperties.env, mergedConfig); - const remoteEnvP = probeRemoteEnv(common, containerProperties, updatedConfig); - const result = await runLifecycleHooks(common, lifecycleCommandOriginMapFromMetadata(imageMetadata), containerProperties, updatedConfig, remoteEnvP, secretsP, stopForPersonalization); - return { - outcome: 'success' as 'success', - result, - dispose, - }; - } catch (originalError) { - const originalStack = originalError?.stack; - const err = originalError instanceof ContainerError ? originalError : new ContainerError({ - description: 'An error occurred running user commands in the container.', - originalError - }); - if (originalStack) { - console.error(originalStack); - } - return { - outcome: 'error' as 'error', - message: err.message, - description: err.description, - dispose, - }; - } -} - - -function readConfigurationOptions(y: Argv) { - return y.options({ - 'user-data-folder': { type: 'string', description: 'Host path to a directory that is intended to be persisted and share state between sessions.' }, - 'docker-path': { type: 'string', description: 'Docker CLI path.' }, - 'docker-compose-path': { type: 'string', description: 'Docker Compose CLI path.' }, - 'workspace-folder': { type: 'string', description: 'Workspace folder path. The devcontainer.json will be looked up relative to this path. If --container-id, --id-label, and --workspace-folder are not provided, this defaults to the current directory.' }, - 'mount-workspace-git-root': { type: 'boolean', default: true, description: 'Mount the workspace using its Git root.' }, - 'mount-git-worktree-common-dir': { type: 'boolean', default: false, description: 'Mount the Git worktree common dir for Git operations to work in the container. This requires the worktree to be created with relative paths (`git worktree add --relative-paths`).' }, - 'container-id': { type: 'string', description: 'Id of the container to run the user commands for.' }, - 'id-label': { type: 'string', description: 'Id label(s) of the format name=value. If no --container-id is given the id labels will be used to look up the container. If no --id-label is given, one will be inferred from the --workspace-folder path.' }, - 'config': { type: 'string', description: 'devcontainer.json path. The default is to use .devcontainer/devcontainer.json or, if that does not exist, .devcontainer.json in the workspace folder.' }, - 'override-config': { type: 'string', description: 'devcontainer.json path to override any devcontainer.json in the workspace folder (or built-in configuration). This is required when there is no devcontainer.json otherwise.' }, - 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level for the --terminal-log-file. When set to trace, the log level for --log-file will also be set to trace.' }, - 'log-format': { choices: ['text' as 'text', 'json' as 'json'], default: 'text' as 'text', description: 'Log format.' }, - 'terminal-columns': { type: 'number', implies: ['terminal-rows'], description: 'Number of columns to render the output for. This is required for some of the subprocesses to correctly render their output.' }, - 'terminal-rows': { type: 'number', implies: ['terminal-columns'], description: 'Number of rows to render the output for. This is required for some of the subprocesses to correctly render their output.' }, - 'include-features-configuration': { type: 'boolean', default: false, description: 'Include features configuration.' }, - 'include-merged-configuration': { type: 'boolean', default: false, description: 'Include merged configuration.' }, - 'additional-features': { type: 'string', description: 'Additional features to apply to the dev container (JSON as per "features" section in devcontainer.json)' }, - 'skip-feature-auto-mapping': { type: 'boolean', default: false, hidden: true, description: 'Temporary option for testing.' }, - }) - .check(argv => { - const idLabels = (argv['id-label'] && (Array.isArray(argv['id-label']) ? argv['id-label'] : [argv['id-label']])) as string[] | undefined; - if (idLabels?.some(idLabel => !/.+=.+/.test(idLabel))) { - throw new Error('Unmatched argument format: id-label must match ='); - } - if (!argv['container-id'] && !idLabels?.length && !argv['workspace-folder']) { - argv['workspace-folder'] = process.cwd(); - } - return true; - }); -} - -type ReadConfigurationArgs = UnpackArgv>; - -function readConfigurationHandler(args: ReadConfigurationArgs) { - runAsyncHandler(readConfiguration.bind(null, args)); -} - -async function readConfiguration({ - // 'user-data-folder': persistedFolder, - 'docker-path': dockerPath, - 'docker-compose-path': dockerComposePath, - 'workspace-folder': workspaceFolderArg, - 'mount-workspace-git-root': mountWorkspaceGitRoot, - 'mount-git-worktree-common-dir': mountGitWorktreeCommonDir, - config: configParam, - 'override-config': overrideConfig, - 'container-id': containerId, - 'id-label': idLabel, - 'log-level': logLevel, - 'log-format': logFormat, - 'terminal-rows': terminalRows, - 'terminal-columns': terminalColumns, - 'include-features-configuration': includeFeaturesConfig, - 'include-merged-configuration': includeMergedConfig, - 'additional-features': additionalFeaturesJson, - 'skip-feature-auto-mapping': skipFeatureAutoMapping, -}: ReadConfigurationArgs) { - const disposables: (() => Promise | undefined)[] = []; - const dispose = async () => { - await Promise.all(disposables.map(d => d())); - }; - let output: Log | undefined; - try { - const workspaceFolder = workspaceFolderArg ? path.resolve(process.cwd(), workspaceFolderArg) : undefined; - const providedIdLabels = idLabel ? Array.isArray(idLabel) ? idLabel as string[] : [idLabel] : undefined; - const configFile = configParam ? URI.file(path.resolve(process.cwd(), configParam)) : undefined; - const overrideConfigFile = overrideConfig ? URI.file(path.resolve(process.cwd(), overrideConfig)) : undefined; - const cwd = workspaceFolder || process.cwd(); - const cliHost = await getCLIHost(cwd, loadNativeModule, logFormat === 'text'); - const extensionPath = path.join(__dirname, '..', '..'); - const sessionStart = new Date(); - const pkg = getPackageConfig(); - output = createLog({ - logLevel: mapLogLevel(logLevel), - logFormat, - log: text => process.stderr.write(text), - terminalDimensions: terminalColumns && terminalRows ? { columns: terminalColumns, rows: terminalRows } : undefined, - }, pkg, sessionStart, disposables); - - const workspace = workspaceFolder ? workspaceFromPath(cliHost.path, workspaceFolder) : undefined; - const configPath = configFile ? configFile : workspace - ? (await getDevContainerConfigPathIn(cliHost, workspace.configFolderPath) - || (overrideConfigFile ? getDefaultDevContainerConfigPath(cliHost, workspace.configFolderPath) : undefined)) - : overrideConfigFile; - const configs = configPath && await readDevContainerConfigFile(cliHost, workspace, configPath, mountWorkspaceGitRoot, mountGitWorktreeCommonDir, output, undefined, overrideConfigFile) || undefined; - if ((configFile || workspaceFolder || overrideConfigFile) && !configs) { - throw new ContainerError({ description: `Dev container config (${uriToFsPath(configFile || getDefaultDevContainerConfigPath(cliHost, workspace!.configFolderPath), cliHost.platform)}) not found.` }); - } - - let configuration = configs?.config || { - raw: {}, - config: {}, - substitute: value => substitute({ platform: cliHost.platform, env: cliHost.env }, value) - }; - - const dockerCLI = dockerPath || 'docker'; - const dockerComposeCLI = dockerComposeCLIConfig({ - exec: cliHost.exec, - env: cliHost.env, - output, - }, dockerCLI, dockerComposePath || 'docker-compose'); - const buildPlatformInfo = { - os: mapNodeOSToGOOS(cliHost.platform), - arch: mapNodeArchitectureToGOARCH(cliHost.arch), - }; - const params: DockerCLIParameters = { - cliHost, - dockerCLI, - dockerComposeCLI, - env: cliHost.env, - output, - buildPlatformInfo, - targetPlatformInfo: buildPlatformInfo - }; - const { container, idLabels } = await findContainerAndIdLabels(params, containerId, providedIdLabels, workspaceFolder, configPath?.fsPath); - if (container) { - configuration = addSubstitution(configuration, config => beforeContainerSubstitute(envListToObj(idLabels), config)); - configuration = addSubstitution(configuration, config => containerSubstitute(cliHost.platform, configuration.config.configFilePath, envListToObj(container.Config.Env), config)); - } - - const additionalFeatures = additionalFeaturesJson ? jsonc.parse(additionalFeaturesJson) as Record> : {}; - const needsFeaturesConfig = includeFeaturesConfig || (includeMergedConfig && !container); - const featuresConfiguration = needsFeaturesConfig ? await readFeaturesConfig(params, pkg, configuration.config, extensionPath, skipFeatureAutoMapping, additionalFeatures) : undefined; - let mergedConfig: MergedDevContainerConfig | undefined; - if (includeMergedConfig) { - let imageMetadata: ImageMetadataEntry[]; - if (container) { - imageMetadata = getImageMetadataFromContainer(container, configuration, featuresConfiguration, idLabels, output).config; - const substitute2: SubstituteConfig = config => containerSubstitute(cliHost.platform, configuration.config.configFilePath, envListToObj(container.Config.Env), config); - imageMetadata = imageMetadata.map(substitute2); - } else { - const imageBuildInfo = await getImageBuildInfo(params, configuration); - imageMetadata = getDevcontainerMetadata(imageBuildInfo.metadata, configuration, featuresConfiguration).config; - } - mergedConfig = mergeConfiguration(configuration.config, imageMetadata); - } - await new Promise((resolve, reject) => { - process.stdout.write(JSON.stringify({ - configuration: configuration.config, - workspace: configs?.workspaceConfig, - featuresConfiguration, - mergedConfiguration: mergedConfig, - }) + '\n', err => err ? reject(err) : resolve()); - }); - } catch (err) { - if (output) { - output.write(err && (err.stack || err.message) || String(err)); - } else { - console.error(err); - } - await dispose(); - process.exit(1); - } - await dispose(); - process.exit(0); -} - -function outdatedOptions(y: Argv) { - return y.options({ - 'user-data-folder': { type: 'string', description: 'Host path to a directory that is intended to be persisted and share state between sessions.' }, - 'workspace-folder': { type: 'string', description: 'Workspace folder path. The devcontainer.json will be looked up relative to this path. If --workspace-folder is not provided, defaults to the current directory.' }, - 'config': { type: 'string', description: 'devcontainer.json path. The default is to use .devcontainer/devcontainer.json or, if that does not exist, .devcontainer.json in the workspace folder.' }, - 'output-format': { choices: ['text' as 'text', 'json' as 'json'], default: 'text', description: 'Output format.' }, - 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level for the --terminal-log-file. When set to trace, the log level for --log-file will also be set to trace.' }, - 'log-format': { choices: ['text' as 'text', 'json' as 'json'], default: 'text' as 'text', description: 'Log format.' }, - 'terminal-columns': { type: 'number', implies: ['terminal-rows'], description: 'Number of columns to render the output for. This is required for some of the subprocesses to correctly render their output.' }, - 'terminal-rows': { type: 'number', implies: ['terminal-columns'], description: 'Number of rows to render the output for. This is required for some of the subprocesses to correctly render their output.' }, - }); -} - -type OutdatedArgs = UnpackArgv>; - -function outdatedHandler(args: OutdatedArgs) { - runAsyncHandler(outdated.bind(null, args)); -} - -async function outdated({ - // 'user-data-folder': persistedFolder, - 'workspace-folder': workspaceFolderArg, - config: configParam, - 'output-format': outputFormat, - 'log-level': logLevel, - 'log-format': logFormat, - 'terminal-rows': terminalRows, - 'terminal-columns': terminalColumns, -}: OutdatedArgs) { - const disposables: (() => Promise | undefined)[] = []; - const dispose = async () => { - await Promise.all(disposables.map(d => d())); - }; - let output: Log | undefined; - try { - const workspaceFolder = workspaceFolderArg ? path.resolve(process.cwd(), workspaceFolderArg) : process.cwd(); - const configFile = configParam ? URI.file(path.resolve(process.cwd(), configParam)) : undefined; - const cliHost = await getCLIHost(workspaceFolder, loadNativeModule, logFormat === 'text'); - const extensionPath = path.join(__dirname, '..', '..'); - const sessionStart = new Date(); - const pkg = getPackageConfig(); - output = createLog({ - logLevel: mapLogLevel(logLevel), - logFormat, - log: text => process.stderr.write(text), - terminalDimensions: terminalColumns && terminalRows ? { columns: terminalColumns, rows: terminalRows } : undefined, - }, pkg, sessionStart, disposables); - - const workspace = workspaceFromPath(cliHost.path, workspaceFolder); - const configPath = configFile ? configFile : await getDevContainerConfigPathIn(cliHost, workspace.configFolderPath); - const configs = configPath && await readDevContainerConfigFile(cliHost, workspace, configPath, true, false, output) || undefined; - if (!configs) { - throw new ContainerError({ description: `Dev container config (${uriToFsPath(configFile || getDefaultDevContainerConfigPath(cliHost, workspace!.configFolderPath), cliHost.platform)}) not found.` }); - } - - const cacheFolder = await getCacheFolder(cliHost); - const params = { - extensionPath, - cacheFolder, - cwd: cliHost.cwd, - output, - env: cliHost.env, - skipFeatureAutoMapping: false, - platform: cliHost.platform, - }; - - const outdated = await loadVersionInfo(params, configs.config.config); - await new Promise((resolve, reject) => { - let text; - if (outputFormat === 'text') { - const rows = Object.keys(outdated.features).map(key => { - const value = outdated.features[key]; - return [ getFeatureIdWithoutVersion(key), value.current, value.wanted, value.latest ] - .map(v => v === undefined ? '-' : v); - }); - const header = ['Feature', 'Current', 'Wanted', 'Latest']; - text = textTable([ - header, - ...rows, - ]); - } else { - text = JSON.stringify(outdated, undefined, process.stdout.isTTY ? ' ' : undefined); - } - process.stdout.write(text + '\n', err => err ? reject(err) : resolve()); - }); - } catch (err) { - if (output) { - output.write(err && (err.stack || err.message) || String(err)); - } else { - console.error(err); - } - await dispose(); - process.exit(1); - } - await dispose(); - process.exit(0); -} - -function execOptions(y: Argv) { - return y.options({ - 'user-data-folder': { type: 'string', description: 'Host path to a directory that is intended to be persisted and share state between sessions.' }, - 'docker-path': { type: 'string', description: 'Docker CLI path.' }, - 'docker-compose-path': { type: 'string', description: 'Docker Compose CLI path.' }, - 'container-data-folder': { type: 'string', description: 'Container data folder where user data inside the container will be stored.' }, - 'container-system-data-folder': { type: 'string', description: 'Container system data folder where system data inside the container will be stored.' }, - 'workspace-folder': { type: 'string', description: 'Workspace folder path. The devcontainer.json will be looked up relative to this path. If --container-id, --id-label, and --workspace-folder are not provided, this defaults to the current directory.' }, - 'mount-workspace-git-root': { type: 'boolean', default: true, description: 'Mount the workspace using its Git root.' }, - 'mount-git-worktree-common-dir': { type: 'boolean', default: false, description: 'Mount the Git worktree common dir for Git operations to work in the container. This requires the worktree to be created with relative paths (`git worktree add --relative-paths`).' }, - 'container-id': { type: 'string', description: 'Id of the container to run the user commands for.' }, - 'id-label': { type: 'string', description: 'Id label(s) of the format name=value. If no --container-id is given the id labels will be used to look up the container. If no --id-label is given, one will be inferred from the --workspace-folder path.' }, - 'config': { type: 'string', description: 'devcontainer.json path. The default is to use .devcontainer/devcontainer.json or, if that does not exist, .devcontainer.json in the workspace folder.' }, - 'override-config': { type: 'string', description: 'devcontainer.json path to override any devcontainer.json in the workspace folder (or built-in configuration). This is required when there is no devcontainer.json otherwise.' }, - 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level for the --terminal-log-file. When set to trace, the log level for --log-file will also be set to trace.' }, - 'log-format': { choices: ['text' as 'text', 'json' as 'json'], default: 'text' as 'text', description: 'Log format.' }, - 'terminal-columns': { type: 'number', implies: ['terminal-rows'], description: 'Number of columns to render the output for. This is required for some of the subprocesses to correctly render their output.' }, - 'terminal-rows': { type: 'number', implies: ['terminal-columns'], description: 'Number of rows to render the output for. This is required for some of the subprocesses to correctly render their output.' }, - 'default-user-env-probe': { choices: ['none' as 'none', 'loginInteractiveShell' as 'loginInteractiveShell', 'interactiveShell' as 'interactiveShell', 'loginShell' as 'loginShell'], default: defaultDefaultUserEnvProbe, description: 'Default value for the devcontainer.json\'s "userEnvProbe".' }, - 'remote-env': { type: 'string', description: 'Remote environment variables of the format name=value. These will be added when executing the user commands.' }, - 'skip-feature-auto-mapping': { type: 'boolean', default: false, hidden: true, description: 'Temporary option for testing.' }, - }) - .positional('cmd', { - type: 'string', - description: 'Command to execute.', - demandOption: true, - }).positional('args', { - type: 'string', - array: true, - description: 'Arguments to the command.', - demandOption: true, - }) - .check(argv => { - const idLabels = (argv['id-label'] && (Array.isArray(argv['id-label']) ? argv['id-label'] : [argv['id-label']])) as string[] | undefined; - if (idLabels?.some(idLabel => !/.+=.+/.test(idLabel))) { - throw new Error('Unmatched argument format: id-label must match ='); - } - const remoteEnvs = (argv['remote-env'] && (Array.isArray(argv['remote-env']) ? argv['remote-env'] : [argv['remote-env']])) as string[] | undefined; - if (remoteEnvs?.some(remoteEnv => !/.+=.*/.test(remoteEnv))) { - throw new Error('Unmatched argument format: remote-env must match ='); - } - if (!argv['container-id'] && !idLabels?.length && !argv['workspace-folder']) { - argv['workspace-folder'] = process.cwd(); - } - return true; - }); -} - -export type ExecArgs = UnpackArgv>; - -function execHandler(args: ExecArgs) { - runAsyncHandler(exec.bind(null, args)); -} - -async function exec(args: ExecArgs) { - const result = await doExec(args); - const exitCode = typeof result.code === 'number' && (result.code || !result.signal) ? result.code : - typeof result.signal === 'number' && result.signal > 0 ? 128 + result.signal : // 128 + signal number convention: https://tldp.org/LDP/abs/html/exitcodes.html - typeof result.signal === 'string' && processSignals[result.signal] ? 128 + processSignals[result.signal]! : 1; - await result.dispose(); - process.exit(exitCode); -} - -export async function doExec({ - 'user-data-folder': persistedFolder, - 'docker-path': dockerPath, - 'docker-compose-path': dockerComposePath, - 'container-data-folder': containerDataFolder, - 'container-system-data-folder': containerSystemDataFolder, - 'workspace-folder': workspaceFolderArg, - 'mount-workspace-git-root': mountWorkspaceGitRoot, - 'mount-git-worktree-common-dir': mountGitWorktreeCommonDir, - 'container-id': containerId, - 'id-label': idLabel, - config: configParam, - 'override-config': overrideConfig, - 'log-level': logLevel, - 'log-format': logFormat, - 'terminal-rows': terminalRows, - 'terminal-columns': terminalColumns, - 'default-user-env-probe': defaultUserEnvProbe, - 'remote-env': addRemoteEnv, - 'skip-feature-auto-mapping': skipFeatureAutoMapping, - _: restArgs, -}: ExecArgs & { _?: string[] }) { - const disposables: (() => Promise | undefined)[] = []; - const dispose = async () => { - await Promise.all(disposables.map(d => d())); - }; - let output: Log | undefined; - const isTTY = process.stdin.isTTY && process.stdout.isTTY || logFormat === 'json'; // If stdin or stdout is a pipe, we don't want to use a PTY. - try { - const workspaceFolder = workspaceFolderArg ? path.resolve(process.cwd(), workspaceFolderArg) : undefined; - const providedIdLabels = idLabel ? Array.isArray(idLabel) ? idLabel as string[] : [idLabel] : undefined; - const addRemoteEnvs = addRemoteEnv ? (Array.isArray(addRemoteEnv) ? addRemoteEnv as string[] : [addRemoteEnv]) : []; - const configFile = configParam ? URI.file(path.resolve(process.cwd(), configParam)) : undefined; - const overrideConfigFile = overrideConfig ? URI.file(path.resolve(process.cwd(), overrideConfig)) : undefined; - const params = await createDockerParams({ - dockerPath, - dockerComposePath, - containerDataFolder, - containerSystemDataFolder, - workspaceFolder, - mountWorkspaceGitRoot, - mountGitWorktreeCommonDir, - configFile, - overrideConfigFile, - logLevel: mapLogLevel(logLevel), - logFormat, - log: text => process.stderr.write(text), - terminalDimensions: terminalColumns && terminalRows ? { columns: terminalColumns, rows: terminalRows } : isTTY ? { columns: process.stdout.columns, rows: process.stdout.rows } : undefined, - onDidChangeTerminalDimensions: terminalColumns && terminalRows ? undefined : isTTY ? createStdoutResizeEmitter(disposables) : undefined, - defaultUserEnvProbe, - removeExistingContainer: false, - buildNoCache: false, - expectExistingContainer: false, - postCreateEnabled: true, - skipNonBlocking: false, - prebuild: false, - persistedFolder, - additionalMounts: [], - updateRemoteUserUIDDefault: 'never', - remoteEnv: envListToObj(addRemoteEnvs), - additionalCacheFroms: [], - useBuildKit: 'auto', - omitLoggerHeader: true, - buildxPlatform: undefined, - buildxPush: false, - additionalLabels: [], - buildxCacheTo: undefined, - skipFeatureAutoMapping, - buildxOutput: undefined, - skipPostAttach: false, - skipPersistingCustomizationsFromFeatures: false, - dotfiles: {} - }, disposables); - - const { common } = params; - const { cliHost } = common; - output = common.output; - const workspace = workspaceFolder ? workspaceFromPath(cliHost.path, workspaceFolder) : undefined; - const configPath = configFile ? configFile : workspace - ? (await getDevContainerConfigPathIn(cliHost, workspace.configFolderPath) - || (overrideConfigFile ? getDefaultDevContainerConfigPath(cliHost, workspace.configFolderPath) : undefined)) - : overrideConfigFile; - const configs = configPath && await readDevContainerConfigFile(cliHost, workspace, configPath, params.mountWorkspaceGitRoot, params.mountGitWorktreeCommonDir, output, undefined, overrideConfigFile) || undefined; - if ((configFile || workspaceFolder || overrideConfigFile) && !configs) { - throw new ContainerError({ description: `Dev container config (${uriToFsPath(configFile || getDefaultDevContainerConfigPath(cliHost, workspace!.configFolderPath), cliHost.platform)}) not found.` }); - } - - const config = configs?.config || { - raw: {}, - config: {}, - substitute: value => substitute({ platform: cliHost.platform, env: cliHost.env }, value) - }; - - const { container, idLabels } = await findContainerAndIdLabels(params, containerId, providedIdLabels, workspaceFolder, configPath?.fsPath); - if (!container) { - bailOut(common.output, 'Dev container not found.'); - } - const imageMetadata = getImageMetadataFromContainer(container, config, undefined, idLabels, output).config; - const mergedConfig = mergeConfiguration(config.config, imageMetadata); - const containerProperties = await createContainerProperties(params, container.Id, configs?.workspaceConfig.workspaceFolder, mergedConfig.remoteUser); - const updatedConfig = containerSubstitute(cliHost.platform, config.config.configFilePath, containerProperties.env, mergedConfig); - const remoteEnv = probeRemoteEnv(common, containerProperties, updatedConfig); - const remoteCwd = containerProperties.remoteWorkspaceFolder || containerProperties.homeFolder; - await runRemoteCommand({ ...common, output, stdin: process.stdin, ...(logFormat !== 'json' ? { stdout: process.stdout, stderr: process.stderr } : {}) }, containerProperties, restArgs || [], remoteCwd, { remoteEnv: await remoteEnv, pty: isTTY, print: 'continuous' }); - return { - code: 0, - dispose, - }; - - } catch (err) { - if (!err?.code && !err?.signal) { - if (output) { - output.write(err?.stack || err?.message || String(err), LogLevel.Error); - } else { - console.error(err?.stack || err?.message || String(err)); - } - } - return { - code: err?.code as number | undefined, - signal: err?.signal as string | number | undefined, - dispose, - }; - } -} - -function createStdoutResizeEmitter(disposables: (() => Promise | void)[]): Event { - const resizeListener = () => { - emitter.fire({ - rows: process.stdout.rows, - columns: process.stdout.columns - }); - }; - const emitter = new NodeEventEmitter({ - on: () => process.stdout.on('resize', resizeListener), - off: () => process.stdout.off('resize', resizeListener), - }); - disposables.push(() => emitter.dispose()); - return emitter.event; -} - -async function readSecretsFromFile(params: { output?: Log; secretsFile?: string; cliHost: CLIHost }) { - const { secretsFile, cliHost, output } = params; - if (!secretsFile) { - return {}; - } - - try { - const fileBuff = await cliHost.readFile(secretsFile); - const parseErrors: jsonc.ParseError[] = []; - const secrets = jsonc.parse(fileBuff.toString(), parseErrors) as Record; - if (parseErrors.length) { - throw new Error('Invalid json data'); - } - - return secrets; - } - catch (e) { - if (output) { - output.write(`Failed to read/parse secrets from file '${secretsFile}'`, LogLevel.Error); - } - - throw new ContainerError({ - description: 'Failed to read/parse secrets', - originalError: e - }); - } -} diff --git a/src/spec-node/disallowedFeatures.ts b/src/spec-node/disallowedFeatures.ts deleted file mode 100644 index 4f7b08a8a..000000000 --- a/src/spec-node/disallowedFeatures.ts +++ /dev/null @@ -1,58 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - *--------------------------------------------------------------------------------------------*/ - -import { DevContainerConfig } from '../spec-configuration/configuration'; -import { ContainerError } from '../spec-common/errors'; -import { DockerCLIParameters, dockerCLI } from '../spec-shutdown/dockerUtils'; -import { findDevContainer } from './singleContainer'; -import { DevContainerControlManifest, DisallowedFeature, getControlManifest } from '../spec-configuration/controlManifest'; -import { getCacheFolder } from './utils'; - - -export async function ensureNoDisallowedFeatures(params: DockerCLIParameters, config: DevContainerConfig, additionalFeatures: Record>, idLabels: string[] | undefined) { - const controlManifest = await getControlManifest(await getCacheFolder(params.cliHost), params.output); - const disallowed = Object.keys({ - ...config.features, - ...additionalFeatures, - }).map(configFeatureId => { - const disallowedFeatureEntry = findDisallowedFeatureEntry(controlManifest, configFeatureId); - return disallowedFeatureEntry ? { configFeatureId, disallowedFeatureEntry } : undefined; - }).filter(Boolean) as { - configFeatureId: string; - disallowedFeatureEntry: DisallowedFeature; - }[]; - - if (!disallowed.length) { - return; - } - - let stopped = false; - if (idLabels) { - const container = await findDevContainer(params, idLabels); - if (container?.State?.Status === 'running') { - await dockerCLI(params, 'stop', '-t', '0', container.Id); - stopped = true; - } - } - - const d = disallowed[0]!; - const documentationURL = d.disallowedFeatureEntry.documentationURL; - throw new ContainerError({ - description: `Cannot use the '${d.configFeatureId}' Feature since it was reported to be problematic. Please remove this Feature from your configuration and rebuild any dev container using it before continuing.${stopped ? ' The existing dev container was stopped.' : ''}${documentationURL ? ` See ${documentationURL} to learn more.` : ''}`, - data: { - disallowedFeatureId: d.configFeatureId, - didStopContainer: stopped, - learnMoreUrl: documentationURL, - }, - }); -} - -export function findDisallowedFeatureEntry(controlManifest: DevContainerControlManifest, featureId: string): DisallowedFeature | undefined { - return controlManifest.disallowedFeatures.find( - disallowedFeature => - featureId.startsWith(disallowedFeature.featureIdPrefix) && - (featureId.length === disallowedFeature.featureIdPrefix.length || // Feature id equal to prefix. - '/:@'.indexOf(featureId[disallowedFeature.featureIdPrefix.length]) !== -1) // Feature id with prefix and continued by separator. - ); -} diff --git a/src/spec-node/dockerCompose.ts b/src/spec-node/dockerCompose.ts deleted file mode 100644 index 8093464cc..000000000 --- a/src/spec-node/dockerCompose.ts +++ /dev/null @@ -1,764 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as yaml from 'js-yaml'; -import * as shellQuote from 'shell-quote'; - -import { createContainerProperties, startEventSeen, ResolverResult, getTunnelInformation, DockerResolverParameters, inspectDockerImage, getEmptyContextFolder, getFolderImageName, SubstitutedConfig, checkDockerSupportForGPU, isBuildKitImagePolicyError } from './utils'; -import { ContainerProperties, setupInContainer, ResolverProgress } from '../spec-common/injectHeadless'; -import { ContainerError } from '../spec-common/errors'; -import { Workspace } from '../spec-utils/workspaces'; -import { equalPaths, parseVersion, isEarlierVersion, CLIHost } from '../spec-common/commonUtils'; -import { ContainerDetails, inspectContainer, listContainers, DockerCLIParameters, dockerComposeCLI, dockerComposePtyCLI, PartialExecParameters, DockerComposeCLI, ImageDetails, toExecParameters, toPtyExecParameters, removeContainer } from '../spec-shutdown/dockerUtils'; -import { DevContainerFromDockerComposeConfig, getDockerComposeFilePaths } from '../spec-configuration/configuration'; -import { Log, LogLevel, makeLog, terminalEscapeSequences } from '../spec-utils/log'; -import { getExtendImageBuildInfo, updateRemoteUserUID } from './containerFeatures'; -import { Mount, parseMount } from '../spec-configuration/containerFeaturesConfiguration'; -import path from 'path'; -import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageBuildInfoFromImage, getImageMetadataFromContainer, ImageBuildInfo, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata'; -import { ensureDockerfileHasFinalStageName } from './dockerfileUtils'; -import { randomUUID } from 'crypto'; - -const projectLabel = 'com.docker.compose.project'; -const serviceLabel = 'com.docker.compose.service'; - -export async function openDockerComposeDevContainer(params: DockerResolverParameters, workspace: Workspace, config: SubstitutedConfig, idLabels: string[], additionalFeatures: Record>): Promise { - const { common, dockerCLI, dockerComposeCLI } = params; - const { cliHost, env, output } = common; - const buildParams: DockerCLIParameters = { cliHost, dockerCLI, dockerComposeCLI, env, output, buildPlatformInfo: params.buildPlatformInfo, targetPlatformInfo: params.targetPlatformInfo }; - return _openDockerComposeDevContainer(params, buildParams, workspace, config, getRemoteWorkspaceFolder(config.config), idLabels, additionalFeatures); -} - -async function _openDockerComposeDevContainer(params: DockerResolverParameters, buildParams: DockerCLIParameters, workspace: Workspace, configWithRaw: SubstitutedConfig, remoteWorkspaceFolder: string, idLabels: string[], additionalFeatures: Record>): Promise { - const { common } = params; - const { cliHost: buildCLIHost } = buildParams; - const { config } = configWithRaw; - - let container: ContainerDetails | undefined; - let containerProperties: ContainerProperties | undefined; - try { - - const composeFiles = await getDockerComposeFilePaths(buildCLIHost, config, buildCLIHost.env, buildCLIHost.cwd); - const cwdEnvFile = buildCLIHost.path.join(buildCLIHost.cwd, '.env'); - const envFile = Array.isArray(config.dockerComposeFile) && config.dockerComposeFile.length === 0 && await buildCLIHost.isFile(cwdEnvFile) ? cwdEnvFile : undefined; - const composeConfig = await readDockerComposeConfig(buildParams, composeFiles, envFile); - const projectName = await getProjectName(buildParams, workspace, composeFiles, composeConfig); - const containerId = await findComposeContainer(params, projectName, config.service); - if (params.expectExistingContainer && !containerId) { - throw new ContainerError({ description: 'The expected container does not exist.' }); - } - container = containerId ? await inspectContainer(params, containerId) : undefined; - - if (container && (params.removeOnStartup === true || params.removeOnStartup === container.Id)) { - const text = 'Removing existing container.'; - const start = common.output.start(text); - await removeContainer(params, container.Id); - common.output.stop(text, start); - container = undefined; - } - - // let collapsedFeaturesConfig: CollapsedFeaturesConfig | undefined; - if (!container || container.State.Status !== 'running') { - const res = await startContainer(params, buildParams, configWithRaw, projectName, composeFiles, envFile, composeConfig, container, idLabels, additionalFeatures); - container = await inspectContainer(params, res.containerId); - // collapsedFeaturesConfig = res.collapsedFeaturesConfig; - // } else { - // const labels = container.Config.Labels || {}; - // const featuresConfig = await generateFeaturesConfig(params.common, (await createFeaturesTempFolder(params.common)), config, async () => labels, getContainerFeaturesFolder); - // collapsedFeaturesConfig = collapseFeaturesConfig(featuresConfig); - } - - const imageMetadata = getImageMetadataFromContainer(container, configWithRaw, undefined, idLabels, common.output).config; - const mergedConfig = mergeConfiguration(configWithRaw.config, imageMetadata); - containerProperties = await createContainerProperties(params, container.Id, remoteWorkspaceFolder, mergedConfig.remoteUser); - - const { - remoteEnv: extensionHostEnv, - updatedConfig, - updatedMergedConfig, - } = await setupInContainer(common, containerProperties, config, mergedConfig, lifecycleCommandOriginMapFromMetadata(imageMetadata)); - - return { - params: common, - properties: containerProperties, - config: updatedConfig, - mergedConfig: updatedMergedConfig, - resolvedAuthority: { - extensionHostEnv, - }, - tunnelInformation: common.isLocalContainer ? getTunnelInformation(container) : {}, - dockerParams: params, - dockerContainerId: container.Id, - composeProjectName: projectName, - }; - - } catch (originalError) { - const err = originalError instanceof ContainerError ? originalError : new ContainerError({ - description: 'An error occurred setting up the container.', - originalError - }); - if (container) { - err.manageContainer = true; - err.params = params.common; - err.containerId = container.Id; - err.dockerParams = params; - } - if (containerProperties) { - err.containerProperties = containerProperties; - } - err.config = config; - throw err; - } -} - -export function getRemoteWorkspaceFolder(config: DevContainerFromDockerComposeConfig) { - return config.workspaceFolder || '/'; -} - -// exported for testing -export function getBuildInfoForService(composeService: any, cliHostPath: typeof path, localComposeFiles: string[]) { - // composeService should taken from readDockerComposeConfig - // the 'build' property can be a string or an object (https://docs.docker.com/compose/compose-file/build/#build-definition) - - const image = composeService.image as string | undefined; - const composeBuild = composeService.build; - if (!composeBuild) { - return { - image - }; - } - if (typeof (composeBuild) === 'string') { - return { - image, - build: { - context: composeBuild, - dockerfilePath: 'Dockerfile' - } - }; - } - return { - image, - build: { - dockerfilePath: (composeBuild.dockerfile as string | undefined) ?? 'Dockerfile', - context: (composeBuild.context as string | undefined) ?? cliHostPath.dirname(localComposeFiles[0]), - target: composeBuild.target as string | undefined, - args: composeBuild.args as Record | undefined, - } - }; -} - -export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConfig, projectName: string, params: DockerResolverParameters, localComposeFiles: string[], envFile: string | undefined, composeGlobalArgs: string[], runServices: string[], noCache: boolean, overrideFilePath: string, overrideFilePrefix: string, versionPrefix: string, additionalFeatures: Record>, canAddLabelsToContainer: boolean, additionalCacheFroms?: string[], noBuild?: boolean) { - - const { common, dockerCLI, dockerComposeCLI: dockerComposeCLIFunc } = params; - const { cliHost, env, output } = common; - const { config } = configWithRaw; - - const cliParams: DockerCLIParameters = { cliHost, dockerCLI, dockerComposeCLI: dockerComposeCLIFunc, env, output, buildPlatformInfo: params.buildPlatformInfo, targetPlatformInfo: params.targetPlatformInfo }; - const composeConfig = await readDockerComposeConfig(cliParams, localComposeFiles, envFile); - const composeService = composeConfig.services[config.service]; - - // determine base imageName for generated features build stage(s) - let baseName = 'dev_container_auto_added_stage_label'; - let dockerfile: string | undefined; - let imageBuildInfo: ImageBuildInfo; - const serviceInfo = getBuildInfoForService(composeService, cliHost.path, localComposeFiles); - if (serviceInfo.build) { - const { context, dockerfilePath, target } = serviceInfo.build; - const resolvedDockerfilePath = cliHost.path.isAbsolute(dockerfilePath) ? dockerfilePath : path.resolve(context, dockerfilePath); - const originalDockerfile = (await cliHost.readFile(resolvedDockerfilePath)).toString(); - dockerfile = originalDockerfile; - if (target) { - // Explictly set build target for the dev container build features on that - baseName = target; - } else { - // Use the last stage in the Dockerfile - // Find the last line that starts with "FROM" (possibly preceeded by white-space) - const { lastStageName, modifiedDockerfile } = ensureDockerfileHasFinalStageName(originalDockerfile, baseName); - baseName = lastStageName; - if (modifiedDockerfile) { - dockerfile = modifiedDockerfile; - } - } - imageBuildInfo = await getImageBuildInfoFromDockerfile(params, originalDockerfile, serviceInfo.build?.args || {}, serviceInfo.build?.target, configWithRaw.substitute); - } else { - imageBuildInfo = await getImageBuildInfoFromImage(params, composeService.image, configWithRaw.substitute); - } - - // determine whether we need to extend with features - const version = parseVersion((await params.dockerComposeCLI()).version); - const supportsAdditionalBuildContexts = !params.isPodman && version && !isEarlierVersion(version, [2, 17, 0]); - const optionalBuildKitParams = supportsAdditionalBuildContexts ? params : { ...params, buildKitVersion: undefined }; - const extendImageBuildInfo = await getExtendImageBuildInfo(optionalBuildKitParams, configWithRaw, baseName, imageBuildInfo, composeService.user, additionalFeatures, canAddLabelsToContainer); - - let overrideImageName: string | undefined; - let buildOverrideContent = ''; - if (extendImageBuildInfo?.featureBuildInfo) { - // Avoid retagging a previously pulled image. - if (!serviceInfo.build) { - overrideImageName = getFolderImageName(common); - buildOverrideContent += ` image: ${overrideImageName}\n`; - } - // Create overridden Dockerfile and generate docker-compose build override content - buildOverrideContent += ' build:\n'; - if (!dockerfile) { - dockerfile = `FROM ${composeService.image} AS ${baseName}\n`; - } - const { featureBuildInfo } = extendImageBuildInfo; - // We add a '# syntax' line at the start, so strip out any existing line - const syntaxMatch = dockerfile.match(/^\s*#\s*syntax\s*=.*[\r\n]/g); - if (syntaxMatch) { - dockerfile = dockerfile.slice(syntaxMatch[0].length); - } - let finalDockerfileContent = `${featureBuildInfo.dockerfilePrefixContent}${dockerfile}\n${featureBuildInfo.dockerfileContent}`; - const finalDockerfilePath = cliHost.path.join(featureBuildInfo?.dstFolder, 'Dockerfile-with-features'); - await cliHost.writeFile(finalDockerfilePath, Buffer.from(finalDockerfileContent)); - buildOverrideContent += ` dockerfile: ${finalDockerfilePath}\n`; - if (serviceInfo.build?.target) { - // Replace target. (Only when set because it is only supported with Docker Compose file version 3.4 and later.) - buildOverrideContent += ` target: ${featureBuildInfo.overrideTarget}\n`; - } - - if (!serviceInfo.build?.context) { - // need to supply a context as we don't have one inherited - const emptyDir = getEmptyContextFolder(common); - await cliHost.mkdirp(emptyDir); - buildOverrideContent += ` context: ${emptyDir}\n`; - } - // track additional build args to include - if (Object.keys(featureBuildInfo.buildArgs).length > 0 || params.buildKitVersion) { - buildOverrideContent += ' args:\n'; - if (params.buildKitVersion) { - buildOverrideContent += ' - BUILDKIT_INLINE_CACHE=1\n'; - } - for (const buildArg in featureBuildInfo.buildArgs) { - buildOverrideContent += ` - ${buildArg}=${featureBuildInfo.buildArgs[buildArg]}\n`; - } - } - - if (Object.keys(featureBuildInfo.buildKitContexts).length > 0) { - buildOverrideContent += ' additional_contexts:\n'; - for (const buildKitContext in featureBuildInfo.buildKitContexts) { - buildOverrideContent += ` - ${buildKitContext}=${featureBuildInfo.buildKitContexts[buildKitContext]}\n`; - } - } - } - - // Generate the docker-compose override and build - const args = ['--project-name', projectName, ...composeGlobalArgs]; - const additionalComposeOverrideFiles: string[] = []; - if (additionalCacheFroms && additionalCacheFroms.length > 0 || buildOverrideContent) { - const composeFolder = cliHost.path.join(overrideFilePath, 'docker-compose'); - await cliHost.mkdirp(composeFolder); - const composeOverrideFile = cliHost.path.join(composeFolder, `${overrideFilePrefix}-${Date.now()}.yml`); - const cacheFromOverrideContent = (additionalCacheFroms && additionalCacheFroms.length > 0) ? ` cache_from:\n${additionalCacheFroms.map(cacheFrom => ` - ${cacheFrom}\n`).join('\n')}` : ''; - const composeOverrideContent = `${versionPrefix}services: - ${config.service}: -${buildOverrideContent?.trimEnd()} -${cacheFromOverrideContent} -`; - output.write(`Docker Compose override file for building image:\n${composeOverrideContent}`); - await cliHost.writeFile(composeOverrideFile, Buffer.from(composeOverrideContent)); - additionalComposeOverrideFiles.push(composeOverrideFile); - args.push('-f', composeOverrideFile); - } - - if (!noBuild) { - args.push('build'); - if (noCache) { - args.push('--no-cache'); - // `docker build --pull` pulls local image: https://github.com/devcontainers/cli/issues/60 - if (!extendImageBuildInfo) { - args.push('--pull'); - } - } - if (runServices.length) { - args.push(...runServices); - if (runServices.indexOf(config.service) === -1) { - args.push(config.service); - } - } - try { - if (params.isTTY) { - const infoParams = { ...toPtyExecParameters(params, await dockerComposeCLIFunc()), output: makeLog(output, LogLevel.Info) }; - await dockerComposePtyCLI(infoParams, ...args); - } else { - const infoParams = { ...toExecParameters(params, await dockerComposeCLIFunc()), output: makeLog(output, LogLevel.Info), print: 'continuous' as 'continuous' }; - await dockerComposeCLI(infoParams, ...args); - } - } catch (err) { - if (isBuildKitImagePolicyError(err)) { - throw new ContainerError({ description: 'Could not resolve image due to policy.', originalError: err, data: { fileWithError: localComposeFiles[0] } }); - } - - throw err instanceof ContainerError ? err : new ContainerError({ description: 'An error occurred building the Docker Compose images.', originalError: err, data: { fileWithError: localComposeFiles[0] } }); - } - } - - return { - imageMetadata: getDevcontainerMetadata(imageBuildInfo.metadata, configWithRaw, extendImageBuildInfo?.featuresConfig), - additionalComposeOverrideFiles, - overrideImageName, - labels: extendImageBuildInfo?.labels, - }; -} - -async function checkForPersistedFile(cliHost: CLIHost, output: Log, files: string[], prefix: string) { - const file = files.find((f) => f.indexOf(prefix) > -1); - if (file) { - const composeFileExists = await cliHost.isFile(file); - - if (composeFileExists) { - output.write(`Restoring ${file} from persisted storage`); - return { - foundLabel: true, - fileExists: true, - file - }; - } else { - output.write(`Expected ${file} to exist, but it did not`, LogLevel.Error); - return { - foundLabel: true, - fileExists: false, - file - }; - } - } else { - output.write(`Expected to find a docker-compose file prefixed with ${prefix}, but did not.`, LogLevel.Error); - } - return { - foundLabel: false - }; -} - -async function startContainer(params: DockerResolverParameters, buildParams: DockerCLIParameters, configWithRaw: SubstitutedConfig, projectName: string, composeFiles: string[], envFile: string | undefined, composeConfig: any, container: ContainerDetails | undefined, idLabels: string[], additionalFeatures: Record>) { - const { common } = params; - const { persistedFolder, output } = common; - const { cliHost: buildCLIHost } = buildParams; - const { config } = configWithRaw; - const featuresBuildOverrideFilePrefix = 'docker-compose.devcontainer.build'; - const featuresStartOverrideFilePrefix = 'docker-compose.devcontainer.containerFeatures'; - - common.progress(ResolverProgress.StartingContainer); - - // If dockerComposeFile is an array, add -f in order. https://docs.docker.com/compose/extends/#multiple-compose-files - const composeGlobalArgs = ([] as string[]).concat(...composeFiles.map(composeFile => ['-f', composeFile])); - if (envFile) { - composeGlobalArgs.push('--env-file', envFile); - } - - const infoOutput = makeLog(buildParams.output, LogLevel.Info); - const services = Object.keys(composeConfig.services || {}); - if (services.indexOf(config.service) === -1) { - throw new ContainerError({ description: `Service '${config.service}' configured in devcontainer.json not found in Docker Compose configuration.`, data: { fileWithError: composeFiles[0] } }); - } - - let cancel: () => void; - const canceled = new Promise((_, reject) => cancel = reject); - const { started } = await startEventSeen(params, { [projectLabel]: projectName, [serviceLabel]: config.service }, canceled, common.output, common.getLogLevel() === LogLevel.Trace); // await getEvents, but only assign started. - - const service = composeConfig.services[config.service]; - const originalImageName = service.image || getDefaultImageName(await buildParams.dockerComposeCLI(), projectName, config.service); - - // Try to restore the 'third' docker-compose file and featuresConfig from persisted storage. - // This file may have been generated upon a Codespace creation. - const labels = container?.Config?.Labels; - output.write(`PersistedPath=${persistedFolder}, ContainerHasLabels=${!!labels}`); - - let didRestoreFromPersistedShare = false; - if (container) { - if (labels) { - // update args for `docker-compose up` to use cached overrides - const configFiles = labels['com.docker.compose.project.config_files']; - output.write(`Container was created with these config files: ${configFiles}`); - - // Parse out the full name of the 'containerFeatures' configFile - const files = configFiles?.split(',') ?? []; - const persistedBuildFile = await checkForPersistedFile(buildCLIHost, output, files, featuresBuildOverrideFilePrefix); - const persistedStartFile = await checkForPersistedFile(buildCLIHost, output, files, featuresStartOverrideFilePrefix); - if ((persistedBuildFile.fileExists || !persistedBuildFile.foundLabel) // require build file if in label - && persistedStartFile.fileExists // always require start file - ) { - didRestoreFromPersistedShare = true; - if (persistedBuildFile.fileExists) { - composeGlobalArgs.push('-f', persistedBuildFile.file); - } - if (persistedStartFile.fileExists) { - composeGlobalArgs.push('-f', persistedStartFile.file); - } - } - } - } - - if (!container || !didRestoreFromPersistedShare) { - const noBuild = !!container; //if we have an existing container, just recreate override files but skip the build - - const versionPrefix = await readVersionPrefix(buildCLIHost, composeFiles); - const infoParams = { ...params, common: { ...params.common, output: infoOutput } }; - const { imageMetadata, additionalComposeOverrideFiles, overrideImageName, labels } = await buildAndExtendDockerCompose(configWithRaw, projectName, infoParams, composeFiles, envFile, composeGlobalArgs, config.runServices ?? [], params.buildNoCache ?? false, persistedFolder, featuresBuildOverrideFilePrefix, versionPrefix, additionalFeatures, true, params.additionalCacheFroms, noBuild); - additionalComposeOverrideFiles.forEach(overrideFilePath => composeGlobalArgs.push('-f', overrideFilePath)); - - const currentImageName = overrideImageName || originalImageName; - let cache: Promise | undefined; - const imageDetails = () => cache || (cache = inspectDockerImage(params, currentImageName, true)); - const mergedConfig = mergeConfiguration(config, imageMetadata.config); - const updatedImageName = noBuild ? currentImageName : await updateRemoteUserUID(params, mergedConfig, currentImageName, imageDetails, service.user); - - // Save override docker-compose file to disk. - // Persisted folder is a path that will be maintained between sessions - // Note: As a fallback, persistedFolder is set to the build's tmpDir() directory - const additionalLabels = labels ? idLabels.concat(Object.keys(labels).map(key => `${key}=${labels[key]}`)) : idLabels; - const overrideFilePath = await writeFeaturesComposeOverrideFile(updatedImageName, currentImageName, mergedConfig, config, versionPrefix, imageDetails, service, additionalLabels, params.additionalMounts, persistedFolder, featuresStartOverrideFilePrefix, buildCLIHost, params, output); - - if (overrideFilePath) { - // Add file path to override file as parameter - composeGlobalArgs.push('-f', overrideFilePath); - } - } - - const args = ['--project-name', projectName, ...composeGlobalArgs]; - args.push('up', '-d'); - if (container || params.expectExistingContainer) { - args.push('--no-recreate'); - } - if (config.runServices && config.runServices.length) { - args.push(...config.runServices); - if (config.runServices.indexOf(config.service) === -1) { - args.push(config.service); - } - } - try { - if (params.isTTY) { - await dockerComposePtyCLI({ ...buildParams, output: infoOutput }, ...args); - } else { - await dockerComposeCLI({ ...buildParams, output: infoOutput }, ...args); - } - } catch (err) { - cancel!(); - - let description = 'An error occurred starting Docker Compose up.'; - if (err?.cmdOutput?.includes('Cannot create container for service app: authorization denied by plugin')) { - description = err.cmdOutput; - } - - throw new ContainerError({ description, originalError: err, data: { fileWithError: composeFiles[0] } }); - } - - await started; - return { - containerId: (await findComposeContainer(params, projectName, config.service))!, - }; -} - -export async function readVersionPrefix(cliHost: CLIHost, composeFiles: string[]) { - if (!composeFiles.length) { - return ''; - } - const firstComposeFile = (await cliHost.readFile(composeFiles[0])).toString(); - const version = (/^\s*(version:.*)$/m.exec(firstComposeFile) || [])[1]; - return version ? `${version}\n\n` : ''; -} - -export function getDefaultImageName(dockerComposeCLI: DockerComposeCLI, projectName: string, serviceName: string) { - const version = parseVersion(dockerComposeCLI.version); - const separator = version && isEarlierVersion(version, [2, 8, 0]) ? '_' : '-'; - return `${projectName}${separator}${serviceName}`; -} - -async function writeFeaturesComposeOverrideFile( - updatedImageName: string, - originalImageName: string, - mergedConfig: MergedDevContainerConfig, - config: DevContainerFromDockerComposeConfig, - versionPrefix: string, - imageDetails: () => Promise, - service: any, - additionalLabels: string[], - additionalMounts: Mount[], - overrideFilePath: string, - overrideFilePrefix: string, - buildCLIHost: CLIHost, - params: DockerResolverParameters, - output: Log, -) { - const composeOverrideContent = await generateFeaturesComposeOverrideContent(updatedImageName, originalImageName, mergedConfig, config, versionPrefix, imageDetails, service, additionalLabels, additionalMounts, params); - const overrideFileHasContents = !!composeOverrideContent && composeOverrideContent.length > 0 && composeOverrideContent.trim() !== ''; - if (overrideFileHasContents) { - output.write(`Docker Compose override file for creating container:\n${composeOverrideContent}`); - - const fileName = `${overrideFilePrefix}-${Date.now()}-${randomUUID()}.yml`; - const composeFolder = buildCLIHost.path.join(overrideFilePath, 'docker-compose'); - const composeOverrideFile = buildCLIHost.path.join(composeFolder, fileName); - output.write(`Writing ${fileName} to ${composeFolder}`); - await buildCLIHost.mkdirp(composeFolder); - await buildCLIHost.writeFile(composeOverrideFile, Buffer.from(composeOverrideContent)); - - return composeOverrideFile; - } else { - output.write('Override file was generated, but was empty and thus not persisted or included in the docker-compose arguments.'); - return undefined; - } -} - -async function generateFeaturesComposeOverrideContent( - updatedImageName: string, - originalImageName: string, - mergedConfig: MergedDevContainerConfig, - config: DevContainerFromDockerComposeConfig, - versionPrefix: string, - imageDetails: () => Promise, - service: any, - additionalLabels: string[], - additionalMounts: Mount[], - params: DockerResolverParameters, -) { - const overrideImage = updatedImageName !== originalImageName; - - const user = mergedConfig.containerUser; - const env = mergedConfig.containerEnv || {}; - const capAdd = mergedConfig.capAdd || []; - const securityOpts = mergedConfig.securityOpt || []; - const mounts = [ - ...mergedConfig.mounts || [], - ...additionalMounts, - ].map(m => typeof m === 'string' ? parseMount(m) : m); - const namedVolumeMounts = mounts.filter(m => m.type === 'volume' && m.source); - const customEntrypoints = mergedConfig.entrypoints || []; - const composeEntrypoint: string[] | undefined = typeof service.entrypoint === 'string' ? shellQuote.parse(service.entrypoint) : service.entrypoint; - const composeCommand: string[] | undefined = typeof service.command === 'string' ? shellQuote.parse(service.command) : service.command; - const { overrideCommand } = mergedConfig; - const userEntrypoint = overrideCommand ? [] : composeEntrypoint /* $ already escaped. */ - || ((await imageDetails()).Config.Entrypoint || []).map(c => c.replace(/\$/g, '$$$$')); // $ > $$ to escape docker-compose.yml's interpolation. - const userCommand = overrideCommand ? [] : composeCommand /* $ already escaped. */ - || (composeEntrypoint ? [/* Ignore image CMD per docker-compose.yml spec. */] : ((await imageDetails()).Config.Cmd || []).map(c => c.replace(/\$/g, '$$$$'))); // $ > $$ to escape docker-compose.yml's interpolation. - - const hasGpuRequirement = config.hostRequirements?.gpu; - const addGpuCapability = hasGpuRequirement && await checkDockerSupportForGPU(params); - if (hasGpuRequirement && hasGpuRequirement !== 'optional' && !addGpuCapability) { - params.common.output.write('No GPU support found yet a GPU was required - consider marking it as "optional"', LogLevel.Warning); - } - const gpuResources = addGpuCapability ? ` - deploy: - resources: - reservations: - devices: - - capabilities: [gpu]` : ''; - - return `${versionPrefix}services: - '${config.service}':${overrideImage ? ` - image: ${updatedImageName}` : ''} - entrypoint: ["/bin/sh", "-c", "echo Container started\\n -trap \\"exit 0\\" 15\\n -${customEntrypoints.join('\\n\n')}\\n -exec \\"$$@\\"\\n -while sleep 1 & wait $$!; do :; done", "-"${userEntrypoint.map(a => `, ${JSON.stringify(a)}`).join('')}]${userCommand !== composeCommand ? ` - command: ${JSON.stringify(userCommand)}` : ''}${mergedConfig.init ? ` - init: true` : ''}${user ? ` - user: ${user}` : ''}${Object.keys(env).length ? ` - environment:${Object.keys(env).map(key => ` - - '${key}=${String(env[key]).replace(/\n/g, '\\n').replace(/\$/g, '$$$$').replace(/'/g, '\'\'')}'`).join('')}` : ''}${mergedConfig.privileged ? ` - privileged: true` : ''}${capAdd.length ? ` - cap_add:${capAdd.map(cap => ` - - ${cap}`).join('')}` : ''}${securityOpts.length ? ` - security_opt:${securityOpts.map(securityOpt => ` - - ${securityOpt}`).join('')}` : ''}${additionalLabels.length ? ` - labels:${additionalLabels.map(label => ` - - '${label.replace(/\$/g, '$$$$').replace(/'/g, '\'\'')}'`).join('')}` : ''}${mounts.length ? ` - volumes:${mounts.map(m => ` - - ${convertMountToVolume(m)}`).join('')}` : ''}${gpuResources}${namedVolumeMounts.length ? ` -volumes:${namedVolumeMounts.map(m => ` - ${convertMountToVolumeTopLevelElement(m)}`).join('')}` : ''} -`; -} - -export async function readDockerComposeConfig(params: DockerCLIParameters, composeFiles: string[], envFile: string | undefined) { - try { - const composeGlobalArgs = ([] as string[]).concat(...composeFiles.map(composeFile => ['-f', composeFile])); - if (envFile) { - composeGlobalArgs.push('--env-file', envFile); - } - const composeCLI = await params.dockerComposeCLI(); - if ((parseVersion(composeCLI.version) || [])[0] >= 2) { - composeGlobalArgs.push('--profile', '*'); - } - try { - const partial = toExecParameters(params, 'dockerComposeCLI' in params ? await params.dockerComposeCLI() : undefined); - const { stdout } = await dockerComposeCLI({ - ...partial, - output: makeLog(params.output, LogLevel.Info), - print: 'onerror' - }, ...composeGlobalArgs, 'config'); - const stdoutStr = stdout.toString(); - params.output.write(stdoutStr); - return yaml.load(stdoutStr) || {} as any; - } catch (err) { - if (!Buffer.isBuffer(err?.stderr) || err?.stderr.toString().indexOf('UnicodeEncodeError') === -1) { - throw err; - } - // Upstream issues. https://github.com/microsoft/vscode-remote-release/issues/5308 - if (params.cliHost.platform === 'win32') { - const { cmdOutput } = await dockerComposePtyCLI({ - ...params, - output: makeLog({ - event: params.output.event, - dimensions: { - columns: 999999, - rows: 1, - }, - }, LogLevel.Info), - }, ...composeGlobalArgs, 'config'); - return yaml.load(cmdOutput.replace(terminalEscapeSequences, '')) || {} as any; - } - const { stdout } = await dockerComposeCLI({ - ...params, - env: { - ...params.env, - LANG: 'en_US.UTF-8', - LC_CTYPE: 'en_US.UTF-8', - } - }, ...composeGlobalArgs, 'config'); - const stdoutStr = stdout.toString(); - params.output.write(stdoutStr); - return yaml.load(stdoutStr) || {} as any; - } - } catch (err) { - throw err instanceof ContainerError ? err : new ContainerError({ description: 'An error occurred retrieving the Docker Compose configuration.', originalError: err, data: { fileWithError: composeFiles[0] } }); - } -} - -export async function findComposeContainer(params: DockerCLIParameters | DockerResolverParameters, projectName: string, serviceName: string): Promise { - const list = await listContainers(params, true, [ - `${projectLabel}=${projectName}`, - `${serviceLabel}=${serviceName}` - ]); - return list && list[0]; -} - -export async function getProjectName(params: DockerCLIParameters | DockerResolverParameters, workspace: Workspace, composeFiles: string[], composeConfig: any) { - const { cliHost } = 'cliHost' in params ? params : params.common; - const newProjectName = await useNewProjectName(params); - const envName = toProjectName(cliHost.env.COMPOSE_PROJECT_NAME || '', newProjectName); - if (envName) { - return envName; - } - try { - const envPath = cliHost.path.join(cliHost.cwd, '.env'); - const buffer = await cliHost.readFile(envPath); - const match = /^COMPOSE_PROJECT_NAME=(.+)$/m.exec(buffer.toString()); - const value = match && match[1].trim(); - const envFileName = toProjectName(value || '', newProjectName); - if (envFileName) { - return envFileName; - } - } catch (err) { - if (!(err && (err.code === 'ENOENT' || err.code === 'EISDIR'))) { - throw err; - } - } - if (composeConfig?.name) { - if (composeConfig.name !== 'devcontainer') { - return toProjectName(composeConfig.name, newProjectName); - } - // Check if 'devcontainer' is from a compose file or just the default. - for (let i = composeFiles.length - 1; i >= 0; i--) { - try { - const fragment = yaml.load((await cliHost.readFile(composeFiles[i])).toString()) || {} as any; - if (fragment.name) { - // Use composeConfig.name ('devcontainer') because fragment.name could include environment variables. - return toProjectName(composeConfig.name, newProjectName); - } - } catch (error) { - // Ignore when parsing fails due to custom yaml tags (e.g., !reset) - } - } - } - const configDir = workspace.configFolderPath; - const workingDir = composeFiles[0] ? cliHost.path.dirname(composeFiles[0]) : cliHost.cwd; // From https://github.com/docker/compose/blob/79557e3d3ab67c3697641d9af91866d7e400cfeb/compose/config/config.py#L290 - if (equalPaths(cliHost.platform, workingDir, cliHost.path.join(configDir, '.devcontainer'))) { - return toProjectName(`${cliHost.path.basename(configDir)}_devcontainer`, newProjectName); - } - return toProjectName(cliHost.path.basename(workingDir), newProjectName); -} - -function toProjectName(basename: string, newProjectName: boolean) { - // From https://github.com/docker/compose/blob/79557e3d3ab67c3697641d9af91866d7e400cfeb/compose/cli/command.py#L152 - if (!newProjectName) { - return basename.toLowerCase().replace(/[^a-z0-9]/g, ''); - } - return basename.toLowerCase().replace(/[^-_a-z0-9]/g, ''); -} - -async function useNewProjectName(params: DockerCLIParameters | DockerResolverParameters) { - try { - const version = parseVersion((await params.dockerComposeCLI()).version); - if (!version) { - return true; // Optimistically continue. - } - return !isEarlierVersion(version, [1, 21, 0]); // 1.21.0 changed allowed characters in project names (added hyphen and underscore). - } catch (err) { - return true; // Optimistically continue. - } -} - -export function dockerComposeCLIConfig(params: Omit, dockerCLICmd: string, dockerComposeCLICmd: string) { - let result: Promise; - return () => { - return result || (result = (async () => { - let v2 = true; - let stdout: Buffer; - try { - stdout = (await dockerComposeCLI({ - ...params, - cmd: dockerCLICmd, - }, 'compose', 'version', '--short')).stdout; - } catch (err) { - stdout = (await dockerComposeCLI({ - ...params, - cmd: dockerComposeCLICmd, - }, 'version', '--short')).stdout; - v2 = false; - } - const version = stdout.toString().trim(); - params.output.write(`Docker Compose version: ${version}`); - return { - version, - cmd: v2 ? dockerCLICmd : dockerComposeCLICmd, - args: v2 ? ['compose'] : [], - }; - })()); - }; -} - -/** - * Convert mount command arguments to Docker Compose volume - * @param mount - * @returns mount command representation for Docker compose - */ -function convertMountToVolume(mount: Mount): string { - let volume: string = ''; - - if (mount.source) { - volume = `${mount.source}:`; - } - - volume += mount.target; - - return volume; -} - -/** - * Convert mount command arguments to volume top-level element - * @param mount - * @returns mount object representation as volumes top-level element - */ -function convertMountToVolumeTopLevelElement(mount: Mount): string { - let volume: string = ` - ${mount.source}:`; - - if (mount.external) { - volume += '\n external: true'; - } - - return volume; -} diff --git a/src/spec-node/dockerfileUtils.ts b/src/spec-node/dockerfileUtils.ts deleted file mode 100644 index 532324f2b..000000000 --- a/src/spec-node/dockerfileUtils.ts +++ /dev/null @@ -1,294 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as semver from 'semver'; -import { Mount } from '../spec-configuration/containerFeaturesConfiguration'; - - -const findFromLines = new RegExp(/^(?\s*FROM.*)/, 'gmi'); -const parseFromLine = /FROM\s+(?--platform=\S+\s+)?(?"?[^\s]+"?)(\s+AS\s+(?