diff --git a/Package.resolved b/Package.resolved index e1d69b93..061dd0c7 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "eb2a57fb4e4c2c83ff1f1fa55631db81988fc6e2e576a05a3cc5c8ed69432c3a", + "originHash" : "a67130742a321d259d3288347f30ac3f5520a64d98c183a85694b6bca44c4a64", "pins" : [ { "identity" : "aexml", @@ -352,6 +352,15 @@ "version" : "2.11.0" } }, + { + "identity" : "swift-subprocess", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swiftlang/swift-subprocess", + "state" : { + "revision" : "13d087685b95d64d6aac9b94500d347bbe84c39b", + "version" : "0.4.0" + } + }, { "identity" : "swift-syntax", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index 01fe34fb..75b84658 100644 --- a/Package.swift +++ b/Package.swift @@ -55,6 +55,7 @@ let package = Package( .package(url: "https://github.com/apple/swift-nio", from: "2.77.0"), .package(url: "https://github.com/swiftlang/swift-docc-plugin", from: "1.0.0"), + .package(url: "https://github.com/swiftlang/swift-subprocess", .upToNextMinor(from: "0.4.0")), .package(url: "https://github.com/swift-server/async-http-client.git", from: "1.23.0"), .package(url: "https://github.com/swift-server/swift-openapi-async-http-client", from: "1.0.0"), @@ -92,7 +93,16 @@ let package = Package( exclude: ["openapi-generator-config.yaml", "patch.js"] ), // common utilities shared across xtool targets - .target(name: "XUtils"), + .target( + name: "XUtils", + dependencies: [ + .product( + name: "Subprocess", + package: "swift-subprocess", + condition: .when(platforms: [.linux, .macOS]) + ), + ] + ), .target( name: "XKit", dependencies: [ diff --git a/Sources/PackLib/BuildSettings.swift b/Sources/PackLib/BuildSettings.swift index 8a3e09db..4ba0efc2 100644 --- a/Sources/PackLib/BuildSettings.swift +++ b/Sources/PackLib/BuildSettings.swift @@ -1,9 +1,11 @@ import Foundation +import Subprocess +import XUtils public struct BuildSettings: Sendable { private static let customBinDir = // this is the same option used by SwiftPM itself for dev builds - ProcessInfo.processInfo.environment["SWIFTPM_CUSTOM_BIN_DIR"].map { URL(fileURLWithPath: $0) } + ProcessInfo.processInfo.environment["SWIFTPM_CUSTOM_BIN_DIR"].map { FilePath($0) } private static let envURL = URL(fileURLWithPath: "/usr/bin/env") @@ -34,18 +36,16 @@ public struct BuildSettings: Sendable { #if os(macOS) private static func xcrun(_ arguments: [String]) async throws -> String { - let xcrun = Process() - let pipe = Pipe() - xcrun.executableURL = URL(fileURLWithPath: "/usr/bin/xcrun") - xcrun.arguments = arguments - xcrun.standardOutput = pipe - try await xcrun.runUntilExit() - return String(decoding: pipe.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self) - .trimmingCharacters(in: .whitespacesAndNewlines) + let result = try await Subprocess.run( + .path("/usr/bin/xcrun"), + arguments: .init(arguments), + output: .string(limit: .max) + ).checkSuccess() + return result.standardOutput?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" } private static let swiftURL = Task { - try await URL(fileURLWithPath: xcrun(["-f", "swift"])) + try await FilePath(xcrun(["-f", "swift"])) } #endif @@ -53,12 +53,11 @@ public struct BuildSettings: Sendable { forTool tool: String, arguments: [String], packagePathOverride: String? = nil - ) async throws -> Process { - let process = Process() - + ) async throws -> Subprocess.Configuration { + let executable: Executable let baseArguments: [String] if let customBinDir = Self.customBinDir { - process.executableURL = customBinDir.appendingPathComponent("swift-\(tool)") + executable = .path(customBinDir.appending("swift-\(tool)")) baseArguments = [] } else { #if os(macOS) @@ -67,9 +66,9 @@ public struct BuildSettings: Sendable { // 1) invoking the real swift executable (located with `xcrun -f`) and // 2) explicitly removing SDKROOT from the env, as it may be inherited // through the `swift run pack` invocation. - process.executableURL = try await Self.swiftURL.value + executable = .path(try await Self.swiftURL.value) #else - process.executableURL = try await ToolRegistry.locate("swift") + executable = .name("swift") #endif baseArguments = [tool] } @@ -79,11 +78,18 @@ public struct BuildSettings: Sendable { "--configuration", configuration.rawValue, ] - var env = ProcessInfo.processInfo.environment - env.removeValue(forKey: "SDKROOT") - process.environment = env - process.arguments = baseArguments + extraArguments + sdkOptions + options + arguments - return process + var rawEnv = ProcessInfo.processInfo.environment + rawEnv.removeValue(forKey: "SDKROOT") + let env = Dictionary(uniqueKeysWithValues: rawEnv.map { + (Environment.Key(rawValue: $0)!, $1) + }) + + return Configuration( + executable, + arguments: .init(baseArguments + extraArguments + sdkOptions + options + arguments), + environment: .custom(env), + platformOptions: .withGracefulShutDown, + ) } } diff --git a/Sources/PackLib/Packer.swift b/Sources/PackLib/Packer.swift index 9712ead2..6b7417ec 100644 --- a/Sources/PackLib/Packer.swift +++ b/Sources/PackLib/Packer.swift @@ -1,5 +1,6 @@ import Foundation import XUtils +import Subprocess public struct Packer: Sendable { public let buildSettings: BuildSettings @@ -54,7 +55,7 @@ public struct Packer: Sendable { try Data().write(to: sources.appendingPathComponent("stub.c", isDirectory: false)) } - let builder = try await buildSettings.swiftPMInvocation( + let buildConfig = try await buildSettings.swiftPMInvocation( forTool: "build", arguments: [ "--package-path", packageDir.path, @@ -68,8 +69,11 @@ public struct Packer: Sendable { "--disable-automatic-resolution", ] ) - builder.standardOutput = FileHandle.standardError - try await builder.runUntilExit() + try await Subprocess.run( + buildConfig, + output: .standardError + ) + .checkSuccess() } public func pack() async throws -> URL { diff --git a/Sources/PackLib/Planner.swift b/Sources/PackLib/Planner.swift index b3f3e93b..2a746e65 100644 --- a/Sources/PackLib/Planner.swift +++ b/Sources/PackLib/Planner.swift @@ -1,5 +1,6 @@ import Foundation import XUtils +import Subprocess public struct Planner: Sendable { public var buildSettings: BuildSettings @@ -243,16 +244,17 @@ public struct Planner: Sendable { } private func _dumpAction(arguments: [String], path: String) async throws -> Data { - let dump = try await buildSettings.swiftPMInvocation( + let dumpConfig = try await buildSettings.swiftPMInvocation( forTool: "package", arguments: arguments, packagePathOverride: path ) - let pipe = Pipe() - dump.standardOutput = pipe - async let task = Data(reading: pipe.fileHandleForReading) - try await dump.runUntilExit() - return try await task + return try await Subprocess.run( + dumpConfig, + output: .data(limit: .max) + ) + .checkSuccess() + .standardOutput } private func selectLibrary( diff --git a/Sources/PackLib/Process+Helpers.swift b/Sources/PackLib/Process+Helpers.swift deleted file mode 100644 index 3b47bcb1..00000000 --- a/Sources/PackLib/Process+Helpers.swift +++ /dev/null @@ -1,63 +0,0 @@ -import Foundation - -extension Process { - /// Launches the process and suspends until the receiver is finished. - /// - /// - Parameter onCancel: The action to take if the current - /// task is cancelled. - public func runUntilExit(onCancel: TaskCancelAction = .interrupt) async throws { - try Task.checkCancellation() - - let (terminationStream, terminationContinuation) = AsyncStream.makeStream() - terminationHandler = { _ in - terminationContinuation.finish() - } - - do { - try run() - } catch { - terminationContinuation.finish() - throw error - } - - await withTaskCancellationHandler { - for await _ in terminationStream {} - } onCancel: { - switch onCancel { - case .interrupt: - interrupt() - case .terminate: - terminate() - case .ignore: - break - } - } - - try Task.checkCancellation() - - switch terminationReason { - case .exit where terminationStatus == 0: - break - case .exit: - throw Failure.exit(terminationStatus) - case .uncaughtSignal: - throw Failure.uncaughtSignal(terminationStatus) - @unknown default: - break - } - } - - public enum Failure: Error { - case exit(CInt) - case uncaughtSignal(CInt) - } - - public enum TaskCancelAction: Sendable { - /// Sends `SIGINT` to the process. - case interrupt - /// Sends `SIGTERM` to the process. - case terminate - /// Don't participate in cooperative cancellation. - case ignore - } -} diff --git a/Sources/PackLib/ToolRegistry.swift b/Sources/PackLib/ToolRegistry.swift index 0d82e280..5643203d 100644 --- a/Sources/PackLib/ToolRegistry.swift +++ b/Sources/PackLib/ToolRegistry.swift @@ -1,4 +1,5 @@ import Foundation +import Subprocess package enum ToolRegistry { package enum Errors: Error, CustomStringConvertible { @@ -16,11 +17,6 @@ package enum ToolRegistry { /// Obtain the full path to a tool in the user's `PATH`. /// - /// This effectively invokes `/bin/sh -c "command -v '$tool'"`. - /// - /// - Warning: Make sure you trust/sanitize the `tool` parameter. If it - /// contains a single quote, it can be used in a shell escape. - /// /// - Throws: `Errors.toolNotFound` if the tool could not be located. package static func locate(_ tool: String) async throws -> URL { try await cache.locate(tool: tool) @@ -30,20 +26,10 @@ package enum ToolRegistry { private var cache: [String: Task] = [:] private func _locate(tool: String) async throws -> URL { - let pipe = Pipe() - let proc = Process() - proc.executableURL = URL(fileURLWithPath: "/bin/sh") - proc.arguments = ["-c", "command -v '\(tool)'"] - proc.standardOutput = pipe - async let bytes = pipe.fileHandleForReading.readToEnd() - do { - try await proc.runUntilExit() - } catch is Process.Failure { - throw Errors.toolNotFound(tool) - } - let path = String(decoding: try await bytes ?? Data(), as: UTF8.self) - .trimmingCharacters(in: .whitespacesAndNewlines) - return URL(fileURLWithPath: path) + guard let path = try? Executable.name(tool).resolveExecutablePath(in: .inherit), + let url = URL(filePath: path) + else { throw Errors.toolNotFound(tool) } + return url } func locate(tool: String) async throws -> URL { diff --git a/Sources/XKit/GrandSlam/Anisette/XADIProvider.swift b/Sources/XKit/GrandSlam/Anisette/XADIProvider.swift index cf5f9d68..3183101c 100644 --- a/Sources/XKit/GrandSlam/Anisette/XADIProvider.swift +++ b/Sources/XKit/GrandSlam/Anisette/XADIProvider.swift @@ -3,6 +3,7 @@ import Foundation import XADI import Dependencies +import Subprocess public actor XADIProvider: RawADIProvider { @MainActor private static var loadTask: Task? @@ -45,17 +46,17 @@ public actor XADIProvider: RawADIProvider { let archDir = "lib/\(arch)" // TODO: Use ZIPFoundation - let proc = Process() - proc.executableURL = URL(filePath: "/usr/bin/env") - proc.arguments = [ - "unzip", "-q", - applemusic.path(), - "\(archDir)/libCoreADI.so", - "\(archDir)/libstoreservicescore.so", - "-d", tmp.path() - ] - try proc.run() - proc.waitUntilExit() + try await Subprocess.run( + .name("unzip"), + arguments: [ + "-q", + applemusic.path, + "\(archDir)/libCoreADI.so", + "\(archDir)/libstoreservicescore.so", + "-d", tmp.path + ], + output: .discarded + ).checkSuccess() try FileManager.default.moveItem( at: tmp.appending(path: archDir), to: libDir diff --git a/Sources/XToolSupport/Console.swift b/Sources/XToolSupport/Console.swift index 07cc77e0..8dc420c1 100644 --- a/Sources/XToolSupport/Console.swift +++ b/Sources/XToolSupport/Console.swift @@ -1,6 +1,6 @@ import Foundation import XKit -import SystemPackage +import XUtils import NIOPosix import NIOCore diff --git a/Sources/XToolSupport/DevCommand.swift b/Sources/XToolSupport/DevCommand.swift index 64d3327f..a8273540 100644 --- a/Sources/XToolSupport/DevCommand.swift +++ b/Sources/XToolSupport/DevCommand.swift @@ -282,6 +282,8 @@ struct DevCommand: AsyncParsableCommand { extension BuildConfiguration: ExpressibleByArgument {} #if os(macOS) +import Subprocess + struct SimInstallOperation { var path: URL @@ -289,10 +291,12 @@ struct SimInstallOperation { var simulator = "booted" func run() async throws { - let process = Process() - process.executableURL = URL(fileURLWithPath: "/usr/bin/xcrun") - process.arguments = ["simctl", "install", simulator, path.path] - try await process.runUntilExit() + try await Subprocess.run( + .path("/usr/bin/xcrun"), + arguments: ["simctl", "install", simulator, path.path], + output: .discarded + ) + .checkSuccess() print("Installed to simulator") } } diff --git a/Sources/XToolSupport/ProcessZIPCompressor.swift b/Sources/XToolSupport/ProcessZIPCompressor.swift index 0a685030..cc04f053 100644 --- a/Sources/XToolSupport/ProcessZIPCompressor.swift +++ b/Sources/XToolSupport/ProcessZIPCompressor.swift @@ -2,6 +2,8 @@ import Foundation import XKit import Dependencies import PackLib +import Subprocess +import XUtils extension ZIPCompressor: DependencyKey { // TODO: Use `powershell Compress-Archive` and `powershell Expand-Archive` on Windows @@ -12,20 +14,23 @@ extension ZIPCompressor: DependencyKey { let dest = dir.deletingLastPathComponent().appendingPathComponent("app.ipa") - let zip = Process() - zip.executableURL = try await ToolRegistry.locate("zip") - zip.currentDirectoryURL = dir.deletingLastPathComponent() - zip.arguments = ["-yqru0", dest.path, dir.lastPathComponent] - try await zip.runUntilExit() + try await Subprocess.run( + .name("zip"), + arguments: ["-yqru0", dest.path, dir.lastPathComponent], + workingDirectory: FilePath(dir.deletingLastPathComponent()), + output: .discarded, + ).checkSuccess() return dest }, decompress: { ipa, directory, progress in progress(nil) - let unzip = Process() - unzip.executableURL = try await ToolRegistry.locate("unzip") - unzip.arguments = ["-q", ipa.path, "-d", directory.path] - try await unzip.runUntilExit() + + try await Subprocess.run( + .name("unzip"), + arguments: ["-q", ipa.path, "-d", directory.path], + output: .discarded, + ).checkSuccess() } ) } diff --git a/Sources/XToolSupport/SDKBuilder.swift b/Sources/XToolSupport/SDKBuilder.swift index 2068e725..44fc8fb6 100644 --- a/Sources/XToolSupport/SDKBuilder.swift +++ b/Sources/XToolSupport/SDKBuilder.swift @@ -1,9 +1,9 @@ import Foundation import Dependencies -import SystemPackage import libunxip +import Subprocess import XKit // HTTPClient, stdoutSafe -import PackLib // ToolRegistry +import XUtils // System.File{Path,Descriptor} struct SDKBuilder { enum Arch: String { @@ -180,14 +180,6 @@ struct SDKBuilder { withIntermediateDirectories: false ) - let pipe = Pipe() - let untar = Process() - untar.currentDirectoryURL = toolsetDir - untar.executableURL = try await ToolRegistry.locate("tar") - untar.arguments = ["xzf", "-"] - untar.standardInput = pipe.fileHandleForReading - async let tarExit: Void = untar.runUntilExit() - @Dependency(\.httpClient) var httpClient let url = URL(string: """ https://github.com/xtool-org/darwin-tools-linux-llvm/releases/download/\ @@ -199,22 +191,30 @@ struct SDKBuilder { case .known(let known): known case .unknown: nil } - let writer = pipe.fileHandleForWriting - var written: Int64 = 0 - do { - defer { try? writer.close() } + + defer { print() } + try await Subprocess.run( + .name("tar"), + arguments: ["xzf", "-"], + workingDirectory: FilePath(toolsetDir), + ) { _, input, _ in + var totalWritten: Int64 = 0 for try await chunk in body { - try writer.write(contentsOf: chunk) - written += Int64(chunk.count) - if let length { - let progress = Int(Double(written) / Double(length) * 100) + var remaining = chunk + while !remaining.isEmpty { + let written = try await input.write(Array(remaining)) + remaining = remaining.dropFirst(written) + totalWritten += Int64(written) + + guard let length else { continue } + let progress = Int(Double(totalWritten) / Double(length) * 100) print("\r[Downloading toolset] \(progress)%", terminator: "") fflush(stdoutSafe) } } + try await input.finish() } - print() - try await tarExit + .checkSuccess() } // swiftlint:disable:next cyclomatic_complexity diff --git a/Sources/XToolSupport/SDKCommand.swift b/Sources/XToolSupport/SDKCommand.swift index 75543f9f..ba92f8cb 100644 --- a/Sources/XToolSupport/SDKCommand.swift +++ b/Sources/XToolSupport/SDKCommand.swift @@ -5,6 +5,7 @@ import ArgumentParser import Dependencies import PackLib import XUtils +import Subprocess struct SDKCommand: AsyncParsableCommand { static let configuration = CommandConfiguration( @@ -151,50 +152,40 @@ struct DarwinSDK { try await addHostClangResourceDir(to: url) - let process = Process() - process.executableURL = try await ToolRegistry.locate("swift") - process.arguments = ["sdk", "install", url.path] - try await process.runUntilExit() + try await Subprocess.run( + .name("swift"), + arguments: ["sdk", "install", url.path], + output: .discarded + ) + .checkSuccess()f6b84ce (wip: use swift-subprocess) } private static func addHostClangResourceDir(to sdk: URL) async throws { - let outPipe = Pipe() - let clang = Process() - clang.executableURL = try await ToolRegistry.locate("clang") - clang.arguments = ["-print-resource-dir"] - clang.standardOutput = outPipe - clang.standardError = FileHandle.standardError - async let outputData = Data(reading: outPipe.fileHandleForReading) - do { - try await clang.runUntilExit() - } catch is Process.Failure { - throw Console.Error("Failed to query host clang -print-resource-dir") - } - let path = String(decoding: try await outputData, as: UTF8.self) - .trimmingCharacters(in: .whitespacesAndNewlines) - guard !path.isEmpty else { - throw Console.Error("Host clang returned an empty resource directory") - } - let hostClangResources = URL(fileURLWithPath: path, isDirectory: true) + let clangURL = try await ToolRegistry.locate("clang") + let process = try await Subprocess.run( + .path(FilePath(clangURL.path)), + arguments: ["-print-resource-dir"], + output: .string(limit: .max) + ).checkSuccess() + let output = process.standardOutput ?? "" + let hostClangResources = URL(filePath: output.trimmingCharacters(in: .whitespacesAndNewlines)) let hostInclude = hostClangResources.appending(path: "include") let sdkInclude = sdk.appending(path: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/clang/include") try FileManager.default.copyItem(at: hostInclude, to: sdkInclude) } static func current() async throws -> DarwinSDK? { - let output = Pipe() - - let process = Process() - process.executableURL = try await ToolRegistry.locate("swift") - process.arguments = ["sdk", "configure", "darwin", "arm64-apple-ios", "--show-configuration"] - process.standardOutput = output - process.standardError = FileHandle.nullDevice - - async let outputData = Data(reading: output.fileHandleForReading) - + let outputString: String do { - try await process.runUntilExit() - } catch Process.Failure.exit { + outputString = try await Subprocess.run( + .name("swift"), + arguments: ["sdk", "configure", "darwin", "arm64-apple-ios", "--show-configuration"], + output: .string(limit: .max) + ) + .checkSuccess() + .standardOutput + ?? "" + } catch SubprocessFailure.exited { return nil } @@ -202,7 +193,6 @@ struct DarwinSDK { // swiftResourcesPath: /home/user/.swiftpm/swift-sdks/darwin.artifactbundle/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift // swiftlint:disable:previous line_length let resourcesPathPrefix = "swiftResourcesPath: " - let outputString = String(decoding: try await outputData, as: UTF8.self) guard let resourcesPath = outputString .split(separator: "\n") @@ -230,21 +220,19 @@ struct DarwinSDK { private enum SwiftVersion {} extension SwiftVersion { static func current() async throws -> Version { - let outPipe = Pipe() - let errPipe = Pipe() - let swift = Process() - swift.executableURL = try await ToolRegistry.locate("swift") - swift.arguments = ["--version"] - swift.standardOutput = outPipe - swift.standardError = errPipe - async let outputTask = outPipe.fileHandleForReading.readToEnd() + let outputString: String? do { - try await swift.runUntilExit() - } catch is Process.Failure { + outputString = try await Subprocess.run( + .name("swift"), + arguments: ["--version"], + output: .string(limit: .max) + ) + .checkSuccess() + .standardOutput + } catch { throw Console.Error("Failed to obtain Swift version") } - let outputData = try await outputTask - var output = String(decoding: outputData ?? Data(), as: UTF8.self)[...] + var output = outputString?[...] ?? "" if output.hasPrefix("Apple ") { output = output.dropFirst("Apple ".count) } diff --git a/Sources/XUtils/Subprocess+Utils.swift b/Sources/XUtils/Subprocess+Utils.swift new file mode 100644 index 00000000..f3641d10 --- /dev/null +++ b/Sources/XUtils/Subprocess+Utils.swift @@ -0,0 +1,65 @@ +#if canImport(Subprocess) +import Subprocess +import Foundation + +extension ExecutionRecord { + @discardableResult + public func checkSuccess() throws(SubprocessFailure) -> Self { + try terminationStatus.checkSuccess() + return self + } +} + +extension ExecutionOutcome { + @discardableResult + public func checkSuccess() throws(SubprocessFailure) -> Self { + try terminationStatus.checkSuccess() + return self + } +} + +extension TerminationStatus { + fileprivate func checkSuccess() throws(SubprocessFailure) { + if let failure { throw failure } + } + + private var failure: SubprocessFailure? { + if isSuccess { return nil } + switch self { + case .exited(let code): return .exited(code) + case .signaled(let code): return .signaled(code) + } + } +} + +public enum SubprocessFailure: Error { + case exited(TerminationStatus.Code) + case signaled(TerminationStatus.Code) +} + +extension PlatformOptions { + public static var withGracefulShutDown: Self { + .init().withGracefulShutDown + } + + public var withGracefulShutDown: Self { + var copy = self + copy.teardownSequence = [ + .gracefulShutDown( + allowedDurationToNextStep: .milliseconds(500) + ) + ] + return copy + } +} + +extension Environment { + public static func currentMap() -> [Environment.Key: String] { + Dictionary( + uniqueKeysWithValues: ProcessInfo.processInfo.environment.map { + (Environment.Key(rawValue: $0)!, $1) + } + ) + } +} +#endif diff --git a/Sources/XUtils/System+Utils.swift b/Sources/XUtils/System+Utils.swift new file mode 100644 index 00000000..139c043c --- /dev/null +++ b/Sources/XUtils/System+Utils.swift @@ -0,0 +1,24 @@ +#if canImport(System) +import System +public typealias FilePath = System.FilePath +public typealias FileDescriptor = System.FileDescriptor +#else +import SystemPackage +import Foundation + +public typealias FilePath = SystemPackage.FilePath +public typealias FileDescriptor = SystemPackage.FileDescriptor + +extension URL { + public init?(filePath: FilePath) { + self.init(filePath: filePath.string) + } +} + +extension FilePath { + public init?(_ url: URL) { + guard url.isFileURL else { return nil } + self.init(url.path) + } +} +#endif