From ff3d17d3ba51cfc90c315ec753bb6d66537dc88a Mon Sep 17 00:00:00 2001 From: William Date: Mon, 29 Sep 2025 12:23:55 +0200 Subject: [PATCH 01/22] refactor(SeleniumGridNodeAutoOldMachineDestroyer): reduce method length by refactoring to helper methods --- ...eniumGridNodeAutoOldMachineDestroyer.swift | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAutoOldMachineDestroyer.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAutoOldMachineDestroyer.swift index af7d75e..2b60121 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAutoOldMachineDestroyer.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAutoOldMachineDestroyer.swift @@ -54,22 +54,11 @@ internal class SeleniumGridNodeAutoOldMachineDestroyer: SeleniumGridNodeMachineA } if machinesToStop.isEmpty { - logger.info( - "None of the \(allMachines.count) machines in a considered old. No machines will be destroyed.", - metadata: [ - "to": .string("\(String(describing: Self.self)).\(#function)"), - ] - ) + logNoMachinesToDestroy(totalMachines: allMachines.count) return } - logger.info( - "Destroying all old node machines started.", - metadata: [ - "to": .string("\(String(describing: Self.self)).\(#function)"), - "total_machines_to_destroy": .string(String(machinesToStop.count)) - ] - ) + logDeleteAllOldNodeMachinesStarted(totalMachines: allMachines.count) for machine in machinesToStop { try await deleteNodeMachine(id: machine.id) @@ -80,16 +69,31 @@ internal class SeleniumGridNodeAutoOldMachineDestroyer: SeleniumGridNodeMachineA let nodeMachineExpirationMinutes = try Environment .getOrThrow("NODE_MACHINE_EXPIRATION_MINUTES") - guard let nodeMachineExpirationMinutesDouble = TimeInterval(nodeMachineExpirationMinutes) + guard let nodeMachineExpirationMinutesInt = TimeInterval(nodeMachineExpirationMinutes) else { - throw AutomaGenericErrors - .guardFailed( - message: """ - Could not convert 'NODE_MACHINE_EXPIRATION_MINUTES' of value '\(nodeMachineExpirationMinutes)' to type `TimeInterval`" - """ - ) + throw Abort(.internalServerError) } - return nodeMachineExpirationMinutesDouble + + return nodeMachineExpirationMinutesInt + } + + private func logNoMachinesToDestroy(totalMachines: Int) { + logger.info( + "None of the \(totalMachines) machines in a considered old. No machines will be destroyed.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + ] + ) + } + + private func logDeleteAllOldNodeMachinesStarted(totalMachines: Int) { + logger.info( + "Destroying all old node machines started.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "total_machines_to_destroy": .string(String(totalMachines)) + ] + ) } deinit {} From 752cd5d0017f49b7e0eef6ca33e459246cf8f076 Mon Sep 17 00:00:00 2001 From: William Date: Mon, 29 Sep 2025 17:31:44 +0200 Subject: [PATCH 02/22] perf(WebsiteHTMLGetter): improve speed of browser startup and navigating to url by making the browser instance headless --- .gitignore | 1 + .../API/WebsiteHTMLGetter/WebsiteHTMLGetter.swift | 4 +++- SeleniumGridNodeMachineAutoscaler/.gitignore | 1 - infra/seleniumgrid-node.toml | 10 +++++----- package.json | 5 +++-- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 64d0287..5cb85a3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .swiftlint.yml .swiftformat +infra/GITHUB_SSH_AUTHENTICATION_TOKEN diff --git a/API/Sources/API/WebsiteHTMLGetter/WebsiteHTMLGetter.swift b/API/Sources/API/WebsiteHTMLGetter/WebsiteHTMLGetter.swift index 60c7f78..3771e1c 100644 --- a/API/Sources/API/WebsiteHTMLGetter/WebsiteHTMLGetter.swift +++ b/API/Sources/API/WebsiteHTMLGetter/WebsiteHTMLGetter.swift @@ -27,7 +27,9 @@ internal struct WebsiteHTMLGetter { private static func getWebDriver(seleniumGridHubBase: URL) -> WebDriver { let chromeOption = ChromeOptions( - args: [] + args: [ + Args(.headless) + ] ) return WebDriver( diff --git a/SeleniumGridNodeMachineAutoscaler/.gitignore b/SeleniumGridNodeMachineAutoscaler/.gitignore index 6f64694..cffd598 100644 --- a/SeleniumGridNodeMachineAutoscaler/.gitignore +++ b/SeleniumGridNodeMachineAutoscaler/.gitignore @@ -9,4 +9,3 @@ db.sqlite .env .env.* !.env.example -infra/GITHUB_SSH_AUTHENTICATION_TOKEN diff --git a/infra/seleniumgrid-node.toml b/infra/seleniumgrid-node.toml index 70fe7b4..47c9f3b 100644 --- a/infra/seleniumgrid-node.toml +++ b/infra/seleniumgrid-node.toml @@ -7,12 +7,12 @@ app = 'automa-web-core-seleniumgrid-node' primary_region = 'jnb' [build] - image = 'selenium/node-chrome:latest' +image = 'selenium/node-chrome:latest' [env] - SE_EVENT_BUS_HOST = 'automa-web-core-seleniumgrid-hub.internal' +SE_EVENT_BUS_HOST = 'automa-web-core-seleniumgrid-hub.internal' [[vm]] - memory = '2gb' - cpu_kind = 'shared' - cpus = 1 +memory = '2gb' +cpu_kind = 'performance' +cpus = 1 diff --git a/package.json b/package.json index c09ca88..cbb5671 100644 --- a/package.json +++ b/package.json @@ -25,8 +25,9 @@ "autoscaler:off_machines_destroyer": "npm run swift:build-and-run ./SeleniumGridNodeMachineAutoscaler SeleniumGridNodeMachineAutoscaler autodestroyer offMachines", "autoscaler:old_machines_destroyer": "npm run swift:build-and-run ./SeleniumGridNodeMachineAutoscaler SeleniumGridNodeMachineAutoscaler autodestroyer oldMachines", "api": "npm run swift:build-and-run -- ./API API", - "deploy:autoscaler": "cd ./SeleniumGridNodeMachineAutoscaler && npm run deploy -- ./SeleniumGridNodeMachineAutoscaler/ --config=./infra/autoscaler.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ./infra/GITHUB_SSH_AUTHENTICATION_TOKEN)", - "deploy:api": "cd ./API && npm run deploy -- ./API/ --config=./infra/api.toml" + "deploy": "flyctl deploy --ha=false", + "deploy:autoscaler": "cd ./SeleniumGridNodeMachineAutoscaler && npm run deploy -- ./SeleniumGridNodeMachineAutoscaler/ --config=./infra/autoscaler.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ../infra/GITHUB_SSH_AUTHENTICATION_TOKEN)", + "deploy:api": "cd ./API && npm run deploy -- ./API/ --config=./infra/api.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ../infra/GITHUB_SSH_AUTHENTICATION_TOKEN)" }, "repository": { "type": "git", From 53f9854b94e9dfeda47fc500c2655013b886b23e Mon Sep 17 00:00:00 2001 From: William Date: Mon, 29 Sep 2025 17:32:33 +0200 Subject: [PATCH 03/22] fix(Dockerfile): set github token to be able to clone private repositories like automa utilities and api.toml file formatting --- API/Dockerfile | 26 +++++++++++++++----------- API/infra/api.toml | 23 ++++++++++++++--------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/API/Dockerfile b/API/Dockerfile index e4093ce..76662d9 100644 --- a/API/Dockerfile +++ b/API/Dockerfile @@ -12,13 +12,17 @@ RUN export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true \ # Set up a build area WORKDIR /build +RUN --mount=type=secret,id=GITHUB_SSH_AUTHENTICATION_TOKEN \ + TOKEN=$(cat /run/secrets/GITHUB_SSH_AUTHENTICATION_TOKEN) && \ + git config --global url."https://x-access-token:${TOKEN}@github.com/".insteadOf "https://github.com/" + # First just resolve dependencies. # This creates a cached layer that can be reused # as long as your Package.swift/Package.resolved # files do not change. COPY ./Package.* ./ RUN swift package resolve \ - $([ -f ./Package.resolved ] && echo "--force-resolved-versions" || true) + $([ -f ./Package.resolved ] && echo "--force-resolved-versions" || true) # Copy entire repo into container COPY . . @@ -29,9 +33,9 @@ RUN mkdir /staging # N.B.: The static version of jemalloc is incompatible with the static Swift runtime. RUN --mount=type=cache,target=/build/.build \ swift build -c release \ - --product API \ - --static-swift-stdlib \ - -Xlinker -ljemalloc && \ + --product API \ + --static-swift-stdlib \ + -Xlinker -ljemalloc && \ # Copy main executable to staging area cp "$(swift build -c release --show-bin-path)/API" /staging && \ # Copy resources bundled by SPM to staging area @@ -59,13 +63,13 @@ RUN export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true \ && apt-get -q update \ && apt-get -q dist-upgrade -y \ && apt-get -q install -y \ - libjemalloc2 \ - ca-certificates \ - tzdata \ -# If your app or its dependencies import FoundationNetworking, also install `libcurl4`. - # libcurl4 \ -# If your app or its dependencies import FoundationXML, also install `libxml2`. - # libxml2 \ + libjemalloc2 \ + ca-certificates \ + tzdata \ + # If your app or its dependencies import FoundationNetworking, also install `libcurl4`. + # libcurl4 \ + # If your app or its dependencies import FoundationXML, also install `libxml2`. + # libxml2 \ && rm -r /var/lib/apt/lists/* # Create a vapor user and group with /app as its home directory diff --git a/API/infra/api.toml b/API/infra/api.toml index 941d03e..7dc52c3 100644 --- a/API/infra/api.toml +++ b/API/infra/api.toml @@ -1,17 +1,22 @@ +# fly.toml app configuration file generated for automa-web-core-api on 2025-09-29T12:32:49+02:00 +# +# See https://fly.io/docs/reference/configuration/ for information about how to use this file. +# + app = 'automa-web-core-api' primary_region = 'jnb' [build] [http_service] -internal_port = 8080 -force_https = true -auto_stop_machines = 'off' -auto_start_machines = false -min_machines_running = 0 -processes = ['app'] + internal_port = 8080 + force_https = true + auto_stop_machines = 'off' + auto_start_machines = false + min_machines_running = 0 + processes = ['app'] [[vm]] -memory = '1gb' -cpu_kind = 'shared' -cpus = 1 + memory = '1gb' + cpu_kind = 'shared' + cpus = 1 From 41eda8cfe3fe0916426cb68811b0199d51e41aa3 Mon Sep 17 00:00:00 2001 From: William Date: Tue, 30 Sep 2025 12:18:20 +0200 Subject: [PATCH 04/22] feat(APIController): created api controller, routes without implementation and some api payload config options that will be uncommented and used after the mvp --- .../API/Controllers/APIController.swift | 72 ++++++++++++++++++ .../WebBrowserClient.swift} | 73 +++++++++++++++---- .../WebsiteHTMLGetter/HTMLGetterPayload.swift | 14 ---- API/Sources/API/configure.swift | 7 ++ API/Sources/API/routes.swift | 6 -- 5 files changed, 137 insertions(+), 35 deletions(-) create mode 100644 API/Sources/API/Controllers/APIController.swift rename API/Sources/API/{WebsiteHTMLGetter/WebsiteHTMLGetter.swift => WebBrowserClient/WebBrowserClient.swift} (52%) delete mode 100644 API/Sources/API/WebsiteHTMLGetter/HTMLGetterPayload.swift diff --git a/API/Sources/API/Controllers/APIController.swift b/API/Sources/API/Controllers/APIController.swift new file mode 100644 index 0000000..b39d6b1 --- /dev/null +++ b/API/Sources/API/Controllers/APIController.swift @@ -0,0 +1,72 @@ +// APIController.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import Vapor + +internal struct APIController: RouteCollection { + public func boot(routes: any RoutesBuilder) throws { + let twitterRoute = routes.grouped("api") + twitterRoute.get(use: get) + twitterRoute.post(use: post) + twitterRoute.put(use: put) + twitterRoute.patch(use: patch) + twitterRoute.delete(use: delete) + } + + @Sendable + public func get(req: Request) async throws -> String { + let payload = try req.content.decode(APIEndpointPayload.self) + return try await WebBrowserClient(logger: req.logger, payload: payload).getHTML() + } + + // Endpoints where implementation becomes required when making any type of request (GET, POST, etc) + // with a body and headers becomes necessary for the next version of the app + @Sendable + public func post(req _: Request) async throws -> String { + "hello world" + } + + @Sendable + public func put(req _: Request) async throws -> String { + "hello world" + } + + @Sendable + public func patch(req _: Request) async throws -> String { + "hello world" + } + + @Sendable + public func delete(req _: Request) async throws -> String { + "hello world" + } +} + +public struct APIEndpointPayload: Content { + let url: URL + let scrollToBottom: Bool + + init(url: URL, scrollToBottom: Bool = false) { + self.url = url + self.scrollToBottom = scrollToBottom + } + + // configuration options required to be implemented and handled in route handlers when + // a service that needs to be able to make a request both with a browser (jsRender) and without a browser + // (jsRender=false), + // + + // let jsRender: Bool + // let residentialProxy: Bool + // let autoCaptchaSolving: Bool + + public enum CodingKeys: String, CodingKey { + case url + case scrollToBottom = "scroll_to_bottom" + // case jsRender = "js_render" + // case residentialProxy = "residential_proxy" + // case autoCaptchaSolving = "auto_captcha_solving" + } +} diff --git a/API/Sources/API/WebsiteHTMLGetter/WebsiteHTMLGetter.swift b/API/Sources/API/WebBrowserClient/WebBrowserClient.swift similarity index 52% rename from API/Sources/API/WebsiteHTMLGetter/WebsiteHTMLGetter.swift rename to API/Sources/API/WebBrowserClient/WebBrowserClient.swift index 3771e1c..f2b432a 100644 --- a/API/Sources/API/WebsiteHTMLGetter/WebsiteHTMLGetter.swift +++ b/API/Sources/API/WebBrowserClient/WebBrowserClient.swift @@ -1,4 +1,4 @@ -// WebsiteHTMLGetter.swift +// WebBrowserClient.swift // Copyright (c) 2025 GetAutomaApp // All source code and related assets are the property of GetAutomaApp. // All rights reserved. @@ -7,25 +7,24 @@ import AutomaUtilities import SwiftWebDriver import Vapor -// TODO: scroll down the page to load all javascript before getting content, some websites (especially blogs/articles) -// get the article content in chunks to have better performance and reduce web scraping success attempts - -internal struct WebsiteHTMLGetter { +internal struct WebBrowserClient { let logger: Logger - let url: URL + let payload: APIEndpointPayload let seleniumGridHubBase: URL let driver: WebDriver - init(logger: Logger, url: URL) async throws { + init(logger: Logger, payload: APIEndpointPayload) async throws { self.logger = logger - self.url = url + self.payload = payload seleniumGridHubBase = try URL .fromString(payload: .init(string: Environment.getOrThrow("SELENIUM_GRID_HUB_BASE"))) - driver = Self.getWebDriver(seleniumGridHubBase: seleniumGridHubBase) + driver = Self.getWebDriver(seleniumGridHubBase: seleniumGridHubBase, logger: logger) try await driver.start() } - private static func getWebDriver(seleniumGridHubBase: URL) -> WebDriver { + private static func getWebDriver(seleniumGridHubBase: URL, logger: Logger) -> WebDriver { + logGetWebDriverStarted(logger: logger) + let chromeOption = ChromeOptions( args: [ Args(.headless) @@ -40,14 +39,32 @@ internal struct WebsiteHTMLGetter { ) } - public func get() async throws -> String { + private static func logGetWebDriverStarted(logger: Logger) { + logger.info( + "Getting new webdriver instance started.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + ] + ) + } + + public func getHTML() async throws -> String { try await navigateDriverToURL() + logger.info( + "API Endpoint Payload: \(payload).", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + ] + ) + if payload.scrollToBottom { + try await scrollToBottom() + } return try await getActiveWindowOuterHTML() } private func navigateDriverToURL() async throws { logNavigateToURLStarted() - try await driver.navigateTo(url: url) + try await driver.navigateTo(url: payload.url) logNavigateToURLSuccess() } @@ -56,7 +73,7 @@ internal struct WebsiteHTMLGetter { "Navigating WebDriver to URL to get HTML content as string.", metadata: [ "to": .string("\(String(describing: Self.self)).\(#function)"), - "url": .string(url.absoluteString), + "url": .string(payload.url.absoluteString), ] ) } @@ -66,7 +83,33 @@ internal struct WebsiteHTMLGetter { "Navigating WebDriver to URL to get HTML content as string success.", metadata: [ "to": .string("\(String(describing: Self.self)).\(#function)"), - "url": .string(url.absoluteString), + "url": .string(payload.url.absoluteString), + ] + ) + } + + private func scrollToBottom() async throws { + logScrollToBottomStarted() + try await driver.execute("window.scrollBy(0, document.querySelector(\"html\").scrollHeight)") + logScrollToBottomCompleted() + } + + private func logScrollToBottomStarted() { + logger.info( + "Scrolling to bottom of page document started.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "url": .string(payload.url.absoluteString), + ] + ) + } + + private func logScrollToBottomCompleted() { + logger.info( + "Scrolling to bottom of page document completed.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "url": .string(payload.url.absoluteString), ] ) } @@ -88,7 +131,7 @@ internal struct WebsiteHTMLGetter { else { throw AutomaGenericErrors .notFound( - message: "'html' element of URL '\(url.absoluteString)' 'outerHTML' property contains an empty value." + message: "'html' element of URL '\(payload.url.absoluteString)' 'outerHTML' property contains an empty value." ) } return outerHTMLString diff --git a/API/Sources/API/WebsiteHTMLGetter/HTMLGetterPayload.swift b/API/Sources/API/WebsiteHTMLGetter/HTMLGetterPayload.swift deleted file mode 100644 index 82450c5..0000000 --- a/API/Sources/API/WebsiteHTMLGetter/HTMLGetterPayload.swift +++ /dev/null @@ -1,14 +0,0 @@ -// HTMLGetterPayload.swift -// Copyright (c) 2025 GetAutomaApp -// All source code and related assets are the property of GetAutomaApp. -// All rights reserved. - -import Vapor - -internal struct HTMLGetterPayload: Content { - let websiteURL: String - - public enum CodingKeys: String, CodingKey { - case websiteURL = "website_url" - } -} diff --git a/API/Sources/API/configure.swift b/API/Sources/API/configure.swift index 7715d7c..2d6da37 100644 --- a/API/Sources/API/configure.swift +++ b/API/Sources/API/configure.swift @@ -1,3 +1,8 @@ +// configure.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + import Vapor // configures your application @@ -7,4 +12,6 @@ public func configure(_ app: Application) async throws { // register routes try routes(app) + + try app.register(collection: APIController()) } diff --git a/API/Sources/API/routes.swift b/API/Sources/API/routes.swift index 27c6875..e3a510e 100644 --- a/API/Sources/API/routes.swift +++ b/API/Sources/API/routes.swift @@ -9,10 +9,4 @@ func routes(_ app: Application) throws { app.get { _ async in "It works!" } - - app.get("get-html") { req async throws -> String in - let content = try req.content.decode(HTMLGetterPayload.self) - let url = try URL.fromString(payload: .init(string: content.websiteURL)) - return try await WebsiteHTMLGetter(logger: req.logger, url: url).get() - } } From ff2a98a410c8b1b106c45e38194cc35acc47d6ea Mon Sep 17 00:00:00 2001 From: William Date: Tue, 30 Sep 2025 19:19:28 +0200 Subject: [PATCH 05/22] feat: use prometheus service from automa utilities instead of legacy prometheus controller `PrometheusService` uses a different port that isn't exposed, which removes the need to pass an authentication token as a query parameer to avoid possible leaks. --- .gitignore | 3 + API/.gitignore | 3 - API/Package.resolved | 34 ++++++++--- API/Sources/API/configure.swift | 2 + API/infra/api.toml | 22 +++++--- SeleniumGridNodeMachineAutoscaler/.gitignore | 3 - .../Package.resolved | 56 ++++++++++++------- .../configure.swift | 5 +- .../entrypoint.swift | 2 +- .../automa.config.json | 16 ++++++ .../infra/autoscaler.toml | 6 +- package.json | 6 +- 12 files changed, 112 insertions(+), 46 deletions(-) create mode 100644 SeleniumGridNodeMachineAutoscaler/automa.config.json diff --git a/.gitignore b/.gitignore index 5cb85a3..db003fc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ .swiftlint.yml .swiftformat infra/GITHUB_SSH_AUTHENTICATION_TOKEN +.env +.env.* +!.env.example diff --git a/API/.gitignore b/API/.gitignore index 745c700..44d437c 100644 --- a/API/.gitignore +++ b/API/.gitignore @@ -6,7 +6,4 @@ DerivedData/ .DS_Store db.sqlite .swiftpm -.env -.env.* -!.env.example default.profraw diff --git a/API/Package.resolved b/API/Package.resolved index a625814..d3be413 100644 --- a/API/Package.resolved +++ b/API/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "8847af8b844c9c7804e98992d8f4898a3b2f5482f54b71586e69017eb57947ed", + "originHash" : "1eb39dba8f43b8ba3a30d60d2114aa24d8aa1c4bf732fe2f87bcc0820ce9939b", "pins" : [ { "identity" : "async-http-client", @@ -25,7 +25,7 @@ "location" : "https://github.com/GetAutomaApp/AutomaUtilities.git", "state" : { "branch" : "main", - "revision" : "bc7212b7994ee496b0d3684dd77be92f23961c8f" + "revision" : "91fa64dab446587a495c9508429408f9d40e1dda" } }, { @@ -37,6 +37,15 @@ "version" : "4.15.2" } }, + { + "identity" : "flyingfox", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swhitty/FlyingFox.git", + "state" : { + "revision" : "76b2d2d0eb54f0b53e8b661cc8007c487d94f8da", + "version" : "0.25.0" + } + }, { "identity" : "multipart-kit", "kind" : "remoteSourceControl", @@ -114,8 +123,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-collections.git", "state" : { - "revision" : "8c0c0a8b49e080e54e5e328cc552821ff07cd341", - "version" : "1.2.1" + "revision" : "7b847a3b7008b2dc2f47ca3110d8c782fb2e5c7e", + "version" : "1.3.0" } }, { @@ -177,8 +186,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio.git", "state" : { - "revision" : "154706efd36d8d8a7d030eea9bcbeca56a947c62", - "version" : "2.86.1" + "revision" : "a18bddb0acf7a40d982b2f128ce73ce4ee31f352", + "version" : "2.86.2" } }, { @@ -222,8 +231,17 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-numerics.git", "state" : { - "revision" : "bbadd4b853a33fd78c4ae977d17bb2af15eb3f2a", - "version" : "1.1.0" + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-prometheus", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swift-server/swift-prometheus.git", + "state" : { + "revision" : "8a8ff47403444e16d8cdfa805a5e3cb8e2efe734", + "version" : "2.2.0" } }, { diff --git a/API/Sources/API/configure.swift b/API/Sources/API/configure.swift index 2d6da37..5c294b1 100644 --- a/API/Sources/API/configure.swift +++ b/API/Sources/API/configure.swift @@ -3,6 +3,7 @@ // All source code and related assets are the property of GetAutomaApp. // All rights reserved. +import AutomaUtilities import Vapor // configures your application @@ -14,4 +15,5 @@ public func configure(_ app: Application) async throws { try routes(app) try app.register(collection: APIController()) + try await PrometheusService().startServer() } diff --git a/API/infra/api.toml b/API/infra/api.toml index 7dc52c3..c5b869e 100644 --- a/API/infra/api.toml +++ b/API/infra/api.toml @@ -9,14 +9,18 @@ primary_region = 'jnb' [build] [http_service] - internal_port = 8080 - force_https = true - auto_stop_machines = 'off' - auto_start_machines = false - min_machines_running = 0 - processes = ['app'] +internal_port = 8080 +force_https = true +auto_stop_machines = 'off' +auto_start_machines = false +min_machines_running = 0 +processes = ['app'] [[vm]] - memory = '1gb' - cpu_kind = 'shared' - cpus = 1 +memory = '1gb' +cpu_kind = 'shared' +cpus = 1 + +[metrics] +port = 6834 +path = "/metrics" diff --git a/SeleniumGridNodeMachineAutoscaler/.gitignore b/SeleniumGridNodeMachineAutoscaler/.gitignore index cffd598..2cd3239 100644 --- a/SeleniumGridNodeMachineAutoscaler/.gitignore +++ b/SeleniumGridNodeMachineAutoscaler/.gitignore @@ -6,6 +6,3 @@ DerivedData/ .DS_Store db.sqlite .swiftpm -.env -.env.* -!.env.example diff --git a/SeleniumGridNodeMachineAutoscaler/Package.resolved b/SeleniumGridNodeMachineAutoscaler/Package.resolved index 506abf5..eda19f6 100644 --- a/SeleniumGridNodeMachineAutoscaler/Package.resolved +++ b/SeleniumGridNodeMachineAutoscaler/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "364aa31ddb591204d017baeb5b7ea1fe5c6ab793835d018b2b9582ff8392a7ff", + "originHash" : "be6392b201136cc104d38ffbc79e947069d4553eba1e14fc3716dc7025cca775", "pins" : [ { "identity" : "async-http-client", @@ -22,10 +22,10 @@ { "identity" : "automautilities", "kind" : "remoteSourceControl", - "location" : "https://github.com/GetAutomaApp/AutomaUtilities", + "location" : "https://github.com/GetAutomaApp/AutomaUtilities.git", "state" : { "branch" : "main", - "revision" : "bc7212b7994ee496b0d3684dd77be92f23961c8f" + "revision" : "91fa64dab446587a495c9508429408f9d40e1dda" } }, { @@ -37,6 +37,15 @@ "version" : "4.15.2" } }, + { + "identity" : "flyingfox", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swhitty/FlyingFox.git", + "state" : { + "revision" : "76b2d2d0eb54f0b53e8b661cc8007c487d94f8da", + "version" : "0.25.0" + } + }, { "identity" : "multipart-kit", "kind" : "remoteSourceControl", @@ -96,8 +105,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-certificates.git", "state" : { - "revision" : "20c451f1ad8e344e61ddbb34ef196653d4b73ea6", - "version" : "1.13.0" + "revision" : "4b092f15164144c24554e0a75e080a960c5190a6", + "version" : "1.14.0" } }, { @@ -105,8 +114,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-collections.git", "state" : { - "revision" : "8c0c0a8b49e080e54e5e328cc552821ff07cd341", - "version" : "1.2.1" + "revision" : "7b847a3b7008b2dc2f47ca3110d8c782fb2e5c7e", + "version" : "1.3.0" } }, { @@ -114,8 +123,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-crypto.git", "state" : { - "revision" : "d1c6b70f7c5f19fb0b8750cb8dcdf2ea6e2d8c34", - "version" : "3.15.0" + "revision" : "95ba0316a9b733e92bb6b071255ff46263bbe7dc", + "version" : "3.15.1" } }, { @@ -159,8 +168,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-metrics.git", "state" : { - "revision" : "4c83e1cdf4ba538ef6e43a9bbd0bcc33a0ca46e3", - "version" : "2.7.0" + "revision" : "0743a9364382629da3bf5677b46a2c4b1ce5d2a6", + "version" : "2.7.1" } }, { @@ -168,8 +177,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio.git", "state" : { - "revision" : "1c30f0f2053b654e3d1302492124aa6d242cdba7", - "version" : "2.86.0" + "revision" : "a18bddb0acf7a40d982b2f128ce73ce4ee31f352", + "version" : "2.86.2" } }, { @@ -195,8 +204,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-ssl.git", "state" : { - "revision" : "737e550e607d82bf15bdfddf158ec61652ce836f", - "version" : "2.34.0" + "revision" : "b2b043a8810ab6d51b3ff4df17f057d87ef1ec7c", + "version" : "2.34.1" } }, { @@ -213,8 +222,17 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-numerics.git", "state" : { - "revision" : "bbadd4b853a33fd78c4ae977d17bb2af15eb3f2a", - "version" : "1.1.0" + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-prometheus", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swift-server/swift-prometheus.git", + "state" : { + "revision" : "8a8ff47403444e16d8cdfa805a5e3cb8e2efe734", + "version" : "2.2.0" } }, { @@ -240,8 +258,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-system.git", "state" : { - "revision" : "890830fff1a577dc83134890c7984020c5f6b43b", - "version" : "1.6.2" + "revision" : "395a77f0aa927f0ff73941d7ac35f2b46d47c9db", + "version" : "1.6.3" } }, { diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/configure.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/configure.swift index ea1df19..8387313 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/configure.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/configure.swift @@ -3,13 +3,16 @@ // All source code and related assets are the property of GetAutomaApp. // All rights reserved. +import AutomaUtilities import Vapor // configures your application -internal func configure(_ app: Application) throws { +internal func configure(_ app: Application) async throws { // uncomment to serve files from /Public folder // app.middleware.use(FileMiddleware(publicDirectory: app.directory.publicDirectory)) app.asyncCommands.use(SeleniumGridNodeAutoCreatorCommand(), as: "autocreator") app.asyncCommands.use(SeleniumGridNodeAutoDestroyerCommand(), as: "autodestroyer") + + try await PrometheusService().startServer() } diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/entrypoint.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/entrypoint.swift index cad2faa..f668fda 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/entrypoint.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/entrypoint.swift @@ -29,7 +29,7 @@ internal enum Entrypoint { // metadata: ["success": .stringConvertible(executorTakeoverSuccess)]) do { - try configure(app) + try await configure(app) try await app.execute() } catch { app.logger.report(error: error) diff --git a/SeleniumGridNodeMachineAutoscaler/automa.config.json b/SeleniumGridNodeMachineAutoscaler/automa.config.json new file mode 100644 index 0000000..b62d6da --- /dev/null +++ b/SeleniumGridNodeMachineAutoscaler/automa.config.json @@ -0,0 +1,16 @@ +{ + "grafana" : { + "current_environment" : "staging" + }, + "actionsSecrets" : { + "owner_repo" : "" + }, + "fly" : { + "config_files_root" : "./infra", + "environments" : [ + "production", + "sandbox", + "staging" + ] + } +} diff --git a/SeleniumGridNodeMachineAutoscaler/infra/autoscaler.toml b/SeleniumGridNodeMachineAutoscaler/infra/autoscaler.toml index af785c5..4d1e6fe 100644 --- a/SeleniumGridNodeMachineAutoscaler/infra/autoscaler.toml +++ b/SeleniumGridNodeMachineAutoscaler/infra/autoscaler.toml @@ -1,4 +1,4 @@ -app = 'automa-web-core-seleniumgrid-node-autoscaler' +app = 'automa-web-core-seleniumgrid-node-autoscaler-__FLY_ENVIRONMENT__' primary_region = 'jnb' [build] @@ -24,3 +24,7 @@ processes = [ memory = '1gb' cpu_kind = 'shared' cpus = 1 + +[metrics] +port = 6834 +path = "/metrics" diff --git a/package.json b/package.json index cbb5671..631057c 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,11 @@ "api": "npm run swift:build-and-run -- ./API API", "deploy": "flyctl deploy --ha=false", "deploy:autoscaler": "cd ./SeleniumGridNodeMachineAutoscaler && npm run deploy -- ./SeleniumGridNodeMachineAutoscaler/ --config=./infra/autoscaler.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ../infra/GITHUB_SSH_AUTHENTICATION_TOKEN)", - "deploy:api": "cd ./API && npm run deploy -- ./API/ --config=./infra/api.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ../infra/GITHUB_SSH_AUTHENTICATION_TOKEN)" + "deploy:api": "cd ./API && npm run deploy -- ./API/ --config=./infra/api.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ../infra/GITHUB_SSH_AUTHENTICATION_TOKEN)", + "fly:config": "npm run cli -- fly-config", + "format": "swiftformat .", + "deploy:backend:sandbox": "npm run fly:config -- ../Backend/infra/fly/fly.toml sandbox | npm run deploy:to:fly", + }, "repository": { "type": "git", From bc2a5fcee7c944e4d534d7ccfce9ddc094ae30b9 Mon Sep 17 00:00:00 2001 From: William Date: Tue, 30 Sep 2025 20:02:13 +0200 Subject: [PATCH 06/22] refactor(infra): moved configs, secrets and dockefiles to their own directories --- .gitignore | 2 +- README.md | 6 +++--- SeleniumGridNodeMachineAutoscaler/README.MD | 1 - docker-compose.yml | 2 +- infra/{ => dockerfiles}/HTTPDDockerfile | 0 infra/{ => fly-configs}/seleniumgrid-hub.toml | 0 infra/{ => fly-configs}/seleniumgrid-node.toml | 0 package.json | 7 +++---- 8 files changed, 8 insertions(+), 10 deletions(-) rename infra/{ => dockerfiles}/HTTPDDockerfile (100%) rename infra/{ => fly-configs}/seleniumgrid-hub.toml (100%) rename infra/{ => fly-configs}/seleniumgrid-node.toml (100%) diff --git a/.gitignore b/.gitignore index db003fc..050c070 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ .swiftlint.yml .swiftformat -infra/GITHUB_SSH_AUTHENTICATION_TOKEN +infra/docker-secrets/* .env .env.* !.env.example diff --git a/README.md b/README.md index 43b258a..4d5a512 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ **Launch all apps** -1. Launch Selenium Grid Hub: `fly launch --ha=false --org --config ./infra/seleniumgrid-hub.toml` -2. Launch Selenium Grid Node App: `fly launch --ha=false --org --config ./infra/seleniumgrid-node.toml` -3. Launch Selenium Grid Node Autoscaler: `cd ./SeleniumGridNodeMachineAutoscaler/ ; fly launch --org --ha=false --config=./infra/autoscaler.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ./infra/GITHUB_SSH_AUTHENTICATION_TOKEN)` +1. Launch Selenium Grid Hub: `fly launch --ha=false --org --config ./infra/fly-configs/seleniumgrid-hub.toml` +2. Launch Selenium Grid Node App: `fly launch --ha=false --org --config ./infra/fly-configs/seleniumgrid-node.toml` +3. Launch Selenium Grid Node Autoscaler: `cd ./SeleniumGridNodeMachineAutoscaler/ ; fly launch --org --ha=false --config=./infra/autoscaler.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ./infra/docker-secrets/GITHUB_SSH_AUTHENTICATION_TOKEN)` - Redeploy: `npm run deploy:autoscaler` - Set all secrets with `cd ./SeleniumGridNodeMachineAutoscaler/ ; fly secrets import < ./.env.production --app automa-web-core-seleniumgrid-node-autoscaler` 4. Launch AutomaWebCore main API: `cd ./API/ ; fly launch --org --ha=false --config=./infra/api.toml` diff --git a/SeleniumGridNodeMachineAutoscaler/README.MD b/SeleniumGridNodeMachineAutoscaler/README.MD index 0bf7245..e69de29 100644 --- a/SeleniumGridNodeMachineAutoscaler/README.MD +++ b/SeleniumGridNodeMachineAutoscaler/README.MD @@ -1 +0,0 @@ -1. Create a file called "GITHUB_SSH_AUTHENTICATION_TOKEN" in "./infra", and put an authentication token with clone access to AutomaUtilities repository. diff --git a/docker-compose.yml b/docker-compose.yml index 550f24a..0fb3a33 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: network_mode: host build: context: . - dockerfile: ./infra/HTTPDDockerfile + dockerfile: ./infra/dockerfiles/HTTPDDockerfile volumes: - ./TestAssets:/usr/local/apache2/htdocs/ ports: diff --git a/infra/HTTPDDockerfile b/infra/dockerfiles/HTTPDDockerfile similarity index 100% rename from infra/HTTPDDockerfile rename to infra/dockerfiles/HTTPDDockerfile diff --git a/infra/seleniumgrid-hub.toml b/infra/fly-configs/seleniumgrid-hub.toml similarity index 100% rename from infra/seleniumgrid-hub.toml rename to infra/fly-configs/seleniumgrid-hub.toml diff --git a/infra/seleniumgrid-node.toml b/infra/fly-configs/seleniumgrid-node.toml similarity index 100% rename from infra/seleniumgrid-node.toml rename to infra/fly-configs/seleniumgrid-node.toml diff --git a/package.json b/package.json index 631057c..a718eb6 100644 --- a/package.json +++ b/package.json @@ -26,12 +26,11 @@ "autoscaler:old_machines_destroyer": "npm run swift:build-and-run ./SeleniumGridNodeMachineAutoscaler SeleniumGridNodeMachineAutoscaler autodestroyer oldMachines", "api": "npm run swift:build-and-run -- ./API API", "deploy": "flyctl deploy --ha=false", - "deploy:autoscaler": "cd ./SeleniumGridNodeMachineAutoscaler && npm run deploy -- ./SeleniumGridNodeMachineAutoscaler/ --config=./infra/autoscaler.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ../infra/GITHUB_SSH_AUTHENTICATION_TOKEN)", - "deploy:api": "cd ./API && npm run deploy -- ./API/ --config=./infra/api.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ../infra/GITHUB_SSH_AUTHENTICATION_TOKEN)", - "fly:config": "npm run cli -- fly-config", + "deploy:autoscaler": "cd ./SeleniumGridNodeMachineAutoscaler && npm run deploy -- ./SeleniumGridNodeMachineAutoscaler/ --config=./infra/autoscaler.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ../infra/docker-secrets/GITHUB_SSH_AUTHENTICATION_TOKEN)", + "deploy:api": "cd ./API && npm run deploy -- ./API/ --config=./infra/api.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ../infra/docker-secrets/GITHUB_SSH_AUTHENTICATION_TOKEN)", + "fly:config": "automa generate fly ", "format": "swiftformat .", "deploy:backend:sandbox": "npm run fly:config -- ../Backend/infra/fly/fly.toml sandbox | npm run deploy:to:fly", - }, "repository": { "type": "git", From a050d13462a8f7ed0844d757cd206c967e2d1ae1 Mon Sep 17 00:00:00 2001 From: William Date: Tue, 30 Sep 2025 20:06:26 +0200 Subject: [PATCH 07/22] fixup! refactor(infra): moved configs, secrets and dockefiles to their own directories --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 4d5a512..846bfa8 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,8 @@ 4. Launch AutomaWebCore main API: `cd ./API/ ; fly launch --org --ha=false --config=./infra/api.toml` - Redeploy: `npm run deploy:api` - Set all secrets with `cd ./API/ ; fly secrets import < ./.env.production --app automa-web-core-api` + +**files to create** +1. All `.env` files in all packages that require environment variables (not in root path, inside packages) +1. Create `./infra/docker-secrets/` directory with the following files and secrets: + 1. "GITHUB_SSH_AUTHENTICATION_TOKEN": A Github fine-grained token that allows cloning AutomaUtilities repository (private) From a41cbe5bd44e2f243553c83943cb3aa7fc59288e Mon Sep 17 00:00:00 2001 From: William Date: Thu, 2 Oct 2025 13:11:06 +0200 Subject: [PATCH 08/22] fix(NodeMachineDeleter,WebCoreMetric,AutoscalerMetric): call metric on machine deletion start, fail and success ; created metrics for machine creation, deletion and getting website html --- API/Sources/API/DataTypes/WebCoreMetric.swift | 25 ++++++++++++ API/infra/automa-webcore-api.toml | 26 ------------- .../Package.resolved | 2 +- .../DataTypes/AutoscalerMetric.swift | 34 +++++++++++++++++ .../NodeMachineDeleter.swift | 38 +++++++++++++++++-- 5 files changed, 94 insertions(+), 31 deletions(-) create mode 100644 API/Sources/API/DataTypes/WebCoreMetric.swift delete mode 100644 API/infra/automa-webcore-api.toml create mode 100644 SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/DataTypes/AutoscalerMetric.swift diff --git a/API/Sources/API/DataTypes/WebCoreMetric.swift b/API/Sources/API/DataTypes/WebCoreMetric.swift new file mode 100644 index 0000000..dfa5299 --- /dev/null +++ b/API/Sources/API/DataTypes/WebCoreMetric.swift @@ -0,0 +1,25 @@ +// WebCoreMetric.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import AutomaUtilities +import Foundation +import Prometheus + +internal enum APIMetric { + public static func getWebsiteHTMLCall( + websiteUrl: URL, + jsRender: Bool, + status: MetricStatus + ) -> Prometheus.Counter { + MetricsService.global.makeCounter( + name: "get_website_html_call", + labels: [ + "status": status.rawValue, + "website_url": websiteUrl, + "js_render": jsRender, + ] + ) + } +} diff --git a/API/infra/automa-webcore-api.toml b/API/infra/automa-webcore-api.toml deleted file mode 100644 index af785c5..0000000 --- a/API/infra/automa-webcore-api.toml +++ /dev/null @@ -1,26 +0,0 @@ -app = 'automa-web-core-seleniumgrid-node-autoscaler' -primary_region = 'jnb' - -[build] - -[processes] -app = './SeleniumGridNodeMachineAutoscaler autocreator' -off_machines_auto_destroyer = './SeleniumGridNodeMachineAutoscaler autodestroyer offMachines' -old_machines_auto_destroyer = './SeleniumGridNodeMachineAutoscaler autodestroyer oldMachines' - -[http_service] -internal_port = 8080 -force_https = true -auto_stop_machines = 'off' -auto_start_machines = false -min_machines_running = 0 -processes = [ - 'app', - 'off_machines_auto_destroyer', - 'old_machines_auto_destroyer', -] - -[[vm]] -memory = '1gb' -cpu_kind = 'shared' -cpus = 1 diff --git a/SeleniumGridNodeMachineAutoscaler/Package.resolved b/SeleniumGridNodeMachineAutoscaler/Package.resolved index eda19f6..23ad6e3 100644 --- a/SeleniumGridNodeMachineAutoscaler/Package.resolved +++ b/SeleniumGridNodeMachineAutoscaler/Package.resolved @@ -25,7 +25,7 @@ "location" : "https://github.com/GetAutomaApp/AutomaUtilities.git", "state" : { "branch" : "main", - "revision" : "91fa64dab446587a495c9508429408f9d40e1dda" + "revision" : "c1b9a9e6930b66dc2139d62dff65578529b08334" } }, { diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/DataTypes/AutoscalerMetric.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/DataTypes/AutoscalerMetric.swift new file mode 100644 index 0000000..cbe5cc1 --- /dev/null +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/DataTypes/AutoscalerMetric.swift @@ -0,0 +1,34 @@ +// AutoscalerMetric.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import AutomaUtilities +import Foundation +import Prometheus + +internal enum AutoscalerMetric { + public static func deleteSeleniumGridNodeAppFlyMachine( + machineID _: String, + status: MetricStatus, + ) -> Prometheus.Counter { + MetricsService.global.makeCounter( + name: "delete_seleniumgrid_node_app_fly_machine_call", + labels: [ + "machine_id": machineID, + "status": status.rawValue + ] + ) + } + + public static func createSeleniumGridNodeAppFlyMachine( + status: MetricStatus + ) -> Prometheus.Counter { + MetricsService.global.makeCounter( + name: "create_seleniumgrid_node_app_fly_machine_call", + labels: [ + "status": status.rawValue + ] + ) + } +} diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift index 025e00a..d8aabc5 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift @@ -16,9 +16,13 @@ internal class NodeMachineDeleter: SeleniumGridNodeAppInteractor { /// Delete node machine using fly.io machines API /// - Throws: An error if deletion of machine failed public func delete() async throws { - logDeleteNodeMachineStarted() + sendTelemetryDataOnDeleteNodeMachineStarted() try await getAndValidateDeleteNodeMachineResponse() - logDeleteNodeMachineSuccess() + } + + private func sendTelemetryDataOnDeleteNodeMachineStarted() { + AutoscalerMetric.deleteSeleniumGridNodeAppFlyMachine(machineID: machineID, status: .start).increment() + logDeleteNodeMachineStarted() } private func logDeleteNodeMachineStarted() { @@ -31,9 +35,19 @@ internal class NodeMachineDeleter: SeleniumGridNodeAppInteractor { ) } + private func sendTelemetryDataOnDeleteNodeMachineSuccess() { + logDeleteNodeMachineSuccess() + AutoscalerMetric.deleteSeleniumGridNodeAppFlyMachine(machineID: machineID, status: .success).increment() + } + private func getAndValidateDeleteNodeMachineResponse() async throws { - let response = try await getDeleteNodeMachineResponse() - try validateDeleteNodeMachineResponseStatus(response: response) + do { + let response = try await getDeleteNodeMachineResponse() + try validateDeleteNodeMachineResponseStatus(response: response) + } catch { + sendTelemetryDataOnGetAndValidateDeleteNodeMachineResponseFail(error: error) + throw error + } } private func getDeleteNodeMachineResponse() async throws -> ClientResponse { @@ -54,6 +68,22 @@ internal class NodeMachineDeleter: SeleniumGridNodeAppInteractor { try handleFlyMachinesAPIError(payload: .init(message: "Failed to delete machine", error: error)) } + private func sendTelemetryDataOnGetAndValidateDeleteNodeMachineResponseFail(error: any Error) { + logGetAndValidateDeleteNodeMachineResponseFail(error: error) + AutoscalerMetric.deleteSeleniumGridNodeAppFlyMachine(machineID: machineID, status: .fail).increment() + } + + private func logGetAndValidateDeleteNodeMachineResponseFail(error: any Error) { + logger.error( + "Failed to delete node machine.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "machine_id": .string(machineID), + "error": .string(error.localizedDescription), + ] + ) + } + private func logDeleteNodeMachineSuccess() { logger.info( "Successfully deleted node machine.", From 98d8f97f3b7b8899c5663411bee1c916a4e4d962 Mon Sep 17 00:00:00 2001 From: William Date: Thu, 2 Oct 2025 18:03:28 +0200 Subject: [PATCH 09/22] fix(NodeMachineCreator): added metrics on node machine creation start, fail and success --- .../NodeMachineCreator.swift | 43 ++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineCreator.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineCreator.swift index b55c56e..a32421b 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineCreator.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineCreator.swift @@ -88,11 +88,16 @@ internal class NodeMachineCreator: NodeMachineCreationBase { } private func createImpl() async throws { - logCreateNodeMachineStarted() + sendTelemetryDataOnCreateNodeMachineStarted() let machineID = try await createNodeMachine() try await updateAndStartMachine(machineID: machineID) } + private func sendTelemetryDataOnCreateNodeMachineStarted() { + AutoscalerMetric.createSeleniumGridNodeAppFlyMachine(status: .start).increment() + logCreateNodeMachineStarted() + } + private func logCreateNodeMachineStarted() { logger.info( "Creating new node machine.", @@ -103,13 +108,26 @@ internal class NodeMachineCreator: NodeMachineCreationBase { } private func createNodeMachine() async throws -> MachineIdentifier { - let response = try await getCreateNodeMachineResponse() - try handleInvalidCreateNodeMachineResponse(response: response) - let machineID = try getMachineIDFromCreateMachineResponse(response) - logNodeMachineCreationSuccess(machineID: machineID) + let machineID: MachineIdentifier + do { + let response = try await getCreateNodeMachineResponse() + try handleInvalidCreateNodeMachineResponse(response: response) + machineID = try getMachineIDFromCreateMachineResponse(response) + } catch { + sendTelemetryDataOnCreateNodeMachineFailed(error: error) + // TODO: refactor throwing direct error in `SeleniumGridNodeMachineAutoscaler` and `NodeMachineDeleter` + // to custom `SeleniumGridNodeMachineAutoscalerError` + throw error + } + sendTelemetryDataOnCreateNodeMachineSuccess(machineID: machineID) return machineID } + private func sendTelemetryDataOnCreateNodeMachineSuccess(machineID: MachineIdentifier) { + AutoscalerMetric.createSeleniumGridNodeAppFlyMachine(status: .success).increment() + logNodeMachineCreationSuccess(machineID: machineID) + } + private func getCreateNodeMachineResponse() async throws -> ClientResponse { @@ -133,6 +151,21 @@ internal class NodeMachineCreator: NodeMachineCreationBase { return try response.content.decode(CreateMachineResponseContent.self).id } + private func sendTelemetryDataOnCreateNodeMachineFailed(error: any Error) { + AutoscalerMetric.createSeleniumGridNodeAppFlyMachine(status: .fail).increment() + logNodeMachineCreationFailed(error: error) + } + + private func logNodeMachineCreationFailed(error: any Error) { + logger.info( + "Node machine creation failed.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "error": .string(error.localizedDescription), + ] + ) + } + private func logNodeMachineCreationSuccess(machineID: MachineIdentifier) { logger.info( "Node machine creation success.", From 924fe61f544a38afc0a441c844de548dc898fee9 Mon Sep 17 00:00:00 2001 From: William Date: Thu, 2 Oct 2025 19:02:59 +0200 Subject: [PATCH 10/22] feat(WebBrowserClient): added metrics to web browser client, todos for autoscaler machine deleter and creator --- API/Package.resolved | 2 +- API/Sources/API/DataTypes/WebCoreMetric.swift | 4 +- .../WebBrowserClient/WebBrowserClient.swift | 129 ++++++++++++++---- .../DataTypes/AutoscalerMetric.swift | 2 +- .../NodeMachineCreator.swift | 11 +- .../NodeMachineDeleter.swift | 58 ++++++-- 6 files changed, 159 insertions(+), 47 deletions(-) diff --git a/API/Package.resolved b/API/Package.resolved index d3be413..bce9f50 100644 --- a/API/Package.resolved +++ b/API/Package.resolved @@ -25,7 +25,7 @@ "location" : "https://github.com/GetAutomaApp/AutomaUtilities.git", "state" : { "branch" : "main", - "revision" : "91fa64dab446587a495c9508429408f9d40e1dda" + "revision" : "c1b9a9e6930b66dc2139d62dff65578529b08334" } }, { diff --git a/API/Sources/API/DataTypes/WebCoreMetric.swift b/API/Sources/API/DataTypes/WebCoreMetric.swift index dfa5299..b561273 100644 --- a/API/Sources/API/DataTypes/WebCoreMetric.swift +++ b/API/Sources/API/DataTypes/WebCoreMetric.swift @@ -17,8 +17,8 @@ internal enum APIMetric { name: "get_website_html_call", labels: [ "status": status.rawValue, - "website_url": websiteUrl, - "js_render": jsRender, + "website_url": websiteUrl.absoluteString, + "js_render": "\(jsRender)", ] ) } diff --git a/API/Sources/API/WebBrowserClient/WebBrowserClient.swift b/API/Sources/API/WebBrowserClient/WebBrowserClient.swift index f2b432a..aa3f8ef 100644 --- a/API/Sources/API/WebBrowserClient/WebBrowserClient.swift +++ b/API/Sources/API/WebBrowserClient/WebBrowserClient.swift @@ -7,6 +7,9 @@ import AutomaUtilities import SwiftWebDriver import Vapor +// TODO: update Autoscaler NodeMachineDeleter and NodeMachineCreator to send a custom error log message on every +// location where an error could be thrown, instead of wrapping multiple try statements with one block. + internal struct WebBrowserClient { let logger: Logger let payload: APIEndpointPayload @@ -49,22 +52,38 @@ internal struct WebBrowserClient { } public func getHTML() async throws -> String { + sendTelemetryDataOnGetHTMLStarted() try await navigateDriverToURL() + if payload.scrollToBottom { + try await scrollDriverWindowToBottom() + } + let html = try await getActiveWindowOuterHTML() + sendTelemetryDataOnGetHTMLSuccess() + return html + } + + private func sendTelemetryDataOnGetHTMLStarted() { + APIMetric.getWebsiteHTMLCall(websiteUrl: payload.url, jsRender: true, status: .start).increment() + logGetHTMLStarted() + } + + private func logGetHTMLStarted() { logger.info( - "API Endpoint Payload: \(payload).", + "Getting HTML of a website started.", metadata: [ "to": .string("\(String(describing: Self.self)).\(#function)"), + "api_payload": .string(String(reflecting: payload)) ] ) - if payload.scrollToBottom { - try await scrollToBottom() - } - return try await getActiveWindowOuterHTML() } private func navigateDriverToURL() async throws { logNavigateToURLStarted() - try await driver.navigateTo(url: payload.url) + do { + try await driver.navigateTo(url: payload.url) + } catch { + sendTelemetryDataOnNavigateDriverToURLFail(error: error) + } logNavigateToURLSuccess() } @@ -78,6 +97,13 @@ internal struct WebBrowserClient { ) } + private func sendTelemetryDataOnNavigateDriverToURLFail(error: any Error) { + sendTelemetryDataOnGetHTMLFail( + reason: "Navigating web driver to URL failed.", + error: error + ) + } + private func logNavigateToURLSuccess() { logger.info( "Navigating WebDriver to URL to get HTML content as string success.", @@ -88,13 +114,17 @@ internal struct WebBrowserClient { ) } - private func scrollToBottom() async throws { - logScrollToBottomStarted() - try await driver.execute("window.scrollBy(0, document.querySelector(\"html\").scrollHeight)") - logScrollToBottomCompleted() + private func scrollDriverWindowToBottom() async throws { + logScrollDriverWindowToBottomStarted() + do { + try await driver.execute("window.scrollBy(0, document.querySelector(\"html\").scrollHeight)") + } catch { + sendTelemetryDataOnScrollDriverWindowToBottomFail(error: error) + } + logScrollDriverWindowToBottomCompleted() } - private func logScrollToBottomStarted() { + private func logScrollDriverWindowToBottomStarted() { logger.info( "Scrolling to bottom of page document started.", metadata: [ @@ -104,7 +134,11 @@ internal struct WebBrowserClient { ) } - private func logScrollToBottomCompleted() { + private func sendTelemetryDataOnScrollDriverWindowToBottomFail(error: any Error) { + sendTelemetryDataOnGetHTMLFail(reason: "Failed to scroll driver window to bottom.", error: error) + } + + private func logScrollDriverWindowToBottomCompleted() { logger.info( "Scrolling to bottom of page document completed.", metadata: [ @@ -115,25 +149,74 @@ internal struct WebBrowserClient { } private func getActiveWindowOuterHTML() async throws -> String { - let response = try await getActiveWindowOuterHTMLProperty() - return try unwrapActiveWindowOuterHTMLPropertyResponse(response) + let response = try await getActiveDriverWindowOuterHTMLProperty() + return try unwrapDriverActiveWindowOuterHTMLPropertyResponse(response) + } + + private func getActiveDriverWindowOuterHTMLProperty() async throws -> PostExecuteResponse { + do { + return try await driver.getProperty( + element: driver.findElement(.tagName("html")), + propertyName: "outerHTML" + ) + } catch { + sendTelemetryDataOnGetDriverWindowOuterHTMLPropertyFailed(error: error) + // TODO: refactor throwing direct error to custom `SeleniumGridNodeMachineAutoscalerError` + throw error + } } - private func getActiveWindowOuterHTMLProperty() async throws -> PostExecuteResponse { - try await driver.getProperty( - element: driver.findElement(.tagName("html")), - propertyName: "outerHTML" + private func sendTelemetryDataOnGetDriverWindowOuterHTMLPropertyFailed(error: any Error) { + sendTelemetryDataOnGetHTMLFail( + reason: "Failed to get 'outerHTML' property on tag", + error: error ) } - private func unwrapActiveWindowOuterHTMLPropertyResponse(_ response: PostExecuteResponse) throws -> String { + private func unwrapDriverActiveWindowOuterHTMLPropertyResponse(_ response: PostExecuteResponse) throws -> String { guard let outerHTMLString = response.value?.stringValue else { - throw AutomaGenericErrors - .notFound( - message: "'html' element of URL '\(payload.url.absoluteString)' 'outerHTML' property contains an empty value." - ) + let reason = """ + 'html' element of URL '\(payload.url.absoluteString)' 'outerHTML' property contains an empty value. + """ + let error = AutomaGenericErrors.notFound( + message: reason + ) + sendTelemetryDataOnGetHTMLFail(reason: reason, error: error) + throw error } return outerHTMLString } + + private func sendTelemetryDataOnGetHTMLSuccess() { + APIMetric.getWebsiteHTMLCall(websiteUrl: payload.url, jsRender: true, status: .success).increment() + logGetHTMLSuccess() + } + + private func logGetHTMLSuccess() { + logger.info( + "Successfully scraped HTML from website.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "website_url": .string(payload.url.absoluteString) + ] + ) + } + + private func sendTelemetryDataOnGetHTMLFail(reason: String, error: any Error) { + APIMetric.getWebsiteHTMLCall(websiteUrl: payload.url, jsRender: true, status: .fail).increment() + logGetHTMLFail(reason: reason, error: error) + } + + private func logGetHTMLFail(reason: String, error: any Error) { + logger.error( + "Failed to get website HTML.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "reason": .string(reason), + "error": .string(error.localizedDescription), + "payload": .string(String(describing: payload)) + ] + ) + } } diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/DataTypes/AutoscalerMetric.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/DataTypes/AutoscalerMetric.swift index cbe5cc1..053d22c 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/DataTypes/AutoscalerMetric.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/DataTypes/AutoscalerMetric.swift @@ -9,7 +9,7 @@ import Prometheus internal enum AutoscalerMetric { public static func deleteSeleniumGridNodeAppFlyMachine( - machineID _: String, + machineID: String, status: MetricStatus, ) -> Prometheus.Counter { MetricsService.global.makeCounter( diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineCreator.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineCreator.swift index a32421b..8561123 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineCreator.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineCreator.swift @@ -114,9 +114,8 @@ internal class NodeMachineCreator: NodeMachineCreationBase { try handleInvalidCreateNodeMachineResponse(response: response) machineID = try getMachineIDFromCreateMachineResponse(response) } catch { - sendTelemetryDataOnCreateNodeMachineFailed(error: error) - // TODO: refactor throwing direct error in `SeleniumGridNodeMachineAutoscaler` and `NodeMachineDeleter` - // to custom `SeleniumGridNodeMachineAutoscalerError` + sendTelemetryDataOnCreateNodeMachineFail(error: error) + // TODO: refactor throwing direct error to custom `SeleniumGridNodeMachineAutoscalerError` throw error } sendTelemetryDataOnCreateNodeMachineSuccess(machineID: machineID) @@ -151,12 +150,12 @@ internal class NodeMachineCreator: NodeMachineCreationBase { return try response.content.decode(CreateMachineResponseContent.self).id } - private func sendTelemetryDataOnCreateNodeMachineFailed(error: any Error) { + private func sendTelemetryDataOnCreateNodeMachineFail(error: any Error) { AutoscalerMetric.createSeleniumGridNodeAppFlyMachine(status: .fail).increment() - logNodeMachineCreationFailed(error: error) + logNodeMachineCreationFail(error: error) } - private func logNodeMachineCreationFailed(error: any Error) { + private func logNodeMachineCreationFail(error: any Error) { logger.info( "Node machine creation failed.", metadata: [ diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift index d8aabc5..4a3bde2 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift @@ -41,22 +41,26 @@ internal class NodeMachineDeleter: SeleniumGridNodeAppInteractor { } private func getAndValidateDeleteNodeMachineResponse() async throws { + let response = try await getDeleteNodeMachineResponse() + try validateDeleteNodeMachineResponseStatus(response: response) + } + + private func getDeleteNodeMachineResponse() async throws -> ClientResponse { do { - let response = try await getDeleteNodeMachineResponse() - try validateDeleteNodeMachineResponseStatus(response: response) + return try await client.delete( + .init(stringLiteral: "\(payload.nodesAppMachineAPIURL)/\(machineID)?force=true"), + headers: .init(flyAPIHTTPRequestAuthenticationHeader) + ) } catch { - sendTelemetryDataOnGetAndValidateDeleteNodeMachineResponseFail(error: error) + sendTelemetryDataOnGetAndValidateDeleteNodeMachineResponseFail( + error: error, + message: "Failed to make HTTP request to delete machine with ID '\(machineID)'." + ) + // TODO: refactor throwing direct error to custom `SeleniumGridNodeMachineAutoscalerError` throw error } } - private func getDeleteNodeMachineResponse() async throws -> ClientResponse { - try await client.delete( - .init(stringLiteral: "\(payload.nodesAppMachineAPIURL)/\(machineID)?force=true"), - headers: .init(flyAPIHTTPRequestAuthenticationHeader) - ) - } - private func validateDeleteNodeMachineResponseStatus(response: ClientResponse) throws { if isInvalidHTTPResponseStatus(status: response.status) { try handleInvalidDeleteNodeMachineResponse(response: response) @@ -64,20 +68,46 @@ internal class NodeMachineDeleter: SeleniumGridNodeAppInteractor { } private func handleInvalidDeleteNodeMachineResponse(response: ClientResponse) throws { - let error = try decodeErrorFromResponse(response) + let error: [String: String] + do { + error = try decodeErrorFromResponse(response) + } catch { + sendTelemetryDataOnUnableToDecodeErrorFromDeleteNodeResponse( + response: response, + error: error + ) + throw error + } try handleFlyMachinesAPIError(payload: .init(message: "Failed to delete machine", error: error)) } - private func sendTelemetryDataOnGetAndValidateDeleteNodeMachineResponseFail(error: any Error) { - logGetAndValidateDeleteNodeMachineResponseFail(error: error) + private func sendTelemetryDataOnUnableToDecodeErrorFromDeleteNodeResponse( + response: ClientResponse, + error: any Error + ) { + let bodyString = String(buffer: response.body ?? .init(string: "<>")) + sendTelemetryDataOnGetAndValidateDeleteNodeMachineResponseFail( + error: error, + message: """ + Invalid HTTP response status for deleting node machine with ID '\(machineID)'. \ + Failed to decode error from response body. Response body: '\(bodyString)' + """ + ) + } + + private func sendTelemetryDataOnGetAndValidateDeleteNodeMachineResponseFail( + error: any Error, message: String + ) { + logGetAndValidateDeleteNodeMachineResponseFail(message: message, error: error) AutoscalerMetric.deleteSeleniumGridNodeAppFlyMachine(machineID: machineID, status: .fail).increment() } - private func logGetAndValidateDeleteNodeMachineResponseFail(error: any Error) { + private func logGetAndValidateDeleteNodeMachineResponseFail(message: String, error: any Error) { logger.error( "Failed to delete node machine.", metadata: [ "to": .string("\(String(describing: Self.self)).\(#function)"), + "reason": .string(message), "machine_id": .string(machineID), "error": .string(error.localizedDescription), ] From 4de79ad1cea0a37469084bd464686bc1d2320f7d Mon Sep 17 00:00:00 2001 From: William Date: Fri, 3 Oct 2025 12:44:24 +0200 Subject: [PATCH 11/22] fix: better error handling and logs --- API/Package.resolved | 2 +- API/Sources/API/DataTypes/APIError.swift | 8 ++ .../WebBrowserClient/WebBrowserClient.swift | 11 +- .../Package.resolved | 2 +- .../NodeMachineCreator.swift | 111 +++++++++++++----- .../NodeMachineDeleter.swift | 29 +++-- .../NodeMachineUpdater.swift | 50 ++++++-- .../SeleniumGridNodeAppInteractor.swift | 15 ++- 8 files changed, 174 insertions(+), 54 deletions(-) create mode 100644 API/Sources/API/DataTypes/APIError.swift diff --git a/API/Package.resolved b/API/Package.resolved index bce9f50..43a5c14 100644 --- a/API/Package.resolved +++ b/API/Package.resolved @@ -25,7 +25,7 @@ "location" : "https://github.com/GetAutomaApp/AutomaUtilities.git", "state" : { "branch" : "main", - "revision" : "c1b9a9e6930b66dc2139d62dff65578529b08334" + "revision" : "d4c0dd0f6b9ed7c79ddf360c0950db8db09a0a87" } }, { diff --git a/API/Sources/API/DataTypes/APIError.swift b/API/Sources/API/DataTypes/APIError.swift new file mode 100644 index 0000000..e70d2bd --- /dev/null +++ b/API/Sources/API/DataTypes/APIError.swift @@ -0,0 +1,8 @@ +// APIError.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +internal enum APIError: Error { + case webBrowserClientError(error: String) +} diff --git a/API/Sources/API/WebBrowserClient/WebBrowserClient.swift b/API/Sources/API/WebBrowserClient/WebBrowserClient.swift index aa3f8ef..f08045a 100644 --- a/API/Sources/API/WebBrowserClient/WebBrowserClient.swift +++ b/API/Sources/API/WebBrowserClient/WebBrowserClient.swift @@ -7,9 +7,6 @@ import AutomaUtilities import SwiftWebDriver import Vapor -// TODO: update Autoscaler NodeMachineDeleter and NodeMachineCreator to send a custom error log message on every -// location where an error could be thrown, instead of wrapping multiple try statements with one block. - internal struct WebBrowserClient { let logger: Logger let payload: APIEndpointPayload @@ -161,8 +158,12 @@ internal struct WebBrowserClient { ) } catch { sendTelemetryDataOnGetDriverWindowOuterHTMLPropertyFailed(error: error) - // TODO: refactor throwing direct error to custom `SeleniumGridNodeMachineAutoscalerError` - throw error + throw APIError.webBrowserClientError( + error: """ + Failed to get active driver window `outerHTML` property for website with URL \ + '\(payload.url.absoluteString)'. Error: \(error.localizedDescription) + """ + ) } } diff --git a/SeleniumGridNodeMachineAutoscaler/Package.resolved b/SeleniumGridNodeMachineAutoscaler/Package.resolved index 23ad6e3..166bcae 100644 --- a/SeleniumGridNodeMachineAutoscaler/Package.resolved +++ b/SeleniumGridNodeMachineAutoscaler/Package.resolved @@ -25,7 +25,7 @@ "location" : "https://github.com/GetAutomaApp/AutomaUtilities.git", "state" : { "branch" : "main", - "revision" : "c1b9a9e6930b66dc2139d62dff65578529b08334" + "revision" : "d4c0dd0f6b9ed7c79ddf360c0950db8db09a0a87" } }, { diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineCreator.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineCreator.swift index 8561123..7cf5bd5 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineCreator.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineCreator.swift @@ -88,7 +88,6 @@ internal class NodeMachineCreator: NodeMachineCreationBase { } private func createImpl() async throws { - sendTelemetryDataOnCreateNodeMachineStarted() let machineID = try await createNodeMachine() try await updateAndStartMachine(machineID: machineID) } @@ -108,17 +107,17 @@ internal class NodeMachineCreator: NodeMachineCreationBase { } private func createNodeMachine() async throws -> MachineIdentifier { + sendTelemetryDataOnCreateNodeMachineStarted() + let machineID: MachineIdentifier - do { - let response = try await getCreateNodeMachineResponse() - try handleInvalidCreateNodeMachineResponse(response: response) - machineID = try getMachineIDFromCreateMachineResponse(response) - } catch { - sendTelemetryDataOnCreateNodeMachineFail(error: error) - // TODO: refactor throwing direct error to custom `SeleniumGridNodeMachineAutoscalerError` - throw error - } + let response = try await getCreateNodeMachineResponse() + + try handleInvalidCreateNodeMachineResponse(response: response) + + machineID = try getMachineIDFromCreateMachineResponse(response) + sendTelemetryDataOnCreateNodeMachineSuccess(machineID: machineID) + return machineID } @@ -130,36 +129,84 @@ internal class NodeMachineCreator: NodeMachineCreationBase { private func getCreateNodeMachineResponse() async throws -> ClientResponse { - return try await client.post(.init(stringLiteral: payload.nodesAppMachineAPIURL)) { req in - req.headers = .init(flyAPIHTTPRequestAuthenticationHeader) - try req.content.encode(machineConfiguration) + do { + return try await client.post(.init(stringLiteral: payload.nodesAppMachineAPIURL)) { req in + req.headers = .init(flyAPIHTTPRequestAuthenticationHeader) + try req.content.encode(machineConfiguration) + } + } catch { + sendTelemetryDataOnCreateNodeMachineFail( + reason: "HTTP request to create node machine failed.", + error: error, + ) + throw AutomaGenericErrors + .httpClientRequestFailed( + requestDescription: "Create node machine with fly.io Machines API.", + error: error.localizedDescription + ) } } private func handleInvalidCreateNodeMachineResponse(response: ClientResponse) throws { if isInvalidHTTPResponseStatus(status: response.status) { - let error = try decodeErrorFromResponse(response) + let error: FlyAPIError + do { + error = try decodeErrorFromResponse(response) + } catch { + sendTelemetryDataOnUnableToDecodeErrorFromCreateNodeMachineResponse( + response: response, + error: error + ) + throw error + } try handleFlyMachinesAPIError(payload: .init(message: "Node machine creation failed", error: error)) } } + private func sendTelemetryDataOnUnableToDecodeErrorFromCreateNodeMachineResponse( + response: ClientResponse, + error: any Error + ) { + let bodyString = getClientResponseBodyAsString(response: response) + sendTelemetryDataOnCreateNodeMachineFail( + reason: """ + Invalid HTTP response status '\(response.status)' for creating node machine. \ + Failed to decode error from response body. Response body: '\(bodyString)' + """, + error: error, + ) + } + private func getMachineIDFromCreateMachineResponse(_ response: ClientResponse) throws -> MachineIdentifier { struct CreateMachineResponseContent: Content { let id: String } - return try response.content.decode(CreateMachineResponseContent.self).id + do { + return try response.content.decode(CreateMachineResponseContent.self).id + } catch { + sendTelemetryDataOnCreateNodeMachineFail( + reason: "Failed to decode create node machine response as 'CreateMachineResponseContent'", + error: error, + ) + throw AutomaGenericErrors + .decodeResponseContentFailed( + contentType: "CreateMachineResponseContent", + error: error.localizedDescription + ) + } } - private func sendTelemetryDataOnCreateNodeMachineFail(error: any Error) { + private func sendTelemetryDataOnCreateNodeMachineFail(reason: String, error: any Error) { AutoscalerMetric.createSeleniumGridNodeAppFlyMachine(status: .fail).increment() - logNodeMachineCreationFail(error: error) + logNodeMachineCreationFail(reason: reason, error: error) } - private func logNodeMachineCreationFail(error: any Error) { - logger.info( + private func logNodeMachineCreationFail(reason: String, error: any Error) { + logger.error( "Node machine creation failed.", metadata: [ "to": .string("\(String(describing: Self.self)).\(#function)"), + "reason": .string(reason), "error": .string(error.localizedDescription), ] ) @@ -176,15 +223,25 @@ internal class NodeMachineCreator: NodeMachineCreationBase { } private func updateAndStartMachine(machineID: MachineIdentifier) async throws { - try await NodeMachineUpdater( - logger: logger, - client: client, - seleniumGridHubBase: seleniumGridHubBase, - machineID: machineID - ) - .updateNodeHostURLEnvironmentVariable() + do { + try await NodeMachineUpdater( + logger: logger, + client: client, + seleniumGridHubBase: seleniumGridHubBase, + machineID: machineID + ) + .updateNodeHostURLEnvironmentVariable() - try await sleepBetweenCycle(config: .init(duration: 20)) + try await sleepBetweenCycle(config: .init(duration: 20)) + } catch { + sendTelemetryDataOnCreateNodeMachineFail( + reason: """ + Failed to update and start new node machine with ID '\(machineID)'. + """, + error: error + ) + throw error + } } deinit {} diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift index 4a3bde2..1856cab 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift @@ -3,6 +3,7 @@ // All source code and related assets are the property of GetAutomaApp. // All rights reserved. +import AutomaUtilities import Vapor internal class NodeMachineDeleter: SeleniumGridNodeAppInteractor { @@ -54,10 +55,13 @@ internal class NodeMachineDeleter: SeleniumGridNodeAppInteractor { } catch { sendTelemetryDataOnGetAndValidateDeleteNodeMachineResponseFail( error: error, - message: "Failed to make HTTP request to delete machine with ID '\(machineID)'." + reason: "Failed to make HTTP request to delete machine with ID '\(machineID)'." ) - // TODO: refactor throwing direct error to custom `SeleniumGridNodeMachineAutoscalerError` - throw error + throw AutomaGenericErrors + .httpClientRequestFailed( + requestDescription: "Delete node machine with ID '\(machineID)' using fly.io Machines API", + error: error.localizedDescription + ) } } @@ -68,7 +72,7 @@ internal class NodeMachineDeleter: SeleniumGridNodeAppInteractor { } private func handleInvalidDeleteNodeMachineResponse(response: ClientResponse) throws { - let error: [String: String] + let error: FlyAPIError do { error = try decodeErrorFromResponse(response) } catch { @@ -85,29 +89,30 @@ internal class NodeMachineDeleter: SeleniumGridNodeAppInteractor { response: ClientResponse, error: any Error ) { - let bodyString = String(buffer: response.body ?? .init(string: "<>")) + let bodyString = getClientResponseBodyAsString(response: response) sendTelemetryDataOnGetAndValidateDeleteNodeMachineResponseFail( error: error, - message: """ - Invalid HTTP response status for deleting node machine with ID '\(machineID)'. \ - Failed to decode error from response body. Response body: '\(bodyString)' + reason: """ + Invalid HTTP response status '\(response.status)' for deleting node machine \ + with ID '\(machineID)'. Failed to decode error from response body. \ + Response body: '\(bodyString)' """ ) } private func sendTelemetryDataOnGetAndValidateDeleteNodeMachineResponseFail( - error: any Error, message: String + error: any Error, reason: String ) { - logGetAndValidateDeleteNodeMachineResponseFail(message: message, error: error) + logGetAndValidateDeleteNodeMachineResponseFail(reason: reason, error: error) AutoscalerMetric.deleteSeleniumGridNodeAppFlyMachine(machineID: machineID, status: .fail).increment() } - private func logGetAndValidateDeleteNodeMachineResponseFail(message: String, error: any Error) { + private func logGetAndValidateDeleteNodeMachineResponseFail(reason: String, error: any Error) { logger.error( "Failed to delete node machine.", metadata: [ "to": .string("\(String(describing: Self.self)).\(#function)"), - "reason": .string(message), + "reason": .string(reason), "machine_id": .string(machineID), "error": .string(error.localizedDescription), ] diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineUpdater.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineUpdater.swift index 7297d4e..d493ae2 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineUpdater.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineUpdater.swift @@ -60,11 +60,18 @@ internal class NodeMachineUpdater: NodeMachineCreationBase { ) async throws -> ClientResponse { let uri = getUpdateMachineURI() - return try await client - .post(uri) { req in - req.headers = .init(flyAPIHTTPRequestAuthenticationHeader) - try req.content.encode(["config": updatedConfig.config]) - } + do { + return try await client + .post(uri) { req in + req.headers = .init(flyAPIHTTPRequestAuthenticationHeader) + try req.content.encode(["config": updatedConfig.config]) + } + } catch { + throw AutomaGenericErrors.httpClientRequestFailed( + requestDescription: "Update node machine configuration using fly.io Machines API for machine with ID '\(machineID).'", + error: error.localizedDescription + ) + } } private func getUpdateMachineURI() -> URI { @@ -73,13 +80,42 @@ internal class NodeMachineUpdater: NodeMachineCreationBase { private func handleInvalidUpdateMachineResponse(response: ClientResponse) throws { if isInvalidHTTPResponseStatus(status: response.status) { + let error: FlyAPIError + do { + error = try decodeErrorFromResponse(response) + } catch { + logDecodeErrorFromUpdateNodeMachineResponseFail(response: response, error: error) + throw error + } try handleFlyMachinesAPIError(payload: .init( message: "Failed to updated machine node 'SE_NODE_HOST' environment variable to URL of the machine", - error: decodeErrorFromResponse(response) + error: error )) } } + private func logDecodeErrorFromUpdateNodeMachineResponseFail(response: ClientResponse, error: any Error) { + let bodyString = getClientResponseBodyAsString(response: response) + logNodeMachineUpdateFail( + reason: """ + Invalid HTTP response status '\(response.status)' for updating node machine. \ + Failed to decode error from response body. Response body: '\(bodyString)' + """, + error: error + ) + } + + private func logNodeMachineUpdateFail(reason: String, error: any Error) { + logger.error( + "Node machine update failed.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "reason": .string(reason), + "error": .string(error.localizedDescription), + ] + ) + } + private func logUpdateMachineSuccess(updateResponseBody: ByteBuffer) { logger.info( "Updating node 'SE_NODE_HOST' environment variable success. Machine will start automatically.", @@ -94,7 +130,7 @@ internal class NodeMachineUpdater: NodeMachineCreationBase { private func getResponseBodyFromUpdateNodeMachineResponse(_ response: ClientResponse) throws -> ByteBuffer { try response .unwrapBodyOrThrow( - errorMessage: "Failed to get update node machine response for machine with ID '\(machineID)'" + errorMessage: "Failed to get update node machine response body for machine with ID '\(machineID)'" ) } diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAppInteractor.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAppInteractor.swift index 58ea97e..21e8c32 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAppInteractor.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAppInteractor.swift @@ -22,7 +22,14 @@ internal extension SeleniumGridNodeAppInteractorBase { typealias FlyAPIError = [String: String] func decodeErrorFromResponse(_ response: ClientResponse) throws -> FlyAPIError { - return try response.content.decode(FlyAPIError.self) + do { + return try response.content.decode(FlyAPIError.self) + } catch { + throw AutomaGenericErrors.decodeResponseContentFailed( + contentType: "FlyAPIError", + error: error.localizedDescription + ) + } } func isInvalidHTTPResponseStatus(status: HTTPStatus) -> Bool { @@ -75,6 +82,12 @@ internal class SeleniumGridNodeAppInteractor: SeleniumGridNodeAppInteractorBase try await CycleSleeper(config, logger: logger).sleep() } + internal func getClientResponseBodyAsString( + response: ClientResponse, + ) -> String { + String(buffer: response.body ?? .init(string: "<>")) + } + deinit {} } From 4066959f14f4e963c9bfcc2098f3a239c28bed39 Mon Sep 17 00:00:00 2001 From: William Date: Fri, 3 Oct 2025 13:48:43 +0200 Subject: [PATCH 12/22] fix(package.json): created working deployment scripts, moved automa cli config to root TODO: launch autoscaler and api for sandbox with the correct name (prefix environment at end of app name). Delete old apps, and set environment variables. --- .../automa.config.json => automa.config.json | 2 +- {API/infra => infra/fly-configs}/api.toml | 0 .../fly-configs}/autoscaler.toml | 0 infra/scripts/deploy-app.sh | 15 +++++++++++++++ package.json | 18 ++++++++++++------ 5 files changed, 28 insertions(+), 7 deletions(-) rename SeleniumGridNodeMachineAutoscaler/automa.config.json => automa.config.json (80%) rename {API/infra => infra/fly-configs}/api.toml (100%) rename {SeleniumGridNodeMachineAutoscaler/infra => infra/fly-configs}/autoscaler.toml (100%) create mode 100755 infra/scripts/deploy-app.sh diff --git a/SeleniumGridNodeMachineAutoscaler/automa.config.json b/automa.config.json similarity index 80% rename from SeleniumGridNodeMachineAutoscaler/automa.config.json rename to automa.config.json index b62d6da..42c39b4 100644 --- a/SeleniumGridNodeMachineAutoscaler/automa.config.json +++ b/automa.config.json @@ -6,7 +6,7 @@ "owner_repo" : "" }, "fly" : { - "config_files_root" : "./infra", + "config_files_root" : "./infra/fly-configs/", "environments" : [ "production", "sandbox", diff --git a/API/infra/api.toml b/infra/fly-configs/api.toml similarity index 100% rename from API/infra/api.toml rename to infra/fly-configs/api.toml diff --git a/SeleniumGridNodeMachineAutoscaler/infra/autoscaler.toml b/infra/fly-configs/autoscaler.toml similarity index 100% rename from SeleniumGridNodeMachineAutoscaler/infra/autoscaler.toml rename to infra/fly-configs/autoscaler.toml diff --git a/infra/scripts/deploy-app.sh b/infra/scripts/deploy-app.sh new file mode 100755 index 0000000..0fbf2e5 --- /dev/null +++ b/infra/scripts/deploy-app.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# deploy-app.sh + +APP_PATH=$1 +FLY_CONFIG_TEMPLATE_FILE_NAME=$2 +ENV=$3 + +CONFIG_PATH=$(npm run fly:config -- "$FLY_CONFIG_TEMPLATE_FILE_NAME" "$ENV" | tail -n 1) +echo "Config Path $CONFIG_PATH" +cp "$CONFIG_PATH" "$APP_PATH/.fly.toml" + +cd "$APP_PATH" && flyctl deploy \ + --ha=false \ + --config=.fly.toml \ + --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN="$(cat ../infra/docker-secrets/GITHUB_SSH_AUTHENTICATION_TOKEN)" diff --git a/package.json b/package.json index a718eb6..8f01cc9 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "install:flyctl": "brew install flyctl", "format": "swiftformat .", "lint": "swiftlint --config=.swiftlint.yml .", + "format": "swiftformat .", "update:submodules": "git submodule foreach --recursive 'branch=$(git remote show origin | awk \"/HEAD branch/ {print \\$NF}\"); git checkout $branch && git pull origin $branch' && CHANGED=$(git status --porcelain | grep '^ M \\.dotfiles' || true) && if [ -n \"$CHANGED\" ]; then npm run config; fi && git add -A && git commit -m \"chore: update submodules\" || echo 'No changes to commit'", "compose:up": "npm run compose -- up -d", "compose:build": "npm run compose -- build", @@ -19,18 +20,23 @@ "compose:down": "npm run compose -- down", "compose:ps": "npm run compose -- ps -a", "compose": "docker compose", + "generate": "npm run cli -- generate", + "cli": "automa", + "fly:config": "npm run generate -- fly", "swift:build-and-run": "sh -c 'cd \"$1\" && swift build && shift && swift run \"$@\"' --", "autoscaler:all": "npx npm-run-all --parallel autoscaler:creator autoscaler:off_machines_destroyer autoscaler:old_machines_destroyer", "autoscaler:creator": "npm run swift:build-and-run ./SeleniumGridNodeMachineAutoscaler SeleniumGridNodeMachineAutoscaler autocreator", "autoscaler:off_machines_destroyer": "npm run swift:build-and-run ./SeleniumGridNodeMachineAutoscaler SeleniumGridNodeMachineAutoscaler autodestroyer offMachines", "autoscaler:old_machines_destroyer": "npm run swift:build-and-run ./SeleniumGridNodeMachineAutoscaler SeleniumGridNodeMachineAutoscaler autodestroyer oldMachines", "api": "npm run swift:build-and-run -- ./API API", - "deploy": "flyctl deploy --ha=false", - "deploy:autoscaler": "cd ./SeleniumGridNodeMachineAutoscaler && npm run deploy -- ./SeleniumGridNodeMachineAutoscaler/ --config=./infra/autoscaler.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ../infra/docker-secrets/GITHUB_SSH_AUTHENTICATION_TOKEN)", - "deploy:api": "cd ./API && npm run deploy -- ./API/ --config=./infra/api.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ../infra/docker-secrets/GITHUB_SSH_AUTHENTICATION_TOKEN)", - "fly:config": "automa generate fly ", - "format": "swiftformat .", - "deploy:backend:sandbox": "npm run fly:config -- ../Backend/infra/fly/fly.toml sandbox | npm run deploy:to:fly", + "autoscaler:all": "npx npm-run-all --parallel autoscaler:creator autoscaler:off_machines_destroyer autoscaler:old_machines_destroyer", + "fly:config": "npm run generate -- fly", + "deploy:autoscaler:sandbox": "bash ./infra/scripts/deply-app.sh ./SeleniumGridNodeMachineAutoscaler autoscaler sandbox", + "deploy:autoscaler:staging": "bash ./infra/scripts/deply-app.sh ./SeleniumGridNodeMachineAutoscaler autoscaler staging", + "deploy:autoscaler:production": "bash ./infra/scripts/deply-app.sh ./SeleniumGridNodeMachineAutoscaler autoscaler production", + "deploy:api:sandbox": "bash ./infra/scripts/deply-app.sh ./API api sandbox", + "deploy:api:staging": "bash ./infra/scripts/deply-app.sh ./API api staging", + "deploy:api:production": "bash ./infra/scripts/deply-app.sh ./API api production" }, "repository": { "type": "git", From 359d12027b1997e925f3e4b7b0dab7c1244d85e6 Mon Sep 17 00:00:00 2001 From: William Date: Tue, 7 Oct 2025 12:17:14 +0200 Subject: [PATCH 13/22] fix(Dockerfile): fixed autoscaler image not working on fly.io libcurl wasn't installed, it's required when using FoundationNetworking --- SeleniumGridNodeMachineAutoscaler/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SeleniumGridNodeMachineAutoscaler/Dockerfile b/SeleniumGridNodeMachineAutoscaler/Dockerfile index 9b26e9a..914222b 100644 --- a/SeleniumGridNodeMachineAutoscaler/Dockerfile +++ b/SeleniumGridNodeMachineAutoscaler/Dockerfile @@ -66,7 +66,7 @@ RUN export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true \ ca-certificates \ tzdata \ # If your app or its dependencies import FoundationNetworking, also install `libcurl4`. - # libcurl4 \ + libcurl4 \ # If your app or its dependencies import FoundationXML, also install `libxml2`. # libxml2 \ && rm -r /var/lib/apt/lists/* From 5354b488fe80fc56f84d86a2fc9d01c93efbc460 Mon Sep 17 00:00:00 2001 From: William Date: Tue, 7 Oct 2025 12:18:44 +0200 Subject: [PATCH 14/22] fix(package.json): small fix in deployment scripts --- infra/scripts/deploy-app.sh | 1 - package.json | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/infra/scripts/deploy-app.sh b/infra/scripts/deploy-app.sh index 0fbf2e5..dc4ea30 100755 --- a/infra/scripts/deploy-app.sh +++ b/infra/scripts/deploy-app.sh @@ -6,7 +6,6 @@ FLY_CONFIG_TEMPLATE_FILE_NAME=$2 ENV=$3 CONFIG_PATH=$(npm run fly:config -- "$FLY_CONFIG_TEMPLATE_FILE_NAME" "$ENV" | tail -n 1) -echo "Config Path $CONFIG_PATH" cp "$CONFIG_PATH" "$APP_PATH/.fly.toml" cd "$APP_PATH" && flyctl deploy \ diff --git a/package.json b/package.json index 8f01cc9..0e810f3 100644 --- a/package.json +++ b/package.json @@ -31,12 +31,12 @@ "api": "npm run swift:build-and-run -- ./API API", "autoscaler:all": "npx npm-run-all --parallel autoscaler:creator autoscaler:off_machines_destroyer autoscaler:old_machines_destroyer", "fly:config": "npm run generate -- fly", - "deploy:autoscaler:sandbox": "bash ./infra/scripts/deply-app.sh ./SeleniumGridNodeMachineAutoscaler autoscaler sandbox", - "deploy:autoscaler:staging": "bash ./infra/scripts/deply-app.sh ./SeleniumGridNodeMachineAutoscaler autoscaler staging", - "deploy:autoscaler:production": "bash ./infra/scripts/deply-app.sh ./SeleniumGridNodeMachineAutoscaler autoscaler production", - "deploy:api:sandbox": "bash ./infra/scripts/deply-app.sh ./API api sandbox", - "deploy:api:staging": "bash ./infra/scripts/deply-app.sh ./API api staging", - "deploy:api:production": "bash ./infra/scripts/deply-app.sh ./API api production" + "deploy:autoscaler:sandbox": "bash ./infra/scripts/deploy-app.sh ./SeleniumGridNodeMachineAutoscaler autoscaler sandbox", + "deploy:autoscaler:staging": "bash ./infra/scripts/deploy-app.sh ./SeleniumGridNodeMachineAutoscaler autoscaler staging", + "deploy:autoscaler:production": "bash ./infra/scripts/deploy-app.sh ./SeleniumGridNodeMachineAutoscaler autoscaler production", + "deploy:api:sandbox": "bash ./infra/scripts/deploy-app.sh ./API api sandbox", + "deploy:api:staging": "bash ./infra/scripts/deploy-app.sh ./API api staging", + "deploy:api:production": "bash ./infra/scripts/deploy-app.sh ./API api production" }, "repository": { "type": "git", From f975141007baddec50c700ef2934a46d3753873b Mon Sep 17 00:00:00 2001 From: William Date: Tue, 7 Oct 2025 12:19:37 +0200 Subject: [PATCH 15/22] docs: created example env files to know what variables are needed in different environments --- .gitignore | 1 + SeleniumGridNodeMachineAutoscaler/.env.example | 6 ++++++ SeleniumGridNodeMachineAutoscaler/.env.example.cli | 1 + SeleniumGridNodeMachineAutoscaler/.env.example.testing | 6 ++++++ 4 files changed, 14 insertions(+) create mode 100644 SeleniumGridNodeMachineAutoscaler/.env.example create mode 100644 SeleniumGridNodeMachineAutoscaler/.env.example.cli create mode 100644 SeleniumGridNodeMachineAutoscaler/.env.example.testing diff --git a/.gitignore b/.gitignore index 050c070..fdce411 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ infra/docker-secrets/* .env .env.* !.env.example +!.env.example.* diff --git a/SeleniumGridNodeMachineAutoscaler/.env.example b/SeleniumGridNodeMachineAutoscaler/.env.example new file mode 100644 index 0000000..50d091e --- /dev/null +++ b/SeleniumGridNodeMachineAutoscaler/.env.example @@ -0,0 +1,6 @@ +SELENIUM_GRID_HUB_BASE= +SELENIUM_GRID_NODE_BASE= +FLY_API_URL= +SELENIUM_GRID_NODE_FLY_APP_API_TOKEN= +FLY_METRICS_TOKEN= +NODE_MACHINE_EXPIRATION_MINUTES=60 diff --git a/SeleniumGridNodeMachineAutoscaler/.env.example.cli b/SeleniumGridNodeMachineAutoscaler/.env.example.cli new file mode 100644 index 0000000..d7c0bc7 --- /dev/null +++ b/SeleniumGridNodeMachineAutoscaler/.env.example.cli @@ -0,0 +1 @@ +ENVIRONMENT= diff --git a/SeleniumGridNodeMachineAutoscaler/.env.example.testing b/SeleniumGridNodeMachineAutoscaler/.env.example.testing new file mode 100644 index 0000000..f1b6a67 --- /dev/null +++ b/SeleniumGridNodeMachineAutoscaler/.env.example.testing @@ -0,0 +1,6 @@ +SELENIUM_GRID_HUB_BASE= +SELENIUM_GRID_NODE_BASE= +FLY_API_URL= +SELENIUM_GRID_NODE_FLY_APP_API_TOKEN= +FLY_METRICS_TOKEN= +NODE_MACHINE_EXPIRATION_MINUTES= From 51380540c94a13164d284a1c2c38f9f4663a45d8 Mon Sep 17 00:00:00 2001 From: William Date: Tue, 7 Oct 2025 12:40:45 +0200 Subject: [PATCH 16/22] fix: fixed running API image failed `AutomaUtilties` is using `FoundationNetworking` package, which needs `libcurl` on Linux --- .gitignore | 1 + API/Dockerfile | 2 +- Dockerfile | 39 ++++++++++++++++++++++++ infra/fly-configs/api.toml | 11 ++----- infra/fly-configs/seleniumgrid-hub.toml | 2 +- infra/fly-configs/seleniumgrid-node.toml | 2 +- infra/scripts/deploy-app.sh | 2 ++ 7 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 Dockerfile diff --git a/.gitignore b/.gitignore index fdce411..12dc7ec 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ infra/docker-secrets/* .env.* !.env.example !.env.example.* +.fly.toml diff --git a/API/Dockerfile b/API/Dockerfile index 76662d9..7cd649d 100644 --- a/API/Dockerfile +++ b/API/Dockerfile @@ -67,7 +67,7 @@ RUN export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true \ ca-certificates \ tzdata \ # If your app or its dependencies import FoundationNetworking, also install `libcurl4`. - # libcurl4 \ + libcurl4 \ # If your app or its dependencies import FoundationXML, also install `libxml2`. # libxml2 \ && rm -r /var/lib/apt/lists/* diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c34db09 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +# syntax = docker/dockerfile:1 + +# Adjust NODE_VERSION as desired +ARG NODE_VERSION=24.9.0 +FROM node:${NODE_VERSION}-slim as base + +LABEL fly_launch_runtime="NodeJS" + +# NodeJS app lives here +WORKDIR /app + +# Set production environment +ENV NODE_ENV=production + + +# Throw-away build stage to reduce size of final image +FROM base as build + +# Install packages needed to build node modules +RUN apt-get update -qq && \ + apt-get install -y python-is-python3 pkg-config build-essential + +# Install node modules +COPY --link package.json . +RUN npm install + +# Copy application code +COPY --link . . + + + +# Final stage for app image +FROM base + +# Copy built application +COPY --from=build /app /app + +# Start the server by default, this can be overwritten at runtime +CMD [ "npm", "run", "start" ] diff --git a/infra/fly-configs/api.toml b/infra/fly-configs/api.toml index c5b869e..fe0490b 100644 --- a/infra/fly-configs/api.toml +++ b/infra/fly-configs/api.toml @@ -1,9 +1,4 @@ -# fly.toml app configuration file generated for automa-web-core-api on 2025-09-29T12:32:49+02:00 -# -# See https://fly.io/docs/reference/configuration/ for information about how to use this file. -# - -app = 'automa-web-core-api' +app = 'automa-web-core-api-__FLY_ENVIRONMENT__' primary_region = 'jnb' [build] @@ -21,6 +16,6 @@ memory = '1gb' cpu_kind = 'shared' cpus = 1 -[metrics] +[[metrics]] port = 6834 -path = "/metrics" +path = '/metrics' diff --git a/infra/fly-configs/seleniumgrid-hub.toml b/infra/fly-configs/seleniumgrid-hub.toml index 2158e71..9d1d860 100644 --- a/infra/fly-configs/seleniumgrid-hub.toml +++ b/infra/fly-configs/seleniumgrid-hub.toml @@ -3,7 +3,7 @@ # See https://fly.io/docs/reference/configuration/ for information about how to use this file. # -app = 'automa-web-core-seleniumgrid-hub' +app = 'automa-web-core-seleniumgrid-hub-__FLY_ENVIRONMENT__' primary_region = 'jnb' [build] diff --git a/infra/fly-configs/seleniumgrid-node.toml b/infra/fly-configs/seleniumgrid-node.toml index 47c9f3b..04db34d 100644 --- a/infra/fly-configs/seleniumgrid-node.toml +++ b/infra/fly-configs/seleniumgrid-node.toml @@ -3,7 +3,7 @@ # See https://fly.io/docs/reference/configuration/ for information about how to use this file. # -app = 'automa-web-core-seleniumgrid-node' +app = 'automa-web-core-seleniumgrid-node-__FLY_ENVIRONMENT__' primary_region = 'jnb' [build] diff --git a/infra/scripts/deploy-app.sh b/infra/scripts/deploy-app.sh index dc4ea30..0d0f17c 100755 --- a/infra/scripts/deploy-app.sh +++ b/infra/scripts/deploy-app.sh @@ -12,3 +12,5 @@ cd "$APP_PATH" && flyctl deploy \ --ha=false \ --config=.fly.toml \ --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN="$(cat ../infra/docker-secrets/GITHUB_SSH_AUTHENTICATION_TOKEN)" + +rm .fly.toml From 9cb483783e0b32c92137899c50ace1ecaa1360c0 Mon Sep 17 00:00:00 2001 From: William Date: Tue, 7 Oct 2025 13:11:00 +0200 Subject: [PATCH 17/22] fix(APIController): fixed problem where scroll_to_bottom must be provided in request body content, default value set in initializer parameter doesn't work when using coding keys --- API/Sources/API/Controllers/APIController.swift | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/API/Sources/API/Controllers/APIController.swift b/API/Sources/API/Controllers/APIController.swift index b39d6b1..ca92554 100644 --- a/API/Sources/API/Controllers/APIController.swift +++ b/API/Sources/API/Controllers/APIController.swift @@ -48,7 +48,19 @@ public struct APIEndpointPayload: Content { let url: URL let scrollToBottom: Bool - init(url: URL, scrollToBottom: Bool = false) { + public init(from decoder: any Decoder) throws { + let values = try decoder.container(keyedBy: CodingKeys.self) + url = try values.decode(URL.self, forKey: .url) + guard + let stb = try? values.decode(Bool.self, forKey: .url) + else { + scrollToBottom = false + return + } + scrollToBottom = stb + } + + public init(url: URL, scrollToBottom: Bool = false) { self.url = url self.scrollToBottom = scrollToBottom } From 4bcb8aa85d70ee366c99a16ea0f8cc38d0aa5d07 Mon Sep 17 00:00:00 2001 From: William Date: Wed, 8 Oct 2025 12:08:10 +0200 Subject: [PATCH 18/22] docs: created example env files for api --- API/.env.example | 3 +++ API/.env.example.cli | 2 ++ API/.env.example.testing | 2 ++ 3 files changed, 7 insertions(+) create mode 100644 API/.env.example create mode 100644 API/.env.example.cli create mode 100644 API/.env.example.testing diff --git a/API/.env.example b/API/.env.example new file mode 100644 index 0000000..b52b952 --- /dev/null +++ b/API/.env.example @@ -0,0 +1,3 @@ +SELENIUM_GRID_HUB_BASE= +FLY_METRICS_TOKEN= + diff --git a/API/.env.example.cli b/API/.env.example.cli new file mode 100644 index 0000000..821c533 --- /dev/null +++ b/API/.env.example.cli @@ -0,0 +1,2 @@ +ENVIRONMENT= + diff --git a/API/.env.example.testing b/API/.env.example.testing new file mode 100644 index 0000000..343e92e --- /dev/null +++ b/API/.env.example.testing @@ -0,0 +1,2 @@ +SELENIUM_GRID_HUB_BASE= +FLY_METRICS_TOKEN= From 2084d3e40def79eb7de9c31b3d87cf7db6da97bf Mon Sep 17 00:00:00 2001 From: William Date: Wed, 8 Oct 2025 12:14:11 +0200 Subject: [PATCH 19/22] fix(autoscaler.toml): send metrics to all processes, not just default app Also, call send telemetry data on machine deletion --- .../NodeMachineDeleter.swift | 1 + infra/fly-configs/autoscaler.toml | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift index 1856cab..1d444b3 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift @@ -19,6 +19,7 @@ internal class NodeMachineDeleter: SeleniumGridNodeAppInteractor { public func delete() async throws { sendTelemetryDataOnDeleteNodeMachineStarted() try await getAndValidateDeleteNodeMachineResponse() + sendTelemetryDataOnDeleteNodeMachineSuccess() } private func sendTelemetryDataOnDeleteNodeMachineStarted() { diff --git a/infra/fly-configs/autoscaler.toml b/infra/fly-configs/autoscaler.toml index 4d1e6fe..1cd95f9 100644 --- a/infra/fly-configs/autoscaler.toml +++ b/infra/fly-configs/autoscaler.toml @@ -28,3 +28,8 @@ cpus = 1 [metrics] port = 6834 path = "/metrics" +processes = [ + 'app', + 'off_machines_auto_destroyer', + 'old_machines_auto_destroyer', +] From bfa080db7e35faaaf12a5626ef52a98d84ac8aaa Mon Sep 17 00:00:00 2001 From: William Date: Wed, 8 Oct 2025 12:37:59 +0200 Subject: [PATCH 20/22] fix(SeleniumGridNodeAutoOldMachineDestroyer): small log typo fix --- .../SeleniumGridNodeAutoOldMachineDestroyer.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAutoOldMachineDestroyer.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAutoOldMachineDestroyer.swift index 2b60121..cc0985a 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAutoOldMachineDestroyer.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAutoOldMachineDestroyer.swift @@ -79,7 +79,7 @@ internal class SeleniumGridNodeAutoOldMachineDestroyer: SeleniumGridNodeMachineA private func logNoMachinesToDestroy(totalMachines: Int) { logger.info( - "None of the \(totalMachines) machines in a considered old. No machines will be destroyed.", + "None of the \(totalMachines) machines are considered old. No machines will be destroyed.", metadata: [ "to": .string("\(String(describing: Self.self)).\(#function)"), ] From 2c1736ed16ddc9f7348b9fb82c36ec61cd34821e Mon Sep 17 00:00:00 2001 From: William Date: Fri, 31 Oct 2025 09:30:25 +0200 Subject: [PATCH 21/22] fix: move webcore api payload to automa utilities as it is a re-usable data type --- API/Package.resolved | 18 ++++---- .../API/Controllers/APIController.swift | 42 +------------------ .../WebBrowserClient/WebBrowserClient.swift | 4 +- 3 files changed, 13 insertions(+), 51 deletions(-) diff --git a/API/Package.resolved b/API/Package.resolved index 43a5c14..fe07cd1 100644 --- a/API/Package.resolved +++ b/API/Package.resolved @@ -6,8 +6,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/swift-server/async-http-client.git", "state" : { - "revision" : "7dc119c7edf3c23f52638faadb89182861dee853", - "version" : "1.28.0" + "revision" : "8430dd49d4e2b417f472141805c9691ec2923cb8", + "version" : "1.29.0" } }, { @@ -25,7 +25,7 @@ "location" : "https://github.com/GetAutomaApp/AutomaUtilities.git", "state" : { "branch" : "main", - "revision" : "d4c0dd0f6b9ed7c79ddf360c0950db8db09a0a87" + "revision" : "e9876286ece603e964a8ef1c9bdc9802d4f680ed" } }, { @@ -141,8 +141,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-distributed-tracing.git", "state" : { - "revision" : "6600888f4cb5bbf1bcac51000f60b2cbd224c91b", - "version" : "1.3.0" + "revision" : "baa932c1336f7894145cbaafcd34ce2dd0b77c97", + "version" : "1.3.1" } }, { @@ -222,8 +222,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-transport-services.git", "state" : { - "revision" : "e645014baea2ec1c2db564410c51a656cf47c923", - "version" : "1.25.1" + "revision" : "df6c28355051c72c884574a6c858bc54f7311ff9", + "version" : "1.25.2" } }, { @@ -258,8 +258,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/swift-server/swift-service-lifecycle.git", "state" : { - "revision" : "e7187309187695115033536e8fc9b2eb87fd956d", - "version" : "2.8.0" + "revision" : "0fcc4c9c2d58dd98504c06f7308c86de775396ff", + "version" : "2.9.0" } }, { diff --git a/API/Sources/API/Controllers/APIController.swift b/API/Sources/API/Controllers/APIController.swift index ca92554..e890926 100644 --- a/API/Sources/API/Controllers/APIController.swift +++ b/API/Sources/API/Controllers/APIController.swift @@ -3,6 +3,7 @@ // All source code and related assets are the property of GetAutomaApp. // All rights reserved. +import AutomaUtilities import Vapor internal struct APIController: RouteCollection { @@ -17,7 +18,7 @@ internal struct APIController: RouteCollection { @Sendable public func get(req: Request) async throws -> String { - let payload = try req.content.decode(APIEndpointPayload.self) + let payload = try req.content.decode(AutomaWebCoreAPIEndpointPayload.self) return try await WebBrowserClient(logger: req.logger, payload: payload).getHTML() } @@ -43,42 +44,3 @@ internal struct APIController: RouteCollection { "hello world" } } - -public struct APIEndpointPayload: Content { - let url: URL - let scrollToBottom: Bool - - public init(from decoder: any Decoder) throws { - let values = try decoder.container(keyedBy: CodingKeys.self) - url = try values.decode(URL.self, forKey: .url) - guard - let stb = try? values.decode(Bool.self, forKey: .url) - else { - scrollToBottom = false - return - } - scrollToBottom = stb - } - - public init(url: URL, scrollToBottom: Bool = false) { - self.url = url - self.scrollToBottom = scrollToBottom - } - - // configuration options required to be implemented and handled in route handlers when - // a service that needs to be able to make a request both with a browser (jsRender) and without a browser - // (jsRender=false), - // - - // let jsRender: Bool - // let residentialProxy: Bool - // let autoCaptchaSolving: Bool - - public enum CodingKeys: String, CodingKey { - case url - case scrollToBottom = "scroll_to_bottom" - // case jsRender = "js_render" - // case residentialProxy = "residential_proxy" - // case autoCaptchaSolving = "auto_captcha_solving" - } -} diff --git a/API/Sources/API/WebBrowserClient/WebBrowserClient.swift b/API/Sources/API/WebBrowserClient/WebBrowserClient.swift index f08045a..d2e619d 100644 --- a/API/Sources/API/WebBrowserClient/WebBrowserClient.swift +++ b/API/Sources/API/WebBrowserClient/WebBrowserClient.swift @@ -9,11 +9,11 @@ import Vapor internal struct WebBrowserClient { let logger: Logger - let payload: APIEndpointPayload + let payload: AutomaWebCoreAPIEndpointPayload let seleniumGridHubBase: URL let driver: WebDriver - init(logger: Logger, payload: APIEndpointPayload) async throws { + init(logger: Logger, payload: AutomaWebCoreAPIEndpointPayload) async throws { self.logger = logger self.payload = payload seleniumGridHubBase = try URL From fdca2ff8ac182f09e731eeb0ed70ae6899969b46 Mon Sep 17 00:00:00 2001 From: William Date: Fri, 31 Oct 2025 09:46:02 +0200 Subject: [PATCH 22/22] fix(Package.resolved,SeleniumGridNodeAutoOldMachineDestroyer): updated packages, better error handling -> no generic internal server errors --- API/Package.resolved | 38 +++++++++---------- ...eniumGridNodeAutoOldMachineDestroyer.swift | 7 +++- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/API/Package.resolved b/API/Package.resolved index fe07cd1..557a770 100644 --- a/API/Package.resolved +++ b/API/Package.resolved @@ -6,8 +6,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/swift-server/async-http-client.git", "state" : { - "revision" : "8430dd49d4e2b417f472141805c9691ec2923cb8", - "version" : "1.29.0" + "revision" : "efb14fec9f79f3f8d4f2a6c0530303efb6fe6533", + "version" : "1.29.1" } }, { @@ -25,7 +25,7 @@ "location" : "https://github.com/GetAutomaApp/AutomaUtilities.git", "state" : { "branch" : "main", - "revision" : "e9876286ece603e964a8ef1c9bdc9802d4f680ed" + "revision" : "331cb7c30e544907a79d0731df268ad96f98700f" } }, { @@ -87,8 +87,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-asn1.git", "state" : { - "revision" : "f70225981241859eb4aa1a18a75531d26637c8cc", - "version" : "1.4.0" + "revision" : "40d25bbb2fc5b557a9aa8512210bded327c0f60d", + "version" : "1.5.0" } }, { @@ -114,8 +114,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-certificates.git", "state" : { - "revision" : "4b092f15164144c24554e0a75e080a960c5190a6", - "version" : "1.14.0" + "revision" : "c399f90e7bbe8874f6cbfda1d5f9023d1f5ce122", + "version" : "1.15.1" } }, { @@ -132,8 +132,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-crypto.git", "state" : { - "revision" : "95ba0316a9b733e92bb6b071255ff46263bbe7dc", - "version" : "3.15.1" + "revision" : "e8ed8867ec23bccf5f3bb9342148fa8deaff9b49", + "version" : "4.1.0" } }, { @@ -150,8 +150,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-http-structured-headers.git", "state" : { - "revision" : "1625f271afb04375bf48737a5572613248d0e7a0", - "version" : "1.4.0" + "revision" : "a9f3c352f4d46afd155e00b3c6e85decae6bcbeb", + "version" : "1.5.0" } }, { @@ -159,8 +159,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-http-types.git", "state" : { - "revision" : "a0a57e949a8903563aba4615869310c0ebf14c03", - "version" : "1.4.0" + "revision" : "45eb0224913ea070ec4fba17291b9e7ecf4749ca", + "version" : "1.5.1" } }, { @@ -186,8 +186,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio.git", "state" : { - "revision" : "a18bddb0acf7a40d982b2f128ce73ce4ee31f352", - "version" : "2.86.2" + "revision" : "a24771a4c228ff116df343c85fcf3dcfae31a06c", + "version" : "2.88.0" } }, { @@ -195,8 +195,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-extras.git", "state" : { - "revision" : "a55c3dd3a81d035af8a20ce5718889c0dcab073d", - "version" : "1.29.0" + "revision" : "b87fdbf492c8fd5ac860e642c714d2d24156990a", + "version" : "1.30.0" } }, { @@ -213,8 +213,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-ssl.git", "state" : { - "revision" : "b2b043a8810ab6d51b3ff4df17f057d87ef1ec7c", - "version" : "2.34.1" + "revision" : "173cc69a058623525a58ae6710e2f5727c663793", + "version" : "2.36.0" } }, { diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAutoOldMachineDestroyer.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAutoOldMachineDestroyer.swift index cc0985a..98e746c 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAutoOldMachineDestroyer.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAutoOldMachineDestroyer.swift @@ -71,7 +71,12 @@ internal class SeleniumGridNodeAutoOldMachineDestroyer: SeleniumGridNodeMachineA guard let nodeMachineExpirationMinutesInt = TimeInterval(nodeMachineExpirationMinutes) else { - throw Abort(.internalServerError) + throw AutomaGenericErrors + .guardFailed( + message: """ + Could not convert 'NODE_MACHINE_EXPIRATION_MINUTES' of value '\(nodeMachineExpirationMinutes)' to type `TimeInterval`" + """ + ) } return nodeMachineExpirationMinutesInt