diff --git a/.gitignore b/.gitignore index 64d0287..12dc7ec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,8 @@ .swiftlint.yml .swiftformat +infra/docker-secrets/* +.env +.env.* +!.env.example +!.env.example.* +.fly.toml diff --git a/API/.env.example b/API/.env.example new file mode 100644 index 0000000..b52b952 --- /dev/null +++ b/API/.env.example @@ -0,0 +1,3 @@ +SELENIUM_GRID_HUB_BASE= +FLY_METRICS_TOKEN= + diff --git a/API/.env.example.cli b/API/.env.example.cli new file mode 100644 index 0000000..821c533 --- /dev/null +++ b/API/.env.example.cli @@ -0,0 +1,2 @@ +ENVIRONMENT= + diff --git a/API/.env.example.testing b/API/.env.example.testing new file mode 100644 index 0000000..343e92e --- /dev/null +++ b/API/.env.example.testing @@ -0,0 +1,2 @@ +SELENIUM_GRID_HUB_BASE= +FLY_METRICS_TOKEN= diff --git a/API/.gitignore b/API/.gitignore index 745c700..44d437c 100644 --- a/API/.gitignore +++ b/API/.gitignore @@ -6,7 +6,4 @@ DerivedData/ .DS_Store db.sqlite .swiftpm -.env -.env.* -!.env.example default.profraw diff --git a/API/Dockerfile b/API/Dockerfile index e4093ce..7cd649d 100644 --- a/API/Dockerfile +++ b/API/Dockerfile @@ -12,13 +12,17 @@ RUN export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true \ # Set up a build area WORKDIR /build +RUN --mount=type=secret,id=GITHUB_SSH_AUTHENTICATION_TOKEN \ + TOKEN=$(cat /run/secrets/GITHUB_SSH_AUTHENTICATION_TOKEN) && \ + git config --global url."https://x-access-token:${TOKEN}@github.com/".insteadOf "https://github.com/" + # First just resolve dependencies. # This creates a cached layer that can be reused # as long as your Package.swift/Package.resolved # files do not change. COPY ./Package.* ./ RUN swift package resolve \ - $([ -f ./Package.resolved ] && echo "--force-resolved-versions" || true) + $([ -f ./Package.resolved ] && echo "--force-resolved-versions" || true) # Copy entire repo into container COPY . . @@ -29,9 +33,9 @@ RUN mkdir /staging # N.B.: The static version of jemalloc is incompatible with the static Swift runtime. RUN --mount=type=cache,target=/build/.build \ swift build -c release \ - --product API \ - --static-swift-stdlib \ - -Xlinker -ljemalloc && \ + --product API \ + --static-swift-stdlib \ + -Xlinker -ljemalloc && \ # Copy main executable to staging area cp "$(swift build -c release --show-bin-path)/API" /staging && \ # Copy resources bundled by SPM to staging area @@ -59,13 +63,13 @@ RUN export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true \ && apt-get -q update \ && apt-get -q dist-upgrade -y \ && apt-get -q install -y \ - libjemalloc2 \ - ca-certificates \ - tzdata \ -# If your app or its dependencies import FoundationNetworking, also install `libcurl4`. - # libcurl4 \ -# If your app or its dependencies import FoundationXML, also install `libxml2`. - # libxml2 \ + libjemalloc2 \ + ca-certificates \ + tzdata \ + # If your app or its dependencies import FoundationNetworking, also install `libcurl4`. + libcurl4 \ + # If your app or its dependencies import FoundationXML, also install `libxml2`. + # libxml2 \ && rm -r /var/lib/apt/lists/* # Create a vapor user and group with /app as its home directory diff --git a/API/Package.resolved b/API/Package.resolved index a625814..557a770 100644 --- a/API/Package.resolved +++ b/API/Package.resolved @@ -1,13 +1,13 @@ { - "originHash" : "8847af8b844c9c7804e98992d8f4898a3b2f5482f54b71586e69017eb57947ed", + "originHash" : "1eb39dba8f43b8ba3a30d60d2114aa24d8aa1c4bf732fe2f87bcc0820ce9939b", "pins" : [ { "identity" : "async-http-client", "kind" : "remoteSourceControl", "location" : "https://github.com/swift-server/async-http-client.git", "state" : { - "revision" : "7dc119c7edf3c23f52638faadb89182861dee853", - "version" : "1.28.0" + "revision" : "efb14fec9f79f3f8d4f2a6c0530303efb6fe6533", + "version" : "1.29.1" } }, { @@ -25,7 +25,7 @@ "location" : "https://github.com/GetAutomaApp/AutomaUtilities.git", "state" : { "branch" : "main", - "revision" : "bc7212b7994ee496b0d3684dd77be92f23961c8f" + "revision" : "331cb7c30e544907a79d0731df268ad96f98700f" } }, { @@ -37,6 +37,15 @@ "version" : "4.15.2" } }, + { + "identity" : "flyingfox", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swhitty/FlyingFox.git", + "state" : { + "revision" : "76b2d2d0eb54f0b53e8b661cc8007c487d94f8da", + "version" : "0.25.0" + } + }, { "identity" : "multipart-kit", "kind" : "remoteSourceControl", @@ -78,8 +87,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-asn1.git", "state" : { - "revision" : "f70225981241859eb4aa1a18a75531d26637c8cc", - "version" : "1.4.0" + "revision" : "40d25bbb2fc5b557a9aa8512210bded327c0f60d", + "version" : "1.5.0" } }, { @@ -105,8 +114,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-certificates.git", "state" : { - "revision" : "4b092f15164144c24554e0a75e080a960c5190a6", - "version" : "1.14.0" + "revision" : "c399f90e7bbe8874f6cbfda1d5f9023d1f5ce122", + "version" : "1.15.1" } }, { @@ -114,8 +123,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-collections.git", "state" : { - "revision" : "8c0c0a8b49e080e54e5e328cc552821ff07cd341", - "version" : "1.2.1" + "revision" : "7b847a3b7008b2dc2f47ca3110d8c782fb2e5c7e", + "version" : "1.3.0" } }, { @@ -123,8 +132,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-crypto.git", "state" : { - "revision" : "95ba0316a9b733e92bb6b071255ff46263bbe7dc", - "version" : "3.15.1" + "revision" : "e8ed8867ec23bccf5f3bb9342148fa8deaff9b49", + "version" : "4.1.0" } }, { @@ -132,8 +141,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-distributed-tracing.git", "state" : { - "revision" : "6600888f4cb5bbf1bcac51000f60b2cbd224c91b", - "version" : "1.3.0" + "revision" : "baa932c1336f7894145cbaafcd34ce2dd0b77c97", + "version" : "1.3.1" } }, { @@ -141,8 +150,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-http-structured-headers.git", "state" : { - "revision" : "1625f271afb04375bf48737a5572613248d0e7a0", - "version" : "1.4.0" + "revision" : "a9f3c352f4d46afd155e00b3c6e85decae6bcbeb", + "version" : "1.5.0" } }, { @@ -150,8 +159,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-http-types.git", "state" : { - "revision" : "a0a57e949a8903563aba4615869310c0ebf14c03", - "version" : "1.4.0" + "revision" : "45eb0224913ea070ec4fba17291b9e7ecf4749ca", + "version" : "1.5.1" } }, { @@ -177,8 +186,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio.git", "state" : { - "revision" : "154706efd36d8d8a7d030eea9bcbeca56a947c62", - "version" : "2.86.1" + "revision" : "a24771a4c228ff116df343c85fcf3dcfae31a06c", + "version" : "2.88.0" } }, { @@ -186,8 +195,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-extras.git", "state" : { - "revision" : "a55c3dd3a81d035af8a20ce5718889c0dcab073d", - "version" : "1.29.0" + "revision" : "b87fdbf492c8fd5ac860e642c714d2d24156990a", + "version" : "1.30.0" } }, { @@ -204,8 +213,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-ssl.git", "state" : { - "revision" : "b2b043a8810ab6d51b3ff4df17f057d87ef1ec7c", - "version" : "2.34.1" + "revision" : "173cc69a058623525a58ae6710e2f5727c663793", + "version" : "2.36.0" } }, { @@ -213,8 +222,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-transport-services.git", "state" : { - "revision" : "e645014baea2ec1c2db564410c51a656cf47c923", - "version" : "1.25.1" + "revision" : "df6c28355051c72c884574a6c858bc54f7311ff9", + "version" : "1.25.2" } }, { @@ -222,8 +231,17 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-numerics.git", "state" : { - "revision" : "bbadd4b853a33fd78c4ae977d17bb2af15eb3f2a", - "version" : "1.1.0" + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-prometheus", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swift-server/swift-prometheus.git", + "state" : { + "revision" : "8a8ff47403444e16d8cdfa805a5e3cb8e2efe734", + "version" : "2.2.0" } }, { @@ -240,8 +258,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/swift-server/swift-service-lifecycle.git", "state" : { - "revision" : "e7187309187695115033536e8fc9b2eb87fd956d", - "version" : "2.8.0" + "revision" : "0fcc4c9c2d58dd98504c06f7308c86de775396ff", + "version" : "2.9.0" } }, { diff --git a/API/Sources/API/Controllers/APIController.swift b/API/Sources/API/Controllers/APIController.swift new file mode 100644 index 0000000..e890926 --- /dev/null +++ b/API/Sources/API/Controllers/APIController.swift @@ -0,0 +1,46 @@ +// APIController.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import AutomaUtilities +import Vapor + +internal struct APIController: RouteCollection { + public func boot(routes: any RoutesBuilder) throws { + let twitterRoute = routes.grouped("api") + twitterRoute.get(use: get) + twitterRoute.post(use: post) + twitterRoute.put(use: put) + twitterRoute.patch(use: patch) + twitterRoute.delete(use: delete) + } + + @Sendable + public func get(req: Request) async throws -> String { + let payload = try req.content.decode(AutomaWebCoreAPIEndpointPayload.self) + return try await WebBrowserClient(logger: req.logger, payload: payload).getHTML() + } + + // Endpoints where implementation becomes required when making any type of request (GET, POST, etc) + // with a body and headers becomes necessary for the next version of the app + @Sendable + public func post(req _: Request) async throws -> String { + "hello world" + } + + @Sendable + public func put(req _: Request) async throws -> String { + "hello world" + } + + @Sendable + public func patch(req _: Request) async throws -> String { + "hello world" + } + + @Sendable + public func delete(req _: Request) async throws -> String { + "hello world" + } +} diff --git a/API/Sources/API/DataTypes/APIError.swift b/API/Sources/API/DataTypes/APIError.swift new file mode 100644 index 0000000..e70d2bd --- /dev/null +++ b/API/Sources/API/DataTypes/APIError.swift @@ -0,0 +1,8 @@ +// APIError.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +internal enum APIError: Error { + case webBrowserClientError(error: String) +} diff --git a/API/Sources/API/DataTypes/WebCoreMetric.swift b/API/Sources/API/DataTypes/WebCoreMetric.swift new file mode 100644 index 0000000..b561273 --- /dev/null +++ b/API/Sources/API/DataTypes/WebCoreMetric.swift @@ -0,0 +1,25 @@ +// WebCoreMetric.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import AutomaUtilities +import Foundation +import Prometheus + +internal enum APIMetric { + public static func getWebsiteHTMLCall( + websiteUrl: URL, + jsRender: Bool, + status: MetricStatus + ) -> Prometheus.Counter { + MetricsService.global.makeCounter( + name: "get_website_html_call", + labels: [ + "status": status.rawValue, + "website_url": websiteUrl.absoluteString, + "js_render": "\(jsRender)", + ] + ) + } +} diff --git a/API/Sources/API/WebBrowserClient/WebBrowserClient.swift b/API/Sources/API/WebBrowserClient/WebBrowserClient.swift new file mode 100644 index 0000000..d2e619d --- /dev/null +++ b/API/Sources/API/WebBrowserClient/WebBrowserClient.swift @@ -0,0 +1,223 @@ +// WebBrowserClient.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import AutomaUtilities +import SwiftWebDriver +import Vapor + +internal struct WebBrowserClient { + let logger: Logger + let payload: AutomaWebCoreAPIEndpointPayload + let seleniumGridHubBase: URL + let driver: WebDriver + + init(logger: Logger, payload: AutomaWebCoreAPIEndpointPayload) async throws { + self.logger = logger + self.payload = payload + seleniumGridHubBase = try URL + .fromString(payload: .init(string: Environment.getOrThrow("SELENIUM_GRID_HUB_BASE"))) + driver = Self.getWebDriver(seleniumGridHubBase: seleniumGridHubBase, logger: logger) + try await driver.start() + } + + private static func getWebDriver(seleniumGridHubBase: URL, logger: Logger) -> WebDriver { + logGetWebDriverStarted(logger: logger) + + let chromeOption = ChromeOptions( + args: [ + Args(.headless) + ] + ) + + return WebDriver( + driver: ChromeDriver( + driverURL: seleniumGridHubBase, + browserObject: chromeOption + ) + ) + } + + private static func logGetWebDriverStarted(logger: Logger) { + logger.info( + "Getting new webdriver instance started.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + ] + ) + } + + public func getHTML() async throws -> String { + sendTelemetryDataOnGetHTMLStarted() + try await navigateDriverToURL() + if payload.scrollToBottom { + try await scrollDriverWindowToBottom() + } + let html = try await getActiveWindowOuterHTML() + sendTelemetryDataOnGetHTMLSuccess() + return html + } + + private func sendTelemetryDataOnGetHTMLStarted() { + APIMetric.getWebsiteHTMLCall(websiteUrl: payload.url, jsRender: true, status: .start).increment() + logGetHTMLStarted() + } + + private func logGetHTMLStarted() { + logger.info( + "Getting HTML of a website started.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "api_payload": .string(String(reflecting: payload)) + ] + ) + } + + private func navigateDriverToURL() async throws { + logNavigateToURLStarted() + do { + try await driver.navigateTo(url: payload.url) + } catch { + sendTelemetryDataOnNavigateDriverToURLFail(error: error) + } + logNavigateToURLSuccess() + } + + private func logNavigateToURLStarted() { + logger.info( + "Navigating WebDriver to URL to get HTML content as string.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "url": .string(payload.url.absoluteString), + ] + ) + } + + private func sendTelemetryDataOnNavigateDriverToURLFail(error: any Error) { + sendTelemetryDataOnGetHTMLFail( + reason: "Navigating web driver to URL failed.", + error: error + ) + } + + private func logNavigateToURLSuccess() { + logger.info( + "Navigating WebDriver to URL to get HTML content as string success.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "url": .string(payload.url.absoluteString), + ] + ) + } + + private func scrollDriverWindowToBottom() async throws { + logScrollDriverWindowToBottomStarted() + do { + try await driver.execute("window.scrollBy(0, document.querySelector(\"html\").scrollHeight)") + } catch { + sendTelemetryDataOnScrollDriverWindowToBottomFail(error: error) + } + logScrollDriverWindowToBottomCompleted() + } + + private func logScrollDriverWindowToBottomStarted() { + logger.info( + "Scrolling to bottom of page document started.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "url": .string(payload.url.absoluteString), + ] + ) + } + + private func sendTelemetryDataOnScrollDriverWindowToBottomFail(error: any Error) { + sendTelemetryDataOnGetHTMLFail(reason: "Failed to scroll driver window to bottom.", error: error) + } + + private func logScrollDriverWindowToBottomCompleted() { + logger.info( + "Scrolling to bottom of page document completed.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "url": .string(payload.url.absoluteString), + ] + ) + } + + private func getActiveWindowOuterHTML() async throws -> String { + let response = try await getActiveDriverWindowOuterHTMLProperty() + return try unwrapDriverActiveWindowOuterHTMLPropertyResponse(response) + } + + private func getActiveDriverWindowOuterHTMLProperty() async throws -> PostExecuteResponse { + do { + return try await driver.getProperty( + element: driver.findElement(.tagName("html")), + propertyName: "outerHTML" + ) + } catch { + sendTelemetryDataOnGetDriverWindowOuterHTMLPropertyFailed(error: error) + throw APIError.webBrowserClientError( + error: """ + Failed to get active driver window `outerHTML` property for website with URL \ + '\(payload.url.absoluteString)'. Error: \(error.localizedDescription) + """ + ) + } + } + + private func sendTelemetryDataOnGetDriverWindowOuterHTMLPropertyFailed(error: any Error) { + sendTelemetryDataOnGetHTMLFail( + reason: "Failed to get 'outerHTML' property on tag", + error: error + ) + } + + private func unwrapDriverActiveWindowOuterHTMLPropertyResponse(_ response: PostExecuteResponse) throws -> String { + guard let outerHTMLString = response.value?.stringValue + else { + let reason = """ + 'html' element of URL '\(payload.url.absoluteString)' 'outerHTML' property contains an empty value. + """ + let error = AutomaGenericErrors.notFound( + message: reason + ) + sendTelemetryDataOnGetHTMLFail(reason: reason, error: error) + throw error + } + return outerHTMLString + } + + private func sendTelemetryDataOnGetHTMLSuccess() { + APIMetric.getWebsiteHTMLCall(websiteUrl: payload.url, jsRender: true, status: .success).increment() + logGetHTMLSuccess() + } + + private func logGetHTMLSuccess() { + logger.info( + "Successfully scraped HTML from website.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "website_url": .string(payload.url.absoluteString) + ] + ) + } + + private func sendTelemetryDataOnGetHTMLFail(reason: String, error: any Error) { + APIMetric.getWebsiteHTMLCall(websiteUrl: payload.url, jsRender: true, status: .fail).increment() + logGetHTMLFail(reason: reason, error: error) + } + + private func logGetHTMLFail(reason: String, error: any Error) { + logger.error( + "Failed to get website HTML.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "reason": .string(reason), + "error": .string(error.localizedDescription), + "payload": .string(String(describing: payload)) + ] + ) + } +} diff --git a/API/Sources/API/WebsiteHTMLGetter/HTMLGetterPayload.swift b/API/Sources/API/WebsiteHTMLGetter/HTMLGetterPayload.swift deleted file mode 100644 index 82450c5..0000000 --- a/API/Sources/API/WebsiteHTMLGetter/HTMLGetterPayload.swift +++ /dev/null @@ -1,14 +0,0 @@ -// HTMLGetterPayload.swift -// Copyright (c) 2025 GetAutomaApp -// All source code and related assets are the property of GetAutomaApp. -// All rights reserved. - -import Vapor - -internal struct HTMLGetterPayload: Content { - let websiteURL: String - - public enum CodingKeys: String, CodingKey { - case websiteURL = "website_url" - } -} diff --git a/API/Sources/API/WebsiteHTMLGetter/WebsiteHTMLGetter.swift b/API/Sources/API/WebsiteHTMLGetter/WebsiteHTMLGetter.swift deleted file mode 100644 index 60c7f78..0000000 --- a/API/Sources/API/WebsiteHTMLGetter/WebsiteHTMLGetter.swift +++ /dev/null @@ -1,94 +0,0 @@ -// WebsiteHTMLGetter.swift -// Copyright (c) 2025 GetAutomaApp -// All source code and related assets are the property of GetAutomaApp. -// All rights reserved. - -import AutomaUtilities -import SwiftWebDriver -import Vapor - -// TODO: scroll down the page to load all javascript before getting content, some websites (especially blogs/articles) -// get the article content in chunks to have better performance and reduce web scraping success attempts - -internal struct WebsiteHTMLGetter { - let logger: Logger - let url: URL - let seleniumGridHubBase: URL - let driver: WebDriver - - init(logger: Logger, url: URL) async throws { - self.logger = logger - self.url = url - seleniumGridHubBase = try URL - .fromString(payload: .init(string: Environment.getOrThrow("SELENIUM_GRID_HUB_BASE"))) - driver = Self.getWebDriver(seleniumGridHubBase: seleniumGridHubBase) - try await driver.start() - } - - private static func getWebDriver(seleniumGridHubBase: URL) -> WebDriver { - let chromeOption = ChromeOptions( - args: [] - ) - - return WebDriver( - driver: ChromeDriver( - driverURL: seleniumGridHubBase, - browserObject: chromeOption - ) - ) - } - - public func get() async throws -> String { - try await navigateDriverToURL() - return try await getActiveWindowOuterHTML() - } - - private func navigateDriverToURL() async throws { - logNavigateToURLStarted() - try await driver.navigateTo(url: url) - logNavigateToURLSuccess() - } - - private func logNavigateToURLStarted() { - logger.info( - "Navigating WebDriver to URL to get HTML content as string.", - metadata: [ - "to": .string("\(String(describing: Self.self)).\(#function)"), - "url": .string(url.absoluteString), - ] - ) - } - - private func logNavigateToURLSuccess() { - logger.info( - "Navigating WebDriver to URL to get HTML content as string success.", - metadata: [ - "to": .string("\(String(describing: Self.self)).\(#function)"), - "url": .string(url.absoluteString), - ] - ) - } - - private func getActiveWindowOuterHTML() async throws -> String { - let response = try await getActiveWindowOuterHTMLProperty() - return try unwrapActiveWindowOuterHTMLPropertyResponse(response) - } - - private func getActiveWindowOuterHTMLProperty() async throws -> PostExecuteResponse { - try await driver.getProperty( - element: driver.findElement(.tagName("html")), - propertyName: "outerHTML" - ) - } - - private func unwrapActiveWindowOuterHTMLPropertyResponse(_ response: PostExecuteResponse) throws -> String { - guard let outerHTMLString = response.value?.stringValue - else { - throw AutomaGenericErrors - .notFound( - message: "'html' element of URL '\(url.absoluteString)' 'outerHTML' property contains an empty value." - ) - } - return outerHTMLString - } -} diff --git a/API/Sources/API/configure.swift b/API/Sources/API/configure.swift index 7715d7c..5c294b1 100644 --- a/API/Sources/API/configure.swift +++ b/API/Sources/API/configure.swift @@ -1,3 +1,9 @@ +// configure.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import AutomaUtilities import Vapor // configures your application @@ -7,4 +13,7 @@ public func configure(_ app: Application) async throws { // register routes try routes(app) + + try app.register(collection: APIController()) + try await PrometheusService().startServer() } diff --git a/API/Sources/API/routes.swift b/API/Sources/API/routes.swift index 27c6875..e3a510e 100644 --- a/API/Sources/API/routes.swift +++ b/API/Sources/API/routes.swift @@ -9,10 +9,4 @@ func routes(_ app: Application) throws { app.get { _ async in "It works!" } - - app.get("get-html") { req async throws -> String in - let content = try req.content.decode(HTMLGetterPayload.self) - let url = try URL.fromString(payload: .init(string: content.websiteURL)) - return try await WebsiteHTMLGetter(logger: req.logger, url: url).get() - } } diff --git a/API/infra/automa-webcore-api.toml b/API/infra/automa-webcore-api.toml deleted file mode 100644 index af785c5..0000000 --- a/API/infra/automa-webcore-api.toml +++ /dev/null @@ -1,26 +0,0 @@ -app = 'automa-web-core-seleniumgrid-node-autoscaler' -primary_region = 'jnb' - -[build] - -[processes] -app = './SeleniumGridNodeMachineAutoscaler autocreator' -off_machines_auto_destroyer = './SeleniumGridNodeMachineAutoscaler autodestroyer offMachines' -old_machines_auto_destroyer = './SeleniumGridNodeMachineAutoscaler autodestroyer oldMachines' - -[http_service] -internal_port = 8080 -force_https = true -auto_stop_machines = 'off' -auto_start_machines = false -min_machines_running = 0 -processes = [ - 'app', - 'off_machines_auto_destroyer', - 'old_machines_auto_destroyer', -] - -[[vm]] -memory = '1gb' -cpu_kind = 'shared' -cpus = 1 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c34db09 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +# syntax = docker/dockerfile:1 + +# Adjust NODE_VERSION as desired +ARG NODE_VERSION=24.9.0 +FROM node:${NODE_VERSION}-slim as base + +LABEL fly_launch_runtime="NodeJS" + +# NodeJS app lives here +WORKDIR /app + +# Set production environment +ENV NODE_ENV=production + + +# Throw-away build stage to reduce size of final image +FROM base as build + +# Install packages needed to build node modules +RUN apt-get update -qq && \ + apt-get install -y python-is-python3 pkg-config build-essential + +# Install node modules +COPY --link package.json . +RUN npm install + +# Copy application code +COPY --link . . + + + +# Final stage for app image +FROM base + +# Copy built application +COPY --from=build /app /app + +# Start the server by default, this can be overwritten at runtime +CMD [ "npm", "run", "start" ] diff --git a/README.md b/README.md index 43b258a..846bfa8 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,15 @@ **Launch all apps** -1. Launch Selenium Grid Hub: `fly launch --ha=false --org --config ./infra/seleniumgrid-hub.toml` -2. Launch Selenium Grid Node App: `fly launch --ha=false --org --config ./infra/seleniumgrid-node.toml` -3. Launch Selenium Grid Node Autoscaler: `cd ./SeleniumGridNodeMachineAutoscaler/ ; fly launch --org --ha=false --config=./infra/autoscaler.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ./infra/GITHUB_SSH_AUTHENTICATION_TOKEN)` +1. Launch Selenium Grid Hub: `fly launch --ha=false --org --config ./infra/fly-configs/seleniumgrid-hub.toml` +2. Launch Selenium Grid Node App: `fly launch --ha=false --org --config ./infra/fly-configs/seleniumgrid-node.toml` +3. Launch Selenium Grid Node Autoscaler: `cd ./SeleniumGridNodeMachineAutoscaler/ ; fly launch --org --ha=false --config=./infra/autoscaler.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ./infra/docker-secrets/GITHUB_SSH_AUTHENTICATION_TOKEN)` - Redeploy: `npm run deploy:autoscaler` - Set all secrets with `cd ./SeleniumGridNodeMachineAutoscaler/ ; fly secrets import < ./.env.production --app automa-web-core-seleniumgrid-node-autoscaler` 4. Launch AutomaWebCore main API: `cd ./API/ ; fly launch --org --ha=false --config=./infra/api.toml` - Redeploy: `npm run deploy:api` - Set all secrets with `cd ./API/ ; fly secrets import < ./.env.production --app automa-web-core-api` + +**files to create** +1. All `.env` files in all packages that require environment variables (not in root path, inside packages) +1. Create `./infra/docker-secrets/` directory with the following files and secrets: + 1. "GITHUB_SSH_AUTHENTICATION_TOKEN": A Github fine-grained token that allows cloning AutomaUtilities repository (private) diff --git a/SeleniumGridNodeMachineAutoscaler/.env.example b/SeleniumGridNodeMachineAutoscaler/.env.example new file mode 100644 index 0000000..50d091e --- /dev/null +++ b/SeleniumGridNodeMachineAutoscaler/.env.example @@ -0,0 +1,6 @@ +SELENIUM_GRID_HUB_BASE= +SELENIUM_GRID_NODE_BASE= +FLY_API_URL= +SELENIUM_GRID_NODE_FLY_APP_API_TOKEN= +FLY_METRICS_TOKEN= +NODE_MACHINE_EXPIRATION_MINUTES=60 diff --git a/SeleniumGridNodeMachineAutoscaler/.env.example.cli b/SeleniumGridNodeMachineAutoscaler/.env.example.cli new file mode 100644 index 0000000..d7c0bc7 --- /dev/null +++ b/SeleniumGridNodeMachineAutoscaler/.env.example.cli @@ -0,0 +1 @@ +ENVIRONMENT= diff --git a/SeleniumGridNodeMachineAutoscaler/.env.example.testing b/SeleniumGridNodeMachineAutoscaler/.env.example.testing new file mode 100644 index 0000000..f1b6a67 --- /dev/null +++ b/SeleniumGridNodeMachineAutoscaler/.env.example.testing @@ -0,0 +1,6 @@ +SELENIUM_GRID_HUB_BASE= +SELENIUM_GRID_NODE_BASE= +FLY_API_URL= +SELENIUM_GRID_NODE_FLY_APP_API_TOKEN= +FLY_METRICS_TOKEN= +NODE_MACHINE_EXPIRATION_MINUTES= diff --git a/SeleniumGridNodeMachineAutoscaler/.gitignore b/SeleniumGridNodeMachineAutoscaler/.gitignore index 6f64694..2cd3239 100644 --- a/SeleniumGridNodeMachineAutoscaler/.gitignore +++ b/SeleniumGridNodeMachineAutoscaler/.gitignore @@ -6,7 +6,3 @@ DerivedData/ .DS_Store db.sqlite .swiftpm -.env -.env.* -!.env.example -infra/GITHUB_SSH_AUTHENTICATION_TOKEN diff --git a/SeleniumGridNodeMachineAutoscaler/Dockerfile b/SeleniumGridNodeMachineAutoscaler/Dockerfile index 9b26e9a..914222b 100644 --- a/SeleniumGridNodeMachineAutoscaler/Dockerfile +++ b/SeleniumGridNodeMachineAutoscaler/Dockerfile @@ -66,7 +66,7 @@ RUN export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true \ ca-certificates \ tzdata \ # If your app or its dependencies import FoundationNetworking, also install `libcurl4`. - # libcurl4 \ + libcurl4 \ # If your app or its dependencies import FoundationXML, also install `libxml2`. # libxml2 \ && rm -r /var/lib/apt/lists/* diff --git a/SeleniumGridNodeMachineAutoscaler/Package.resolved b/SeleniumGridNodeMachineAutoscaler/Package.resolved index 506abf5..166bcae 100644 --- a/SeleniumGridNodeMachineAutoscaler/Package.resolved +++ b/SeleniumGridNodeMachineAutoscaler/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "364aa31ddb591204d017baeb5b7ea1fe5c6ab793835d018b2b9582ff8392a7ff", + "originHash" : "be6392b201136cc104d38ffbc79e947069d4553eba1e14fc3716dc7025cca775", "pins" : [ { "identity" : "async-http-client", @@ -22,10 +22,10 @@ { "identity" : "automautilities", "kind" : "remoteSourceControl", - "location" : "https://github.com/GetAutomaApp/AutomaUtilities", + "location" : "https://github.com/GetAutomaApp/AutomaUtilities.git", "state" : { "branch" : "main", - "revision" : "bc7212b7994ee496b0d3684dd77be92f23961c8f" + "revision" : "d4c0dd0f6b9ed7c79ddf360c0950db8db09a0a87" } }, { @@ -37,6 +37,15 @@ "version" : "4.15.2" } }, + { + "identity" : "flyingfox", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swhitty/FlyingFox.git", + "state" : { + "revision" : "76b2d2d0eb54f0b53e8b661cc8007c487d94f8da", + "version" : "0.25.0" + } + }, { "identity" : "multipart-kit", "kind" : "remoteSourceControl", @@ -96,8 +105,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-certificates.git", "state" : { - "revision" : "20c451f1ad8e344e61ddbb34ef196653d4b73ea6", - "version" : "1.13.0" + "revision" : "4b092f15164144c24554e0a75e080a960c5190a6", + "version" : "1.14.0" } }, { @@ -105,8 +114,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-collections.git", "state" : { - "revision" : "8c0c0a8b49e080e54e5e328cc552821ff07cd341", - "version" : "1.2.1" + "revision" : "7b847a3b7008b2dc2f47ca3110d8c782fb2e5c7e", + "version" : "1.3.0" } }, { @@ -114,8 +123,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-crypto.git", "state" : { - "revision" : "d1c6b70f7c5f19fb0b8750cb8dcdf2ea6e2d8c34", - "version" : "3.15.0" + "revision" : "95ba0316a9b733e92bb6b071255ff46263bbe7dc", + "version" : "3.15.1" } }, { @@ -159,8 +168,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-metrics.git", "state" : { - "revision" : "4c83e1cdf4ba538ef6e43a9bbd0bcc33a0ca46e3", - "version" : "2.7.0" + "revision" : "0743a9364382629da3bf5677b46a2c4b1ce5d2a6", + "version" : "2.7.1" } }, { @@ -168,8 +177,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio.git", "state" : { - "revision" : "1c30f0f2053b654e3d1302492124aa6d242cdba7", - "version" : "2.86.0" + "revision" : "a18bddb0acf7a40d982b2f128ce73ce4ee31f352", + "version" : "2.86.2" } }, { @@ -195,8 +204,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-ssl.git", "state" : { - "revision" : "737e550e607d82bf15bdfddf158ec61652ce836f", - "version" : "2.34.0" + "revision" : "b2b043a8810ab6d51b3ff4df17f057d87ef1ec7c", + "version" : "2.34.1" } }, { @@ -213,8 +222,17 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-numerics.git", "state" : { - "revision" : "bbadd4b853a33fd78c4ae977d17bb2af15eb3f2a", - "version" : "1.1.0" + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-prometheus", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swift-server/swift-prometheus.git", + "state" : { + "revision" : "8a8ff47403444e16d8cdfa805a5e3cb8e2efe734", + "version" : "2.2.0" } }, { @@ -240,8 +258,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-system.git", "state" : { - "revision" : "890830fff1a577dc83134890c7984020c5f6b43b", - "version" : "1.6.2" + "revision" : "395a77f0aa927f0ff73941d7ac35f2b46d47c9db", + "version" : "1.6.3" } }, { diff --git a/SeleniumGridNodeMachineAutoscaler/README.MD b/SeleniumGridNodeMachineAutoscaler/README.MD index 0bf7245..e69de29 100644 --- a/SeleniumGridNodeMachineAutoscaler/README.MD +++ b/SeleniumGridNodeMachineAutoscaler/README.MD @@ -1 +0,0 @@ -1. Create a file called "GITHUB_SSH_AUTHENTICATION_TOKEN" in "./infra", and put an authentication token with clone access to AutomaUtilities repository. diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/DataTypes/AutoscalerMetric.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/DataTypes/AutoscalerMetric.swift new file mode 100644 index 0000000..053d22c --- /dev/null +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/DataTypes/AutoscalerMetric.swift @@ -0,0 +1,34 @@ +// AutoscalerMetric.swift +// Copyright (c) 2025 GetAutomaApp +// All source code and related assets are the property of GetAutomaApp. +// All rights reserved. + +import AutomaUtilities +import Foundation +import Prometheus + +internal enum AutoscalerMetric { + public static func deleteSeleniumGridNodeAppFlyMachine( + machineID: String, + status: MetricStatus, + ) -> Prometheus.Counter { + MetricsService.global.makeCounter( + name: "delete_seleniumgrid_node_app_fly_machine_call", + labels: [ + "machine_id": machineID, + "status": status.rawValue + ] + ) + } + + public static func createSeleniumGridNodeAppFlyMachine( + status: MetricStatus + ) -> Prometheus.Counter { + MetricsService.global.makeCounter( + name: "create_seleniumgrid_node_app_fly_machine_call", + labels: [ + "status": status.rawValue + ] + ) + } +} diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineCreator.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineCreator.swift index b55c56e..7cf5bd5 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineCreator.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineCreator.swift @@ -88,11 +88,15 @@ internal class NodeMachineCreator: NodeMachineCreationBase { } private func createImpl() async throws { - logCreateNodeMachineStarted() let machineID = try await createNodeMachine() try await updateAndStartMachine(machineID: machineID) } + private func sendTelemetryDataOnCreateNodeMachineStarted() { + AutoscalerMetric.createSeleniumGridNodeAppFlyMachine(status: .start).increment() + logCreateNodeMachineStarted() + } + private func logCreateNodeMachineStarted() { logger.info( "Creating new node machine.", @@ -103,34 +107,109 @@ internal class NodeMachineCreator: NodeMachineCreationBase { } private func createNodeMachine() async throws -> MachineIdentifier { + sendTelemetryDataOnCreateNodeMachineStarted() + + let machineID: MachineIdentifier let response = try await getCreateNodeMachineResponse() + try handleInvalidCreateNodeMachineResponse(response: response) - let machineID = try getMachineIDFromCreateMachineResponse(response) - logNodeMachineCreationSuccess(machineID: machineID) + + machineID = try getMachineIDFromCreateMachineResponse(response) + + sendTelemetryDataOnCreateNodeMachineSuccess(machineID: machineID) + return machineID } + private func sendTelemetryDataOnCreateNodeMachineSuccess(machineID: MachineIdentifier) { + AutoscalerMetric.createSeleniumGridNodeAppFlyMachine(status: .success).increment() + logNodeMachineCreationSuccess(machineID: machineID) + } + private func getCreateNodeMachineResponse() async throws -> ClientResponse { - return try await client.post(.init(stringLiteral: payload.nodesAppMachineAPIURL)) { req in - req.headers = .init(flyAPIHTTPRequestAuthenticationHeader) - try req.content.encode(machineConfiguration) + do { + return try await client.post(.init(stringLiteral: payload.nodesAppMachineAPIURL)) { req in + req.headers = .init(flyAPIHTTPRequestAuthenticationHeader) + try req.content.encode(machineConfiguration) + } + } catch { + sendTelemetryDataOnCreateNodeMachineFail( + reason: "HTTP request to create node machine failed.", + error: error, + ) + throw AutomaGenericErrors + .httpClientRequestFailed( + requestDescription: "Create node machine with fly.io Machines API.", + error: error.localizedDescription + ) } } private func handleInvalidCreateNodeMachineResponse(response: ClientResponse) throws { if isInvalidHTTPResponseStatus(status: response.status) { - let error = try decodeErrorFromResponse(response) + let error: FlyAPIError + do { + error = try decodeErrorFromResponse(response) + } catch { + sendTelemetryDataOnUnableToDecodeErrorFromCreateNodeMachineResponse( + response: response, + error: error + ) + throw error + } try handleFlyMachinesAPIError(payload: .init(message: "Node machine creation failed", error: error)) } } + private func sendTelemetryDataOnUnableToDecodeErrorFromCreateNodeMachineResponse( + response: ClientResponse, + error: any Error + ) { + let bodyString = getClientResponseBodyAsString(response: response) + sendTelemetryDataOnCreateNodeMachineFail( + reason: """ + Invalid HTTP response status '\(response.status)' for creating node machine. \ + Failed to decode error from response body. Response body: '\(bodyString)' + """, + error: error, + ) + } + private func getMachineIDFromCreateMachineResponse(_ response: ClientResponse) throws -> MachineIdentifier { struct CreateMachineResponseContent: Content { let id: String } - return try response.content.decode(CreateMachineResponseContent.self).id + do { + return try response.content.decode(CreateMachineResponseContent.self).id + } catch { + sendTelemetryDataOnCreateNodeMachineFail( + reason: "Failed to decode create node machine response as 'CreateMachineResponseContent'", + error: error, + ) + throw AutomaGenericErrors + .decodeResponseContentFailed( + contentType: "CreateMachineResponseContent", + error: error.localizedDescription + ) + } + } + + private func sendTelemetryDataOnCreateNodeMachineFail(reason: String, error: any Error) { + AutoscalerMetric.createSeleniumGridNodeAppFlyMachine(status: .fail).increment() + logNodeMachineCreationFail(reason: reason, error: error) + } + + private func logNodeMachineCreationFail(reason: String, error: any Error) { + logger.error( + "Node machine creation failed.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "reason": .string(reason), + "error": .string(error.localizedDescription), + ] + ) } private func logNodeMachineCreationSuccess(machineID: MachineIdentifier) { @@ -144,15 +223,25 @@ internal class NodeMachineCreator: NodeMachineCreationBase { } private func updateAndStartMachine(machineID: MachineIdentifier) async throws { - try await NodeMachineUpdater( - logger: logger, - client: client, - seleniumGridHubBase: seleniumGridHubBase, - machineID: machineID - ) - .updateNodeHostURLEnvironmentVariable() - - try await sleepBetweenCycle(config: .init(duration: 20)) + do { + try await NodeMachineUpdater( + logger: logger, + client: client, + seleniumGridHubBase: seleniumGridHubBase, + machineID: machineID + ) + .updateNodeHostURLEnvironmentVariable() + + try await sleepBetweenCycle(config: .init(duration: 20)) + } catch { + sendTelemetryDataOnCreateNodeMachineFail( + reason: """ + Failed to update and start new node machine with ID '\(machineID)'. + """, + error: error + ) + throw error + } } deinit {} diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift index 025e00a..1d444b3 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineDeleter.swift @@ -3,6 +3,7 @@ // All source code and related assets are the property of GetAutomaApp. // All rights reserved. +import AutomaUtilities import Vapor internal class NodeMachineDeleter: SeleniumGridNodeAppInteractor { @@ -16,9 +17,14 @@ internal class NodeMachineDeleter: SeleniumGridNodeAppInteractor { /// Delete node machine using fly.io machines API /// - Throws: An error if deletion of machine failed public func delete() async throws { - logDeleteNodeMachineStarted() + sendTelemetryDataOnDeleteNodeMachineStarted() try await getAndValidateDeleteNodeMachineResponse() - logDeleteNodeMachineSuccess() + sendTelemetryDataOnDeleteNodeMachineSuccess() + } + + private func sendTelemetryDataOnDeleteNodeMachineStarted() { + AutoscalerMetric.deleteSeleniumGridNodeAppFlyMachine(machineID: machineID, status: .start).increment() + logDeleteNodeMachineStarted() } private func logDeleteNodeMachineStarted() { @@ -31,16 +37,33 @@ internal class NodeMachineDeleter: SeleniumGridNodeAppInteractor { ) } + private func sendTelemetryDataOnDeleteNodeMachineSuccess() { + logDeleteNodeMachineSuccess() + AutoscalerMetric.deleteSeleniumGridNodeAppFlyMachine(machineID: machineID, status: .success).increment() + } + private func getAndValidateDeleteNodeMachineResponse() async throws { let response = try await getDeleteNodeMachineResponse() try validateDeleteNodeMachineResponseStatus(response: response) } private func getDeleteNodeMachineResponse() async throws -> ClientResponse { - try await client.delete( - .init(stringLiteral: "\(payload.nodesAppMachineAPIURL)/\(machineID)?force=true"), - headers: .init(flyAPIHTTPRequestAuthenticationHeader) - ) + do { + return try await client.delete( + .init(stringLiteral: "\(payload.nodesAppMachineAPIURL)/\(machineID)?force=true"), + headers: .init(flyAPIHTTPRequestAuthenticationHeader) + ) + } catch { + sendTelemetryDataOnGetAndValidateDeleteNodeMachineResponseFail( + error: error, + reason: "Failed to make HTTP request to delete machine with ID '\(machineID)'." + ) + throw AutomaGenericErrors + .httpClientRequestFailed( + requestDescription: "Delete node machine with ID '\(machineID)' using fly.io Machines API", + error: error.localizedDescription + ) + } } private func validateDeleteNodeMachineResponseStatus(response: ClientResponse) throws { @@ -50,10 +73,53 @@ internal class NodeMachineDeleter: SeleniumGridNodeAppInteractor { } private func handleInvalidDeleteNodeMachineResponse(response: ClientResponse) throws { - let error = try decodeErrorFromResponse(response) + let error: FlyAPIError + do { + error = try decodeErrorFromResponse(response) + } catch { + sendTelemetryDataOnUnableToDecodeErrorFromDeleteNodeResponse( + response: response, + error: error + ) + throw error + } try handleFlyMachinesAPIError(payload: .init(message: "Failed to delete machine", error: error)) } + private func sendTelemetryDataOnUnableToDecodeErrorFromDeleteNodeResponse( + response: ClientResponse, + error: any Error + ) { + let bodyString = getClientResponseBodyAsString(response: response) + sendTelemetryDataOnGetAndValidateDeleteNodeMachineResponseFail( + error: error, + reason: """ + Invalid HTTP response status '\(response.status)' for deleting node machine \ + with ID '\(machineID)'. Failed to decode error from response body. \ + Response body: '\(bodyString)' + """ + ) + } + + private func sendTelemetryDataOnGetAndValidateDeleteNodeMachineResponseFail( + error: any Error, reason: String + ) { + logGetAndValidateDeleteNodeMachineResponseFail(reason: reason, error: error) + AutoscalerMetric.deleteSeleniumGridNodeAppFlyMachine(machineID: machineID, status: .fail).increment() + } + + private func logGetAndValidateDeleteNodeMachineResponseFail(reason: String, error: any Error) { + logger.error( + "Failed to delete node machine.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "reason": .string(reason), + "machine_id": .string(machineID), + "error": .string(error.localizedDescription), + ] + ) + } + private func logDeleteNodeMachineSuccess() { logger.info( "Successfully deleted node machine.", diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineUpdater.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineUpdater.swift index 7297d4e..d493ae2 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineUpdater.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/NodeMachineUpdater.swift @@ -60,11 +60,18 @@ internal class NodeMachineUpdater: NodeMachineCreationBase { ) async throws -> ClientResponse { let uri = getUpdateMachineURI() - return try await client - .post(uri) { req in - req.headers = .init(flyAPIHTTPRequestAuthenticationHeader) - try req.content.encode(["config": updatedConfig.config]) - } + do { + return try await client + .post(uri) { req in + req.headers = .init(flyAPIHTTPRequestAuthenticationHeader) + try req.content.encode(["config": updatedConfig.config]) + } + } catch { + throw AutomaGenericErrors.httpClientRequestFailed( + requestDescription: "Update node machine configuration using fly.io Machines API for machine with ID '\(machineID).'", + error: error.localizedDescription + ) + } } private func getUpdateMachineURI() -> URI { @@ -73,13 +80,42 @@ internal class NodeMachineUpdater: NodeMachineCreationBase { private func handleInvalidUpdateMachineResponse(response: ClientResponse) throws { if isInvalidHTTPResponseStatus(status: response.status) { + let error: FlyAPIError + do { + error = try decodeErrorFromResponse(response) + } catch { + logDecodeErrorFromUpdateNodeMachineResponseFail(response: response, error: error) + throw error + } try handleFlyMachinesAPIError(payload: .init( message: "Failed to updated machine node 'SE_NODE_HOST' environment variable to URL of the machine", - error: decodeErrorFromResponse(response) + error: error )) } } + private func logDecodeErrorFromUpdateNodeMachineResponseFail(response: ClientResponse, error: any Error) { + let bodyString = getClientResponseBodyAsString(response: response) + logNodeMachineUpdateFail( + reason: """ + Invalid HTTP response status '\(response.status)' for updating node machine. \ + Failed to decode error from response body. Response body: '\(bodyString)' + """, + error: error + ) + } + + private func logNodeMachineUpdateFail(reason: String, error: any Error) { + logger.error( + "Node machine update failed.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "reason": .string(reason), + "error": .string(error.localizedDescription), + ] + ) + } + private func logUpdateMachineSuccess(updateResponseBody: ByteBuffer) { logger.info( "Updating node 'SE_NODE_HOST' environment variable success. Machine will start automatically.", @@ -94,7 +130,7 @@ internal class NodeMachineUpdater: NodeMachineCreationBase { private func getResponseBodyFromUpdateNodeMachineResponse(_ response: ClientResponse) throws -> ByteBuffer { try response .unwrapBodyOrThrow( - errorMessage: "Failed to get update node machine response for machine with ID '\(machineID)'" + errorMessage: "Failed to get update node machine response body for machine with ID '\(machineID)'" ) } diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAppInteractor.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAppInteractor.swift index 58ea97e..21e8c32 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAppInteractor.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAppInteractor.swift @@ -22,7 +22,14 @@ internal extension SeleniumGridNodeAppInteractorBase { typealias FlyAPIError = [String: String] func decodeErrorFromResponse(_ response: ClientResponse) throws -> FlyAPIError { - return try response.content.decode(FlyAPIError.self) + do { + return try response.content.decode(FlyAPIError.self) + } catch { + throw AutomaGenericErrors.decodeResponseContentFailed( + contentType: "FlyAPIError", + error: error.localizedDescription + ) + } } func isInvalidHTTPResponseStatus(status: HTTPStatus) -> Bool { @@ -75,6 +82,12 @@ internal class SeleniumGridNodeAppInteractor: SeleniumGridNodeAppInteractorBase try await CycleSleeper(config, logger: logger).sleep() } + internal func getClientResponseBodyAsString( + response: ClientResponse, + ) -> String { + String(buffer: response.body ?? .init(string: "<>")) + } + deinit {} } diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAutoOldMachineDestroyer.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAutoOldMachineDestroyer.swift index af7d75e..98e746c 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAutoOldMachineDestroyer.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/SeleniumGridNodeAutoOldMachineDestroyer.swift @@ -54,22 +54,11 @@ internal class SeleniumGridNodeAutoOldMachineDestroyer: SeleniumGridNodeMachineA } if machinesToStop.isEmpty { - logger.info( - "None of the \(allMachines.count) machines in a considered old. No machines will be destroyed.", - metadata: [ - "to": .string("\(String(describing: Self.self)).\(#function)"), - ] - ) + logNoMachinesToDestroy(totalMachines: allMachines.count) return } - logger.info( - "Destroying all old node machines started.", - metadata: [ - "to": .string("\(String(describing: Self.self)).\(#function)"), - "total_machines_to_destroy": .string(String(machinesToStop.count)) - ] - ) + logDeleteAllOldNodeMachinesStarted(totalMachines: allMachines.count) for machine in machinesToStop { try await deleteNodeMachine(id: machine.id) @@ -80,7 +69,7 @@ internal class SeleniumGridNodeAutoOldMachineDestroyer: SeleniumGridNodeMachineA let nodeMachineExpirationMinutes = try Environment .getOrThrow("NODE_MACHINE_EXPIRATION_MINUTES") - guard let nodeMachineExpirationMinutesDouble = TimeInterval(nodeMachineExpirationMinutes) + guard let nodeMachineExpirationMinutesInt = TimeInterval(nodeMachineExpirationMinutes) else { throw AutomaGenericErrors .guardFailed( @@ -89,7 +78,27 @@ internal class SeleniumGridNodeAutoOldMachineDestroyer: SeleniumGridNodeMachineA """ ) } - return nodeMachineExpirationMinutesDouble + + return nodeMachineExpirationMinutesInt + } + + private func logNoMachinesToDestroy(totalMachines: Int) { + logger.info( + "None of the \(totalMachines) machines are considered old. No machines will be destroyed.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + ] + ) + } + + private func logDeleteAllOldNodeMachinesStarted(totalMachines: Int) { + logger.info( + "Destroying all old node machines started.", + metadata: [ + "to": .string("\(String(describing: Self.self)).\(#function)"), + "total_machines_to_destroy": .string(String(totalMachines)) + ] + ) } deinit {} diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/configure.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/configure.swift index ea1df19..8387313 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/configure.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/configure.swift @@ -3,13 +3,16 @@ // All source code and related assets are the property of GetAutomaApp. // All rights reserved. +import AutomaUtilities import Vapor // configures your application -internal func configure(_ app: Application) throws { +internal func configure(_ app: Application) async throws { // uncomment to serve files from /Public folder // app.middleware.use(FileMiddleware(publicDirectory: app.directory.publicDirectory)) app.asyncCommands.use(SeleniumGridNodeAutoCreatorCommand(), as: "autocreator") app.asyncCommands.use(SeleniumGridNodeAutoDestroyerCommand(), as: "autodestroyer") + + try await PrometheusService().startServer() } diff --git a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/entrypoint.swift b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/entrypoint.swift index cad2faa..f668fda 100644 --- a/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/entrypoint.swift +++ b/SeleniumGridNodeMachineAutoscaler/Sources/SeleniumGridNodeMachineAutoscaler/entrypoint.swift @@ -29,7 +29,7 @@ internal enum Entrypoint { // metadata: ["success": .stringConvertible(executorTakeoverSuccess)]) do { - try configure(app) + try await configure(app) try await app.execute() } catch { app.logger.report(error: error) diff --git a/automa.config.json b/automa.config.json new file mode 100644 index 0000000..42c39b4 --- /dev/null +++ b/automa.config.json @@ -0,0 +1,16 @@ +{ + "grafana" : { + "current_environment" : "staging" + }, + "actionsSecrets" : { + "owner_repo" : "" + }, + "fly" : { + "config_files_root" : "./infra/fly-configs/", + "environments" : [ + "production", + "sandbox", + "staging" + ] + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 550f24a..0fb3a33 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: network_mode: host build: context: . - dockerfile: ./infra/HTTPDDockerfile + dockerfile: ./infra/dockerfiles/HTTPDDockerfile volumes: - ./TestAssets:/usr/local/apache2/htdocs/ ports: diff --git a/infra/HTTPDDockerfile b/infra/dockerfiles/HTTPDDockerfile similarity index 100% rename from infra/HTTPDDockerfile rename to infra/dockerfiles/HTTPDDockerfile diff --git a/API/infra/api.toml b/infra/fly-configs/api.toml similarity index 72% rename from API/infra/api.toml rename to infra/fly-configs/api.toml index 941d03e..fe0490b 100644 --- a/API/infra/api.toml +++ b/infra/fly-configs/api.toml @@ -1,4 +1,4 @@ -app = 'automa-web-core-api' +app = 'automa-web-core-api-__FLY_ENVIRONMENT__' primary_region = 'jnb' [build] @@ -15,3 +15,7 @@ processes = ['app'] memory = '1gb' cpu_kind = 'shared' cpus = 1 + +[[metrics]] +port = 6834 +path = '/metrics' diff --git a/SeleniumGridNodeMachineAutoscaler/infra/autoscaler.toml b/infra/fly-configs/autoscaler.toml similarity index 73% rename from SeleniumGridNodeMachineAutoscaler/infra/autoscaler.toml rename to infra/fly-configs/autoscaler.toml index af785c5..1cd95f9 100644 --- a/SeleniumGridNodeMachineAutoscaler/infra/autoscaler.toml +++ b/infra/fly-configs/autoscaler.toml @@ -1,4 +1,4 @@ -app = 'automa-web-core-seleniumgrid-node-autoscaler' +app = 'automa-web-core-seleniumgrid-node-autoscaler-__FLY_ENVIRONMENT__' primary_region = 'jnb' [build] @@ -24,3 +24,12 @@ processes = [ memory = '1gb' cpu_kind = 'shared' cpus = 1 + +[metrics] +port = 6834 +path = "/metrics" +processes = [ + 'app', + 'off_machines_auto_destroyer', + 'old_machines_auto_destroyer', +] diff --git a/infra/seleniumgrid-hub.toml b/infra/fly-configs/seleniumgrid-hub.toml similarity index 84% rename from infra/seleniumgrid-hub.toml rename to infra/fly-configs/seleniumgrid-hub.toml index 2158e71..9d1d860 100644 --- a/infra/seleniumgrid-hub.toml +++ b/infra/fly-configs/seleniumgrid-hub.toml @@ -3,7 +3,7 @@ # See https://fly.io/docs/reference/configuration/ for information about how to use this file. # -app = 'automa-web-core-seleniumgrid-hub' +app = 'automa-web-core-seleniumgrid-hub-__FLY_ENVIRONMENT__' primary_region = 'jnb' [build] diff --git a/infra/seleniumgrid-node.toml b/infra/fly-configs/seleniumgrid-node.toml similarity index 54% rename from infra/seleniumgrid-node.toml rename to infra/fly-configs/seleniumgrid-node.toml index 70fe7b4..04db34d 100644 --- a/infra/seleniumgrid-node.toml +++ b/infra/fly-configs/seleniumgrid-node.toml @@ -3,16 +3,16 @@ # See https://fly.io/docs/reference/configuration/ for information about how to use this file. # -app = 'automa-web-core-seleniumgrid-node' +app = 'automa-web-core-seleniumgrid-node-__FLY_ENVIRONMENT__' primary_region = 'jnb' [build] - image = 'selenium/node-chrome:latest' +image = 'selenium/node-chrome:latest' [env] - SE_EVENT_BUS_HOST = 'automa-web-core-seleniumgrid-hub.internal' +SE_EVENT_BUS_HOST = 'automa-web-core-seleniumgrid-hub.internal' [[vm]] - memory = '2gb' - cpu_kind = 'shared' - cpus = 1 +memory = '2gb' +cpu_kind = 'performance' +cpus = 1 diff --git a/infra/scripts/deploy-app.sh b/infra/scripts/deploy-app.sh new file mode 100755 index 0000000..0d0f17c --- /dev/null +++ b/infra/scripts/deploy-app.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# deploy-app.sh + +APP_PATH=$1 +FLY_CONFIG_TEMPLATE_FILE_NAME=$2 +ENV=$3 + +CONFIG_PATH=$(npm run fly:config -- "$FLY_CONFIG_TEMPLATE_FILE_NAME" "$ENV" | tail -n 1) +cp "$CONFIG_PATH" "$APP_PATH/.fly.toml" + +cd "$APP_PATH" && flyctl deploy \ + --ha=false \ + --config=.fly.toml \ + --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN="$(cat ../infra/docker-secrets/GITHUB_SSH_AUTHENTICATION_TOKEN)" + +rm .fly.toml diff --git a/package.json b/package.json index c09ca88..0e810f3 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "install:flyctl": "brew install flyctl", "format": "swiftformat .", "lint": "swiftlint --config=.swiftlint.yml .", + "format": "swiftformat .", "update:submodules": "git submodule foreach --recursive 'branch=$(git remote show origin | awk \"/HEAD branch/ {print \\$NF}\"); git checkout $branch && git pull origin $branch' && CHANGED=$(git status --porcelain | grep '^ M \\.dotfiles' || true) && if [ -n \"$CHANGED\" ]; then npm run config; fi && git add -A && git commit -m \"chore: update submodules\" || echo 'No changes to commit'", "compose:up": "npm run compose -- up -d", "compose:build": "npm run compose -- build", @@ -19,14 +20,23 @@ "compose:down": "npm run compose -- down", "compose:ps": "npm run compose -- ps -a", "compose": "docker compose", + "generate": "npm run cli -- generate", + "cli": "automa", + "fly:config": "npm run generate -- fly", "swift:build-and-run": "sh -c 'cd \"$1\" && swift build && shift && swift run \"$@\"' --", "autoscaler:all": "npx npm-run-all --parallel autoscaler:creator autoscaler:off_machines_destroyer autoscaler:old_machines_destroyer", "autoscaler:creator": "npm run swift:build-and-run ./SeleniumGridNodeMachineAutoscaler SeleniumGridNodeMachineAutoscaler autocreator", "autoscaler:off_machines_destroyer": "npm run swift:build-and-run ./SeleniumGridNodeMachineAutoscaler SeleniumGridNodeMachineAutoscaler autodestroyer offMachines", "autoscaler:old_machines_destroyer": "npm run swift:build-and-run ./SeleniumGridNodeMachineAutoscaler SeleniumGridNodeMachineAutoscaler autodestroyer oldMachines", "api": "npm run swift:build-and-run -- ./API API", - "deploy:autoscaler": "cd ./SeleniumGridNodeMachineAutoscaler && npm run deploy -- ./SeleniumGridNodeMachineAutoscaler/ --config=./infra/autoscaler.toml --build-secret GITHUB_SSH_AUTHENTICATION_TOKEN=$(cat ./infra/GITHUB_SSH_AUTHENTICATION_TOKEN)", - "deploy:api": "cd ./API && npm run deploy -- ./API/ --config=./infra/api.toml" + "autoscaler:all": "npx npm-run-all --parallel autoscaler:creator autoscaler:off_machines_destroyer autoscaler:old_machines_destroyer", + "fly:config": "npm run generate -- fly", + "deploy:autoscaler:sandbox": "bash ./infra/scripts/deploy-app.sh ./SeleniumGridNodeMachineAutoscaler autoscaler sandbox", + "deploy:autoscaler:staging": "bash ./infra/scripts/deploy-app.sh ./SeleniumGridNodeMachineAutoscaler autoscaler staging", + "deploy:autoscaler:production": "bash ./infra/scripts/deploy-app.sh ./SeleniumGridNodeMachineAutoscaler autoscaler production", + "deploy:api:sandbox": "bash ./infra/scripts/deploy-app.sh ./API api sandbox", + "deploy:api:staging": "bash ./infra/scripts/deploy-app.sh ./API api staging", + "deploy:api:production": "bash ./infra/scripts/deploy-app.sh ./API api production" }, "repository": { "type": "git",