From 82fac91980a8ead64ec0f4769e0e8874039e0512 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sun, 15 Mar 2026 16:32:30 +0100 Subject: [PATCH 01/52] refactor: rename onclose in RunHandler --- src/engine/docker/action/reattach.ts | 2 +- src/engine/engine.ts | 2 +- src/engine/manager.ts | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/engine/docker/action/reattach.ts b/src/engine/docker/action/reattach.ts index c194a27..1f57135 100644 --- a/src/engine/docker/action/reattach.ts +++ b/src/engine/docker/action/reattach.ts @@ -42,7 +42,7 @@ export default function reattach(self: ServiceEngine, client: DockerClient): Ser const handleClosed = async () => { await deleteContainer(container.id, client, { deleteNetwork: true }); - await listener.onclose?.(); + await listener.onClose?.(); } const info = await container.inspect(); diff --git a/src/engine/engine.ts b/src/engine/engine.ts index 9534d27..88da06e 100644 --- a/src/engine/engine.ts +++ b/src/engine/engine.ts @@ -49,7 +49,7 @@ export type RunListener = { /** * Called when the container is closed, either by stop or kill, or by itself. */ - onclose?: () => Promise|void; + onClose?: () => Promise|void; } export type DockerServiceEngine = ServiceEngineI & { diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 7d35151..c5eb5c2 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -800,9 +800,6 @@ export { whenUnlocked } - -// Utils - function reqCompatibleEngine() { if (noTAlternateSett && !engine.supportsNoTemplateMode) { throw new Error('No-template mode is enabled, but current engine does not support it! Please switch to different engine.'); @@ -831,7 +828,7 @@ function buildRunListener(serviceId: string): RunListener { onMessage: (msg) => { logService(serviceId, msg); }, - onclose: async () => { + onClose: async () => { // Remove session when container is closed, because the service is not running anymore await db.deleteSession(serviceId); From dfc696a456e531038184288f624cb0b70d7d9bf2 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sun, 15 Mar 2026 22:11:55 +0100 Subject: [PATCH 02/52] TODO.txt --- TODO.txt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 TODO.txt diff --git a/TODO.txt b/TODO.txt new file mode 100644 index 0000000..d7b080a --- /dev/null +++ b/TODO.txt @@ -0,0 +1,6 @@ +Odebrat ukládání běžících sessions do databáze, držet jen v runtimu +odebrat no template mode +vyřešit logging u servis a logování změn stavů +abstrahovat template management a udělat další sources jako například používání images z docker registry +předělat usages monitoring na nějaký standardizovaný formát/způsob +udělat nějaký auto packaging do balíčků na githubu jako bundle aby se dal spustit jako systémový proces v systemctl například \ No newline at end of file From f49022c072ec3622d3cbe3e772f47afaaf7b6110 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sun, 15 Mar 2026 22:12:43 +0100 Subject: [PATCH 03/52] TODO.txt --- TODO.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/TODO.txt b/TODO.txt index d7b080a..6c50667 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,5 +1,6 @@ Odebrat ukládání běžících sessions do databáze, držet jen v runtimu odebrat no template mode +odebrat nodeId, cluster support se bude řešit jinak vyřešit logging u servis a logování změn stavů abstrahovat template management a udělat další sources jako například používání images z docker registry předělat usages monitoring na nějaký standardizovaný formát/způsob From 28b88370cca4da86346a913d0ab033c6fbb47d82 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sun, 15 Mar 2026 22:14:41 +0100 Subject: [PATCH 04/52] TODO.txt --- TODO.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/TODO.txt b/TODO.txt index 6c50667..28379ec 100644 --- a/TODO.txt +++ b/TODO.txt @@ -4,4 +4,5 @@ odebrat nodeId, cluster support se bude řešit jinak vyřešit logging u servis a logování změn stavů abstrahovat template management a udělat další sources jako například používání images z docker registry předělat usages monitoring na nějaký standardizovaný formát/způsob +pořešit /resources/config.yml aby měl víc možností a možná byl uložený jinde v systému podle platformy udělat nějaký auto packaging do balíčků na githubu jako bundle aby se dal spustit jako systémový proces v systemctl například \ No newline at end of file From e95bf1960c83ea33b42a78a0f8002b0ad8e95b3f Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sun, 15 Mar 2026 22:15:15 +0100 Subject: [PATCH 05/52] refactor: remove import --- src/database/manager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/manager.ts b/src/database/manager.ts index e05b244..09b48ac 100644 --- a/src/database/manager.ts +++ b/src/database/manager.ts @@ -1,4 +1,4 @@ -import {PrismaClient, Image} from "@prisma/client"; +import {PrismaClient} from "@prisma/client"; import {ImageModel, PermaModel, SessionModel} from "./models"; import {optionsDiffer} from "@nsm/engine/image"; From 042c08190ecdfb1abe104442a76ca526daafeebb Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 16 Mar 2026 12:58:05 +0100 Subject: [PATCH 06/52] TODO.txt --- TODO.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO.txt b/TODO.txt index 28379ec..c26e260 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,7 +1,7 @@ +vyřešit logging u servis a logování změn stavů Odebrat ukládání běžících sessions do databáze, držet jen v runtimu odebrat no template mode odebrat nodeId, cluster support se bude řešit jinak -vyřešit logging u servis a logování změn stavů abstrahovat template management a udělat další sources jako například používání images z docker registry předělat usages monitoring na nějaký standardizovaný formát/způsob pořešit /resources/config.yml aby měl víc možností a možná byl uložený jinde v systému podle platformy From 8b0f5353cd5c929585b5edc4ce4edb2cea1c0412 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 16 Mar 2026 13:04:27 +0100 Subject: [PATCH 07/52] refactor: await state message calls --- src/engine/docker/action/reattach.ts | 2 +- src/engine/docker/action/run.ts | 12 ++++++------ src/engine/engine.ts | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/engine/docker/action/reattach.ts b/src/engine/docker/action/reattach.ts index 1f57135..41d2f9f 100644 --- a/src/engine/docker/action/reattach.ts +++ b/src/engine/docker/action/reattach.ts @@ -70,6 +70,6 @@ export default function reattach(self: ServiceEngine, client: DockerClient): Ser }); (self as DockerServiceEngine).rws[container.id] = rws; - listener.onStateMessage?.('Watching changes'); + await listener.onStateMessage?.('Watching changes'); } } \ No newline at end of file diff --git a/src/engine/docker/action/run.ts b/src/engine/docker/action/run.ts index 2cdf671..f62452d 100644 --- a/src/engine/docker/action/run.ts +++ b/src/engine/docker/action/run.ts @@ -103,15 +103,15 @@ export default function run(self: ServiceEngine, client: DockerClient): ServiceE // Prepare volume let creating = await prepareVolume(client, volumeId); if (creating) { - listener.onStateMessage('Created new volume'); + await listener.onStateMessage('Created new volume'); } - listener.onStateMessage('Preparing network'); + await listener.onStateMessage('Preparing network'); const net = await prepareNetwork(client, options.network, meta, creating); // Port decorator that takes port and according to network changes it to : or keeps the same. - listener.onStateMessage('Preparing container'); + await listener.onStateMessage('Preparing container'); container = await prepareContainer(client, imageId, buildDir(templateId), volumeId, options, net); - listener.onStateMessage('Starting container'); + await listener.onStateMessage('Starting container'); await container.start(); const info = await container.inspect(); @@ -130,8 +130,8 @@ export default function run(self: ServiceEngine, client: DockerClient): ServiceE }); const msg = logs.toString("utf8"); - listener.onStateMessage("Container failed to start"); - listener.onMessage(msg); + await listener.onStateMessage("Container failed to start"); + await listener.onMessage(msg); } catch (e) { ctx.logger.error("Error while fetching logs for failed container " + container.id, e); } diff --git a/src/engine/engine.ts b/src/engine/engine.ts index 88da06e..d35852f 100644 --- a/src/engine/engine.ts +++ b/src/engine/engine.ts @@ -121,6 +121,8 @@ export type ServiceEngine = { /** * Reattaches to a container. * + * When this completes, the service is up and running. + * * @param id Container ID * @param listener Listener for container messages and state changes */ From 41bc1098cf1932129e06796d42a2b76c5fb944f1 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 16 Mar 2026 13:05:36 +0100 Subject: [PATCH 08/52] refactor: comments --- src/engine/manager.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/engine/manager.ts b/src/engine/manager.ts index c5eb5c2..605d59a 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -847,8 +847,8 @@ function buildRunListener(serviceId: string): RunListener { async function getPermaModel(id: string) { const perma_ = await db.getPerma(id); - // Service does not exist if (!perma_) { + // Service does not exist throw new _InternalError('Not found.', 3); } @@ -888,9 +888,9 @@ export default async function ({db, appConfig, logger}: { await db.deleteSession(session.serviceId); } } - // + await new Promise((resolve) => whenUnlockedAll(() => resolve(null))); - // + const running = await engine.listRunning(); for (const id of running) { const volumeId = await engine.getAttachedVolume(id); From fcb7a3c3a3d3836865fee0bbb9653e94e20eb269 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 16 Mar 2026 13:09:45 +0100 Subject: [PATCH 09/52] refactor: remove async from functions where is not needed --- src/engine/manager.ts | 2 +- src/engine/monitoring/templateDirWatcher.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 605d59a..2c224c7 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -362,7 +362,7 @@ async function init(db_: Database, appConfig_: any) { nodeId = appConfig['node_id'] as string; initImageEngine(engine, templateManager, templateDirWatcher, db_, currentContext.logger); - await watchTemplateDirChanges(currentContext.logger); + watchTemplateDirChanges(currentContext.logger); } export async function expandEngine(exp?: T): Promise { diff --git a/src/engine/monitoring/templateDirWatcher.ts b/src/engine/monitoring/templateDirWatcher.ts index 03f08c5..1fc74eb 100644 --- a/src/engine/monitoring/templateDirWatcher.ts +++ b/src/engine/monitoring/templateDirWatcher.ts @@ -12,7 +12,7 @@ export type TemplateDirWatcher = { * Starts watching the template directories for changes. * When a change is detected, the template hash is updated and cached. */ - watchTemplateDirChanges(logger: winston.Logger): Promise; + watchTemplateDirChanges(logger: winston.Logger): void; /** * Gets the cached hash of a template directory. @@ -28,13 +28,13 @@ const hashCache: Map = new Map(); const watchers: Map = new Map(); const hashingInProgress: Set = new Set(); -export const watchTemplateDirChanges = async (logger: winston.Logger) => { +export const watchTemplateDirChanges = (logger: winston.Logger) => { const templates = getAllTemplates(); // Populate on startup - await Promise.all(templates.map(template => watchTemplateDir(template.id))); + templates.forEach(template => watchTemplateDir(template.id)); // Watch the base directory for new templates - await watchBaseDir(logger); + watchBaseDir(logger); logger.info("Watching template directories for changes..."); } @@ -45,7 +45,7 @@ export const watchTemplateDirChanges = async (logger: winston.Logger) => { * When a new template directory is added, it starts watching that directory for changes. * When a template directory is removed, it stops watching that directory and removes its hash from the cache. */ -const watchBaseDir = async (logger: winston.Logger) => { +const watchBaseDir = (logger: winston.Logger) => { const dir = baseTemplatesDir(); const watcher = chokidar.watch(dir, { From f3d83fb741c2264dd5c9c05bdc59e02b980d1b74 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 16 Mar 2026 13:15:02 +0100 Subject: [PATCH 10/52] refactor: remove comment --- src/database/manager.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/database/manager.ts b/src/database/manager.ts index 09b48ac..cd3baf1 100644 --- a/src/database/manager.ts +++ b/src/database/manager.ts @@ -8,8 +8,6 @@ export const initClientForTest = (client_: PrismaClient) => { client = client_; } -// Database manager implementation - export async function saveSession({ serviceId, nodeId, containerId }: SessionModel): Promise { try { await client.session.upsert({ From a7f7eb1a74c4e07fc1350fd89603b530192d016e Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 16 Mar 2026 13:30:07 +0100 Subject: [PATCH 11/52] refactor: TODO.txt --- TODO.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/TODO.txt b/TODO.txt index c26e260..b79e1fc 100644 --- a/TODO.txt +++ b/TODO.txt @@ -5,4 +5,5 @@ odebrat nodeId, cluster support se bude řešit jinak abstrahovat template management a udělat další sources jako například používání images z docker registry předělat usages monitoring na nějaký standardizovaný formát/způsob pořešit /resources/config.yml aby měl víc možností a možná byl uložený jinde v systému podle platformy +port retrieval strategy pro spouštění servis, aby nebyla jenom random ale i jiné strategies udělat nějaký auto packaging do balíčků na githubu jako bundle aby se dal spustit jako systémový proces v systemctl například \ No newline at end of file From 86aa27c67cb4b5b5ad44c8bc34593db63b654154 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 16 Mar 2026 13:54:10 +0100 Subject: [PATCH 12/52] refactor --- src/app.ts | 5 ++--- src/engine/index.ts | 6 +----- src/engine/manager.ts | 35 ++++++++++++++++++++++------------- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/app.ts b/src/app.ts index 4a73ac6..86d3a05 100644 --- a/src/app.ts +++ b/src/app.ts @@ -13,10 +13,9 @@ import {ServiceManager} from "@nsm/engine"; import loadAddons from "./addon"; import loadAppRoutes from '@nsm/router'; import createDbManager from '@nsm/database'; -import initServiceManager from '@nsm/engine'; import loadSecurity from "@nsm/security"; import * as r from "@nsm/configuration/resources"; -import * as manager from "@nsm/engine"; +import * as manager from "@nsm/engine/manager"; import * as logging from "./logger"; import winston from "winston"; import {Application} from "express-ws"; @@ -114,7 +113,7 @@ export const init = async (router: Application, options?: AppBootOptions): Promi // Service (virtualization) layer steps('BEFORE_ENGINE', ctx); - await initServiceManager({ db: database, appConfig, logger }); + await manager.init(database, appConfig, logger); // Bring back original manager ctx.manager = currentContext.manager = middleLayer(manager); diff --git a/src/engine/index.ts b/src/engine/index.ts index 0c36c88..abdc154 100644 --- a/src/engine/index.ts +++ b/src/engine/index.ts @@ -1,6 +1,2 @@ -import manager from "./manager"; - export * from "./manager"; -export * from "./engine"; - -export default manager; \ No newline at end of file +export * from "./engine"; \ No newline at end of file diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 2c224c7..89ffe10 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -161,6 +161,15 @@ export type ServiceManager = ServiceManagerEventBus & { */ engine: ServiceEngineI; + /** + * Initialize the service manager. + * + * @param db The database + * @param appConfig The app config + * @param logger The global logger + */ + init(db: Database, appConfig: any, logger: winston.Logger): Promise; + /** * Create a new service. * @@ -352,7 +361,11 @@ const evtHandlers: Map[]> = new Map(); }; }); -async function init(db_: Database, appConfig_: any) { +export async function init(db_: Database, appConfig_: any, logger: winston.Logger) { + const nodeId_ = appConfig['node_id'] as string; + + logger.info(`Initializing service manager for node ${nodeId_}...`); + db = db_; appConfig = appConfig_; if (!engine) { @@ -363,6 +376,11 @@ async function init(db_: Database, appConfig_: any) { initImageEngine(engine, templateManager, templateDirWatcher, db_, currentContext.logger); watchTemplateDirChanges(currentContext.logger); + + await clearSessions(db, nodeId_, logger); + await stopRunningServices(); // TODO: is this necessary? + + logger.info(`Using ${engine.defaultEngine ? 'default' : 'custom'} engine`); } export async function expandEngine(exp?: T): Promise { @@ -864,17 +882,8 @@ async function getServiceSession(id: string) { return session; } -export default async function ({db, appConfig, logger}: { - db: Database, - appConfig: any, - logger: winston.Logger -}) { - const nodeId = appConfig['node_id'] as string; - - logger.info(`Initializing service manager for node ${nodeId}...`); - +async function clearSessions(db: Database, nodeId: string, logger: winston.Logger) { const unclearedSessions = await db.listSessions(nodeId); - await init(db, appConfig); if (unclearedSessions.length > 0) { logger.info('There are ' + unclearedSessions.length + ' uncleared sessions, trying to reattach to them...'); } @@ -890,7 +899,9 @@ export default async function ({db, appConfig, logger}: { } await new Promise((resolve) => whenUnlockedAll(() => resolve(null))); +} +async function stopRunningServices() { const running = await engine.listRunning(); for (const id of running) { const volumeId = await engine.getAttachedVolume(id); @@ -900,6 +911,4 @@ export default async function ({db, appConfig, logger}: { } await engine.stop(id, metaStorageForService(volumeId)); } - - logger.info(`Using ${engine.defaultEngine ? 'default' : 'custom'} engine`); } \ No newline at end of file From b8f6e1f1dfb0241edd07b3dec5c2f9d27232aa75 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 16 Mar 2026 13:59:38 +0100 Subject: [PATCH 13/52] jsdoc --- src/engine/middle.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/engine/middle.ts b/src/engine/middle.ts index f697bfe..d5cf6a4 100644 --- a/src/engine/middle.ts +++ b/src/engine/middle.ts @@ -33,6 +33,8 @@ const publishers: ErrorPublisher[] = [ /** * Registers a new error publisher to publish notifications of service action errors. + * This is called when fails an ServiceManager call that manipulates with service lifecycle state, + * so registering this is quite useful for handling service action request errors. * * @param publisher The error publisher to register. */ From d4f475ff1be3b998538936c5a45fce2d2ae117ce Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 16 Mar 2026 14:04:04 +0100 Subject: [PATCH 14/52] jsdoc --- src/engine/engine.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/engine/engine.ts b/src/engine/engine.ts index d35852f..59896e8 100644 --- a/src/engine/engine.ts +++ b/src/engine/engine.ts @@ -3,6 +3,9 @@ import buildDockerEngine from "./docker"; import {getSingleton} from "../depend"; import {MetaStorage} from "./manager"; +/** + * The options for running a service. + */ export type RunOptions = { port: number; ports: number[]; @@ -18,6 +21,9 @@ export type RunOptions = { }; } +/** + * The stats of a container, used for monitoring. + */ export type ContainerStat = { id: string, memory: { From 89ef6e8b4b82c46f5d1a6d4e197fd84bcd024f79 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 16 Mar 2026 15:05:33 +0100 Subject: [PATCH 15/52] refactor --- src/engine/manager.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 89ffe10..e828b70 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -439,9 +439,19 @@ export async function createService(template: string, options: Options) { portRange.max as number ); + const perma: PermaModel = { + serviceId, + template, + nodeId, + port, + options: {ram, cpu, disk, ports}, + meta, + env: env ?? {}, + network + }; let err: any; // Save permanent info - if (!await db.savePerma({ serviceId, template, nodeId, port, options: {ram, cpu, disk, ports}, meta, env: env ?? {}, network })) { + if (!await db.savePerma(perma)) { err = new _InternalError('Failed to save perma info to database'); } From 32791882fbedbe68ded8ae98097e4fa708529bef Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 16 Mar 2026 15:28:45 +0100 Subject: [PATCH 16/52] refactor: remove no-template mode --- README.md | 1 - TODO.txt | 1 - src/engine/docker/action/build.ts | 4 -- src/engine/docker/index.ts | 1 - src/engine/engine.ts | 10 +--- src/engine/manager.ts | 87 ++-------------------------- src/router/v1/service/createRoute.ts | 4 +- 7 files changed, 8 insertions(+), 100 deletions(-) diff --git a/README.md b/README.md index b4804a4..1b2073f 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,6 @@ NSM is a robust service manager built on Docker Engine. Its primary purpose is t - **Dynamic Service Generation**: Create and manage services on-the-fly using RESTful APIs. - **Template-Based Configuration**: Use customizable templates to define service configurations. -- **No-template mode**: NSM supports integrating custom engine with no template mode to disable templates completely. - **Docker Integration**: Leverage Docker Engine for reliable and scalable service management. - **Resources usage management**: NSM provides ability to limit or extend resources limits and view current usage. diff --git a/TODO.txt b/TODO.txt index b79e1fc..73b7e9a 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,6 +1,5 @@ vyřešit logging u servis a logování změn stavů Odebrat ukládání běžících sessions do databáze, držet jen v runtimu -odebrat no template mode odebrat nodeId, cluster support se bude řešit jinak abstrahovat template management a udělat další sources jako například používání images z docker registry předělat usages monitoring na nějaký standardizovaný formát/způsob diff --git a/src/engine/docker/action/build.ts b/src/engine/docker/action/build.ts index 06ba8ea..267cfed 100644 --- a/src/engine/docker/action/build.ts +++ b/src/engine/docker/action/build.ts @@ -125,10 +125,6 @@ export default function (client: DockerClient): ServiceEngine['build'] { } return async (imageId, buildDir, options) => { - if (!buildDir) { - throw new Error('Docker engine does not support no-template mode!'); - } - const imageBuildClock = clock(); const imageTag = await prepareImage({imageName: imageId, client, arDir, buildDir, env: options}); currentContext.logger.info('Image built in ' + imageBuildClock.durFromCreation() + 'ms'); diff --git a/src/engine/docker/index.ts b/src/engine/docker/index.ts index 8498a6e..d90984b 100644 --- a/src/engine/docker/index.ts +++ b/src/engine/docker/index.ts @@ -22,7 +22,6 @@ export default function buildDockerEngine(appConfig: any) { const client = initDockerClient(appConfig); const engine = {} as DockerServiceEngine; engine.dockerClient = client; - engine.supportsNoTemplateMode = false; engine.rws = {}; // engine.cast - Being replaced in manager. engine.build = build(client); diff --git a/src/engine/engine.ts b/src/engine/engine.ts index 59896e8..ae10fab 100644 --- a/src/engine/engine.ts +++ b/src/engine/engine.ts @@ -12,7 +12,7 @@ export type RunOptions = { ram: number; // in MB cpu: number; // in cores disk: number; - env: {[key: string]: string}; + env: { [key: string]: string }; network?: { address: string, // If only ports should be exposed to this @@ -81,12 +81,6 @@ export type ServiceEngineI = ServiceEngine & { // Internal * containers themselves. */ export type ServiceEngine = { - /** - * If this engine supports no-t mode from engine/manager. - * If enabled, buildDir from build() method can be undefined. - */ - supportsNoTemplateMode: boolean; - /** * Builds an image from build dir. * @@ -96,7 +90,7 @@ export type ServiceEngine = { */ build( imageId: string|undefined, - buildDir: string|undefined, buildOptions: { [key: string]: string }): Promise; + buildDir: string, buildOptions: { [key: string]: string }): Promise; run( templateId: string, diff --git a/src/engine/manager.ts b/src/engine/manager.ts index e828b70..9ef2003 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -92,23 +92,6 @@ export type MetaStorage = { get: (key: string, def?: T) => Promise; } -export type NoTAlternateSettings = { - port_range: { - min: number, - max: number, - }, - defaults: { - ram: number, - cpu: number, - disk: number, - }, - meta: { - stopCmd: string, - }, - env: { - }, -} - export type EngineExpansion = { [k in keyof ServiceEngineI | string]: any; }; @@ -274,26 +257,6 @@ export type ServiceManager = ServiceManagerEventBus & { */ stopRunning(): Promise; - /** - * Enable no-template mode. - * - * When this is enabled, all services are being treated as same template, and it's - * up to the internal engine to handle creating them without templates. Services - * created using template mode can't be manipulated in this mode. Also, internal - * engine must support this mode. - * - * @param alternateSettings The global template default settings, replacement for - * settings.yml in template mode. This will be used as - * the settings for the global template used for every - * service created in this mode. - */ - enableNoTemplateMode(alternateSettings: NoTAlternateSettings): Promise; - - /** - * Whether the no-template mode is enabled. - */ - noTemplateMode(): boolean; - isRunning(id: string): boolean; waitForBusyAction(id: string): Promise; @@ -333,7 +296,6 @@ export let nodeId: string; let db: Database; let appConfig: any; -let noTAlternateSett: NoTAlternateSettings|undefined = undefined; // Returns the settings.yml file for the template function settings(template: string) { @@ -345,7 +307,6 @@ function settings(template: string) { const errors = {}; // Service IDs that are currently running const started = []; -const noTTemplate = '__no_t__'; const evtHandlers: Map[]> = new Map(); ["push", "splice"].forEach(funcName => { @@ -406,9 +367,6 @@ export async function expandEngine(exp?: T): Promise< } export async function createService(template: string, options: Options) { - reqCompatibleEngine(); - template = noTAlternateSett ? noTTemplate : template; - const { ram, cpu, @@ -418,7 +376,7 @@ export async function createService(template: string, options: Options) { network } = options; - const serviceSettings = noTAlternateSett ? {...noTAlternateSett} : settings(template); + const serviceSettings = settings(template); // Join meta supplied by user and template meta const meta = { @@ -469,7 +427,6 @@ export async function createService(template: string, options: Options) { } export async function resumeService(id: string) { - reqCompatibleEngine(); // Service is already running if (await db.getSession(id)) { throw new _InternalError('Already running.', 2); @@ -481,20 +438,9 @@ export async function resumeService(id: string) { env, network, port, - nodeId: permaNodeId, } = await getPermaModel(id); - if (noTAlternateSett && template !== noTTemplate) { - throw new Error('Tried to resume template-based service from within no-template mode.'); - } else if (template === noTTemplate && !noTAlternateSett) { - throw new Error('Tried to resume no-t service from within template mode.'); - } - - if (noTAlternateSett && permaNodeId !== nodeId) { - throw new Error('In no-template mode, only services that came from this node can be resumed here.'); - } - - const {defaults, env: settingsEnv} = noTAlternateSett ? {...noTAlternateSett} : settings(template); + const {defaults, env: settingsEnv} = settings(template); // Filter env to only those that are defined in settings.yml, because those are the only ones that // we can guarantee to be used and will not make problems when handling images. env = { @@ -700,16 +646,7 @@ export async function updateOptions(id: string, options: Options) { } export function getTemplate(id: string) { - if (noTemplateMode()) { - return { - id: noTTemplate, - name: "Built-in", - description: "A no-template template.", - settings: noTAlternateSett, - } - } else { - return loadTemplate(id); - } + return loadTemplate(id); } export async function getService(from: string, options?: { includeSession?: boolean, otherNodes?: boolean }) { @@ -760,11 +697,6 @@ export async function stopRunning() { ))); } -export async function enableNoTemplateMode(alternateSettings: NoTAlternateSettings) { - noTAlternateSett = alternateSettings; - currentContext.logger.info("No-template mode has been enabled."); -} - export async function waitForBusyAction(id: string) { return new Promise((resolve, reject) => { whenUnlocked(id, (_, status, err) => { @@ -792,10 +724,6 @@ function metaStorageForService(id: string): MetaStorage { // service id }; } -export function noTemplateMode() { - return noTAlternateSett !== undefined; -} - export function initialized() { return engine !== undefined; } @@ -828,12 +756,6 @@ export { whenUnlocked } -function reqCompatibleEngine() { - if (noTAlternateSett && !engine.supportsNoTemplateMode) { - throw new Error('No-template mode is enabled, but current engine does not support it! Please switch to different engine.'); - } -} - function callManagerEvent(e: T, event: ServiceManagerEvents[T]) { if (!evtHandlers.has(e)) { return; @@ -900,8 +822,7 @@ async function clearSessions(db: Database, nodeId: string, logger: winston.Logge for (const session of unclearedSessions) { try { // Reattach to the container - await engine.reattach( - session.containerId, buildRunListener(session.serviceId)); + await engine.reattach(session.containerId, buildRunListener(session.serviceId)); } catch (e) { // Delete session if failed to reattach, probably the container is not running anymore await db.deleteSession(session.serviceId); diff --git a/src/router/v1/service/createRoute.ts b/src/router/v1/service/createRoute.ts index 4203ce4..2ad6b33 100644 --- a/src/router/v1/service/createRoute.ts +++ b/src/router/v1/service/createRoute.ts @@ -5,13 +5,13 @@ import {Options} from "@nsm/engine"; import {clock} from "@nsm/util/clock"; import {prepareEnvForTemplate} from "@nsm/engine/template"; -export default async function ({manager, logger}: AppContext): Promise { +export default async function ({manager}: AppContext): Promise { return { url: '/service/create', routes: { post: async (req, res) => { const clk = clock(); - if (!req.body || (!manager.noTemplateMode() && !req.body.template)) { + if (!req.body || !req.body.template) { res.status(400).json({status: 400, message: 'Missing body or template key.'}).end(); return; } From d35b8b51a39d8ecd81eede90acbfd57b1ecc2cbe Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 16 Mar 2026 20:47:27 +0100 Subject: [PATCH 17/52] fix: bad reference --- src/engine/manager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 9ef2003..798077a 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -323,7 +323,7 @@ const evtHandlers: Map[]> = new Map(); }); export async function init(db_: Database, appConfig_: any, logger: winston.Logger) { - const nodeId_ = appConfig['node_id'] as string; + const nodeId_ = appConfig_['node_id'] as string; logger.info(`Initializing service manager for node ${nodeId_}...`); From d1ee07e25b8e8b53ac48ffaf46874f08bf0a9270 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Tue, 17 Mar 2026 00:25:32 +0100 Subject: [PATCH 18/52] feat: remove session info from db --- TODO.txt | 2 +- openapi.yml | 4 - .../migration.sql | 8 ++ prisma/schema.prisma | 7 - src/database/manager.ts | 38 +---- src/database/models.ts | 9 -- src/engine/manager.ts | 132 ++++++++++-------- 7 files changed, 83 insertions(+), 117 deletions(-) create mode 100644 prisma/migrations/20260316232423_remove_session/migration.sql diff --git a/TODO.txt b/TODO.txt index 73b7e9a..b57c669 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,5 +1,5 @@ -vyřešit logging u servis a logování změn stavů Odebrat ukládání běžících sessions do databáze, držet jen v runtimu +vyřešit logging u servis a logování změn stavů odebrat nodeId, cluster support se bude řešit jinak abstrahovat template management a udělat další sources jako například používání images z docker registry předělat usages monitoring na nějaký standardizovaný formát/způsob diff --git a/openapi.yml b/openapi.yml index 12efadc..e80dbc5 100644 --- a/openapi.yml +++ b/openapi.yml @@ -86,10 +86,6 @@ components: type: "object" description: "The session information about a running service, if running" properties: - serviceId: - type: "string" - nodeId: - type: "string" containerId: type: "string" stats: diff --git a/prisma/migrations/20260316232423_remove_session/migration.sql b/prisma/migrations/20260316232423_remove_session/migration.sql new file mode 100644 index 0000000..c6dfddf --- /dev/null +++ b/prisma/migrations/20260316232423_remove_session/migration.sql @@ -0,0 +1,8 @@ +/* + Warnings: + + - You are about to drop the `Session` table. If the table is not empty, all the data it contains will be lost. + +*/ +-- DropTable +DROP TABLE `Session`; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 910fc1b..3bf2757 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -14,13 +14,6 @@ datasource db { shadowDatabaseUrl = env("SHADOW_DATABASE_URL") } -model Session { - serviceId String @id @default(uuid()) - createdAt DateTime @default(now()) - nodeId String - containerId String -} - model Service { serviceId String @id @default(uuid()) nodeId String diff --git a/src/database/manager.ts b/src/database/manager.ts index cd3baf1..b360e1a 100644 --- a/src/database/manager.ts +++ b/src/database/manager.ts @@ -1,5 +1,5 @@ import {PrismaClient} from "@prisma/client"; -import {ImageModel, PermaModel, SessionModel} from "./models"; +import {ImageModel, PermaModel} from "./models"; import {optionsDiffer} from "@nsm/engine/image"; let client = new PrismaClient(); @@ -8,20 +8,6 @@ export const initClientForTest = (client_: PrismaClient) => { client = client_; } -export async function saveSession({ serviceId, nodeId, containerId }: SessionModel): Promise { - try { - await client.session.upsert({ - where: { serviceId }, - update: { nodeId, containerId }, - create: { serviceId, nodeId, containerId } - }); - return true; - } catch (e) { - console.log(e); - return false; - } -} - export async function savePerma(data: PermaModel): Promise { const { serviceId } = data; try { @@ -73,19 +59,6 @@ export async function deletePerma(serviceId: string): Promise { } } -export async function getSession(serviceId: string): Promise { - try { - const session = await client.session.findUnique({ where: { serviceId } }); - if (!session) { - return undefined; - } - return session; - } catch (e) { - console.log(e); - return undefined; - } -} - export async function getPerma(serviceId: string): Promise { try { const service = await client.service.findUnique({ where: { serviceId } }); @@ -162,15 +135,6 @@ export async function listAllUsingImage(imageId: string): Promise } } -export async function listSessions(nodeId: string): Promise { - try { - return await client.session.findMany({ where: { nodeId } }); - } catch (e) { - console.log(e); - return []; - } -} - export async function count(nodeId: string): Promise { try { return await client.service.count({ where: { nodeId } }); diff --git a/src/database/models.ts b/src/database/models.ts index e8631eb..c441c3e 100644 --- a/src/database/models.ts +++ b/src/database/models.ts @@ -1,15 +1,12 @@ export type Database = { - saveSession(info: SessionModel): Promise; savePerma(info: PermaModel): Promise; deleteSession(serviceId: string): Promise; deleteSessions(nodeId: string): Promise; deletePerma(serviceId: string): Promise; - getSession(serviceId: string): Promise; getPerma(serviceId: string): Promise; getMetaVal(key: string, defaultVal?: string): Promise; list(nodeId: string, page?: number, pageSize?: number, meta?: {[key: string]: any}): Promise; listAllUsingImage(imageId: string): Promise; - listSessions(nodeId: string): Promise; count(nodeId: string): Promise; setServiceMeta(serviceId: string, key: string, value: any): Promise; getServiceMeta(serviceId: string, key: string): Promise; @@ -19,12 +16,6 @@ export type Database = { listImagesByOptions(templateId: string, buildOptions: {[key: string]: string}): Promise; }; -export type SessionModel = { - serviceId: string, - nodeId: string, - containerId: string -}; - export type PermaModel = { serviceId: string, template: string, diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 798077a..331edbf 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -6,7 +6,7 @@ import * as templateDirWatcher from "./monitoring/templateDirWatcher"; import crypto from "crypto"; import {randomPort as retrieveRandomPort} from "@nsm/util/port"; import {loadYamlFile} from "@nsm/util/yaml"; -import {PermaModel, SessionModel} from "../database"; +import {PermaModel} from "../database"; import { isServicePending, lckStatusTp, @@ -270,11 +270,21 @@ export type ServiceManager = ServiceManagerEventBus & { whenUnlocked: typeof whenUnlocked }; +type RunningService = { + id: string; + session: Session; +} + +export type Session = { + containerId: string; + // TODO: add more useful information? +} + export type ServiceInfo = PermaModel & { optionsRam: number, // From options.ram optionsCpu: number, // From options.cpu optionsDisk: number, // From options.disk - session?: SessionModel + session?: Session } // 1 = unknown, 2 = conflict, 3 = not found @@ -306,7 +316,7 @@ function settings(template: string) { // Could it be a memory leak if there are tons of them?? const errors = {}; // Service IDs that are currently running -const started = []; +const started: RunningService[] = []; const evtHandlers: Map[]> = new Map(); ["push", "splice"].forEach(funcName => { @@ -338,7 +348,7 @@ export async function init(db_: Database, appConfig_: any, logger: winston.Logge initImageEngine(engine, templateManager, templateDirWatcher, db_, currentContext.logger); watchTemplateDirChanges(currentContext.logger); - await clearSessions(db, nodeId_, logger); + await reattachStaleContainers(logger); await stopRunningServices(); // TODO: is this necessary? logger.info(`Using ${engine.defaultEngine ? 'default' : 'custom'} engine`); @@ -427,10 +437,7 @@ export async function createService(template: string, options: Options) { } export async function resumeService(id: string) { - // Service is already running - if (await db.getSession(id)) { - throw new _InternalError('Already running.', 2); - } + reqNotRunning(id); let { template, @@ -504,15 +511,14 @@ export async function resumeService(id: string) { let success: boolean = false; if (containerId) { - const saved = await db.saveSession({ serviceId: id, nodeId, containerId }); - // Save new session info - if (saved) { - started.push(id); - success = true; - } else { - // Cleanup if err with db - await engine.stop(containerId, meta); - } + const runningService: RunningService = { + id, + session: { + containerId + } + }; + started.push(runningService); + success = true; } if (success == true) { @@ -520,7 +526,7 @@ export async function resumeService(id: string) { callManagerEvent('resume', { id }); } else { errors[id] = new Error('Failed to resume service'); - started.splice(started.indexOf(id), 1); + clearRunningServiceIfExists(id); callManagerEvent('resume', { id, error: errors[id] }); } @@ -530,14 +536,9 @@ export async function resumeService(id: string) { } export async function stopService(id: string, force?: boolean) { - if (!await db.getPerma(id)) { - throw new _InternalError("Service not found.", 3); - } + await reqExists(id); - const session = await db.getSession(id); - if (!session) { - throw new _InternalError("This service is not running.", 2); - } + const { session } = reqRunning(id); lckStatusTp(session.containerId, 'stop'); const unlock = lockBusyAction(id, 'stop'); @@ -574,15 +575,15 @@ export async function stopServiceForcibly(id: string) { } export async function sendStopSignal(id: string) { - const perma_ = await getPermaModel(id); - const session = await getServiceSession(id); - const stopCmd = perma_.meta?.stopCmd; + const perma = await reqExists(id); + const { session } = reqRunning(id); + + const stopCmd = perma.meta?.stopCmd; if (!stopCmd) { throw new _InternalError('Service does not have stop command set.'); } await engine.cmd(session.containerId, stopCmd); - return true; } @@ -654,7 +655,7 @@ export async function getService(from: string, options?: { includeSession?: bool if (data && (data.nodeId == nodeId || options?.otherNodes === true)) { let session = undefined; if (options?.includeSession === true) { - session = await currentContext.database.getSession(data.serviceId); + session = getRunningService(data.serviceId); } return { ...data, @@ -684,8 +685,8 @@ export async function listTemplates(): Promise { } export async function stopRunning() { - await Promise.all(started.map(id => ( - new Promise((resolve, reject) => { + await Promise.all(started.map(({id}) => ( + new Promise((resolve) => { whenUnlocked(id, () => { stopService(id) .catch(e => console.log(e)) @@ -710,7 +711,11 @@ export async function waitForBusyAction(id: string) { } export function isRunning(id: string) { - return started.includes(id); + return getRunningService(id) != undefined; +} + +function getRunningService(id: string) { + return started.find(service => service.id === id); } function metaStorageForService(id: string): MetaStorage { // service id @@ -756,6 +761,37 @@ export { whenUnlocked } +async function reqExists(id: string) { + const perma = await db.getPerma(id); + if (!perma) { + throw new _InternalError("Service not found.", 3); + } + + return perma; +} + +function reqRunning(id: string) { + const session = getRunningService(id); + if (!session) { + throw new _InternalError("This service is not running.", 2); + } + + return session; +} + +function reqNotRunning(id: string) { + if (isRunning(id)) { + throw new _InternalError('Already running.', 2); + } +} + +function clearRunningServiceIfExists(id: string) { + const service = getRunningService(id); + if (service) { + started.splice(started.indexOf(service, 1)); + } +} + function callManagerEvent(e: T, event: ServiceManagerEvents[T]) { if (!evtHandlers.has(e)) { return; @@ -782,9 +818,7 @@ function buildRunListener(serviceId: string): RunListener { // Remove session when container is closed, because the service is not running anymore await db.deleteSession(serviceId); - if (started.includes(serviceId)) { - started.splice(started.indexOf(serviceId, 1)); - } + clearRunningServiceIfExists(serviceId); // Call stop event on the manager for the stopService() to potentially // unlock a busy action @@ -805,29 +839,9 @@ async function getPermaModel(id: string) { return perma_; } -async function getServiceSession(id: string) { - const session = await db.getSession(id); - if (!session) { - throw new _InternalError("This service is not running.", 2); - } - - return session; -} - -async function clearSessions(db: Database, nodeId: string, logger: winston.Logger) { - const unclearedSessions = await db.listSessions(nodeId); - if (unclearedSessions.length > 0) { - logger.info('There are ' + unclearedSessions.length + ' uncleared sessions, trying to reattach to them...'); - } - for (const session of unclearedSessions) { - try { - // Reattach to the container - await engine.reattach(session.containerId, buildRunListener(session.serviceId)); - } catch (e) { - // Delete session if failed to reattach, probably the container is not running anymore - await db.deleteSession(session.serviceId); - } - } +async function reattachStaleContainers(logger: winston.Logger) { + // TODO: find containers marked and find serviceId marked on them + // TODO: engine.reattach(containerId, buildRunListener(serviceId)); await new Promise((resolve) => whenUnlockedAll(() => resolve(null))); } From 39477f21e6fe33354258fbeee7ea39bd0e28a933 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Wed, 18 Mar 2026 01:18:44 +0100 Subject: [PATCH 19/52] feat: move some of the label responsibility to service manager from engine --- TODO.txt | 1 - src/engine/docker/action/getAttachedVolume.ts | 13 ---- src/engine/docker/action/listRunning.ts | 8 +- src/engine/docker/action/listc.ts | 20 +++-- src/engine/docker/action/run.ts | 14 +--- src/engine/docker/action/statall.ts | 15 ++-- src/engine/docker/index.ts | 4 +- src/engine/docker/util/labels.ts | 16 ++++ src/engine/engine.ts | 77 ++++++++++++++++--- src/engine/manager.ts | 73 +++++++++++++----- src/router/v1/service/lookupRoute.ts | 8 +- src/router/v1/status/index.ts | 13 ++-- 12 files changed, 168 insertions(+), 94 deletions(-) delete mode 100644 src/engine/docker/action/getAttachedVolume.ts create mode 100644 src/engine/docker/util/labels.ts diff --git a/TODO.txt b/TODO.txt index b57c669..e4ce032 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,4 +1,3 @@ -Odebrat ukládání běžících sessions do databáze, držet jen v runtimu vyřešit logging u servis a logování změn stavů odebrat nodeId, cluster support se bude řešit jinak abstrahovat template management a udělat další sources jako například používání images z docker registry diff --git a/src/engine/docker/action/getAttachedVolume.ts b/src/engine/docker/action/getAttachedVolume.ts deleted file mode 100644 index 9b9b460..0000000 --- a/src/engine/docker/action/getAttachedVolume.ts +++ /dev/null @@ -1,13 +0,0 @@ -import DockerClient from "dockerode"; - -export default function getAttachedVolume(client: DockerClient) { - return async (id: string) => { - const c = client.getContainer(id); - try { - const i = await c.inspect(); - return i.Config.Labels['nsm.volumeId']; - } catch (e) { - return undefined; - } - } -} \ No newline at end of file diff --git a/src/engine/docker/action/listRunning.ts b/src/engine/docker/action/listRunning.ts index 2d0fbd6..d91fff3 100644 --- a/src/engine/docker/action/listRunning.ts +++ b/src/engine/docker/action/listRunning.ts @@ -1,11 +1,13 @@ import DockerClient from "dockerode"; +import {ContainerFilter} from "@nsm/engine"; +import {toDockerFilters} from "@nsm/engine/docker/util/labels"; export default function listRunningFunc(client: DockerClient) { - return async () => { + return async (filter: ContainerFilter) => { const list = await client.listContainers({ all: true, - filters: JSON.stringify({ 'label': ['nsm=true'] }) } - ); + filters: toDockerFilters(filter) + }); return list .filter(c => c.State === 'running') .map(c => c.Id); diff --git a/src/engine/docker/action/listc.ts b/src/engine/docker/action/listc.ts index 8149cd3..00b7831 100644 --- a/src/engine/docker/action/listc.ts +++ b/src/engine/docker/action/listc.ts @@ -1,18 +1,16 @@ import DockerClient from "dockerode"; -import {ServiceEngine} from "../../engine"; -import {currentContext} from "../../../app"; +import {ServiceEngine} from "@nsm/engine"; +import {toDockerFilters} from "@nsm/engine/docker/util/labels"; export default function (self: ServiceEngine, client: DockerClient): ServiceEngine['listContainers'] { - return async (templates) => { - if (templates === undefined) { - templates = await currentContext.manager.listTemplates(); - } + return async (filter) => { try { - return (await client.listContainers()) - .filter(c => templates.some(function (t) { - return c.Labels.hasOwnProperty('nsm.templateId') && c.Labels['nsm.templateId'] == t; - })) - .map(c => c.Id); + const containers = await client.listContainers({ + all: true, + filters: toDockerFilters(filter) + }); + + return containers.map(c => c.Id); } catch (e) { console.log(e); return []; diff --git a/src/engine/docker/action/run.ts b/src/engine/docker/action/run.ts index f62452d..183d16d 100644 --- a/src/engine/docker/action/run.ts +++ b/src/engine/docker/action/run.ts @@ -2,8 +2,6 @@ import DockerClient from "dockerode"; import {RunOptions, MetaStorage, ServiceEngine} from "@nsm/engine"; import {accessNetwork, createNetwork} from "@nsm/networking/manager"; import {constructObjectLabels} from "@nsm/util/services"; -import path from "path"; -import {buildDir} from "@nsm/engine/monitoring/util"; import {currentContext as ctx} from "@nsm/app"; import {propagateOptionsToEnv} from "@nsm/engine/docker/util/env"; @@ -53,7 +51,6 @@ async function prepareNetwork( async function prepareContainer( client: DockerClient, imageTag: string, - buildDir: string, volumeId: string, options: RunOptions, net: DockerClient.Network|undefined @@ -66,12 +63,7 @@ async function prepareContainer( // Create container const container = await client.createContainer({ Image: imageTag, - Labels: { - ...constructObjectLabels({ id: volumeId }), - 'nsm.buildDir': buildDir, - 'nsm.volumeId': volumeId, - 'nsm.templateId': buildDir ? path.basename(buildDir) : '__no_t__' - }, + Labels: options.labels, HostConfig: { Memory: ram, CpuShares: cpu, @@ -98,7 +90,7 @@ async function prepareContainer( } export default function run(self: ServiceEngine, client: DockerClient): ServiceEngine["run"] { - return async (templateId, imageId, volumeId, options, meta, listener) => { + return async (imageId, volumeId, options, meta, listener) => { let container: DockerClient.Container; // Prepare volume let creating = await prepareVolume(client, volumeId); @@ -110,7 +102,7 @@ export default function run(self: ServiceEngine, client: DockerClient): ServiceE const net = await prepareNetwork(client, options.network, meta, creating); // Port decorator that takes port and according to network changes it to : or keeps the same. await listener.onStateMessage('Preparing container'); - container = await prepareContainer(client, imageId, buildDir(templateId), volumeId, options, net); + container = await prepareContainer(client, imageId, volumeId, options, net); await listener.onStateMessage('Starting container'); await container.start(); diff --git a/src/engine/docker/action/statall.ts b/src/engine/docker/action/statall.ts index 6f22acb..c8320a2 100644 --- a/src/engine/docker/action/statall.ts +++ b/src/engine/docker/action/statall.ts @@ -1,12 +1,9 @@ -import {ServiceEngine} from "../../engine"; -import DockerClient from "dockerode"; +import {ServiceEngine} from "@nsm/engine"; -export default function (self: ServiceEngine, client: DockerClient): ServiceEngine['statAll'] { - return async () => { - const list = []; - for (const c of await self.listContainers()) { - list.push(await self.stat(c)); - } - return list; +export default function (self: ServiceEngine): ServiceEngine['statAll'] { + return async (filter) => { + const containers = await self.listContainers(filter); + + return Promise.all(containers.map(c => self.stat(c))); } } \ No newline at end of file diff --git a/src/engine/docker/index.ts b/src/engine/docker/index.ts index d90984b..d91ce13 100644 --- a/src/engine/docker/index.ts +++ b/src/engine/docker/index.ts @@ -9,7 +9,6 @@ import reattach from "./action/reattach"; import delVolume from './action/deletev'; import delImage from './action/deletei'; import cmd from './action/cmd'; -import getAttachedVolume from "./action/getAttachedVolume"; import listContainers from './action/listc'; import listAttachedPorts from './action/listp'; import stat from "./action/stat"; @@ -32,11 +31,10 @@ export default function buildDockerEngine(appConfig: any) { engine.deleteVolume = delVolume(engine, client); engine.deleteImage = delImage(client); engine.cmd = cmd(engine, client); - engine.getAttachedVolume = getAttachedVolume(client); engine.listContainers = listContainers(engine, client); engine.listAttachedPorts = listAttachedPorts(engine, client); engine.stat = stat(engine, client); - engine.statAll = statAll(engine, client); + engine.statAll = statAll(engine); engine.calcHostUsage = calcHostUsage(client); engine.listRunning = listRunning(client); return engine; diff --git a/src/engine/docker/util/labels.ts b/src/engine/docker/util/labels.ts new file mode 100644 index 0000000..c9e168c --- /dev/null +++ b/src/engine/docker/util/labels.ts @@ -0,0 +1,16 @@ +import {ContainerFilter} from "@nsm/engine"; + +/** + * Convert a ContainerFilter to Docker filters format. + * + * @param filter The ContainerFilter to convert + * @returns The Docker filters JSON + */ +export const toDockerFilters = (filter: ContainerFilter) => { + const dockerFilters: any = {}; + if (filter.labels) { + dockerFilters.label = Object.entries(filter.labels).map(([key, value]) => `${key}=${value}`); + } + + return JSON.stringify(dockerFilters); +} \ No newline at end of file diff --git a/src/engine/engine.ts b/src/engine/engine.ts index ae10fab..116618f 100644 --- a/src/engine/engine.ts +++ b/src/engine/engine.ts @@ -19,6 +19,9 @@ export type RunOptions = { // IP address. portsOnly: boolean, }; + labels?: { + [key: string]: string; + } } /** @@ -38,6 +41,13 @@ export type ContainerStat = { }, } +export type ContainerFilter = { + /** + * Filter containers that have all those labels. + */ + labels?: { [key: string]: string }; +} + export type RunListener = { /** * Called when there is a state change in the container, with the message of the state change. @@ -93,7 +103,6 @@ export type ServiceEngine = { buildDir: string, buildOptions: { [key: string]: string }): Promise; run( - templateId: string, imageId: string, volumeId: string, options: RunOptions, @@ -104,10 +113,9 @@ export type ServiceEngine = { * Stops a container. * * @param id Container ID - * @param meta Meta storage for this unique context * @return Success state */ - stop(id: string, meta: MetaStorage): Promise; + stop(id: string): Promise; /** * Kills a container. @@ -153,40 +161,85 @@ export type ServiceEngine = { cmd(id: string, cmd: string): Promise; /** - * Get ID of volume that this container is attached to, or undefined - * if not found or no volume. Meta is not present here because this func - * is used to determine ID for building the meta. + * Gets the labels of a container. * - * @param id The volume ID. + * @param id Container ID */ - getAttachedVolume(id: string): Promise; + getLabels(id: string): Promise<{ [key: string]: string }>; /** * Lists container ids of containers by templates. * - * @param templates The templates + * @param filter The filter to apply * @return List of container IDs */ - listContainers(templates?: string[]): Promise; + listContainers(filter: ContainerFilter): Promise; /** * List running containers owned by this engine on this machine. * + * @param filter The filter to apply * @return List of container IDs */ - listRunning(): Promise; + listRunning(filter: ContainerFilter): Promise; listAttachedPorts(): Promise; stat(id: string): Promise; - statAll(): Promise; + statAll(filter: ContainerFilter): Promise; // Disk usage of all services here // [0]: free, [1]: size calcHostUsage(): Promise; } +/** + * Standard labels that NSM uses to identify and manage containers. + * Used by the manager to keep consistency across the codebase. + */ +export enum StandardLabel { + // The default label identifying a NSM-managed container. + Nsm = 'nsm', + // The service ID that owns the container. + ServiceId = 'nsm.id', + // The volume ID that the container is using. + VolumeId = 'nsm.volumeId', + // The template ID that the container is created from. + TemplateId = 'nsm.templateId', + // The node ID of the managing worker. + NodeId = 'nsm.nodeId', + // The templare build dir. + BuildDir = 'nsm.buildDir', +} + +export const Filters = { + /** + * The standard filter for NSM-managed containers, which + * filters containers that have the label "nsm" with value "true". + */ + nsm() { + return { + labels: { + [StandardLabel.Nsm]: 'true' + } + } + }, + /** + * The filter for containers belonging to a node with the given node ID. + * + * @param nodeId The node ID + */ + node(nodeId: string) { + return { + labels: { + ...this.nsm().labels, + [StandardLabel.NodeId]: nodeId, + } + } + } +} + export default function (appConfig: any): ServiceEngineI { let engine = getSingleton('engine'); const usingBuiltInEngine = engine == undefined; diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 331edbf..a035aa8 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -1,5 +1,5 @@ import {currentContext, Database} from "../app"; -import createEngine, {RunOptions, RunListener, ServiceEngineI} from "./engine"; +import createEngine, {RunOptions, RunListener, ServiceEngineI, StandardLabel, Filters} from "./engine"; import {Template, getTemplate as loadTemplate, getAllTemplates} from "./template"; import * as templateManager from "./template"; import * as templateDirWatcher from "./monitoring/templateDirWatcher"; @@ -235,7 +235,14 @@ export type ServiceManager = ServiceManagerEventBus & { /** * Get list of running services on this node. */ - getRunningServices(): string[]; + getRunningServices(): RunningService[]; + + /** + * Get the running service by ID. + * + * @param id The service ID + */ + getRunningService(id: string): RunningService|undefined; /** * List all available services. @@ -349,7 +356,6 @@ export async function init(db_: Database, appConfig_: any, logger: winston.Logge watchTemplateDirChanges(currentContext.logger); await reattachStaleContainers(logger); - await stopRunningServices(); // TODO: is this necessary? logger.info(`Using ${engine.defaultEngine ? 'default' : 'custom'} engine`); } @@ -468,6 +474,13 @@ export async function resumeService(id: string) { port, ports: options.ports ?? [], network, + labels: { + [StandardLabel.Nsm]: 'true', + [StandardLabel.ServiceId]: id, + [StandardLabel.VolumeId]: id, + [StandardLabel.TemplateId]: template, + // TODO: nsm.buildDir + } }; const perma = await db.getPerma(id); @@ -497,7 +510,6 @@ export async function resumeService(id: string) { if (image) { currentContext.logger.info('Running service ' + id + "..."); containerId = await engine.run( - template, image, id, runOptions, @@ -561,7 +573,7 @@ export async function stopService(id: string, force?: boolean) { if (force) { await engine.kill(session.containerId, meta); } else { - await engine.stop(session.containerId, meta); + await engine.stop(session.containerId); } } catch (e) { currentContext.logger.error(e); @@ -714,7 +726,7 @@ export function isRunning(id: string) { return getRunningService(id) != undefined; } -function getRunningService(id: string) { +export function getRunningService(id: string) { return started.find(service => service.id === id); } @@ -839,21 +851,44 @@ async function getPermaModel(id: string) { return perma_; } +/** + * Reattach to containers that are still running from the previous session. + * This may happen if NSM was force-stopped and not properly cleared up resources. + * + * @param logger The logger to use + */ async function reattachStaleContainers(logger: winston.Logger) { - // TODO: find containers marked and find serviceId marked on them - // TODO: engine.reattach(containerId, buildRunListener(serviceId)); + const running = await engine.listRunning(Filters.node(nodeId)) + .then(containerIds => containerIds + // Filter out those that we have already started in this session, just in case + // this was started more than once a session + .filter(id => !started.find(runningService => runningService.session.containerId === id))); + + for (let containerId of running) { + const labels = await engine.getLabels(containerId); + if (!labels[StandardLabel.ServiceId]) { + // The container was in the running list, but does not have the required labels + // Should not happen, but just in case + logger.warn(`Found a running container with id ${containerId} that does not have a service id label, stopping.`); + + await engine.stop(containerId); + } - await new Promise((resolve) => whenUnlockedAll(() => resolve(null))); -} + const serviceId = labels[StandardLabel.ServiceId]; + logger.info(`Reattaching container ${containerId} for service ${serviceId}...`); -async function stopRunningServices() { - const running = await engine.listRunning(); - for (const id of running) { - const volumeId = await engine.getAttachedVolume(id); - if (!volumeId) { - // The container does not exist or does not have a volume attached? - continue; - } - await engine.stop(id, metaStorageForService(volumeId)); + // Reattach and watch the container + await engine.reattach(containerId, buildRunListener(serviceId)); + + // Save session in-memory + const info: RunningService = { + id: serviceId, + session: { + containerId + } + }; + started.push(info); } + + await new Promise((resolve) => whenUnlockedAll(() => resolve(null))); } \ No newline at end of file diff --git a/src/router/v1/service/lookupRoute.ts b/src/router/v1/service/lookupRoute.ts index 0abae9d..e7ae2ad 100644 --- a/src/router/v1/service/lookupRoute.ts +++ b/src/router/v1/service/lookupRoute.ts @@ -1,18 +1,18 @@ -import {AppContext} from "../../../app"; +import {AppContext} from "@nsm/app"; import {RouterHandler} from "../../index"; -export default async function ({database, manager}: AppContext): Promise { +export default async function ({manager}: AppContext): Promise { return { url: '/service/:id', routes: { get: async (req, res) => { const id = req.params.id; - const service = await database.getPerma(id); + const service = await manager.getService(id, { includeSession: true }); if (!service) { res.status(404).json({status: 404, message: 'Invalid service ID.'}).end(); return; } - const session = await database.getSession(id); + const session = service.session; let stats: any; if (session && req.query.stats === 'true') { stats = await manager.engine.stat(session.containerId); diff --git a/src/router/v1/status/index.ts b/src/router/v1/status/index.ts index 7ea761b..a9056d7 100644 --- a/src/router/v1/status/index.ts +++ b/src/router/v1/status/index.ts @@ -1,9 +1,10 @@ -import {AppContext, Database, ServiceManager} from "../../../app"; +import {AppContext, Database, ServiceManager} from "@nsm/app"; import {RouterHandler} from "../../index"; import * as os from "os"; +import {Filters} from "@nsm/engine"; async function checkNsmResources(engine: ServiceManager, db: Database) { - const stats = await engine.engine.statAll(); + const stats = await engine.engine.statAll(Filters.node(engine.nodeId)); const servicesGlobal = await db.list(engine.nodeId); const res = stats.reduce((acc, s) => { acc.memory.used += s.memory.used; @@ -55,9 +56,6 @@ export default async function ({manager, appConfig, database}: AppContext): Prom routes: { get: async (req, res) => { const nodeId = appConfig['node_id']; - const templates = await manager.listTemplates(); - const runningContainers = await manager.engine.listContainers(templates); - const sessions = await database.listSessions(manager.nodeId); const all = await database.list(nodeId); const [free, size] = await manager.engine.calcHostUsage(); const system = { @@ -68,9 +66,8 @@ export default async function ({manager, appConfig, database}: AppContext): Prom } res.json({ nodeId, - running: sessions - .filter(s => runningContainers.includes(s.containerId)) - .map(s => s.serviceId), + running: manager.getRunningServices() + .map(s => s.id), all: all.length, system, ...(req.query.stats === 'true' ? { stats: await checkNsmResources(manager, database) } : {}) From bbb889f4417b4c839c47c329cf10055f17424d68 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Wed, 18 Mar 2026 01:41:07 +0100 Subject: [PATCH 20/52] fix: implement label listing for container --- src/engine/docker/action/getLabels.ts | 12 ++++++++++++ src/engine/docker/index.ts | 2 ++ src/engine/manager.ts | 3 ++- 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 src/engine/docker/action/getLabels.ts diff --git a/src/engine/docker/action/getLabels.ts b/src/engine/docker/action/getLabels.ts new file mode 100644 index 0000000..52f3205 --- /dev/null +++ b/src/engine/docker/action/getLabels.ts @@ -0,0 +1,12 @@ +import DockerClient from "dockerode"; +import {ServiceEngine} from "@nsm/engine"; + +export default function (client: DockerClient): ServiceEngine['getLabels'] { + return async (id) => { + const container = client.getContainer(id); + + const inspect = await container.inspect(); + + return inspect.Config.Labels; + } +} \ No newline at end of file diff --git a/src/engine/docker/index.ts b/src/engine/docker/index.ts index d91ce13..8ee52e1 100644 --- a/src/engine/docker/index.ts +++ b/src/engine/docker/index.ts @@ -9,6 +9,7 @@ import reattach from "./action/reattach"; import delVolume from './action/deletev'; import delImage from './action/deletei'; import cmd from './action/cmd'; +import getLabels from './action/getLabels'; import listContainers from './action/listc'; import listAttachedPorts from './action/listp'; import stat from "./action/stat"; @@ -31,6 +32,7 @@ export default function buildDockerEngine(appConfig: any) { engine.deleteVolume = delVolume(engine, client); engine.deleteImage = delImage(client); engine.cmd = cmd(engine, client); + engine.getLabels = getLabels(client); engine.listContainers = listContainers(engine, client); engine.listAttachedPorts = listAttachedPorts(engine, client); engine.stat = stat(engine, client); diff --git a/src/engine/manager.ts b/src/engine/manager.ts index a035aa8..d50625d 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -477,6 +477,7 @@ export async function resumeService(id: string) { labels: { [StandardLabel.Nsm]: 'true', [StandardLabel.ServiceId]: id, + [StandardLabel.NodeId]: nodeId, [StandardLabel.VolumeId]: id, [StandardLabel.TemplateId]: template, // TODO: nsm.buildDir @@ -875,7 +876,6 @@ async function reattachStaleContainers(logger: winston.Logger) { } const serviceId = labels[StandardLabel.ServiceId]; - logger.info(`Reattaching container ${containerId} for service ${serviceId}...`); // Reattach and watch the container await engine.reattach(containerId, buildRunListener(serviceId)); @@ -888,6 +888,7 @@ async function reattachStaleContainers(logger: winston.Logger) { } }; started.push(info); + logger.info(`Reattached container ${containerId} for service ${serviceId}`); } await new Promise((resolve) => whenUnlockedAll(() => resolve(null))); From fefdb03a2a5f2daa9218a73810ca70fccf5ce8e1 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Wed, 18 Mar 2026 01:44:42 +0100 Subject: [PATCH 21/52] fix: tests --- tests/api/api.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/api/api.test.ts b/tests/api/api.test.ts index f187cc9..8730f03 100644 --- a/tests/api/api.test.ts +++ b/tests/api/api.test.ts @@ -127,8 +127,6 @@ describe("Test v1 API models", () => { "port", undefined, "options", undefined, "env", undefined, - "session.serviceId", undefined, - "session.nodeId", undefined, "session.containerId", undefined, ]); }, 20000); From 703dee4d82d823a2dd948a377af22c13046728bc Mon Sep 17 00:00:00 2001 From: ZorTik Date: Wed, 18 Mar 2026 01:46:01 +0100 Subject: [PATCH 22/52] feat: stack trace --- TODO.txt | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/TODO.txt b/TODO.txt index e4ce032..56a4d86 100644 --- a/TODO.txt +++ b/TODO.txt @@ -4,4 +4,25 @@ abstrahovat template management a udělat další sources jako například použ předělat usages monitoring na nějaký standardizovaný formát/způsob pořešit /resources/config.yml aby měl víc možností a možná byl uložený jinde v systému podle platformy port retrieval strategy pro spouštění servis, aby nebyla jenom random ale i jiné strategies -udělat nějaký auto packaging do balíčků na githubu jako bundle aby se dal spustit jako systémový proces v systemctl například \ No newline at end of file +udělat nějaký auto packaging do balíčků na githubu jako bundle aby se dal spustit jako systémový proces v systemctl například + + +2026-03-18T00:45:00.041Z [NSM] info: Server started on port 3000 +2026-03-18T00:45:11.731Z [NSM] info: Running service f052b43b-3b04-4c75-957a-5ff0cbec65df... +2026-03-18T00:45:12.119Z [NSM] info: Service f052b43b-3b04-4c75-957a-5ff0cbec65df resumed +C:\Users\zortl\WebstormProjects\node-server-manager\node_modules\docker-modem\lib\modem.js:383 + var msg = new Error( + ^ + +Error: (HTTP code 404) no such container - No such container: undefined + at C:\Users\zortl\WebstormProjects\node-server-manager\node_modules\docker-modem\lib\modem.js:383:17 + at getCause (C:\Users\zortl\WebstormProjects\node-server-manager\node_modules\docker-modem\lib\modem.js:418:7) + at Modem.buildPayload (C:\Users\zortl\WebstormProjects\node-server-manager\node_modules\docker-modem\lib\modem.js:379:5) + at IncomingMessage. (C:\Users\zortl\WebstormProjects\node-server-manager\node_modules\docker-modem\lib\modem.js:347:16) + at IncomingMessage.emit (node:events:531:35) + at endReadableNT (node:internal/streams/readable:1698:12) + at process.processTicksAndRejections (node:internal/process/task_queues:90:21) { + reason: 'no such container', + statusCode: 404, + json: { message: 'No such container: undefined' } +} From 4d645847fff0b0c11bfeaf8dfb45989a706ebc9b Mon Sep 17 00:00:00 2001 From: ZorTik Date: Wed, 18 Mar 2026 11:47:11 +0100 Subject: [PATCH 23/52] feat: fixes --- src/engine/docker/action/stat.ts | 2 +- src/engine/manager.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engine/docker/action/stat.ts b/src/engine/docker/action/stat.ts index 4730691..0cf729a 100644 --- a/src/engine/docker/action/stat.ts +++ b/src/engine/docker/action/stat.ts @@ -1,4 +1,4 @@ -import {ServiceEngine} from "../../engine"; +import {ServiceEngine} from "@nsm/engine"; import DockerClient from "dockerode"; import {adaptContainerStatsFromDocker} from "@nsm/util/docker"; diff --git a/src/engine/manager.ts b/src/engine/manager.ts index d50625d..3f7a2a8 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -668,7 +668,7 @@ export async function getService(from: string, options?: { includeSession?: bool if (data && (data.nodeId == nodeId || options?.otherNodes === true)) { let session = undefined; if (options?.includeSession === true) { - session = getRunningService(data.serviceId); + session = getRunningService(data.serviceId)?.session; } return { ...data, From b0a3f4379bfd64bd100573fbfdc44f6c1dc31ea2 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Wed, 18 Mar 2026 15:59:46 +0100 Subject: [PATCH 24/52] feat: refactor engine name display --- TODO.txt | 24 +----------------------- src/engine/docker/index.ts | 1 + src/engine/engine.ts | 14 ++++++-------- src/engine/manager.ts | 2 +- 4 files changed, 9 insertions(+), 32 deletions(-) diff --git a/TODO.txt b/TODO.txt index 56a4d86..dbc5ea8 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,28 +1,6 @@ -vyřešit logging u servis a logování změn stavů -odebrat nodeId, cluster support se bude řešit jinak +vyřešit logging u servis a logování změn stavů takže i stavy služeb abstrahovat template management a udělat další sources jako například používání images z docker registry předělat usages monitoring na nějaký standardizovaný formát/způsob pořešit /resources/config.yml aby měl víc možností a možná byl uložený jinde v systému podle platformy port retrieval strategy pro spouštění servis, aby nebyla jenom random ale i jiné strategies udělat nějaký auto packaging do balíčků na githubu jako bundle aby se dal spustit jako systémový proces v systemctl například - - -2026-03-18T00:45:00.041Z [NSM] info: Server started on port 3000 -2026-03-18T00:45:11.731Z [NSM] info: Running service f052b43b-3b04-4c75-957a-5ff0cbec65df... -2026-03-18T00:45:12.119Z [NSM] info: Service f052b43b-3b04-4c75-957a-5ff0cbec65df resumed -C:\Users\zortl\WebstormProjects\node-server-manager\node_modules\docker-modem\lib\modem.js:383 - var msg = new Error( - ^ - -Error: (HTTP code 404) no such container - No such container: undefined - at C:\Users\zortl\WebstormProjects\node-server-manager\node_modules\docker-modem\lib\modem.js:383:17 - at getCause (C:\Users\zortl\WebstormProjects\node-server-manager\node_modules\docker-modem\lib\modem.js:418:7) - at Modem.buildPayload (C:\Users\zortl\WebstormProjects\node-server-manager\node_modules\docker-modem\lib\modem.js:379:5) - at IncomingMessage. (C:\Users\zortl\WebstormProjects\node-server-manager\node_modules\docker-modem\lib\modem.js:347:16) - at IncomingMessage.emit (node:events:531:35) - at endReadableNT (node:internal/streams/readable:1698:12) - at process.processTicksAndRejections (node:internal/process/task_queues:90:21) { - reason: 'no such container', - statusCode: 404, - json: { message: 'No such container: undefined' } -} diff --git a/src/engine/docker/index.ts b/src/engine/docker/index.ts index 8ee52e1..c8d3178 100644 --- a/src/engine/docker/index.ts +++ b/src/engine/docker/index.ts @@ -21,6 +21,7 @@ export default function buildDockerEngine(appConfig: any) { // Default engine implementation const client = initDockerClient(appConfig); const engine = {} as DockerServiceEngine; + engine.name = "Docker"; engine.dockerClient = client; engine.rws = {}; // engine.cast - Being replaced in manager. diff --git a/src/engine/engine.ts b/src/engine/engine.ts index 116618f..24de4dc 100644 --- a/src/engine/engine.ts +++ b/src/engine/engine.ts @@ -79,9 +79,6 @@ export type DockerServiceEngine = ServiceEngineI & { } export type ServiceEngineI = ServiceEngine & { // Internal - // If we are using the default (not custom) engine - defaultEngine: boolean, - cast(): T; } @@ -91,6 +88,9 @@ export type ServiceEngineI = ServiceEngine & { // Internal * containers themselves. */ export type ServiceEngine = { + // Just for display purposes + name: string; + /** * Builds an image from build dir. * @@ -242,19 +242,17 @@ export const Filters = { export default function (appConfig: any): ServiceEngineI { let engine = getSingleton('engine'); - const usingBuiltInEngine = engine == undefined; - const engineId = process.env.NSM_ENGINE ?? 'docker'; - if (usingBuiltInEngine) { + if (!engine) { + const engineId = process.env.NSM_ENGINE ?? 'docker'; switch (engineId) { case 'docker': engine = buildDockerEngine(appConfig); break; default: - throw new Error('Invalid engine ID ' + engineId); + throw new Error('Invalid engine ID: ' + engineId); } } return { - defaultEngine: usingBuiltInEngine && engineId === 'docker', cast: undefined, // Being set in manager ...engine, }; diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 3f7a2a8..0048df0 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -357,7 +357,7 @@ export async function init(db_: Database, appConfig_: any, logger: winston.Logge await reattachStaleContainers(logger); - logger.info(`Using ${engine.defaultEngine ? 'default' : 'custom'} engine`); + logger.info(`Using engine: ${engine.name}`); } export async function expandEngine(exp?: T): Promise { From 099f714368714ac4866b3729667f88d141d850bd Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 20 Mar 2026 04:17:04 +0100 Subject: [PATCH 25/52] feat: big update --- .../migration.sql | 24 ++ .../migration.sql | 8 + prisma/schema.prisma | 34 ++- src/database/image.ts | 101 +++++++ src/database/index.ts | 33 ++- src/database/manager.ts | 260 ------------------ src/database/meta.ts | 24 ++ src/database/models.ts | 81 ++++-- src/database/perma.ts | 115 ++++++++ src/database/serviceMeta.ts | 38 +++ src/database/session.ts | 22 ++ src/engine/asyncp.ts | 5 +- src/engine/docker/action/reattach.ts | 4 +- src/engine/docker/action/run.ts | 35 ++- src/engine/docker/util/logging.ts | 15 + src/engine/engine.ts | 48 +++- src/engine/image.ts | 10 +- src/engine/manager.ts | 162 ++++++----- src/engine/session.ts | 157 +++++++++++ src/router/v1/service/listRoute.ts | 2 +- src/router/v1/service/lookupRoute.ts | 2 +- src/router/v1/status/index.ts | 4 +- src/security/token/index.ts | 2 +- tests/database/manager.test.ts | 16 +- tests/engine/image.test.ts | 4 +- tests/engine/middle.test.ts | 2 +- 26 files changed, 806 insertions(+), 402 deletions(-) create mode 100644 prisma/migrations/20260319213658_service_session/migration.sql create mode 100644 prisma/migrations/20260320025242_service_log_source/migration.sql create mode 100644 src/database/image.ts delete mode 100644 src/database/manager.ts create mode 100644 src/database/meta.ts create mode 100644 src/database/perma.ts create mode 100644 src/database/serviceMeta.ts create mode 100644 src/database/session.ts create mode 100644 src/engine/docker/util/logging.ts create mode 100644 src/engine/session.ts diff --git a/prisma/migrations/20260319213658_service_session/migration.sql b/prisma/migrations/20260319213658_service_session/migration.sql new file mode 100644 index 0000000..adc4313 --- /dev/null +++ b/prisma/migrations/20260319213658_service_session/migration.sql @@ -0,0 +1,24 @@ +-- CreateTable +CREATE TABLE `ServiceSession` ( + `id` VARCHAR(191) NOT NULL, + `serviceId` VARCHAR(191) NOT NULL, + + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + +-- CreateTable +CREATE TABLE `ServiceLogRecord` ( + `id` BIGINT NOT NULL AUTO_INCREMENT, + `sessionId` VARCHAR(191) NOT NULL, + `timestamp` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + `logLevel` VARCHAR(191) NOT NULL, + `message` VARCHAR(191) NOT NULL, + + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + +-- AddForeignKey +ALTER TABLE `ServiceSession` ADD CONSTRAINT `ServiceSession_serviceId_fkey` FOREIGN KEY (`serviceId`) REFERENCES `Service`(`serviceId`) ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE `ServiceLogRecord` ADD CONSTRAINT `ServiceLogRecord_sessionId_fkey` FOREIGN KEY (`sessionId`) REFERENCES `ServiceSession`(`id`) ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/prisma/migrations/20260320025242_service_log_source/migration.sql b/prisma/migrations/20260320025242_service_log_source/migration.sql new file mode 100644 index 0000000..4e6016d --- /dev/null +++ b/prisma/migrations/20260320025242_service_log_source/migration.sql @@ -0,0 +1,8 @@ +/* + Warnings: + + - Added the required column `source` to the `ServiceLogRecord` table without a default value. This is not possible if the table is not empty. + +*/ +-- AlterTable +ALTER TABLE `ServiceLogRecord` ADD COLUMN `source` ENUM('ENGINE', 'CONTAINER') NOT NULL; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 3bf2757..d2303d8 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -9,22 +9,27 @@ generator client { } datasource db { - provider = "mysql" - url = env("DATABASE_URL") - shadowDatabaseUrl = env("SHADOW_DATABASE_URL") + provider = "mysql" + url = env("DATABASE_URL") +} + +enum ServiceLogSource { + ENGINE + CONTAINER } model Service { - serviceId String @id @default(uuid()) + serviceId String @id @default(uuid()) nodeId String imageId String? template String port Int options Json - meta Json @default("{}") + meta Json @default("{}") env Json network Json? - image Image? @relation(fields: [imageId], references: [id]) + image Image? @relation(fields: [imageId], references: [id]) + sessions ServiceSession[] } model ServiceMeta { @@ -33,6 +38,23 @@ model ServiceMeta { value Json } +model ServiceSession { + id String @id @default(uuid()) + serviceId String + service Service @relation(fields: [serviceId], references: [serviceId], onDelete: Cascade) + logs ServiceLogRecord[] +} + +model ServiceLogRecord { + id BigInt @id @default(autoincrement()) + sessionId String + timestamp DateTime @default(now()) + source ServiceLogSource + logLevel String + message String + session ServiceSession @relation(fields: [sessionId], references: [id], onDelete: Cascade) +} + // Persistent key-value storage for system-wide attributes. model Meta { id Int @id @default(autoincrement()) diff --git a/src/database/image.ts b/src/database/image.ts new file mode 100644 index 0000000..189a778 --- /dev/null +++ b/src/database/image.ts @@ -0,0 +1,101 @@ +import {ImageRepository} from "@nsm/database/models"; +import {optionsDiffer} from "@nsm/engine/image"; +import {PrismaClient} from "@prisma/client"; + +let client: PrismaClient; + +export const init = (client_: PrismaClient) => { + client = client_; +} + +export const saveImage: ImageRepository["saveImage"] = async (info) => { + const { id, templateId, hash, buildOptions } = info; + + try { + await client.image.upsert({ + where: { id }, + update: { + templateId, + hash, + buildOptions: { + deleteMany: {}, + create: Object.entries(buildOptions).map(([key, value]) => ({ key, value })), + } + }, + create: { + id, + templateId, + hash, + buildOptions: { + create: Object.entries(buildOptions).map(([key, value]) => ({ key, value })), + } + } + }); + return true; + } catch (e) { + console.log(e); + return false; + } +} + +export const getImage: ImageRepository["getImage"] = async (id) => { + const image = await client.image.findUnique({ + where: { id }, + include: { + buildOptions: { + select: { key: true, value: true }, + } + } + }); + if (image) { + const buildOptions = {}; + image.buildOptions.forEach((option) => buildOptions[option.key] = option.value); + + return { + ...image, + buildOptions, + } + } else { + return undefined; + } +} + +export const deleteImage: ImageRepository["deleteImage"] = async (id) => { + try { + await client.image.delete({ + where: { id } + }); + return true; + } catch (e) { + if (e.code !== 'P2025') { + console.log(e); + } + + return false; + } +} + +export const listImagesByOptions: ImageRepository["listImagesByOptions"] = async (templateId, buildOptions) => { + return ( + client.image.findMany({ + include: { + buildOptions: { + select: { key: true, value: true }, + } + } + }) + ).then((images) => ( + images.map(image => ({ + ...image, + buildOptions: image.buildOptions.reduce((acc, option) => { + acc[option.key] = option.value; + return acc; + }, {}) + })) + )) + .then((images) => ( + images.filter( + (image) => image.templateId === templateId && !optionsDiffer(image.buildOptions, buildOptions) + ) + )); +} \ No newline at end of file diff --git a/src/database/index.ts b/src/database/index.ts index 942a0cd..f447f3a 100644 --- a/src/database/index.ts +++ b/src/database/index.ts @@ -1,8 +1,35 @@ import {Database} from "./models"; -import * as manager from './manager'; +import {PrismaClient} from "@prisma/client"; + +import * as permaRepository from "./perma"; +import * as metaRepository from "./meta"; +import * as serviceMetaRepository from "./serviceMeta"; +import * as imageRepository from "./image"; +import * as sessionRepository from "./session"; export * from './models'; -export default function (): Database { - return manager; +export default function (client?: PrismaClient): Database { + if (!client) { + client = new PrismaClient(); + } + + // Propagate client + ( + [ + permaRepository, + metaRepository, + serviceMetaRepository, + imageRepository, + sessionRepository + ] as unknown as { init: (client: PrismaClient) => void }[] + ).forEach(repository => repository.init(client)); + + return { + permaRepository, + metaRepository, + serviceMetaRepository, + imageRepository, + sessionRepository + } } \ No newline at end of file diff --git a/src/database/manager.ts b/src/database/manager.ts deleted file mode 100644 index b360e1a..0000000 --- a/src/database/manager.ts +++ /dev/null @@ -1,260 +0,0 @@ -import {PrismaClient} from "@prisma/client"; -import {ImageModel, PermaModel} from "./models"; -import {optionsDiffer} from "@nsm/engine/image"; - -let client = new PrismaClient(); - -export const initClientForTest = (client_: PrismaClient) => { - client = client_; -} - -export async function savePerma(data: PermaModel): Promise { - const { serviceId } = data; - try { - await client.service.upsert({ - where: { serviceId }, - update: data, - create: data - }); - return true; - } catch (e) { - console.log(e); - return false; - } -} - -export async function deleteSession(serviceId: string): Promise { - try { - await client.session.delete({ where: { serviceId } }); - return true; - } catch (e) { - if (!e.message.includes('does not exist')) { - console.log(e); - } - return false; - } -} - -export async function deleteSessions(nodeId: string): Promise { - try { - await client.session.deleteMany({ where: { nodeId } }); - return true; - } catch (e) { - if (!e.message.includes('does not exist')) { - console.log(e); - } - return false; - } -} - -export async function deletePerma(serviceId: string): Promise { - try { - await client.service.delete({ where: { serviceId } }); - return true; - } catch (e) { - if (e.code !== 'P2025') { - console.log(e); - } - return false; - } -} - -export async function getPerma(serviceId: string): Promise { - try { - const service = await client.service.findUnique({ where: { serviceId } }); - if (!service) { - return undefined; - } - return service as PermaModel; - } catch (e) { - console.log(e); - return undefined; - } -} - -export async function getMetaVal(key: string, defaultVal?: string): Promise { - try { - let meta = await client.meta.findUnique({ where: { key } }); - if (!meta) { - if (!defaultVal) { - return defaultVal; - } - meta = await client.meta.create({ data: { key, value: defaultVal } }); - } - return meta.value; - } catch (e) { - console.log(e); - return ''; - } -} - -export async function list(nodeId: string|undefined, page?: number, pageSize?: number, meta?: {[key: string]: any}): Promise { - try { - // SELECT * FROM Service WHERE JSON_EXTRACT(Meta, "$.tag1") IS NOT NULL; - let where = " WHERE 1"; - // Pagination part - let pg = ""; - // Values for prepared statement - let values = []; - - if (nodeId != undefined) { - where += " AND nodeId = ?"; - // Store for prepare statement - values.push(nodeId); - } - if (page != undefined && pageSize != undefined) { - // Insert pagination - pg += " LIMIT " + pageSize; - pg += " OFFSET " + page * pageSize; - } - if (meta != undefined) { - // AND clause for every key,value pair - for (const key in meta) { - // Add another AND clause for specific key,value pair - where += " AND JSON_EXTRACT(meta, ?) = ?"; - - // Push key and value to be replaced in prepared statement - values.push("$." + key, meta[key]); - } - } - return client - .$queryRawUnsafe(`SELECT * FROM Service${where}${pg};`, ...values) - .then(result => result as PermaModel[]); - } catch (e) { - console.log(e); - return []; - } -} - -export async function listAllUsingImage(imageId: string): Promise { - try { - return await client.service.findMany({ where: { imageId } }) as PermaModel[]; - } catch (e) { - console.log(e); - return []; - } -} - -export async function count(nodeId: string): Promise { - try { - return await client.service.count({ where: { nodeId } }); - } catch (e) { - console.log(e); - return -1; - } -} - -export async function setServiceMeta(serviceId: string, key: string, value: any): Promise { - try { - await client.serviceMeta.upsert({ - where: { serviceId }, - update: { serviceId, key, value }, - create: { serviceId, key, value } - }); - return true; - } catch (e) { - console.log(e); - return false; - } -} - -export async function getServiceMeta(serviceId: string, key: string): Promise { - const meta = await client.serviceMeta.findUnique({ where: { serviceId, key } }); - if (meta) { - return meta.value; - } else { - return undefined; - } -} - -export async function saveImage(info: ImageModel): Promise { - const { id, templateId, hash, buildOptions } = info; - - try { - await client.image.upsert({ - where: { id }, - update: { - templateId, - hash, - buildOptions: { - deleteMany: {}, - create: Object.entries(buildOptions).map(([key, value]) => ({ key, value })), - } - }, - create: { - id, - templateId, - hash, - buildOptions: { - create: Object.entries(buildOptions).map(([key, value]) => ({ key, value })), - } - } - }); - return true; - } catch (e) { - console.log(e); - return false; - } -} - -export async function getImage(id: string): Promise { - const image = await client.image.findUnique({ - where: { id }, - include: { - buildOptions: { - select: { key: true, value: true }, - } - } - }); - if (image) { - const buildOptions = {}; - image.buildOptions.forEach((option) => buildOptions[option.key] = option.value); - - return { - ...image, - buildOptions, - } - } else { - return undefined; - } -} - -export async function deleteImage(id: string): Promise { - try { - await client.image.delete({ - where: { id } - }); - return true; - } catch (e) { - if (e.code !== 'P2025') { - console.log(e); - } - - return false; - } -} - -export async function listImagesByOptions(templateId: string, buildOptions: {[key: string]: string}): Promise { - return ( - client.image.findMany({ - include: { - buildOptions: { - select: { key: true, value: true }, - } - } - }) - ).then((images) => ( - images.map(image => ({ - ...image, - buildOptions: image.buildOptions.reduce((acc, option) => { - acc[option.key] = option.value; - return acc; - }, {}) - })) - )) - .then((images) => ( - images.filter( - (image) => image.templateId === templateId && !optionsDiffer(image.buildOptions, buildOptions) - ) - )); -} \ No newline at end of file diff --git a/src/database/meta.ts b/src/database/meta.ts new file mode 100644 index 0000000..738e840 --- /dev/null +++ b/src/database/meta.ts @@ -0,0 +1,24 @@ +import {PrismaClient} from "@prisma/client"; +import {MetaRepository} from "@nsm/database/models"; + +let client: PrismaClient; + +export const init = (client_: PrismaClient) => { + client = client_; +} + +export const getMetaVal: MetaRepository["getMetaVal"] = async (key, defaultVal) => { + try { + let meta = await client.meta.findUnique({ where: { key } }); + if (!meta) { + if (!defaultVal) { + return defaultVal; + } + meta = await client.meta.create({ data: { key, value: defaultVal } }); + } + return meta.value; + } catch (e) { + console.log(e); + return ''; + } +} \ No newline at end of file diff --git a/src/database/models.ts b/src/database/models.ts index c441c3e..c2355a3 100644 --- a/src/database/models.ts +++ b/src/database/models.ts @@ -1,39 +1,66 @@ -export type Database = { +export interface Database { + permaRepository: PermaRepository; + metaRepository: MetaRepository; + serviceMetaRepository: ServiceMetaRepository; + imageRepository: ImageRepository; + sessionRepository: SessionRepository; + serviceLogRepository: ServiceLogRepository; +} + +export interface PermaRepository { savePerma(info: PermaModel): Promise; - deleteSession(serviceId: string): Promise; - deleteSessions(nodeId: string): Promise; deletePerma(serviceId: string): Promise; getPerma(serviceId: string): Promise; + listPerma(nodeId: string, page?: number, pageSize?: number, meta?: {[key: string]: any}): Promise; + listPermaUsingImage(imageId: string): Promise; + countPerma(nodeId: string): Promise; +} + +export interface MetaRepository { getMetaVal(key: string, defaultVal?: string): Promise; - list(nodeId: string, page?: number, pageSize?: number, meta?: {[key: string]: any}): Promise; - listAllUsingImage(imageId: string): Promise; - count(nodeId: string): Promise; +} + +export interface ServiceMetaRepository { setServiceMeta(serviceId: string, key: string, value: any): Promise; getServiceMeta(serviceId: string, key: string): Promise; +} + +export interface ImageRepository { saveImage(info: ImageModel): Promise; getImage(id: string): Promise; deleteImage(id: string): Promise; listImagesByOptions(templateId: string, buildOptions: {[key: string]: string}): Promise; -}; +} + +export interface SessionRepository { + createSession(serviceId: string): Promise; + // TODO: store session log record, list session log records etc +} + +export interface ServiceLogRepository { + createRecords(records: CreateLogRecordArgs[]): Promise; +} + +export type CreateLogRecordArgs = Omit; export type PermaModel = { - serviceId: string, - template: string, - nodeId: string, - imageId?: string, - port: number, + serviceId: string; + template: string; + nodeId: string; + imageId?: string; + port: number; options: { - [key: string]: any, - }, + [key: string]: any; + }; meta?: { - stopCmd?: string, - } + stopCmd?: string; + }; env: { - [key: string]: string, - }, + [key: string]: string; + }; network?: { - address: string, - portsOnly: boolean, + address: string; + portsOnly: boolean; } }; @@ -44,4 +71,18 @@ export type ImageModel = { buildOptions: { [key: string]: string, } +} + +export type ServiceSessionModel = { + id: string, + serviceId: string, +} + +export type ServiceLogRecordModel = { + id: number; + sessionId: string; + source: 'ENGINE' | 'CONTAINER' + timestamp: Date; + logLevel: string; + message: string; } \ No newline at end of file diff --git a/src/database/perma.ts b/src/database/perma.ts new file mode 100644 index 0000000..642b5bd --- /dev/null +++ b/src/database/perma.ts @@ -0,0 +1,115 @@ +import {PrismaClient} from "@prisma/client"; +import {PermaModel, PermaRepository} from "@nsm/database/models"; + +let client: PrismaClient; + +export const init = (client_: PrismaClient) => { + client = client_; +} + +export const savePerma: PermaRepository["savePerma"] = async (data) => { + const { serviceId } = data; + try { + await client.service.upsert({ + where: { serviceId }, + update: data, + create: data + }); + return true; + } catch (e) { + console.log(e); + return false; + } +} + +export const deletePerma: PermaRepository["deletePerma"] = async ( + serviceId +) => { + try { + await client.service.delete({ where: { serviceId } }); + return true; + } catch (e) { + if (e.code !== 'P2025') { + console.log(e); + } + return false; + } +} + +export const getPerma: PermaRepository["getPerma"] = async ( + serviceId +) => { + try { + const service = await client.service.findUnique({ where: { serviceId } }); + if (!service) { + return undefined; + } + return service as PermaModel; + } catch (e) { + console.log(e); + return undefined; + } +} + +export const listPerma: PermaRepository["listPerma"] = async ( + nodeId, + page, + pageSize, + meta +) => { + try { + // SELECT * FROM Service WHERE JSON_EXTRACT(Meta, "$.tag1") IS NOT NULL; + let where = " WHERE 1"; + // Pagination part + let pg = ""; + // Values for prepared statement + let values = []; + + if (nodeId != undefined) { + where += " AND nodeId = ?"; + // Store for prepare statement + values.push(nodeId); + } + if (page != undefined && pageSize != undefined) { + // Insert pagination + pg += " LIMIT " + pageSize; + pg += " OFFSET " + page * pageSize; + } + if (meta != undefined) { + // AND clause for every key,value pair + for (const key in meta) { + // Add another AND clause for specific key,value pair + where += " AND JSON_EXTRACT(meta, ?) = ?"; + + // Push key and value to be replaced in prepared statement + values.push("$." + key, meta[key]); + } + } + return client + .$queryRawUnsafe(`SELECT * FROM Service${where}${pg};`, ...values) + .then(result => result as PermaModel[]); + } catch (e) { + console.log(e); + return []; + } +} + +export const listPermaUsingImage: PermaRepository["listPermaUsingImage"] = async ( + imageId +) => { + try { + return await client.service.findMany({ where: { imageId } }) as PermaModel[]; + } catch (e) { + console.log(e); + return []; + } +} + +export const countPerma: PermaRepository["countPerma"] = async (nodeId) => { + try { + return await client.service.count({ where: { nodeId } }); + } catch (e) { + console.log(e); + return -1; + } +} diff --git a/src/database/serviceMeta.ts b/src/database/serviceMeta.ts new file mode 100644 index 0000000..8799340 --- /dev/null +++ b/src/database/serviceMeta.ts @@ -0,0 +1,38 @@ +import {PrismaClient} from "@prisma/client"; +import {ServiceMetaRepository} from "@nsm/database/models"; + +let client: PrismaClient; + +export const init = (client_: PrismaClient) => { + client = client_; +} + +export const setServiceMeta: ServiceMetaRepository["setServiceMeta"] = async ( + serviceId, + key, + value +) => { + try { + await client.serviceMeta.upsert({ + where: { serviceId }, + update: { serviceId, key, value }, + create: { serviceId, key, value } + }); + return true; + } catch (e) { + console.log(e); + return false; + } +} + +export const getServiceMeta: ServiceMetaRepository["getServiceMeta"] = async ( + serviceId, + key +) => { + const meta = await client.serviceMeta.findUnique({ where: { serviceId, key } }); + if (meta) { + return meta.value; + } else { + return undefined; + } +} \ No newline at end of file diff --git a/src/database/session.ts b/src/database/session.ts new file mode 100644 index 0000000..ff3da66 --- /dev/null +++ b/src/database/session.ts @@ -0,0 +1,22 @@ +import {Prisma, PrismaClient} from "@prisma/client"; +import {SessionRepository} from "@nsm/database/models"; + +let client: PrismaClient; + +export const init = (client_: PrismaClient) => { + client = client_; +} + +export const createSession: SessionRepository["createSession"] = async (serviceId) => { + const data: Prisma.ServiceSessionUncheckedCreateInput = { + serviceId + }; + + try { + await client.serviceSession.create({ data }); + } catch (e) { + console.log(e); + + return undefined; + } +} \ No newline at end of file diff --git a/src/engine/asyncp.ts b/src/engine/asyncp.ts index d41969c..2f94191 100644 --- a/src/engine/asyncp.ts +++ b/src/engine/asyncp.ts @@ -18,13 +18,14 @@ export function lockBusyAction(id: string, tp: string) { reqNotPending(id); statuses[id] = true; status_types[id] = tp; // type of action + return (err?: any) => { delete statuses[id]; delete status_types[id]; - // + (obs.get(id) ?? []).forEach(o => o(id, tp, err)); obs.delete(id); - // + if (pendingCount() == 0) { obsAll.forEach(o => o()); obsAll.splice(0, obsAll.length); diff --git a/src/engine/docker/action/reattach.ts b/src/engine/docker/action/reattach.ts index 41d2f9f..89ffbc2 100644 --- a/src/engine/docker/action/reattach.ts +++ b/src/engine/docker/action/reattach.ts @@ -1,7 +1,7 @@ import DockerClient from "dockerode"; import {DockerServiceEngine, ServiceEngine} from "@nsm/engine"; import {getActionType} from "@nsm/engine/asyncp"; -import {currentContext, currentContext as ctx} from "@nsm/app"; +import {currentContext} from "@nsm/app"; import {deleteNetwork as doDeleteNetwork, isInNetwork} from "@nsm/networking/manager"; async function deleteContainer(id: string, client: DockerClient, options: { deleteNetwork?: boolean }) { @@ -70,6 +70,6 @@ export default function reattach(self: ServiceEngine, client: DockerClient): Ser }); (self as DockerServiceEngine).rws[container.id] = rws; - await listener.onStateMessage?.('Watching changes'); + await listener.onStateChange?.({ id: 'watching_changes', description: 'Watching changes' }); } } \ No newline at end of file diff --git a/src/engine/docker/action/run.ts b/src/engine/docker/action/run.ts index 183d16d..dcf9cb6 100644 --- a/src/engine/docker/action/run.ts +++ b/src/engine/docker/action/run.ts @@ -1,9 +1,10 @@ import DockerClient from "dockerode"; -import {RunOptions, MetaStorage, ServiceEngine} from "@nsm/engine"; +import {RunOptions, MetaStorage, ServiceEngine, ServiceState} from "@nsm/engine"; import {accessNetwork, createNetwork} from "@nsm/networking/manager"; import {constructObjectLabels} from "@nsm/util/services"; import {currentContext as ctx} from "@nsm/app"; import {propagateOptionsToEnv} from "@nsm/engine/docker/util/env"; +import {infoRecord as info} from "@nsm/engine/docker/util/logging"; async function prepareVolume(client: DockerClient, volumeId: string) { try { @@ -89,25 +90,37 @@ async function prepareContainer( return container; } +const createState = (id: string, description: string): ServiceState => { + return { + id, + description + } +}; + +const createErrorState = (description: string): ServiceState => { + return { + id: 'error', + description + } +} + export default function run(self: ServiceEngine, client: DockerClient): ServiceEngine["run"] { return async (imageId, volumeId, options, meta, listener) => { let container: DockerClient.Container; // Prepare volume + let creating = await prepareVolume(client, volumeId); - if (creating) { - await listener.onStateMessage('Created new volume'); - } - await listener.onStateMessage('Preparing network'); + await listener.onStateChange?.(createState('preparing_network', 'Preparing network')); const net = await prepareNetwork(client, options.network, meta, creating); // Port decorator that takes port and according to network changes it to : or keeps the same. - await listener.onStateMessage('Preparing container'); + await listener.onStateChange?.(createState('preparing_container', 'Preparing container')); container = await prepareContainer(client, imageId, volumeId, options, net); - await listener.onStateMessage('Starting container'); + await listener.onStateChange?.(createState('starting_container', 'Starting container')); await container.start(); - const info = await container.inspect(); - if (!info.State.Running) { + const inspectInfo = await container.inspect(); + if (!inspectInfo.State.Running) { // Wait a bit for logs to be available await new Promise(r => setTimeout(r, 300)); @@ -122,8 +135,8 @@ export default function run(self: ServiceEngine, client: DockerClient): ServiceE }); const msg = logs.toString("utf8"); - await listener.onStateMessage("Container failed to start"); - await listener.onMessage(msg); + await listener.onStateChange?.(createErrorState('Container failed to start')); + await listener.onMessage(info(msg)); } catch (e) { ctx.logger.error("Error while fetching logs for failed container " + container.id, e); } diff --git a/src/engine/docker/util/logging.ts b/src/engine/docker/util/logging.ts new file mode 100644 index 0000000..48ae6b1 --- /dev/null +++ b/src/engine/docker/util/logging.ts @@ -0,0 +1,15 @@ +import {ServiceLogRecord} from "@nsm/engine"; + +export const infoRecord = (message: string): ServiceLogRecord => { + return { + level: 'info', + message + } +} + +export const errorRecord = (message: string): ServiceLogRecord => { + return { + level: 'error', + message + } +} \ No newline at end of file diff --git a/src/engine/engine.ts b/src/engine/engine.ts index 24de4dc..df3dc82 100644 --- a/src/engine/engine.ts +++ b/src/engine/engine.ts @@ -48,20 +48,31 @@ export type ContainerFilter = { labels?: { [key: string]: string }; } +export type ServiceLogRecord = { + level: 'error' | 'info'; + message: string, +} + +export type ServiceState = { + id: string; + description: string; +} + export type RunListener = { /** - * Called when there is a state change in the container, with the message of the state change. - * This is used to update the logs in NSM. + * Called when the container progress changes state. * - * @param message The message of the state change, e.g. "Creating container", etc. + * @param state The new state. */ - onStateMessage?: (message: string) => Promise|void; + onStateChange?: (state: ServiceState) => Promise|void; + /** * Called when there is a message from the container, with the message. * * @param message The message from the container */ - onMessage?: (message: string) => Promise|void; + onMessage?: (message: ServiceLogRecord) => Promise|void; + /** * Called when the container is closed, either by stop or kill, or by itself. */ @@ -209,8 +220,6 @@ export enum StandardLabel { TemplateId = 'nsm.templateId', // The node ID of the managing worker. NodeId = 'nsm.nodeId', - // The templare build dir. - BuildDir = 'nsm.buildDir', } export const Filters = { @@ -240,6 +249,31 @@ export const Filters = { } } +/** + * Combines multiple run listeners into one, by calling them in sequence. + * + * @param listeners The listeners to combine. + */ +export const combineRunListeners = (listeners: RunListener[]): RunListener => { + return { + onStateChange: async (state) => { + for (let listener of listeners) { + await listener.onStateChange?.(state); + } + }, + onMessage: async (record) => { + for (let listener of listeners) { + await listener.onMessage?.(record); + } + }, + onClose: () => { + for (let listener of listeners) { + listener.onClose?.(); + } + } + } +} + export default function (appConfig: any): ServiceEngineI { let engine = getSingleton('engine'); if (!engine) { diff --git a/src/engine/image.ts b/src/engine/image.ts index 4225bc7..edfa97d 100644 --- a/src/engine/image.ts +++ b/src/engine/image.ts @@ -128,7 +128,7 @@ export const optionsDiffer = (options1: BuildOptionsMap, options2: BuildOptionsM * @throws Error if the image with the given ID is not found in the database */ const getImage = async (id: string) => { - const image = await db.getImage(id); + const image = await db.imageRepository.getImage(id); if (!image) { throw new Error(`Image with ID ${id} not found`); } @@ -149,7 +149,7 @@ const buildImage = async (templateId: string, options: BuildOptionsMap, imageId? const hash = templateDirWatcher.getTemplateHash(templateId); imageId = await engine.build(imageId, buildDir(templateId), options); - await db.saveImage({ + await db.imageRepository.saveImage({ id: imageId, templateId, hash, @@ -159,7 +159,7 @@ const buildImage = async (templateId: string, options: BuildOptionsMap, imageId? } const pickImage = async (templateId: string, options: BuildOptionsMap): Promise => { - const images = await db.listImagesByOptions(templateId, options); + const images = await db.imageRepository.listImagesByOptions(templateId, options); if (images.length == 0) { return null; } @@ -174,7 +174,7 @@ const rebuildImage = async (image: ImageModel) => { } export const deleteImageIfUnused = async (image: ImageModel) => { - const servicesUsingImage = await db.listAllUsingImage(image.id); + const servicesUsingImage = await db.permaRepository.listPermaUsingImage(image.id); if (servicesUsingImage.length > 0) { // Image is still in use, do not delete return; @@ -187,5 +187,5 @@ export const deleteImageIfUnused = async (image: ImageModel) => { } catch (e) { logger.error(`Failed to delete image ${image.id}`, e); } - await db.deleteImage(image.id); + await db.imageRepository.deleteImage(image.id); } \ No newline at end of file diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 0048df0..3076299 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -1,5 +1,11 @@ import {currentContext, Database} from "../app"; -import createEngine, {RunOptions, RunListener, ServiceEngineI, StandardLabel, Filters} from "./engine"; +import createEngine, { + RunOptions, + RunListener, + ServiceEngineI, + StandardLabel, + Filters, combineRunListeners +} from "./engine"; import {Template, getTemplate as loadTemplate, getAllTemplates} from "./template"; import * as templateManager from "./template"; import * as templateDirWatcher from "./monitoring/templateDirWatcher"; @@ -22,9 +28,9 @@ import {isDebug} from "../helpers"; import {resolveSequentially} from "@nsm/util/promises"; import {buildDir} from "@nsm/engine/monitoring/util"; import {watchTemplateDirChanges} from "@nsm/engine/monitoring/templateDirWatcher"; -import {logService} from "@nsm/logger"; import {processImage, init as initImageEngine, deleteImageIfUnused} from "@nsm/engine/image"; import {propagateOptionsToEnv} from "@nsm/engine/docker/util/env"; +import {ActiveServiceSession, beginServiceSession, ServiceSession, init as initSessionEngine} from "@nsm/engine/session"; export type Options = { /** @@ -279,10 +285,11 @@ export type ServiceManager = ServiceManagerEventBus & { type RunningService = { id: string; - session: Session; + session: ServiceSession; + internalSession: InternalSession; } -export type Session = { +export type InternalSession = { containerId: string; // TODO: add more useful information? } @@ -291,7 +298,7 @@ export type ServiceInfo = PermaModel & { optionsRam: number, // From options.ram optionsCpu: number, // From options.cpu optionsDisk: number, // From options.disk - session?: Session + internalSession?: InternalSession } // 1 = unknown, 2 = conflict, 3 = not found @@ -353,6 +360,7 @@ export async function init(db_: Database, appConfig_: any, logger: winston.Logge nodeId = appConfig['node_id'] as string; initImageEngine(engine, templateManager, templateDirWatcher, db_, currentContext.logger); + initSessionEngine(db_); watchTemplateDirChanges(currentContext.logger); await reattachStaleContainers(logger); @@ -391,7 +399,6 @@ export async function createService(template: string, options: Options) { env, network } = options; - const serviceSettings = settings(template); // Join meta supplied by user and template meta @@ -425,7 +432,7 @@ export async function createService(template: string, options: Options) { }; let err: any; // Save permanent info - if (!await db.savePerma(perma)) { + if (!await db.permaRepository.savePerma(perma)) { err = new _InternalError('Failed to save perma info to database'); } @@ -444,7 +451,6 @@ export async function createService(template: string, options: Options) { export async function resumeService(id: string) { reqNotRunning(id); - let { template, options, @@ -480,11 +486,10 @@ export async function resumeService(id: string) { [StandardLabel.NodeId]: nodeId, [StandardLabel.VolumeId]: id, [StandardLabel.TemplateId]: template, - // TODO: nsm.buildDir } }; - const perma = await db.getPerma(id); + const perma = await db.permaRepository.getPerma(id); let image = perma.imageId; // Propagate other options to env, so they can be used in image processing and building @@ -502,20 +507,21 @@ export async function resumeService(id: string) { // Update image in database if it was changed by processing perma.imageId = image; - await db.savePerma(perma); + await db.permaRepository.savePerma(perma); } + let session: ActiveServiceSession|undefined; let containerId: string|undefined; try { // Run the container with the built image and save the container id for later use. if (image) { - currentContext.logger.info('Running service ' + id + "..."); + session = await beginServiceSession(id); containerId = await engine.run( image, id, runOptions, meta, - buildRunListener(id) + buildRunListener(session) ); } } catch (e) { @@ -526,7 +532,8 @@ export async function resumeService(id: string) { if (containerId) { const runningService: RunningService = { id, - session: { + session, + internalSession: { containerId } }; @@ -551,9 +558,9 @@ export async function resumeService(id: string) { export async function stopService(id: string, force?: boolean) { await reqExists(id); - const { session } = reqRunning(id); + const { internalSession } = reqRunning(id); - lckStatusTp(session.containerId, 'stop'); + lckStatusTp(internalSession.containerId, 'stop'); const unlock = lockBusyAction(id, 'stop'); try { @@ -566,15 +573,15 @@ export async function stopService(id: string, force?: boolean) { if (isServicePending(id)) { unlock(error); } - ulckStatusTp(session.containerId); + ulckStatusTp(internalSession.containerId); return true; }) const meta = metaStorageForService(id); if (force) { - await engine.kill(session.containerId, meta); + await engine.kill(internalSession.containerId, meta); } else { - await engine.stop(session.containerId); + await engine.stop(internalSession.containerId); } } catch (e) { currentContext.logger.error(e); @@ -589,14 +596,14 @@ export async function stopServiceForcibly(id: string) { export async function sendStopSignal(id: string) { const perma = await reqExists(id); - const { session } = reqRunning(id); + const { internalSession } = reqRunning(id); const stopCmd = perma.meta?.stopCmd; if (!stopCmd) { throw new _InternalError('Service does not have stop command set.'); } - await engine.cmd(session.containerId, stopCmd); + await engine.cmd(internalSession.containerId, stopCmd); return true; } @@ -612,9 +619,9 @@ export async function deleteService(id: string) { const unlockHandler: UnlockObserver = (_, __, ___) => { const resolveDeleteImageFunc = async () => { - const image = await db.getPerma(id) + const image = await db.permaRepository.getPerma(id) .then((perma) => perma.imageId - ? db.getImage(perma.imageId) + ? db.imageRepository.getImage(perma.imageId) : undefined); return async () => { @@ -629,7 +636,7 @@ export async function deleteService(id: string) { .then((deleteImageFunc) => ( resolveSequentially( async () => engine.deleteVolume(id), - async () => db.deletePerma(id), + async () => db.permaRepository.deletePerma(id), deleteImageFunc, ) )) @@ -643,7 +650,7 @@ export async function deleteService(id: string) { export async function updateOptions(id: string, options: Options) { reqNotPending(id); - const perma = await db.getPerma(id); + const perma = await db.permaRepository.getPerma(id); const data: PermaModel = { ...perma, ...options, @@ -656,7 +663,7 @@ export async function updateOptions(id: string, options: Options) { ...options.env, }, }; - return db.savePerma(data); + return db.permaRepository.savePerma(data); } export function getTemplate(id: string) { @@ -664,11 +671,11 @@ export function getTemplate(id: string) { } export async function getService(from: string, options?: { includeSession?: boolean, otherNodes?: boolean }) { - const data = typeof from === 'string' ? await db.getPerma(from) : from; + const data = typeof from === 'string' ? await db.permaRepository.getPerma(from) : from; if (data && (data.nodeId == nodeId || options?.otherNodes === true)) { let session = undefined; if (options?.includeSession === true) { - session = getRunningService(data.serviceId)?.session; + session = getRunningService(data.serviceId)?.internalSession; } return { ...data, @@ -688,8 +695,8 @@ export function getLastPowerError(id: string) { export async function listServices(options: ListServicesOptions) { const meta = options.filter?.meta; - return db - .list(nodeId, options.page, options.pageSize, meta) + return db.permaRepository + .listPerma(nodeId, options.page, options.pageSize, meta) .then(list => list.map(d => d.serviceId)); } @@ -698,29 +705,34 @@ export async function listTemplates(): Promise { } export async function stopRunning() { - await Promise.all(started.map(({id}) => ( - new Promise((resolve) => { - whenUnlocked(id, () => { - stopService(id) - .catch(e => console.log(e)) - .then(() => { - whenUnlocked(id, () => resolve(null)); - }); - }); - }) - ))); + const tasks = started.map(({id}) => ( + new Promise((resolve) => { + whenUnlocked(id, () => { + stopService(id) + .catch(e => console.log(e)) + .then(() => { + whenUnlocked(id, () => resolve(null)); + }); + }); + }) + )); + + await Promise.all(tasks); } export async function waitForBusyAction(id: string) { - return new Promise((resolve, reject) => { - whenUnlocked(id, (_, status, err) => { - if (err) { - reject(err); - return; - } - resolve(null); - }); - }); + return new Promise( + (resolve, reject) => { + whenUnlocked(id, (_, __, err) => { + if (err) { + reject(err); + return; + } + + resolve(null); + }); + } + ); } export function isRunning(id: string) { @@ -734,10 +746,12 @@ export function getRunningService(id: string) { function metaStorageForService(id: string): MetaStorage { // service id return { set: async (key, value) => { - return db.setServiceMeta(id, key, value); + return db.serviceMetaRepository.setServiceMeta(id, key, value); }, get: async (key, def) => { - return (await db.getServiceMeta(id, key)) ?? def; + const meta = await db.serviceMetaRepository.getServiceMeta(id, key); + + return meta ?? def; }, }; } @@ -775,7 +789,7 @@ export { } async function reqExists(id: string) { - const perma = await db.getPerma(id); + const perma = await db.permaRepository.getPerma(id); if (!perma) { throw new _InternalError("Service not found.", 3); } @@ -800,6 +814,7 @@ function reqNotRunning(id: string) { function clearRunningServiceIfExists(id: string) { const service = getRunningService(id); + if (service) { started.splice(started.indexOf(service, 1)); } @@ -819,18 +834,20 @@ function callManagerEvent(e: T, event: Ser evtHandlers.set(e, newArray); } -function buildRunListener(serviceId: string): RunListener { - return { - onStateMessage: (msg) => { - // TODO: handle state messages in a better way - }, - onMessage: (msg) => { - logService(serviceId, msg); - }, - onClose: async () => { - // Remove session when container is closed, because the service is not running anymore - await db.deleteSession(serviceId); +/** + * Collects all relevant run listeners and builds a composite one + * to be used directly when running/attaching service container. + * + * @param session The session for whom to create the session. + */ +function buildRunListener(session: ActiveServiceSession): RunListener { + const { + serviceId + } = session; + // The internal run listener of this manager + const internalRunListener: RunListener = { + onClose: async () => { clearRunningServiceIfExists(serviceId); // Call stop event on the manager for the stopService() to potentially @@ -840,10 +857,16 @@ function buildRunListener(serviceId: string): RunListener { currentContext.logger.info("Service " + serviceId + " stopped"); } }; + // Combine collected listeners + return combineRunListeners([ + internalRunListener, + // Add listener from the session + session.runListener + ]) } async function getPermaModel(id: string) { - const perma_ = await db.getPerma(id); + const perma_ = await db.permaRepository.getPerma(id); if (!perma_) { // Service does not exist throw new _InternalError('Not found.', 3); @@ -863,7 +886,7 @@ async function reattachStaleContainers(logger: winston.Logger) { .then(containerIds => containerIds // Filter out those that we have already started in this session, just in case // this was started more than once a session - .filter(id => !started.find(runningService => runningService.session.containerId === id))); + .filter(id => !started.find(runningService => runningService.internalSession.containerId === id))); for (let containerId of running) { const labels = await engine.getLabels(containerId); @@ -877,13 +900,16 @@ async function reattachStaleContainers(logger: winston.Logger) { const serviceId = labels[StandardLabel.ServiceId]; + // We must begin a new session since the previous was interrupted + const session = await beginServiceSession(serviceId); // Reattach and watch the container - await engine.reattach(containerId, buildRunListener(serviceId)); + await engine.reattach(containerId, buildRunListener(session)); // Save session in-memory const info: RunningService = { id: serviceId, - session: { + session, + internalSession: { containerId } }; diff --git a/src/engine/session.ts b/src/engine/session.ts new file mode 100644 index 0000000..23ed604 --- /dev/null +++ b/src/engine/session.ts @@ -0,0 +1,157 @@ +import {RunListener} from "@nsm/engine/engine"; +import {CreateLogRecordArgs, Database} from "@nsm/database"; + +export interface ServiceSession { + id: string; + serviceId: string; + // TODO: begin timestamp, end timestamp +} + +export interface ActiveServiceSession extends ServiceSession { + /** + * A run listener for this session. + */ + runListener: RunListener; +} + +let db: Database; + +export const init = (db_: Database) => { + db = db_; +} + +/** + * Begins a new service session for the given service ID. + * + * @param serviceId The ID of the service for which to begin a session. + * @return An object representing the active service session. + */ +export const beginServiceSession = async (serviceId: string): Promise => { + let session = await db.sessionRepository.createSession(serviceId); + + // Debounce the push in bulk to prevent database overhead + const { + flush: flushRecords, + debounce: pushRecord + } = debounceBulkPush(); + + const runListener: RunListener = { + onStateChange: async (state) => { + const log: CreateLogRecordArgs = { + sessionId: session.id, + source: 'ENGINE', + logLevel: 'INFO', + message: state.description + } + + pushRecord(log); + }, + onMessage: async (record) => { + const log: CreateLogRecordArgs = { + sessionId: session.id, + source: 'CONTAINER', + logLevel: record.level.toUpperCase(), + message: record.message + } + + pushRecord(log); + }, + onClose: async () => { + // Push remaining logs now + await flushRecords(); + + // TODO: mark session as closed + } + } + + return { + ...session, + runListener + } +} + +/** + * Creates a debounced function for pushing log records in bulk to the database. + * + * This function maintains an internal buffer of log records and pushes them to the database + * after a certain delay or when the buffer reaches a certain size, whichever comes first. + * + * @return A function that can be called to push a log record, which will be debounced and pushed in bulk. + */ +const debounceBulkPush = () => { + const logRecordsBulk: CreateLogRecordArgs[] = []; + + const MAX_BATCH_SIZE = 50; + const DEBOUNCE_MS = 500; + + let timeout: NodeJS.Timeout|null = null; + let isFlushing = false; + + const flush = async () => { + if (logRecordsBulk.length === 0) { + return; + } + if (isFlushing) { + // If currently flushing, move the timer + renew(); + return; + } + + isFlushing = true; + + let success = false; + + const batch = logRecordsBulk.splice(0, logRecordsBulk.length); + try { + success = await db.serviceLogRepository.createRecords(batch); + } catch (err) { + console.error("Failed to flush log records:", err); + } finally { + if (!success) { + // If the operation did not complete for whatever reason, + // bring back the records + logRecordsBulk.unshift(...batch); + } + + isFlushing = false; + } + }; + + const renew = () => { + // Renew timer + if (timeout) { + clearTimeout(timeout); + } + timeout = setTimeout(() => { + timeout = null; + + void flush(); + }, DEBOUNCE_MS); + }; + + return { + // Also return flush to be able to forcibly push logs into the database + flush, + debounce: (log: CreateLogRecordArgs) => { + logRecordsBulk.push(log); + + // If we reached the bulk size limit, flush immediately + if (logRecordsBulk.length >= MAX_BATCH_SIZE) { + if (timeout) { + clearTimeout(timeout); + timeout = null; + } + + void flush(); + return; + } + + // Renew timer + renew(); + } + } +} + +// TODO: get service session + +// TODO: list service session logs \ No newline at end of file diff --git a/src/router/v1/service/listRoute.ts b/src/router/v1/service/listRoute.ts index 0dfa252..75dd67d 100644 --- a/src/router/v1/service/listRoute.ts +++ b/src/router/v1/service/listRoute.ts @@ -52,7 +52,7 @@ export default async function ({manager, database}: AppContext): Promise { res.status(404).json({status: 404, message: 'Invalid service ID.'}).end(); return; } - const session = service.session; + const session = service.internalSession; let stats: any; if (session && req.query.stats === 'true') { stats = await manager.engine.stat(session.containerId); diff --git a/src/router/v1/status/index.ts b/src/router/v1/status/index.ts index a9056d7..232005c 100644 --- a/src/router/v1/status/index.ts +++ b/src/router/v1/status/index.ts @@ -5,7 +5,7 @@ import {Filters} from "@nsm/engine"; async function checkNsmResources(engine: ServiceManager, db: Database) { const stats = await engine.engine.statAll(Filters.node(engine.nodeId)); - const servicesGlobal = await db.list(engine.nodeId); + const servicesGlobal = await db.listPerma(engine.nodeId); const res = stats.reduce((acc, s) => { acc.memory.used += s.memory.used; acc.memory.total += s.memory.total; @@ -56,7 +56,7 @@ export default async function ({manager, appConfig, database}: AppContext): Prom routes: { get: async (req, res) => { const nodeId = appConfig['node_id']; - const all = await database.list(nodeId); + const all = await database.listPerma(nodeId); const [free, size] = await manager.engine.calcHostUsage(); const system = { totalmem: os.totalmem(), diff --git a/src/security/token/index.ts b/src/security/token/index.ts index d808775..0cd367e 100644 --- a/src/security/token/index.ts +++ b/src/security/token/index.ts @@ -3,7 +3,7 @@ import crypto from "crypto"; export default async function ({database, router, logger}: AppContext) { const token_new = crypto.randomBytes(30).toString('hex'); - const token = process.env.NSM_TOKEN ?? await database.getMetaVal('auth:basic_token', token_new); + const token = process.env.NSM_TOKEN ?? await database.metaRepository.getMetaVal('auth:basic_token', token_new); router.use((req, res, next) => { if (!req.header('Authorization') || req.header('Authorization') != token) { res.status(401).json({ status: 401, message: 'Unauthorized. Invalid \'Authorization\' header.' }); diff --git a/tests/database/manager.test.ts b/tests/database/manager.test.ts index 036ba95..881edd4 100644 --- a/tests/database/manager.test.ts +++ b/tests/database/manager.test.ts @@ -1,7 +1,6 @@ import {afterEach, beforeEach, expect, it} from "@jest/globals"; import {StartedMariaDbContainer} from "@testcontainers/mariadb"; import getDb, {Database} from "@nsm/database"; -import {initClientForTest} from "@nsm/database/manager"; import {PrismaClient} from "@prisma/client"; import {initDbContainerForTest} from "../testUtils"; @@ -15,8 +14,7 @@ beforeEach(async () => { ] = await initDbContainerForTest(); container = container_; - db = getDb(); - initClientForTest(new PrismaClient({ + db = getDb(new PrismaClient({ datasourceUrl: dbUrl_, })); }, 20000); @@ -28,7 +26,7 @@ afterEach(async () => { }, 20000); it("saves image", async () => { - const success = await db.saveImage({ + const success = await db.imageRepository.saveImage({ id: "test-image", templateId: "test-template", hash: "test-hash", @@ -39,7 +37,7 @@ it("saves image", async () => { }); expect(success).toBe(true); - const image = await db.getImage("test-image"); + const image = await db.imageRepository.getImage("test-image"); expect(image).toBeDefined(); expect(image?.id).toBe("test-image"); expect(image?.templateId).toBe("test-template"); @@ -51,7 +49,7 @@ it("saves image", async () => { }); it("finds image by options", async () => { - await db.saveImage({ + await db.imageRepository.saveImage({ id: "test-image", templateId: "test-template", hash: "test-hash", @@ -61,21 +59,21 @@ it("finds image by options", async () => { }, }); - let imagesByOptions = await db.listImagesByOptions("test-template", { + let imagesByOptions = await db.imageRepository.listImagesByOptions("test-template", { option1: "value1", option2: "value2", }); expect(imagesByOptions).toHaveLength(1); expect(imagesByOptions[0]?.id).toBe("test-image"); - imagesByOptions = await db.listImagesByOptions("test-template", { + imagesByOptions = await db.imageRepository.listImagesByOptions("test-template", { option2: "value2", option1: "value1", }); expect(imagesByOptions).toHaveLength(1); expect(imagesByOptions[0]?.id).toBe("test-image"); - imagesByOptions = await db.listImagesByOptions("test-template", { + imagesByOptions = await db.imageRepository.listImagesByOptions("test-template", { option1: "value1", option2: "value2", option3: "value3", diff --git a/tests/engine/image.test.ts b/tests/engine/image.test.ts index e111dd9..af236a7 100644 --- a/tests/engine/image.test.ts +++ b/tests/engine/image.test.ts @@ -6,7 +6,6 @@ import getDb from "@nsm/database"; import {Database} from "@nsm/database"; import {StartedMariaDbContainer} from "@testcontainers/mariadb"; import {initDbContainerForTest} from "../testUtils"; -import {initClientForTest} from "@nsm/database/manager"; import {PrismaClient} from "@prisma/client"; import {processImage} from "@nsm/engine/image"; import {createLogger} from "@nsm/logger"; @@ -31,8 +30,7 @@ beforeAll(async () => { // Just to prevent assertion errors docker_host: "///var/run/docker.sock" }); - db = getDb(); - initClientForTest(new PrismaClient({ + db = getDb(new PrismaClient({ datasourceUrl: dbUrl_, })); }, 20000); diff --git a/tests/engine/middle.test.ts b/tests/engine/middle.test.ts index 457eacb..d9627fc 100644 --- a/tests/engine/middle.test.ts +++ b/tests/engine/middle.test.ts @@ -48,7 +48,7 @@ it("test sets service id in action error", async () => { let customManager: ServiceManager = { ...manager, - async resumeService(serviceId: string) { + async resumeService(_: string) { throw new Error("Failed to resume service"); } }; From 98481ae8f626b47562f3f5c97d7168b86c8c22a2 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 20 Mar 2026 04:24:09 +0100 Subject: [PATCH 26/52] feat: small compilation fixes --- src/database/index.ts | 7 +++++-- src/database/serviceLog.ts | 21 +++++++++++++++++++++ src/router/v1/service/listRoute.ts | 2 +- src/router/v1/status/index.ts | 4 ++-- 4 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 src/database/serviceLog.ts diff --git a/src/database/index.ts b/src/database/index.ts index f447f3a..e8bbdee 100644 --- a/src/database/index.ts +++ b/src/database/index.ts @@ -6,6 +6,7 @@ import * as metaRepository from "./meta"; import * as serviceMetaRepository from "./serviceMeta"; import * as imageRepository from "./image"; import * as sessionRepository from "./session"; +import * as serviceLogRepository from "./serviceLog"; export * from './models'; @@ -21,7 +22,8 @@ export default function (client?: PrismaClient): Database { metaRepository, serviceMetaRepository, imageRepository, - sessionRepository + sessionRepository, + serviceLogRepository ] as unknown as { init: (client: PrismaClient) => void }[] ).forEach(repository => repository.init(client)); @@ -30,6 +32,7 @@ export default function (client?: PrismaClient): Database { metaRepository, serviceMetaRepository, imageRepository, - sessionRepository + sessionRepository, + serviceLogRepository } } \ No newline at end of file diff --git a/src/database/serviceLog.ts b/src/database/serviceLog.ts new file mode 100644 index 0000000..d6d9894 --- /dev/null +++ b/src/database/serviceLog.ts @@ -0,0 +1,21 @@ +import {PrismaClient} from "@prisma/client"; +import {ServiceLogRepository} from "@nsm/database/models"; + +let client: PrismaClient; + +export const init = (client_: PrismaClient) => { + client = client_; +} + +export const createRecords: ServiceLogRepository["createRecords"] = async ( + records +) => { + try { + await client.serviceLogRecord.createMany({ data: records }); + return true; + } catch (e) { + console.error(e); + + return false; + } +} \ No newline at end of file diff --git a/src/router/v1/service/listRoute.ts b/src/router/v1/service/listRoute.ts index 75dd67d..27cb1f2 100644 --- a/src/router/v1/service/listRoute.ts +++ b/src/router/v1/service/listRoute.ts @@ -52,7 +52,7 @@ export default async function ({manager, database}: AppContext): Promise { acc.memory.used += s.memory.used; acc.memory.total += s.memory.total; @@ -56,7 +56,7 @@ export default async function ({manager, appConfig, database}: AppContext): Prom routes: { get: async (req, res) => { const nodeId = appConfig['node_id']; - const all = await database.listPerma(nodeId); + const all = await database.permaRepository.listPerma(nodeId); const [free, size] = await manager.engine.calcHostUsage(); const system = { totalmem: os.totalmem(), From 6989fa8f5ea874eac7a0c2c6dd418a2f68f659c3 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 20 Mar 2026 13:16:55 +0100 Subject: [PATCH 27/52] feat: fixes and refactorings, remove template lookup from service lookup and add templateId instead --- openapi.yml | 8 ++++-- .../migration.sql | 2 ++ prisma/schema.prisma | 2 +- src/app.ts | 2 +- src/database/session.ts | 2 +- src/engine/docker/action/reattach.ts | 16 +++++++++-- src/engine/manager.ts | 27 ++++++++++++------- src/logger.ts | 18 ++++++++----- src/router/v1/service/lookupRoute.ts | 9 +++---- 9 files changed, 57 insertions(+), 29 deletions(-) create mode 100644 prisma/migrations/20260320114548_log_record_message_length/migration.sql diff --git a/openapi.yml b/openapi.yml index e80dbc5..32c0a52 100644 --- a/openapi.yml +++ b/openapi.yml @@ -86,6 +86,9 @@ components: type: "object" description: "The session information about a running service, if running" properties: + id: + type: "string" + description: "The currently active session ID" containerId: type: "string" stats: @@ -132,8 +135,9 @@ components: id: type: "string" description: "The service UID in the system" - template: - $ref: "#/components/schemas/TemplateLookup" + templateId: + type: "string" + description: "The template ID used to create the service" port: type: "integer" format: "int32" diff --git a/prisma/migrations/20260320114548_log_record_message_length/migration.sql b/prisma/migrations/20260320114548_log_record_message_length/migration.sql new file mode 100644 index 0000000..ed3b5e5 --- /dev/null +++ b/prisma/migrations/20260320114548_log_record_message_length/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE `ServiceLogRecord` MODIFY `message` LONGTEXT NOT NULL; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index d2303d8..4c9969d 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -51,7 +51,7 @@ model ServiceLogRecord { timestamp DateTime @default(now()) source ServiceLogSource logLevel String - message String + message String @db.LongText session ServiceSession @relation(fields: [sessionId], references: [id], onDelete: Cascade) } diff --git a/src/app.ts b/src/app.ts index 86d3a05..3557174 100644 --- a/src/app.ts +++ b/src/app.ts @@ -54,7 +54,7 @@ function prepareServiceLogs(appConfig: any, logger: winston.Logger) { } function initGlobalLogger() { - logging.createNewLatest(); + logging.createLatestLogFile(); return logging.createLogger(); } diff --git a/src/database/session.ts b/src/database/session.ts index ff3da66..4a40781 100644 --- a/src/database/session.ts +++ b/src/database/session.ts @@ -13,7 +13,7 @@ export const createSession: SessionRepository["createSession"] = async (serviceI }; try { - await client.serviceSession.create({ data }); + return await client.serviceSession.create({ data }); } catch (e) { console.log(e); diff --git a/src/engine/docker/action/reattach.ts b/src/engine/docker/action/reattach.ts index 89ffbc2..a40d96b 100644 --- a/src/engine/docker/action/reattach.ts +++ b/src/engine/docker/action/reattach.ts @@ -1,8 +1,9 @@ import DockerClient from "dockerode"; -import {DockerServiceEngine, ServiceEngine} from "@nsm/engine"; +import {DockerServiceEngine, ServiceEngine, ServiceLogRecord} from "@nsm/engine"; import {getActionType} from "@nsm/engine/asyncp"; import {currentContext} from "@nsm/app"; import {deleteNetwork as doDeleteNetwork, isInNetwork} from "@nsm/networking/manager"; +import winston from "winston"; async function deleteContainer(id: string, client: DockerClient, options: { deleteNetwork?: boolean }) { try { @@ -38,6 +39,7 @@ async function deleteContainer(id: string, client: DockerClient, options: { dele export default function reattach(self: ServiceEngine, client: DockerClient): ServiceEngine["reattach"] { return async (id, listener) => { const container = client.getContainer(id); + const logger = currentContext.logger; const handleClosed = async () => { await deleteContainer(container.id, client, { deleteNetwork: true }); @@ -55,7 +57,17 @@ export default function reattach(self: ServiceEngine, client: DockerClient): Ser const attachOptions = { stream: true, stdin: true, stdout: true, stderr: true, hijack: true }; const rws = await container.attach(attachOptions); rws.on('data', (data) => { - listener.onMessage?.(data); + try { + data = Buffer.from(data).toString('ascii'); + const record: ServiceLogRecord = { + level: 'info', + message: data + }; + + listener.onMessage?.(record); + } catch (e) { + logger.error("Error producing container output: " + e); + } }); // no-op, keepalive rws.on('end', async () => { if (getActionType(container.id) != 'stop') { diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 3076299..902efb8 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -295,9 +295,10 @@ export type InternalSession = { } export type ServiceInfo = PermaModel & { - optionsRam: number, // From options.ram - optionsCpu: number, // From options.cpu - optionsDisk: number, // From options.disk + optionsRam: number; // From options.ram + optionsCpu: number; // From options.cpu + optionsDisk: number; // From options.disk + session?: ServiceSession; internalSession?: InternalSession } @@ -525,7 +526,8 @@ export async function resumeService(id: string) { ); } } catch (e) { - currentContext.logger.error('Failed to run container for service ' + id, e); + currentContext.logger.error('Failed to run container for service ' + id); + currentContext.logger.error(e); } let success: boolean = false; @@ -542,7 +544,7 @@ export async function resumeService(id: string) { } if (success == true) { - currentContext.logger.info('Service ' + id + ' resumed'); + currentContext.logger.debug('Service ' + id + ' resumed'); callManagerEvent('resume', { id }); } else { errors[id] = new Error('Failed to resume service'); @@ -670,19 +672,26 @@ export function getTemplate(id: string) { return loadTemplate(id); } -export async function getService(from: string, options?: { includeSession?: boolean, otherNodes?: boolean }) { +export async function getService(from: string, options?: { includeSession?: boolean, otherNodes?: boolean }): ReturnType { const data = typeof from === 'string' ? await db.permaRepository.getPerma(from) : from; if (data && (data.nodeId == nodeId || options?.otherNodes === true)) { let session = undefined; + let internalSession = undefined; if (options?.includeSession === true) { - session = getRunningService(data.serviceId)?.internalSession; + const runningService = getRunningService(data.serviceId); + if (runningService) { + session = runningService.session; + internalSession = runningService.internalSession; + } } + return { ...data, optionsRam: data.env.SERVICE_RAM ? Number(data.env.SERVICE_RAM) : 0, optionsCpu: data.env.SERVICE_CPU ? Number(data.env.SERVICE_CPU) : 0, optionsDisk: data.env.SERVICE_DISK ? Number(data.env.SERVICE_DISK) : 0, - session + session, + internalSession } } else { return undefined; @@ -854,7 +863,7 @@ function buildRunListener(session: ActiveServiceSession): RunListener { // unlock a busy action callManagerEvent("stop", { id: serviceId }); - currentContext.logger.info("Service " + serviceId + " stopped"); + currentContext.logger.debug("Service " + serviceId + " stopped"); } }; // Combine collected listeners diff --git a/src/logger.ts b/src/logger.ts index 0ac1525..68aa03c 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,13 +1,14 @@ import winston from "winston"; import fs from "fs"; -const { combine, timestamp, label, printf } = winston.format; +const { combine, timestamp, label, errors, printf } = winston.format; -export function createNewLatest() { +export function createLatestLogFile() { if (fs.existsSync(process.cwd() + '/logs/latest.log')) { const date = new Date(Date.now()).toJSON().slice(2, 10) + '.' + new Date(Date.now()).getHours() + '.' + new Date(Date.now()).getMinutes(); + fs.renameSync(process.cwd() + '/logs/latest.log', process.cwd() + '/logs/' + date + '.log'); } } @@ -17,11 +18,14 @@ export function createLogger(options?: { label?: string }) { return winston.createLogger({ level: debug ? 'debug' : 'info', format: combine( - label({ label: options?.label ?? 'NSM' }), - timestamp(), - printf(({ level, message, label, timestamp }) => { - return `${timestamp} [${label}] ${level}: ${message}`; - }) + errors({ stack: true }), + label({ label: options?.label ?? 'NSM' }), + timestamp(), + printf(({ level, message, label, timestamp, stack }) => { + let row = `${timestamp} [${label}] ${level}: ${message}`; + + return stack ? row + `\n${stack}` : row; + }) ), transports: [ new winston.transports.Console(), diff --git a/src/router/v1/service/lookupRoute.ts b/src/router/v1/service/lookupRoute.ts index b496f3e..5d4272f 100644 --- a/src/router/v1/service/lookupRoute.ts +++ b/src/router/v1/service/lookupRoute.ts @@ -19,21 +19,18 @@ export default async function ({manager}: AppContext): Promise { } else { stats = null; } - const template = manager.getTemplate(service.template); // Build that info res.json({ id: service.serviceId, - template: { - id: service.template, - ...template - }, + templateId: service.template, port: service.port, options: service.options, env: service.env, ...(session ? { session: { + id: service.session.id, ...session, - ...stats, + stats, } } : {}) }).end(); From c29799194b6402f0998fab8cedd91f7ccf1eebf1 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 20 Mar 2026 13:23:56 +0100 Subject: [PATCH 28/52] feat: improve log record timestamps & todos --- src/engine/session.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/engine/session.ts b/src/engine/session.ts index 23ed604..108a819 100644 --- a/src/engine/session.ts +++ b/src/engine/session.ts @@ -1,5 +1,5 @@ import {RunListener} from "@nsm/engine/engine"; -import {CreateLogRecordArgs, Database} from "@nsm/database"; +import {CreateLogRecordArgs, Database, ServiceLogRecordModel} from "@nsm/database"; export interface ServiceSession { id: string; @@ -79,7 +79,7 @@ export const beginServiceSession = async (serviceId: string): Promise { - const logRecordsBulk: CreateLogRecordArgs[] = []; + const logRecordsBulk: Omit[] = []; const MAX_BATCH_SIZE = 50; const DEBOUNCE_MS = 500; @@ -133,7 +133,7 @@ const debounceBulkPush = () => { // Also return flush to be able to forcibly push logs into the database flush, debounce: (log: CreateLogRecordArgs) => { - logRecordsBulk.push(log); + logRecordsBulk.push({ ...log, timestamp: new Date(Date.now()) }); // If we reached the bulk size limit, flush immediately if (logRecordsBulk.length >= MAX_BATCH_SIZE) { @@ -154,4 +154,6 @@ const debounceBulkPush = () => { // TODO: get service session +// TODO: get service session history for service + // TODO: list service session logs \ No newline at end of file From 8060b2782227e067b2cc94965f667fa0ce566c7c Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 20 Mar 2026 21:43:58 +0100 Subject: [PATCH 29/52] feat: service sessions ep --- TODO.txt | 2 +- openapi.yml | 4 +- .../migration.sql | 2 + prisma/schema.prisma | 1 + src/database/models.ts | 35 ++++- src/database/serviceLog.ts | 30 ++++- src/database/session.ts | 28 ++++ src/engine/manager.ts | 125 +++++++++--------- src/engine/session.ts | 10 +- src/router/index.ts | 5 +- src/router/v1/index.ts | 26 ++-- src/router/v1/service/sessionsRoute.ts | 40 ++++++ 12 files changed, 222 insertions(+), 86 deletions(-) create mode 100644 prisma/migrations/20260320184831_service_session_startedat/migration.sql create mode 100644 src/router/v1/service/sessionsRoute.ts diff --git a/TODO.txt b/TODO.txt index dbc5ea8..73d1bda 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,4 +1,4 @@ -vyřešit logging u servis a logování změn stavů takže i stavy služeb +vyřešit logging u servis a logování změn stavů takže i stavy služeb - už jen stavy služeb abstrahovat template management a udělat další sources jako například používání images z docker registry předělat usages monitoring na nějaký standardizovaný formát/způsob pořešit /resources/config.yml aby měl víc možností a možná byl uložený jinde v systému podle platformy diff --git a/openapi.yml b/openapi.yml index 32c0a52..026f11e 100644 --- a/openapi.yml +++ b/openapi.yml @@ -537,4 +537,6 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/Result" \ No newline at end of file + $ref: "#/components/schemas/Result" + +# TODO: /v1/service/{serviceId}/sessions \ No newline at end of file diff --git a/prisma/migrations/20260320184831_service_session_startedat/migration.sql b/prisma/migrations/20260320184831_service_session_startedat/migration.sql new file mode 100644 index 0000000..9bf4d35 --- /dev/null +++ b/prisma/migrations/20260320184831_service_session_startedat/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE `ServiceSession` ADD COLUMN `startedAt` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 4c9969d..a92fc77 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -43,6 +43,7 @@ model ServiceSession { serviceId String service Service @relation(fields: [serviceId], references: [serviceId], onDelete: Cascade) logs ServiceLogRecord[] + startedAt DateTime @default(now()) } model ServiceLogRecord { diff --git a/src/database/models.ts b/src/database/models.ts index c2355a3..1645a30 100644 --- a/src/database/models.ts +++ b/src/database/models.ts @@ -34,15 +34,46 @@ export interface ImageRepository { export interface SessionRepository { createSession(serviceId: string): Promise; - // TODO: store session log record, list session log records etc + + listSessions(args: ListSessionsArgs): Promise; +} + +export type ListSessionsArgs = { + filter?: { + serviceId?: string; + } + sort?: { + by?: 'startedAt' + direction?: 'asc' | 'desc' + } + page?: { + index: number; + size: number; + } } export interface ServiceLogRepository { createRecords(records: CreateLogRecordArgs[]): Promise; + + listRecords(args: ListRecordsArgs): Promise; } export type CreateLogRecordArgs = Omit; +export type ListRecordsArgs = { + filter?: { + sessionId?: string; + } + sort?: { + by?: 'timestamp', + direction?: 'asc' | 'desc' + } + page?: { + index: number; + size: number; + } +} + export type PermaModel = { serviceId: string; template: string; @@ -79,7 +110,7 @@ export type ServiceSessionModel = { } export type ServiceLogRecordModel = { - id: number; + id: bigint; sessionId: string; source: 'ENGINE' | 'CONTAINER' timestamp: Date; diff --git a/src/database/serviceLog.ts b/src/database/serviceLog.ts index d6d9894..f79c6d7 100644 --- a/src/database/serviceLog.ts +++ b/src/database/serviceLog.ts @@ -1,4 +1,4 @@ -import {PrismaClient} from "@prisma/client"; +import {Prisma, PrismaClient} from "@prisma/client"; import {ServiceLogRepository} from "@nsm/database/models"; let client: PrismaClient; @@ -18,4 +18,32 @@ export const createRecords: ServiceLogRepository["createRecords"] = async ( return false; } +} + +export const listRecords: ServiceLogRepository["listRecords"] = async (args) => { + const { + filter, + sort, + page + } = args; + + const query: Prisma.ServiceLogRecordFindManyArgs = {}; + if (filter?.sessionId) { + query.where = filter; + } + query.orderBy = { + [sort?.by ?? "timestamp"]: sort?.direction ?? "desc" + }; + if (page) { + query.skip = page.index * page.size; + query.take = page.size; + } + + try { + return await client.serviceLogRecord.findMany(query); + } catch (e) { + console.log(e); + + return undefined; + } } \ No newline at end of file diff --git a/src/database/session.ts b/src/database/session.ts index 4a40781..2d33d6e 100644 --- a/src/database/session.ts +++ b/src/database/session.ts @@ -17,6 +17,34 @@ export const createSession: SessionRepository["createSession"] = async (serviceI } catch (e) { console.log(e); + return undefined; + } +} + +export const listSessions: SessionRepository["listSessions"] = async (args) => { + const { + filter, + sort, + page + } = args; + + const query: Prisma.ServiceSessionFindManyArgs = {}; + if (filter?.serviceId) { + query.where = filter; + } + query.orderBy = { + [sort?.by ?? "startedAt"]: sort?.direction ?? "desc" + }; + if (page) { + query.skip = page.index * page.size; + query.take = page.size; + } + + try { + return await client.serviceSession.findMany(query); + } catch (e) { + console.log(e); + return undefined; } } \ No newline at end of file diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 902efb8..d58185b 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -369,6 +369,51 @@ export async function init(db_: Database, appConfig_: any, logger: winston.Logge logger.info(`Using engine: ${engine.name}`); } +/** + * Reattach to containers that are still running from the previous session. + * This may happen if NSM was force-stopped and not properly cleared up resources. + * + * @param logger The logger to use + */ +async function reattachStaleContainers(logger: winston.Logger) { + const running = await engine.listRunning(Filters.node(nodeId)) + .then(containerIds => containerIds + // Filter out those that we have already started in this session, just in case + // this was started more than once a session + .filter(id => !started.find(runningService => runningService.internalSession.containerId === id))); + + for (let containerId of running) { + const labels = await engine.getLabels(containerId); + if (!labels[StandardLabel.ServiceId]) { + // The container was in the running list, but does not have the required labels + // Should not happen, but just in case + logger.warn(`Found a running container with id ${containerId} that does not have a service id label, stopping.`); + + await engine.stop(containerId); + } + + const serviceId = labels[StandardLabel.ServiceId]; + + // We must begin a new session since the previous was interrupted + const session = await beginServiceSession(serviceId); + // Reattach and watch the container + await engine.reattach(containerId, buildRunListener(session)); + + // Save session in-memory + const info: RunningService = { + id: serviceId, + session, + internalSession: { + containerId + } + }; + started.push(info); + logger.info(`Reattached container ${containerId} for service ${serviceId}`); + } + + await new Promise((resolve) => whenUnlockedAll(() => resolve(null))); +} + export async function expandEngine(exp?: T): Promise { if (exp) { if (!engine && (!currentContext || !currentContext.appConfig)) { @@ -407,7 +452,6 @@ export async function createService(template: string, options: Options) { ...(options.meta ?? {}), ...(serviceSettings.meta ?? {}) }; - if (!meta || !meta.stopCmd) { throw new _InternalError('Invalid template meta for ' + template); } @@ -797,30 +841,6 @@ export { whenUnlocked } -async function reqExists(id: string) { - const perma = await db.permaRepository.getPerma(id); - if (!perma) { - throw new _InternalError("Service not found.", 3); - } - - return perma; -} - -function reqRunning(id: string) { - const session = getRunningService(id); - if (!session) { - throw new _InternalError("This service is not running.", 2); - } - - return session; -} - -function reqNotRunning(id: string) { - if (isRunning(id)) { - throw new _InternalError('Already running.', 2); - } -} - function clearRunningServiceIfExists(id: string) { const service = getRunningService(id); @@ -884,47 +904,26 @@ async function getPermaModel(id: string) { return perma_; } -/** - * Reattach to containers that are still running from the previous session. - * This may happen if NSM was force-stopped and not properly cleared up resources. - * - * @param logger The logger to use - */ -async function reattachStaleContainers(logger: winston.Logger) { - const running = await engine.listRunning(Filters.node(nodeId)) - .then(containerIds => containerIds - // Filter out those that we have already started in this session, just in case - // this was started more than once a session - .filter(id => !started.find(runningService => runningService.internalSession.containerId === id))); - - for (let containerId of running) { - const labels = await engine.getLabels(containerId); - if (!labels[StandardLabel.ServiceId]) { - // The container was in the running list, but does not have the required labels - // Should not happen, but just in case - logger.warn(`Found a running container with id ${containerId} that does not have a service id label, stopping.`); +async function reqExists(id: string) { + const perma = await db.permaRepository.getPerma(id); + if (!perma) { + throw new _InternalError("Service not found.", 3); + } - await engine.stop(containerId); - } + return perma; +} - const serviceId = labels[StandardLabel.ServiceId]; +function reqRunning(id: string) { + const session = getRunningService(id); + if (!session) { + throw new _InternalError("This service is not running.", 2); + } - // We must begin a new session since the previous was interrupted - const session = await beginServiceSession(serviceId); - // Reattach and watch the container - await engine.reattach(containerId, buildRunListener(session)); + return session; +} - // Save session in-memory - const info: RunningService = { - id: serviceId, - session, - internalSession: { - containerId - } - }; - started.push(info); - logger.info(`Reattached container ${containerId} for service ${serviceId}`); +function reqNotRunning(id: string) { + if (isRunning(id)) { + throw new _InternalError('Already running.', 2); } - - await new Promise((resolve) => whenUnlockedAll(() => resolve(null))); } \ No newline at end of file diff --git a/src/engine/session.ts b/src/engine/session.ts index 108a819..974c774 100644 --- a/src/engine/session.ts +++ b/src/engine/session.ts @@ -1,5 +1,5 @@ import {RunListener} from "@nsm/engine/engine"; -import {CreateLogRecordArgs, Database, ServiceLogRecordModel} from "@nsm/database"; +import {CreateLogRecordArgs, Database, ListRecordsArgs, ListSessionsArgs, ServiceLogRecordModel} from "@nsm/database"; export interface ServiceSession { id: string; @@ -154,6 +154,10 @@ const debounceBulkPush = () => { // TODO: get service session -// TODO: get service session history for service +export const listSessions = async (args: ListSessionsArgs) => { + return db.sessionRepository.listSessions(args); +} -// TODO: list service session logs \ No newline at end of file +export const listSessionLogs = async (args: ListRecordsArgs) => { + return db.serviceLogRepository.listRecords(args); +} \ No newline at end of file diff --git a/src/router/index.ts b/src/router/index.ts index b292f13..30e9a95 100644 --- a/src/router/index.ts +++ b/src/router/index.ts @@ -12,7 +12,6 @@ type RouterInit = (context: AppContext) => Promise; // Load API by version async function api(ver: string, context: AppContext, routes: RouterInit[]) { - context.logger.info(`API ${ver} routes`); const router = Router(); router.use(json()); if (context.debug) { @@ -35,7 +34,7 @@ async function api(ver: string, context: AppContext, routes: RouterInit[]) { // used specifically for this API version const handler = await init({ ...context, router }); let reg = false; - // + for (const method of ['get', 'post', 'put', 'delete']) { if (handler.routes[method]) { // Register handler to express @@ -47,7 +46,7 @@ async function api(ver: string, context: AppContext, routes: RouterInit[]) { } } if (reg) { - context.logger.info(`-- ${handler.url}`); + context.logger.debug(`Registered route ${handler.url}`); } } context.router.use(`/${ver}`, router); diff --git a/src/router/v1/index.ts b/src/router/v1/index.ts index d198fff..23b6576 100644 --- a/src/router/v1/index.ts +++ b/src/router/v1/index.ts @@ -9,18 +9,20 @@ import rebootRoute from "./service/rebootRoute"; import powerStatusRoute from "./service/powerStatusRoute"; import stopCmdRoute from "@nsm/router/v1/service/stopCmdRoute"; import optionsRoute from "@nsm/router/v1/service/optionsRoute"; +import sessionsRoute from "@nsm/router/v1/service/sessionsRoute"; export default [ - // v1 routes - statusRoute, - createRoute, - lookupRoute, - deleteRoute, - resumeRoute, - rebootRoute, - stopCmdRoute, - stopRoute, - powerStatusRoute, - optionsRoute, - listRoute, + // v1 routes + statusRoute, + createRoute, + lookupRoute, + deleteRoute, + resumeRoute, + rebootRoute, + stopCmdRoute, + stopRoute, + powerStatusRoute, + optionsRoute, + listRoute, + sessionsRoute ] \ No newline at end of file diff --git a/src/router/v1/service/sessionsRoute.ts b/src/router/v1/service/sessionsRoute.ts new file mode 100644 index 0000000..ee3cfb8 --- /dev/null +++ b/src/router/v1/service/sessionsRoute.ts @@ -0,0 +1,40 @@ +import {AppContext} from "@nsm/app"; +import {RouterHandler} from "@nsm/router"; +import {checkServiceExists} from "@nsm/router/util/preconditions"; +import {listSessions} from "@nsm/engine/session"; +import {ListSessionsArgs} from "@nsm/database"; + +export default async function(ctx: AppContext): Promise { + return { + url: '/service/:id/sessions', + routes: { + get: async (req, res) => { + const id = req.params.id; + const pageIndex = req.query.pageIndex ? Number(req.query.pageIndex) : 0; + const pageSize = req.query.pageSize ? Number(req.query.pageSize) : 10; + + if (!await checkServiceExists(id, ctx.manager, res)) { + return; + } + + const args: ListSessionsArgs = { + filter: { + serviceId: id + }, + sort: { + by: "startedAt", + direction: "desc" + }, + page: { + index: pageIndex, + size: pageSize + } + }; + const sessionIds = await listSessions(args) + .then(sessions => sessions.map(session => session.id)); + + res.status(200).json({ sessions: sessionIds }); + } + } + } +} \ No newline at end of file From 80fe5e0d6271f70b4592acd4ddcb2ef33993a989 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 20 Mar 2026 22:05:35 +0100 Subject: [PATCH 30/52] feat: service state --- openapi.yml | 7 +++++++ src/engine/docker/action/reattach.ts | 2 +- src/engine/docker/action/run.ts | 8 +++++--- src/engine/engine.ts | 11 +++++++++++ src/engine/manager.ts | 23 +++++++++++++++++++++-- src/router/v1/service/lookupRoute.ts | 1 + 6 files changed, 46 insertions(+), 6 deletions(-) diff --git a/openapi.yml b/openapi.yml index 026f11e..28085bc 100644 --- a/openapi.yml +++ b/openapi.yml @@ -138,6 +138,13 @@ components: templateId: type: "string" description: "The template ID used to create the service" + state: + type: string + description: "The current state of the service. One of: 'RUNNING', 'BUILDING', 'STOPPED'." + enum: + - "RUNNING" + - "BUILDING" + - "STOPPED" port: type: "integer" format: "int32" diff --git a/src/engine/docker/action/reattach.ts b/src/engine/docker/action/reattach.ts index a40d96b..cdc9ac3 100644 --- a/src/engine/docker/action/reattach.ts +++ b/src/engine/docker/action/reattach.ts @@ -82,6 +82,6 @@ export default function reattach(self: ServiceEngine, client: DockerClient): Ser }); (self as DockerServiceEngine).rws[container.id] = rws; - await listener.onStateChange?.({ id: 'watching_changes', description: 'Watching changes' }); + await listener.onStateChange?.({ id: 'watching_changes', description: 'Watching changes', ready: true }); } } \ No newline at end of file diff --git a/src/engine/docker/action/run.ts b/src/engine/docker/action/run.ts index dcf9cb6..cc22b4d 100644 --- a/src/engine/docker/action/run.ts +++ b/src/engine/docker/action/run.ts @@ -90,17 +90,19 @@ async function prepareContainer( return container; } -const createState = (id: string, description: string): ServiceState => { +const createState = (id: string, description: string, ready?: boolean): ServiceState => { return { id, - description + description, + ready: ready ?? false } }; const createErrorState = (description: string): ServiceState => { return { id: 'error', - description + description, + ready: false } } diff --git a/src/engine/engine.ts b/src/engine/engine.ts index df3dc82..46b1f05 100644 --- a/src/engine/engine.ts +++ b/src/engine/engine.ts @@ -54,8 +54,19 @@ export type ServiceLogRecord = { } export type ServiceState = { + /** + * Internal ID of the state. + */ id: string; + /** + * A brief description of the state, for display purposes. + */ description: string; + /** + * Whether the service is ready to accept commands and connections + * in this state, thus is running. + */ + ready: boolean; } export type RunListener = { diff --git a/src/engine/manager.ts b/src/engine/manager.ts index d58185b..14d63c6 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -298,10 +298,13 @@ export type ServiceInfo = PermaModel & { optionsRam: number; // From options.ram optionsCpu: number; // From options.cpu optionsDisk: number; // From options.disk + state: State; session?: ServiceSession; - internalSession?: InternalSession + internalSession?: InternalSession; } +export type State = 'RUNNING' | 'BUILDING' | 'STOPPED'; + // 1 = unknown, 2 = conflict, 3 = not found export type StatusCode = 1 | 2 | 3; @@ -332,6 +335,7 @@ function settings(template: string) { const errors = {}; // Service IDs that are currently running const started: RunningService[] = []; +const startedStates: Map = new Map(); const evtHandlers: Map[]> = new Map(); ["push", "splice"].forEach(funcName => { @@ -734,6 +738,7 @@ export async function getService(from: string, options?: { includeSession?: bool optionsRam: data.env.SERVICE_RAM ? Number(data.env.SERVICE_RAM) : 0, optionsCpu: data.env.SERVICE_CPU ? Number(data.env.SERVICE_CPU) : 0, optionsDisk: data.env.SERVICE_DISK ? Number(data.env.SERVICE_DISK) : 0, + state: getServiceState(data.serviceId), session, internalSession } @@ -762,7 +767,7 @@ export async function stopRunning() { new Promise((resolve) => { whenUnlocked(id, () => { stopService(id) - .catch(e => console.log(e)) + .catch(e => currentContext.logger.error(e)) .then(() => { whenUnlocked(id, () => resolve(null)); }); @@ -876,8 +881,12 @@ function buildRunListener(session: ActiveServiceSession): RunListener { // The internal run listener of this manager const internalRunListener: RunListener = { + onStateChange: (state) => { + startedStates.set(serviceId, state.ready ? 'RUNNING' : 'BUILDING'); + }, onClose: async () => { clearRunningServiceIfExists(serviceId); + startedStates.delete(serviceId); // Call stop event on the manager for the stopService() to potentially // unlock a busy action @@ -894,6 +903,16 @@ function buildRunListener(session: ActiveServiceSession): RunListener { ]) } +/** + * Returns the local service state managed by this manager. + * + * @param id The id of the service. + * @returns The state of the service + */ +function getServiceState(id: string) { + return startedStates.get(id) ?? 'STOPPED'; +} + async function getPermaModel(id: string) { const perma_ = await db.permaRepository.getPerma(id); if (!perma_) { diff --git a/src/router/v1/service/lookupRoute.ts b/src/router/v1/service/lookupRoute.ts index 5d4272f..d53857f 100644 --- a/src/router/v1/service/lookupRoute.ts +++ b/src/router/v1/service/lookupRoute.ts @@ -23,6 +23,7 @@ export default async function ({manager}: AppContext): Promise { res.json({ id: service.serviceId, templateId: service.template, + state: service.state, port: service.port, options: service.options, env: service.env, From 2db42cc5317986b022a915af539b2ebd43294b44 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 20 Mar 2026 22:08:18 +0100 Subject: [PATCH 31/52] refactor --- TODO.txt | 2 +- src/engine/session.ts | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/TODO.txt b/TODO.txt index 73d1bda..f74a4d0 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,6 +1,6 @@ -vyřešit logging u servis a logování změn stavů takže i stavy služeb - už jen stavy služeb abstrahovat template management a udělat další sources jako například používání images z docker registry předělat usages monitoring na nějaký standardizovaný formát/způsob pořešit /resources/config.yml aby měl víc možností a možná byl uložený jinde v systému podle platformy port retrieval strategy pro spouštění servis, aby nebyla jenom random ale i jiné strategies udělat nějaký auto packaging do balíčků na githubu jako bundle aby se dal spustit jako systémový proces v systemctl například +restart policy \ No newline at end of file diff --git a/src/engine/session.ts b/src/engine/session.ts index 974c774..4facd55 100644 --- a/src/engine/session.ts +++ b/src/engine/session.ts @@ -37,24 +37,20 @@ export const beginServiceSession = async (serviceId: string): Promise { - const log: CreateLogRecordArgs = { + pushRecord({ sessionId: session.id, source: 'ENGINE', logLevel: 'INFO', message: state.description - } - - pushRecord(log); + }); }, onMessage: async (record) => { - const log: CreateLogRecordArgs = { + pushRecord({ sessionId: session.id, source: 'CONTAINER', logLevel: record.level.toUpperCase(), message: record.message - } - - pushRecord(log); + }); }, onClose: async () => { // Push remaining logs now From b1bde6545e7c55d28c403a28d6de1ae91d9615a4 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 20 Mar 2026 22:13:27 +0100 Subject: [PATCH 32/52] todo --- src/engine/session.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/session.ts b/src/engine/session.ts index 4facd55..aaaaeb4 100644 --- a/src/engine/session.ts +++ b/src/engine/session.ts @@ -154,6 +154,6 @@ export const listSessions = async (args: ListSessionsArgs) => { return db.sessionRepository.listSessions(args); } -export const listSessionLogs = async (args: ListRecordsArgs) => { +export const listSessionLogs = async (args: ListRecordsArgs) => { // TODO: implement this in ep return db.serviceLogRepository.listRecords(args); } \ No newline at end of file From 1d9ccfd6b07ea9e4d52b0d219571a50ddd0d5759 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 20 Mar 2026 23:51:09 +0100 Subject: [PATCH 33/52] feat: session logs ep --- src/app.ts | 4 +++ src/engine/session.ts | 26 +++++++++++--- src/router/v1/index.ts | 4 ++- src/router/v1/service/sessionsRoute.ts | 5 +-- src/router/v1/session/sessionLogsRoute.ts | 41 +++++++++++++++++++++++ 5 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 src/router/v1/session/sessionLogsRoute.ts diff --git a/src/app.ts b/src/app.ts index 3557174..ce7eb37 100644 --- a/src/app.ts +++ b/src/app.ts @@ -16,12 +16,14 @@ import createDbManager from '@nsm/database'; import loadSecurity from "@nsm/security"; import * as r from "@nsm/configuration/resources"; import * as manager from "@nsm/engine/manager"; +import * as sessionManager from "@nsm/engine/session"; import * as logging from "./logger"; import winston from "winston"; import {Application} from "express-ws"; import fs from "fs"; import isInsideContainer from "@nsm/lib/isInsideContainer"; import {middleLayer} from "@nsm/engine/middle"; +import {SessionManager} from "@nsm/engine/session"; export type AppBootContext = AppContext & { steps: any }; @@ -29,6 +31,7 @@ export type AppBootContext = AppContext & { steps: any }; export type AppContext = { router: Router; manager: ServiceManager; + sessionManager: SessionManager; database: Database; appConfig: any; logger: winston.Logger; @@ -104,6 +107,7 @@ export const init = async (router: Application, options?: AppBootOptions): Promi const ctx = currentContext = { router, manager: managerForUnsafeUse(), + sessionManager, database, appConfig, logger, diff --git a/src/engine/session.ts b/src/engine/session.ts index aaaaeb4..e51b9a5 100644 --- a/src/engine/session.ts +++ b/src/engine/session.ts @@ -1,5 +1,21 @@ import {RunListener} from "@nsm/engine/engine"; -import {CreateLogRecordArgs, Database, ListRecordsArgs, ListSessionsArgs, ServiceLogRecordModel} from "@nsm/database"; +import { + CreateLogRecordArgs, + Database, + ListRecordsArgs, + ListSessionsArgs, + ServiceLogRecordModel, + ServiceSessionModel +} from "@nsm/database"; + +export interface SessionManager { + init(db: Database): void; + + beginServiceSession(serviceId: string): Promise; + + listSessions(args: ListSessionsArgs): Promise; + listSessionLogs(args: ListRecordsArgs): Promise; +} export interface ServiceSession { id: string; @@ -26,7 +42,9 @@ export const init = (db_: Database) => { * @param serviceId The ID of the service for which to begin a session. * @return An object representing the active service session. */ -export const beginServiceSession = async (serviceId: string): Promise => { +export const beginServiceSession: SessionManager["beginServiceSession"] = async ( + serviceId: string +): Promise => { let session = await db.sessionRepository.createSession(serviceId); // Debounce the push in bulk to prevent database overhead @@ -150,10 +168,10 @@ const debounceBulkPush = () => { // TODO: get service session -export const listSessions = async (args: ListSessionsArgs) => { +export const listSessions: SessionManager["listSessions"] = async (args: ListSessionsArgs) => { return db.sessionRepository.listSessions(args); } -export const listSessionLogs = async (args: ListRecordsArgs) => { // TODO: implement this in ep +export const listSessionLogs: SessionManager["listSessionLogs"] = async (args: ListRecordsArgs) => { return db.serviceLogRepository.listRecords(args); } \ No newline at end of file diff --git a/src/router/v1/index.ts b/src/router/v1/index.ts index 23b6576..8df235c 100644 --- a/src/router/v1/index.ts +++ b/src/router/v1/index.ts @@ -10,6 +10,7 @@ import powerStatusRoute from "./service/powerStatusRoute"; import stopCmdRoute from "@nsm/router/v1/service/stopCmdRoute"; import optionsRoute from "@nsm/router/v1/service/optionsRoute"; import sessionsRoute from "@nsm/router/v1/service/sessionsRoute"; +import sessionLogsRoute from "@nsm/router/v1/session/sessionLogsRoute"; export default [ // v1 routes @@ -24,5 +25,6 @@ export default [ powerStatusRoute, optionsRoute, listRoute, - sessionsRoute + sessionsRoute, + sessionLogsRoute ] \ No newline at end of file diff --git a/src/router/v1/service/sessionsRoute.ts b/src/router/v1/service/sessionsRoute.ts index ee3cfb8..8426652 100644 --- a/src/router/v1/service/sessionsRoute.ts +++ b/src/router/v1/service/sessionsRoute.ts @@ -1,7 +1,6 @@ import {AppContext} from "@nsm/app"; import {RouterHandler} from "@nsm/router"; import {checkServiceExists} from "@nsm/router/util/preconditions"; -import {listSessions} from "@nsm/engine/session"; import {ListSessionsArgs} from "@nsm/database"; export default async function(ctx: AppContext): Promise { @@ -10,6 +9,7 @@ export default async function(ctx: AppContext): Promise { routes: { get: async (req, res) => { const id = req.params.id; + const pageIndex = req.query.pageIndex ? Number(req.query.pageIndex) : 0; const pageSize = req.query.pageSize ? Number(req.query.pageSize) : 10; @@ -30,7 +30,8 @@ export default async function(ctx: AppContext): Promise { size: pageSize } }; - const sessionIds = await listSessions(args) + const sessionIds = await ctx.sessionManager + .listSessions(args) .then(sessions => sessions.map(session => session.id)); res.status(200).json({ sessions: sessionIds }); diff --git a/src/router/v1/session/sessionLogsRoute.ts b/src/router/v1/session/sessionLogsRoute.ts new file mode 100644 index 0000000..dc111bc --- /dev/null +++ b/src/router/v1/session/sessionLogsRoute.ts @@ -0,0 +1,41 @@ +import {AppContext} from "@nsm/app"; +import {RouterHandler} from "@nsm/router"; +import {ListRecordsArgs} from "@nsm/database"; + +export default async function(ctx: AppContext): Promise { + return { + url: '/session/:id/logs', + routes: { + get: async (req, res) => { + const id = req.params.id; + + const pageIndex = req.query.pageIndex ? Number(req.query.pageIndex) : 0; + const pageSize = req.query.pageSize ? Number(req.query.pageSize) : 10; + + // Use pagination only if it was requested by params + const page = req.query.pageIndex || req.query.pageSize + ? ( + { + index: pageIndex, + size: pageSize + } + ) + : undefined; + + const args: ListRecordsArgs = { + filter: { + sessionId: id + }, + sort: { + by: "timestamp", + direction: "asc" + }, + page + }; + const logs = await ctx.sessionManager.listSessionLogs(args); + + res.status(200).json({ logs }); + } + } + } +} \ No newline at end of file From 26613a59c6ce05d554f9e336db83fb784e051a08 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 20 Mar 2026 23:52:19 +0100 Subject: [PATCH 34/52] todo --- openapi.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openapi.yml b/openapi.yml index 28085bc..8ec541e 100644 --- a/openapi.yml +++ b/openapi.yml @@ -546,4 +546,5 @@ paths: schema: $ref: "#/components/schemas/Result" -# TODO: /v1/service/{serviceId}/sessions \ No newline at end of file +# TODO: /v1/service/{serviceId}/sessions +# TODO: /v1/session/{sessionId}/logs \ No newline at end of file From 131cd6021f0a38dc868aeeb54a7e4254a8245e6b Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 20 Mar 2026 23:58:48 +0100 Subject: [PATCH 35/52] todo --- TODO.txt | 1 + src/engine/manager.ts | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/TODO.txt b/TODO.txt index f74a4d0..7cfc0b3 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,3 +1,4 @@ +fixnout stop flow pomocí stop cmd abstrahovat template management a udělat další sources jako například používání images z docker registry předělat usages monitoring na nějaký standardizovaný formát/způsob pořešit /resources/config.yml aby měl víc možností a možná byl uložený jinde v systému podle platformy diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 14d63c6..7130bc2 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -784,10 +784,9 @@ export async function waitForBusyAction(id: string) { whenUnlocked(id, (_, __, err) => { if (err) { reject(err); - return; + } else { + resolve(null); } - - resolve(null); }); } ); From fe649dad8cc87eaafa6a367dd7ac2fefd3576192 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 21 Mar 2026 00:03:37 +0100 Subject: [PATCH 36/52] feat: change log level for base dir watcher --- src/engine/monitoring/templateDirWatcher.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/engine/monitoring/templateDirWatcher.ts b/src/engine/monitoring/templateDirWatcher.ts index 1fc74eb..447be99 100644 --- a/src/engine/monitoring/templateDirWatcher.ts +++ b/src/engine/monitoring/templateDirWatcher.ts @@ -55,7 +55,7 @@ const watchBaseDir = (logger: winston.Logger) => { watcher.on("addDir", async (path_) => { const template = path.basename(path_); if (template && !watchers.has(template)) { - logger.info(`New template directory detected: ${template}. Starting to watch for changes...`); + logger.debug(`New template directory detected: ${template}. Starting to watch for changes...`); await watchTemplateDir(template); } @@ -65,7 +65,7 @@ const watchBaseDir = (logger: winston.Logger) => { if (template && watchers.has(template)) { const tWatcher = watchers.get(template); if (tWatcher) { - logger.info( + logger.debug( `Template directory removed: ${template}. Stopping watch and removing hash from cache...`); await tWatcher.close(); @@ -110,6 +110,11 @@ const watchTemplateDir = async (template: string) => { watchers.set(template, watcher); } +/** + * Recalculates the hash of a template directory and updates the cache. + * + * @param template The name of the template to recalculate the hash for. + */ const recalculateTemplateHash = async (template: string) => { const dir = buildDir(template); const excluded = getFilteredPaths(dir); From 67d6aac684d8da98ac530af6367f6b93404335cd Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 21 Mar 2026 01:29:44 +0100 Subject: [PATCH 37/52] refactor --- package.json | 1 + src/engine/docker/action/build.ts | 43 ++++++++++++++++--------------- src/engine/engine.ts | 32 ++++++++++++++++------- src/engine/image.ts | 21 ++++++++++----- src/engine/manager.ts | 2 +- 5 files changed, 61 insertions(+), 38 deletions(-) diff --git a/package.json b/package.json index cf8f79b..19688ce 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "crypto": "^1.0.1", "dockerode": "^4.0.2", "dotenv": "^16.4.5", + "env-paths": "^4.0.0", "express": "^4.18.2", "express-fileupload": "^1.5.0", "folder-hash": "^4.1.1", diff --git a/src/engine/docker/action/build.ts b/src/engine/docker/action/build.ts index 267cfed..9accc22 100644 --- a/src/engine/docker/action/build.ts +++ b/src/engine/docker/action/build.ts @@ -3,37 +3,40 @@ import fs from "fs"; import path from "path"; import tar from "tar"; import {currentContext, currentContext as ctx} from "../../../app"; -import {ServiceEngine} from "@nsm/engine"; +import {MessageListener, ServiceEngine} from "@nsm/engine"; import {clock} from "@nsm/util/clock"; import {Worker} from "worker_threads"; import {getRootFilesFiltered} from "@nsm/engine/ignore"; +import {Paths} from "env-paths"; async function prepareImage( - options: { + args: { imageName: string|undefined, client: DockerClient, arDir: string, buildDir: string, - env: any - }, - logger = currentContext.logger, + env: any, + messageListener?: MessageListener + } ): Promise { let { - imageName, - client, - arDir, - buildDir, - env - } = options; + imageName, + client, + arDir, + buildDir, + env, + messageListener + } = args; if (!imageName) { // Generate an unique image name imageName = "nsm-template-" + path.basename(buildDir) + '-' + Date.now() + ':latest'; // TODO: better unique name generation, maybe hash of the build context? } - // TODO: make this in temp folder - const archive = arDir + '/' + imageName + '.tar'; + // temp archive + const archive = path.join(arDir, imageName + '.tar'); try { + // try to delete if there is already a file fs.unlinkSync(archive); } catch (e) { if (!e.message.includes('ENOENT')) { @@ -54,7 +57,8 @@ async function prepareImage( const msgHandler = (msg: any) => { if (Array.isArray(msg)) { msg.forEach(m => { - // TODO: log message line into service logs? + // Push service log record + // TODO: publish log record using messageListener }); } else { // Final message, resolve the promise with the image tag. @@ -118,15 +122,12 @@ async function prepareImage( ); } -export default function (client: DockerClient): ServiceEngine['build'] { - const arDir = process.cwd() + path.sep + "archives"; - if (!fs.existsSync(arDir)) { - fs.mkdirSync(arDir); - } +export default function (client: DockerClient, paths: Paths): ServiceEngine['build'] { + const arDir = path.join(paths.temp, "archives"); - return async (imageId, buildDir, options) => { + return async (imageId, buildDir, options, messageListener) => { const imageBuildClock = clock(); - const imageTag = await prepareImage({imageName: imageId, client, arDir, buildDir, env: options}); + const imageTag = await prepareImage({imageName: imageId, client, arDir, buildDir, env: options, messageListener}); currentContext.logger.info('Image built in ' + imageBuildClock.durFromCreation() + 'ms'); return imageTag; diff --git a/src/engine/engine.ts b/src/engine/engine.ts index 46b1f05..e5f97f5 100644 --- a/src/engine/engine.ts +++ b/src/engine/engine.ts @@ -50,7 +50,7 @@ export type ContainerFilter = { export type ServiceLogRecord = { level: 'error' | 'info'; - message: string, + message: string; } export type ServiceState = { @@ -69,20 +69,22 @@ export type ServiceState = { ready: boolean; } -export type RunListener = { +export type MessageListener = { /** - * Called when the container progress changes state. + * Called when there is a message from the container, with the message. * - * @param state The new state. + * @param message The message from the container */ - onStateChange?: (state: ServiceState) => Promise|void; + onMessage?: (message: ServiceLogRecord) => Promise|void; +} +export type RunListener = MessageListener & { /** - * Called when there is a message from the container, with the message. + * Called when the container progress changes state. * - * @param message The message from the container + * @param state The new state. */ - onMessage?: (message: ServiceLogRecord) => Promise|void; + onStateChange?: (state: ServiceState) => Promise|void; /** * Called when the container is closed, either by stop or kill, or by itself. @@ -119,11 +121,23 @@ export type ServiceEngine = { * @param imageId The image ID to build. If this is undefined, the engine should generate a random image ID and return it. * @param buildDir The build dir path * @param buildOptions The build options + * @param listener The listener to use for calling back up messages from the process */ build( imageId: string|undefined, - buildDir: string, buildOptions: { [key: string]: string }): Promise; + buildDir: string, + buildOptions: { [key: string]: string }, + listener?: MessageListener): Promise; + /** + * Runs a container from an image, with the given options. + * + * @param imageId The ID of the image to use + * @param volumeId The ID of the volume to use + * @param options The options + * @param meta The meta storage + * @param listener An optional listener for back propagation + */ run( imageId: string, volumeId: string, diff --git a/src/engine/image.ts b/src/engine/image.ts index edfa97d..09454a6 100644 --- a/src/engine/image.ts +++ b/src/engine/image.ts @@ -1,6 +1,6 @@ import {Database, ImageModel} from "@nsm/database"; import winston from "winston"; -import {ServiceEngineI} from "@nsm/engine/engine"; +import {MessageListener, ServiceEngineI} from "@nsm/engine/engine"; import {buildDir} from "@nsm/engine/monitoring/util"; import {TemplateManager} from "@nsm/engine/template"; import {TemplateDirWatcher} from "@nsm/engine/monitoring/templateDirWatcher"; @@ -38,11 +38,12 @@ export const init = ( * @param id The ID of the current image * @param templateId The ID of the template * @param buildOptions Build arguments used when building the image + * @param messageListener A message listener to use when building the image * @returns The ID of the image that should be used */ export const processImage = async ( id: string | undefined | null, - templateId: string, buildOptions: BuildOptionsMap + templateId: string, buildOptions: BuildOptionsMap, messageListener?: MessageListener ) => { const template = templateManager.getTemplate(templateId); // Checks if the provided options are still compatible with the template @@ -72,7 +73,7 @@ export const processImage = async ( logger.info(`Image ${id} is outdated due to template changes. Rebuilding...`); // Template changed, we need to rebuild the image - await rebuildImage(imageModel); + await rebuildImage(imageModel, messageListener); } } @@ -143,11 +144,17 @@ const getImage = async (id: string) => { * @param templateId The ID of the template to build the image from * @param options Build options to use when building the image * @param imageId (Optional) The ID of the image to overwrite. If not provided, a new image will be created. + * @param messageListener A message listener to use for logs propagation * @return The ID of the built image */ -const buildImage = async (templateId: string, options: BuildOptionsMap, imageId?: string): Promise => { +const buildImage = async ( + templateId: string, + options: BuildOptionsMap, + imageId?: string, + messageListener?: MessageListener +): Promise => { const hash = templateDirWatcher.getTemplateHash(templateId); - imageId = await engine.build(imageId, buildDir(templateId), options); + imageId = await engine.build(imageId, buildDir(templateId), options, messageListener); await db.imageRepository.saveImage({ id: imageId, @@ -169,8 +176,8 @@ const pickImage = async (templateId: string, options: BuildOptionsMap): Promise< return image.id; } -const rebuildImage = async (image: ImageModel) => { - return buildImage(image.templateId, image.buildOptions, image.id); +const rebuildImage = async (image: ImageModel, messageListener?: MessageListener) => { + return buildImage(image.templateId, image.buildOptions, image.id, messageListener); } export const deleteImageIfUnused = async (image: ImageModel) => { diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 7130bc2..8872525 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -549,7 +549,7 @@ export async function resumeService(id: string) { // Omit the always-changing args from build env, since they would always trigger an // image rebuild const { SERVICE_ID, SERVICE_PORT, SERVICE_PORTS, ...buildEnv } = runOptions.env; - const processedImage = await processImage(image, template, buildEnv); + const processedImage = await processImage(image, template, buildEnv); // TODO: tato funkce má poslední parametr messageListener, vymyslet jak sem propagovat message listener z session // If the image was changed by processing (e.g. it was built or rebuilt), update the image id in database if (processedImage != image) { image = processedImage; From 9acf0981e1c734f7a5412656fd5be6ab868647da Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 21 Mar 2026 02:41:46 +0100 Subject: [PATCH 38/52] refactor: move resources to data folder --- resources/config.yml | 3 +- src/app.ts | 53 ++++++++++--------- src/configuration/appConfig.ts | 18 ++++++- src/configuration/resources.ts | 32 ------------ src/engine/docker/index.ts | 3 +- src/engine/image.ts | 4 +- src/engine/manager.ts | 8 +-- src/engine/monitoring/templateDirWatcher.ts | 9 ++-- src/engine/monitoring/util.ts | 9 ++-- src/engine/template.ts | 12 ++--- src/filestructure.ts | 8 +++ src/logger.ts | 14 ++--- src/resources.ts | 58 +++++++++++++++++++++ src/router/v1/status/index.ts | 5 +- tests/configuration/resources.test.ts | 7 --- 15 files changed, 142 insertions(+), 101 deletions(-) delete mode 100644 src/configuration/resources.ts create mode 100644 src/filestructure.ts create mode 100644 src/resources.ts delete mode 100644 tests/configuration/resources.test.ts diff --git a/resources/config.yml b/resources/config.yml index 5e449df..ed25ea7 100644 --- a/resources/config.yml +++ b/resources/config.yml @@ -8,5 +8,4 @@ port: 3000 # Security # Supported types: 'none', 'auth_token' auth: 'none' -docker_host: 'unix:///var/run/docker.sock' -service_logs: true \ No newline at end of file +docker_host: 'unix:///var/run/docker.sock' \ No newline at end of file diff --git a/src/app.ts b/src/app.ts index ce7eb37..6a9aef8 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,11 +1,16 @@ +import envPaths, {Paths} from "env-paths"; +// This app's paths object +// Pre-define to avoid circular dependency +export const currentPaths: Paths = envPaths("nsm"); + import dotenv from "dotenv"; -import config from "@nsm/configuration/appConfig"; +import {getAppConfig, saveAppConfig} from "@nsm/configuration/appConfig"; // Load .env dotenv.config(); // Preload app config here to set needed env variables // before some modules require them. -config(); +saveAppConfig(); import {Router} from 'express'; import {Database} from "@nsm/database"; @@ -14,7 +19,6 @@ import loadAddons from "./addon"; import loadAppRoutes from '@nsm/router'; import createDbManager from '@nsm/database'; import loadSecurity from "@nsm/security"; -import * as r from "@nsm/configuration/resources"; import * as manager from "@nsm/engine/manager"; import * as sessionManager from "@nsm/engine/session"; import * as logging from "./logger"; @@ -24,6 +28,9 @@ import fs from "fs"; import isInsideContainer from "@nsm/lib/isInsideContainer"; import {middleLayer} from "@nsm/engine/middle"; import {SessionManager} from "@nsm/engine/session"; +import {mkdirResource, saveResource} from "@nsm/resources"; +import path from "path"; +import {resourcesTargetPath} from "@nsm/filestructure"; export type AppBootContext = AppContext & { steps: any }; @@ -44,17 +51,7 @@ export type AppBootOptions = { disableWorkers?: boolean; } -let currentContext: AppContext; - -function prepareServiceLogs(appConfig: any, logger: winston.Logger) { - if (appConfig.service_logs === true) { - logger.info('Service logs are enabled'); - const path = process.cwd() + '/service_logs'; - if (!fs.existsSync(path)) { - fs.mkdirSync(path); - } - } -} +export let currentContext: AppContext; function initGlobalLogger() { logging.createLatestLogFile(); @@ -81,23 +78,27 @@ function managerForUnsafeUse() { return new Proxy(manager, handler); } -// App orchestration code +/** + * App orchestration code. + * + * @param router The app router. + * @param options The optional boot options. + */ export const init = async (router: Application, options?: AppBootOptions): Promise => { // Prepare logging const logger = initGlobalLogger(); - r.prepareTemplatesFolder(); + // Prepare templates folder + mkdirResource("templates"); if (options?.test === true) { - r.prepareTestResources(); // Copy resources for test + prepareTestResources(); // Copy resources for test } // Load addon steps const steps = await loadAddons(logger); steps('BEFORE_CONFIG', { logger }); - const appConfig = config(); - - prepareServiceLogs(appConfig, logger); + const appConfig = getAppConfig(); // Database connection layer steps('BEFORE_DB', { logger, appConfig }); @@ -150,8 +151,12 @@ export const init = async (router: Application, options?: AppBootOptions): Promi return { ...ctx, steps }; } -export { - Database, - ServiceManager, - currentContext +const prepareTestResources = () => { + if (fs.existsSync(path.join(resourcesTargetPath, 'templates', 'test'))) { + return; + } + + saveResource('template/test/test_settings.yml', 'templates/test/settings.yml') + saveResource('template/test/test_dockerfile', 'templates/test/Dockerfile') + saveResource('template/test/test_nsmignore', 'templates/test/.nsmignore') } \ No newline at end of file diff --git a/src/configuration/appConfig.ts b/src/configuration/appConfig.ts index 04c8c2d..b45070c 100644 --- a/src/configuration/appConfig.ts +++ b/src/configuration/appConfig.ts @@ -1,12 +1,26 @@ import {loadYamlFile} from "@nsm/util/yaml"; +import {saveResource} from "@nsm/resources"; +import path from "path"; +import {resourcesTargetPath} from "@nsm/filestructure"; let cached: any = undefined; -export default function getAppConfig() { +export const saveAppConfig = () => { + saveResource('config.yml', 'config.yml', true); +} + +export const getAppConfig = () => { + saveAppConfig(); + + return loadAppConfig(); +} + +const loadAppConfig = () => { if (cached) { return cached; } - const config = loadYamlFile(`${process.cwd()}/resources/config.yml`); + + const config = loadYamlFile(path.join(resourcesTargetPath, 'config.yml')); for (let key in config) { // Overwrite with env variable if exists. // Sync diff --git a/src/configuration/resources.ts b/src/configuration/resources.ts deleted file mode 100644 index a886dce..0000000 --- a/src/configuration/resources.ts +++ /dev/null @@ -1,32 +0,0 @@ -import * as fs from "fs"; - -// Read resource from resources dir -export function readResource(name: string) { - return fs.readFileSync(process.cwd() + '/resources/' + name, 'utf8'); -} - -// Copy resource to target dir -export function saveResource(name: string, target: string) { - fs.writeFileSync(target, readResource(name)); -} - -export function prepareTemplatesFolder() { - const cwd = process.cwd(); - if (fs.existsSync(cwd + '/templates')) { - return; - } - - fs.mkdirSync(cwd + '/templates'); -} - -export function prepareTestResources() { - const cwd = process.cwd(); - if (fs.existsSync(cwd + '/templates/test')) { - return; - } - - fs.mkdirSync(cwd + '/templates/test'); - saveResource('template/test/test_settings.yml', cwd + '/templates/test/settings.yml'); - saveResource('template/test/test_dockerfile', cwd + '/templates/test/Dockerfile'); - saveResource('template/test/test_nsmignore', cwd + '/templates/test/.nsmignore'); -} \ No newline at end of file diff --git a/src/engine/docker/index.ts b/src/engine/docker/index.ts index c8d3178..36c0753 100644 --- a/src/engine/docker/index.ts +++ b/src/engine/docker/index.ts @@ -1,3 +1,4 @@ +import {currentPaths} from "@nsm/app"; import {DockerServiceEngine} from "@nsm/engine"; import {initDockerClient} from "@nsm/engine/docker/client"; @@ -25,7 +26,7 @@ export default function buildDockerEngine(appConfig: any) { engine.dockerClient = client; engine.rws = {}; // engine.cast - Being replaced in manager. - engine.build = build(client); + engine.build = build(client, currentPaths); engine.run = run(engine, client); engine.stop = stop(client); engine.kill = kill(client); diff --git a/src/engine/image.ts b/src/engine/image.ts index 09454a6..3c23818 100644 --- a/src/engine/image.ts +++ b/src/engine/image.ts @@ -1,7 +1,7 @@ import {Database, ImageModel} from "@nsm/database"; import winston from "winston"; import {MessageListener, ServiceEngineI} from "@nsm/engine/engine"; -import {buildDir} from "@nsm/engine/monitoring/util"; +import {templateBuildDir} from "@nsm/engine/monitoring/util"; import {TemplateManager} from "@nsm/engine/template"; import {TemplateDirWatcher} from "@nsm/engine/monitoring/templateDirWatcher"; @@ -154,7 +154,7 @@ const buildImage = async ( messageListener?: MessageListener ): Promise => { const hash = templateDirWatcher.getTemplateHash(templateId); - imageId = await engine.build(imageId, buildDir(templateId), options, messageListener); + imageId = await engine.build(imageId, templateBuildDir(templateId), options, messageListener); await db.imageRepository.saveImage({ id: imageId, diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 8872525..a638050 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -1,4 +1,4 @@ -import {currentContext, Database} from "../app"; +import {currentContext} from "../app"; import createEngine, { RunOptions, RunListener, @@ -12,7 +12,7 @@ import * as templateDirWatcher from "./monitoring/templateDirWatcher"; import crypto from "crypto"; import {randomPort as retrieveRandomPort} from "@nsm/util/port"; import {loadYamlFile} from "@nsm/util/yaml"; -import {PermaModel} from "../database"; +import {Database, PermaModel} from "../database"; import { isServicePending, lckStatusTp, @@ -26,7 +26,7 @@ import winston from "winston"; import path from "path"; import {isDebug} from "../helpers"; import {resolveSequentially} from "@nsm/util/promises"; -import {buildDir} from "@nsm/engine/monitoring/util"; +import {templateBuildDir} from "@nsm/engine/monitoring/util"; import {watchTemplateDirChanges} from "@nsm/engine/monitoring/templateDirWatcher"; import {processImage, init as initImageEngine, deleteImageIfUnused} from "@nsm/engine/image"; import {propagateOptionsToEnv} from "@nsm/engine/docker/util/env"; @@ -327,7 +327,7 @@ let appConfig: any; // Returns the settings.yml file for the template function settings(template: string) { - return loadYamlFile(buildDir(template) + path.sep + 'settings.yml'); + return loadYamlFile(templateBuildDir(template) + path.sep + 'settings.yml'); } // Save errors somewhere else? diff --git a/src/engine/monitoring/templateDirWatcher.ts b/src/engine/monitoring/templateDirWatcher.ts index 447be99..00c763c 100644 --- a/src/engine/monitoring/templateDirWatcher.ts +++ b/src/engine/monitoring/templateDirWatcher.ts @@ -1,10 +1,11 @@ -import {baseTemplatesDir, buildDir, debounce} from "@nsm/engine/monitoring/util"; +import {templateBuildDir, debounce} from "@nsm/engine/monitoring/util"; import {hashElement} from "folder-hash"; import {getFilteredPaths} from "@nsm/engine/ignore"; import {getAllTemplates} from "@nsm/engine/template"; import winston from "winston"; import chokidar, {FSWatcher} from "chokidar"; import path from "path"; +import {templatesPath} from "@nsm/filestructure"; export type TemplateDirWatcher = { @@ -46,7 +47,7 @@ export const watchTemplateDirChanges = (logger: winston.Logger) => { * When a template directory is removed, it stops watching that directory and removes its hash from the cache. */ const watchBaseDir = (logger: winston.Logger) => { - const dir = baseTemplatesDir(); + const dir = templatesPath; const watcher = chokidar.watch(dir, { ignoreInitial: true, @@ -89,7 +90,7 @@ const watchTemplateDir = async (template: string) => { await recalculateTemplateHash(template); - const dir = buildDir(template); + const dir = templateBuildDir(template); const excluded = getFilteredPaths(dir); const recalc = debounce(() => recalculateTemplateHash(template), 2000); @@ -116,7 +117,7 @@ const watchTemplateDir = async (template: string) => { * @param template The name of the template to recalculate the hash for. */ const recalculateTemplateHash = async (template: string) => { - const dir = buildDir(template); + const dir = templateBuildDir(template); const excluded = getFilteredPaths(dir); if (hashingInProgress.has(template)) { diff --git a/src/engine/monitoring/util.ts b/src/engine/monitoring/util.ts index d035092..80cd1b2 100644 --- a/src/engine/monitoring/util.ts +++ b/src/engine/monitoring/util.ts @@ -1,12 +1,9 @@ import path from 'path'; - -export function baseTemplatesDir() { - return `${process.cwd()}${path.sep}templates`; -} +import {templatesPath} from "@nsm/filestructure"; // Returns the build directory for the template -export function buildDir(template: string) { - return `${baseTemplatesDir()}${path.sep}${template}`; +export function templateBuildDir(template: string) { + return path.join(templatesPath, template); } /** diff --git a/src/engine/template.ts b/src/engine/template.ts index ec6a6b3..33bce8e 100644 --- a/src/engine/template.ts +++ b/src/engine/template.ts @@ -1,6 +1,7 @@ import {loadYamlFile} from "@nsm/util/yaml"; import * as fs from "fs"; -import {baseTemplatesDir} from "@nsm/engine/monitoring/util"; +import path from "path"; +import {templatesPath} from "@nsm/filestructure"; export type Template = { /** @@ -81,7 +82,7 @@ export const getTemplate = (id: string): Template|null => { if (templateCache[id]) { return templateCache[id]; } - const settingsPath = `${baseTemplatesDir()}/${id}/settings.yml`; + const settingsPath = path.join(templatesPath, id, 'settings.yml'); if (!fs.existsSync(settingsPath)) { return null; } @@ -97,14 +98,13 @@ export const getTemplate = (id: string): Template|null => { } export const getAllTemplates = () => { - const templatesDir = baseTemplatesDir(); - if (!fs.existsSync(templatesDir)) { + if (!fs.existsSync(templatesPath)) { return []; } return fs - .readdirSync(templatesDir) - .filter(file => fs.statSync(`${templatesDir}/${file}`).isDirectory()) + .readdirSync(templatesPath) + .filter(file => fs.statSync(path.join(templatesPath, file)).isDirectory()) .map(id => getTemplate(id)) .filter(template => template !== null); } \ No newline at end of file diff --git a/src/filestructure.ts b/src/filestructure.ts new file mode 100644 index 0000000..3ed1df3 --- /dev/null +++ b/src/filestructure.ts @@ -0,0 +1,8 @@ +import path from "path"; +import {currentPaths} from "@nsm/app"; + +// The local resources dir (not the source of truth) +export const resourcesPath = path.join(process.cwd(), "resources"); +// The target (platform-agnostic) resources dir (the source of truth) +export const resourcesTargetPath = path.join(currentPaths.data); +export const templatesPath = path.join(resourcesTargetPath, 'templates'); \ No newline at end of file diff --git a/src/logger.ts b/src/logger.ts index 68aa03c..184e6e7 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,15 +1,17 @@ import winston from "winston"; import fs from "fs"; +import path from "path"; +import {resourcesTargetPath} from "@nsm/filestructure"; const { combine, timestamp, label, errors, printf } = winston.format; export function createLatestLogFile() { - if (fs.existsSync(process.cwd() + '/logs/latest.log')) { + if (fs.existsSync(path.join(resourcesTargetPath, 'logs', 'latest.log'))) { const date = new Date(Date.now()).toJSON().slice(2, 10) + '.' + new Date(Date.now()).getHours() + '.' + new Date(Date.now()).getMinutes(); - fs.renameSync(process.cwd() + '/logs/latest.log', process.cwd() + '/logs/' + date + '.log'); + fs.renameSync(path.join(resourcesTargetPath, 'logs', 'latest.log'), path.join(resourcesTargetPath, 'logs', date + '.log')); } } @@ -29,13 +31,7 @@ export function createLogger(options?: { label?: string }) { ), transports: [ new winston.transports.Console(), - new winston.transports.File({dirname: 'logs', filename: 'latest.log'}) + new winston.transports.File({dirname: path.join(resourcesTargetPath, 'logs'), filename: 'latest.log'}) ] }); -} - -export function logService(id: string, str: any) { - // Isn't this thing blocking??? Look at it later, zort - by zort xdd - const log_path = process.cwd() + '/service_logs/' + id + '.log'; - fs.appendFileSync(log_path, (str ?? '').toString() + '\n'); } \ No newline at end of file diff --git a/src/resources.ts b/src/resources.ts new file mode 100644 index 0000000..962ff87 --- /dev/null +++ b/src/resources.ts @@ -0,0 +1,58 @@ +import path from "path"; +import fs from "fs"; +import {resourcesPath, resourcesTargetPath} from "@nsm/filestructure"; + +/** + * Reads resource from target dir. + * + * @param name The name of the resource in the target dir. + */ +export const readResource = (name: string) => { + const p = path.join(resourcesTargetPath, name); + + return fs.readFileSync(p, 'utf8'); +} + +/** + * Creates a directory in the target dir. Creates parent dirs if missing. + * + * @param name The name of the dir in the target dir. + */ +export const mkdirResource = (name: string) => { + const p = path.join(resourcesTargetPath, name); + + fs.mkdirSync(p, { recursive: true }); +} + +/** + * Saves resource to target dir. Creates parent dirs if missing. + * + * @param name The name of the resource in the resources dir. + * @param targetName The target name where to copy. + * @param skipIfExists If true, the resource will not be copied if a file with the same name already exists in the target dir. Default is false. + */ +export const saveResource = ( + name: string, + targetName: string, + skipIfExists: boolean = false +) => { + const targetPath = path.join(resourcesTargetPath, targetName); + // Create parent dirs if missing + fs.mkdirSync(path.dirname(targetPath), { recursive: true }); + + if (skipIfExists == true && fs.existsSync(targetPath)) { + return; + } + fs.writeFileSync(targetPath, readCwdResource(name)); +} + +/** + * Reads resource from resources dir. + * + * @param name The name of the resource in the resources dir. + */ +export const readCwdResource = (name: string) => { + const p = path.join(resourcesPath, name); + + return fs.readFileSync(p, 'utf8'); +} \ No newline at end of file diff --git a/src/router/v1/status/index.ts b/src/router/v1/status/index.ts index 1092a92..ed1615e 100644 --- a/src/router/v1/status/index.ts +++ b/src/router/v1/status/index.ts @@ -1,7 +1,8 @@ -import {AppContext, Database, ServiceManager} from "@nsm/app"; +import {AppContext} from "@nsm/app"; import {RouterHandler} from "../../index"; import * as os from "os"; -import {Filters} from "@nsm/engine"; +import {Filters, ServiceManager} from "@nsm/engine"; +import {Database} from "@nsm/database"; async function checkNsmResources(engine: ServiceManager, db: Database) { const stats = await engine.engine.statAll(Filters.node(engine.nodeId)); diff --git a/tests/configuration/resources.test.ts b/tests/configuration/resources.test.ts deleted file mode 100644 index e8c95aa..0000000 --- a/tests/configuration/resources.test.ts +++ /dev/null @@ -1,7 +0,0 @@ -import {expect, test} from "@jest/globals"; -import {readResource} from "@nsm/configuration/resources"; -import * as fs from "fs"; - -test('Reads the resource', () => { - expect(readResource('template/example/example_settings.yml')).toBe(fs.readFileSync(process.cwd() + '/resources/template/example/example_settings.yml', 'utf8')); -}); \ No newline at end of file From 39e7d9e0812a92a525a6828bc925fdb883702178 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 21 Mar 2026 02:43:10 +0100 Subject: [PATCH 39/52] todo --- src/engine/ignore.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/engine/ignore.ts b/src/engine/ignore.ts index a8420cb..9884d48 100644 --- a/src/engine/ignore.ts +++ b/src/engine/ignore.ts @@ -1,6 +1,8 @@ import fs from "fs"; import ignore from "ignore"; +// TODO: předělat tento modul aby používal novou resources složku + export const getRootFilesFiltered = (dir: string) => { let filtered = fs.readdirSync(dir); if (fs.existsSync(dir + '/.nsmignore')) { From 6576a7ea092f0376d64c2e8125eb0cee17b7ea7b Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 21 Mar 2026 12:45:51 +0100 Subject: [PATCH 40/52] feat: service logs ep --- openapi.yml | 1 + src/app.ts | 6 +-- src/database/models.ts | 13 +++--- src/engine/docker/index.ts | 2 +- src/engine/ignore.ts | 14 +++--- src/engine/session.ts | 3 +- src/filestructure.ts | 4 +- src/router/v1/index.ts | 2 + src/router/v1/service/logsRoute.ts | 72 ++++++++++++++++++++++++++++++ 9 files changed, 96 insertions(+), 21 deletions(-) create mode 100644 src/router/v1/service/logsRoute.ts diff --git a/openapi.yml b/openapi.yml index 8ec541e..31d9e96 100644 --- a/openapi.yml +++ b/openapi.yml @@ -547,4 +547,5 @@ paths: $ref: "#/components/schemas/Result" # TODO: /v1/service/{serviceId}/sessions +# TODO: /v1/service/{serviceId}/logs # TODO: /v1/session/{sessionId}/logs \ No newline at end of file diff --git a/src/app.ts b/src/app.ts index 6a9aef8..0194fbd 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,8 +1,3 @@ -import envPaths, {Paths} from "env-paths"; -// This app's paths object -// Pre-define to avoid circular dependency -export const currentPaths: Paths = envPaths("nsm"); - import dotenv from "dotenv"; import {getAppConfig, saveAppConfig} from "@nsm/configuration/appConfig"; @@ -55,6 +50,7 @@ export let currentContext: AppContext; function initGlobalLogger() { logging.createLatestLogFile(); + return logging.createLogger(); } diff --git a/src/database/models.ts b/src/database/models.ts index 1645a30..c44f9a3 100644 --- a/src/database/models.ts +++ b/src/database/models.ts @@ -96,17 +96,18 @@ export type PermaModel = { }; export type ImageModel = { - id: string, - templateId: string, - hash: string, + id: string; + templateId: string; + hash: string; buildOptions: { - [key: string]: string, + [key: string]: string; } } export type ServiceSessionModel = { - id: string, - serviceId: string, + id: string; + serviceId: string; + startedAt: Date; } export type ServiceLogRecordModel = { diff --git a/src/engine/docker/index.ts b/src/engine/docker/index.ts index 36c0753..d31ead2 100644 --- a/src/engine/docker/index.ts +++ b/src/engine/docker/index.ts @@ -1,4 +1,3 @@ -import {currentPaths} from "@nsm/app"; import {DockerServiceEngine} from "@nsm/engine"; import {initDockerClient} from "@nsm/engine/docker/client"; @@ -17,6 +16,7 @@ import stat from "./action/stat"; import statAll from "./action/statall"; import calcHostUsage from "./action/calcHostUsage"; import listRunning from "./action/listRunning"; +import {currentPaths} from "@nsm/filestructure"; export default function buildDockerEngine(appConfig: any) { // Default engine implementation diff --git a/src/engine/ignore.ts b/src/engine/ignore.ts index 9884d48..3ba09ea 100644 --- a/src/engine/ignore.ts +++ b/src/engine/ignore.ts @@ -1,11 +1,10 @@ import fs from "fs"; import ignore from "ignore"; - -// TODO: předělat tento modul aby používal novou resources složku +import path from "path"; export const getRootFilesFiltered = (dir: string) => { let filtered = fs.readdirSync(dir); - if (fs.existsSync(dir + '/.nsmignore')) { + if (fs.existsSync(path.join(dir, '.nsmignore'))) { const ig = buildIgnore(dir); filtered = ig.filter(filtered); @@ -24,8 +23,8 @@ export const getFilteredPaths = (dir: string) => { const walk = (currentDir: string) => { const files = fs.readdirSync(currentDir); for (const file of files) { - const relativePath = currentDir === dir ? file : currentDir.substring(dir.length + 1) + '/' + file; - const fullPath = currentDir + '/' + file; + const relativePath = currentDir === dir ? file : currentDir.substring(dir.length + 1) + path.sep + file; + const fullPath = currentDir + path.sep + file; if (ig.ignores(relativePath)) { const isDir = fs.statSync(fullPath).isDirectory(); @@ -53,8 +52,9 @@ export const getFilteredPaths = (dir: string) => { const buildIgnore = (dir: string) => { const ig = ignore(); - if (fs.existsSync(dir + '/.nsmignore')) { - ig.add(fs.readFileSync(dir + '/.nsmignore', 'utf8')); + const ignorePath = path.join(dir, '.nsmignore'); + if (fs.existsSync(ignorePath)) { + ig.add(fs.readFileSync(ignorePath, 'utf8')); } return ig; diff --git a/src/engine/session.ts b/src/engine/session.ts index e51b9a5..20eb06c 100644 --- a/src/engine/session.ts +++ b/src/engine/session.ts @@ -20,7 +20,8 @@ export interface SessionManager { export interface ServiceSession { id: string; serviceId: string; - // TODO: begin timestamp, end timestamp + startedAt: Date; + // TODO: end timestamp } export interface ActiveServiceSession extends ServiceSession { diff --git a/src/filestructure.ts b/src/filestructure.ts index 3ed1df3..7adf2ab 100644 --- a/src/filestructure.ts +++ b/src/filestructure.ts @@ -1,5 +1,7 @@ import path from "path"; -import {currentPaths} from "@nsm/app"; +import envPaths, {Paths} from "env-paths"; + +export const currentPaths: Paths = envPaths("nsm"); // The local resources dir (not the source of truth) export const resourcesPath = path.join(process.cwd(), "resources"); diff --git a/src/router/v1/index.ts b/src/router/v1/index.ts index 8df235c..2534223 100644 --- a/src/router/v1/index.ts +++ b/src/router/v1/index.ts @@ -11,6 +11,7 @@ import stopCmdRoute from "@nsm/router/v1/service/stopCmdRoute"; import optionsRoute from "@nsm/router/v1/service/optionsRoute"; import sessionsRoute from "@nsm/router/v1/service/sessionsRoute"; import sessionLogsRoute from "@nsm/router/v1/session/sessionLogsRoute"; +import logsRoute from "@nsm/router/v1/service/logsRoute"; export default [ // v1 routes @@ -26,5 +27,6 @@ export default [ optionsRoute, listRoute, sessionsRoute, + logsRoute, sessionLogsRoute ] \ No newline at end of file diff --git a/src/router/v1/service/logsRoute.ts b/src/router/v1/service/logsRoute.ts new file mode 100644 index 0000000..61ea880 --- /dev/null +++ b/src/router/v1/service/logsRoute.ts @@ -0,0 +1,72 @@ +import {AppContext} from "@nsm/app"; +import {RouterHandler} from "@nsm/router"; +import {ListRecordsArgs} from "@nsm/database"; +import {checkServiceExists} from "@nsm/router/util/preconditions"; + +export default async function(ctx: AppContext): Promise { + return { + url: '/service/:id/logs', + routes: { + get: async (req, res) => { + const id = req.params.id; + if (!id) { + res.status(400).json({status: 400, message: 'Required \'id\' field not present in the body.'}); + return; + } + if (!await checkServiceExists(id, ctx.manager, res)) { + return; + } + + let sessionId: string; + + const runningService = ctx.manager.getRunningService(id); + if (runningService) { + // Service currently running, we can use logs from the current session + sessionId = runningService.session.id; + } else { + // Service not running, so we need to retrieve last session ID + const lastSession = await ctx.sessionManager.listSessions({ + filter: { serviceId: id }, + sort: { by: "startedAt", direction: "desc" }, + page: { index: 0, size: 1 } + }); + if (lastSession && lastSession.length > 0) { + sessionId = lastSession[0].id; + } + } + + if (!sessionId) { + res.status(400).json({status: 400, message: 'Service was never active.'}); + return; + } + + const pageIndex = req.query.pageIndex ? Number(req.query.pageIndex) : 0; + const pageSize = req.query.pageSize ? Number(req.query.pageSize) : 10; + + // Use pagination only if it was requested by params + const page = req.query.pageIndex || req.query.pageSize + ? ( + { + index: pageIndex, + size: pageSize + } + ) + : undefined; + + const args: ListRecordsArgs = { + filter: { + sessionId + }, + sort: { + by: "timestamp", + direction: "asc" + }, + page + }; + const logs = await ctx.sessionManager.listSessionLogs(args); + + res.status(200).json({ logs }); + } + } + } +} \ No newline at end of file From b3ea56d32b6f8aa1c41fca0a0803068584ca41ae Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 21 Mar 2026 12:48:21 +0100 Subject: [PATCH 41/52] todo --- src/engine/manager.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/engine/manager.ts b/src/engine/manager.ts index a638050..51d9f77 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -368,11 +368,16 @@ export async function init(db_: Database, appConfig_: any, logger: winston.Logge initSessionEngine(db_); watchTemplateDirChanges(currentContext.logger); + await deleteGarbage(logger); await reattachStaleContainers(logger); logger.info(`Using engine: ${engine.name}`); } +async function deleteGarbage(logger: winston.Logger) { + // TODO: delete containers that are not running and remained from last session +} + /** * Reattach to containers that are still running from the previous session. * This may happen if NSM was force-stopped and not properly cleared up resources. From c165a2701240b6f83fb032c20c0538e3abdcf12c Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 21 Mar 2026 13:17:02 +0100 Subject: [PATCH 42/52] fix: handle engine power errors at the top layer --- src/engine/manager.ts | 23 ++++++++------- src/helpers.ts | 7 +++++ src/router/v1/service/createRoute.ts | 7 ++--- src/router/v1/service/rebootRoute.ts | 25 +++++++++------- src/router/v1/service/resumeRoute.ts | 43 ++++++++++++---------------- src/router/v1/service/stopRoute.ts | 16 +++++------ 6 files changed, 62 insertions(+), 59 deletions(-) diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 51d9f77..e1a6208 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -11,7 +11,6 @@ import * as templateManager from "./template"; import * as templateDirWatcher from "./monitoring/templateDirWatcher"; import crypto from "crypto"; import {randomPort as retrieveRandomPort} from "@nsm/util/port"; -import {loadYamlFile} from "@nsm/util/yaml"; import {Database, PermaModel} from "../database"; import { isServicePending, @@ -23,10 +22,8 @@ import { whenUnlocked, whenUnlockedAll } from "./asyncp"; import winston from "winston"; -import path from "path"; import {isDebug} from "../helpers"; import {resolveSequentially} from "@nsm/util/promises"; -import {templateBuildDir} from "@nsm/engine/monitoring/util"; import {watchTemplateDirChanges} from "@nsm/engine/monitoring/templateDirWatcher"; import {processImage, init as initImageEngine, deleteImageIfUnused} from "@nsm/engine/image"; import {propagateOptionsToEnv} from "@nsm/engine/docker/util/env"; @@ -325,11 +322,6 @@ export let nodeId: string; let db: Database; let appConfig: any; -// Returns the settings.yml file for the template -function settings(template: string) { - return loadYamlFile(templateBuildDir(template) + path.sep + 'settings.yml'); -} - // Save errors somewhere else? // Could it be a memory leak if there are tons of them?? const errors = {}; @@ -454,7 +446,7 @@ export async function createService(template: string, options: Options) { env, network } = options; - const serviceSettings = settings(template); + const serviceSettings = reqTemplate(template).settings; // Join meta supplied by user and template meta const meta = { @@ -513,7 +505,7 @@ export async function resumeService(id: string) { port, } = await getPermaModel(id); - const {defaults, env: settingsEnv} = settings(template); + const {defaults, env: settingsEnv} = reqTemplate(template).settings; // Filter env to only those that are defined in settings.yml, because those are the only ones that // we can guarantee to be used and will not make problems when handling images. env = { @@ -917,6 +909,8 @@ function getServiceState(id: string) { return startedStates.get(id) ?? 'STOPPED'; } +// --------------------------------------------------------------------------------------- + async function getPermaModel(id: string) { const perma_ = await db.permaRepository.getPerma(id); if (!perma_) { @@ -949,4 +943,13 @@ function reqNotRunning(id: string) { if (isRunning(id)) { throw new _InternalError('Already running.', 2); } +} + +function reqTemplate(id: string) { + const template = getTemplate(id); + if (!template) { + throw new _InternalError('Template not found.', 3); + } + + return template; } \ No newline at end of file diff --git a/src/helpers.ts b/src/helpers.ts index ed19f19..8a6da96 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -1,3 +1,10 @@ export function isDebug() { return process.env.DEBUG === 'true'; +} + +export function consumeEnginePowerAction(action: () => Promise) { + action().catch((e) => { + // Manager service power action errors are ignored since + // they are handled by the middle-layer defined in engine/middle.ts + }); } \ No newline at end of file diff --git a/src/router/v1/service/createRoute.ts b/src/router/v1/service/createRoute.ts index 2ad6b33..49e634b 100644 --- a/src/router/v1/service/createRoute.ts +++ b/src/router/v1/service/createRoute.ts @@ -1,9 +1,9 @@ - import {RouterHandler} from "../../index"; import {AppContext} from "@nsm/app"; import {Options} from "@nsm/engine"; import {clock} from "@nsm/util/clock"; import {prepareEnvForTemplate} from "@nsm/engine/template"; +import {consumeEnginePowerAction} from "@nsm/helpers"; export default async function ({manager}: AppContext): Promise { return { @@ -36,10 +36,7 @@ export default async function ({manager}: AppContext): Promise { const serviceId = await manager.createService(template.id, options); // Resume right afterward - manager.resumeService(serviceId) - .then(() => { - // Service resumed successfully, do nothing here for now. - }); + consumeEnginePowerAction(() => manager.resumeService(serviceId)); res.status(200).json({ status: 200, diff --git a/src/router/v1/service/rebootRoute.ts b/src/router/v1/service/rebootRoute.ts index f0e4c58..9820372 100644 --- a/src/router/v1/service/rebootRoute.ts +++ b/src/router/v1/service/rebootRoute.ts @@ -1,6 +1,7 @@ import {AppContext} from "@nsm/app"; import {RouterHandler} from "../../index"; import {checkServiceExists, checkServicePending} from "@nsm/router/util/preconditions"; +import {consumeEnginePowerAction} from "@nsm/helpers"; export default async function ({manager, logger}: AppContext): Promise { return { @@ -19,18 +20,20 @@ export default async function ({manager, logger}: AppContext): Promise { - // Service stopped successfully, now wait for it to be unlocked before resuming. + consumeEnginePowerAction(() => ( + manager.stopService(id) + .then(() => { + // Service stopped successfully, now wait for it to be unlocked before resuming. - manager.whenUnlocked(id, (_, __, err) => { - if (err) { - logger.error(err); - } else { - manager.resumeService(id); - } - }); - }); + manager.whenUnlocked(id, (_, __, err) => { + if (err) { + logger.error(err); + } else { + manager.resumeService(id); + } + }); + }) + )); res.status(200).json({ status: 200, diff --git a/src/router/v1/service/resumeRoute.ts b/src/router/v1/service/resumeRoute.ts index 439a4fe..54e8e9f 100644 --- a/src/router/v1/service/resumeRoute.ts +++ b/src/router/v1/service/resumeRoute.ts @@ -1,9 +1,9 @@ import {AppContext} from "@nsm/app"; import {RouterHandler} from "../../index"; -import {handleErr} from "@nsm/util/routes"; import {checkServiceExists, checkServicePending} from "@nsm/router/util/preconditions"; +import {consumeEnginePowerAction} from "@nsm/helpers"; -export default async function ({manager, logger}: AppContext): Promise { +export default async function ({manager}: AppContext): Promise { return { url: '/service/:id/resume', routes: { @@ -13,31 +13,24 @@ export default async function ({manager, logger}: AppContext): Promise { - // Service resumed successfully, do nothing here for now. - }); + consumeEnginePowerAction(() => manager.resumeService(id)); - res.status(200).json({ - status: 200, - message: 'Service resume action successfully registered to be completed in a moment.', - statusPath: '/v1/service/' + id + '/powerstatus', - }); - } catch (e) { - handleErr(e, res); - } + res.status(200).json({ + status: 200, + message: 'Service resume action successfully registered to be completed in a moment.', + statusPath: '/v1/service/' + id + '/powerstatus', + }); } }, } diff --git a/src/router/v1/service/stopRoute.ts b/src/router/v1/service/stopRoute.ts index 5320b5f..dd782a8 100644 --- a/src/router/v1/service/stopRoute.ts +++ b/src/router/v1/service/stopRoute.ts @@ -1,6 +1,7 @@ import {AppContext} from "@nsm/app"; import {RouterHandler} from "../../index"; import {checkServiceExists, checkServicePending} from "@nsm/router/util/preconditions"; +import {consumeEnginePowerAction} from "@nsm/helpers"; export default async function ({manager, logger}: AppContext): Promise { return { @@ -23,14 +24,13 @@ export default async function ({manager, logger}: AppContext): Promise { - // Service stopped successfully, do nothing here for now. - }); + consumeEnginePowerAction(async () => { + if (req.query.force === 'true') { + await manager.stopServiceForcibly(id); + } else { + await manager.stopService(id) + } + }); res.status(200).json({ status: 200, From ad52fd81c113271140c7037574d4e236e7f1e4ee Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 21 Mar 2026 13:21:10 +0100 Subject: [PATCH 43/52] todo --- TODO.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/TODO.txt b/TODO.txt index 7cfc0b3..be37234 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,4 +1,5 @@ fixnout stop flow pomocí stop cmd +přidat created at a last started at do service info abstrahovat template management a udělat další sources jako například používání images z docker registry předělat usages monitoring na nějaký standardizovaný formát/způsob pořešit /resources/config.yml aby měl víc možností a možná byl uložený jinde v systému podle platformy From a4af0bf6015c892200a1dc91073ea6acc213766a Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 21 Mar 2026 13:25:31 +0100 Subject: [PATCH 44/52] todo --- TODO.txt | 4 ++- src/engine/template.ts | 58 +++++++++++++++++++++--------------------- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/TODO.txt b/TODO.txt index be37234..635e1c8 100644 --- a/TODO.txt +++ b/TODO.txt @@ -5,4 +5,6 @@ předělat usages monitoring na nějaký standardizovaný formát/způsob pořešit /resources/config.yml aby měl víc možností a možná byl uložený jinde v systému podle platformy port retrieval strategy pro spouštění servis, aby nebyla jenom random ale i jiné strategies udělat nějaký auto packaging do balíčků na githubu jako bundle aby se dal spustit jako systémový proces v systemctl například -restart policy \ No newline at end of file +restart policy +předělat addons systém +flagy servis, každý flag bude mít jiný účel a půjdou přidávat přes addony které je budou taky managovat \ No newline at end of file diff --git a/src/engine/template.ts b/src/engine/template.ts index 33bce8e..81c5560 100644 --- a/src/engine/template.ts +++ b/src/engine/template.ts @@ -49,35 +49,6 @@ export type TemplateManager = { const templateCache = {}; -export const prepareEnvForTemplate = (template: Template | string, env: any) => { - env = { ...env }; // Shallow copy to avoid mutating the original object - if (typeof template === 'string') { - template = getTemplate(template); // Load the template if ID provided - } - - for (const key of Object.keys(template.settings['env'])) { - if (env[key] && typeof env[key] == typeof template.settings['env'][key]) { - // Keep the value - } else if (env[key]) { - throw new Error('Invalid option type for ' + key + '. Got ' + typeof env[key] + ' but expected ' + typeof template.settings['env'][key] + '.'); - } else if (isRequiredOption(template.settings['env'][key])) { - throw new Error('Missing required option ' + key); - } else { - // Set default - env[key] = template.settings['env'][key]; - } - } - return env; -} - -// Defines if the value represents required option. -const isRequiredOption = (value: any) => { - return ( - (typeof value == "string" && value === "") || - (typeof value === "number" && value == -1) - ) -} - export const getTemplate = (id: string): Template|null => { if (templateCache[id]) { return templateCache[id]; @@ -107,4 +78,33 @@ export const getAllTemplates = () => { .filter(file => fs.statSync(path.join(templatesPath, file)).isDirectory()) .map(id => getTemplate(id)) .filter(template => template !== null); +} + +export const prepareEnvForTemplate = (template: Template | string, env: any) => { + env = { ...env }; // Shallow copy to avoid mutating the original object + if (typeof template === 'string') { + template = getTemplate(template); // Load the template if ID provided + } + + for (const key of Object.keys(template.settings['env'])) { + if (env[key] && typeof env[key] == typeof template.settings['env'][key]) { + // Keep the value + } else if (env[key]) { + throw new Error('Invalid option type for ' + key + '. Got ' + typeof env[key] + ' but expected ' + typeof template.settings['env'][key] + '.'); + } else if (isRequiredOption(template.settings['env'][key])) { + throw new Error('Missing required option ' + key); + } else { + // Set default + env[key] = template.settings['env'][key]; + } + } + return env; +} + +// Defines if the value represents required option. +const isRequiredOption = (value: any) => { + return ( + (typeof value == "string" && value === "") || + (typeof value === "number" && value == -1) + ) } \ No newline at end of file From 83f6e34153efef2f317104c5d3a7f1a9c5fc959f Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 21 Mar 2026 13:35:00 +0100 Subject: [PATCH 45/52] refactor --- src/engine/manager.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/engine/manager.ts b/src/engine/manager.ts index e1a6208..c3a9b2c 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -688,7 +688,7 @@ export async function deleteService(id: string) { ) )) .then(() => { - currentContext.logger.info(`Service ${id} deleted`); + currentContext.logger.debug(`Service ${id} deleted`); }); }; @@ -778,13 +778,7 @@ export async function stopRunning() { export async function waitForBusyAction(id: string) { return new Promise( (resolve, reject) => { - whenUnlocked(id, (_, __, err) => { - if (err) { - reject(err); - } else { - resolve(null); - } - }); + whenUnlocked(id, (_, __, err) => err ? reject(err) : resolve(null)); } ); } From e75b8e92f6694887c11759b55c8c35601f156641 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 21 Mar 2026 19:41:27 +0100 Subject: [PATCH 46/52] feat: reworked configuration handling --- openapi.yml | 6 ++- src/app.ts | 12 ++--- src/config.ts | 67 ++++++++++++++++++++++++++++ src/configuration/appConfig.ts | 35 --------------- src/engine/docker/client.ts | 18 ++++---- src/engine/docker/index.ts | 3 +- src/engine/engine.ts | 3 +- src/engine/manager.ts | 9 ++-- src/router/v1/service/lookupRoute.ts | 26 ++++++----- src/router/v1/status/index.ts | 2 +- src/security/index.ts | 4 +- tests/engine/image.test.ts | 7 +-- 12 files changed, 117 insertions(+), 75 deletions(-) create mode 100644 src/config.ts delete mode 100644 src/configuration/appConfig.ts diff --git a/openapi.yml b/openapi.yml index 31d9e96..bed694d 100644 --- a/openapi.yml +++ b/openapi.yml @@ -89,8 +89,10 @@ components: id: type: "string" description: "The currently active session ID" - containerId: - type: "string" + startedAt: + type: "number" + format: "int64" + description: "The timestamp when the session started, in milliseconds since epoch" stats: type: "object" properties: diff --git a/src/app.ts b/src/app.ts index 0194fbd..93ed931 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,11 +1,11 @@ import dotenv from "dotenv"; -import {getAppConfig, saveAppConfig} from "@nsm/configuration/appConfig"; +import {loadAppConfig} from "@nsm/config"; // Load .env dotenv.config(); // Preload app config here to set needed env variables // before some modules require them. -saveAppConfig(); +const appConfig = loadAppConfig(); import {Router} from 'express'; import {Database} from "@nsm/database"; @@ -26,6 +26,7 @@ import {SessionManager} from "@nsm/engine/session"; import {mkdirResource, saveResource} from "@nsm/resources"; import path from "path"; import {resourcesTargetPath} from "@nsm/filestructure"; +import {AppConfig} from "@nsm/config"; export type AppBootContext = AppContext & { steps: any }; @@ -35,7 +36,7 @@ export type AppContext = { manager: ServiceManager; sessionManager: SessionManager; database: Database; - appConfig: any; + appConfig: AppConfig; logger: winston.Logger; debug: boolean; workers: boolean; @@ -94,7 +95,6 @@ export const init = async (router: Application, options?: AppBootOptions): Promi const steps = await loadAddons(logger); steps('BEFORE_CONFIG', { logger }); - const appConfig = getAppConfig(); // Database connection layer steps('BEFORE_DB', { logger, appConfig }); @@ -139,8 +139,8 @@ export const init = async (router: Application, options?: AppBootOptions): Promi let srv = undefined; if (options?.test == undefined || options.test == false) { logger.info(`Starting server`); - srv = router.listen(appConfig.port, () => { - logger.info(`Server started on port ${appConfig.port}`); + srv = router.listen(appConfig.getPort(), () => { + logger.info(`Server started on port ${appConfig.getPort()}`); }); } steps('BOOT', ctx, srv); diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..b74db0a --- /dev/null +++ b/src/config.ts @@ -0,0 +1,67 @@ +import {loadYamlFile} from "@nsm/util/yaml"; +import path from "path"; +import {resourcesTargetPath} from "@nsm/filestructure"; +import {saveResource} from "@nsm/resources"; + +export interface AppConfig { + getNodeId(): string; + + getPort(): number; + + getAuth(): string; + + getDockerHost(): string; +} + +/** + * An implementation of AppConfig stored in config.yml file. + * + * @author ZorTik + */ +export class YamlAppConfig implements AppConfig { + private readonly data: any; + + constructor() { + this.data = YamlAppConfig.loadData(); + + // TODO: validate + } + + getNodeId(): string { + return this.data["node_id"]; + } + + getPort(): number { + return this.data["port"]; + } + + getAuth(): string { + return this.data["auth"]; + } + + getDockerHost(): string { + return this.data["docker_host"]; + } + + private static loadData = () => { + // Copy if it does not exist + saveResource('config.yml', 'config.yml', true); + + const config = loadYamlFile(path.join(resourcesTargetPath, 'config.yml')); + for (let key in config) { + // Overwrite with env variable if exists. + // Sync + const envKey = 'CONFIG_' + key.toUpperCase(); + if (process.env[envKey]) { + config[key] = process.env[envKey]; + } else { + process.env[envKey] = config[key]; + } + } + return config; + } +} + +export const loadAppConfig = (): AppConfig => { + return new YamlAppConfig(); +} \ No newline at end of file diff --git a/src/configuration/appConfig.ts b/src/configuration/appConfig.ts deleted file mode 100644 index b45070c..0000000 --- a/src/configuration/appConfig.ts +++ /dev/null @@ -1,35 +0,0 @@ -import {loadYamlFile} from "@nsm/util/yaml"; -import {saveResource} from "@nsm/resources"; -import path from "path"; -import {resourcesTargetPath} from "@nsm/filestructure"; - -let cached: any = undefined; - -export const saveAppConfig = () => { - saveResource('config.yml', 'config.yml', true); -} - -export const getAppConfig = () => { - saveAppConfig(); - - return loadAppConfig(); -} - -const loadAppConfig = () => { - if (cached) { - return cached; - } - - const config = loadYamlFile(path.join(resourcesTargetPath, 'config.yml')); - for (let key in config) { - // Overwrite with env variable if exists. - // Sync - const envKey = 'CONFIG_' + key.toUpperCase(); - if (process.env[envKey]) { - config[key] = process.env[envKey]; - } else { - process.env[envKey] = config[key]; - } - } - return cached = config; -} \ No newline at end of file diff --git a/src/engine/docker/client.ts b/src/engine/docker/client.ts index 6f947b2..6833e0f 100644 --- a/src/engine/docker/client.ts +++ b/src/engine/docker/client.ts @@ -1,18 +1,20 @@ import DockerClient from "dockerode"; +import {AppConfig} from "@nsm/config"; + +export function initDockerClient(appConfig: AppConfig) { + let host = appConfig.getDockerHost(); -export function initDockerClient(appConfig: { docker_host: string }) { let client: DockerClient; - if (appConfig.docker_host && ( - appConfig.docker_host.endsWith('.sock') || - appConfig.docker_host.startsWith('\\\\.\\pipe') + if (host && ( + host.endsWith('.sock') || + host.startsWith('\\\\.\\pipe') )) { - client = new DockerClient({ socketPath: appConfig.docker_host }); - } else if (appConfig.docker_host) { + client = new DockerClient({ socketPath: host }); + } else if (host) { // http(s)://host:port - let host = appConfig.docker_host; host = host.substring(0, host.lastIndexOf(':') + 1); - let port = parseInt(appConfig.docker_host.replace(host, '')); + let port = parseInt(appConfig.getDockerHost().replace(host, '')); host = host.substring(0, host.length - 1); diff --git a/src/engine/docker/index.ts b/src/engine/docker/index.ts index d31ead2..4814c81 100644 --- a/src/engine/docker/index.ts +++ b/src/engine/docker/index.ts @@ -17,8 +17,9 @@ import statAll from "./action/statall"; import calcHostUsage from "./action/calcHostUsage"; import listRunning from "./action/listRunning"; import {currentPaths} from "@nsm/filestructure"; +import {AppConfig} from "@nsm/config"; -export default function buildDockerEngine(appConfig: any) { +export default function buildDockerEngine(appConfig: AppConfig) { // Default engine implementation const client = initDockerClient(appConfig); const engine = {} as DockerServiceEngine; diff --git a/src/engine/engine.ts b/src/engine/engine.ts index e5f97f5..dd77c09 100644 --- a/src/engine/engine.ts +++ b/src/engine/engine.ts @@ -2,6 +2,7 @@ import DockerClient from "dockerode"; import buildDockerEngine from "./docker"; import {getSingleton} from "../depend"; import {MetaStorage} from "./manager"; +import {AppConfig} from "@nsm/config"; /** * The options for running a service. @@ -299,7 +300,7 @@ export const combineRunListeners = (listeners: RunListener[]): RunListener => { } } -export default function (appConfig: any): ServiceEngineI { +export default function (appConfig: AppConfig): ServiceEngineI { let engine = getSingleton('engine'); if (!engine) { const engineId = process.env.NSM_ENGINE ?? 'docker'; diff --git a/src/engine/manager.ts b/src/engine/manager.ts index c3a9b2c..8e7b99b 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -28,6 +28,7 @@ import {watchTemplateDirChanges} from "@nsm/engine/monitoring/templateDirWatcher import {processImage, init as initImageEngine, deleteImageIfUnused} from "@nsm/engine/image"; import {propagateOptionsToEnv} from "@nsm/engine/docker/util/env"; import {ActiveServiceSession, beginServiceSession, ServiceSession, init as initSessionEngine} from "@nsm/engine/session"; +import {AppConfig} from "@nsm/config"; export type Options = { /** @@ -320,7 +321,6 @@ export let engine: ServiceEngineI = undefined; export let nodeId: string; let db: Database; -let appConfig: any; // Save errors somewhere else? // Could it be a memory leak if there are tons of them?? @@ -343,18 +343,17 @@ const evtHandlers: Map[]> = new Map(); }; }); -export async function init(db_: Database, appConfig_: any, logger: winston.Logger) { - const nodeId_ = appConfig_['node_id'] as string; +export async function init(db_: Database, appConfig_: AppConfig, logger: winston.Logger) { + const nodeId_ = appConfig_.getNodeId(); logger.info(`Initializing service manager for node ${nodeId_}...`); db = db_; - appConfig = appConfig_; if (!engine) { // Init only if it has not already been force-initialized await initEngineForcibly(); } - nodeId = appConfig['node_id'] as string; + nodeId = nodeId_ as string; initImageEngine(engine, templateManager, templateDirWatcher, db_, currentContext.logger); initSessionEngine(db_); diff --git a/src/router/v1/service/lookupRoute.ts b/src/router/v1/service/lookupRoute.ts index d53857f..f9d10ec 100644 --- a/src/router/v1/service/lookupRoute.ts +++ b/src/router/v1/service/lookupRoute.ts @@ -7,11 +7,13 @@ export default async function ({manager}: AppContext): Promise { routes: { get: async (req, res) => { const id = req.params.id; + const service = await manager.getService(id, { includeSession: true }); if (!service) { res.status(404).json({status: 404, message: 'Invalid service ID.'}).end(); return; } + const session = service.internalSession; let stats: any; if (session && req.query.stats === 'true') { @@ -19,22 +21,24 @@ export default async function ({manager}: AppContext): Promise { } else { stats = null; } - // Build that info - res.json({ + + const data: any = { id: service.serviceId, templateId: service.template, state: service.state, port: service.port, options: service.options, - env: service.env, - ...(session ? { - session: { - id: service.session.id, - ...session, - stats, - } - } : {}) - }).end(); + env: service.env + }; + if (session) { + data.session = { + id: service.session.id, + startedAt: service.session.startedAt.getTime(), + stats, + }; + } + + res.json(data).end(); }, }, } diff --git a/src/router/v1/status/index.ts b/src/router/v1/status/index.ts index ed1615e..aa8497d 100644 --- a/src/router/v1/status/index.ts +++ b/src/router/v1/status/index.ts @@ -56,7 +56,7 @@ export default async function ({manager, appConfig, database}: AppContext): Prom url: '/status', routes: { get: async (req, res) => { - const nodeId = appConfig['node_id']; + const nodeId = appConfig.getNodeId(); const all = await database.permaRepository.listPerma(nodeId); const [free, size] = await manager.engine.calcHostUsage(); const system = { diff --git a/src/security/index.ts b/src/security/index.ts index 17a7141..ee20b05 100644 --- a/src/security/index.ts +++ b/src/security/index.ts @@ -3,9 +3,9 @@ import token from './token'; export default async function (ctx: AppContext) { // This code block is initialized before app routes. - if (ctx.appConfig['auth'] == 'auth_token') { + if (ctx.appConfig.getAuth() == 'auth_token') { // Basic credentials auth type await token(ctx); } - ctx.logger.info('Using ' + ctx.appConfig['auth'] + ' auth.'); + ctx.logger.info('Using ' + ctx.appConfig.getAuth() + ' auth.'); } \ No newline at end of file diff --git a/tests/engine/image.test.ts b/tests/engine/image.test.ts index af236a7..9a2bc0d 100644 --- a/tests/engine/image.test.ts +++ b/tests/engine/image.test.ts @@ -13,6 +13,7 @@ import {Template, TemplateManager} from "@nsm/engine/template"; import {TemplateDirWatcher} from "@nsm/engine/monitoring/templateDirWatcher"; import * as templateManager from "@nsm/engine/template"; import * as templateDirWatcher from "@nsm/engine/monitoring/templateDirWatcher"; +import {YamlAppConfig} from "@nsm/config"; let container: StartedMariaDbContainer; @@ -26,10 +27,10 @@ beforeAll(async () => { ] = await initDbContainerForTest(); container = container_; - engine = createEngine({ + engine = createEngine( // Just to prevent assertion errors - docker_host: "///var/run/docker.sock" - }); + new YamlAppConfig() + ); db = getDb(new PrismaClient({ datasourceUrl: dbUrl_, })); From 3ec24b892ccd8cb7fca8beb653984911c12db6ec Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 21 Mar 2026 20:19:56 +0100 Subject: [PATCH 47/52] feat: configuration rework, resources_path config option --- resources/config.yml | 5 ++- src/app.ts | 5 +-- src/config.ts | 34 +++++++++++++++++---- src/engine/monitoring/templateDirWatcher.ts | 4 +-- src/engine/monitoring/util.ts | 4 +-- src/engine/template.ts | 10 +++--- src/filestructure.ts | 18 +++++++++-- src/logger.ts | 8 ++--- src/resources.ts | 12 +++++--- 9 files changed, 71 insertions(+), 29 deletions(-) diff --git a/resources/config.yml b/resources/config.yml index ed25ea7..a610785 100644 --- a/resources/config.yml +++ b/resources/config.yml @@ -8,4 +8,7 @@ port: 3000 # Security # Supported types: 'none', 'auth_token' auth: 'none' -docker_host: 'unix:///var/run/docker.sock' \ No newline at end of file +docker_host: 'unix:///var/run/docker.sock' +# Override resources path if needed. +# By default, an explicit system-specific data path is used. +# resources_path: '/srv/resources' \ No newline at end of file diff --git a/src/app.ts b/src/app.ts index 93ed931..1deffd3 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,11 +1,13 @@ import dotenv from "dotenv"; import {loadAppConfig} from "@nsm/config"; +import {init as initFileStructure, getResourcesTargetPath} from "@nsm/filestructure"; // Load .env dotenv.config(); // Preload app config here to set needed env variables // before some modules require them. const appConfig = loadAppConfig(); +initFileStructure(appConfig); import {Router} from 'express'; import {Database} from "@nsm/database"; @@ -25,7 +27,6 @@ import {middleLayer} from "@nsm/engine/middle"; import {SessionManager} from "@nsm/engine/session"; import {mkdirResource, saveResource} from "@nsm/resources"; import path from "path"; -import {resourcesTargetPath} from "@nsm/filestructure"; import {AppConfig} from "@nsm/config"; export type AppBootContext = AppContext & { steps: any }; @@ -148,7 +149,7 @@ export const init = async (router: Application, options?: AppBootOptions): Promi } const prepareTestResources = () => { - if (fs.existsSync(path.join(resourcesTargetPath, 'templates', 'test'))) { + if (fs.existsSync(path.join(getResourcesTargetPath(), 'templates', 'test'))) { return; } diff --git a/src/config.ts b/src/config.ts index b74db0a..1076587 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,7 +1,8 @@ import {loadYamlFile} from "@nsm/util/yaml"; import path from "path"; -import {resourcesTargetPath} from "@nsm/filestructure"; +import {currentPaths} from "@nsm/filestructure"; import {saveResource} from "@nsm/resources"; +import z from "zod"; export interface AppConfig { getNodeId(): string; @@ -11,6 +12,8 @@ export interface AppConfig { getAuth(): string; getDockerHost(): string; + + getResourcesPath(): string|undefined; } /** @@ -19,12 +22,20 @@ export interface AppConfig { * @author ZorTik */ export class YamlAppConfig implements AppConfig { + private static readonly schema: z.ZodObject = z.object({ + node_id: z.string(), + port: z.number().int().positive(), + auth: z.string(), + docker_host: z.string(), + resources_path: z.string().optional() + }).strict(); + private readonly data: any; constructor() { this.data = YamlAppConfig.loadData(); - // TODO: validate + this.validate(); } getNodeId(): string { @@ -43,18 +54,29 @@ export class YamlAppConfig implements AppConfig { return this.data["docker_host"]; } + getResourcesPath(): string | undefined { + return this.data["resources_path"]; + } + + private validate = () => { + const result = YamlAppConfig.schema.safeParse(this.data); + if (!result.success) { + throw new Error('Invalid config file. ' + result.error.toString()); + } + } + private static loadData = () => { // Copy if it does not exist - saveResource('config.yml', 'config.yml', true); + saveResource('config.yml', 'config.yml', true, currentPaths.config); - const config = loadYamlFile(path.join(resourcesTargetPath, 'config.yml')); - for (let key in config) { + const config = loadYamlFile(path.join(currentPaths.config, 'config.yml')); + for (let key in YamlAppConfig.schema.shape) { // Overwrite with env variable if exists. // Sync const envKey = 'CONFIG_' + key.toUpperCase(); if (process.env[envKey]) { config[key] = process.env[envKey]; - } else { + } else if (config[key]) { process.env[envKey] = config[key]; } } diff --git a/src/engine/monitoring/templateDirWatcher.ts b/src/engine/monitoring/templateDirWatcher.ts index 00c763c..4cda3b5 100644 --- a/src/engine/monitoring/templateDirWatcher.ts +++ b/src/engine/monitoring/templateDirWatcher.ts @@ -5,7 +5,7 @@ import {getAllTemplates} from "@nsm/engine/template"; import winston from "winston"; import chokidar, {FSWatcher} from "chokidar"; import path from "path"; -import {templatesPath} from "@nsm/filestructure"; +import {getTemplatesPath} from "@nsm/filestructure"; export type TemplateDirWatcher = { @@ -47,7 +47,7 @@ export const watchTemplateDirChanges = (logger: winston.Logger) => { * When a template directory is removed, it stops watching that directory and removes its hash from the cache. */ const watchBaseDir = (logger: winston.Logger) => { - const dir = templatesPath; + const dir = getTemplatesPath(); const watcher = chokidar.watch(dir, { ignoreInitial: true, diff --git a/src/engine/monitoring/util.ts b/src/engine/monitoring/util.ts index 80cd1b2..112e2c7 100644 --- a/src/engine/monitoring/util.ts +++ b/src/engine/monitoring/util.ts @@ -1,9 +1,9 @@ import path from 'path'; -import {templatesPath} from "@nsm/filestructure"; +import {getTemplatesPath} from "@nsm/filestructure"; // Returns the build directory for the template export function templateBuildDir(template: string) { - return path.join(templatesPath, template); + return path.join(getTemplatesPath(), template); } /** diff --git a/src/engine/template.ts b/src/engine/template.ts index 81c5560..f6b7828 100644 --- a/src/engine/template.ts +++ b/src/engine/template.ts @@ -1,7 +1,7 @@ import {loadYamlFile} from "@nsm/util/yaml"; import * as fs from "fs"; import path from "path"; -import {templatesPath} from "@nsm/filestructure"; +import {getTemplatesPath} from "@nsm/filestructure"; export type Template = { /** @@ -53,7 +53,7 @@ export const getTemplate = (id: string): Template|null => { if (templateCache[id]) { return templateCache[id]; } - const settingsPath = path.join(templatesPath, id, 'settings.yml'); + const settingsPath = path.join(getTemplatesPath(), id, 'settings.yml'); if (!fs.existsSync(settingsPath)) { return null; } @@ -69,13 +69,13 @@ export const getTemplate = (id: string): Template|null => { } export const getAllTemplates = () => { - if (!fs.existsSync(templatesPath)) { + if (!fs.existsSync(getTemplatesPath())) { return []; } return fs - .readdirSync(templatesPath) - .filter(file => fs.statSync(path.join(templatesPath, file)).isDirectory()) + .readdirSync(getTemplatesPath()) + .filter(file => fs.statSync(path.join(getTemplatesPath(), file)).isDirectory()) .map(id => getTemplate(id)) .filter(template => template !== null); } diff --git a/src/filestructure.ts b/src/filestructure.ts index 7adf2ab..6f8fd95 100644 --- a/src/filestructure.ts +++ b/src/filestructure.ts @@ -1,10 +1,24 @@ import path from "path"; import envPaths, {Paths} from "env-paths"; +import {AppConfig} from "@nsm/config"; export const currentPaths: Paths = envPaths("nsm"); +let appConfig: AppConfig; + +export const init = (appConfig_: AppConfig) => { + appConfig = appConfig_; +} + // The local resources dir (not the source of truth) export const resourcesPath = path.join(process.cwd(), "resources"); // The target (platform-agnostic) resources dir (the source of truth) -export const resourcesTargetPath = path.join(currentPaths.data); -export const templatesPath = path.join(resourcesTargetPath, 'templates'); \ No newline at end of file +const defaultResourcesTargetPath = path.join(currentPaths.data); + +export const getResourcesTargetPath = () => { + return appConfig.getResourcesPath() ?? defaultResourcesTargetPath; +} + +export const getTemplatesPath = () => { + return path.join(getResourcesTargetPath(), 'templates') +} \ No newline at end of file diff --git a/src/logger.ts b/src/logger.ts index 184e6e7..9e88488 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,17 +1,17 @@ import winston from "winston"; import fs from "fs"; import path from "path"; -import {resourcesTargetPath} from "@nsm/filestructure"; +import {getResourcesTargetPath} from "@nsm/filestructure"; const { combine, timestamp, label, errors, printf } = winston.format; export function createLatestLogFile() { - if (fs.existsSync(path.join(resourcesTargetPath, 'logs', 'latest.log'))) { + if (fs.existsSync(path.join(getResourcesTargetPath(), 'logs', 'latest.log'))) { const date = new Date(Date.now()).toJSON().slice(2, 10) + '.' + new Date(Date.now()).getHours() + '.' + new Date(Date.now()).getMinutes(); - fs.renameSync(path.join(resourcesTargetPath, 'logs', 'latest.log'), path.join(resourcesTargetPath, 'logs', date + '.log')); + fs.renameSync(path.join(getResourcesTargetPath(), 'logs', 'latest.log'), path.join(getResourcesTargetPath(), 'logs', date + '.log')); } } @@ -31,7 +31,7 @@ export function createLogger(options?: { label?: string }) { ), transports: [ new winston.transports.Console(), - new winston.transports.File({dirname: path.join(resourcesTargetPath, 'logs'), filename: 'latest.log'}) + new winston.transports.File({dirname: path.join(getResourcesTargetPath(), 'logs'), filename: 'latest.log'}) ] }); } \ No newline at end of file diff --git a/src/resources.ts b/src/resources.ts index 962ff87..d735a3c 100644 --- a/src/resources.ts +++ b/src/resources.ts @@ -1,6 +1,6 @@ import path from "path"; import fs from "fs"; -import {resourcesPath, resourcesTargetPath} from "@nsm/filestructure"; +import {getResourcesTargetPath, resourcesPath} from "@nsm/filestructure"; /** * Reads resource from target dir. @@ -8,7 +8,7 @@ import {resourcesPath, resourcesTargetPath} from "@nsm/filestructure"; * @param name The name of the resource in the target dir. */ export const readResource = (name: string) => { - const p = path.join(resourcesTargetPath, name); + const p = path.join(getResourcesTargetPath(), name); return fs.readFileSync(p, 'utf8'); } @@ -19,7 +19,7 @@ export const readResource = (name: string) => { * @param name The name of the dir in the target dir. */ export const mkdirResource = (name: string) => { - const p = path.join(resourcesTargetPath, name); + const p = path.join(getResourcesTargetPath(), name); fs.mkdirSync(p, { recursive: true }); } @@ -30,13 +30,15 @@ export const mkdirResource = (name: string) => { * @param name The name of the resource in the resources dir. * @param targetName The target name where to copy. * @param skipIfExists If true, the resource will not be copied if a file with the same name already exists in the target dir. Default is false. + * @param targetDirPath The target dir path. Default is the platform-agnostic resources dir. */ export const saveResource = ( name: string, targetName: string, - skipIfExists: boolean = false + skipIfExists: boolean = false, + targetDirPath: string = getResourcesTargetPath() ) => { - const targetPath = path.join(resourcesTargetPath, targetName); + const targetPath = path.join(targetDirPath, targetName); // Create parent dirs if missing fs.mkdirSync(path.dirname(targetPath), { recursive: true }); From 71b09f2ded89e02ef03958446b36c9eaf7b70ed9 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Tue, 31 Mar 2026 18:05:12 +0200 Subject: [PATCH 48/52] fixes --- src/app.ts | 4 +- src/engine/docker/action/build.ts | 65 +++++++++++++++++-------------- src/engine/middle.ts | 9 ++--- src/filestructure.ts | 22 +++++++++++ 4 files changed, 63 insertions(+), 37 deletions(-) diff --git a/src/app.ts b/src/app.ts index 1deffd3..a84cc8b 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,6 +1,6 @@ import dotenv from "dotenv"; import {loadAppConfig} from "@nsm/config"; -import {init as initFileStructure, getResourcesTargetPath} from "@nsm/filestructure"; +import {init as initFileStructure, getResourcesTargetPath, prepareFolders} from "@nsm/filestructure"; // Load .env dotenv.config(); @@ -86,6 +86,8 @@ export const init = async (router: Application, options?: AppBootOptions): Promi // Prepare logging const logger = initGlobalLogger(); + prepareFolders(); + // Prepare templates folder mkdirResource("templates"); if (options?.test === true) { diff --git a/src/engine/docker/action/build.ts b/src/engine/docker/action/build.ts index 9accc22..370e4eb 100644 --- a/src/engine/docker/action/build.ts +++ b/src/engine/docker/action/build.ts @@ -8,6 +8,7 @@ import {clock} from "@nsm/util/clock"; import {Worker} from "worker_threads"; import {getRootFilesFiltered} from "@nsm/engine/ignore"; import {Paths} from "env-paths"; +import {getTempPath} from "@nsm/filestructure"; async function prepareImage( args: { @@ -65,7 +66,7 @@ async function prepareImage( resolve(msg); } } - if (ctx.workers) { + /*if (ctx.workers) { // Build using workers const w = new Worker(__dirname + path.sep + 'build.worker.js', { workerData: { @@ -78,37 +79,38 @@ async function prepareImage( }); w.on('message', msgHandler); } else { - // In container, worker threads are not supported. Or they - // are disabled. - client.buildImage(archive, { t: imageTag, buildargs: env }).then(stream => { - logs.push('--------- Begin Build Log ---------'); - client.modem.followProgress(stream, (err, res) => { - if (err) { - console.error(err); - } else { - let errorOccurred = false; - res.forEach(r => { - if (r.errorDetail) { - errorOccurred = true; + // Here comes the normal build + }*/ + // In container, worker threads are not supported. Or they + // are disabled. + client.buildImage(archive, { t: imageTag, buildargs: env }).then(stream => { + logs.push('--------- Begin Build Log ---------'); + client.modem.followProgress(stream, (err, res) => { + if (err) { + console.error(err); + } else { + let errorOccurred = false; + res.forEach(r => { + if (r.errorDetail) { + errorOccurred = true; - reject(r.errorDetail); - } else { - const msg = r.stream?.trim(); + reject(r.errorDetail); + } else { + const msg = r.stream?.trim(); - logs.push(msg); - } - }); - if (errorOccurred) { - return; - } - logs.push('--------- End Of Build Log ---------\n'); - fs.unlinkSync(archive); - msgHandler(logs); - msgHandler(imageTag); - } - }); + logs.push(msg); + } }); - } + if (errorOccurred) { + return; + } + logs.push('--------- End Of Build Log ---------\n'); + fs.unlinkSync(archive); + msgHandler(logs); + msgHandler(imageTag); + } + }); + }); }).finally(() => { // Clean up archive file if it still exists try { @@ -123,7 +125,10 @@ async function prepareImage( } export default function (client: DockerClient, paths: Paths): ServiceEngine['build'] { - const arDir = path.join(paths.temp, "archives"); + const arDir = path.join(getTempPath(), "archives"); + if (!fs.existsSync(arDir)) { + fs.mkdirSync(arDir, { recursive: true }); + } return async (imageId, buildDir, options, messageListener) => { const imageBuildClock = clock(); diff --git a/src/engine/middle.ts b/src/engine/middle.ts index d5cf6a4..2b736c6 100644 --- a/src/engine/middle.ts +++ b/src/engine/middle.ts @@ -23,12 +23,7 @@ export interface ErrorPublisher { } const publishers: ErrorPublisher[] = [ - { - // Logger publisher - async publishError(action: ServiceActionError) { - currentContext.logger.error(`${action.serviceId ? `Service ${action.serviceId} f` : "F"}ailed action ${action.type}: ${action.message}`); - } - } + // The publishers ]; /** @@ -74,6 +69,8 @@ const decorateFunc = ) => Promise>( }; await publishError(action); + currentContext.logger.error(`${action.serviceId ? `Service ${action.serviceId} f` : "F"}ailed action ${action.type}: ${action.message}`, e); + throw e; } } diff --git a/src/filestructure.ts b/src/filestructure.ts index 6f8fd95..84186e9 100644 --- a/src/filestructure.ts +++ b/src/filestructure.ts @@ -1,6 +1,7 @@ import path from "path"; import envPaths, {Paths} from "env-paths"; import {AppConfig} from "@nsm/config"; +import fs from "fs"; export const currentPaths: Paths = envPaths("nsm"); @@ -21,4 +22,25 @@ export const getResourcesTargetPath = () => { export const getTemplatesPath = () => { return path.join(getResourcesTargetPath(), 'templates') +} + +export const getTempPath = () => { + return currentPaths.temp; +} + +export const prepareFolders = () => { + const resourcesTargetPath = getResourcesTargetPath(); + if (!fs.existsSync(resourcesTargetPath)) { + fs.mkdirSync(resourcesTargetPath, { recursive: true }); + } + + const templatesPath = getTemplatesPath(); + if (!fs.existsSync(templatesPath)) { + fs.mkdirSync(templatesPath, { recursive: true }); + } + + const tempPath = getTempPath(); + if (!fs.existsSync(tempPath)) { + fs.mkdirSync(tempPath, { recursive: true }); + } } \ No newline at end of file From 9233791a4419d2bfd3a8f745e5172a1afb85cf6e Mon Sep 17 00:00:00 2001 From: ZorTik Date: Tue, 31 Mar 2026 18:17:46 +0200 Subject: [PATCH 49/52] refactor: change node version in Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 839eba5..7cfc4b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18 +FROM node:22 WORKDIR /data From ff382b0968bddbccef033270fe4a76bde06e05a0 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 18 Apr 2026 00:28:56 +0200 Subject: [PATCH 50/52] feat: defaultResourcesTargetPath --- src/filestructure.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/filestructure.ts b/src/filestructure.ts index 6f8fd95..a90a78e 100644 --- a/src/filestructure.ts +++ b/src/filestructure.ts @@ -12,11 +12,10 @@ export const init = (appConfig_: AppConfig) => { // The local resources dir (not the source of truth) export const resourcesPath = path.join(process.cwd(), "resources"); -// The target (platform-agnostic) resources dir (the source of truth) -const defaultResourcesTargetPath = path.join(currentPaths.data); +// The target (platform-agnostic) resources dir (the source of truth) export const getResourcesTargetPath = () => { - return appConfig.getResourcesPath() ?? defaultResourcesTargetPath; + return appConfig.getResourcesPath() ?? path.join(currentPaths.data); } export const getTemplatesPath = () => { From 943d77b09597a6c210763c13f441eb91c814142d Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 18 Apr 2026 02:40:58 +0200 Subject: [PATCH 51/52] fix: tests --- jest.config.js | 6 +++--- package.json | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jest.config.js b/jest.config.js index af3369b..216f8fc 100644 --- a/jest.config.js +++ b/jest.config.js @@ -2,10 +2,10 @@ const tsconfig = require("./tsconfig.json") const moduleNameMapper = require("tsconfig-paths-jest")(tsconfig) module.exports = { - "modulePathIgnorePatterns": [ - "/tests/" - ], moduleNameMapper, + transformIgnorePatterns: [ + "/node_modules/(?!(env-paths)/)", + ], reporters: [ 'default', ['jest-ctrf-json-reporter', {}], diff --git a/package.json b/package.json index 19688ce..6ad3e3b 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "node installTempDeps.js && tsc && tscp", "start": "cross-env TS_NODE_BASEURL=./dist node -r tsconfig-paths/register dist/index.js", - "test": "npm run build && cross-env TS_NODE_BASEURL=./dist jest" + "test": "jest" }, "keywords": [], "author": "ZorTik", @@ -54,7 +54,7 @@ "crypto": "^1.0.1", "dockerode": "^4.0.2", "dotenv": "^16.4.5", - "env-paths": "^4.0.0", + "env-paths": "^3.0.0", "express": "^4.18.2", "express-fileupload": "^1.5.0", "folder-hash": "^4.1.1", From 0dc61f61dc34e6eaee1d764c847d73baea714621 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 18 Apr 2026 03:04:01 +0200 Subject: [PATCH 52/52] fix: port parsing when inserted through env --- src/config.ts | 3 ++- tests/api/api.test.ts | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/config.ts b/src/config.ts index 1076587..a032dbd 100644 --- a/src/config.ts +++ b/src/config.ts @@ -24,7 +24,8 @@ export interface AppConfig { export class YamlAppConfig implements AppConfig { private static readonly schema: z.ZodObject = z.object({ node_id: z.string(), - port: z.number().int().positive(), + // Coerce port to auto-parse from env if overwritten + port: z.coerce.number().int().positive(), auth: z.string(), docker_host: z.string(), resources_path: z.string().optional() diff --git a/tests/api/api.test.ts b/tests/api/api.test.ts index 8730f03..4e79651 100644 --- a/tests/api/api.test.ts +++ b/tests/api/api.test.ts @@ -120,14 +120,12 @@ describe("Test v1 API models", () => { expect(res.status).toBe(200); expectProps(res.body, [ "id", id, - "template.id", "test", - "template.name", undefined, - "template.description", undefined, - "template.settings", undefined, + "templateId", "test", "port", undefined, "options", undefined, "env", undefined, - "session.containerId", undefined, + "session.id", undefined, + "session.startedAt", undefined, ]); }, 20000);