From db5f3a38e571072677ed9f6d541cde1ca62630b8 Mon Sep 17 00:00:00 2001 From: Spencer Bliven Date: Tue, 20 May 2025 09:47:35 +0200 Subject: [PATCH 1/6] Use yaml to parse all JSON files (fix #1120) This is backwards compatible. No changes to existing json files are needed (although yaml features like comments can be added if desired). --- src/common/utils.ts | 10 +++++++++ src/config/configuration.ts | 33 ++++++++++++++++------------- src/users/dto/create-user.dto.ts | 36 ++++++++++++++++++++++++++++++++ src/users/users.service.ts | 11 ++++++++-- 4 files changed, 74 insertions(+), 16 deletions(-) diff --git a/src/common/utils.ts b/src/common/utils.ts index 5907c736e..401658fbf 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -1135,6 +1135,16 @@ export function oneOrMore(x: T[] | T): T[] { return Array.isArray(x) ? x : [x]; } +/** + * Given a type guard for T, check whether an object is an array of T + */ +export function isArrayOf( + itemGuard: (item: unknown) => item is T, +): (arr: unknown) => arr is T[] { + return (arr: unknown): arr is T[] => + Array.isArray(arr) && arr.every(itemGuard); +} + /** * Helper function to generate HttpExceptions * @param message error message diff --git a/src/config/configuration.ts b/src/config/configuration.ts index 6975135c0..17f73a6b4 100644 --- a/src/config/configuration.ts +++ b/src/config/configuration.ts @@ -4,6 +4,7 @@ import localconfiguration from "./localconfiguration"; import { boolean } from "mathjs"; import { DEFAULT_PROPOSAL_TYPE } from "src/proposals/schemas/proposal.schema"; import { DatasetType } from "src/datasets/types/dataset-type.enum"; +import { load } from "js-yaml"; const configuration = () => { const accessGroupsStaticValues = @@ -48,11 +49,11 @@ const configuration = () => { modulePath: "./loggingProviders/defaultLogger", config: {}, }; - const jsonConfigMap: { [key: string]: object | object[] | boolean } = { + const jsonConfigMap: { [key: string]: unknown } = { datasetTypes: {}, proposalTypes: {}, }; - const jsonConfigFileList: { [key: string]: string } = { + const yamlConfigFileList: { [key: string]: string } = { frontendConfig: process.env.FRONTEND_CONFIG_FILE || "./src/config/frontend.config.json", frontendTheme: @@ -62,12 +63,12 @@ const configuration = () => { proposalTypes: process.env.PROPOSAL_TYPES_FILE || "proposalTypes.json", metricsConfig: process.env.METRICS_CONFIG_FILE || "metricsConfig.json", }; - Object.keys(jsonConfigFileList).forEach((key) => { - const filePath = jsonConfigFileList[key]; + Object.keys(yamlConfigFileList).forEach((key) => { + const filePath = yamlConfigFileList[key]; if (fs.existsSync(filePath)) { const data = fs.readFileSync(filePath, "utf8"); try { - jsonConfigMap[key] = JSON.parse(data); + jsonConfigMap[key] = load(data, { filename: filePath }); } catch (error) { console.error( "Error json config file parsing " + filePath + " : " + error, @@ -77,15 +78,19 @@ const configuration = () => { } }); - // NOTE: Add the default dataset types here - Object.assign(jsonConfigMap.datasetTypes, { - Raw: DatasetType.Raw, - Derived: DatasetType.Derived, - }); - // NOTE: Add the default proposal type here - Object.assign(jsonConfigMap.proposalTypes, { - DefaultProposal: DEFAULT_PROPOSAL_TYPE, - }); + if (jsonConfigMap.datasetTypes) { + // NOTE: Add the default dataset types here + Object.assign(jsonConfigMap.datasetTypes, { + Raw: DatasetType.Raw, + Derived: DatasetType.Derived, + }); + } + if (jsonConfigMap.proposalTypes) { + // NOTE: Add the default proposal type here + Object.assign(jsonConfigMap.proposalTypes, { + DefaultProposal: DEFAULT_PROPOSAL_TYPE, + }); + } const oidcFrontendClients = (() => { const clients = ["scicat"]; diff --git a/src/users/dto/create-user.dto.ts b/src/users/dto/create-user.dto.ts index dd30ebd99..983a59a7f 100644 --- a/src/users/dto/create-user.dto.ts +++ b/src/users/dto/create-user.dto.ts @@ -1,4 +1,5 @@ import { ApiProperty } from "@nestjs/swagger"; +import { isArrayOf } from "../../common/utils"; export class CreateUserDto { @ApiProperty({ required: true }) @@ -19,3 +20,38 @@ export class CreateUserDto { @ApiProperty({ required: false }) readonly authStrategy?: string; } + +// Type guard for CreateUserDto +export function isCreateUserDto(obj: unknown): obj is CreateUserDto { + if (typeof obj !== "object" || obj === null) return false; + + const allowedKeys = [ + "username", + "email", + // optional + "password", + "role", + "global", + "authStrategy", + ]; + + // Check for extra fields + const objKeys = Object.keys(obj); + if (!objKeys.every((key) => allowedKeys.includes(key))) return false; + if (!allowedKeys.slice(0, 2).every((key) => objKeys.includes(key))) + return false; // username & email required + + // Type checks + if (typeof obj.username !== "string") return false; + if (typeof obj.email !== "string") return false; + if ("password" in obj && typeof obj.password !== "string") return false; + if ("role" in obj && typeof obj.role !== "string") return false; + if ("global" in obj && typeof obj.global !== "boolean") return false; + if ("authStrategy" in obj && typeof obj.authStrategy !== "string") + return false; + + return true; +} + +// Type guard for CreateUserDto[] +export const isCreateUserDtoArray = isArrayOf(isCreateUserDto); diff --git a/src/users/users.service.ts b/src/users/users.service.ts index 767e18d30..2700aca13 100644 --- a/src/users/users.service.ts +++ b/src/users/users.service.ts @@ -4,7 +4,7 @@ import { InjectModel } from "@nestjs/mongoose"; import { genSalt, hash } from "bcrypt"; import { FilterQuery, Model } from "mongoose"; import { CreateUserIdentityDto } from "./dto/create-user-identity.dto"; -import { CreateUserDto } from "./dto/create-user.dto"; +import { CreateUserDto, isCreateUserDtoArray } from "./dto/create-user.dto"; import { RolesService } from "./roles.service"; import { UserIdentity, @@ -31,6 +31,7 @@ import { UserPayload } from "src/auth/interfaces/userPayload.interface"; import { AccessGroupService } from "src/auth/access-group-provider/access-group.service"; import { ReturnedUserDto } from "./dto/returned-user.dto"; import { UpdateUserDto } from "./dto/update-user.dto"; +import { load } from "js-yaml"; @Injectable() export class UsersService implements OnModuleInit { @@ -55,7 +56,13 @@ export class UsersService implements OnModuleInit { const filePath = this.configService.get("functionalAccounts.file"); if (fs.existsSync(filePath)) { const data = fs.readFileSync(filePath, "utf8"); - functionalAccounts = JSON.parse(data); + const parsedData = load(data, { filename: filePath }); + if (!parsedData || !isCreateUserDtoArray(parsedData)) { + throw new Error( + `Invalid functional accounts file format. Expected an object but got ${typeof parsedData}`, + ); + } + functionalAccounts = parsedData; } if (functionalAccounts && functionalAccounts.length > 0) { From 9c60378751df0245d4fad9bb74fb68a66ec21a14 Mon Sep 17 00:00:00 2001 From: Spencer Bliven Date: Tue, 20 May 2025 10:52:06 +0200 Subject: [PATCH 2/6] Mention yaml support in README --- README.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 69ad78e16..26cf78515 100644 --- a/README.md +++ b/README.md @@ -99,30 +99,40 @@ automatically create the specified accounts on startup. If this file is not prov Follow the structure of [functionalAccounts.json.minimal.example](/functionalAccounts.json.minimal.example) to create your own _functionalAccounts.json_ file. +YAML syntax is also supported. Set `FUNCTIONAL_ACCOUNTS_FILE=functionalAccounts.yaml` in your environment. + ## Frontend configuration and theme The SciCat backend provides functionality to serve a configuration and theme to a connected frontend. The default files are _frontend.config.json_ and _frontend.theme.json_ located in the `/src/config` directory, locally or in the container. The file names and locations can be configured via the environment variables `FRONTEND_CONFIG_FILE` and `FRONTEND_THEME_FILE`. Follow the structure of the provided [frontend.config.json](/frontend.config.json) and [frontend.theme.json](/frontend.theme.json) to create your own files. +YAML syntax is also supported. Set `FRONTEND_CONFIG_FILE=frontend.config.yaml` in your environment. + ### Loggers configuration Providing a file called _loggers.json_ at the root of the project, locally or in the container, and create an external logger class in the `src/loggers/loggingProviders/`directory will automatically create specified one or multiple loggers instances. The `loggers.json.example` file in the root directory showcases the example of configuration structure for the one or multiple loggers. `logger.service.ts` file contains the configuration handling process logic, and `src/loggers/loggingProviders/grayLogger.ts` includes actual usecase of grayLogger. +YAML syntax is also supported. Set `LOGGERS_CONFIG_FILE=loggers.yaml` in your environment. + ### Proposal types configuration Providing a file called _proposalTypes.json_ at the root of the project, locally or in the container, will be automatically loaded into the application configuration service under property called `proposalTypes` and used for validation against proposal creation and update. The `proposalTypes.json.example` file in the root directory showcases the example of configuration structure for proposal types. +YAML syntax is also supported. Set `PROPOSAL_TYPES_FILE=proposalTypes.yaml` in your environment. + ### Dataset types configuration When providing a file called _datasetTypes.json_ at the root of the project, locally or in the container, it will be automatically loaded into the application configuration service under property called `datasetTypes` and used for validation against dataset creation and update. The types `Raw` and `Derived` are always valid dataset types by default. The `datasetTypes.json.example` file in the root directory showcases an example of configuration structure for dataset types. +YAML syntax is also supported. Set `DATASET_TYPES_FILE=datasetTypes.yaml` in your environment. + ## Environment variables Valid environment variables for the .env file. See [.env.example](/.env.example) for examples value formats. @@ -210,13 +220,13 @@ Valid environment variables for the .env file. See [.env.example](/.env.example) | `ES_MAX_RESULT` | number | | Maximum records that can be indexed into Elasticsearch. | 10000 | | `ES_FIELDS_LIMIT` | number | | The total number of fields in an index. | 1000 | | `ES_REFRESH` | string | | If set to `wait_for`, Elasticsearch will wait till data is inserted into the specified index before returning a response. | false | -| `FRONTEND_CONFIG_FILE` | string | | The file name for frontend configuration, located in the `/src/config` directory by default. | "./src/config/frontend.config.json" | -| `FRONTEND_THEME_FILE` | string | | The file name for frontend theme, located in the `/src/config` directory by default. | "./src/config/frontend.theme.json" | -| `LOGGERS_CONFIG_FILE` | string | | The file name for loggers configuration, located in the project root directory. | "loggers.json" | -| `PROPOSAL_TYPES_FILE` | string | | The file name for proposal types configuration, located in the project root directory. | "proposalTypes.json" | +| `FRONTEND_CONFIG_FILE` | string | | The file name for frontend configuration (yaml/json), located in the `/src/config` directory by default. | "./src/config/frontend.config.json" | +| `FRONTEND_THEME_FILE` | string | | The file name for frontend theme (yaml/json), located in the `/src/config` directory by default. | "./src/config/frontend.theme.json" | +| `LOGGERS_CONFIG_FILE` | string | | The file name for loggers configuration (yaml/json), located in the project root directory. | "loggers.json" | +| `PROPOSAL_TYPES_FILE` | string | | The file name for proposal types configuration (yaml/json), located in the project root directory. | "proposalTypes.json" | | `SWAGGER_PATH` | string | Yes | swaggerPath is the path where the swagger UI will be available. | "explorer"| | `MAX_FILE_UPLOAD_SIZE` | string | Yes | Maximum allowed file upload size. | "16mb"| -| `FUNCTIONAL_ACCOUNTS_FILE` | string | Yes | The file name for functional accounts, relative to the project root directory | "functionalAccounts.json"| +| `FUNCTIONAL_ACCOUNTS_FILE` | string | Yes | The file name for functional accounts (yaml/json), relative to the project root directory | "functionalAccounts.json"| | `JOB_CONFIGURATION_FILE` | string | Yes | Path of a job configuration file (conventionally `"jobConfig.yaml"`). If unset, jobs are disabled | | | `JOB_DEFAULT_STATUS_CODE` | string | Yes | Default statusCode for new jobs | "jobSubmitted" | | `JOB_DEFAULT_STATUS_MESSAGE | string | Yes | Default statusMessage for new jobs | "Job submitted." | From c62289bca26f0b97899240b357ba031fda0134ed Mon Sep 17 00:00:00 2001 From: Spencer Bliven Date: Tue, 20 May 2025 11:46:33 +0200 Subject: [PATCH 3/6] Fix error --- src/users/dto/create-user.dto.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/users/dto/create-user.dto.ts b/src/users/dto/create-user.dto.ts index 983a59a7f..2528d3d19 100644 --- a/src/users/dto/create-user.dto.ts +++ b/src/users/dto/create-user.dto.ts @@ -38,12 +38,10 @@ export function isCreateUserDto(obj: unknown): obj is CreateUserDto { // Check for extra fields const objKeys = Object.keys(obj); if (!objKeys.every((key) => allowedKeys.includes(key))) return false; - if (!allowedKeys.slice(0, 2).every((key) => objKeys.includes(key))) - return false; // username & email required // Type checks - if (typeof obj.username !== "string") return false; - if (typeof obj.email !== "string") return false; + if (!("username" in obj) || typeof obj.username !== "string") return false; + if (!("email" in obj) || typeof obj.email !== "string") return false; if ("password" in obj && typeof obj.password !== "string") return false; if ("role" in obj && typeof obj.role !== "string") return false; if ("global" in obj && typeof obj.global !== "boolean") return false; From dcad919c00ed88151534365a703d3d5006c8bae6 Mon Sep 17 00:00:00 2001 From: Max Novelli Date: Tue, 2 Sep 2025 15:28:19 +0200 Subject: [PATCH 4/6] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e76979b9..2a70f185b 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,7 @@ Valid environment variables for the .env file. See [.env.example](/.env.example) | `RABBITMQ_HOSTNAME` | string | Yes | The hostname of the RabbitMQ message broker. Only required if RabbitMQ is enabled. | | | `RABBITMQ_USERNAME` | string | Yes | The username used to authenticate to the RabbitMQ message broker. Only required if RabbitMQ is enabled. | | | `RABBITMQ_PASSWORD` | string | Yes | The password used to authenticate to the RabbitMQ message broker. Only required if RabbitMQ is enabled. | | +| `RABBITMQ_PORT`| number | Yes | The port of the RabbitMQ message broker. Only required if RabbitMQ is enabled. |5672| | `REGISTER_DOI_URI`| string | | URI to the organization that registers the facility's DOIs. |"https://mds.test.datacite.org/doi"| | `REGISTER_METADATA_URI`| string | | URI to the organization that registers the facility's published data metadata. |"https://mds.test.datacite.org/metadata"| | `DOI_USERNAME`| string | Yes | DOI Username. |"username"| @@ -242,7 +243,7 @@ Valid environment variables for the .env file. See [.env.example](/.env.example) | `FUNCTIONAL_ACCOUNTS_FILE` | string | Yes | The file name for functional accounts (yaml/json), relative to the project root directory | "functionalAccounts.json"| | `JOB_CONFIGURATION_FILE` | string | Yes | Path of a job configuration file (conventionally `"jobConfig.yaml"`). If unset, jobs are disabled | | | `JOB_DEFAULT_STATUS_CODE` | string | Yes | Default statusCode for new jobs | "jobSubmitted" | -| `JOB_DEFAULT_STATUS_MESSAGE | string | Yes | Default statusMessage for new jobs | "Job submitted." | +| `JOB_DEFAULT_STATUS_MESSAGE' | string | Yes | Default statusMessage for new jobs | "Job submitted." | | `MASK_PERSONAL_INFO` | string | Yes | When enabled all emails and orcid from HTTP responses are masked. Values "yes" or "no". | "no" | ## Migrating from the old SciCat Backend From cb485635bb6e4d8f5df576de430cc80e38814bb3 Mon Sep 17 00:00:00 2001 From: Jay Date: Thu, 6 Nov 2025 14:54:43 +0100 Subject: [PATCH 5/6] Resolve merge conflicts --- src/common/utils.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/common/utils.ts b/src/common/utils.ts index fd5cbbfee..9cd041fc1 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -1212,6 +1212,9 @@ export function isArrayOf( ): (arr: unknown) => arr is T[] { return (arr: unknown): arr is T[] => Array.isArray(arr) && arr.every(itemGuard); +} + +/** * Make a single property K of T optional */ export type MakeOptional = Omit & From 111b7237570f349a9510dbbffabe909751e07dad Mon Sep 17 00:00:00 2001 From: Jay Date: Thu, 6 Nov 2025 15:00:43 +0100 Subject: [PATCH 6/6] Update README --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 06af0837b..49661fd8f 100644 --- a/README.md +++ b/README.md @@ -243,14 +243,14 @@ Valid environment variables for the .env file. See [.env.example](/.env.example) |`STACK_VERSION` | string | Yes | Defines the Elasticsearch version to deploy | "8.8.2" | |`CLUSTER_NAME` | string | Yes | Sets the name of the Elasticsearch cluster | "es-cluster" | |`MEM_LIMIT`| string | Yes | Specifies the max memory for Elasticsearch container (or process) | "4G" | -|`FRONTEND_CONFIG_FILE`| string | | The file name for frontend configuration, located in the`/src/config`directory by default. | "./src/config/frontend.config.json" | -|`FRONTEND_THEME_FILE`| string | | The file name for frontend theme, located in the`/src/config`directory by default. | "./src/config/frontend.theme.json" | -|`LOGGERS_CONFIG_FILE`| string | | The file name for loggers configuration, located in the project root directory. | "loggers.json" | -|`PROPOSAL_TYPES_FILE`| string | | The file name for proposal types configuration, located in the project root directory. | "proposalTypes.json" | +|`FRONTEND_CONFIG_FILE`| string | | The file name for frontend configuration (yaml/json), located in the`/src/config`directory by default. | "./src/config/frontend.config.json" | +|`FRONTEND_THEME_FILE`| string | | The file name for frontend theme (yaml/json), located in the`/src/config`directory by default. | "./src/config/frontend.theme.json" | +|`LOGGERS_CONFIG_FILE`| string | | The file name for loggers configuration (yaml/json), located in the project root directory. | "loggers.json" | +|`PROPOSAL_TYPES_FILE`| string | | The file name for proposal types configuration (yaml/json), located in the project root directory. | "proposalTypes.json" | |`DATASET_TYPES_FILE`| string | | | "datasetTypes.json" | |`SWAGGER_PATH`| string | Yes | swaggerPath is the path where the swagger UI will be available. | "explorer"| |`MAX_FILE_UPLOAD_SIZE`| string | Yes | Maximum allowed file upload size. | "16mb"| -|`FUNCTIONAL_ACCOUNTS_FILE`| string | Yes | The file name for functional accounts, relative to the project root directory | "functionalAccounts.json"| +|`FUNCTIONAL_ACCOUNTS_FILE`| string | Yes | The file name for functional accounts (yaml/json), relative to the project root directory | "functionalAccounts.json"| |`JOB_CONFIGURATION_FILE`| string | Yes | Path of a job configuration file (conventionally`"jobConfig.yaml"`). If unset, jobs are disabled | | |`JOB_DEFAULT_STATUS_CODE`| string | Yes | Default statusCode for new jobs | "jobSubmitted" | |`JOB_DEFAULT_STATUS_MESSAGE` | string | Yes | Default statusMessage for new jobs | "Job submitted." |