chore(master): release 2.1.1

.github/workflows/pull_request.yaml

@@ -4,7 +4,7 @@ on: [pull_request]
 
 jobs:
   pull_request:
-    uses: MapColonies/shared-workflows/.github/workflows/pull_request.yaml@v5
+    uses: MapColonies/shared-workflows/.github/workflows/pull_request.yaml@v5.1.0
     secrets: inherit
     with:
       openApiFilePath: './bundledApi.yaml'

.release-please-manifest.json

@@ -1 +1 @@
-{".":"2.0.3"}
+{".":"2.1.1"}

CHANGELOG.md

@@ -2,6 +2,20 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [2.1.1](https://github.com/MapColonies/store-trigger/compare/v2.1.0...v2.1.1) (2026-06-14)
+
+
+### Bug Fixes
+
+* fixed tilesetFilename added to the path (MAPCO-10817) ([#63](https://github.com/MapColonies/store-trigger/issues/63)) ([8f14997](https://github.com/MapColonies/store-trigger/commit/8f149978ea57d2f01b9900f86c48205fc4a766ed))
+
+## [2.1.0](https://github.com/MapColonies/store-trigger/compare/v2.0.3...v2.1.0) (2026-05-24)
+
+
+### Features
+
+* removed blacklist and updated crawling (MAPCO-10530) ([#60](https://github.com/MapColonies/store-trigger/issues/60)) ([00cc3a1](https://github.com/MapColonies/store-trigger/commit/00cc3a1613e8c7140999ba045a70a3905c782b56))
+
 ## [2.0.3](https://github.com/MapColonies/store-trigger/compare/v2.0.2...v2.0.3) (2026-01-05)
 
 

bundledApi.yaml

@@ -2,7 +2,7 @@ openapi: 3.0.1
 info:
   title: 3d store-trigger
   description: Ingesting and exporting 3d model to S3/NFS
-  version: 2.0.3
+  version: 2.1.1
   license:
     name: MIT
     url: https://opensource.org/licenses/MIT

config/custom-environment-variables.json

@@ -68,9 +68,16 @@
   "NFS": {
     "pvPath": "PV_SOURCE_PATH"
   },
+  "crawling": {
+    "extension": "CRAWLING_EXTENSION",
+    "nestedJsonPath": "CRAWLING_NESTED_JSON_PATH",
+    "ignoreNotFound": {
+      "__name": "CRAWLING_IGNORE_NOT_FOUND",
+      "__format": "boolean"
+    }
+  },
   "ingestion": {
-    "provider": "PROVIDER_FROM",
-    "blackList": "BLACK_LIST"
+    "provider": "PROVIDER_FROM"
   },
   "jobManager": {
     "url": "JOB_MANAGER_URL",

config/default.json

@@ -46,9 +46,13 @@
   "NFS": {
     "pvPath": "/Path/To/Models"
   },
+  "crawling": {
+    "extension": ".json",
+    "nestedJsonPath": "$..['uri','url']",
+    "ignoreNotFound": true
+  },
   "ingestion": {
-    "provider": "NFS",
-    "blackList": ["tar", "zip", "rar", "7z"]
+    "provider": "NFS"
   },
   "jobManager": {
     "url": "http://127.0.0.1:8080",

config/test.json

@@ -1,4 +1,9 @@
 {
+  "crawling": {
+    "extension": ".json",
+    "nestedJsonPath": "$..['uri','url']",
+    "ignoreNotFound": true
+  },
   "S3": {
     "accessKeyId": "minioadmin",
     "secretAccessKey": "minioadmin",
@@ -13,8 +18,7 @@
     "pvPath": "./tests/helpers/3DModels"
   },
   "ingestion": {
-    "provider": "S3",
-    "blackList": ["tar", "zip", "rar", "7z"]
+    "provider": "S3"
   },
   "jobManager": {
     "url": "http://127.0.0.1:8080",

helm/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: store-trigger
 description: A Helm chart for store-trigger service
 type: application
-version: 2.0.3
-appVersion: 2.0.3
+version: 2.1.1
+appVersion: 2.1.1
 dependencies:
   - name: mc-labels-and-annotations
     version: 0.7.0

helm/templates/configmap.yaml

@@ -26,7 +26,6 @@ data:
   REQUESTS_CA_BUNDLE: {{ printf "%s/%s" $ca.path $ca.key | quote }}
   NODE_EXTRA_CA_CERTS: {{ printf "%s/%s" $ca.path $ca.key | quote }}
   {{- end }}
-  BLACK_LIST: {{ .Values.env.blackList | quote }}
   MAX_CONCURRENCY: {{ .Values.env.maxConcurrency | quote }}
   {{ if eq $provider "S3" }}
   {{- $S3 := (include "merged.S3" . ) | fromYaml }}
@@ -50,3 +49,6 @@ data:
   INGESTION_TASK_BATCHES: {{ $jobManager.ingestion.batches | quote}}
   JOB_DELETE_TYPE: {{ $jobManager.delete.jobType | quote }} 
   TASK_DELETE_TYPE: {{ $jobManager.delete.taskType | quote }}
+  CRAWLING_EXTENSION: {{ .Values.env.crawling.extension | quote }}
+  CRAWLING_NESTED_JSON_PATH: {{ .Values.env.crawling.nestedJsonPath | quote }}
+  CRAWLING_IGNORE_NOT_FOUND: {{ .Values.env.crawling.ignoreNotFound | quote }}

helm/values.yaml

@@ -116,6 +116,7 @@ jobManager:
   delete:
     jobType:
     taskType:
+
 env:
   port: 80
   targetPort: 8080
@@ -130,9 +131,13 @@ env:
   metrics:
     buckets: '[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 50, 250, 500]'
     url: ''
-  blackList: ["tar", "zip", "rar", "7z"]
   maxConcurrency: 5
 
+  crawling:
+    extension: '.json'
+    nestedJsonPath: "$..['uri','url']"
+    ignoreNotFound: true
+
 resources:
   enabled: true
   value:

openapi3.yaml

@@ -3,7 +3,7 @@ security: []
 info:
   title: 3d store-trigger
   description: Ingesting and exporting 3d model to S3/NFS
-  version: 2.0.3
+  version: 2.1.1
   license:
     name: MIT
     url: https://opensource.org/licenses/MIT

package.json

@@ -1,6 +1,6 @@
 {
   "name": "store-trigger",
-  "version": "2.0.3",
+  "version": "2.1.1",
   "description": "This service responsible for creating job and their tasks.",
   "main": "./src/index.ts",
   "scripts": {
@@ -48,12 +48,12 @@
     "@map-colonies/error-express-handler": "^2.1.0",
     "@map-colonies/express-access-log-middleware": "^2.0.1",
     "@map-colonies/js-logger": "^1.0.1",
-    "@map-colonies/types": "^1.7.0",
     "@map-colonies/mc-model-types": "^19.0.0",
     "@map-colonies/mc-priority-queue": "^8.2.1",
     "@map-colonies/openapi-express-viewer": "^3.0.0",
     "@map-colonies/read-pkg": "0.0.1",
     "@map-colonies/telemetry": "^6.1.0",
+    "@map-colonies/types": "^1.7.0",
     "@opentelemetry/api": "1.7.0",
     "@opentelemetry/api-metrics": "0.23.0",
     "@opentelemetry/context-async-hooks": "^1.24.1",
@@ -65,6 +65,7 @@
     "express": "^4.18.2",
     "express-openapi-validator": "^5.0.4",
     "http-status-codes": "^2.2.0",
+    "jsonpath": "^1.3.0",
     "n-readlines": "^1.0.1",
     "prom-client": "^15.1.1",
     "reflect-metadata": "^0.1.13",
@@ -76,12 +77,13 @@
     "@faker-js/faker": "^8.4.1",
     "@map-colonies/eslint-config": "^4.0.0",
     "@map-colonies/prettier-config": "0.0.1",
-    "@redocly/openapi-cli": "^1.0.0-beta.94",
     "@redocly/cli": "^1.34.3",
+    "@redocly/openapi-cli": "^1.0.0-beta.94",
     "@types/compression": "^1.7.2",
     "@types/config": "^3.3.0",
     "@types/express": "^4.17.17",
     "@types/jest": "^29.5.2",
+    "@types/jsonpath": "^0.2.4",
     "@types/multer": "^1.4.7",
     "@types/n-readlines": "^1.0.3",
     "@types/supertest": "^2.0.12",

src/common/interfaces.ts

@@ -30,7 +30,8 @@ export interface DeletePayload {
 }
 
 export interface Provider {
-  streamModelPathsToQueueFile: (modelId: string, pathToTileset: string, productName: string) => Promise<number>;
+  streamModelPathsToQueueFile: (modelId: string, pathToTileset: string, tilesetFilename: string, productName: string) => Promise<number>;
+  getFile: (filePath: string) => Promise<Buffer>;
 }
 
 export interface IngestionJobParameters {
@@ -56,7 +57,7 @@ export interface DeleteTaskParameters {
   blockDuplication?: boolean;
 }
 
-export interface S3Config {
+export interface S3Config extends BaseProviderConfig {
   accessKeyId: string;
   secretAccessKey: string;
   endpointUrl: string;
@@ -66,10 +67,16 @@ export interface S3Config {
   forcePathStyle: boolean;
 }
 
-export interface NFSConfig {
+export interface NFSConfig extends BaseProviderConfig {
   pvPath: string;
 }
 
+export interface BaseProviderConfig {
+  extension: string;
+  nestedJsonPath: string;
+  ignoreNotFound: boolean;
+}
+
 export type ProviderConfig = S3Config | NFSConfig;
 
 export interface JobOperationResponse {