fix: Download of binary DICOM and study index writes wrong files

packages/create-dicomweb/lib/instance/instanceFromStream.mjs

@@ -1,4 +1,5 @@
 import { async, utilities, data } from 'dcmjs';
+import { v4 as uuid } from 'uuid';
 import { Tags, StatusMonitor, createPromiseTracker } from '@radicalimaging/static-wado-util';
 import { writeMultipartFramesFilter } from './writeMultipartFramesFilter.mjs';
 import { writeBulkdataFilter } from './writeBulkdataFilter.mjs';
@@ -28,6 +29,60 @@ function isReadBufferStreamLike(stream) {
 
 const PARSE_JOB_TYPE = 'stowInstanceParse';
 
+/** Modalities whose original Part 10 binary is stored as instances/<sop>/index.mht.gz */
+const RAW_PART10_MODALITIES = new Set(['SEG', 'SR']);
+/** SOP Classes whose original Part 10 binary is stored (Basic Structured Display) */
+const RAW_PART10_SOP_CLASSES = new Set(['1.2.840.10008.5.1.4.1.1.30']);
+
+/**
+ * Returns true when the original Part 10 binary should be stored for this
+ * instance (segmentations and structured reports), mirroring the
+ * static-wado-creator RawDicomWriter selector.
+ * @param {Object} dict - Parsed instance dataset (DICOM JSON model)
+ * @returns {boolean}
+ */
+function shouldStoreRawPart10(dict) {
+  const modality = dict?.[Tags.Modality]?.Value?.[0];
+  const sopClass = dict?.[Tags.SOPClassUID]?.Value?.[0];
+  return RAW_PART10_MODALITIES.has(modality) || RAW_PART10_SOP_CLASSES.has(sopClass);
+}
+
+/**
+ * Writes the original Part 10 bytes (as received) to the instance-level
+ * rendition instances/<sop>/index.mht.gz - a gzipped multipart/related wrapper
+ * with Content-Type: application/dicom, the same format RawDicomWriter used
+ * and the format dicomwebserver serves for instance retrieval.
+ * The source bytes are re-read from the parse stream, which retains its
+ * buffers on the STOW path (clearBuffers is false there).
+ * @param {Object} writer - DicomWebWriter used for this instance
+ * @param {Object} stream - The stream the instance was parsed from
+ * @param {string} sopInstanceUID - For logging
+ * @returns {Promise<void>}
+ */
+async function writeRawPart10(writer, stream, sopInstanceUID) {
+  const rawSize = stream?.size;
+  const canReRead =
+    typeof stream?.getBuffer === 'function' &&
+    rawSize > 0 &&
+    (typeof stream.hasData !== 'function' || stream.hasData(0, rawSize));
+  if (!canReRead) {
+    console.verbose('Raw Part 10 not stored for', sopInstanceUID, '- source bytes unavailable');
+    return;
+  }
+  const raw = stream.getBuffer(0, rawSize);
+  const rawBuffer = ArrayBuffer.isView(raw)
+    ? Buffer.from(raw.buffer, raw.byteOffset, raw.byteLength)
+    : Buffer.from(raw);
+  const rawStream = await writer.openInstanceStream('index.mht', {
+    gzip: true,
+    multipart: true,
+    contentType: 'application/dicom',
+    boundary: `BOUNDARY_${uuid()}`,
+  });
+  rawStream.stream.write(rawBuffer);
+  await writer.closeStream(rawStream.streamKey);
+}
+
 /**
  * Creates a filter that counts addTag/value calls and reports progress to StatusMonitor.
  * Uses a per-instance parse job so counts are not overwritten when multiple instances parse concurrently.
@@ -325,6 +380,20 @@ export async function instanceFromStream(stream, options = {}) {
       await writer.closeStream(metadataStream.streamKey);
     }
 
+    // Store the original Part 10 binary for segmentations/structured reports so
+    // instance retrieval can serve it directly instead of re-creating it.
+    if (writer && options.writeRawPart10 !== false && shouldStoreRawPart10(dict)) {
+      try {
+        await writeRawPart10(writer, reader.stream, information.sopInstanceUid);
+      } catch (err) {
+        console.warn(
+          'Unable to store raw Part 10 for',
+          information.sopInstanceUid,
+          err?.message ?? err
+        );
+      }
+    }
+
     // Wait for all frame writes to complete before returning
     await writer?.awaitAllStreams();
     console.verbose('Finished writing metadata to file', information.sopInstanceUid);

packages/s3-deploy/lib/S3Ops.mjs

@@ -178,6 +178,54 @@ class S3Ops {
     }
   }
 
+  /**
+   * Restores the local file name for a multipart object whose ".mht"
+   * extension was stripped by fileToKey on upload. The directory listing
+   * can only guess "<key>.gz" for extension-less keys; the object's
+   * content type and encoding (known once it is fetched) determine the
+   * real name: <key>.mht for plain multipart, <key>.mht.gz when the
+   * object is gzip encoded.
+   */
+  retrieveFileName(destFile, contentType, contentEncoding) {
+    if (!contentType || !contentType.startsWith(multipartRelated)) return destFile;
+    if (!endsWith(destFile, '.gz')) return destFile;
+    let base = destFile.substring(0, destFile.length - 3);
+    const lastSegment = base.substring(
+      Math.max(base.lastIndexOf('/'), base.lastIndexOf('\\')) + 1
+    );
+    if (lastSegment === 'index.json') {
+      // Directory-index guess for a key with children (e.g. the instance-level
+      // Part 10 rendition at .../instances/<sop>): multipart content really
+      // lives at <dir>/index.mht rather than <dir>/index.json.
+      base = base.substring(0, base.length - '.json'.length);
+    } else if (lastSegment.indexOf('.') !== -1) {
+      return destFile;
+    }
+    return contentEncoding === 'gzip' ? `${base}.mht.gz` : `${base}.mht`;
+  }
+
+  /**
+   * The local file names a listed object may be stored under. An
+   * extension-less key guessed as "<key>.gz" may really be a multipart
+   * file stored locally as <key>.mht or <key>.mht.gz.
+   */
+  localCandidates(fileName) {
+    const candidates = [fileName];
+    if (endsWith(fileName, '.gz')) {
+      const base = fileName.substring(0, fileName.length - 3);
+      const lastSegment = base.substring(base.lastIndexOf('/') + 1);
+      if (lastSegment.indexOf('.') === -1) {
+        candidates.push(`${base}.mht`, `${base}.mht.gz`);
+      } else if (lastSegment === 'index.json') {
+        // Directory-index guess may really be a multipart rendition at
+        // <dir>/index.mht(.gz) (e.g. instance-level Part 10 files).
+        const indexBase = base.substring(0, base.length - '.json'.length);
+        candidates.push(`${indexBase}.mht`, `${indexBase}.mht.gz`);
+      }
+    }
+    return candidates;
+  }
+
   /** Retrieves the given s3 URI to the specified destination path */
   async retrieve(uri, destFile, options = { force: false }) {
     const remoteUri = this.remoteRelativeToUri(uri);
@@ -204,7 +252,8 @@ class S3Ops {
 
     try {
       const result = await this.client.send(command);
-      const { Body } = result;
+      const { Body, ContentType, ContentEncoding } = result;
+      destFile = this.retrieveFileName(destFile, ContentType, ContentEncoding);
       await copyTo(Body, destFile);
       console.info('Retrieved', Key);
     } catch (e) {
@@ -219,16 +268,47 @@ class S3Ops {
     return this.group.path ? contentItem.Key.substring(this.group.path.length) : contentItem.Key;
   }
 
-  contentItemToFileName(contentItem) {
+  contentItemToFileName(contentItem, isDirectory = false) {
     const s = this.getPath(contentItem);
     if (endsWith(s, 'thumbnail')) return s;
     if (endsWith(s, '/')) return `${s}index.json.gz`;
-    if (endsWith(s, '/series') || endsWith(s, '/studies') || endsWith(s, '/instances'))
+    if (
+      isDirectory ||
+      endsWith(s, '/series') ||
+      endsWith(s, '/studies') ||
+      endsWith(s, '/instances')
+    )
       return `${s}/index.json.gz`;
     if (endsWith(s, '.gz') || endsWith(s, '.jls')) return s;
     return `${s}.gz`;
   }
 
+  /**
+   * Assigns the local fileName to each listed item. An object whose key also
+   * has children in the listing (e.g. "studies/<uid>" alongside
+   * "studies/<uid>/series/...") is a directory index object created by
+   * fileToKey stripping "/index.json.gz" on upload, so it maps back to
+   * <key>/index.json.gz rather than <key>.gz.
+   */
+  assignFileNames(results) {
+    const keys = results.map(it => it.Key).sort();
+    const hasChildren = key => {
+      const childPrefix = `${key}/`;
+      let lo = 0;
+      let hi = keys.length;
+      while (lo < hi) {
+        const mid = (lo + hi) >> 1;
+        if (keys[mid] < childPrefix) lo = mid + 1;
+        else hi = mid;
+      }
+      return lo < keys.length && keys[lo].startsWith(childPrefix);
+    };
+    for (const item of results) {
+      item.fileName = this.contentItemToFileName(item, hasChildren(item.Key));
+    }
+    return results;
+  }
+
   async dir(uri) {
     const remoteUri = this.remoteRelativeToUri(uri);
     if (!remoteUri) {
@@ -256,22 +336,21 @@ class S3Ops {
             ...it,
             size: it.Size,
             relativeUri: this.getPath(it),
-            fileName: this.contentItemToFileName(it),
           });
         });
         if (!result.IsTruncated) {
-          return results;
+          return this.assignFileNames(results);
         }
         ContinuationToken = result.NextContinuationToken;
         if (!ContinuationToken) {
           throw new Error('No continuation token');
         }
       } catch (e) {
         console.log('Error sending', Bucket, remoteUri, e);
-        return results;
+        return this.assignFileNames(results);
       }
     }
-    return results;
+    return this.assignFileNames(results);
   }
 
   async upload(dir, file, hash, excludeExisting = {}) {

packages/static-wado-creator/lib/index.mjs

@@ -4,7 +4,10 @@ import programIndex from './program/index.js';
 import createMain from './createMain.js';
 import deleteMain from './deleteMain.js';
 import adaptProgramOpts from './util/adaptProgramOpts.js';
-import { uids } from '@radicalimaging/static-wado-util';
+// Import uids via its subpath rather than the package root: mixing an ESM
+// import of the package root with the CJS require() calls in the lib/*.js
+// files makes Bun treat the root as an async module and fail the require().
+import uids from '@radicalimaging/static-wado-util/uids.mjs';
 
 const { configureProgram } = programIndex;
 

packages/static-wado-deploy/lib/DeployGroup.mjs

@@ -1,4 +1,5 @@
 import fs from 'fs';
+import zlib from 'zlib';
 import { configGroup, handleHomeRelative } from '@radicalimaging/static-wado-util';
 import path from 'path';
 import { plugins } from '@radicalimaging/static-wado-plugins';
@@ -333,6 +334,69 @@ class DeployGroup {
     }
   }
 
+  /**
+   * Finds the local file a listed object is already stored under, checking
+   * the alternate .mht names a multipart object may have locally. A file
+   * left under the generic ".gz" name by an older retrieve is renamed to
+   * its correct multipart name, determined from its leading bytes.
+   * @returns {string|undefined} Full path of the existing local file
+   */
+  findExistingLocal(fileName) {
+    const candidates = this.ops.localCandidates ? this.ops.localCandidates(fileName) : [fileName];
+    const existing = candidates.find(name => fs.existsSync(path.join(this.baseDir, name)));
+    if (existing === undefined) return undefined;
+    const existingPath = path.join(this.baseDir, existing);
+    if (existing === fileName && candidates.length > 1 && !this.options.dryRun) {
+      return this.renameLegacyMultipart(existingPath) || existingPath;
+    }
+    return existingPath;
+  }
+
+  /**
+   * Renames a multipart file stored under a plain ".gz" name to
+   * <base>.mht or <base>.mht.gz according to its content. Files that are
+   * not multipart (e.g. gzipped JSON such as metadata.gz) are left alone.
+   * @returns {string|undefined} The corrected path, if renamed
+   */
+  renameLegacyMultipart(existingPath) {
+    const fd = fs.openSync(existingPath, 'r');
+    const header = Buffer.alloc(1024);
+    let bytesRead;
+    try {
+      bytesRead = fs.readSync(fd, header, 0, header.length, 0);
+    } finally {
+      fs.closeSync(fd);
+    }
+    if (bytesRead < 2) return undefined;
+    let base = existingPath.substring(0, existingPath.length - 3);
+    if (base.endsWith('.json')) {
+      // A directory-index guess (index.json.gz) holding multipart content
+      // belongs at index.mht(.gz)
+      base = base.substring(0, base.length - '.json'.length);
+    }
+    let corrected;
+    if (header[0] === 0x2d && header[1] === 0x2d) {
+      // Multipart boundary "--" - an uncompressed .mht stored as .gz
+      corrected = `${base}.mht`;
+    } else if (header[0] === 0x1f && header[1] === 0x8b) {
+      // Gzip data - only rename when the compressed content is multipart
+      try {
+        const inflated = zlib.gunzipSync(header.subarray(0, bytesRead), {
+          finishFlush: zlib.constants.Z_SYNC_FLUSH,
+        });
+        if (inflated[0] === 0x2d && inflated[1] === 0x2d) {
+          corrected = `${base}.mht.gz`;
+        }
+      } catch (e) {
+        console.verbose('Unable to inspect gzip content of', existingPath, e.message);
+      }
+    }
+    if (!corrected || fs.existsSync(corrected)) return undefined;
+    fs.renameSync(existingPath, corrected);
+    console.noQuiet('Renamed', existingPath, 'to', corrected);
+    return corrected;
+  }
+
   async dir(uri) {
     const list = await this.ops.dir(uri);
     return list.reduce((acc, value) => {
@@ -388,9 +452,10 @@ class DeployGroup {
       if (isExcluded) {
         continue;
       }
-      if (fs.existsSync(destName) && !force) {
-        if (this.ops.shouldSkip(item, destName)) {
-          console.verbose('Skipping', destName);
+      if (!force) {
+        const existingPath = this.findExistingLocal(item.fileName);
+        if (existingPath && this.ops.shouldSkip(item, existingPath)) {
+          console.verbose('Skipping', existingPath);
           skippedItems += 1;
           continue;
         }

packages/static-wado-deploy/lib/deployConfig.mjs

@@ -91,9 +91,8 @@ const { deployConfig } = ConfigPoint.register({
 
     programs: [
       {
-        command: 'studies',
-        arguments: ['studies'],
-        helpShort: 'deploydicomweb studies studyUID',
+        command: 'studies <studyUIDs...>',
+        helpShort: 'deploydicomweb studies <studyUIDs...>',
         helpDescription: 'Deploy DICOMweb files to the cloud',
         options: [
           {

packages/static-wado-deploy/lib/studiesMain.mjs

@@ -3,10 +3,9 @@ import uploadDeploy from './uploadDeploy.mjs';
 import uploadIndex from './uploadIndex.mjs';
 import retrieveDeploy from './retrieveDeploy.mjs';
 
-export default async function (options, program) {
-  const { args: studies } = program;
+export default async function (studies, options) {
   if (!studies?.length) {
-    return;
+    throw new Error("No study UIDs specified - provide one or more study UIDs, or '*' for all studies.");
   }
 
   if (studies.length === 1 && studies[0] === '*') {