feat(ImageOverlayViewerTool): add ImageOverlayViewer tool that can render image overlay (pixel overlay) of the DICOM images (#3163)

Co-authored-by: Joe Boccanfuso <joe.boccanfuso@radicalimaging.com>

extensions/cornerstone-dicom-sr/package.json

@@ -44,9 +44,9 @@
   },
   "dependencies": {
     "@babel/runtime": "^7.20.13",
-    "@cornerstonejs/adapters": "^1.11.4",
-    "@cornerstonejs/core": "^1.11.4",
-    "@cornerstonejs/tools": "^1.11.4",
+    "@cornerstonejs/adapters": "^1.13.2",
+    "@cornerstonejs/core": "^1.13.2",
+    "@cornerstonejs/tools": "^1.13.2",
     "classnames": "^2.3.2"
   }
 }

extensions/cornerstone/package.json

@@ -36,7 +36,7 @@
     "@cornerstonejs/codec-libjpeg-turbo-8bit": "^1.2.2",
     "@cornerstonejs/codec-openjpeg": "^1.2.2",
     "@cornerstonejs/codec-openjph": "^2.4.2",
-    "@cornerstonejs/dicom-image-loader": "^1.11.4",
+    "@cornerstonejs/dicom-image-loader": "^1.13.2",
     "@ohif/core": "3.7.0-beta.64",
     "@ohif/ui": "3.7.0-beta.64",
     "dcmjs": "^0.29.6",
@@ -52,10 +52,10 @@
   },
   "dependencies": {
     "@babel/runtime": "^7.20.13",
-    "@cornerstonejs/adapters": "^1.11.4",
-    "@cornerstonejs/core": "^1.11.4",
-    "@cornerstonejs/streaming-image-volume-loader": "^1.11.4",
-    "@cornerstonejs/tools": "^1.11.4",
+    "@cornerstonejs/adapters": "^1.13.2",
+    "@cornerstonejs/core": "^1.13.2",
+    "@cornerstonejs/streaming-image-volume-loader": "^1.13.2",
+    "@cornerstonejs/tools": "^1.13.2",
     "@kitware/vtk.js": "27.3.1",
     "html2canvas": "^1.4.1",
     "lodash.debounce": "4.0.8",

extensions/cornerstone/src/commandsModule.ts

@@ -18,6 +18,7 @@ import toggleStackImageSync from './utils/stackSync/toggleStackImageSync';
 import { getFirstAnnotationSelected } from './utils/measurementServiceMappings/utils/selection';
 import getActiveViewportEnabledElement from './utils/getActiveViewportEnabledElement';
 import { CornerstoneServices } from './types';
+import ImageOverlayViewerTool from './tools/ImageOverlayViewerTool';
 
 function commandsModule({
   servicesManager,
@@ -266,7 +267,7 @@ function commandsModule({
       toolbarService.recordInteraction(props);
     },
 
-    setToolActive: ({ toolName, toolGroupId = null }) => {
+    setToolActive: ({ toolName, toolGroupId = null, toggledState }) => {
       if (toolName === 'Crosshairs') {
         const activeViewportToolGroup = toolGroupService.getToolGroup(null);
 
@@ -327,6 +328,14 @@ function commandsModule({
           toolGroup.setToolPassive(activeToolName);
         }
       }
+
+      // If there is a toggle state, then simply set the enabled/disabled state without
+      // setting the tool active.
+      if (toggledState != null) {
+        toggledState ? toolGroup.setToolEnabled(toolName) : toolGroup.setToolDisabled(toolName);
+        return;
+      }
+
       // Set the new toolName to be active
       toolGroup.setToolActive(toolName, {
         bindings: [
@@ -547,17 +556,13 @@ function commandsModule({
         toggledState,
       });
     },
-    toggleReferenceLines: ({ toggledState }) => {
+    setSourceViewportForReferenceLinesTool: ({ toggledState }) => {
       const { activeViewportId } = viewportGridService.getState();
       const viewportInfo = cornerstoneViewportService.getViewportInfo(activeViewportId);
 
       const viewportId = viewportInfo.getViewportId();
       const toolGroup = toolGroupService.getToolGroupForViewport(viewportId);
 
-      if (!toggledState) {
-        toolGroup.setToolDisabled(ReferenceLinesTool.toolName);
-      }
-
       toolGroup.setToolConfiguration(
         ReferenceLinesTool.toolName,
         {
@@ -565,7 +570,6 @@ function commandsModule({
         },
         true // overwrite
       );
-      toolGroup.setToolEnabled(ReferenceLinesTool.toolName);
     },
     storePresentation: ({ viewportId }) => {
       cornerstoneViewportService.storePresentation({ viewportId });
@@ -692,8 +696,8 @@ function commandsModule({
     toggleStackImageSync: {
       commandFn: actions.toggleStackImageSync,
     },
-    toggleReferenceLines: {
-      commandFn: actions.toggleReferenceLines,
+    setSourceViewportForReferenceLinesTool: {
+      commandFn: actions.setSourceViewportForReferenceLinesTool,
     },
     storePresentation: {
       commandFn: actions.storePresentation,

extensions/cornerstone/src/initCornerstoneTools.js

@@ -28,6 +28,7 @@ import {
 } from '@cornerstonejs/tools';
 
 import CalibrationLineTool from './tools/CalibrationLineTool';
+import ImageOverlayViewerTool from './tools/ImageOverlayViewerTool';
 
 export default function initCornerstoneTools(configuration = {}) {
   CrosshairsTool.isAnnotation = false;
@@ -58,6 +59,7 @@ export default function initCornerstoneTools(configuration = {}) {
   addTool(ReferenceLinesTool);
   addTool(CalibrationLineTool);
   addTool(TrackballRotateTool);
+  addTool(ImageOverlayViewerTool);
 
   // Modify annotation tools to use dashed lines on SR
   const annotationStyle = {
@@ -99,6 +101,7 @@ const toolNames = {
   ReferenceLines: ReferenceLinesTool.toolName,
   CalibrationLine: CalibrationLineTool.toolName,
   TrackballRotateTool: TrackballRotateTool.toolName,
+  ImageOverlayViewer: ImageOverlayViewerTool.toolName,
 };
 
 export { toolNames };

extensions/cornerstone/src/tools/ImageOverlayViewerTool.tsx

@@ -0,0 +1,247 @@
+import { metaData } from '@cornerstonejs/core';
+import { utilities } from '@cornerstonejs/core';
+import { AnnotationDisplayTool, drawing } from '@cornerstonejs/tools';
+import { guid } from '@ohif/core/src/utils';
+
+interface CachedStat {
+  color: number[]; // [r, g, b, a]
+  overlays: {
+    // ...overlayPlaneModule
+    _id: string;
+    type: 'G' | 'R'; // G for Graphics, R for ROI
+    color?: number[]; // Rendered color [r, g, b, a]
+    dataUrl?: string; // Rendered image in Data URL expression
+  }[];
+}
+
+/**
+ * Image Overlay Viewer tool is not a traditional tool that requires user interactin.
+ * But it is used to display Pixel Overlays. And it will provide toggling capability.
+ *
+ * The documentation for Overlay Plane Module of DICOM can be found in [C.9.2 of
+ * Part-3 of DICOM standard](https://dicom.nema.org/medical/dicom/2018b/output/chtml/part03/sect_C.9.2.html)
+ *
+ * Image Overlay rendered by this tool can be toggled on and off using
+ * toolGroup.setToolEnabled() and toolGroup.setToolDisabled()
+ */
+class ImageOverlayViewerTool extends AnnotationDisplayTool {
+  static toolName = 'ImageOverlayViewer';
+  private _cachedOverlayMetadata: Map<string, any[]> = new Map();
+  private _cachedStats: { [key: string]: CachedStat } = {};
+
+  constructor(
+    toolProps = {},
+    defaultToolProps = {
+      supportedInteractionTypes: [],
+      configuration: {
+        fillColor: [255, 127, 127, 255],
+      },
+    }
+  ) {
+    super(toolProps, defaultToolProps);
+  }
+
+  onSetToolDisabled = (): void => {
+    this._cachedStats = {};
+    this._cachedOverlayMetadata = new Map();
+  };
+
+  renderAnnotation = (enabledElement, svgDrawingHelper) => {
+    const { viewport } = enabledElement;
+
+    const imageId = this.getReferencedImageId(viewport);
+    if (!imageId) {
+      return;
+    }
+
+    const overlays =
+      this._cachedOverlayMetadata.get(imageId) ??
+      metaData.get('overlayPlaneModule', imageId)?.overlays;
+
+    // no overlays
+    if (!overlays?.length) {
+      return;
+    }
+
+    this._cachedOverlayMetadata.set(imageId, overlays);
+
+    this._getCachedStat(imageId, overlays, this.configuration.fillColor).then(cachedStat => {
+      cachedStat.overlays.forEach(overlay => {
+        this._renderOverlay(enabledElement, svgDrawingHelper, overlay);
+      });
+    });
+
+    return true;
+  };
+
+  /**
+   * Render to DOM
+   *
+   * @param enabledElement
+   * @param svgDrawingHelper
+   * @param overlayData
+   * @returns
+   */
+  private _renderOverlay(enabledElement, svgDrawingHelper, overlayData) {
+    const { viewport } = enabledElement;
+    const imageId = this.getReferencedImageId(viewport);
+    if (!imageId) {
+      return;
+    }
+
+    // Decide the rendering position of the overlay image on the current canvas
+    const { _id, columns: width, rows: height, x, y } = overlayData;
+    const overlayTopLeftWorldPos = utilities.imageToWorldCoords(imageId, [
+      x - 1, // Remind that top-left corner's (x, y) is be (1, 1)
+      y - 1,
+    ]);
+    const overlayTopLeftOnCanvas = viewport.worldToCanvas(overlayTopLeftWorldPos);
+    const overlayBottomRightWorldPos = utilities.imageToWorldCoords(imageId, [width, height]);
+    const overlayBottomRightOnCanvas = viewport.worldToCanvas(overlayBottomRightWorldPos);
+
+    // add image to the annotations svg layer
+    const svgns = 'http://www.w3.org/2000/svg';
+    const svgNodeHash = `image-overlay-${_id}`;
+    const existingImageElement = svgDrawingHelper.getSvgNode(svgNodeHash);
+
+    const attributes = {
+      'data-id': svgNodeHash,
+      width: overlayBottomRightOnCanvas[0] - overlayTopLeftOnCanvas[0],
+      height: overlayBottomRightOnCanvas[1] - overlayTopLeftOnCanvas[1],
+      x: overlayTopLeftOnCanvas[0],
+      y: overlayTopLeftOnCanvas[1],
+      href: overlayData.dataUrl,
+    };
+
+    if (
+      isNaN(attributes.x) ||
+      isNaN(attributes.y) ||
+      isNaN(attributes.width) ||
+      isNaN(attributes.height)
+    ) {
+      console.warn('Invalid rendering attribute for image overlay', attributes['data-id']);
+      return false;
+    }
+
+    if (existingImageElement) {
+      drawing.setAttributesIfNecessary(attributes, existingImageElement);
+      svgDrawingHelper.setNodeTouched(svgNodeHash);
+    } else {
+      const newImageElement = document.createElementNS(svgns, 'image');
+      drawing.setNewAttributesIfValid(attributes, newImageElement);
+      svgDrawingHelper.appendNode(newImageElement, svgNodeHash);
+    }
+    return true;
+  }
+
+  private async _getCachedStat(
+    imageId: string,
+    overlayMetadata: any[],
+    color: number[]
+  ): Promise<CachedStat> {
+    if (this._cachedStats[imageId] && this._isSameColor(this._cachedStats[imageId].color, color)) {
+      return this._cachedStats[imageId];
+    }
+
+    const overlays = await Promise.all(
+      overlayMetadata
+        .filter(overlay => overlay.pixelData)
+        .map(async (overlay, idx) => {
+          let pixelData = null;
+          if (overlay.pixelData.Value) {
+            pixelData = overlay.pixelData.Value;
+          } else if (overlay.pixelData.retrieveBulkData) {
+            pixelData = await overlay.pixelData.retrieveBulkData();
+          }
+
+          if (!pixelData) {
+            return;
+          }
+
+          const dataUrl = this._renderOverlayToDataUrl(
+            { width: overlay.columns, height: overlay.rows },
+            color,
+            pixelData
+          );
+
+          return {
+            ...overlay,
+            _id: guid(),
+            dataUrl, // this will be a data url expression of the rendered image
+            color,
+          };
+        })
+    );
+
+    this._cachedStats[imageId] = {
+      color: color,
+      overlays: overlays.filter(overlay => overlay),
+    };
+
+    return this._cachedStats[imageId];
+  }
+
+  /**
+   * compare two RGBA expression of colors.
+   *
+   * @param color1
+   * @param color2
+   * @returns
+   */
+  private _isSameColor(color1: number[], color2: number[]) {
+    return (
+      color1 &&
+      color2 &&
+      color1[0] === color2[0] &&
+      color1[1] === color2[1] &&
+      color1[2] === color2[2] &&
+      color1[3] === color2[3]
+    );
+  }
+
+  /**
+   * pixelData of overlayPlane module is an array of bits corresponding
+   * to each of the underlying pixels of the image.
+   * Let's create pixel data from bit array of overlay data
+   *
+   * @param pixelDataRaw
+   * @param color
+   * @returns
+   */
+  private _renderOverlayToDataUrl({ width, height }, color, pixelDataRaw) {
+    const pixelDataView = new DataView(pixelDataRaw);
+    const totalBits = width * height;
+
+    const canvas = document.createElement('canvas');
+    canvas.width = width;
+    canvas.height = height;
+
+    const ctx = canvas.getContext('2d');
+    ctx.clearRect(0, 0, width, height); // make it transparent
+    ctx.globalCompositeOperation = 'copy';
+
+    const imageData = ctx.getImageData(0, 0, width, height);
+    const data = imageData.data;
+    for (let i = 0, bitIdx = 0, byteIdx = 0; i < totalBits; i++) {
+      if (pixelDataView.getUint8(byteIdx) & (1 << bitIdx)) {
+        data[i * 4] = color[0];
+        data[i * 4 + 1] = color[1];
+        data[i * 4 + 2] = color[2];
+        data[i * 4 + 3] = color[3];
+      }
+
+      // next bit, byte
+      if (bitIdx >= 7) {
+        bitIdx = 0;
+        byteIdx++;
+      } else {
+        bitIdx++;
+      }
+    }
+    ctx.putImageData(imageData, 0, 0);
+
+    return canvas.toDataURL();
+  }
+}
+
+export default ImageOverlayViewerTool;

extensions/measurement-tracking/package.json

@@ -30,8 +30,8 @@
     "start": "yarn run dev"
   },
   "peerDependencies": {
-    "@cornerstonejs/core": "^1.11.4",
-    "@cornerstonejs/tools": "^1.11.4",
+    "@cornerstonejs/core": "^1.13.2",
+    "@cornerstonejs/tools": "^1.13.2",
     "@ohif/core": "3.7.0-beta.64",
     "@ohif/extension-cornerstone-dicom-sr": "3.7.0-beta.64",
     "@ohif/ui": "3.7.0-beta.64",