feat: Add CS3D WSI and Video Viewports and add annotation navigation for MPR (#4182)

platform/app/.webpack/webpack.pwa.js

@@ -35,6 +35,8 @@ const setHeaders = (res, path) => {
   }
   if (path.indexOf('.pdf') !== -1) {
     res.setHeader('Content-Type', 'application/pdf');
+  } else if (path.indexOf('mp4') !== -1) {
+    res.setHeader('Content-Type', 'video/mp4');
   } else if (path.indexOf('frames') !== -1) {
     res.setHeader('Content-Type', 'multipart/related');
   } else {

platform/app/cypress/integration/measurement-tracking/OHIFMeasurementPanel.spec.js

@@ -42,6 +42,10 @@ describe('OHIF Measurement Panel', function () {
   });
 
   it('checks if image would jump when clicked on a measurement item', function () {
+    cy.get('[data-cy="study-browser-thumbnail"][data-series="1"]').dblclick();
+    cy.wait(250);
+    cy.scrollToIndex(0);
+
     // Add length measurement
     cy.addLengthMeasurement().wait(250);
     cy.get('[data-cy="prompt-begin-tracking-yes-btn"]').as('yesBtn').click();

platform/app/cypress/support/commands.js

@@ -36,9 +36,9 @@ Cypress.Commands.add('openStudy', PatientName => {
   cy.openStudyList();
   cy.get('#filter-patientNameOrId').type(PatientName);
   // cy.get('@getStudies').then(() => {
-  cy.waitQueryList();
+  // cy.waitQueryList();
 
-  cy.get('[data-cy="study-list-results"]', { timeout: 5000 })
+  cy.get('[data-cy="study-list-results"]', { timeout: 15000 })
     .contains(PatientName)
     .first()
     .click({ force: true });
@@ -79,8 +79,9 @@ Cypress.Commands.add(
 );
 
 Cypress.Commands.add('waitQueryList', () => {
-  cy.get('[data-querying="false"]');
+  cy.get('[data-querying="false"]', { timeout: 15000 });
 });
+
 /**
  * Command to search for a Modality and open the study.
  *
@@ -109,7 +110,7 @@ Cypress.Commands.add('openStudyList', () => {
   cy.visit('/', { timeout: 15000 });
 
   // For some reason cypress 12.x does not like to stub the network request
-  // so we just wait here for 1 second
+  // so we just wait here for querying to be done.
   // cy.wait('@getStudies');
   cy.waitQueryList();
 });

platform/app/public/config/default.js

@@ -59,6 +59,7 @@ window.config = {
         bulkDataURI: {
           enabled: true,
           relativeResolution: 'studies',
+          transform: url => url.replace('/pixeldata.mp4', '/rendered'),
         },
         omitQuotationForMultipartRequest: true,
       },

platform/app/public/config/e2e.js

@@ -44,6 +44,7 @@ window.config = {
         bulkDataURI: {
           enabled: true,
           relativeResolution: 'studies',
+          transform: url => url.replace('/pixeldata.mp4', '/index.mp4'),
         },
       },
     },
@@ -115,6 +116,7 @@ window.config = {
         bulkDataURI: {
           enabled: true,
           relativeResolution: 'studies',
+          transform: url => url.replace('/pixeldata.mp4', '/rendered'),
         },
       },
     },

platform/app/public/config/netlify.js

@@ -38,6 +38,7 @@ window.config = {
         bulkDataURI: {
           enabled: true,
           relativeResolution: 'studies',
+          transform: url => url.replace('/pixeldata.mp4', '/rendered'),
         },
         omitQuotationForMultipartRequest: true,
       },

platform/app/src/components/ViewportGrid.tsx

@@ -168,33 +168,31 @@ function ViewerViewportGrid(props: withAppTypes) {
         const { displaySetInstanceUID: referencedDisplaySetInstanceUID } = measurement;
 
         const updatedViewports = _getUpdatedViewports(viewportId, referencedDisplaySetInstanceUID);
-        // Arbitrarily assign the viewport to element 0
-        const viewport = updatedViewports?.[0];
-
-        if (!viewport) {
+        if (!updatedViewports[0]) {
           console.warn(
             'ViewportGrid::Unable to navigate to viewport containing',
             referencedDisplaySetInstanceUID
           );
           return;
         }
+        // Arbitrarily assign the viewport to element 0
+        // TODO - this should perform a search to find the most suitable viewport.
+        updatedViewports[0] = { ...updatedViewports[0] };
+        const [viewport] = updatedViewports;
 
-        viewport.viewportOptions ||= {};
-        viewport.viewportOptions.orientation = 'acquisition';
+        // Copy the viewport options to prevent modifying the internal data
+        viewport.viewportOptions = {
+          ...viewport.viewportOptions,
+          orientation: 'acquisition',
+          // The preferred way to jump to the measurement view is to set the
+          // view reference, as this can hold information such as the orientation
+          // or zoom level required to display an annotation.  The metadata attribute
+          // of the measurement is a viewReference, so use it to show the measurement.
+          // Longer term this should clear the view reference data
+          viewReference: measurement.metadata,
+          viewportType: measurement.metadata.volumeId ? 'volume' : null,
+        };
 
-        const displaySet = displaySetService.getDisplaySetByUID(referencedDisplaySetInstanceUID);
-        // jump straight to the initial image index if we can
-        if (displaySet.images && measurement.SOPInstanceUID) {
-          for (let index = 0; index < displaySet.images.length; index++) {
-            const image = displaySet.images[index];
-            if (image.SOPInstanceUID === measurement.SOPInstanceUID) {
-              viewport.viewportOptions.initialImageOptions = {
-                index,
-              };
-              break;
-            }
-          }
-        }
         viewportGridService.setDisplaySetsForViewports(updatedViewports);
       }
     );
@@ -338,11 +336,6 @@ function ViewerViewportGrid(props: withAppTypes) {
   );
 }
 
-ViewerViewportGrid.propTypes = {
-  viewportComponents: PropTypes.array.isRequired,
-  servicesManager: PropTypes.instanceOf(Object).isRequired,
-};
-
 function _getViewportComponent(displaySets, viewportComponents, uiNotificationService) {
   if (!displaySets || !displaySets.length) {
     return EmptyViewport;

platform/core/src/classes/MetadataProvider.ts

@@ -389,6 +389,7 @@ class MetadataProvider {
       case WADO_IMAGE_LOADER_TAGS.CINE_MODULE:
         metadata = {
           frameTime: instance.FrameTime,
+          numberOfFrames: instance.NumberOfFrames ? Number(instance.NumberOfFrames) : 1,
         };
 
         break;

platform/core/src/services/DisplaySetService/DisplaySet.ts

@@ -1,9 +1,12 @@
-interface IDisplaySet {
+type DisplaySet {
   displaySetInstanceUID: string;
   StudyInstanceUID: string;
   SeriesInstanceUID?: string;
   SeriesNumber?: string;
   unsupported?: boolean;
+  viewportType?: string;
+  instances: any[];
+  instance?: any;
 }
 
-export default IDisplaySet;
+export default DisplaySet;

platform/core/src/services/DisplaySetService/DisplaySetService.ts

@@ -131,7 +131,7 @@ export default class DisplaySetService extends PubSubService {
       : [...this.getDisplaySetCache().values()];
 
     const displaySet = displaySets.find(ds => {
-      return ds.images && ds.images.some(i => i.SOPInstanceUID === sopInstanceUID);
+      return ds.instances?.some(i => i.SOPInstanceUID === sopInstanceUID);
     });
 
     return displaySet;

platform/core/src/services/HangingProtocolService/HangingProtocolService.ts

@@ -4,7 +4,7 @@ import { PubSubService } from '../_shared/pubSubServiceInterface';
 import sortBy from '../../utils/sortBy';
 import ProtocolEngine from './ProtocolEngine';
 import { StudyMetadata } from '../../types/StudyMetadata';
-import IDisplaySet from '../DisplaySetService/IDisplaySet';
+import DisplaySet from '../DisplaySetService/DisplaySet';
 import { CommandsManager } from '../../classes';
 import * as HangingProtocol from '../../types/HangingProtocol';
 import { isDisplaySetFromUrl, sopInstanceLocation } from './custom-attribute/isDisplaySetFromUrl';
@@ -62,7 +62,7 @@ export default class HangingProtocolService extends PubSubService {
   _servicesManager: AppTypes.ServicesManager;
   protocolEngine: ProtocolEngine;
   customViewportSettings = [];
-  displaySets: IDisplaySet[] = [];
+  displaySets: DisplaySet[] = [];
   activeStudy: StudyMetadata;
   debugLogging: false;
 
@@ -1126,10 +1126,7 @@ export default class HangingProtocolService extends PubSubService {
    * to match display sets to viewports.
    * @returns a display set sort function
    */
-  public getDisplaySetSortFunction(): (
-    displaySetA: IDisplaySet,
-    displaySetB: IDisplaySet
-  ) => number {
+  public getDisplaySetSortFunction(): (displaySetA: DisplaySet, displaySetB: DisplaySet) => number {
     return (displaySetA, displaySetB) => {
       const seriesA = this._getSeriesSortInfoForDisplaySetSort(displaySetA);
       const seriesB = this._getSeriesSortInfoForDisplaySetSort(displaySetB);

platform/core/src/types/index.ts

@@ -7,6 +7,7 @@ import type {
   BaseDataSourceConfigurationAPI,
   BaseDataSourceConfigurationAPIItem,
 } from './DataSourceConfigurationAPI';
+import type DisplaySet from '../services/DisplaySetService/DisplaySet';
 
 export type * from '../services/ViewportGridService';
 export type * from '../services/CustomizationService/types';
@@ -31,4 +32,5 @@ export {
   DataSourceDefinition,
   BaseDataSourceConfigurationAPI,
   BaseDataSourceConfigurationAPIItem,
+  DisplaySet,
 };

platform/core/src/utils/index.test.js

@@ -22,6 +22,7 @@ describe('Top level exports', () => {
       'imageIdToURI',
       'roundNumber',
       'b64toBlob',
+      'sopClassDictionary',
       'createStudyBrowserTabs',
       'formatDate',
       'formatTime',

platform/core/src/utils/index.ts

@@ -39,6 +39,7 @@ import {
 import { subscribeToNextViewportGridChange } from './subscribeToNextViewportGridChange';
 import { splitComma, getSplitParam } from './splitComma';
 import { createStudyBrowserTabs } from './createStudyBrowserTabs';
+import { sopClassDictionary } from './sopClassDictionary';
 
 // Commented out unused functionality.
 // Need to implement new mechanism for derived displaySets using the displaySetManager.
@@ -69,6 +70,7 @@ const utils = {
   Queue,
   isDicomUid,
   isEqualWithin,
+  sopClassDictionary,
   addAccessors,
   resolveObjectPath,
   hierarchicalListUtils,

platform/core/src/utils/sortStudy.ts

@@ -26,6 +26,8 @@ function seriesInfoSortingCriteria(firstSeries, secondSeries) {
   const bLowPriority = isLowPriorityModality(secondSeries.Modality ?? secondSeries.modality);
 
   if (aLowPriority) {
+    // Use the reverse sort order for low priority modalities so that the
+    // most recent one comes up first as usually that is the one of interest.
     return bLowPriority ? defaultSeriesSort(secondSeries, firstSeries) : 1;
   } else if (bLowPriority) {
     return -1;

platform/docs/docs/migration-guide/from-3p8-to-3p9-beta.md

@@ -267,3 +267,43 @@ We have removed the Polyfill.io script from the Viewer. If you require polyfills
 ## Dynamic Modules
 
 TBD
+
+## Renaming some interfaces
+A few interfaces are being renamed to simple types to reflect the fact that
+they don't contain methods and are thus more properly simple types.
+
+* IDisplaySet renamed to DisplaySet
+  * Adding some field declarations to agree with actual usage
+
+
+## Use of ViewReference for navigation
+When navigating to measurements and storing/remembering navigation positions,
+the `viewport.getViewReference` is used to get a position, and `viewport.isReferenceViewable`
+used to check if a reference can be applied, and finally `viewport.setViewReference` to
+navigate to a view.  Note that this changes the behaviour of navigation between
+MPR and Stack viewports, and also enables navigation of video and microscopy
+viewports in CS3D.  This can cause some unexpected behaviour depending on how the
+frame of reference values are configured to allow for navigation.
+
+The isReferenceViewable is used to determine when a view or measurement can be
+shown on a given view.  For stack versus volume viewports, this can cause unexpected
+behaviour to be seen depending on how the view reference was fetched.
+
+### `getViewReference` with `forFrameOfReference`
+When a view reference is fetched with the for frame of reference flag set to true,
+a reference will be returned which can be displayed on any viewport containing
+the same frame of reference and encompassing the given FOR and able to display the required
+orientation.  Without this flag, a view reference is returned which will be
+displayed on a stack with the given image id, or a volume containing said image id
+or the specified volume.
+
+### `isReferenceViewable` with navigation and/or orientation
+The is reference viewable will return false unless the given reference is directly
+viewable in the viewport as is.  However, it can be passed various flags to determine
+whether the reference could be displayed if the viewport was modified in various ways,
+for example, by changing the position or orientation of the viewport.  This allows
+checking for degrees of closeness so that the correct viewport can be chosen.
+
+Note that this may result in displaying a measurement from one viewport on a completely
+different viewport, for example, showing a Probe tool from the stack viewport on
+an MPR view.

platform/ui/src/components/StudyBrowser/StudyBrowser.tsx

@@ -1,6 +1,7 @@
 import React from 'react';
 import PropTypes from 'prop-types';
 import { useTranslation } from 'react-i18next';
+import { utils } from '@ohif/core';
 
 import StudyItem from '../StudyItem';
 import LegacyButtonGroup from '../LegacyButtonGroup';
@@ -9,6 +10,8 @@ import ThumbnailList from '../ThumbnailList';
 import { StringNumber } from '../../types';
 import StudyBrowserSort from '../StudyBrowserSort';
 
+const { sortStudySeries } = utils;
+
 const getTrackedSeries = displaySets => {
   let trackedSeries = 0;
   displaySets.forEach(displaySet => {
@@ -20,7 +23,7 @@ const getTrackedSeries = displaySets => {
   return trackedSeries;
 };
 
-const noop = () => {};
+const noop = () => { };
 
 const StudyBrowser = ({
   tabs,
@@ -36,11 +39,14 @@ const StudyBrowser = ({
 }: withAppTypes) => {
   const { t } = useTranslation('StudyBrowser');
   const { customizationService } = servicesManager?.services || {};
-
+  const { experimentalStudyBrowserSort } = window.config;
   const getTabContent = () => {
     const tabData = tabs.find(tab => tab.name === activeTabName);
     return tabData.studies.map(
       ({ studyInstanceUid, date, description, numInstances, modalities, displaySets }) => {
+        if (!experimentalStudyBrowserSort) {
+          sortStudySeries(displaySets);
+        }
         const isExpanded = expandedStudyInstanceUIDs.includes(studyInstanceUid);
         return (
           <React.Fragment key={studyInstanceUid}>
@@ -112,9 +118,7 @@ const StudyBrowser = ({
             );
           })}
         </LegacyButtonGroup>
-        {window.config.experimentalStudyBrowserSort && (
-          <StudyBrowserSort servicesManager={servicesManager} />
-        )}
+        {experimentalStudyBrowserSort && <StudyBrowserSort servicesManager={servicesManager} />}
       </div>
       <div className="ohif-scrollbar invisible-scrollbar flex flex-1 flex-col overflow-auto">
         {getTabContent()}

platform/ui/src/components/Thumbnail/Thumbnail.tsx

@@ -101,14 +101,14 @@ const Thumbnail = ({
             {loadingProgress && loadingProgress === 1 && (
               <Icon
                 name={'database'}
-                className="w-3"
-              />
-            )}
-          </div>
-          <DisplaySetMessageListTooltip
-            messages={messages}
-            id={`display-set-tooltip-${displaySetInstanceUID}`}
-          />
+              className="w-3"
+            />
+          )}
+        </div>
+        <DisplaySetMessageListTooltip
+          messages={messages}
+          id={`display-set-tooltip-${displaySetInstanceUID}`}
+        />
         </div>
         <div className="break-all text-base text-white">{description}</div>
       </div>