feat(DASH): Handle mixed-codec variants. (#5950)

With the addition of the changeType API for MediaSource, it is theoretically possible for a variant to change between multiple codecs for a given buffer, over the course of playback. This adds support for the DASH player to stitch together periods which have such multi-codec variants, but only as a last resort. For example, if one period only has audio in aac, and another period only has opus audio, the player will now stitch those periods together as one, but if there is a throughline that does not involve changing codecs it will go for that instead. Closes #5961
shaka-project · Dec 1, 2023 · 24e3255 · 24e3255
1 parent 90bc6a7
commit 24e3255
Show file tree

Hide file tree

Showing 16 changed files with 290 additions and 87 deletions.
diff --git a/README.md b/README.md
@@ -104,6 +104,7 @@ DASH features supported:
  - Trick mode tracks
  - WebVTT and TTML
  - CEA-608/708 captions
+ - Multi-codec variants (on platforms with changeType support)
 
 DASH features **not** supported:
  - Xlink with actuate=onRequest

diff --git a/demo/config.js b/demo/config.js
@@ -195,6 +195,8 @@ shakaDemo.Config = class {
             'manifest.dash.ignoreEmptyAdaptationSet')
         .addBoolInput_('Ignore DASH maxSegmentDuration',
             'manifest.dash.ignoreMaxSegmentDuration')
+        .addBoolInput_('Allow DASH multi type variants',
+            'manifest.dash.multiTypeVariantsAllowed')
         .addBoolInput_('Ignore HLS Text Stream Failures',
             'manifest.hls.ignoreTextStreamFailures')
         .addBoolInput_('Ignore HLS Image Stream Failures',

diff --git a/externs/shaka/player.js b/externs/shaka/player.js
@@ -855,7 +855,8 @@ shaka.extern.InitDataTransform;
  *   keySystemsByURI: !Object.<string, string>,
  *   manifestPreprocessor: function(!Element),
  *   sequenceMode: boolean,
- *   enableAudioGroups: boolean
+ *   enableAudioGroups: boolean,
+ *   multiTypeVariantsAllowed: boolean
  * }}
  *
  * @property {string} clockSyncUri
@@ -917,6 +918,16 @@ shaka.extern.InitDataTransform;
  *   If set, audio streams will be grouped and filtered by their parent
  *   adaptation set ID.
  *   <i>Defaults to <code>false</code>.</i>
+ * @property {boolean} multiTypeVariantsAllowed
+ *   If true, the manifest parser will create variants that have multiple
+ *   mimeTypes or codecs for video or for audio if there is no other choice.
+ *   Meant for content where some periods are only available in one mimeType or
+ *   codec, and other periods are only available in a different mimeType or
+ *   codec. For example, a stream with baked-in ads where the audio codec does
+ *   not match the main content.
+ *   Might result in undesirable behavior if mediaSource.codecSwitchingStrategy
+ *   is not set to SMOOTH.
+ *   Defaults to true if SMOOTH codec switching is supported, RELOAD overwise.
  * @exportDoc
  */
 shaka.extern.DashManifestConfiguration;

diff --git a/lib/dash/dash_parser.js b/lib/dash/dash_parser.js
@@ -14,6 +14,7 @@ goog.require('shaka.dash.SegmentBase');
 goog.require('shaka.dash.SegmentList');
 goog.require('shaka.dash.SegmentTemplate');
 goog.require('shaka.log');
+goog.require('shaka.media.Capabilities');
 goog.require('shaka.media.ManifestParser');
 goog.require('shaka.media.PresentationTimeline');
 goog.require('shaka.media.SegmentIndex');
@@ -134,6 +135,10 @@ shaka.dash.DashParser = class {
     if (this.contentSteeringManager_) {
       this.contentSteeringManager_.configure(this.config_);
     }
+
+    this.periodCombiner_.setAllowMultiTypeVariants(
+        this.config_.dash.multiTypeVariantsAllowed &&
+        shaka.media.Capabilities.isChangeTypeSupported());
   }
 
   /**

diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js
@@ -1091,7 +1091,8 @@ shaka.media.MediaSourceEngine = class {
    *   value will be dropped.
    * @param {boolean} ignoreTimestampOffset  If true, the timestampOffset will
    *   not be applied in this step.
-   * @param {shaka.extern.Stream} stream The current stream.
+   * @param {string} mimeType
+   * @param {string} codecs
    * @param {!Map.<shaka.util.ManifestParserUtils.ContentType,
    *               shaka.extern.Stream>} streamsByType
    *   A map of content types to streams.  All streams must be supported
@@ -1101,7 +1102,7 @@ shaka.media.MediaSourceEngine = class {
    */
   async setStreamProperties(
       contentType, timestampOffset, appendWindowStart, appendWindowEnd,
-      ignoreTimestampOffset, stream, streamsByType) {
+      ignoreTimestampOffset, mimeType, codecs, streamsByType) {
     const ContentType = shaka.util.ManifestParserUtils.ContentType;
     if (contentType == ContentType.TEXT) {
       if (!ignoreTimestampOffset) {
@@ -1112,8 +1113,8 @@ shaka.media.MediaSourceEngine = class {
     }
     const operations = [];
 
-    const hasChangedCodecs =
-        await this.codecSwitchIfNecessary_(contentType, stream, streamsByType);
+    const hasChangedCodecs = await this.codecSwitchIfNecessary_(
+        contentType, mimeType, codecs, streamsByType);
 
     if (!hasChangedCodecs) {
       // Queue an abort() to help MSE splice together overlapping segments.
@@ -1827,15 +1828,17 @@ shaka.media.MediaSourceEngine = class {
    * Codec switch if necessary, this will not resolve until the codec
    * switch is over.
    * @param {shaka.util.ManifestParserUtils.ContentType} contentType
-   * @param {shaka.extern.Stream} stream
+   * @param {string} mimeType
+   * @param {string} codecs
    * @param {!Map.<shaka.util.ManifestParserUtils.ContentType,
    *               shaka.extern.Stream>} streamsByType
    * @return {!Promise.<boolean>} true if there was a codec switch,
    *                              false otherwise.
    * @private
    */
-  async codecSwitchIfNecessary_(contentType, stream, streamsByType) {
-    if (contentType == shaka.util.ManifestParserUtils.ContentType.TEXT) {
+  async codecSwitchIfNecessary_(contentType, mimeType, codecs, streamsByType) {
+    const ContentType = shaka.util.ManifestParserUtils.ContentType;
+    if (contentType == ContentType.TEXT) {
       return false;
     }
     const MimeUtils = shaka.util.MimeUtils;
@@ -1846,30 +1849,43 @@ shaka.media.MediaSourceEngine = class {
 
     /** @type {?shaka.extern.Transmuxer} */
     let transmuxer;
-    let newMimeType = shaka.util.MimeUtils.getFullType(
-        stream.mimeType, stream.codecs);
+    let transmuxerMuxed = false;
+    let newMimeType = shaka.util.MimeUtils.getFullType(mimeType, codecs);
     let needTransmux = this.config_.forceTransmux;
     if (!shaka.media.Capabilities.isTypeSupported(newMimeType) ||
         (!this.sequenceMode_ &&
         shaka.util.MimeUtils.RAW_FORMATS.includes(newMimeType))) {
       needTransmux = true;
     }
-    const newMimeTypeWithAllCodecs =
-        shaka.util.MimeUtils.getFullTypeWithAllCodecs(
-            stream.mimeType, stream.codecs);
     const TransmuxerEngine = shaka.transmuxer.TransmuxerEngine;
     if (needTransmux) {
+      const newMimeTypeWithAllCodecs =
+          shaka.util.MimeUtils.getFullTypeWithAllCodecs(mimeType, codecs);
       const transmuxerPlugin =
           TransmuxerEngine.findTransmuxer(newMimeTypeWithAllCodecs);
       if (transmuxerPlugin) {
         transmuxer = transmuxerPlugin();
-        newMimeType =
-            transmuxer.convertCodecs(contentType, newMimeTypeWithAllCodecs);
+        const audioCodec = shaka.util.ManifestParserUtils.guessCodecsSafe(
+            ContentType.AUDIO, (codecs || '').split(','));
+        const videoCodec = shaka.util.ManifestParserUtils.guessCodecsSafe(
+            ContentType.VIDEO, (codecs || '').split(','));
+        if (audioCodec && videoCodec) {
+          transmuxerMuxed = true;
+          let codec = videoCodec;
+          if (contentType == ContentType.AUDIO) {
+            codec = audioCodec;
+          }
+          newMimeType = transmuxer.convertCodecs(contentType,
+              shaka.util.MimeUtils.getFullTypeWithAllCodecs(mimeType, codec));
+        } else {
+          newMimeType =
+              transmuxer.convertCodecs(contentType, newMimeTypeWithAllCodecs);
+        }
       }
     }
 
-    const newAllCodecs = MimeUtils.getCodecs(newMimeType);
-    const newCodec = MimeUtils.getCodecBase(newAllCodecs);
+    const newCodec = MimeUtils.getCodecBase(
+        MimeUtils.getCodecs(newMimeType));
     const newBasicType = MimeUtils.getBasicType(newMimeType);
 
     // Current/new codecs base and basic type match then no need to switch
@@ -1878,7 +1894,7 @@ shaka.media.MediaSourceEngine = class {
     }
 
     let allowChangeType = true;
-    if (this.needSplitMuxedContent_ || (newAllCodecs.split(',').length &&
+    if (this.needSplitMuxedContent_ || (transmuxerMuxed &&
         transmuxer && !this.transmuxers_[contentType])) {
       allowChangeType = false;
     }

diff --git a/lib/media/segment_reference.js b/lib/media/segment_reference.js
@@ -57,6 +57,12 @@ shaka.media.InitSegmentReference = class {
 
     /** @type {?shaka.extern.aes128Key} */
     this.aes128Key = aes128Key;
+
+    /** @type {?string} */
+    this.codecs = null;
+
+    /** @type {?string} */
+    this.mimeType = null;
   }
 
   /**
@@ -305,6 +311,12 @@ shaka.media.SegmentReference = class {
           this.partialReferences[this.partialReferences.length - 1];
       lastPartial.markAsLastPartial();
     }
+
+    /** @type {?string} */
+    this.codecs = null;
+
+    /** @type {?string} */
+    this.mimeType = null;
   }
 
   /**

diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js
@@ -1769,7 +1769,8 @@ shaka.media.StreamingEngine = class {
           await this.playerInterface_.mediaSourceEngine.setStreamProperties(
               mediaState.type, timestampOffset, appendWindowStart,
               appendWindowEnd, ignoreTimestampOffset,
-              mediaState.stream, streamsByType);
+              reference.mimeType || mediaState.stream.mimeType,
+              reference.codecs || mediaState.stream.codecs, streamsByType);
           if (isResetMediaSourceNecessary) {
             let otherState = null;
             if (mediaState.type === ContentType.VIDEO) {

diff --git a/lib/util/mime_utils.js b/lib/util/mime_utils.js
@@ -181,8 +181,12 @@ shaka.util.MimeUtils = class {
    * @return {string}
    */
   static getCodecBase(codecString) {
-    const parts = shaka.util.MimeUtils.getCodecParts_(codecString);
-    return parts[0];
+    const codecsBase = [];
+    for (const codec of codecString.split(',')) {
+      const parts = shaka.util.MimeUtils.getCodecParts_(codec);
+      codecsBase.push(parts[0]);
+    }
+    return codecsBase.sort().join(',');
   }
 
   /**