diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js
index 726b3a6d23..9a4ba731fc 100644
--- a/lib/hls/hls_parser.js
+++ b/lib/hls/hls_parser.js
@@ -2792,10 +2792,21 @@ shaka.hls.HlsParser = class {
       }
     }
     this.notifySegmentsForStreams_(streamInfos.map((s) => s.stream));
+
+    const activeStreamInfos = Array.from(this.uriToStreamInfosMap_.values())
+        .filter((s) => s.stream.segmentIndex);
+    const ContentType = shaka.util.ManifestParserUtils.ContentType;
+    const hasAudio =
+        activeStreamInfos.some((s) => s.stream.type == ContentType.AUDIO);
+    const hasVideo =
+        activeStreamInfos.some((s) => s.stream.type == ContentType.VIDEO);
+
     const liveWithNoProgramDateTime =
         this.isLive_() && !this.usesProgramDateTime_;
+    const vodWithOnlyAudioOrVideo = !this.isLive_() &&
+        this.usesProgramDateTime_ && !(hasAudio && hasVideo);
     if (this.config_.hls.ignoreManifestProgramDateTime ||
-        liveWithNoProgramDateTime) {
+        liveWithNoProgramDateTime || vodWithOnlyAudioOrVideo) {
       this.syncStreamsWithSequenceNumber_(
           streamInfos, liveWithNoProgramDateTime);
     } else {