Skip to content

Commit

Permalink
fix(Transmuxer): Fix transmuxer with overflow data (video nalus and a…
Browse files Browse the repository at this point in the history 
…ac samples) between PES
  • Loading branch information
@avelad
avelad committed Dec 31, 2024
1 parent df8eff9 commit 662b9a8
Show file tree
Hide file tree
Showing 10 changed files with 386 additions and 180 deletions.
21 changes: 21 additions & 0 deletions externs/shaka/codecs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,27 @@
shaka.extern.MPEG_PES;


/**
* @typedef {{
* data: !Uint8Array,
* frame: boolean,
* isKeyframe: boolean,
* pts: ?number,
* dts: ?number,
* nalus: !Array.<!shaka.extern.VideoNalu>
* }}
*
* @summary VideoSample.
* @property {!Uint8Array} data
* @property {boolean} frame
* @property {boolean} isKeyframe
* @property {?number} pts
* @property {?number} dts
* @property {!Array.<!shaka.extern.VideoNalu>} nalus
*/
shaka.extern.VideoSample;


/**
* @typedef {{
* data: !Uint8Array,
Expand Down
4 changes: 4 additions & 0 deletions karma.conf.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,10 @@ module.exports = (config) => {
{pattern: 'test/test/assets/hls-ts-h265/*', included: false},
{pattern: 'test/test/assets/hls-ts-mp3/*', included: false},
{pattern: 'test/test/assets/hls-ts-muxed-aac-h264/*', included: false},
// eslint-disable-next-line max-len
{pattern: 'test/test/assets/hls-ts-muxed-aac-h264-with-overflow-nalus/*', included: false},
// eslint-disable-next-line max-len
{pattern: 'test/test/assets/hls-ts-muxed-aac-h264-with-overflow-samples/*', included: false},
{pattern: 'test/test/assets/hls-ts-muxed-aac-h265/*', included: false},
{pattern: 'test/test/assets/hls-ts-muxed-ac3-h264/*', included: false},
{pattern: 'test/test/assets/hls-ts-muxed-mp3-h264/*', included: false},
Expand Down
213 changes: 143 additions & 70 deletions lib/transmuxer/h264.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,71 +196,28 @@ shaka.transmuxer.H264 = class {
}

/**
* @param {!Array.<shaka.extern.VideoNalu>} nalus
* @return {?{data: !Uint8Array, isKeyframe: boolean}}
* @param {!Array.<shaka.extern.MPEG_PES>} videoData
* @return {!Array.<shaka.extern.VideoSample>}
*/
static parseFrame(nalus) {
static getVideoSamples(videoData) {
const H264 = shaka.transmuxer.H264;
let isKeyframe = false;
const nalusData = [];
const spsNalu = nalus.find((nalu) => {
return nalu.type == H264.NALU_TYPE_SPS_;
});
let avcSample = false;
for (const nalu of nalus) {
let push = false;
switch (nalu.type) {
case H264.NALU_TYPE_NDR_: {
avcSample = true;
push = true;
const data = nalu.data;
// Only check slice type to detect KF in case SPS found in same packet
// (any keyframe is preceded by SPS ...)
if (spsNalu && data.length > 4) {
// retrieve slice type by parsing beginning of NAL unit (follow
// H264 spec,slice_header definition) to detect keyframe embedded
// in NDR
const sliceType = new shaka.util.ExpGolomb(data).readSliceType();
// 2 : I slice, 4 : SI slice, 7 : I slice, 9: SI slice
// SI slice : A slice that is coded using intra prediction only and
// using quantisation of the prediction samples.
// An SI slice can be coded such that its decoded samples can be
// constructed identically to an SP slice.
// I slice: A slice that is not an SI slice that is decoded using
// intra prediction only.
if (sliceType === 2 || sliceType === 4 ||
sliceType === 7 || sliceType === 9) {
isKeyframe = true;
}
}
break;
}
case H264.NALU_TYPE_IDR_:
avcSample = true;
push = true;
isKeyframe = true;
break;
case H264.NALU_TYPE_SEI_:
push = true;
break;
case H264.NALU_TYPE_SPS_:
push = true;
break;
case H264.NALU_TYPE_PPS_:
push = true;
break;
case H264.NALU_TYPE_AUD_:
push = true;
avcSample = true;
break;
case H264.NALU_TYPE_FILLER_DATA_:
push = true;
break;
default:
push = false;
break;

/** @type {!Array.<shaka.extern.VideoSample>} */
const videoSamples = [];
/** @type {?shaka.extern.VideoSample} */
let lastVideoSample = null;
/** @type {boolean} */
let audFound = false;

const addLastVideoSample = () => {
if (!lastVideoSample) {
return;
}
if (avcSample && push) {
if (!lastVideoSample.nalus.length || !lastVideoSample.frame) {
return;
}
const nalusData = [];
for (const nalu of lastVideoSample.nalus) {
const size = nalu.fullData.byteLength;
const naluLength = new Uint8Array(4);
naluLength[0] = (size >> 24) & 0xff;
Expand All @@ -270,15 +227,131 @@ shaka.transmuxer.H264 = class {
nalusData.push(naluLength);
nalusData.push(nalu.fullData);
}
}
if (!nalusData.length) {
return null;
}
const data = shaka.util.Uint8ArrayUtils.concat(...nalusData);
return {
data,
isKeyframe,
lastVideoSample.data = shaka.util.Uint8ArrayUtils.concat(...nalusData);
videoSamples.push(lastVideoSample);
};

const createLastVideoSample = (pes) => {
lastVideoSample = {
data: new Uint8Array([]),
frame: false,
isKeyframe: false,
pts: pes.pts,
dts: pes.dts,
nalus: [],
};
};

for (let i = 0; i < videoData.length; i++) {
const pes = videoData[i];
const nalus = pes.nalus;
let spsFound = false;

// If new NAL units found and last sample still there, let's push ...
// This helps parsing streams with missing AUD
// (only do this if AUD never found)
if (lastVideoSample && nalus.length && !audFound) {
addLastVideoSample();
createLastVideoSample(pes);
}

for (const nalu of pes.nalus) {
let push = false;
switch (nalu.type) {
case H264.NALU_TYPE_NDR_: {
let isKeyframe = false;
push = true;
const data = nalu.data;
// Only check slice type to detect KF in case SPS found in same
// packet (any keyframe is preceded by SPS ...)
if (spsFound && data.length > 4) {
// retrieve slice type by parsing beginning of NAL unit (follow
// H264 spec,slice_header definition) to detect keyframe embedded
// in NDR
const sliceType = new shaka.util.ExpGolomb(data).readSliceType();
// 2 : I slice, 4 : SI slice, 7 : I slice, 9: SI slice
// SI slice : A slice that is coded using intra prediction only
// and using quantisation of the prediction samples.
// An SI slice can be coded such that its decoded samples can be
// constructed identically to an SP slice.
// I slice: A slice that is not an SI slice that is decoded using
// intra prediction only.
if (sliceType === 2 || sliceType === 4 ||
sliceType === 7 || sliceType === 9) {
isKeyframe = true;
}
}
if (isKeyframe) {
// If we have non-keyframe data already, that cannot belong to
// the same frame as a keyframe, so force a push
if (lastVideoSample &&
lastVideoSample.frame && !lastVideoSample.isKeyframe) {
addLastVideoSample();
lastVideoSample = null;
}
}
if (!lastVideoSample) {
createLastVideoSample(pes);
}
lastVideoSample.frame = true;
lastVideoSample.isKeyframe = isKeyframe;
break;
}
case H264.NALU_TYPE_IDR_: {
push = true;
// Handle PES not starting with AUD
// If we have frame data already, that cannot belong to the same
// frame, so force a push
if (lastVideoSample &&
lastVideoSample.frame && !lastVideoSample.isKeyframe) {
addLastVideoSample();
lastVideoSample = null;
}
if (!lastVideoSample) {
createLastVideoSample(pes);
}
lastVideoSample.frame = true;
lastVideoSample.isKeyframe = true;
break;
}
case H264.NALU_TYPE_SEI_:
push = true;
break;
case H264.NALU_TYPE_SPS_:
push = true;
spsFound = true;
break;
case H264.NALU_TYPE_PPS_:
push = true;
break;
case H264.NALU_TYPE_AUD_:
push = true;
audFound = true;
if (lastVideoSample && lastVideoSample.frame) {
addLastVideoSample();
lastVideoSample = null;
}
if (!lastVideoSample) {
createLastVideoSample(pes);
}
break;
case H264.NALU_TYPE_FILLER_DATA_:
push = true;
break;
default:
push = false;
break;
}
if (lastVideoSample && push) {
lastVideoSample.nalus.push(nalu);
}
}
}

// If last PES packet, push samples
addLastVideoSample();

return videoSamples;
}
};

Expand Down
Loading

0 comments on commit 662b9a8

Please sign in to comment.