Skip to content

Commit d9900c7

Browse files
julijanejulijane
committed
fix(TTML): Correctly handle multiple samples in a segment
Fixes #8087 Implements handling of multiple samples in a segment. Also fixes the testdata for the multiple MDAT test, as the prior data was invalid (not conforming to ISO14496-12).
1 parent c42169e commit d9900c7

File tree

6 files changed

+110
-34
lines changed

6 files changed

+110
-34
lines changed

‎AUTHORS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ Jesper Haug Karsrud <jesper.karsrud@gmail.com>
5656
Johan Sundström <oyasumi@gmail.com>
5757
Jonas Birmé <jonas.birme@eyevinn.se>
5858
Jozef Chúťka <jozefchutka@gmail.com>
59+
Juliane Holzt <juliane@box.fqdn.org>
5960
Jun Hong Chong <chongjunhong@gmail.com>
6061
Jürgen Kartnaller <kartnaller@lovelysystems.com>
6162
Justin Swaney <justin.mark.swaney@gmail.com>

‎CONTRIBUTORS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ Jono Ward <jonoward@gmail.com>
8686
Jozef Chúťka <jozefchutka@gmail.com>
8787
Juan Manuel Tomás <juant@qualabs.com>
8888
Julian Domingo <juliandomingo@google.com>
89+
Juliane Holzt <juliane@box.fqdn.org>
8990
Jun Hong Chong <chongjunhong@gmail.com>
9091
Jürgen Kartnaller <kartnaller@lovelysystems.com>
9192
Justin Swaney <justin.mark.swaney@gmail.com>

‎lib/text/mp4_ttml_parser.js‎

Lines changed: 81 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@
66

77
goog.provide('shaka.text.Mp4TtmlParser');
88

9+
goog.require('goog.asserts');
910
goog.require('shaka.text.TextEngine');
1011
goog.require('shaka.text.TtmlTextParser');
1112
goog.require('shaka.util.BufferUtils');
1213
goog.require('shaka.util.Error');
14+
goog.require('shaka.util.Mp4BoxParsers');
1315
goog.require('shaka.util.Mp4Parser');
1416
goog.require('shaka.util.Uint8ArrayUtils');
1517

@@ -80,67 +82,117 @@ shaka.text.Mp4TtmlParser = class {
8082
parseMedia(data, time, uri) {
8183
const Mp4Parser = shaka.util.Mp4Parser;
8284

83-
let sawMDAT = false;
8485
let payload = [];
86+
let defaultSampleSize = null;
87+
88+
/** @type {!Array<Uint8Array>} */
89+
const mdats = [];
90+
91+
/* @type {!Map<number,!Array<number>>} */
92+
const subSampleSizesPerSample = new Map();
8593

8694
/** @type {!Array<number>} */
87-
let subSizes = [];
95+
const sampleSizes = [];
8896

8997
const parser = new Mp4Parser()
9098
.box('moof', Mp4Parser.children)
9199
.box('traf', Mp4Parser.children)
100+
.fullBox('tfhd', (box) => {
101+
goog.asserts.assert(
102+
box.flags != null,
103+
'A TFHD box should have a valid flags value');
104+
const parsedTFHDBox = shaka.util.Mp4BoxParsers.parseTFHD(
105+
box.reader, box.flags);
106+
defaultSampleSize = parsedTFHDBox.defaultSampleSize;
107+
})
108+
.fullBox('trun', (box) => {
109+
goog.asserts.assert(
110+
box.version != null,
111+
'A TRUN box should have a valid version value');
112+
goog.asserts.assert(
113+
box.flags != null,
114+
'A TRUN box should have a valid flags value');
115+
116+
const parsedTRUNBox = shaka.util.Mp4BoxParsers.parseTRUN(
117+
box.reader, box.version, box.flags);
118+
119+
for (const sample of parsedTRUNBox.sampleData) {
120+
const sampleSize =
121+
sample.sampleSize || defaultSampleSize || 0;
122+
sampleSizes.push(sampleSize);
123+
}
124+
})
92125
.fullBox('subs', (box) => {
93-
subSizes = [];
94126
const reader = box.reader;
95127
const entryCount = reader.readUint32();
128+
let currentSampleNum = -1;
96129
for (let i = 0; i < entryCount; i++) {
97-
reader.readUint32(); // sample_delta
130+
const sampleDelta = reader.readUint32();
131+
currentSampleNum += sampleDelta;
98132
const subsampleCount = reader.readUint16();
133+
const subsampleSizes = [];
99134
for (let j = 0; j < subsampleCount; j++) {
100135
if (box.version == 1) {
101-
subSizes.push(reader.readUint32());
136+
subsampleSizes.push(reader.readUint32());
102137
} else {
103-
subSizes.push(reader.readUint16());
138+
subsampleSizes.push(reader.readUint16());
104139
}
105140
reader.readUint8(); // priority
106141
reader.readUint8(); // discardable
107142
reader.readUint32(); // codec_specific_parameters
108143
}
144+
subSampleSizesPerSample.set(currentSampleNum, subsampleSizes);
109145
}
110146
})
111147
.box('mdat', Mp4Parser.allData((data) => {
112-
sawMDAT = true;
113-
// Join this to any previous payload, in case the mp4 has multiple
114-
// mdats.
115-
if (subSizes.length) {
116-
const contentData =
117-
shaka.util.BufferUtils.toUint8(data, 0, subSizes[0]);
118-
const images = [];
119-
let offset = subSizes[0];
120-
for (let i = 1; i < subSizes.length; i++) {
121-
const imageData =
122-
shaka.util.BufferUtils.toUint8(data, offset, subSizes[i]);
123-
const raw =
124-
shaka.util.Uint8ArrayUtils.toStandardBase64(imageData);
125-
images.push('data:image/png;base64,' + raw);
126-
offset += subSizes[i];
127-
}
128-
payload = payload.concat(
129-
this.parser_.parseMedia(contentData, time, uri, images));
130-
} else {
131-
payload = payload.concat(
132-
this.parser_.parseMedia(data, time, uri, /* images= */ []));
133-
}
148+
// We collect all of the mdats first, before parsing any of them.
149+
// This is necessary in case the mp4 has multiple mdats.
150+
mdats.push(data);
134151
}));
135152
parser.parse(data, /* partialOkay= */ false);
136153

137-
if (!sawMDAT) {
154+
if (mdats.length == 0) {
138155
throw new shaka.util.Error(
139156
shaka.util.Error.Severity.CRITICAL,
140157
shaka.util.Error.Category.TEXT,
141158
shaka.util.Error.Code.INVALID_MP4_TTML);
142159
}
143160

161+
const fullData =
162+
shaka.util.Uint8ArrayUtils.concat(...mdats);
163+
164+
let sampleOffset = 0;
165+
for (let sampleNum = 0; sampleNum < sampleSizes.length; sampleNum++) {
166+
const sampleData =
167+
shaka.util.BufferUtils.toUint8(fullData, sampleOffset,
168+
sampleSizes[sampleNum]);
169+
sampleOffset += sampleSizes[sampleNum];
170+
171+
const subSampleSizes = subSampleSizesPerSample.get(sampleNum);
172+
173+
if (subSampleSizes && subSampleSizes.length) {
174+
const contentData =
175+
shaka.util.BufferUtils.toUint8(sampleData, 0, subSampleSizes[0]);
176+
const images = [];
177+
let subOffset = subSampleSizes[0];
178+
for (let i = 1; i < subSampleSizes.length; i++) {
179+
const imageData =
180+
shaka.util.BufferUtils.toUint8(data, subOffset,
181+
subSampleSizes[i]);
182+
const raw =
183+
shaka.util.Uint8ArrayUtils.toStandardBase64(imageData);
184+
images.push('data:image/png;base64,' + raw);
185+
subOffset += subSampleSizes[i];
186+
}
187+
payload = payload.concat(
188+
this.parser_.parseMedia(contentData, time, uri, images));
189+
} else {
190+
payload = payload.concat(
191+
this.parser_.parseMedia(sampleData, time, uri,
192+
/* images= */ []));
193+
}
194+
}
195+
144196
return payload;
145197
}
146198
};

‎test/test/assets/ttml-segment-multiple-mdat.mp4‎

-1.16 KB
Binary file not shown.

‎test/test/assets/ttml-segment-multiple-sample.mp4‎

2.97 KB
Binary file not shown.

‎test/text/mp4_ttml_parser_unit.js‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ describe('Mp4TtmlParser', () => {
99
const ttmlSegmentUri = '/base/test/test/assets/ttml-segment.mp4';
1010
const ttmlSegmentMultipleMDATUri =
1111
'/base/test/test/assets/ttml-segment-multiple-mdat.mp4';
12+
const ttmlSegmentMultipleSampleUri =
13+
'/base/test/test/assets/ttml-segment-multiple-sample.mp4';
1214
const imscImageInitSegmentUri =
1315
'/base/test/test/assets/imsc-image-init.cmft';
1416
const imscImageSegmentUri =
@@ -22,6 +24,8 @@ describe('Mp4TtmlParser', () => {
2224
/** @type {!Uint8Array} */
2325
let ttmlSegmentMultipleMDAT;
2426
/** @type {!Uint8Array} */
27+
let ttmlSegmentMultipleSample;
28+
/** @type {!Uint8Array} */
2529
let imscImageInitSegment;
2630
/** @type {!Uint8Array} */
2731
let imscImageSegment;
@@ -33,16 +37,18 @@ describe('Mp4TtmlParser', () => {
3337
shaka.test.Util.fetch(ttmlInitSegmentUri),
3438
shaka.test.Util.fetch(ttmlSegmentUri),
3539
shaka.test.Util.fetch(ttmlSegmentMultipleMDATUri),
40+
shaka.test.Util.fetch(ttmlSegmentMultipleSampleUri),
3641
shaka.test.Util.fetch(imscImageInitSegmentUri),
3742
shaka.test.Util.fetch(imscImageSegmentUri),
3843
shaka.test.Util.fetch(audioInitSegmentUri),
3944
]);
4045
ttmlInitSegment = shaka.util.BufferUtils.toUint8(responses[0]);
4146
ttmlSegment = shaka.util.BufferUtils.toUint8(responses[1]);
4247
ttmlSegmentMultipleMDAT = shaka.util.BufferUtils.toUint8(responses[2]);
43-
imscImageInitSegment = shaka.util.BufferUtils.toUint8(responses[3]);
44-
imscImageSegment = shaka.util.BufferUtils.toUint8(responses[4]);
45-
audioInitSegment = shaka.util.BufferUtils.toUint8(responses[5]);
48+
ttmlSegmentMultipleSample = shaka.util.BufferUtils.toUint8(responses[3]);
49+
imscImageInitSegment = shaka.util.BufferUtils.toUint8(responses[4]);
50+
imscImageSegment = shaka.util.BufferUtils.toUint8(responses[5]);
51+
audioInitSegment = shaka.util.BufferUtils.toUint8(responses[6]);
4652
});
4753

4854
it('parses init segment', () => {
@@ -62,8 +68,24 @@ describe('Mp4TtmlParser', () => {
6268
expect(ret[0].nestedCues.length).toBe(1);
6369
expect(ret[1].nestedCues.length).toBe(1);
6470
// Cues.
65-
expect(ret[0].nestedCues[0].nestedCues.length).toBe(10);
66-
expect(ret[1].nestedCues[0].nestedCues.length).toBe(10);
71+
expect(ret[0].nestedCues[0].nestedCues.length).toBe(5);
72+
expect(ret[1].nestedCues[0].nestedCues.length).toBe(5);
73+
});
74+
75+
it('handles media segments with multiple sample', () => {
76+
const parser = new shaka.text.Mp4TtmlParser();
77+
parser.parseInit(ttmlInitSegment);
78+
const time =
79+
{periodStart: 0, segmentStart: 0, segmentEnd: 60, vttOffset: 0};
80+
const ret = parser.parseMedia(ttmlSegmentMultipleSample, time, null);
81+
// Bodies.
82+
expect(ret.length).toBe(2);
83+
// Divs.
84+
expect(ret[0].nestedCues.length).toBe(1);
85+
expect(ret[1].nestedCues.length).toBe(1);
86+
// Cues.
87+
expect(ret[0].nestedCues[0].nestedCues.length).toBe(5);
88+
expect(ret[1].nestedCues[0].nestedCues.length).toBe(5);
6789
});
6890

6991
it('accounts for offset', () => {

0 commit comments

Comments
 (0)