Source: lib/cea/mp4_cea_parser.js

  1. /*! @license
  2.  * Shaka Player
  3.  * Copyright 2016 Google LLC
  4.  * SPDX-License-Identifier: Apache-2.0
  5.  */
  7. goog.provide('shaka.cea.Mp4CeaParser');
  9. goog.require('goog.asserts');
  10. goog.require('shaka.cea.CeaUtils');
  11. goog.require('shaka.cea.SeiProcessor');
  12. goog.require('shaka.log');
  13. goog.require('shaka.media.ClosedCaptionParser');
  14. goog.require('shaka.util.DataViewReader');
  15. goog.require('shaka.util.Error');
  16. goog.require('shaka.util.Mp4Parser');
  17. goog.require('shaka.util.Mp4BoxParsers');
  19. /**
  20.  * MPEG4 stream parser used for extracting 708 closed captions data.
  21.  * @implements {shaka.extern.ICeaParser}
  22.  * @export
  23.  */
  24. shaka.cea.Mp4CeaParser = class {
  25.   /** */
  26.   constructor() {
  27.     /**
  28.      * SEI data processor.
  29.      * @private
  30.      * @const {!shaka.cea.SeiProcessor}
  31.      */
  32.     this.seiProcessor_ = new shaka.cea.SeiProcessor();
  34.     /**
  35.      * Map of track id to corresponding timescale.
  36.      * @private {!Map<number, number>}
  37.      */
  38.     this.trackIdToTimescale_ = new Map();
  40.     /**
  41.      * Default sample duration, as specified by the TREX box.
  42.      * @private {number}
  43.      */
  44.     this.defaultSampleDuration_ = 0;
  46.     /**
  47.      * Default sample size, as specified by the TREX box.
  48.      * @private {number}
  49.      */
  50.     this.defaultSampleSize_ = 0;
  52.     /**
  53.      * @private {shaka.cea.Mp4CeaParser.BitstreamFormat}
  54.      */
  55.     this.bitstreamFormat_ = shaka.cea.Mp4CeaParser.BitstreamFormat.UNKNOWN;
  56.   }
  58.   /**
  59.    * Parses the init segment. Gets Default Sample Duration and Size from the
  60.    * TREX box, and constructs a map of Track IDs to timescales. Each TRAK box
  61.    * contains a track header (TKHD) containing track ID, and a media header box
  62.    * (MDHD) containing the timescale for the track
  63.    * @override
  64.    */
  65.   init(initSegment) {
  66.     const Mp4Parser = shaka.util.Mp4Parser;
  67.     const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
  68.     const trackIds = [];
  69.     const timescales = [];
  71.     const codecBoxParser = (box) => this.setBitstreamFormat_(box.name);
  73.     new Mp4Parser()
  74.         .box('moov', Mp4Parser.children)
  75.         .box('mvex', Mp4Parser.children)
  76.         .fullBox('trex', (box) => {
  77.           const parsedTREXBox = shaka.util.Mp4BoxParsers.parseTREX(
  78.               box.reader);
  80.           this.defaultSampleDuration_ = parsedTREXBox.defaultSampleDuration;
  81.           this.defaultSampleSize_ = parsedTREXBox.defaultSampleSize;
  82.         })
  83.         .box('trak', Mp4Parser.children)
  84.         .fullBox('tkhd', (box) => {
  85.           goog.asserts.assert(
  86.               box.version != null,
  87.               'TKHD is a full box and should have a valid version.');
  88.           const parsedTKHDBox = shaka.util.Mp4BoxParsers.parseTKHD(
  89.               box.reader, box.version);
  90.           trackIds.push(parsedTKHDBox.trackId);
  91.         })
  92.         .box('mdia', Mp4Parser.children)
  93.         .fullBox('mdhd', (box) => {
  94.           goog.asserts.assert(
  95.               box.version != null,
  96.               'MDHD is a full box and should have a valid version.');
  97.           const parsedMDHDBox = shaka.util.Mp4BoxParsers.parseMDHD(
  98.               box.reader, box.version);
  99.           timescales.push(parsedMDHDBox.timescale);
  100.         })
  101.         .box('minf', Mp4Parser.children)
  102.         .box('stbl', Mp4Parser.children)
  103.         .fullBox('stsd', Mp4Parser.sampleDescription)
  105.         // These are the various boxes that signal a codec.
  106.         .box('avc1', codecBoxParser)
  107.         .box('avc3', codecBoxParser)
  108.         .box('dvav', codecBoxParser)
  109.         .box('dva1', codecBoxParser)
  110.         .box('hev1', codecBoxParser)
  111.         .box('hvc1', codecBoxParser)
  112.         .box('dvh1', codecBoxParser)
  113.         .box('dvhe', codecBoxParser)
  114.         .box('vvc1', codecBoxParser)
  115.         .box('vvi1', codecBoxParser)
  116.         .box('dvc1', codecBoxParser)
  117.         .box('dvi1', codecBoxParser)
  119.         // This signals an encrypted sample, which we can go inside of to find
  120.         // the codec used.
  121.         .box('encv', Mp4Parser.visualSampleEntry)
  122.         .box('sinf', Mp4Parser.children)
  123.         .box('frma', (box) => {
  124.           const {codec} = shaka.util.Mp4BoxParsers.parseFRMA(box.reader);
  125.           this.setBitstreamFormat_(codec);
  126.         })
  128.         .parse(initSegment, /* partialOkay= */ true);
  130.     // At least one track should exist, and each track should have a
  131.     // corresponding Id in TKHD box, and timescale in its MDHD box
  132.     if (!trackIds.length|| !timescales.length ||
  133.       trackIds.length != timescales.length) {
  134.       throw new shaka.util.Error(
  135.           shaka.util.Error.Severity.CRITICAL,
  136.           shaka.util.Error.Category.TEXT,
  137.           shaka.util.Error.Code.INVALID_MP4_CEA);
  138.     }
  140.     if (this.bitstreamFormat_ == BitstreamFormat.UNKNOWN) {
  141.       shaka.log.alwaysWarn(
  142.           'Unable to determine bitstream format for CEA parsing!');
  143.     }
  145.     // Populate the map from track Id to timescale
  146.     trackIds.forEach((trackId, idx) => {
  147.       this.trackIdToTimescale_.set(trackId, timescales[idx]);
  148.     });
  149.   }
  151.   /**
  152.    * Parses each video segment. In fragmented MP4s, MOOF and MDAT come in
  153.    * pairs. The following logic gets the necessary info from MOOFs to parse
  154.    * MDATs (base media decode time, sample sizes/offsets/durations, etc),
  155.    * and then parses the MDAT boxes for CEA-708 packets using this information.
  156.    * CEA-708 packets are returned in the callback.
  157.    * @override
  158.    */
  159.   parse(mediaSegment) {
  160.     const Mp4Parser = shaka.util.Mp4Parser;
  161.     const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
  163.     if (this.bitstreamFormat_ == BitstreamFormat.UNKNOWN) {
  164.       // We don't know how to extract SEI from this.
  165.       return [];
  166.     }
  168.     /** @type {!Array<!shaka.extern.ICeaParser.CaptionPacket>} **/
  169.     const captionPackets = [];
  171.     let moofOffset = 0;
  173.     /** @type {!Array<!shaka.cea.Mp4CeaParser.ParsedTRAF>} */
  174.     let parsedTRAFs = [];
  176.     new Mp4Parser()
  177.         .box('moof', (box) => {
  178.           moofOffset = box.start;
  179.           // traf box parsing is reset on each moof.
  180.           parsedTRAFs = [];
  181.           Mp4Parser.children(box);
  182.         })
  183.         .box('traf', (box) => {
  184.           parsedTRAFs.push({
  185.             baseMediaDecodeTime: null,
  186.             defaultSampleDuration: this.defaultSampleDuration_,
  187.             defaultSampleSize: this.defaultSampleSize_,
  188.             parsedTRUNs: [],
  189.             timescale: shaka.cea.CeaUtils.DEFAULT_TIMESCALE_VALUE,
  190.           });
  191.           Mp4Parser.children(box);
  192.         })
  193.         .fullBox('trun', (box) => {
  194.           goog.asserts.assert(
  195.               box.version != null && box.flags != null,
  196.               'TRUN is a full box and should have a valid version & flags.');
  198.           const lastTRAF = parsedTRAFs[parsedTRAFs.length - 1];
  200.           const parsedTRUN = shaka.util.Mp4BoxParsers.parseTRUN(
  201.               box.reader, box.version, box.flags);
  202.           lastTRAF.parsedTRUNs.push(parsedTRUN);
  203.         })
  204.         .fullBox('tfhd', (box) => {
  205.           goog.asserts.assert(
  206.               box.flags != null,
  207.               'TFHD is a full box and should have valid flags.');
  209.           const lastTRAF = parsedTRAFs[parsedTRAFs.length - 1];
  211.           const parsedTFHD = shaka.util.Mp4BoxParsers.parseTFHD(
  212.               box.reader, box.flags);
  214.           // If specified, defaultSampleDuration and defaultSampleSize
  215.           // override the ones specified in the TREX box
  216.           lastTRAF.defaultSampleDuration = parsedTFHD.defaultSampleDuration ||
  217.               this.defaultSampleDuration_;
  219.           lastTRAF.defaultSampleSize = parsedTFHD.defaultSampleSize ||
  220.               this.defaultSampleSize_;
  222.           const trackId = parsedTFHD.trackId;
  224.           // Get the timescale from the track Id
  225.           if (this.trackIdToTimescale_.has(trackId)) {
  226.             lastTRAF.timescale = this.trackIdToTimescale_.get(trackId);
  227.           }
  228.         })
  229.         .fullBox('tfdt', (box) => {
  230.           goog.asserts.assert(
  231.               box.version != null,
  232.               'TFDT is a full box and should have a valid version.');
  234.           const lastTRAF = parsedTRAFs[parsedTRAFs.length - 1];
  236.           const parsedTFDT = shaka.util.Mp4BoxParsers.parseTFDTInaccurate(
  237.               box.reader, box.version);
  239.           lastTRAF.baseMediaDecodeTime = parsedTFDT.baseMediaDecodeTime;
  240.         })
  241.         .box('mdat', (box) => {
  242.           const offset = moofOffset - box.start - 8;
  243.           const initialPosition = box.reader.getPosition();
  244.           for (const parsedTRAF of parsedTRAFs) {
  245.             if (parsedTRAF.baseMediaDecodeTime === null) {
  246.               // This field should have been populated by the Base Media Decode
  247.               // Time in the tfdt box.
  248.               shaka.log.alwaysWarn(
  249.                   'Unable to find base media decode time for CEA captions!');
  250.               throw new shaka.util.Error(
  251.                   shaka.util.Error.Severity.CRITICAL,
  252.                   shaka.util.Error.Category.TEXT,
  253.                   shaka.util.Error.Code.INVALID_MP4_CEA);
  254.             }
  255.             box.reader.seek(initialPosition);
  256.             this.parseMdat_(box.reader,
  257.                 parsedTRAF.baseMediaDecodeTime,
  258.                 parsedTRAF.timescale,
  259.                 parsedTRAF.defaultSampleDuration,
  260.                 parsedTRAF.defaultSampleSize,
  261.                 offset,
  262.                 parsedTRAF.parsedTRUNs,
  263.                 captionPackets);
  264.           }
  265.         })
  266.         .parse(mediaSegment, /* partialOkay= */ false);
  268.     return captionPackets;
  269.   }
  271.   /**
  272.    * Parse MDAT box.
  273.    * @param {!shaka.util.DataViewReader} reader
  274.    * @param {number} time
  275.    * @param {number} timescale
  276.    * @param {number} defaultSampleDuration
  277.    * @param {number} defaultSampleSize
  278.    * @param {number} offset
  279.    * @param {!Array<shaka.util.ParsedTRUNBox>} parsedTRUNs
  280.    * @param {!Array<!shaka.extern.ICeaParser.CaptionPacket>} captionPackets
  281.    * @private
  282.    */
  283.   parseMdat_(reader, time, timescale, defaultSampleDuration,
  284.       defaultSampleSize, offset, parsedTRUNs, captionPackets) {
  285.     const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
  286.     const CeaUtils = shaka.cea.CeaUtils;
  287.     let sampleIndex = 0;
  289.     // The fields in each ParsedTRUNSample contained in the sampleData
  290.     // array are nullable. In the case of sample data and sample duration,
  291.     // we use the defaults provided by the TREX/TFHD boxes. For sample
  292.     // composition time offset, we default to 0.
  293.     let sampleSize = defaultSampleSize;
  295.     // Combine all sample data.  This assumes that the samples described across
  296.     // multiple trun boxes are still continuous in the mdat box.
  297.     const sampleDatas = parsedTRUNs.map((t) => t.sampleData);
  298.     const sampleData = [].concat(...sampleDatas);
  300.     if (sampleData.length) {
  301.       sampleSize = sampleData[0].sampleSize || defaultSampleSize;
  302.     }
  304.     reader.skip(offset + parsedTRUNs[0].dataOffset);
  306.     while (reader.hasMoreData()) {
  307.       const naluSize = reader.readUint32();
  308.       const naluHeader = reader.readUint8();
  309.       let naluType = null;
  310.       let isSeiMessage = false;
  311.       let naluHeaderSize = 1;
  313.       goog.asserts.assert(this.bitstreamFormat_ != BitstreamFormat.UNKNOWN,
  314.           'Bitstream format should have been checked before now!');
  315.       switch (this.bitstreamFormat_) {
  316.         case BitstreamFormat.H264:
  317.           naluType = naluHeader & 0x1f;
  318.           isSeiMessage = naluType == CeaUtils.H264_NALU_TYPE_SEI;
  319.           break;
  321.         case BitstreamFormat.H265:
  322.           naluHeaderSize = 2;
  323.           reader.skip(1);
  324.           naluType = (naluHeader >> 1) & 0x3f;
  325.           isSeiMessage =
  326.               naluType == CeaUtils.H265_PREFIX_NALU_TYPE_SEI ||
  327.               naluType == CeaUtils.H265_SUFFIX_NALU_TYPE_SEI;
  328.           break;
  330.         case BitstreamFormat.H266:
  331.           naluHeaderSize = 2;
  332.           reader.skip(1);
  333.           naluType = (naluHeader >> 1) & 0x3f;
  334.           isSeiMessage =
  335.               naluType == CeaUtils.H266_PREFIX_NALU_TYPE_SEI ||
  336.               naluType == CeaUtils.H266_SUFFIX_NALU_TYPE_SEI;
  337.           break;
  339.         default:
  340.           return;
  341.       }
  343.       if (isSeiMessage) {
  344.         let timeOffset = 0;
  346.         if (sampleIndex < sampleData.length) {
  347.           timeOffset = sampleData[sampleIndex].sampleCompositionTimeOffset || 0;
  348.         }
  350.         const pts = (time + timeOffset) / timescale;
  351.         for (const packet of this.seiProcessor_
  352.             .process(reader.readBytes(naluSize - naluHeaderSize))) {
  353.           captionPackets.push({
  354.             packet,
  355.             pts,
  356.           });
  357.         }
  358.       } else {
  359.         try {
  360.           reader.skip(naluSize - naluHeaderSize);
  361.         } catch (e) {
  362.           // It is necessary to ignore this error because it can break the start
  363.           // of playback even if the user does not want to see the subtitles.
  364.           break;
  365.         }
  366.       }
  367.       sampleSize -= (naluSize + 4);
  368.       if (sampleSize == 0) {
  369.         if (sampleIndex < sampleData.length) {
  370.           time += sampleData[sampleIndex].sampleDuration ||
  371.               defaultSampleDuration;
  372.         } else {
  373.           time += defaultSampleDuration;
  374.         }
  376.         sampleIndex++;
  378.         if (sampleIndex < sampleData.length) {
  379.           sampleSize = sampleData[sampleIndex].sampleSize || defaultSampleSize;
  380.         } else {
  381.           sampleSize = defaultSampleSize;
  382.         }
  383.       }
  384.     }
  385.   }
  387.   /**
  388.    * @param {string} codec A fourcc for a codec.
  389.    * @private
  390.    */
  391.   setBitstreamFormat_(codec) {
  392.     if (codec in shaka.cea.Mp4CeaParser.CodecBitstreamMap_) {
  393.       this.bitstreamFormat_ = shaka.cea.Mp4CeaParser.CodecBitstreamMap_[codec];
  394.     }
  395.   }
  396. };
  398. /** @enum {number} */
  399. shaka.cea.Mp4CeaParser.BitstreamFormat = {
  400.   UNKNOWN: 0,
  401.   H264: 1,
  402.   H265: 2,
  403.   H266: 3,
  404. };
  406. /** @private {Object.<string, shaka.cea.Mp4CeaParser.BitstreamFormat>} */
  407. shaka.cea.Mp4CeaParser.CodecBitstreamMap_ = {
  408.   // AVC
  409.   'avc1': shaka.cea.Mp4CeaParser.BitstreamFormat.H264,
  410.   'avc3': shaka.cea.Mp4CeaParser.BitstreamFormat.H264,
  411.   // Dolby Vision based in AVC
  412.   'dvav': shaka.cea.Mp4CeaParser.BitstreamFormat.H264,
  413.   'dva1': shaka.cea.Mp4CeaParser.BitstreamFormat.H264,
  414.   // HEVC
  415.   'hev1': shaka.cea.Mp4CeaParser.BitstreamFormat.H265,
  416.   'hvc1': shaka.cea.Mp4CeaParser.BitstreamFormat.H265,
  417.   // Dolby Vision based in HEVC
  418.   'dvh1': shaka.cea.Mp4CeaParser.BitstreamFormat.H265,
  419.   'dvhe': shaka.cea.Mp4CeaParser.BitstreamFormat.H265,
  420.   // VVC
  421.   'vvc1': shaka.cea.Mp4CeaParser.BitstreamFormat.H266,
  422.   'vvi1': shaka.cea.Mp4CeaParser.BitstreamFormat.H266,
  423.   // Dolby Vision based in VVC
  424.   'dvc1': shaka.cea.Mp4CeaParser.BitstreamFormat.H266,
  425.   'dvi1': shaka.cea.Mp4CeaParser.BitstreamFormat.H266,
  426. };
  428. /**
  429.  * @typedef {{
  430.  *    baseMediaDecodeTime: ?number,
  431.  *    defaultSampleDuration: number,
  432.  *    defaultSampleSize: number,
  433.  *    parsedTRUNs: !Array<shaka.util.ParsedTRUNBox>,
  434.  *    timescale: number
  435.  * }}
  436.  *
  437.  * @property {?number} baseMediaDecodeTime
  438.  * @property {number} defaultSampleDuration
  439.  * @property {number} defaultSampleSize
  440.  * @property {!Array<shaka.util.ParsedTRUNBox>} parsedTRUNs
  441.  * @property {?number} timescale
  442.  */
  443. shaka.cea.Mp4CeaParser.ParsedTRAF;
  445. shaka.media.ClosedCaptionParser.registerParser('video/mp4',
  446.     () => new shaka.cea.Mp4CeaParser());