diff --git a/source/Lib/CommonLib/SEI.cpp b/source/Lib/CommonLib/SEI.cpp index 6b3d7148b72f3d0557e1cc7f643b314b4ed89de6..7d979c6ea851f9466666578449bbe090cbe93691 100644 --- a/source/Lib/CommonLib/SEI.cpp +++ b/source/Lib/CommonLib/SEI.cpp @@ -52,11 +52,11 @@ void xTraceSEIMessageType( SEI::PayloadType payloadType ) } #endif -SEIMessages getSeisByType(SEIMessages &seiList, SEI::PayloadType seiType) +SEIMessages getSeisByType(const SEIMessages &seiList, SEI::PayloadType seiType) { SEIMessages result; - for (SEIMessages::iterator it=seiList.begin(); it!=seiList.end(); it++) + for (SEIMessages::const_iterator it=seiList.begin(); it!=seiList.end(); it++) { if ((*it)->payloadType() == seiType) { diff --git a/source/Lib/CommonLib/SEI.h b/source/Lib/CommonLib/SEI.h index fa88f605289daa69a956ee76c70a0c74ecde5819..311d7f1631ca069a4af1d553a92ed707e6c96f79 100644 --- a/source/Lib/CommonLib/SEI.h +++ b/source/Lib/CommonLib/SEI.h @@ -478,7 +478,7 @@ public: typedef std::list<SEI*> SEIMessages; /// output a selection of SEI messages by payload type. Ownership stays in original message list. -SEIMessages getSeisByType(SEIMessages &seiList, SEI::PayloadType seiType); +SEIMessages getSeisByType(const SEIMessages &seiList, SEI::PayloadType seiType); /// remove a selection of SEI messages by payload type from the original list and return them in a new list. SEIMessages extractSeisByType(SEIMessages &seiList, SEI::PayloadType seiType); @@ -491,7 +491,20 @@ class SEIScalableNesting : public SEI public: PayloadType payloadType() const { return SCALABLE_NESTING; } - SEIScalableNesting() {} + SEIScalableNesting() + : m_snOlsFlag (false) +#if JVET_Q0397_SCAL_NESTING + , m_snSubpicFlag (false) +#endif + , m_snNumOlssMinus1 (0) + , m_snAllLayersFlag (false) + , m_snNumLayersMinus1 (0) +#if JVET_Q0397_SCAL_NESTING + , m_snNumSubpics (1) + , m_snSubpicIdLen (0) +#endif + , m_snNumSEIs(0) + {} virtual ~SEIScalableNesting() { @@ -499,12 +512,21 @@ public: } bool m_snOlsFlag; +#if JVET_Q0397_SCAL_NESTING + bool m_snSubpicFlag; +#endif uint32_t m_snNumOlssMinus1; uint32_t m_snOlsIdxDeltaMinus1[MAX_NESTING_NUM_LAYER]; uint32_t m_snOlsIdx[MAX_NESTING_NUM_LAYER]; bool m_snAllLayersFlag; //value valid if m_nestingOlsFlag == 0 uint32_t m_snNumLayersMinus1; //value valid if m_nestingOlsFlag == 0 and m_nestingAllLayersFlag == 0 uint8_t m_snLayerId[MAX_NESTING_NUM_LAYER]; //value valid if m_nestingOlsFlag == 0 and m_nestingAllLayersFlag == 0. This can e.g. be a static array of 64 uint8_t values +#if JVET_Q0397_SCAL_NESTING + uint32_t m_snNumSubpics; + uint8_t m_snSubpicIdLen; + std::vector<uint16_t> m_snSubpicId; +#endif + uint32_t m_snNumSEIs; SEIMessages m_nestedSEIs; }; diff --git a/source/Lib/CommonLib/Slice.h b/source/Lib/CommonLib/Slice.h index 3e600d341ed3ff23f44f5a75eeb933f0dce4ee5a..e0d621986df8ade7b5d70dfb90eb1cb86883c1d3 100644 --- a/source/Lib/CommonLib/Slice.h +++ b/source/Lib/CommonLib/Slice.h @@ -1553,6 +1553,7 @@ public: void setSubPicId( int i, uint16_t u ) { m_subPicId[i] = u; } uint16_t getSubPicId( int i ) const { return m_subPicId[i]; } void setSubPicId(const std::vector<uint16_t> &v) { CHECK(v.size()!=m_numSubPics, "number of vector entries must be equal to numSubPics") ; m_subPicId = v; } + const std::vector<uint16_t> getSubPicIds() const { return m_subPicId; } uint32_t getNumLongTermRefPicSPS() const { return m_numLongTermRefPicSPS; } void setNumLongTermRefPicSPS(uint32_t val) { m_numLongTermRefPicSPS = val; } @@ -2085,6 +2086,7 @@ public: void setSubPicId( int i, uint16_t u ) { m_subPicId[i] = u; } void setSubPicId(const std::vector<uint16_t> &v) { CHECK(v.size()!=m_numSubPics, "number of vector entries must be equal to numSubPics") ; m_subPicId = v; } uint16_t getSubPicId( int i ) const { return m_subPicId[i]; } + const std::vector<uint16_t> getSubPicIds() const { return m_subPicId; } uint32_t getSubPicIdxFromSubPicId( uint32_t subPicId ) const; void setNoPicPartitionFlag( bool b ) { m_noPicPartitionFlag = b; } bool getNoPicPartitionFlag( ) const { return m_noPicPartitionFlag; } diff --git a/source/Lib/CommonLib/TypeDef.h b/source/Lib/CommonLib/TypeDef.h index 97d5b52aad7c715885c27a0e7b4456cebd0dc33f..2f307705eb805932fc6b8281230c80388f7243ac 100644 --- a/source/Lib/CommonLib/TypeDef.h +++ b/source/Lib/CommonLib/TypeDef.h @@ -178,6 +178,8 @@ #define JVET_R0100 1 // JVET-R0100: Proposal 1 DUI Signalling and inference +#define JVET_Q0397_SCAL_NESTING 1 // JVET-Q0397: Scalable Nesting SEI related aspects + #define JVET_R0413_HRD_TIMING_INFORMATION 1 // JVET-R0413: HRD timing parameters signalling #define JVET_R0214_MMVD_SYNTAX_MODIFICATION 1 // JVET-R0214: MMVD syntax modifications diff --git a/source/Lib/DecoderLib/SEIread.cpp b/source/Lib/DecoderLib/SEIread.cpp index 93846dff5340faba5df904d04e559ef3ee92d478..efe6681f54ffdfdae78095d3e521348731a6a31b 100644 --- a/source/Lib/DecoderLib/SEIread.cpp +++ b/source/Lib/DecoderLib/SEIread.cpp @@ -464,6 +464,9 @@ void SEIReader::xParseSEIScalableNesting(SEIScalableNesting& sei, const NalUnitT output_sei_message_header(sei, decodedMessageOutputStream, payloadSize); sei_read_flag(decodedMessageOutputStream, symbol, "sn_ols_flag"); sei.m_snOlsFlag = symbol; +#if JVET_Q0397_SCAL_NESTING + sei_read_flag(decodedMessageOutputStream, symbol, "sn_subpic_flag"); sei.m_snSubpicFlag = symbol; +#endif if (sei.m_snOlsFlag) { sei_read_uvlc(decodedMessageOutputStream, symbol, "sn_num_olss_minus1"); sei.m_snNumOlssMinus1 = symbol; @@ -511,6 +514,21 @@ void SEIReader::xParseSEIScalableNesting(SEIScalableNesting& sei, const NalUnitT } } } +#if JVET_Q0397_SCAL_NESTING + if (sei.m_snSubpicFlag) + { + sei_read_uvlc(decodedMessageOutputStream, symbol, "sn_num_subpics_minus1"); sei.m_snNumSubpics = symbol + 1; + sei_read_uvlc(decodedMessageOutputStream, symbol, "sn_subpic_id_len_minus1"); sei.m_snSubpicIdLen = symbol + 1; + sei.m_snSubpicId.resize(sei.m_snNumSubpics); + for (uint32_t i = 0; i < sei.m_snNumSubpics; i++) + { + sei_read_code(decodedMessageOutputStream, sei.m_snSubpicIdLen, symbol, "sn_subpic_id[i]"); sei.m_snSubpicId[i] = symbol; + } + } +#endif + + sei_read_uvlc(decodedMessageOutputStream, symbol, "sn_num_seis_minus1"); sei.m_snNumSEIs = symbol + 1; + CHECK (sei.m_snNumSEIs > 64, "The value of sn_num_seis_minus1 shall be in the range of 0 to 63"); // byte alignment while (m_pcBitstream->getNumBitsRead() % 8 != 0) @@ -518,7 +536,21 @@ void SEIReader::xParseSEIScalableNesting(SEIScalableNesting& sei, const NalUnitT sei_read_flag(decodedMessageOutputStream, symbol, "sn_zero_bit"); } - bool containBPorPTorDUI = false; + // read nested SEI messages + for (int32_t i=0; i<sei.m_snNumSEIs; i++) + { + SEIMessages tmpSEIs; + xReadSEImessage(tmpSEIs, nalUnitType, nuhLayerId, 0, vps, sps, m_nestedHrd, decodedMessageOutputStream); + if (tmpSEIs.front()->payloadType() == SEI::BUFFERING_PERIOD) + { + SEIBufferingPeriod *bp = (SEIBufferingPeriod*) tmpSEIs.front(); + m_nestedHrd.setBufferingPeriodSEI(bp); + } + sei.m_nestedSEIs.push_back(tmpSEIs.front()); + tmpSEIs.clear(); + } + + bool containBPorPTorDUI = false; bool containNoBPorPTorDUI = false; for (auto nestedsei : sei.m_nestedSEIs) { @@ -531,13 +563,7 @@ void SEIReader::xParseSEIScalableNesting(SEIScalableNesting& sei, const NalUnitT containNoBPorPTorDUI = true; } } - CHECK(containBPorPTorDUI && containNoBPorPTorDUI, "nested SEI cannot have timing-related SEI and none-timing-related SEI at the same time"); - // read nested SEI messages - do - { - HRD hrd; - xReadSEImessage(sei.m_nestedSEIs, nalUnitType, nuhLayerId, 0, vps, sps, hrd, decodedMessageOutputStream); - } while (m_pcBitstream->getNumBitsLeft() > 8); + CHECK(containBPorPTorDUI && containNoBPorPTorDUI, "Scalable Nesting SEI cannot contain timing-related SEI and none-timing-related SEIs at the same time"); if (decodedMessageOutputStream) { diff --git a/source/Lib/DecoderLib/SEIread.h b/source/Lib/DecoderLib/SEIread.h index c02c65b9773ba87c150a6da65fbb403b72a6b410..b0b196f2a381f142b360fda43e8f0cb23c9c7083 100644 --- a/source/Lib/DecoderLib/SEIread.h +++ b/source/Lib/DecoderLib/SEIread.h @@ -90,6 +90,9 @@ protected: void sei_read_uvlc(std::ostream *pOS, uint32_t& ruiCode, const char *pSymbolName); void sei_read_svlc(std::ostream *pOS, int& ruiCode, const char *pSymbolName); void sei_read_flag(std::ostream *pOS, uint32_t& ruiCode, const char *pSymbolName); + +protected: + HRD m_nestedHrd; }; diff --git a/source/Lib/EncoderLib/EncGOP.cpp b/source/Lib/EncoderLib/EncGOP.cpp index 7910bdf7a1bc89094c1d54257dca1690a8466cdc..d77473714644b9bbea838d57dcaa7aa92482f306 100644 --- a/source/Lib/EncoderLib/EncGOP.cpp +++ b/source/Lib/EncoderLib/EncGOP.cpp @@ -741,7 +741,11 @@ void EncGOP::xCreatePerPictureSEIMessages (int picInGOP, SEIMessages& seiMessage } +#if JVET_Q0397_SCAL_NESTING +void EncGOP::xCreateScalableNestingSEI(SEIMessages& seiMessages, SEIMessages& nestedSeiMessages, const std::vector<uint16_t>& subpicIDs) +#else void EncGOP::xCreateScalableNestingSEI(SEIMessages& seiMessages, SEIMessages& nestedSeiMessages) +#endif { SEIMessages tmpMessages; while (!nestedSeiMessages.empty()) @@ -750,7 +754,11 @@ void EncGOP::xCreateScalableNestingSEI(SEIMessages& seiMessages, SEIMessages& ne nestedSeiMessages.pop_front(); tmpMessages.push_back(sei); SEIScalableNesting *nestingSEI = new SEIScalableNesting(); +#if JVET_Q0397_SCAL_NESTING + m_seiEncoder.initSEIScalableNesting(nestingSEI, tmpMessages, subpicIDs); +#else m_seiEncoder.initSEIScalableNesting(nestingSEI, tmpMessages); +#endif seiMessages.push_back(nestingSEI); tmpMessages.clear(); } @@ -3327,7 +3335,38 @@ void EncGOP::compressGOP( int iPOCLast, int iNumPicRcvd, PicList& rcListPic, if (m_pcCfg->getScalableNestingSEIEnabled()) { +#if JVET_Q0397_SCAL_NESTING + const SPS* sps = pcSlice->getSPS(); + const PPS* pps = pcSlice->getPPS(); + + std::vector<uint16_t> subpicIDs; + if (sps->getSubPicInfoPresentFlag()) + { + if(sps->getSubPicIdMappingExplicitlySignalledFlag()) + { + if(sps->getSubPicIdMappingInSpsFlag()) + { + subpicIDs = sps->getSubPicIds(); + } + else + { + subpicIDs = pps->getSubPicIds(); + } + } + else + { + const int numSubPics = sps->getNumSubPics(); + subpicIDs.resize(numSubPics); + for (int i = 0 ; i < numSubPics; i++) + { + subpicIDs[i] = (uint16_t) i; + } + } + } + xCreateScalableNestingSEI(leadingSeiMessages, nestedSeiMessages, subpicIDs); +#else xCreateScalableNestingSEI(leadingSeiMessages, nestedSeiMessages); +#endif } xWriteLeadingSEIMessages( leadingSeiMessages, duInfoSeiMessages, accessUnit, pcSlice->getTLayer(), pcSlice->getSPS(), duData ); diff --git a/source/Lib/EncoderLib/EncGOP.h b/source/Lib/EncoderLib/EncGOP.h index 5a4c51ef8a2f9eac32da5419349f076cc9636ae7..26b890e196b12d55925a1b648131f2dfb15b021f 100644 --- a/source/Lib/EncoderLib/EncGOP.h +++ b/source/Lib/EncoderLib/EncGOP.h @@ -306,7 +306,11 @@ protected: void xUpdateDuData(AccessUnit &testAU, std::deque<DUData> &duData); void xUpdateTimingSEI(SEIPictureTiming *pictureTimingSEI, std::deque<DUData> &duData, const SPS *sps); void xUpdateDuInfoSEI(SEIMessages &duInfoSeiMessages, SEIPictureTiming *pictureTimingSEI, int maxSubLayers); +#if JVET_Q0397_SCAL_NESTING + void xCreateScalableNestingSEI(SEIMessages& seiMessages, SEIMessages& nestedSeiMessages, const std::vector<uint16_t>& subpicIDs); +#else void xCreateScalableNestingSEI(SEIMessages& seiMessages, SEIMessages& nestedSeiMessages); +#endif void xWriteSEI (NalUnitType naluType, SEIMessages& seiMessages, AccessUnit &accessUnit, AccessUnit::iterator &auPos, int temporalId, const SPS *sps); void xWriteSEISeparately (NalUnitType naluType, SEIMessages& seiMessages, AccessUnit &accessUnit, AccessUnit::iterator &auPos, int temporalId, const SPS *sps); void xClearSEIs(SEIMessages& seiMessages, bool deleteMessages); diff --git a/source/Lib/EncoderLib/SEIEncoder.cpp b/source/Lib/EncoderLib/SEIEncoder.cpp index 07013ea25eb21e67b1b3da8e88374ae3056ee07c..58ce350ed01be76372d918e93eabcf6cf8011d4c 100644 --- a/source/Lib/EncoderLib/SEIEncoder.cpp +++ b/source/Lib/EncoderLib/SEIEncoder.cpp @@ -389,11 +389,17 @@ void SEIEncoder::initSEISampleAspectRatioInfo(SEISampleAspectRatioInfo* seiSampl //! initialize scalable nesting SEI message. //! Note: The SEI message structures input into this function will become part of the scalable nesting SEI and will be //! automatically freed, when the nesting SEI is disposed. +#if JVET_Q0397_SCAL_NESTING +void SEIEncoder::initSEIScalableNesting(SEIScalableNesting *scalableNestingSEI, SEIMessages &nestedSEIs, const std::vector<uint16_t> &subpictureIDs) +#else void SEIEncoder::initSEIScalableNesting(SEIScalableNesting *scalableNestingSEI, SEIMessages &nestedSEIs) +#endif { - CHECK(!(m_isInitialized), "Unspecified error"); - CHECK(!(scalableNestingSEI != NULL), "Unspecified error"); - scalableNestingSEI->m_snOlsFlag = 1; // by If the nested SEI messages are picture buffering SEI messages, picture timing SEI messages or sub-picture timing SEI messages, nesting_ols_flag shall be equal to 1, by default case + CHECK(!(m_isInitialized), "Scalable Nesting SEI already initialized "); + CHECK(!(scalableNestingSEI != NULL), "No Scalable Nesting SEI object passed"); + + //KJS: OLS and layer targeting needs to be fixed for the actual OLSs in the bitstream + scalableNestingSEI->m_snOlsFlag = 1; // If the nested SEI messages are picture buffering SEI messages, picture timing SEI messages or sub-picture timing SEI messages, nesting_ols_flag shall be equal to 1, by default case scalableNestingSEI->m_snNumOlssMinus1 = 1; // by default the nesting scalable SEI message applies to two OLSs. for (int i = 0; i <= scalableNestingSEI->m_snNumOlssMinus1; i++) { @@ -414,7 +420,16 @@ void SEIEncoder::initSEIScalableNesting(SEIScalableNesting *scalableNestingSEI, scalableNestingSEI->m_snAllLayersFlag = 1; // nesting is not applied to all layers scalableNestingSEI->m_snNumLayersMinus1 = 2 - 1; //nesting_num_layers_minus1 scalableNestingSEI->m_snLayerId[0] = 0; - +#if JVET_Q0397_SCAL_NESTING + if (!subpictureIDs.empty()) + { + scalableNestingSEI->m_snSubpicFlag = 1; + scalableNestingSEI->m_snNumSubpics = (uint32_t) subpictureIDs.size(); + scalableNestingSEI->m_snSubpicId = subpictureIDs; + scalableNestingSEI->m_snSubpicIdLen = max(1, ceilLog2((*std::max_element(subpictureIDs.begin(), subpictureIDs.end())) + 1)); + CHECK ( scalableNestingSEI->m_snSubpicIdLen > 15, "Subpicture ID too large. Length must be <= 15 bits"); + } +#endif scalableNestingSEI->m_nestedSEIs.clear(); for (SEIMessages::iterator it = nestedSEIs.begin(); it != nestedSEIs.end(); it++) { diff --git a/source/Lib/EncoderLib/SEIEncoder.h b/source/Lib/EncoderLib/SEIEncoder.h index 0506df6ed80780459998d77bf78c9cc72d8c17dd..5b8c36a11e609ca81379fc6383f75011b2d48a26 100644 --- a/source/Lib/EncoderLib/SEIEncoder.h +++ b/source/Lib/EncoderLib/SEIEncoder.h @@ -71,7 +71,11 @@ public: #if U0033_ALTERNATIVE_TRANSFER_CHARACTERISTICS_SEI void initSEIAlternativeTransferCharacteristics(SEIAlternativeTransferCharacteristics *sei); #endif +#if JVET_Q0397_SCAL_NESTING + void initSEIScalableNesting(SEIScalableNesting *scalableNestingSEI, SEIMessages &nestedSEIs, const std::vector<uint16_t> &subpictureIDs); +#else void initSEIScalableNesting(SEIScalableNesting *sei, SEIMessages &nestedSEIs); +#endif // trailing SEIs void initDecodedPictureHashSEI(SEIDecodedPictureHash *sei, PelUnitBuf& pic, std::string &rHashString, const BitDepths &bitDepths); void initSEIErp(SEIEquirectangularProjection *sei); diff --git a/source/Lib/EncoderLib/SEIwrite.cpp b/source/Lib/EncoderLib/SEIwrite.cpp index 2160886524c39252c3198ea84d4bbcc70a67db4e..7a004ebab11b323b83a6223e20499162e2aeca5b 100644 --- a/source/Lib/EncoderLib/SEIwrite.cpp +++ b/source/Lib/EncoderLib/SEIwrite.cpp @@ -499,7 +499,12 @@ void SEIWriter::xWriteSEIDependentRAPIndication(const SEIDependentRAPIndication& void SEIWriter::xWriteSEIScalableNesting(OutputBitstream& bs, const SEIScalableNesting& sei, const SPS *sps) { + CHECK (sei.m_nestedSEIs.size()<1, "There must be at lease one SEI message nested in the scalable nesting SEI.") + WRITE_FLAG(sei.m_snOlsFlag, "sn_ols_flag"); +#if JVET_Q0397_SCAL_NESTING + WRITE_FLAG(sei.m_snSubpicFlag, "sn_subpic_flag"); +#endif if (sei.m_snOlsFlag) { WRITE_UVLC(sei.m_snNumOlssMinus1, "sn_num_olss_minus1"); @@ -520,6 +525,20 @@ void SEIWriter::xWriteSEIScalableNesting(OutputBitstream& bs, const SEIScalableN } } } +#if JVET_Q0397_SCAL_NESTING + if (sei.m_snSubpicFlag) + { + WRITE_UVLC( sei.m_snNumSubpics - 1, "sn_num_subpics_minus1"); + CHECK(sei.m_snSubpicIdLen < 1, "sn_subpic_id_len_minus1 must be >= 0"); + WRITE_UVLC( sei.m_snSubpicIdLen - 1, "sn_subpic_id_len_minus1"); + for (uint32_t i = 0; i < sei.m_snNumSubpics; i++) + { + WRITE_CODE(sei.m_snSubpicId[i], sei.m_snSubpicIdLen, "sn_subpic_id[i]"); + } + } +#endif + + WRITE_UVLC( (uint32_t)sei.m_nestedSEIs.size() - 1, "sn_num_seis_minus1"); // byte alignment while (m_pcBitIf->getNumberOfWrittenBits() % 8 != 0) @@ -527,9 +546,15 @@ void SEIWriter::xWriteSEIScalableNesting(OutputBitstream& bs, const SEIScalableN WRITE_FLAG(0, "sn_zero_bit"); } + SEIMessages bufferingPeriod = getSeisByType(sei.m_nestedSEIs, SEI::BUFFERING_PERIOD); + if (!bufferingPeriod.empty()) + { + SEIBufferingPeriod *bp = (SEIBufferingPeriod*)bufferingPeriod.front(); + m_nestingHrd.setBufferingPeriodSEI(bp); + } + // write nested SEI messages - HRD hrd; - writeSEImessages(bs, sei.m_nestedSEIs, sps, hrd, true, 0); + writeSEImessages(bs, sei.m_nestedSEIs, sps, m_nestingHrd, true, 0); } void SEIWriter::xWriteSEIFramePacking(const SEIFramePacking& sei) diff --git a/source/Lib/EncoderLib/SEIwrite.h b/source/Lib/EncoderLib/SEIwrite.h index 54315c0d1e7365c82f4ad6a41449385366a8beaf..0f108f333242a20f318aa8111c84b6747f10d240 100644 --- a/source/Lib/EncoderLib/SEIwrite.h +++ b/source/Lib/EncoderLib/SEIwrite.h @@ -80,6 +80,8 @@ protected: void xWriteSEIContentColourVolume(const SEIContentColourVolume &sei); void xWriteSEIpayloadData(OutputBitstream &bs, const SEI& sei, const SPS *sps, HRD &hrd, const uint32_t temporalId); void xWriteByteAlign(); +protected: + HRD m_nestingHrd; }; //! \}