From 51330cb68a9080db661dafa9a9347ca12cdd6d44 Mon Sep 17 00:00:00 2001 From: Yang Wang <wangyang.cs@bytedance.com> Date: Mon, 9 Nov 2020 14:53:12 +0800 Subject: [PATCH] JVET-T0055 aspect4 --- source/App/DecoderApp/DecApp.cpp | 4 + source/Lib/CommonLib/TypeDef.h | 2 + source/Lib/DecoderLib/DecLib.cpp | 172 ++++++++++++++++++++++++++++ source/Lib/DecoderLib/DecLib.h | 7 ++ source/Lib/DecoderLib/SEIread.cpp | 184 ++++++++++++++++++++++++++++++ source/Lib/DecoderLib/SEIread.h | 6 + 6 files changed, 375 insertions(+) diff --git a/source/App/DecoderApp/DecApp.cpp b/source/App/DecoderApp/DecApp.cpp index 100720131..1dca32afd 100644 --- a/source/App/DecoderApp/DecApp.cpp +++ b/source/App/DecoderApp/DecApp.cpp @@ -506,6 +506,10 @@ uint32_t DecApp::decode() m_cDecLib.checkSEIInAccessUnit(); #endif m_cDecLib.resetAccessUnitSeiPayLoadTypes(); +#if JVET_T0055_ASPECT4 + m_cDecLib.checkSeiContentInAccessUnit(); + m_cDecLib.resetAccessUnitSeiNalus(); +#endif m_cDecLib.resetAccessUnitNals(); m_cDecLib.resetAccessUnitApsNals(); m_cDecLib.resetAccessUnitPicInfo(); diff --git a/source/Lib/CommonLib/TypeDef.h b/source/Lib/CommonLib/TypeDef.h index 57ba502bf..18d0d0c45 100644 --- a/source/Lib/CommonLib/TypeDef.h +++ b/source/Lib/CommonLib/TypeDef.h @@ -52,6 +52,8 @@ // clang-format off +#define JVET_T0055_ASPECT4 1 // JVET-T0055 aspect4: When there are multiple SEI messages with a particular value of payloadType not equal to 133 that are associated with a particular AU or DU and apply to a particular OLS or layer, regardless of whether some or all of these SEI messages are scalable-nested, the SEI messages shall have the same SEI payload content. + //########### place macros to be removed in next cycle below this line ############### #define JVET_W0133_CONSTRAINED_RASL_ENCODING 1 // SEI message for Constrained RASL encoding for bitstream switching diff --git a/source/Lib/DecoderLib/DecLib.cpp b/source/Lib/DecoderLib/DecLib.cpp index 1347b946c..3d8c09edc 100644 --- a/source/Lib/DecoderLib/DecLib.cpp +++ b/source/Lib/DecoderLib/DecLib.cpp @@ -1439,6 +1439,172 @@ void DecLib::resetPictureSeiNalus() } } +#if JVET_T0055_ASPECT4 +void DecLib::checkSeiContentInAccessUnit() +{ + std::vector<std::tuple<int, int, bool, uint32_t, uint8_t*>> seiList; + + // get the OLSs that cover all layers + std::vector<uint32_t> olsIds; + for (uint32_t i = 0; i < m_vps->getNumOutputLayerSets(); i++) + { + bool olsIncludeAllLayersFind = false; + for (auto pic = m_firstAccessUnitPicInfo.begin(); pic != m_firstAccessUnitPicInfo.end(); pic++) + { + int targetLayerId = pic->m_nuhLayerId; + for (int j = 0; j < m_vps->getNumLayersInOls(i); j++) + { + olsIncludeAllLayersFind = m_vps->getLayerIdInOls(i, j) == targetLayerId ? true : false; + if (olsIncludeAllLayersFind) + { + break; + } + } + if (!olsIncludeAllLayersFind) + { + break; + } + } + if (olsIncludeAllLayersFind) + { + olsIds.push_back(i); + } + } + + // extract SEI messages from NAL units + for (auto &sei : m_accessUnitSeiNalus) + { + InputBitstream bs = sei->getBitstream(); + + do + { + int payloadType = 0; + int payloadLayerId = sei->m_nuhLayerId; + uint32_t val = 0; + + do + { + bs.readByte(val); + payloadType += val; + } while (val==0xFF); + + uint32_t payloadSize = 0; + do + { + bs.readByte(val); + payloadSize += val; + } while (val==0xFF); + + if (payloadType != SEI::SCALABLE_NESTING && payloadType != SEI::USER_DATA_REGISTERED_ITU_T_T35 && payloadType != SEI::USER_DATA_UNREGISTERED) + { + if (payloadType == SEI::BUFFERING_PERIOD || payloadType == SEI::PICTURE_TIMING || payloadType == SEI::DECODING_UNIT_INFO || payloadType == SEI::SUBPICTURE_LEVEL_INFO) + { + uint8_t *payload = new uint8_t[payloadSize]; + for (uint32_t i = 0; i < payloadSize; i++) + { + bs.readByte(val); + payload[i] = (uint8_t)val; + } + for (uint32_t i = 0; i < olsIds.size(); i++) + { + if (i == 0) + { + seiList.push_back(std::tuple<int, int, bool, uint32_t, uint8_t*>(payloadType, olsIds.at(i), false, payloadSize, payload)); + } + else + { + uint8_t *payloadTemp = new uint8_t[payloadSize]; + memcpy(payloadTemp, payload, payloadSize *sizeof(uint8_t)); + seiList.push_back(std::tuple<int, int, bool, uint32_t, uint8_t*>(payloadType, olsIds.at(i), false, payloadSize, payloadTemp)); + } + } + } + else + { + uint8_t *payload = new uint8_t[payloadSize]; + for (uint32_t i = 0; i < payloadSize; i++) + { + bs.readByte(val); + payload[i] = (uint8_t)val; + } + seiList.push_back(std::tuple<int, int, bool, uint32_t, uint8_t*>(payloadType, payloadLayerId, false, payloadSize, payload)); + } + } + else + { + const SPS *sps = m_parameterSetManager.getActiveSPS(); + const VPS *vps = m_parameterSetManager.getVPS(sps->getVPSId()); + m_seiReader.parseAndExtractSEIScalableNesting(&bs, sei->m_nalUnitType, payloadLayerId, vps, sps, m_HRD, payloadSize, &seiList); + } + } + while (bs.getNumBitsLeft() > 8); + } + + // check contents of the repeated messages in list + for (uint32_t i = 0; i < seiList.size(); i++) + { + int payloadType1 = std::get<0>(seiList[i]); + int payLoadLayerId1 = std::get<1>(seiList[i]); + bool payLoadNested1 = std::get<2>(seiList[i]); + uint32_t payloadSize1 = std::get<3>(seiList[i]); + uint8_t *payload1 = std::get<4>(seiList[i]); + + // compare current SEI message with remaining messages in the list + for (uint32_t j = i+1; j < seiList.size(); j++) + { + int payloadType2 = std::get<0>(seiList[j]); + int payLoadLayerId2 = std::get<1>(seiList[j]); + bool payLoadNested2 = std::get<2>(seiList[j]); + uint32_t payloadSize2 = std::get<3>(seiList[j]); + uint8_t *payload2 = std::get<4>(seiList[j]); + + // check for identical SEI type, olsId or layerId, size, and payload + if (payloadType1 == SEI::BUFFERING_PERIOD || payloadType1 == SEI::PICTURE_TIMING || payloadType1 == SEI::DECODING_UNIT_INFO || payloadType1 == SEI::SUBPICTURE_LEVEL_INFO) + { + CHECK((payloadType1 == payloadType2) && (payLoadLayerId1 == payLoadLayerId2) && ((payloadSize1 != payloadSize2) || memcmp(payload1, payload2, payloadSize1*sizeof(uint8_t))), "When there are multiple SEI messages with a particular value of payloadType not equal to 133 that are associated with a particular AU or DU and apply to a particular OLS or layer, regardless of whether some or all of these SEI messages are scalable-nested, the SEI messages shall have the same SEI payload content."); + } + else + { + bool bSameLayer = false; + if (!payLoadNested1 && !payLoadNested2) + { + bSameLayer = (payLoadLayerId1 == payLoadLayerId2); + } + else if (payLoadNested1 && payLoadNested2) + { + bSameLayer = true; + } + else + { + bSameLayer = payLoadNested1 ? payLoadLayerId2 >= payLoadLayerId1 : payLoadLayerId1 >= payLoadLayerId2; + } + CHECK(payloadType1 == payloadType2 && bSameLayer && ((payloadSize1 != payloadSize2) || memcmp(payload1, payload2, payloadSize1*sizeof(uint8_t))), "When there are multiple SEI messages with a particular value of payloadType not equal to 133 that are associated with a particular AU or DU and apply to a particular OLS or layer, regardless of whether some or all of these SEI messages are scalable-nested, the SEI messages shall have the same SEI payload content."); + } + } + } + + // free SEI message list memory + for (uint32_t i = 0; i < seiList.size(); i++) + { + uint8_t *payload = std::get<4>(seiList[i]); + delete[] payload; + } + seiList.clear(); +} + +/** + - Reset list of SEI NAL units from the current access unit + */ +void DecLib::resetAccessUnitSeiNalus() +{ + while (!m_accessUnitSeiNalus.empty()) + { + delete m_accessUnitSeiNalus.front(); + m_accessUnitSeiNalus.pop_front(); + } +} +#endif + /** - Process buffered list of suffix APS NALUs */ @@ -2230,6 +2396,9 @@ void DecLib::xParsePrefixSEImessages() while (!m_prefixSEINALUs.empty()) { InputNALUnit &nalu=*m_prefixSEINALUs.front(); +#if JVET_T0055_ASPECT4 + m_accessUnitSeiNalus.push_back(new InputNALUnit(nalu)); +#endif m_accessUnitSeiTids.push_back(nalu.m_temporalId); const SPS *sps = m_parameterSetManager.getActiveSPS(); const VPS *vps = m_parameterSetManager.getVPS(sps->getVPSId()); @@ -3449,6 +3618,9 @@ bool DecLib::decode(InputNALUnit& nalu, int& iSkipFrame, int& iPOCLastDisplay, i return false; } m_pictureSeiNalus.push_back(new InputNALUnit(nalu)); +#if JVET_T0055_ASPECT4 + m_accessUnitSeiNalus.push_back(new InputNALUnit(nalu)); +#endif m_accessUnitSeiTids.push_back(nalu.m_temporalId); const SPS *sps = m_parameterSetManager.getActiveSPS(); const VPS *vps = m_parameterSetManager.getVPS(sps->getVPSId()); diff --git a/source/Lib/DecoderLib/DecLib.h b/source/Lib/DecoderLib/DecLib.h index 5e6192754..aa3f755c7 100644 --- a/source/Lib/DecoderLib/DecLib.h +++ b/source/Lib/DecoderLib/DecLib.h @@ -205,6 +205,9 @@ private: std::vector<NalUnitType> m_pictureUnitNals; std::list<InputNALUnit*> m_pictureSeiNalus; std::list<InputNALUnit*> m_suffixApsNalus; +#if JVET_T0055_ASPECT4 + std::list<InputNALUnit*> m_accessUnitSeiNalus; +#endif OPI* m_opi; bool m_mTidExternalSet; @@ -279,6 +282,10 @@ public: void checkTidLayerIdInAccessUnit(); void resetAccessUnitSeiPayLoadTypes() { m_accessUnitSeiPayLoadTypes.clear(); } void checkSEIInAccessUnit(); +#if JVET_T0055_ASPECT4 + void checkSeiContentInAccessUnit(); + void resetAccessUnitSeiNalus(); +#endif void checkLayerIdIncludedInCvss(); void CheckNoOutputPriorPicFlagsInAccessUnit(); void resetAccessUnitNoOutputPriorPicFlags() { m_accessUnitNoOutputPriorPicFlags.clear(); } diff --git a/source/Lib/DecoderLib/SEIread.cpp b/source/Lib/DecoderLib/SEIread.cpp index d6dd927c0..93a83fd59 100644 --- a/source/Lib/DecoderLib/SEIread.cpp +++ b/source/Lib/DecoderLib/SEIread.cpp @@ -141,6 +141,48 @@ void SEIReader::parseSEImessage(InputBitstream* bs, SEIMessages& seis, const Nal xReadRbspTrailingBits(); } +#if JVET_T0055_ASPECT4 +void SEIReader::parseAndExtractSEIScalableNesting(InputBitstream* bs, const NalUnitType nalUnitType, const uint32_t nuh_layer_id, const VPS* vps, const SPS* sps, HRD &hrd, uint32_t payloadSize, std::vector<std::tuple<int, int, bool, uint32_t, uint8_t*>> *seiList) +{ + SEI *sei = NULL; + sei = new SEIScalableNesting; + setBitstream(bs); + xParseSEIScalableNestingBinary((SEIScalableNesting&)*sei, nalUnitType, nuh_layer_id, payloadSize, vps, sps, hrd, NULL, seiList); + int payloadBitsRemaining = getBitstream()->getNumBitsLeft(); + if (payloadBitsRemaining) /* more_data_in_payload() */ + { + for (; payloadBitsRemaining > 9; payloadBitsRemaining--) + { + uint32_t reservedPayloadExtensionData; + sei_read_code ( NULL, 1, reservedPayloadExtensionData, "reserved_payload_extension_data"); + } + + /* 2 */ + int finalBits = getBitstream()->peekBits(payloadBitsRemaining); + int finalPayloadBits = 0; + for (int mask = 0xff; finalBits & (mask >> finalPayloadBits); finalPayloadBits++) + { + continue; + } + + /* 3 */ + for (; payloadBitsRemaining > 9 - finalPayloadBits; payloadBitsRemaining--) + { + uint32_t reservedPayloadExtensionData; + sei_read_flag ( 0, reservedPayloadExtensionData, "reserved_payload_extension_data"); + } + + uint32_t dummy; + sei_read_flag( 0, dummy, "payload_bit_equal_to_one"); payloadBitsRemaining--; + while (payloadBitsRemaining) + { + sei_read_flag( 0, dummy, "payload_bit_equal_to_zero"); payloadBitsRemaining--; + } + } + delete sei; +} +#endif + void SEIReader::xReadSEImessage(SEIMessages& seis, const NalUnitType nalUnitType, const uint32_t nuh_layer_id, const uint32_t temporalId, const VPS *vps, const SPS *sps, HRD &hrd, std::ostream *pDecodedMessageOutputStream) { #if ENABLE_TRACING @@ -623,6 +665,148 @@ void SEIReader::xParseSEIScalableNesting(SEIScalableNesting& sei, const NalUnitT } } +#if JVET_T0055_ASPECT4 +void SEIReader::xParseSEIScalableNestingBinary(SEIScalableNesting& sei, const NalUnitType nalUnitType, const uint32_t nuhLayerId, uint32_t payloadSize, const VPS* vps, const SPS* sps, HRD &hrd, std::ostream* decodedMessageOutputStream, std::vector<std::tuple<int, int, bool, uint32_t, uint8_t*>> *seiList) +{ + uint32_t symbol; + SEIMessages seis; + output_sei_message_header(sei, decodedMessageOutputStream, payloadSize); + + sei_read_flag(decodedMessageOutputStream, symbol, "sn_ols_flag"); sei.m_snOlsFlag = symbol; + sei_read_flag(decodedMessageOutputStream, symbol, "sn_subpic_flag"); sei.m_snSubpicFlag = symbol; + if (sei.m_snOlsFlag) + { + sei_read_uvlc(decodedMessageOutputStream, symbol, "sn_num_olss_minus1"); sei.m_snNumOlssMinus1 = symbol; + for (uint32_t i = 0; i <= sei.m_snNumOlssMinus1; i++) + { + sei_read_uvlc(decodedMessageOutputStream, symbol, "sn_ols_idx_delta_minus1[i]"); sei.m_snOlsIdxDeltaMinus1[i] = symbol; + } + for (uint32_t i = 0; i <= sei.m_snNumOlssMinus1; i++) + { + if (i == 0) + { + sei.m_snOlsIdx[i] = sei.m_snOlsIdxDeltaMinus1[i]; + } + else + { + sei.m_snOlsIdx[i] = sei.m_snOlsIdxDeltaMinus1[i] + sei.m_snOlsIdxDeltaMinus1[i - 1] + 1; + } + } + if (vps && vps->getVPSId() != 0) + { + uint32_t lowestLayerId = MAX_UINT; + for (uint32_t olsIdxForSEI = 0; olsIdxForSEI <= sei.m_snNumOlssMinus1; olsIdxForSEI++) + { + int olsIdx = sei.m_snOlsIdx[olsIdxForSEI]; + for (int layerIdx = 0; layerIdx < vps->getNumLayersInOls(olsIdx); layerIdx++) + { + if (lowestLayerId > vps->getLayerIdInOls(olsIdx, layerIdx)) + { + lowestLayerId = vps->getLayerIdInOls(olsIdx, layerIdx); + } + } + } + CHECK(lowestLayerId!= nuhLayerId, "nuh_layer_id is not equal to the lowest layer among Olss that the scalable SEI applies"); + } + } + else + { + sei_read_flag(decodedMessageOutputStream, symbol, "sn_all_layers_flag"); sei.m_snAllLayersFlag = symbol; + if (!sei.m_snAllLayersFlag) + { + sei_read_uvlc(decodedMessageOutputStream, symbol, "sn_num_layers_minus1"); sei.m_snNumLayersMinus1 = symbol; + sei.m_snLayerId[0] = nuhLayerId; + for (uint32_t i = 1; i <= sei.m_snNumLayersMinus1; i++) + { + sei_read_code(decodedMessageOutputStream, 6, symbol, "sn_layer_id[i]"); sei.m_snLayerId[i] = symbol; + } + } + } + if (sei.m_snSubpicFlag) + { + sei_read_uvlc(decodedMessageOutputStream, symbol, "sn_num_subpics_minus1"); sei.m_snNumSubpics = symbol + 1; + sei_read_uvlc(decodedMessageOutputStream, symbol, "sn_subpic_id_len_minus1"); sei.m_snSubpicIdLen = symbol + 1; + sei.m_snSubpicId.resize(sei.m_snNumSubpics); + for (uint32_t i = 0; i < sei.m_snNumSubpics; i++) + { + sei_read_code(decodedMessageOutputStream, sei.m_snSubpicIdLen, symbol, "sn_subpic_id[i]"); sei.m_snSubpicId[i] = symbol; + } + } + + sei_read_uvlc(decodedMessageOutputStream, symbol, "sn_num_seis_minus1"); sei.m_snNumSEIs = symbol + 1; + CHECK (sei.m_snNumSEIs > 64, "The value of sn_num_seis_minus1 shall be in the range of 0 to 63"); + + // byte alignment + while (m_pcBitstream->getNumBitsRead() % 8 != 0) + { + sei_read_flag(decodedMessageOutputStream, symbol, "sn_zero_bit"); + } + + // above codes are exactly same as those in xParseSEIScalableNesting() + // read and save nested SEI messages in binary form + for (int32_t i=0; i<sei.m_snNumSEIs; i++) + { + int payloadType = 0; + uint32_t val = 0; + do + { + sei_read_code(NULL, 8, val, "payload_type"); + payloadType += val; + } while (val==0xFF); + + uint32_t payloadSize = 0; + do + { + sei_read_code(NULL, 8, val, "payload_size"); + payloadSize += val; + } while (val==0xFF); + + uint8_t *payload = new uint8_t[payloadSize]; + for (uint32_t j = 0; j < payloadSize; j++) + { + sei_read_code(NULL, 8, val, "payload_content"); + payload[j] = (uint8_t)val; + } + if (sei.m_snOlsFlag) + { + for (uint32_t j = 0; j <= sei.m_snNumOlssMinus1; j++) + { + if (j == 0) + { + seiList->push_back(std::tuple<int, int, bool, uint32_t, uint8_t*>(payloadType, sei.m_snOlsIdx[j], false, payloadSize, payload)); + } + else + { + uint8_t *payloadTemp = new uint8_t[payloadSize]; + memcpy(payloadTemp, payload, payloadSize *sizeof(uint8_t)); + seiList->push_back(std::tuple<int, int, bool, uint32_t, uint8_t*>(payloadType, sei.m_snOlsIdx[j], false, payloadSize, payloadTemp)); + } + } + } + else if (sei.m_snAllLayersFlag) + { + seiList->push_back(std::tuple<int, int, bool, uint32_t, uint8_t*>(payloadType, nuhLayerId, true, payloadSize, payload)); + } + else // !sei.m_snOlsFlag && !sei.m_snAllLayersFlag + { + for (uint32_t j = 0; j <= sei.m_snNumLayersMinus1; j++) + { + if (j == 0) + { + seiList->push_back(std::tuple<int, int, bool, uint32_t, uint8_t*>(payloadType, sei.m_snLayerId[j], false, payloadSize, payload)); + } + else + { + uint8_t *payloadTemp = new uint8_t[payloadSize]; + memcpy(payloadTemp, payload, payloadSize *sizeof(uint8_t)); + seiList->push_back(std::tuple<int, int, bool, uint32_t, uint8_t*>(payloadType, sei.m_snLayerId[j], false, payloadSize, payloadTemp)); + } + } + } + } +} +#endif + void SEIReader::xCheckScalableNestingConstraints(const SEIScalableNesting& sei, const NalUnitType nalUnitType, const VPS* vps) { const std::vector<int> vclAssociatedSeiList { 3, 19, 45, 129, 137, 144, 145, 147, 148, 149, 150, 153, 154, 155, 156, 168, 204 }; diff --git a/source/Lib/DecoderLib/SEIread.h b/source/Lib/DecoderLib/SEIread.h index 8a9f0aea7..a9d687914 100644 --- a/source/Lib/DecoderLib/SEIread.h +++ b/source/Lib/DecoderLib/SEIread.h @@ -56,6 +56,9 @@ public: SEIReader() {}; virtual ~SEIReader() {}; void parseSEImessage(InputBitstream* bs, SEIMessages& seis, const NalUnitType nalUnitType, const uint32_t nuh_layer_id, const uint32_t temporalId,const VPS *vps, const SPS *sps, HRD &hrd, std::ostream *pDecodedMessageOutputStream); +#if JVET_T0055_ASPECT4 + void parseAndExtractSEIScalableNesting(InputBitstream* bs, const NalUnitType nalUnitType, const uint32_t nuh_layer_id, const VPS* vps, const SPS* sps, HRD &hrd, uint32_t payloadSize, std::vector<std::tuple<int, int, bool, uint32_t, uint8_t*>> *seiList); +#endif protected: void xReadSEImessage (SEIMessages& seis, const NalUnitType nalUnitType, const uint32_t nuh_layer_id, const uint32_t temporalId, const VPS *vps, const SPS *sps, HRD &hrd, std::ostream *pDecodedMessageOutputStream); @@ -65,6 +68,9 @@ protected: void xParseSEIBufferingPeriod (SEIBufferingPeriod& sei, uint32_t payloadSize, std::ostream *pDecodedMessageOutputStream); void xParseSEIPictureTiming (SEIPictureTiming& sei, uint32_t payloadSize, const uint32_t temporalId, const SEIBufferingPeriod& bp, std::ostream *pDecodedMessageOutputStream); void xParseSEIScalableNesting (SEIScalableNesting& sei, const NalUnitType nalUnitType, const uint32_t nuhLayerId, uint32_t payloadSize, const VPS* vps, const SPS* sps, HRD &hrd, std::ostream* decodedMessageOutputStream); +#if JVET_T0055_ASPECT4 + void xParseSEIScalableNestingBinary (SEIScalableNesting& sei, const NalUnitType nalUnitType, const uint32_t nuhLayerId, uint32_t payloadSize, const VPS* vps, const SPS* sps, HRD &hrd, std::ostream* decodedMessageOutputStream, std::vector<std::tuple<int, int, bool, uint32_t, uint8_t*>> *seiList); +#endif void xCheckScalableNestingConstraints (const SEIScalableNesting& sei, const NalUnitType nalUnitType, const VPS* vps); void xParseSEIFrameFieldinfo (SEIFrameFieldInfo& sei, uint32_t payloadSize, std::ostream *pDecodedMessageOutputStream); void xParseSEIDependentRAPIndication (SEIDependentRAPIndication& sei, uint32_t payLoadSize, std::ostream *pDecodedMessageOutputStream); -- GitLab