From a5b3492ef05e4ad836c3f82973714cb66eb614da Mon Sep 17 00:00:00 2001
From: biaowang <biao.wang@huawei.com>
Date: Tue, 3 Mar 2020 22:27:51 +0100
Subject: [PATCH] JVET-P0124/P0095/P0222: mixed Nal Unit type within a picture
---
source/Lib/CommonLib/Slice.h | 10 +++
source/Lib/CommonLib/TypeDef.h | 5 +-
source/Lib/DecoderLib/DecLib.cpp | 125 ++++++++++++++++++++++++++++
source/Lib/DecoderLib/DecLib.h | 14 +++-
source/Lib/DecoderLib/VLCReader.cpp | 3 +
source/Lib/EncoderLib/VLCWriter.cpp | 3 +
6 files changed, 158 insertions(+), 2 deletions(-)
diff --git a/source/Lib/CommonLib/Slice.h b/source/Lib/CommonLib/Slice.h
index 0749dfce94..791e42d12b 100644
--- a/source/Lib/CommonLib/Slice.h
+++ b/source/Lib/CommonLib/Slice.h
@@ -293,6 +293,9 @@ class ConstraintInfo
bool m_noQpDeltaConstraintFlag;
bool m_noDepQuantConstraintFlag;
bool m_noSignDataHidingConstraintFlag;
+#if JVET_P0124_MIXED_NALU
+ bool m_noMixedNaluTypesInPicConstraintFlag;
+#endif
bool m_noTrailConstraintFlag;
bool m_noStsaConstraintFlag;
bool m_noRaslConstraintFlag;
@@ -353,6 +356,9 @@ public:
, m_noQpDeltaConstraintFlag (false)
, m_noDepQuantConstraintFlag (false)
, m_noSignDataHidingConstraintFlag(false)
+#if JVET_P0124_MIXED_NALU
+ , m_noMixedNaluTypesInPicConstraintFlag(false)
+#endif
, m_noTrailConstraintFlag (false)
, m_noStsaConstraintFlag (false)
, m_noRaslConstraintFlag (false)
@@ -468,6 +474,10 @@ public:
void setNoDepQuantConstraintFlag(bool bVal) { m_noDepQuantConstraintFlag = bVal; }
bool getNoSignDataHidingConstraintFlag() const { return m_noSignDataHidingConstraintFlag; }
void setNoSignDataHidingConstraintFlag(bool bVal) { m_noSignDataHidingConstraintFlag = bVal; }
+#if JVET_P0124_MIXED_NALU
+ bool getNoMixedNaluTypesInPicConstraintFlag() const { return m_noMixedNaluTypesInPicConstraintFlag; }
+ void setNoMixedNaluTypesInPicConstraintFlag(bool bVal) { m_noMixedNaluTypesInPicConstraintFlag = bVal; }
+#endif
bool getNoTrailConstraintFlag() const { return m_noTrailConstraintFlag; }
void setNoTrailConstraintFlag(bool bVal) { m_noTrailConstraintFlag = bVal; }
bool getNoStsaConstraintFlag() const { return m_noStsaConstraintFlag; }
diff --git a/source/Lib/CommonLib/TypeDef.h b/source/Lib/CommonLib/TypeDef.h
index 5340766aa9..2e9d499036 100644
--- a/source/Lib/CommonLib/TypeDef.h
+++ b/source/Lib/CommonLib/TypeDef.h
@@ -167,6 +167,7 @@
#define JVET_Q0814_DPB 1 // JVET-Q0814: DPB capacity is based on picture units regardless of the resoltuion
#define ENABLING_MULTI_SPS 1 // Bug fix to enable multiple SPS
#define SPS_ID_CHECK 1 // add SPS id check to be the same within CLVS, related to mixed_nalu_types_in_pic_flag
+#define JVET_P0124_MIXED_NALU 1 // JVET-P0124/P0095/P0222: MIxed IRAP/non-IRAP VCL NAL units within a picture
#define JVET_Q0117_PARAMETER_SETS_CLEANUP 1 // JVET-Q0117: cleanups on parameter sets
@@ -991,7 +992,9 @@ enum NalUnitType
NAL_UNIT_RESERVED_IRAP_VCL_11,
NAL_UNIT_RESERVED_IRAP_VCL_12,
-
+#if JVET_P0124_MIXED_NALU
+ NAL_UNIT_RESERVED_VCL_15,
+#endif
#if JVET_Q0117_PARAMETER_SETS_CLEANUP
NAL_UNIT_DCI, // 13
#else
diff --git a/source/Lib/DecoderLib/DecLib.cpp b/source/Lib/DecoderLib/DecLib.cpp
index d8ddbcc10b..02dc5f5b5a 100644
--- a/source/Lib/DecoderLib/DecLib.cpp
+++ b/source/Lib/DecoderLib/DecLib.cpp
@@ -2145,6 +2145,15 @@ bool DecLib::xDecodeSlice(InputNALUnit &nalu, int &iSkipFrame, int iPOCLastDispl
//---------------
pcSlice->setRefPOCList();
+#if JVET_P0124_MIXED_NALU
+ NalUnitInfo naluInfo;
+ naluInfo.m_nalUnitType = nalu.m_nalUnitType;
+ naluInfo.m_nuhLayerId = nalu.m_nuhLayerId;
+ naluInfo.m_firstCTUinSlice = pcSlice->getFirstCtuRsAddrInSlice();
+ naluInfo.m_POC = pcSlice->getPOC();
+ xCheckMixedNalUnit(pcSlice, sps, nalu);
+ m_nalUnitInfo.push_back(naluInfo);
+#endif
SEIMessages drapSEIs = getSeisByType(m_pcPic->SEIs, SEI::DEPENDENT_RAP_INDICATION );
if (!drapSEIs.empty())
{
@@ -2559,5 +2568,121 @@ void DecLib::xCheckNalUnitConstraintFlags( const ConstraintInfo *cInfo, uint32_t
"Non-conforming bitstream. no_aps_constraint_flag is equal to 1 but bitstream contains NAL unit of type APS_SUFFIX_NUT.");
}
}
+#if JVET_P0124_MIXED_NALU
+void DecLib::xCheckMixedNalUnit(Slice* pcSlice, SPS *sps, InputNALUnit &nalu)
+{
+ if (pcSlice->getPPS()->getMixedNaluTypesInPicFlag())
+ {
+ CHECK(pcSlice->getPPS()->getNumSlicesInPic() < 2, "mixed nal unit type picture, but with less than 2 slices");
+
+ const unsigned ctuRsAddr = pcSlice->getCtuAddrInSlice(0);
+ const unsigned ctuXPosInCtus = ctuRsAddr % pcSlice->getPPS()->getPicWidthInCtu();
+ const unsigned ctuYPosInCtus = ctuRsAddr / pcSlice->getPPS()->getPicWidthInCtu();
+ const unsigned maxCUSize = sps->getMaxCUWidth();
+ Position pos(ctuXPosInCtus*maxCUSize, ctuYPosInCtus*maxCUSize);
+ SubPic curSubPic = pcSlice->getPPS()->getSubPicFromPos(pos);
+ // check subpicture constraints
+ if ((pcSlice->getNalUnitType() >= NAL_UNIT_CODED_SLICE_IDR_W_RADL) && (pcSlice->getNalUnitType() <= NAL_UNIT_CODED_SLICE_CRA))
+ {
+ CHECK(curSubPic.getTreatedAsPicFlag() != true, "a slice of IDR_W_RADL to CRA_NUT shall have its subpic's sub_pic_treated_as_pic_flag equal to 1");
+ }
+ else
+ {
+ // check reference list constraint
+ if (!m_nalUnitInfo.empty())
+ {
+ //find out the closest IRAP nal unit that are in the same layer and in the corresponding subpicture
+ NalUnitInfo *latestIRAPNalUnit = nullptr;
+ int size = (int)m_nalUnitInfo.size();
+ int naluIdx;
+ for (naluIdx = size - 1; naluIdx >= 0; naluIdx--)
+ {
+ NalUnitInfo *iterNalu = &m_nalUnitInfo[naluIdx];
+ bool isTheSameLayer = iterNalu->m_nuhLayerId == nalu.m_nuhLayerId;
+ bool isIRAPSlice = iterNalu->m_nalUnitType >= NAL_UNIT_CODED_SLICE_IDR_W_RADL && iterNalu->m_nalUnitType <= NAL_UNIT_CODED_SLICE_CRA;
+ if (isTheSameLayer && isIRAPSlice)
+ {
+ latestIRAPNalUnit = iterNalu;
+ break;
+ }
+ }
+ if (latestIRAPNalUnit != nullptr)
+ {
+ // only check the current slice, as previous slice have been checked
+ const unsigned ctuRsAddrIRAP = latestIRAPNalUnit->m_firstCTUinSlice;
+ const unsigned ctuXPosInCtusIRAP = ctuRsAddrIRAP % pcSlice->getPPS()->getPicWidthInCtu();
+ const unsigned ctuYPosInCtusIRAP = ctuRsAddrIRAP / pcSlice->getPPS()->getPicWidthInCtu();
+ Position posIRAP(ctuXPosInCtusIRAP*maxCUSize, ctuXPosInCtusIRAP*maxCUSize);
+ bool isInCorrespondingSubpic = curSubPic.isContainingPos(posIRAP);
+ if (isInCorrespondingSubpic)
+ {
+ // check RefPicList[0]
+ for (int refIdx = 0; refIdx < pcSlice->getNumRefIdx(REF_PIC_LIST_0); refIdx++)
+ {
+ int POC = pcSlice->getRefPOC(REF_PIC_LIST_0, refIdx);
+ for (int i = 0; i < naluIdx; i++)
+ {
+ NalUnitInfo *iterNalu = &m_nalUnitInfo[i];
+ if (iterNalu->m_nuhLayerId == nalu.m_nuhLayerId)
+ {
+ CHECK(POC == iterNalu->m_POC, "preIRAP picture shall not be used as the reference picture of a slice after the IRAP picture");
+ }
+ }
+ }
+ // check RefPicList[1]
+ for (int refIdx = 0; refIdx < pcSlice->getNumRefIdx(REF_PIC_LIST_1); refIdx++)
+ {
+ int POC = pcSlice->getRefPOC(REF_PIC_LIST_1, refIdx);
+ for (int i = 0; i < naluIdx; i++)
+ {
+ NalUnitInfo *iterNalu = &m_nalUnitInfo[i];
+ if (iterNalu->m_nuhLayerId == nalu.m_nuhLayerId)
+ {
+ CHECK(POC == iterNalu->m_POC, "preIRAP picture shall not be used as the reference picture of a slice after the IRAP picture");
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ // check whether the nalu type of slices meet the nal unit type constraints
+ if (pcSlice->getPPS()->getNumSlicesInPic() == (m_uiSliceSegmentIdx + 1))
+ {
+ int num1stSetSlice = 0;
+ int num2ndSetSlice = 0;
+ int num3rdSetSlice = 0;
+ for (int i = 0; i < pcSlice->getPPS()->getNumSlicesInPic(); i++)
+ {
+ NalUnitType naluType = pcSlice->getNalUnitType();
+ if (naluType >= NAL_UNIT_CODED_SLICE_IDR_W_RADL && naluType <= NAL_UNIT_CODED_SLICE_CRA)
+ {
+ num1stSetSlice++;
+ }
+ else if ((naluType >= NAL_UNIT_CODED_SLICE_TRAIL && naluType <= NAL_UNIT_RESERVED_VCL_15) || naluType == NAL_UNIT_CODED_SLICE_GDR)
+ {
+ num2ndSetSlice++;
+ }
+ else
+ {
+ num3rdSetSlice++;
+ }
+ }
+ CHECK((num1stSetSlice + num2ndSetSlice) != pcSlice->getPPS()->getNumSlicesInPic() || num3rdSetSlice != 0, "mixed nal unit picture contain more than two nal unit types");
+ }
+ }
+ else // all slices shall have the same nal unit type
+ {
+ bool sameNalUnitType = true;
+ for (int i = 0; i < m_uiSliceSegmentIdx; i++)
+ {
+ Slice *PreSlice = m_pcPic->slices[i];
+ if (PreSlice->getNalUnitType() != pcSlice->getNalUnitType())
+ sameNalUnitType = false;
+ }
+ CHECK(!sameNalUnitType, "mixed_nalu_types_in_pic_flag is zero, but have different nal unit types");
+ }
+}
+#endif
//! \}
diff --git a/source/Lib/DecoderLib/DecLib.h b/source/Lib/DecoderLib/DecLib.h
index 1edbe9cc3a..160512cce1 100644
--- a/source/Lib/DecoderLib/DecLib.h
+++ b/source/Lib/DecoderLib/DecLib.h
@@ -145,6 +145,16 @@ private:
};
std::vector<AccessUnitPicInfo> m_accessUnitPicInfo;
#endif
+#if JVET_P0124_MIXED_NALU
+ struct NalUnitInfo
+ {
+ NalUnitType m_nalUnitType; ///< nal_unit_type
+ uint32_t m_nuhLayerId; ///< nuh_layer_id
+ uint32_t m_firstCTUinSlice; /// the first CTU in slice, specified with raster scan order ctu address
+ int m_POC; /// the picture order
+ };
+ std::vector<NalUnitInfo> m_nalUnitInfo;
+#endif
std::vector<int> m_accessUnitApsNals;
#if JVET_P0125_ASPECT_TID_LAYER_ID_NUH
std::vector<int> m_accessUnitSeiTids;
@@ -261,7 +271,9 @@ protected:
void xParsePrefixSEIsForUnknownVCLNal();
void xCheckNalUnitConstraintFlags( const ConstraintInfo *cInfo, uint32_t naluType );
-
+#if JVET_P0124_MIXED_NALU
+ void xCheckMixedNalUnit(Slice* pcSlice, SPS *sps, InputNALUnit &nalu);
+#endif
};// END CLASS DEFINITION DecLib
diff --git a/source/Lib/DecoderLib/VLCReader.cpp b/source/Lib/DecoderLib/VLCReader.cpp
index 1da70efb16..609d5a7756 100644
--- a/source/Lib/DecoderLib/VLCReader.cpp
+++ b/source/Lib/DecoderLib/VLCReader.cpp
@@ -5075,6 +5075,9 @@ void HLSyntaxReader::parseConstraintInfo(ConstraintInfo *cinfo)
READ_FLAG(symbol, "no_qp_delta_constraint_flag"); cinfo->setNoQpDeltaConstraintFlag(symbol > 0 ? true : false);
READ_FLAG(symbol, "no_dep_quant_constraint_flag"); cinfo->setNoDepQuantConstraintFlag(symbol > 0 ? true : false);
READ_FLAG(symbol, "no_sign_data_hiding_constraint_flag"); cinfo->setNoSignDataHidingConstraintFlag(symbol > 0 ? true : false);
+#if JVET_P0124_MIXED_NALU
+ READ_FLAG(symbol, "no_mixed_nalu_types_in_pic_constraint_flag"); cinfo->setNoMixedNaluTypesInPicConstraintFlag(symbol > 0 ? true : false);
+#endif
READ_FLAG(symbol, "no_trail_constraint_flag"); cinfo->setNoTrailConstraintFlag(symbol > 0 ? true : false);
READ_FLAG(symbol, "no_stsa_constraint_flag"); cinfo->setNoStsaConstraintFlag(symbol > 0 ? true : false);
READ_FLAG(symbol, "no_rasl_constraint_flag"); cinfo->setNoRaslConstraintFlag(symbol > 0 ? true : false);
diff --git a/source/Lib/EncoderLib/VLCWriter.cpp b/source/Lib/EncoderLib/VLCWriter.cpp
index 5a7c67b246..77ae5ca50f 100644
--- a/source/Lib/EncoderLib/VLCWriter.cpp
+++ b/source/Lib/EncoderLib/VLCWriter.cpp
@@ -3310,6 +3310,9 @@ void HLSWriter::codeConstraintInfo ( const ConstraintInfo* cinfo )
WRITE_FLAG(cinfo->getNoQpDeltaConstraintFlag() ? 1 : 0, "no_qp_delta_constraint_flag");
WRITE_FLAG(cinfo->getNoDepQuantConstraintFlag() ? 1 : 0, "no_dep_quant_constraint_flag");
WRITE_FLAG(cinfo->getNoSignDataHidingConstraintFlag() ? 1 : 0, "no_sign_data_hiding_constraint_flag");
+#if JVET_P0124_MIXED_NALU
+ WRITE_FLAG(cinfo->getNoMixedNaluTypesInPicConstraintFlag() ? 1 : 0, "no_mixed_nalu_types_in_pic_constraint_flag");
+#endif
WRITE_FLAG(cinfo->getNoTrailConstraintFlag() ? 1 : 0, "no_trail_constraint_flag");
WRITE_FLAG(cinfo->getNoStsaConstraintFlag() ? 1 : 0, "no_stsa_constraint_flag");
WRITE_FLAG(cinfo->getNoRaslConstraintFlag() ? 1 : 0, "no_rasl_constraint_flag");
--
GitLab