From c1a58c7401aa294960cc8f305f3b908b50fb8ade Mon Sep 17 00:00:00 2001
From: Hendry <hendry197@gmail.com>
Date: Thu, 30 Mar 2023 15:57:35 -0700
Subject: [PATCH] Implementation of nnpfc_base_flag
- Add the control for that flag in the cfg file & update the manual doc
- Implement checking at encoder side to ensure that when sending an update the base filter is already present
- Implement detection at decoder whether the given base filter is a new or a repetition filter.
- Implement checking at decoder side to ensure that when receiving an update filter, the base filter has been received earlier / already present.
---
...al_network_post_filter_characteristics.cfg | 1 +
doc/software-manual.tex | 4 ++
source/App/EncoderApp/EncApp.cpp | 16 +++++++
source/App/EncoderApp/EncAppCfg.cpp | 6 +++
source/App/EncoderApp/EncAppCfg.h | 3 ++
source/Lib/CommonLib/SEI.h | 6 +++
source/Lib/CommonLib/TypeDef.h | 2 +-
source/Lib/DecoderLib/SEIread.cpp | 44 ++++++++++++++++++-
source/Lib/DecoderLib/SEIread.h | 3 ++
source/Lib/EncoderLib/EncCfg.h | 7 +++
source/Lib/EncoderLib/SEIEncoder.cpp | 3 ++
source/Lib/EncoderLib/SEIwrite.cpp | 3 ++
12 files changed, 96 insertions(+), 2 deletions(-)
diff --git a/cfg/sei_vui/neural_network_post_filter_characteristics.cfg b/cfg/sei_vui/neural_network_post_filter_characteristics.cfg
index a69af8b28..19a7cd521 100644
--- a/cfg/sei_vui/neural_network_post_filter_characteristics.cfg
+++ b/cfg/sei_vui/neural_network_post_filter_characteristics.cfg
@@ -6,6 +6,7 @@ SEINNPostFilterCharacteristicsNumFilters: 1
SEINNPostFilterCharacteristicsId0: 1
SEINNPostFilterCharacteristicsModeIdc0: 1
SEINNPostFilterCharacteristicsPropertyPresentFlag0: 1
+SEINNPostFilterCharacteristicsBaseFlag0: 1
SEINNPostFilterCharacteristicsPurpose0: 2
SEINNPostFilterCharacteristicsOutSubCFlag0: 1
SEINNPostFilterCharacteristicsComponentLastFlag0: 0
diff --git a/doc/software-manual.tex b/doc/software-manual.tex
index 28627fb0c..2b3f5837d 100644
--- a/doc/software-manual.tex
+++ b/doc/software-manual.tex
@@ -5702,6 +5702,10 @@ Specifies sii_num_units_in_shutter_interval for single entry.If multiple entries
\Default{false} &
When true (non-zero) specifies, for the \emph{i}-th neural network post-filter, that the filter input formatting, output formatting, and complexity are present.
\\
+ \Option{SEINNPostFilterCharacteristicsBaseFlag\emph{i}} &
+ \Default{false} &
+ When true (non-zero) specifies, for the \emph{i}-th neural network post-filter, that the filter is a base filter.
+ \\
\Option{SEINNPostFilterCharacteristicsPurpose\emph{i}} &
\Default{0} &
Specifies the purpose of the \emph{i}-th neural network post-filter.
diff --git a/source/App/EncoderApp/EncApp.cpp b/source/App/EncoderApp/EncApp.cpp
index 2f8957421..f3b313e8b 100644
--- a/source/App/EncoderApp/EncApp.cpp
+++ b/source/App/EncoderApp/EncApp.cpp
@@ -1187,6 +1187,22 @@ void EncApp::xInitLibCfg( int layerIdx )
m_cEncLib.setNNPostFilterSEICharacteristicsPropertyPresentFlag( m_nnPostFilterSEICharacteristicsPropertyPresentFlag[i], i);
if (m_cEncLib.getNNPostFilterSEICharacteristicsPropertyPresentFlag(i))
{
+#if JVET_AC0353_NNPFC_BASE_FLAG
+ m_cEncLib.setNNPostFilterSEICharacteristicsBaseFlag (m_nnPostFilterSEICharacteristicsBaseFlag[i], i);
+ if (!m_nnPostFilterSEICharacteristicsBaseFlag[i])
+ {
+ bool baseFilterExist = false;
+ for (int j = i - 1; j >= 0; j--)
+ {
+ if (m_cEncLib.getNNPostFilterSEICharacteristicsId(i) == m_cEncLib.getNNPostFilterSEICharacteristicsId(j))
+ {
+ baseFilterExist = true;
+ break;
+ }
+ }
+ CHECK(!baseFilterExist, "No base filter found! Cannot have an update filter without base filter.")
+ }
+#endif
m_cEncLib.setNNPostFilterSEICharacteristicsPurpose (m_nnPostFilterSEICharacteristicsPurpose[i], i);
#if JVET_AC0127_BIT_MASKING_NNPFC_PURPOSE
if ((m_cEncLib.getNNPostFilterSEICharacteristicsPurpose(i) & NNPC_PurposeType::CHROMA_UPSAMPLING) != 0)
diff --git a/source/App/EncoderApp/EncAppCfg.cpp b/source/App/EncoderApp/EncAppCfg.cpp
index 4e92b0777..25fba2094 100644
--- a/source/App/EncoderApp/EncAppCfg.cpp
+++ b/source/App/EncoderApp/EncAppCfg.cpp
@@ -1771,6 +1771,12 @@ bool EncAppCfg::parseCfg( int argc, char* argv[] )
propertyPresentFlag << "SEINNPostFilterCharacteristicsPropertyPresentFlag" << i;
opts.addOptions()(propertyPresentFlag.str(), m_nnPostFilterSEICharacteristicsPropertyPresentFlag[i], false, "Specifies whether the filter purpose, input formatting, output formatting and complexity are present in the Neural Network Post Filter Characteristics SEI message");
+#if JVET_AC0353_NNPFC_BASE_FLAG
+ std::ostringstream nnpfcBaseFlag;
+ nnpfcBaseFlag << "SEINNPostFilterCharacteristicsBaseFlag" << i;
+ opts.addOptions()(nnpfcBaseFlag.str(), m_nnPostFilterSEICharacteristicsBaseFlag[i], false, "Specifies whether the filter is a base filter or not");
+#endif
+
std::ostringstream purpose;
purpose << "SEINNPostFilterCharacteristicsPurpose" << i;
opts.addOptions()(purpose.str(), m_nnPostFilterSEICharacteristicsPurpose[i], 0u, "Specifies the purpose in the Neural Network Post Filter Characteristics SEI message");
diff --git a/source/App/EncoderApp/EncAppCfg.h b/source/App/EncoderApp/EncAppCfg.h
index b7c278083..ce088b438 100644
--- a/source/App/EncoderApp/EncAppCfg.h
+++ b/source/App/EncoderApp/EncAppCfg.h
@@ -732,6 +732,9 @@ protected:
uint32_t m_nnPostFilterSEICharacteristicsId[MAX_NUM_NN_POST_FILTERS];
uint32_t m_nnPostFilterSEICharacteristicsModeIdc[MAX_NUM_NN_POST_FILTERS];
bool m_nnPostFilterSEICharacteristicsPropertyPresentFlag[MAX_NUM_NN_POST_FILTERS];
+#if JVET_AC0353_NNPFC_BASE_FLAG
+ bool m_nnPostFilterSEICharacteristicsBaseFlag[MAX_NUM_NN_POST_FILTERS];
+#endif
uint32_t m_nnPostFilterSEICharacteristicsPurpose[MAX_NUM_NN_POST_FILTERS];
bool m_nnPostFilterSEICharacteristicsOutSubCFlag[MAX_NUM_NN_POST_FILTERS];
#if JVET_AC0154
diff --git a/source/Lib/CommonLib/SEI.h b/source/Lib/CommonLib/SEI.h
index 821b2ed21..58f48aca0 100644
--- a/source/Lib/CommonLib/SEI.h
+++ b/source/Lib/CommonLib/SEI.h
@@ -1301,6 +1301,9 @@ public:
, m_modeIdc(0)
, m_propertyPresentFlag(false)
, m_purpose(0)
+#if JVET_AC0353_NNPFC_BASE_FLAG
+ , m_baseFlag(false)
+#endif
, m_outSubCFlag(0)
, m_outSubWidthC(1)
, m_outSubHeightC(1)
@@ -1367,6 +1370,9 @@ public:
uint32_t m_modeIdc;
bool m_propertyPresentFlag;
uint32_t m_purpose;
+#if JVET_AC0353_NNPFC_BASE_FLAG
+ bool m_baseFlag;
+#endif
bool m_outSubCFlag;
uint8_t m_outSubWidthC;
uint8_t m_outSubHeightC;
diff --git a/source/Lib/CommonLib/TypeDef.h b/source/Lib/CommonLib/TypeDef.h
index e2c71ddaf..0bee174db 100644
--- a/source/Lib/CommonLib/TypeDef.h
+++ b/source/Lib/CommonLib/TypeDef.h
@@ -65,7 +65,7 @@
#define JVET_AC0062_CONSTRAINT_CHECK 1
#define JVET_AC0344_NNPFC_PATCH 1
#define JVET_AC0074_USE_OF_NNPFC_FOR_PIC_RATE_UPSAMPLING 1
-
+#define JVET_AC0353_NNPFC_BASE_FLAG 1
//########### place macros to be be kept below this line ###############
diff --git a/source/Lib/DecoderLib/SEIread.cpp b/source/Lib/DecoderLib/SEIread.cpp
index 54ede8ae6..d9aed5b27 100644
--- a/source/Lib/DecoderLib/SEIread.cpp
+++ b/source/Lib/DecoderLib/SEIread.cpp
@@ -219,6 +219,38 @@ void SEIReader::getSEIDecodingUnitInfoDuiIdx(InputBitstream* bs, const NalUnitTy
}
}
+#if JVET_AC0353_NNPFC_BASE_FLAG
+bool SEIReader::xCheckNnpfcSeiMsg(uint32_t seiId, bool baseFlag, const std::vector<int> nnpfcValueList)
+{
+ if (baseFlag)
+ {
+ //Check if this is a new filter or a repetition of an existing base flag
+ for (auto val : nnpfcValueList)
+ {
+ if (val == seiId)
+ {
+ //The filter is a repetition.
+ return false;
+ }
+ }
+ }
+ else
+ {
+ bool filterHasPresent = false;
+ for(auto val : nnpfcValueList)
+ {
+ if (val == seiId)
+ {
+ filterHasPresent = true;
+ break;
+ }
+ }
+ CHECK(!filterHasPresent, "Cannot have update filter without base filter already present!")
+ }
+ return true;
+}
+#endif
+
bool SEIReader::xReadSEImessage(SEIMessages& seis, const NalUnitType nalUnitType, const uint32_t nuh_layer_id, const uint32_t temporalId, const VPS *vps, const SPS *sps, HRD &hrd, std::ostream *pDecodedMessageOutputStream)
{
#if ENABLE_TRACING
@@ -444,8 +476,14 @@ bool SEIReader::xReadSEImessage(SEIMessages& seis, const NalUnitType nalUnitType
xParseSEINNPostFilterCharacteristics((SEINeuralNetworkPostFilterCharacteristics &) *sei, payloadSize, sps,
pDecodedMessageOutputStream);
-
+#if JVET_AC0353_NNPFC_BASE_FLAG
+ if (xCheckNnpfcSeiMsg( ((SEINeuralNetworkPostFilterCharacteristics*)sei)->m_id, ((SEINeuralNetworkPostFilterCharacteristics*)sei)->m_baseFlag, nnpfcValues) )
+ {
+ nnpfcValues.push_back(((SEINeuralNetworkPostFilterCharacteristics*)sei)->m_id);
+ }
+#else
nnpfcValues.push_back(((SEINeuralNetworkPostFilterCharacteristics*)sei)->m_id);
+#endif
break;
case SEI::PayloadType::NEURAL_NETWORK_POST_FILTER_ACTIVATION:
sei = new SEINeuralNetworkPostFilterActivation;
@@ -2722,6 +2760,10 @@ void SEIReader::xParseSEINNPostFilterCharacteristics(SEINeuralNetworkPostFilterC
if (sei.m_propertyPresentFlag)
{
+#if JVET_AC0353_NNPFC_BASE_FLAG
+ sei_read_flag(pDecodedMessageOutputStream, val, "nnpfc_base_flag");
+ sei.m_baseFlag = val;
+#endif
#if !JVET_AC0127_BIT_MASKING_NNPFC_PURPOSE
sei_read_uvlc(pDecodedMessageOutputStream, val, "nnpfc_purpose");
sei.m_purpose = val;
diff --git a/source/Lib/DecoderLib/SEIread.h b/source/Lib/DecoderLib/SEIread.h
index 3892468a7..07531f7e0 100644
--- a/source/Lib/DecoderLib/SEIread.h
+++ b/source/Lib/DecoderLib/SEIread.h
@@ -63,6 +63,9 @@ public:
std::vector<int> nnpfcValues;
protected:
+#if JVET_AC0353_NNPFC_BASE_FLAG
+ bool xCheckNnpfcSeiMsg (uint32_t seiId, bool baseFlag, const std::vector<int> nnpfcValueList);
+#endif
bool xReadSEImessage (SEIMessages& seis, const NalUnitType nalUnitType, const uint32_t nuh_layer_id, const uint32_t temporalId, const VPS *vps, const SPS *sps, HRD &hrd, std::ostream *pDecodedMessageOutputStream);
void xParseSEIFillerPayload (SEIFillerPayload &sei, uint32_t payloadSize, std::ostream *pDecodedMessageOutputStream);
void xParseSEIuserDataUnregistered (SEIuserDataUnregistered &sei, uint32_t payloadSize, std::ostream *pDecodedMessageOutputStream);
diff --git a/source/Lib/EncoderLib/EncCfg.h b/source/Lib/EncoderLib/EncCfg.h
index 68af43ab0..dfe3c544e 100644
--- a/source/Lib/EncoderLib/EncCfg.h
+++ b/source/Lib/EncoderLib/EncCfg.h
@@ -689,6 +689,9 @@ protected:
uint32_t m_nnPostFilterSEICharacteristicsId[MAX_NUM_NN_POST_FILTERS];
uint32_t m_nnPostFilterSEICharacteristicsModeIdc[MAX_NUM_NN_POST_FILTERS];
bool m_nnPostFilterSEICharacteristicsPropertyPresentFlag[MAX_NUM_NN_POST_FILTERS];
+#if JVET_AC0353_NNPFC_BASE_FLAG
+ bool m_nnPostFilterSEICharacteristicsBaseFlag[MAX_NUM_NN_POST_FILTERS];
+#endif
uint32_t m_nnPostFilterSEICharacteristicsPurpose[MAX_NUM_NN_POST_FILTERS];
bool m_nnPostFilterSEICharacteristicsOutSubCFlag[MAX_NUM_NN_POST_FILTERS];
#if JVET_AC0154
@@ -1913,6 +1916,10 @@ public:
uint32_t getNNPostFilterSEICharacteristicsModeIdc(int filterIdx) const { return m_nnPostFilterSEICharacteristicsModeIdc[filterIdx]; }
void setNNPostFilterSEICharacteristicsPropertyPresentFlag(bool propertyPresentFlag, int filterIdx) { m_nnPostFilterSEICharacteristicsPropertyPresentFlag[filterIdx] = propertyPresentFlag; }
bool getNNPostFilterSEICharacteristicsPropertyPresentFlag(int filterIdx) const { return m_nnPostFilterSEICharacteristicsPropertyPresentFlag[filterIdx]; }
+#if JVET_AC0353_NNPFC_BASE_FLAG
+ void setNNPostFilterSEICharacteristicsBaseFlag(bool baseFlag, int filterIdx) { m_nnPostFilterSEICharacteristicsBaseFlag[filterIdx] = baseFlag; }
+ bool getNNPostFilterSEICharacteristicsBaseFlag(int filterIdx) const { return m_nnPostFilterSEICharacteristicsBaseFlag[filterIdx]; }
+#endif
void setNNPostFilterSEICharacteristicsPurpose(uint32_t purpose, int filterIdx) { m_nnPostFilterSEICharacteristicsPurpose[filterIdx] = purpose; }
uint32_t getNNPostFilterSEICharacteristicsPurpose(int filterIdx) const { return m_nnPostFilterSEICharacteristicsPurpose[filterIdx]; }
diff --git a/source/Lib/EncoderLib/SEIEncoder.cpp b/source/Lib/EncoderLib/SEIEncoder.cpp
index 28c1042b6..0fbd9ef7f 100644
--- a/source/Lib/EncoderLib/SEIEncoder.cpp
+++ b/source/Lib/EncoderLib/SEIEncoder.cpp
@@ -1325,6 +1325,9 @@ void SEIEncoder::initSEINeuralNetworkPostFilterCharacteristics(SEINeuralNetworkP
sei->m_propertyPresentFlag = m_pcCfg->getNNPostFilterSEICharacteristicsPropertyPresentFlag(filterIdx);
if (sei->m_propertyPresentFlag)
{
+#if JVET_AC0353_NNPFC_BASE_FLAG
+ sei->m_baseFlag = m_pcCfg->getNNPostFilterSEICharacteristicsBaseFlag(filterIdx);
+#endif
#if !JVET_AC0127_BIT_MASKING_NNPFC_PURPOSE
sei->m_purpose = m_pcCfg->getNNPostFilterSEICharacteristicsPurpose(filterIdx);
#endif
diff --git a/source/Lib/EncoderLib/SEIwrite.cpp b/source/Lib/EncoderLib/SEIwrite.cpp
index c65991d51..64ec77847 100644
--- a/source/Lib/EncoderLib/SEIwrite.cpp
+++ b/source/Lib/EncoderLib/SEIwrite.cpp
@@ -1715,6 +1715,9 @@ void SEIWriter::xWriteSEINeuralNetworkPostFilterCharacteristics(const SEINeuralN
xWriteFlag(sei.m_propertyPresentFlag, "nnpfc_property_present_flag");
if (sei.m_propertyPresentFlag)
{
+#if JVET_AC0353_NNPFC_BASE_FLAG
+ xWriteFlag(sei.m_baseFlag, "nnpfc_base_flag");
+#endif
#if !JVET_AC0127_BIT_MASKING_NNPFC_PURPOSE
xWriteUvlc(sei.m_purpose, "nnpfc_purpose");
#endif
--
GitLab