diff --git a/.gitattributes b/.gitattributes
index 6cf967cf006dee546047ab924bfb494a29d1e8d1..acf3ec92dd9bf76897b6449376a2c0530f28df04 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -10,4 +10,3 @@
*.index filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.data-* filter=lfs diff=lfs merge=lfs -text
-
diff --git a/models/nnlf_hop_temporal_model_float.sadl b/models/nnlf_hop_temporal_model_float.sadl
new file mode 100644
index 0000000000000000000000000000000000000000..1388fd9a35fa598fb8136b3cb2fdf21120c8bd9d
--- /dev/null
+++ b/models/nnlf_hop_temporal_model_float.sadl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b70e1d808e2eeb01760691c0f1885e80191cef13a3363b062bb1f86939a009a8
+size 6166219
diff --git a/models/nnlf_hop_temporal_model_int16.sadl b/models/nnlf_hop_temporal_model_int16.sadl
new file mode 100644
index 0000000000000000000000000000000000000000..914f483f0d50072cbc003f2bb28be54db954e2af
--- /dev/null
+++ b/models/nnlf_hop_temporal_model_int16.sadl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba3b4e74166936eba83cf57e566a004c34ff555fca42e918d4a972d5b09daac6
+size 3086941
diff --git a/source/App/DecoderApp/DecApp.cpp b/source/App/DecoderApp/DecApp.cpp
index d8667b8969a43fd55bb8e06c32ee67301f3a7628..7a904ade7c279c32955b2c737e3180e301356e0b 100644
--- a/source/App/DecoderApp/DecApp.cpp
+++ b/source/App/DecoderApp/DecApp.cpp
@@ -651,6 +651,9 @@ void DecApp::xCreateDecLib()
#if NN_HOP_UNIFIED_FORCE_USE
m_cDecLib.setNnlfHopOption(m_nnlfHopOption);
#endif
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ m_cDecLib.setNnlfHopTemporalModelName(m_nnlfHopTemporalModelName);
+#endif
#endif
#if NN_FILTERING_SET_0
m_cDecLib.setModelPath(m_ModelPath);
diff --git a/source/App/DecoderApp/DecAppCfg.cpp b/source/App/DecoderApp/DecAppCfg.cpp
index 7910efd5810c63c7aa0c365d3ddd9e4a90040659..1ac419fbc078a653aecfb24094b8b932c94b5f02 100644
--- a/source/App/DecoderApp/DecAppCfg.cpp
+++ b/source/App/DecoderApp/DecAppCfg.cpp
@@ -85,6 +85,9 @@ bool DecAppCfg::parseCfg( int argc, char* argv[] )
#if NN_HOP_UNIFIED_FORCE_USE
( "NnlfHopDebugOption", m_nnlfHopOption, 0, "Option used to debug stage 1 model. 0: default, 1: apply only on I slice, 2: apply on all slices using I type as input" )
#endif
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ ("NnlfHopTemporalModelName", m_nnlfHopTemporalModelName, string("models/nnlf_hop_temporal_model_int16.sadl"), "HOP temporal loop filter model name\n")
+#endif
#endif
diff --git a/source/App/DecoderApp/DecAppCfg.h b/source/App/DecoderApp/DecAppCfg.h
index 914b61207f0fd696f2137f5d505919fbaa8ddd5d..72f8280ab0440473606ee40e1f06318c8a1b8ed4 100644
--- a/source/App/DecoderApp/DecAppCfg.h
+++ b/source/App/DecoderApp/DecAppCfg.h
@@ -63,6 +63,9 @@ protected:
#if NN_HOP_UNIFIED_FORCE_USE
int m_nnlfHopOption;
#endif
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ std::string m_nnlfHopTemporalModelName; ///<nnlf temporal hop model
+#endif
#endif
#if NN_FILTERING_SET_0
std::string m_ModelPath; ///< model path
diff --git a/source/App/EncoderApp/EncApp.cpp b/source/App/EncoderApp/EncApp.cpp
index 6b488f8b7eddc24692eecef6b65d2574ef937e5c..fbcf2b4e2b4b067381da8e565cf97a2c0b34c75a 100644
--- a/source/App/EncoderApp/EncApp.cpp
+++ b/source/App/EncoderApp/EncApp.cpp
@@ -1123,6 +1123,10 @@ void EncApp::xInitLibCfg()
#if NN_HOP_UNIFIED_FORCE_USE
m_cEncLib.setNnlfHopOption (m_nnlfHopOption);
#endif
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ m_cEncLib.setUseNnlfHopTemporalFiltering (m_nnlfHopTemporalFiltering);
+ m_cEncLib.setNnlfHopTemporalModelName (m_nnlfHopTemporalModelName);
+#endif
#endif
#if NN_FILTERING_SET_0
diff --git a/source/App/EncoderApp/EncAppCfg.cpp b/source/App/EncoderApp/EncAppCfg.cpp
index 97d6c13c0a85fa918105dbf95548c7b2f9f3ce25..bdc72c7d2774108f59c72b79395580001ce65d64 100644
--- a/source/App/EncoderApp/EncAppCfg.cpp
+++ b/source/App/EncoderApp/EncAppCfg.cpp
@@ -1163,13 +1163,17 @@ bool EncAppCfg::parseCfg( int argc, char* argv[] )
#endif
#if NN_HOP_UNIFIED
( "NnlfHop", m_nnlfHop, true, "High operation point (HOP) of NN-based loop filter" )
- ( "NnlfHOPBlockSize", m_nnlfHopBlockSize, 128u, "Base inference size of NN-based in-loop filter for HOP")
+ ( "NnlfHopBlockSize", m_nnlfHopBlockSize, 128u, "Base inference size of NN-based in-loop filter for HOP")
( "NnlfHopInfSizeExt", m_nnlfHopInfSizeExt, 8u, "Extension of inference size of HOP loop filter" )
( "NnlfHopMaxNumPrms", m_nnlfHopMaxNumPrms, 2u, "Number of conditional parameters of HOP loop filter" )
( "NnlfHopModelName", m_nnlfHopModelName, string("models/nnlf_hop_model_int16.sadl"), "HOP loop filter model name" )
#if NN_HOP_UNIFIED_FORCE_USE
( "NnlfHopDebugOption", m_nnlfHopOption, 0, "Option used to debug stage 1 model. 0: default, 1: apply only on I slice, 2: apply on all slices using I type as input" )
#endif
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ ( "NnlfHopTemporalFiltering", m_nnlfHopTemporalFiltering, false, "Use HOP temporal filtering" )
+ ( "NnlfHopTemporalModelName", m_nnlfHopTemporalModelName, string("models/nnlf_hop_temporal_model_int16.sadl"), "HOP temporal loop filter model name")
+#endif
#endif
("SAO", m_bUseSAO, true, "Enable Sample Adaptive Offset")
("TestSAODisableAtPictureLevel", m_bTestSAODisableAtPictureLevel, false, "Enables the testing of disabling SAO at the picture level after having analysed all blocks")
@@ -3172,7 +3176,13 @@ bool EncAppCfg::xCheckParameter()
{
xConfirmPara( m_ccalf, "CCALF cannot be enabled when ALF is disabled" );
}
-
+
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ if (!m_nnlfHop)
+ {
+ xConfirmPara( m_nnlfHopTemporalFiltering, "Temporal HOP cannot be enabled when HOP is disabled" );
+ }
+#endif
xConfirmPara( m_iSourceWidth % SPS::getWinUnitX(m_chromaFormatIDC) != 0, "Picture width must be an integer multiple of the specified chroma subsampling");
xConfirmPara( m_iSourceHeight % SPS::getWinUnitY(m_chromaFormatIDC) != 0, "Picture height must be an integer multiple of the specified chroma subsampling");
@@ -4270,7 +4280,12 @@ void EncAppCfg::xPrintParameter()
#endif
#if NN_HOP_UNIFIED
msg(VERBOSE, "NnlfHop:%d ", m_nnlfHop ? 1 : 0);
- msg(VERBOSE, "NnlfHopOption:%d ", m_nnlfHopOption);
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ msg(VERBOSE, "NnlfHopTemporalFiltering:%d ", m_nnlfHopTemporalFiltering ? 1 : 0);
+#endif
+#if NN_HOP_UNIFIED_FORCE_USE
+ msg(VERBOSE, "NnlfHopDebugOption:%d ", m_nnlfHopOption);
+#endif
#endif
#if JVET_AB0068_AC0328_NNLF_RDO
msg( VERBOSE, "EncNnlfOpt:%d ", m_encNnlfOpt ? 1 : 0);
diff --git a/source/App/EncoderApp/EncAppCfg.h b/source/App/EncoderApp/EncAppCfg.h
index 720a74d64aca164248239af9b6606e3ef886daf9..1b60d4dda60670ff4760a472fb2becbf59f285b2 100644
--- a/source/App/EncoderApp/EncAppCfg.h
+++ b/source/App/EncoderApp/EncAppCfg.h
@@ -763,6 +763,10 @@ protected:
#if NN_HOP_UNIFIED_FORCE_USE
int m_nnlfHopOption;
#endif
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ bool m_nnlfHopTemporalFiltering;
+ std::string m_nnlfHopTemporalModelName; ///<nnlf temporal hop model
+#endif
#endif
#if NN_FILTERING_SET_0
diff --git a/source/Lib/CommonLib/NNFilterHOP.cpp b/source/Lib/CommonLib/NNFilterHOP.cpp
index abe37e7231ffc6888a5e222e9a54ba299f8a2383..fc2f9ecd280c170ed56ad997f0654ad7d355c430 100644
--- a/source/Lib/CommonLib/NNFilterHOP.cpp
+++ b/source/Lib/CommonLib/NNFilterHOP.cpp
@@ -55,7 +55,20 @@ struct Input
nbInputs
};
};
-
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+struct InputTemporal
+{
+ enum
+ {
+ Rec = 0,
+ Pred,
+ List0,
+ List1,
+ QPbase,
+ nbInputsTemporal
+ };
+};
+#endif
void NNFilterHOP::init(const std::string &filename, int picWidth, int picHeight, ChromaFormat format, int prmNum)
{
ifstream file(filename, ios::binary);
@@ -68,6 +81,7 @@ void NNFilterHOP::init(const std::string &filename, int picWidth, int picHeight,
exit(-1);
}
}
+
// prepare inputs
m_inputs.resize(Input::nbInputs);
resizeInputs(defaultInputSize + defaultBlockExt * 2, defaultInputSize + defaultBlockExt * 2);
@@ -91,7 +105,24 @@ void NNFilterHOP::init(const std::string &filename, int picWidth, int picHeight,
}
}
}
-
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+void NNFilterHOP::initTemporal(const std::string &filename)
+{
+ ifstream file(filename, ios::binary);
+ if (!m_modelTemporal)
+ {
+ m_modelTemporal.reset(new sadl::Model<TypeSadlHOP>());
+ if (!m_modelTemporal->load(file))
+ {
+ cerr << "[ERROR] issue loading temporal model NNFilterHOP " << filename << endl;
+ exit(-1);
+ }
+ }
+ // prepare inputs
+ m_inputsTemporal.resize(InputTemporal::nbInputsTemporal);
+ resizeInputsTemporal(defaultInputSize + defaultBlockExt * 2, defaultInputSize + defaultBlockExt * 2);
+}
+#endif
void NNFilterHOP::destroy()
{
for (int i = 0; i < (int)m_filtered.size(); i++)
@@ -132,8 +163,36 @@ void NNFilterHOP::resizeInputs(int width, int height)
exit(-1);
}
m_input_quantizer = m_model->getInputsTemplate()[0].quantizer; // assume all image inputs have same quantizer
+
}
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+// default is square block + extension
+void NNFilterHOP::resizeInputsTemporal(int width, int height) // assume using same quantizer as the primary model
+{
+ int sizeW = width;
+ int sizeH = height;
+ if (sizeH == m_blocksizeTemporal[0] && sizeW == m_blocksizeTemporal[1])
+ {
+ return;
+ }
+ m_blocksizeTemporal[0] = sizeH;
+ m_blocksizeTemporal[1] = sizeW;
+ // note: later QP inputs can be optimized to avoid some duplicate computation with a 1x1 input
+ m_inputsTemporal[InputTemporal::Rec].resize(sadl::Dimensions({ 1, sizeH, sizeW, 1 }));
+ m_inputsTemporal[InputTemporal::Pred].resize(sadl::Dimensions({ 1, sizeH, sizeW, 1 }));
+ m_inputsTemporal[InputTemporal::List0].resize(sadl::Dimensions({ 1, sizeH, sizeW, 1 }));
+ m_inputsTemporal[InputTemporal::List1].resize(sadl::Dimensions({ 1, sizeH, sizeW, 1 }));
+ m_inputsTemporal[InputTemporal::QPbase].resize(sadl::Dimensions({ 1, sizeH, sizeW, 1 }));
+
+ if (!m_modelTemporal->init(m_inputsTemporal))
+ {
+ cerr << "[ERROR] issue init temporal model NNFilterHOP " << endl;
+ exit(-1);
+ }
+ m_input_quantizer_temporal = m_modelTemporal->getInputsTemplate()[0].quantizer; // assume all image inputs have same quantizer
+}
+#endif
void roundToOutputBitdepth(const PelUnitBuf &src, PelUnitBuf &dst, const ClpRngs &clpRngs)
{
for (int c = 0; c < MAX_NUM_COMPONENT; c++)
@@ -233,7 +292,76 @@ static void extractOutputs(const Picture &pic, sadl::Model<T> &m, PelUnitBuf &bu
}
}
}
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+// bufDst: temporary buffer to store results
+// inferArea: area used for inference (include extension)
+template<typename T>
+static void extractOutputsTemporal(const Picture &pic, sadl::Model<T> &m, PelUnitBuf &bufDst, UnitArea inferArea, int extLeft,
+ int extRight, int extTop, int extBottom)
+{
+ const int log2InputBitdepth = pic.cs->slice->clpRng(COMPONENT_Y).bd; // internal bitdepth
+ auto output = m.result(0);
+ const int q_output_sadl = output.quantizer;
+ const int shiftInput = NNFilterHOP::log2OutputScale - log2InputBitdepth;
+ const int shiftOutput = NNFilterHOP::log2OutputScale - q_output_sadl;
+ assert(shiftInput >= 0);
+ assert(shiftOutput >= 0);
+
+ const int width = bufDst.Y().width;
+ const int height = bufDst.Y().height;
+ PelBuf & bufDstY = bufDst.get(COMPONENT_Y);
+ CPelBuf bufRecY = pic.getRecBeforeDbfBuf(inferArea).get(COMPONENT_Y);
+
+ for (int c = 0; c < 4; ++c) // unshuffle on C++ side
+ {
+ for (int y = 0; y < height / 2; ++y)
+ {
+ for (int x = 0; x < width / 2; ++x)
+ {
+ int yy = (y * 2) + c / 2;
+ int xx = (x * 2) + c % 2;
+ if (xx < extLeft || yy < extTop || xx >= width - extRight || yy >= height - extBottom)
+ {
+ continue;
+ }
+ int out;
+ if constexpr (std::is_same<TypeSadlHOP, float>::value)
+ {
+ out = round((output(0, y, x, c) * (1 << shiftOutput) + (float) bufRecY.at(xx, yy) * (1 << shiftInput)));
+ }
+ else
+ {
+ out = ((output(0, y, x, c) << shiftOutput) + (bufRecY.at(xx, yy) << shiftInput));
+ }
+
+ bufDstY.at(xx, yy) = Pel(Clip3<int>(0, (1 << NNFilterHOP::log2OutputScale) - 1, out));
+ }
+ }
+ }
+
+ PelBuf &bufDstCb = bufDst.get(COMPONENT_Cb);
+ PelBuf &bufDstCr = bufDst.get(COMPONENT_Cr);
+ CPelBuf bufRecCb = pic.getRecoBuf(inferArea).get(COMPONENT_Cb);
+ CPelBuf bufRecCr = pic.getRecoBuf(inferArea).get(COMPONENT_Cr);
+
+ for (int y = 0; y < height / 2; ++y)
+ {
+ for (int x = 0; x < width / 2; ++x)
+ {
+ if (x < extLeft / 2 || y < extTop / 2 || x >= width / 2 - extRight / 2 || y >= height / 2 - extBottom / 2)
+ {
+ continue;
+ }
+ int outCb = (bufRecCb.at(x, y) << shiftInput);
+ int outCr = (bufRecCr.at(x, y) << shiftInput);
+
+ bufDstCb.at(x, y) = Pel(Clip3<int>(0, (1 << NNFilterHOP::log2OutputScale) - 1, outCb));
+ bufDstCr.at(x, y) = Pel(Clip3<int>(0, (1 << NNFilterHOP::log2OutputScale) - 1, outCr));
+ }
+ }
+}
+#endif
void NNFilterHOP::filterBlock(Picture &pic, UnitArea inferArea, int extLeft, int extRight, int extTop, int extBottom,
int prmId)
{
@@ -271,7 +399,37 @@ void NNFilterHOP::filterBlock(Picture &pic, UnitArea inferArea, int extLeft, int
extractOutputs(pic, model, bufDst, inferArea, extLeft, extRight, extTop, extBottom);
}
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+void NNFilterHOP::filterBlockTemporal(Picture &pic, UnitArea inferArea, int extLeft, int extRight, int extTop, int extBottom,
+ int prmId)
+{
+ // get model
+ auto &model = *m_modelTemporal;
+ bool inter = pic.slices[0]->getSliceType() != I_SLICE ? true : false;
+ int qpOffset = (inter ? prmId * 5 : prmId * 2) * (pic.slices[0]->getTLayer() >= 4 ? 1 : -1);
+ int seqQp = pic.slices[0]->getPPS()->getPicInitQPMinus26() + 26 + qpOffset;
+ resizeInputsTemporal(inferArea.Y().width, inferArea.Y().height);
+ const int log2InputBitdepth = pic.cs->slice->clpRng(COMPONENT_Y).bd; // internal bitdepth
+ const double inputScalePred = (1 << log2InputBitdepth);
+ const double inputScaleQp = (1 << log2InputQpScale);
+
+ std::vector<InputData> listInputData;
+ listInputData.push_back({ NN_INPUT_REC, 0, inputScalePred, m_input_quantizer_temporal - log2InputBitdepth, true, false });
+ listInputData.push_back({ NN_INPUT_PRED, 1, inputScalePred, m_input_quantizer_temporal - log2InputBitdepth, true, false });
+ listInputData.push_back({ NN_INPUT_REF_LIST_0, 2, inputScalePred, m_input_quantizer_temporal - log2InputBitdepth, true, false });
+ listInputData.push_back({ NN_INPUT_REF_LIST_1, 3, inputScalePred, m_input_quantizer_temporal - log2InputBitdepth, true, false });
+ listInputData.push_back({ NN_INPUT_GLOBAL_QP, 4, inputScaleQp, m_input_quantizer_temporal - log2InputQpScale, true, false });
+
+ NNInference::prepareInputs<TypeSadlHOP>(&pic, inferArea, m_inputsTemporal, seqQp, -1 /* localQp */, -1 /* sliceType */, listInputData);
+
+ NNInference::infer<TypeSadlHOP>(model, m_inputsTemporal);
+
+ PelUnitBuf bufDst = m_scaled[0][prmId].getBuf(inferArea);
+
+ extractOutputsTemporal(pic, model, bufDst, inferArea, extLeft, extRight, extTop, extBottom);
+}
+#endif
void NNFilterHOP::filter(Picture &pic, const bool isDec)
{
const CodingStructure &cs = *pic.cs;
@@ -307,8 +465,12 @@ void NNFilterHOP::filter(Picture &pic, const bool isDec)
int extWidth = width + extLeft + extRight;
int extHeight = height + extTop + extBottom;
const UnitArea inferArea(cs.area.chromaFormat, Area(extXPos, extYPos, extWidth, extHeight));
-
- filterBlock(pic, inferArea, extLeft, extRight, extTop, extBottom, prmId);
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ if (m_picprm->temporal)
+ filterBlockTemporal(pic, inferArea, extLeft, extRight, extTop, extBottom, prmId);
+ else
+#endif
+ filterBlock(pic, inferArea, extLeft, extRight, extTop, extBottom, prmId);
const UnitArea inferAreaNoExt(cs.area.chromaFormat, Area(xPos, yPos, width, height));
PelUnitBuf filteredBuf = getFilteredBuf(prmId, inferAreaNoExt);
diff --git a/source/Lib/CommonLib/NNFilterHOP.h b/source/Lib/CommonLib/NNFilterHOP.h
index d5e474838f69525cc809f9317891105800b7efb7..ea48a0df1e48e646cec08e6ae3cbfe5d048c2ea5 100644
--- a/source/Lib/CommonLib/NNFilterHOP.h
+++ b/source/Lib/CommonLib/NNFilterHOP.h
@@ -54,6 +54,9 @@ public:
static constexpr float nnResidueScaleDerivationUpBound = 1.25f;
static constexpr float nnResidueScaleDerivationLowBound = 0.0625f;
static constexpr int max_scale=(1<<log2ResidueScale);
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ static constexpr int minimumTidUseTemporalFiltering = 3;
+#endif
// parameters signaled at slice level
struct SliceParameters {
@@ -83,6 +86,7 @@ public:
int prmNum;
std::vector<int> prmId; // -1: off
SliceParameters sprm;
+ bool temporal;
};
static constexpr int scale_candidates[4] = { 0, max_scale, (max_scale + (max_scale << 1)) >> 2, max_scale >> 1 };
@@ -97,6 +101,11 @@ public:
// just filter the block, output on log2OutputScale bits
void filterBlock(Picture &pic, UnitArea inferArea, int extLeft, int extRight, int extTop, int extBottom,
int prmId);
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ void initTemporal(const std::string &filename);
+ void filterBlockTemporal(Picture &pic, UnitArea inferArea, int extLeft, int extRight, int extTop, int extBottom,
+ int prmId);
+#endif
// put scaled res+rec in tgt from the filtered src
void scaleResidualBlock(Picture &pic, ComponentID compID, UnitArea inferArea, CPelBuf src, PelBuf tgt,
@@ -125,6 +134,13 @@ private:
void resizeInputs(int width, int height);
std::unique_ptr<sadl::Model<TypeSadlHOP>> m_model;
std::vector<sadl::Tensor<TypeSadlHOP>> m_inputs;
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ int m_blocksizeTemporal[2]; // current inputs size of temporal model
+ int m_input_quantizer_temporal = 0;
+ void resizeInputsTemporal(int width, int height);
+ std::unique_ptr<sadl::Model<TypeSadlHOP>> m_modelTemporal;
+ std::vector<sadl::Tensor<TypeSadlHOP>> m_inputsTemporal;
+#endif
std::vector<PelStorage> m_filtered; // filtered results of each parameter
std::vector<PelStorage> m_scaled[4]; // residue scaling results
FilterParameters *m_picprm; // filtering parameters
diff --git a/source/Lib/CommonLib/NNInference.h b/source/Lib/CommonLib/NNInference.h
index c8e4448167d5bf45cdce4c5f4f90e3e5ac5c5bd2..9b475a42387c53b4ba024f1db30b53cb887b3e45 100644
--- a/source/Lib/CommonLib/NNInference.h
+++ b/source/Lib/CommonLib/NNInference.h
@@ -452,14 +452,12 @@ public:
case NN_INPUT_SLICE_TYPE:
fillInputFromConstant<T>(pic, inferArea, inputs[inputData.index], sliceType, inputData.luma, inputData.scale, inputData.shift);
break;
-#if JVET_AC0177_MULTI_FRAME
case NN_INPUT_REF_LIST_0:
fillInputFromBuf<T>(pic, inferArea, inputs[inputData.index], pic->slices[0]->getRefPic(REF_PIC_LIST_0, 0)->getRecoBuf(inferArea), inputData.luma, inputData.chroma, inputData.scale, inputData.shift);
break;
case NN_INPUT_REF_LIST_1:
fillInputFromBuf<T>(pic, inferArea, inputs[inputData.index], pic->slices[0]->getRefPic(REF_PIC_LIST_1, 0)->getRecoBuf(inferArea), inputData.luma, inputData.chroma, inputData.scale, inputData.shift);
break;
-#endif
#if JVET_AC0089_COMBINE_INTRA_INTER
case NN_INPUT_IPB:
fillInputFromBufIpb<T>(pic, inferArea, inputs[inputData.index], pic->getBlockPredModeBuf(inferArea), inputData.luma, inputData.chroma, inputData.scale, inputData.shift);
@@ -507,14 +505,12 @@ public:
case NN_INPUT_SLICE_TYPE:
fillInputFromConstant<T>(pic, inferArea, inputs[inputData.index], sliceType, inputData.luma, inputData.scale, inputData.shift);
break;
-#if JVET_AC0177_MULTI_FRAME
case NN_INPUT_REF_LIST_0:
fillInputFromBuf<T>(pic, inferArea, inputs[inputData.index], pic->slices[0]->getRefPic(REF_PIC_LIST_0, 0)->getRecoBuf(inferArea), inputData.luma, inputData.chroma, inputData.scale, inputData.shift, flip);
break;
case NN_INPUT_REF_LIST_1:
fillInputFromBuf<T>(pic, inferArea, inputs[inputData.index], pic->slices[0]->getRefPic(REF_PIC_LIST_1, 0)->getRecoBuf(inferArea), inputData.luma, inputData.chroma, inputData.scale, inputData.shift, flip);
break;
-#endif
#if JVET_AC0089_COMBINE_INTRA_INTER
case NN_INPUT_IPB:
fillInputFromBufIpb<T>(pic, inferArea, inputs[inputData.index], pic->getBlockPredModeBuf(inferArea), inputData.luma, inputData.chroma, inputData.scale, inputData.shift, flip);
diff --git a/source/Lib/CommonLib/Picture.h b/source/Lib/CommonLib/Picture.h
index 24dd030fee9fad59fc17faa05bc3a2ba66a0e800..978d9776e3d5062ee57c60d9f25564234062df99 100644
--- a/source/Lib/CommonLib/Picture.h
+++ b/source/Lib/CommonLib/Picture.h
@@ -418,6 +418,9 @@ public:
m_picprm.nb_blocks_width = (sps.getMaxPicWidthInLumaSamples() + m_picprm.block_size - 1) / m_picprm.block_size;
m_picprm.prmId.resize(m_picprm.nb_blocks_height * m_picprm.nb_blocks_width);
fill(m_picprm.prmId.begin(), m_picprm.prmId.end(), -1);
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ m_picprm.temporal = sps.getNnlfHopTemporalFilteringEnabledFlag() && slice.getSliceType() != I_SLICE && slice.getTLayer() >= NNFilterHOP::minimumTidUseTemporalFiltering;
+#endif
}
#endif
diff --git a/source/Lib/CommonLib/Slice.h b/source/Lib/CommonLib/Slice.h
index 841efabccec114eb3b687524ba07e13cc357a4cb..f287e0c5521f9182a95197fff0fd9ef59bb8b950 100644
--- a/source/Lib/CommonLib/Slice.h
+++ b/source/Lib/CommonLib/Slice.h
@@ -1490,6 +1490,9 @@ private:
uint32_t m_nnlfHopInferSize[MAX_NUM_NNLF_HOP_INFER_GRANULARITY];
uint32_t m_nnlfHopInfSizeExt;
uint32_t m_nnlfHopMaxNumPrms;
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ bool m_nnlfHopTemporalFilteringEnabledFlag;
+#endif
#endif
#if NN_FILTERING_SET_0
@@ -1783,6 +1786,10 @@ public:
void setNnlfHopInfSizeExt( uint32_t i ) { m_nnlfHopInfSizeExt = i; }
uint32_t getNnlfHopMaxNumPrms() const { return m_nnlfHopMaxNumPrms; }
void setNnlfHopMaxNumPrms( uint32_t i ) { m_nnlfHopMaxNumPrms = i; }
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ bool getNnlfHopTemporalFilteringEnabledFlag() const { return m_nnlfHopTemporalFilteringEnabledFlag; }
+ void setNnlfHopTemporalFilteringEnabledFlag( bool b ) { m_nnlfHopTemporalFilteringEnabledFlag = b; }
+#endif
#endif
#if NN_FILTERING_SET_0
bool getNnlfSet0EnabledFlag() const { return m_nnlfSet0EnabledFlag; }
diff --git a/source/Lib/CommonLib/TypeDef.h b/source/Lib/CommonLib/TypeDef.h
index 0328cfb2073b1498fb2c77fb5321e2cea9f09ee1..c68051e8826e22ed368c3ea17b656ef65df9c171 100644
--- a/source/Lib/CommonLib/TypeDef.h
+++ b/source/Lib/CommonLib/TypeDef.h
@@ -54,7 +54,8 @@
#define NN_HOP_UNIFIED 1
#if NN_HOP_UNIFIED
using TypeSadlHOP=float;
-#define NN_HOP_UNIFIED_FORCE_USE 1 // for debug of stage 1
+#define NN_HOP_UNIFIED_FORCE_USE 1 // for debug of stage 1
+#define NN_HOP_UNIFIED_TEMPORAL_FILTERING 1 // hop temporal filtering
#endif
@@ -564,11 +565,9 @@ enum NNInputType
NN_INPUT_SLICE_TYPE = 6,
#if JVET_AC0089_NNVC_USE_BPM_INFO
NN_INPUT_IPB = 7,
-#if JVET_AC0177_MULTI_FRAME
- NN_INPUT_REF_LIST_0 = 8,
- NN_INPUT_REF_LIST_1 = 9,
-#endif
#endif
+ NN_INPUT_REF_LIST_0,
+ NN_INPUT_REF_LIST_1,
#if NN_HOP_UNIFIED_FORCE_USE
NN_INPUT_ZERO,
#endif
diff --git a/source/Lib/DecoderLib/DecLib.cpp b/source/Lib/DecoderLib/DecLib.cpp
index 70c6b560d4c750a59c80653333be62677aa81cf4..3dafe96ec819f75f3abd12b19ca1d90719198374 100644
--- a/source/Lib/DecoderLib/DecLib.cpp
+++ b/source/Lib/DecoderLib/DecLib.cpp
@@ -1846,6 +1846,12 @@ void DecLib::xActivateParameterSets( const InputNALUnit nalu )
m_pcPic->initPicprms(*pSlice);
}
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ if (sps->getNnlfHopTemporalFilteringEnabledFlag())
+ {
+ m_nnfilterHOP.initTemporal(m_nnlfHopTemporalModelName);
+ }
+#endif
#endif
}
else
diff --git a/source/Lib/DecoderLib/DecLib.h b/source/Lib/DecoderLib/DecLib.h
index 8b28fcc9b26c9ce1f0127abe338ca8474a66ad62..967bf57274332477ebd0aaf1d544be2efd26e060 100644
--- a/source/Lib/DecoderLib/DecLib.h
+++ b/source/Lib/DecoderLib/DecLib.h
@@ -151,6 +151,9 @@ private:
#if NN_HOP_UNIFIED_FORCE_USE
int m_nnlfHopOption;
#endif
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ std::string m_nnlfHopTemporalModelName; ///<nnlf temporal hop model
+#endif
#endif
#if NN_FILTERING_SET_0
NNFilterSet0 m_cCNNLF;
@@ -281,6 +284,10 @@ public:
void setNnlfHopOption( int i ) { m_nnlfHopOption = i; }
int getNnlfHopOption() const { return m_nnlfHopOption; }
#endif
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ void setNnlfHopTemporalModelName(std::string s) { m_nnlfHopTemporalModelName = std::move(s); }
+ const std::string &getNnlfHopTemporalModelName() { return m_nnlfHopTemporalModelName; }
+#endif
#endif
#if NN_FILTERING_SET_1
std::string getNnlfSet1InterLumaModelName() { return m_nnlfSet1InterLumaModelName; }
diff --git a/source/Lib/DecoderLib/VLCReader.cpp b/source/Lib/DecoderLib/VLCReader.cpp
index 91fea3b161ff5f1f67a5b6b18cde8991825b5f30..cef74644a8903f722c73c74f805ee26029143737 100644
--- a/source/Lib/DecoderLib/VLCReader.cpp
+++ b/source/Lib/DecoderLib/VLCReader.cpp
@@ -1752,6 +1752,9 @@ void HLSyntaxReader::parseSPS(SPS* pcSPS)
pcSPS->setNnlfHopInferSize(nnlfHopInferSize);
READ_UVLC( uiCode, "sps_nnlf_hop_inf_size_ext" ); pcSPS->setNnlfHopInfSizeExt ( uiCode );
READ_UVLC( uiCode, "sps_nnlf_hop_max_num_prms" ); pcSPS->setNnlfHopMaxNumPrms (uiCode );
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ READ_FLAG( uiCode, "sps_nnlf_hop_temporal_filtering_enabled_flag" ); pcSPS->setNnlfHopTemporalFilteringEnabledFlag ( uiCode ? true : false );
+#endif
}
#endif
diff --git a/source/Lib/EncoderLib/EncCfg.h b/source/Lib/EncoderLib/EncCfg.h
index 94e017c20faeff18ef50f88cce58f44b81a7db2c..34ec51527974f8fa4b44534cefa12989226e3b82 100644
--- a/source/Lib/EncoderLib/EncCfg.h
+++ b/source/Lib/EncoderLib/EncCfg.h
@@ -818,6 +818,10 @@ protected:
#if NN_HOP_UNIFIED_FORCE_USE
int m_nnlfHopOption;
#endif
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ bool m_nnlfHopTemporalFiltering;
+ std::string m_nnlfHopTemporalModelName; ///<nnlf temporal hop model
+#endif
#endif
#if NN_FILTERING_SET_0
@@ -2191,6 +2195,12 @@ public:
void setNnlfHopOption( int i ) { m_nnlfHopOption = i; }
int getNnlfHopOption() const { return m_nnlfHopOption; }
#endif
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ void setUseNnlfHopTemporalFiltering( bool b ) { m_nnlfHopTemporalFiltering = b; }
+ bool getUseNnlfHopTemporalFiltering() const { return m_nnlfHopTemporalFiltering; }
+ void setNnlfHopTemporalModelName(std::string s) { m_nnlfHopTemporalModelName = std::move(s); }
+ const std::string &getNnlfHopTemporalModelName() { return m_nnlfHopTemporalModelName; }
+#endif
#endif
#if NN_FILTERING_SET_0
diff --git a/source/Lib/EncoderLib/EncGOP.cpp b/source/Lib/EncoderLib/EncGOP.cpp
index 0156ed69c298911e4c815ad6364afa00003d4136..6e67b8f425d6e0329ad96fc81d2a0a868ceb0ac8 100644
--- a/source/Lib/EncoderLib/EncGOP.cpp
+++ b/source/Lib/EncoderLib/EncGOP.cpp
@@ -257,6 +257,12 @@ void EncGOP::init ( EncLib* pcEncLib )
{
m_nnfilterHOP.init(m_pcCfg->getNnlfHopModelName(), m_pcCfg->getSourceWidth(), m_pcCfg->getSourceHeight(), m_pcCfg->getChromaFormatIdc(), m_pcCfg->getNnlfHopMaxNumPrms());
}
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ if (m_pcCfg->getUseNnlfHopTemporalFiltering())
+ {
+ m_nnfilterHOP.initTemporal(m_pcCfg->getNnlfHopTemporalModelName());
+ }
+#endif
#endif
#if WCG_EXT
diff --git a/source/Lib/EncoderLib/EncLib.cpp b/source/Lib/EncoderLib/EncLib.cpp
index 68fed853d2aa376a9c5bee29ed84c77292f0e506..cd2de9f70b7eee0d2f9db0a1237c027ad15ec82a 100644
--- a/source/Lib/EncoderLib/EncLib.cpp
+++ b/source/Lib/EncoderLib/EncLib.cpp
@@ -1582,6 +1582,9 @@ void EncLib::xInitSPS( SPS& sps )
sps.setNnlfHopInferSize(nnlfHopInferSize);
sps.setNnlfHopInfSizeExt(m_nnlfHopInfSizeExt);
sps.setNnlfHopMaxNumPrms(m_intraPeriod == 1 ? 1 : m_nnlfHopMaxNumPrms); // only on/off for all-intra
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ sps.setNnlfHopTemporalFilteringEnabledFlag(m_nnlfHopTemporalFiltering);
+#endif
}
#endif
diff --git a/source/Lib/EncoderLib/VLCWriter.cpp b/source/Lib/EncoderLib/VLCWriter.cpp
index f8a9d54bde6897de152a5c13a53d0f4dd95cfcc2..9dc8dbfce6d8a9e543b91674b5c9fc97381e5bee 100644
--- a/source/Lib/EncoderLib/VLCWriter.cpp
+++ b/source/Lib/EncoderLib/VLCWriter.cpp
@@ -1022,6 +1022,9 @@ void HLSWriter::codeSPS( const SPS* pcSPS )
WRITE_UVLC( pcSPS->getNnlfHopInferSize(NNLF_HOP_INFER_GRANULARITY_BASE), "sps_nnlf_hop_infer_size_base");
WRITE_UVLC( pcSPS->getNnlfHopInfSizeExt(), "sps_nnlf_hop_inf_size_ext" );
WRITE_UVLC( pcSPS->getNnlfHopMaxNumPrms(), "sps_nnlf_hop_max_num_prms" );
+#if NN_HOP_UNIFIED_TEMPORAL_FILTERING
+ WRITE_FLAG( pcSPS->getNnlfHopTemporalFilteringEnabledFlag(), "sps_nnlf_hop_temporal_filtering_enabled_flag" );
+#endif
}
#endif