diff --git a/.gitattributes b/.gitattributes
index b0f57055dd7623f95d8803b44f7d102eb7ef1dc1..54fda3c6473c15566049239068d7d725d58b5d56 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -493,3 +493,9 @@ models/post_filter/float/overfitted_models_float/nnr_F_SlideShow_37.sadl filter=
models/post_filter/float/overfitted_models_float/nnr_D_BQSquare_32.sadl filter=lfs diff=lfs merge=lfs -text
models/post_filter/int16/overfitted_models_int16/nnr_B_BasketBallDrive_27.sadl filter=lfs diff=lfs merge=lfs -text
models/post_filter/int16/overfitted_models_int16/nnr_C_BasketballDrill_32.sadl filter=lfs diff=lfs merge=lfs -text
+models/super_resolution/Nnsr_ChromaCNNSR_Inter_Intra_float.sadl filter=lfs diff=lfs merge=lfs -text
+models/super_resolution/Nnsr_ChromaCNNSR_Inter_Intra_int16.sadl filter=lfs diff=lfs merge=lfs -text
+models/super_resolution/Nnsr_LumaCNNSR_Inter_float.sadl filter=lfs diff=lfs merge=lfs -text
+models/super_resolution/Nnsr_LumaCNNSR_Inter_int16.sadl filter=lfs diff=lfs merge=lfs -text
+models/super_resolution/Nnsr_LumaCNNSR_Intra_float.sadl filter=lfs diff=lfs merge=lfs -text
+models/super_resolution/Nnsr_LumaCNNSR_Intra_int16.sadl filter=lfs diff=lfs merge=lfs -text
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 175e5ad33c6761b46e9c9878b9c66416197210d6..b62498d973ca6cefea704eb5dbd9789953c5791f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -127,8 +127,7 @@ if( CMAKE_CXX_COMPILER_ID STREQUAL "GNU" )
endif()
endif()
-set(PATH_TO_DIRECTORY_ROOT_SADL "../../../sadl")
-
+include_directories( "./sadl" )
# modify .lldbinit for lldb custom data formatters
if( XCODE )
diff --git a/README.md b/README.md
index 5c85ddcd2ba2f064cc39d6d5b90f753cd3482f9a..381fcaabb6133939fa123cf73ea24797c282ed34 100644
--- a/README.md
+++ b/README.md
@@ -419,6 +419,10 @@ please add the following argument when running the VTM-11-NNVC encoder/decoder e
where `path_to_directory_models_intra` is the path to the directory "models/intra" relatively to the directory from which the
VTM-11-NNVC encoder/decoder executable is run.
+NN-based super resolution
+------------------------------------------------------------------------
+To activate NN-based super resolution, use --NnsrOption=1. And to successfully inference the codec, the decoder should output decoded YUVs.
+For example, the decoder commands should contain "-o /dev/null".
Content-adaptive post-filter
------------------------------------------------------------------------
diff --git a/models/super_resolution/Nnsr_ChromaCNNSR_Inter_Intra_float.sadl b/models/super_resolution/Nnsr_ChromaCNNSR_Inter_Intra_float.sadl
new file mode 100644
index 0000000000000000000000000000000000000000..3c2069a5b2c673a9b25523a412bcbe0d2847d61d
--- /dev/null
+++ b/models/super_resolution/Nnsr_ChromaCNNSR_Inter_Intra_float.sadl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6faf2337d085a578612fdb231c37683b10c58746b686f4902bb77a02d450240d
+size 6007029
diff --git a/models/super_resolution/Nnsr_ChromaCNNSR_Inter_Intra_int16.sadl b/models/super_resolution/Nnsr_ChromaCNNSR_Inter_Intra_int16.sadl
new file mode 100644
index 0000000000000000000000000000000000000000..d558ce38f42726ebc1f7d5a9aac6da6ba1b62129
--- /dev/null
+++ b/models/super_resolution/Nnsr_ChromaCNNSR_Inter_Intra_int16.sadl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87674dbee5326dd0bf0e57245f8cf3266f312c8ea52170439cc881beb26939a2
+size 3013371
diff --git a/models/super_resolution/Nnsr_LumaCNNSR_Inter_float.sadl b/models/super_resolution/Nnsr_LumaCNNSR_Inter_float.sadl
new file mode 100644
index 0000000000000000000000000000000000000000..98d9820d9b9ffae0704823843941c8e19f80593b
--- /dev/null
+++ b/models/super_resolution/Nnsr_LumaCNNSR_Inter_float.sadl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e691791b18e4947489c253c517df2ca09909e01dadbf2334aaf8e05ae6c53e2a
+size 5995184
diff --git a/models/super_resolution/Nnsr_LumaCNNSR_Inter_int16.sadl b/models/super_resolution/Nnsr_LumaCNNSR_Inter_int16.sadl
new file mode 100644
index 0000000000000000000000000000000000000000..2739257ac72b8c60a18c4c46bd6050403bf20f8a
--- /dev/null
+++ b/models/super_resolution/Nnsr_LumaCNNSR_Inter_int16.sadl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2031718d04a4352e4a75d6f06a6343b1d960888bbc15670c96016599dd0a34df
+size 3007422
diff --git a/models/super_resolution/Nnsr_LumaCNNSR_Intra_float.sadl b/models/super_resolution/Nnsr_LumaCNNSR_Intra_float.sadl
new file mode 100644
index 0000000000000000000000000000000000000000..e5934cf75fe807ac48f89d47982ea3bf2b18bb07
--- /dev/null
+++ b/models/super_resolution/Nnsr_LumaCNNSR_Intra_float.sadl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbb66bf78a6fe2f1ed528dbc266c039a55ec50a20ef43243b53d9f593aff94ad
+size 5994879
diff --git a/models/super_resolution/Nnsr_LumaCNNSR_Intra_int16.sadl b/models/super_resolution/Nnsr_LumaCNNSR_Intra_int16.sadl
new file mode 100644
index 0000000000000000000000000000000000000000..7b26d05ce3a469f68d922236bddd0fbe169c2df1
--- /dev/null
+++ b/models/super_resolution/Nnsr_LumaCNNSR_Intra_int16.sadl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1730fd3ed2142109032c404be1de4399028502ee61517d4ac83470035250680b
+size 3007245
diff --git a/source/App/DecoderApp/DecApp.cpp b/source/App/DecoderApp/DecApp.cpp
index 0fffc9f072073ab289c8a425602b14d1d229cb61..c44cc1b837babc028e01b2f8ae2fc1379a4a56e5 100644
--- a/source/App/DecoderApp/DecApp.cpp
+++ b/source/App/DecoderApp/DecApp.cpp
@@ -913,7 +913,18 @@ void DecApp::xWriteOutput( PicList* pcListPic, uint32_t tId )
ChromaFormat chromaFormatIDC = sps->getChromaFormatIdc();
if( m_upscaledOutput )
{
- m_cVideoIOYuvReconFile[pcPic->layerId].writeUpscaledPicture( *sps, *pcPic->cs->pps, pcPic->getRecoBuf(), m_outputColourSpaceConvert, m_packedYUVMode, m_upscaledOutput, NUM_CHROMA_FORMAT, m_bClipOutputVideoToRec709Range );
+ m_cVideoIOYuvReconFile[pcPic->layerId].writeUpscaledPicture( *sps, *pcPic->cs->pps, pcPic->getRecoBuf(), m_outputColourSpaceConvert, m_packedYUVMode
+#if JVET_AC0196_NNSR
+ , pcPic->getPredBufCustom()
+ , pcPic->cs->slice->getSliceQp()
+ , pcPic->cs->pps->getPicInitQPMinus26() + 26
+ , pcPic->cs->slice->getSliceType()
+#endif
+ , m_upscaledOutput, NUM_CHROMA_FORMAT, m_bClipOutputVideoToRec709Range
+ );
+#if JVET_AC0196_NNSR && NNVC_USE_PRED
+ pcPic->m_bufs[PIC_PREDICTION_CUSTOM].destroy();
+#endif
}
else
{
@@ -926,6 +937,9 @@ void DecApp::xWriteOutput( PicList* pcListPic, uint32_t tId )
conf.getWindowBottomOffset() * SPS::getWinUnitY( chromaFormatIDC ),
NUM_CHROMA_FORMAT, m_bClipOutputVideoToRec709Range );
}
+#if JVET_AC0196_NNSR
+ pcPic->destroyTempBuffers();
+#endif
}
#if NNVC_DUMP_DATA
if (!m_dumpBasename.empty())
@@ -1013,7 +1027,14 @@ void DecApp::xWriteOutput( PicList* pcListPic, uint32_t tId )
ChromaFormat chromaFormatIDC = sps->getChromaFormatIdc();
if( m_upscaledOutput )
{
- m_cVideoIOYuvReconPostFile[pcPic->layerId].writeUpscaledPicture( *sps, *pcPic->cs->pps, pcPic->getNnPostFilteredBuf(), m_outputColourSpaceConvert, m_packedYUVMode, m_upscaledOutput, NUM_CHROMA_FORMAT, m_bClipOutputVideoToRec709Range );
+ m_cVideoIOYuvReconPostFile[pcPic->layerId].writeUpscaledPicture( *sps, *pcPic->cs->pps, pcPic->getNnPostFilteredBuf(), m_outputColourSpaceConvert, m_packedYUVMode,
+#if JVET_AC0196_NNSR
+ pcPic->getPredBufCustom(),
+ pcPic->cs->slice->getSliceQp(),
+ pcPic->cs->pps->getPicInitQPMinus26() + 26,
+ pcPic->cs->slice->getSliceType(),
+#endif
+ m_upscaledOutput, NUM_CHROMA_FORMAT, m_bClipOutputVideoToRec709Range );
}
else
{
@@ -1178,7 +1199,15 @@ void DecApp::xFlushOutput( PicList* pcListPic, const int layerId )
ChromaFormat chromaFormatIDC = sps->getChromaFormatIdc();
if( m_upscaledOutput )
{
- m_cVideoIOYuvReconFile[pcPic->layerId].writeUpscaledPicture( *sps, *pcPic->cs->pps, pcPic->getRecoBuf(), m_outputColourSpaceConvert, m_packedYUVMode, m_upscaledOutput, NUM_CHROMA_FORMAT, m_bClipOutputVideoToRec709Range );
+ m_cVideoIOYuvReconFile[pcPic->layerId].writeUpscaledPicture( *sps, *pcPic->cs->pps, pcPic->getRecoBuf(), m_outputColourSpaceConvert, m_packedYUVMode
+#if JVET_AC0196_NNSR
+ , pcPic->getPredBufCustom()
+ , pcPic->cs->slice->getSliceQp()
+ , pcPic->cs->pps->getPicInitQPMinus26() + 26
+ , pcPic->cs->slice->getSliceType()
+#endif
+ , m_upscaledOutput, NUM_CHROMA_FORMAT, m_bClipOutputVideoToRec709Range
+ );
}
else
{
@@ -1190,7 +1219,10 @@ void DecApp::xFlushOutput( PicList* pcListPic, const int layerId )
conf.getWindowTopOffset() * SPS::getWinUnitY( chromaFormatIDC ),
conf.getWindowBottomOffset() * SPS::getWinUnitY( chromaFormatIDC ),
NUM_CHROMA_FORMAT, m_bClipOutputVideoToRec709Range );
- }
+ }
+#if JVET_AC0196_NNSR
+ pcPic->destroyTempBuffers();
+#endif
}
// write to file
#if NNVC_DUMP_DATA
@@ -1278,7 +1310,14 @@ void DecApp::xFlushOutput( PicList* pcListPic, const int layerId )
ChromaFormat chromaFormatIDC = sps->getChromaFormatIdc();
if( m_upscaledOutput )
{
- m_cVideoIOYuvReconPostFile[pcPic->layerId].writeUpscaledPicture( *sps, *pcPic->cs->pps, pcPic->getNnPostFilteredBuf(), m_outputColourSpaceConvert, m_packedYUVMode, m_upscaledOutput, NUM_CHROMA_FORMAT, m_bClipOutputVideoToRec709Range );
+ m_cVideoIOYuvReconPostFile[pcPic->layerId].writeUpscaledPicture( *sps, *pcPic->cs->pps, pcPic->getNnPostFilteredBuf(), m_outputColourSpaceConvert, m_packedYUVMode,
+#if JVET_AC0196_NNSR
+ pcPic->getPredBufCustom(),
+ pcPic->cs->slice->getSliceQp(),
+ pcPic->cs->pps->getPicInitQPMinus26() + 26,
+ pcPic->cs->slice->getSliceType(),
+#endif
+ m_upscaledOutput, NUM_CHROMA_FORMAT, m_bClipOutputVideoToRec709Range );
}
else
{
diff --git a/source/App/DecoderApp/DecAppCfg.cpp b/source/App/DecoderApp/DecAppCfg.cpp
index 992409f7c46554bb7eda58f1937f8cf94128e4df..c4343d695bc64a7f9fcb599bd4b67c571284e33b 100644
--- a/source/App/DecoderApp/DecAppCfg.cpp
+++ b/source/App/DecoderApp/DecAppCfg.cpp
@@ -146,7 +146,11 @@ bool DecAppCfg::parseCfg( int argc, char* argv[] )
#endif
("MCTSCheck", m_mctsCheck, false, "If enabled, the decoder checks for violations of mc_exact_sample_value_match_flag in Temporal MCTS ")
("targetSubPicIdx", m_targetSubPicIdx, 0, "Specify which subpicture shall be written to output, using subpic index, 0: disabled, subpicIdx=m_targetSubPicIdx-1 \n" )
+#if JVET_AC0196_NNSR
+ ( "UpscaledOutput", m_upscaledOutput, 2, "Upscaled output for RPR" )
+#else
( "UpscaledOutput", m_upscaledOutput, 0, "Upscaled output for RPR" )
+#endif
#if JVET_AB0149_INTRA_PRED
("DescriptionPairHeightWidthPathToGraphOutput", m_descriptionPairHeightWidthPathToGraphOutput, string("4,4,graph_output_4_4_int16.sadl;4,8,graph_output_4_8_int16.sadl;4,16,graph_output_4_16_int16.sadl;4,32,graph_output_4_32_int16.sadl;8,8,graph_output_8_8_int16.sadl;8,16,graph_output_8_16_int16.sadl;16,16,graph_output_16_16_int16.sadl;"), "Description of each pair of block height and width being a map key and the path to the output graph of the prediction neural network being its string value.")
("PrefixAbsolutePathsToGraphsOutput", m_prefixAbsolutePathsToGraphsOutput, string("models/intra"), "Prefix of the absolute path to the output graph of each prediction neural network.")
diff --git a/source/App/EncoderApp/EncApp.cpp b/source/App/EncoderApp/EncApp.cpp
index c8b2845cb1f9fcce7e84ad85174472bd624a543f..b5c80aabc2f2a4d56157828e778e94c0a572bc53 100644
--- a/source/App/EncoderApp/EncApp.cpp
+++ b/source/App/EncoderApp/EncApp.cpp
@@ -260,6 +260,11 @@ void EncApp::xInitLibCfg()
m_cEncLib.setRdoCnnlfInterLumaModelNameNNFilter1 (m_rdoCnnlfInterLumaModelNameNNFilter1);
m_cEncLib.setRdoCnnlfIntraLumaModelNameNNFilter1 (m_rdoCnnlfIntraLumaModelNameNNFilter1);
#endif
+
+#if JVET_AC0196_NNSR
+ m_cEncLib.setUseNnsr (m_nnsrOption);
+#endif
+
m_cEncLib.setProfile ( m_profile);
m_cEncLib.setLevel ( m_levelTier, m_level);
m_cEncLib.setFrameOnlyConstraintFlag ( m_frameOnlyConstraintFlag);
@@ -1273,11 +1278,22 @@ void EncApp::createLib( const int layerIdx )
#endif
const int sourceHeight = m_isField ? m_iSourceHeightOrg : m_iSourceHeight;
UnitArea unitArea( m_chromaFormatIDC, Area( 0, 0, m_iSourceWidth, sourceHeight ) );
-
+
+#if ADAPTIVE_RPR
+ UnitArea unitAreaRPR10(m_chromaFormatIDC, Area(0, 0, int(m_iSourceWidth), int(sourceHeight)));
+ UnitArea unitAreaRPR20(m_chromaFormatIDC, Area(0, 0, int(m_iSourceWidth / 2.0), int(sourceHeight / 2.0)));
+#endif
+
m_orgPic = new PelStorage;
m_trueOrgPic = new PelStorage;
m_orgPic->create( unitArea );
m_trueOrgPic->create( unitArea );
+#if ADAPTIVE_RPR
+ m_rprPic[0] = new PelStorage;
+ m_rprPic[0]->create(unitAreaRPR10);
+ m_rprPic[1] = new PelStorage;
+ m_rprPic[1]->create(unitAreaRPR20);
+#endif
if(m_gopBasedTemporalFilterEnabled)
{
m_filteredOrgPic = new PelStorage;
@@ -1345,6 +1361,13 @@ void EncApp::destroyLib()
m_trueOrgPic->destroy();
delete m_trueOrgPic;
delete m_orgPic;
+#if ADAPTIVE_RPR
+ for (int i = 0; i < 2; i++)
+ {
+ m_rprPic[i]->destroy();
+ delete m_rprPic[i];
+ }
+#endif
if(m_gopBasedTemporalFilterEnabled)
{
m_filteredOrgPic->destroy();
@@ -1417,7 +1440,11 @@ bool EncApp::encodePrep( bool& eos )
}
else
{
- keepDoing = m_cEncLib.encodePrep( eos, m_flush ? 0 : m_orgPic, m_flush ? 0 : m_trueOrgPic, m_flush ? 0 : m_filteredOrgPic, snrCSC, m_recBufList, m_numEncoded );
+ keepDoing = m_cEncLib.encodePrep( eos, m_flush ? 0 : m_orgPic, m_flush ? 0 : m_trueOrgPic, m_flush ? 0 : m_filteredOrgPic, snrCSC, m_recBufList, m_numEncoded
+#if ADAPTIVE_RPR
+ , m_rprPic
+#endif
+ );
}
return keepDoing;
@@ -1480,10 +1507,17 @@ void EncApp::xWriteOutput( int iNumEncoded, std::list<PelUnitBuf*>& recBufList )
const InputColourSpaceConversion ipCSC = (!m_outputInternalColourSpace) ? m_inputColourSpaceConvert : IPCOLOURSPACE_UNCHANGED;
std::list<PelUnitBuf*>::iterator iterPicYuvRec = recBufList.end();
int i;
-
+#if JVET_AC0196_NNSR
+ PicList tempPicList = *m_cEncLib.getListPic();
+ Slice::sortPicList(tempPicList);
+ std::list<Picture *>::iterator iterPicYuvPred = tempPicList.end();
+#endif
for ( i = 0; i < iNumEncoded; i++ )
{
--iterPicYuvRec;
+#if JVET_AC0196_NNSR
+ --iterPicYuvPred;
+#endif
}
if (m_isField)
@@ -1508,20 +1542,37 @@ void EncApp::xWriteOutput( int iNumEncoded, std::list<PelUnitBuf*>& recBufList )
for ( i = 0; i < iNumEncoded; i++ )
{
const PelUnitBuf* pcPicYuvRec = *(iterPicYuvRec++);
+#if JVET_AC0196_NNSR
+ Picture* tempPic = *(iterPicYuvPred++);
+#endif
if (!m_reconFileName.empty())
{
if( m_cEncLib.isResChangeInClvsEnabled() && m_cEncLib.getUpscaledOutput() )
{
const SPS& sps = *m_cEncLib.getSPS( 0 );
+#if ADAPTIVE_RPR
+ const PPS& pps = *m_cEncLib.getPPS((sps.getMaxPicWidthInLumaSamples() != pcPicYuvRec->get(COMPONENT_Y).width || sps.getMaxPicHeightInLumaSamples() != pcPicYuvRec->get(COMPONENT_Y).height) ? m_cEncLib.m_iGOPRprPpsId : 0);
+#else
const PPS& pps = *m_cEncLib.getPPS( ( sps.getMaxPicWidthInLumaSamples() != pcPicYuvRec->get( COMPONENT_Y ).width || sps.getMaxPicHeightInLumaSamples() != pcPicYuvRec->get( COMPONENT_Y ).height ) ? ENC_PPS_ID_RPR : 0 );
-
- m_cVideoIOYuvReconFile.writeUpscaledPicture( sps, pps, *pcPicYuvRec, ipCSC, m_packedYUVMode, m_cEncLib.getUpscaledOutput(), NUM_CHROMA_FORMAT, m_bClipOutputVideoToRec709Range );
+#endif
+ m_cVideoIOYuvReconFile.writeUpscaledPicture( sps, pps, *pcPicYuvRec, ipCSC, m_packedYUVMode
+#if JVET_AC0196_NNSR
+ , tempPic->getPredBufCustom()
+ , tempPic->cs->slice->getSliceQp()
+ , tempPic->cs->pps->getPicInitQPMinus26() + 26
+ , tempPic->cs->slice->getSliceType()
+#endif
+ , m_cEncLib.getUpscaledOutput(), NUM_CHROMA_FORMAT, m_bClipOutputVideoToRec709Range
+ );
}
else
{
m_cVideoIOYuvReconFile.write( pcPicYuvRec->get( COMPONENT_Y ).width, pcPicYuvRec->get( COMPONENT_Y ).height, *pcPicYuvRec, ipCSC, m_packedYUVMode,
m_confWinLeft, m_confWinRight, m_confWinTop, m_confWinBottom, NUM_CHROMA_FORMAT, m_bClipOutputVideoToRec709Range );
}
+#if JVET_AC0196_NNSR
+ tempPic->destroyTempBuffers();
+#endif
}
}
}
diff --git a/source/App/EncoderApp/EncApp.h b/source/App/EncoderApp/EncApp.h
index b3b2f3e8c43782d848e81ffafddaf3c80c463cdc..332c7aa33393fa7f66ca388e47f18d81995b2c86 100644
--- a/source/App/EncoderApp/EncApp.h
+++ b/source/App/EncoderApp/EncApp.h
@@ -98,6 +98,9 @@ private:
PelStorage* m_trueOrgPic;
PelStorage* m_orgPic;
PelStorage* m_filteredOrgPic;
+#if ADAPTIVE_RPR
+ PelStorage* m_rprPic[2];
+#endif
#if EXTENSION_360_VIDEO
TExt360AppEncTop* m_ext360;
#endif
diff --git a/source/App/EncoderApp/EncAppCfg.cpp b/source/App/EncoderApp/EncAppCfg.cpp
index 470ee7dd6f3c93b4dd08475c11905feddbb62fcc..3f5d257c121a6017149d44d17a9f3ad721b9552f 100644
--- a/source/App/EncoderApp/EncAppCfg.cpp
+++ b/source/App/EncoderApp/EncAppCfg.cpp
@@ -1463,11 +1463,20 @@ bool EncAppCfg::parseCfg( int argc, char* argv[] )
( "RdoCnnlfIntraLumaModelNNFilter1", m_rdoCnnlfIntraLumaModelNameNNFilter1, string("models/RdNnlfSet1_LumaCNNFilter_IntraSlice_int16.sadl"), "NnlfSet1 intra luma model name for RDO")
#endif
( "RPR", m_rprEnabledFlag, true, "Reference Sample Resolution" )
+#if JVET_AC0196_NNSR
+ ( "NnsrOption", m_nnsrOption, false, "NN-based super resolution option")
+ ( "ScalingRatioHor", m_scalingRatioHor, 2.0, "Scaling ratio in hor direction" )
+ ( "ScalingRatioVer", m_scalingRatioVer, 2.0, "Scaling ratio in ver direction" )
+ ( "FractionNumFrames", m_fractionOfFrames, 1.0, "Encode a fraction of the specified in FramesToBeEncoded frames" )
+ ( "SwitchPocPeriod", m_switchPocPeriod, 0, "Switch POC period for RPR" )
+ ( "UpscaledOutput", m_upscaledOutput, 2, "Output upscaled (2), decoded but in full resolution buffer (1) or decoded cropped (0, default) picture for RPR" )
+#else
( "ScalingRatioHor", m_scalingRatioHor, 1.0, "Scaling ratio in hor direction" )
( "ScalingRatioVer", m_scalingRatioVer, 1.0, "Scaling ratio in ver direction" )
( "FractionNumFrames", m_fractionOfFrames, 1.0, "Encode a fraction of the specified in FramesToBeEncoded frames" )
( "SwitchPocPeriod", m_switchPocPeriod, 0, "Switch POC period for RPR" )
( "UpscaledOutput", m_upscaledOutput, 0, "Output upscaled (2), decoded but in full resolution buffer (1) or decoded cropped (0, default) picture for RPR" )
+#endif
( "MaxLayers", m_maxLayers, 1, "Max number of layers" )
#if JVET_S0163_ON_TARGETOLS_SUBLAYERS
( "EnableOperatingPointInformation", m_OPIEnabled, false, "Enables writing of Operating Point Information (OPI)" )
@@ -1661,6 +1670,12 @@ bool EncAppCfg::parseCfg( int argc, char* argv[] )
po::ErrorReporter err;
const list<const char*>& argv_unhandled = po::scanArgv(opts, argc, (const char**) argv, err);
+#if JVET_AC0196_NNSR
+ m_nnsrOption = m_nnsrOption ? true : false;
+ m_scalingRatioHor = m_nnsrOption ? 2.0 : 1.0;
+ m_scalingRatioVer = m_nnsrOption ? 2.0 : 1.0;
+ m_upscaledOutput = (m_nnsrOption && (m_upscaledOutput != 0)) ? 2 : 0;
+#endif
m_resChangeInClvsEnabled = m_scalingRatioHor != 1.0 || m_scalingRatioVer != 1.0;
m_resChangeInClvsEnabled = m_resChangeInClvsEnabled && m_rprEnabledFlag;
if( m_fractionOfFrames != 1.0 )
@@ -4207,6 +4222,9 @@ void EncAppCfg::xPrintParameter()
#if JVET_AC0328_NNLF_RDO
msg( VERBOSE, "EncNnlfOpt:%d ", m_encNnlfOpt ? 1 : 0);
#endif
+#if JVET_AC0196_NNSR
+ msg( VERBOSE, "NnsrOption:%d ", m_nnsrOption ? 1 : 0);
+#endif
#if JVET_AC0055_NN_POST_FILTERING
msg(VERBOSE, "Nnpf:%d", m_nnpf ? 1 : 0);
#endif
diff --git a/source/App/EncoderApp/EncAppCfg.h b/source/App/EncoderApp/EncAppCfg.h
index 94db3117a3fed1ed1519466351f4c9852552f554..aef6a5a51f9af03361dcb49f5436c8b76ca1c68d 100644
--- a/source/App/EncoderApp/EncAppCfg.h
+++ b/source/App/EncoderApp/EncAppCfg.h
@@ -781,6 +781,9 @@ protected:
bool m_encNnlfOpt;
#endif
+#if JVET_AC0196_NNSR
+ bool m_nnsrOption;
+#endif
#if JVET_AC0055_NN_POST_FILTERING
bool m_nnpf;
#endif
diff --git a/source/Lib/CommonLib/NNSuperResolution.cpp b/source/Lib/CommonLib/NNSuperResolution.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..1e202e231c5bfb11af2c028cb76b687befff8a48
--- /dev/null
+++ b/source/Lib/CommonLib/NNSuperResolution.cpp
@@ -0,0 +1,296 @@
+/* The copyright in this software is being made available under the BSD
+ * License, included below. This software may be subject to other third party
+ * and contributor rights, including patent rights, and no such rights are
+ * granted under this license.
+ *
+ * Copyright (c) 2010-2018, ITU/ISO/IEC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of the ITU/ISO/IEC nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file NNSuperResolution.cpp
+ \brief nn super resolution class
+*/
+
+#include "NNSuperResolution.h"
+
+#if JVET_AC0196_NNSR
+#include "CodingStructure.h"
+#include "Picture.h"
+
+NNSuperResolution::NNSuperResolution()
+{
+ m_initFlag = false;
+}
+
+void NNSuperResolution::initCNNModel()
+{
+ if (m_initFlag)
+ {
+ return;
+ }
+ static const std::string modelDir = "./models/super_resolution/";
+#if NN_FIXED_POINT_IMPLEMENTATION
+ static const std::string lumaIModel = modelDir + "Nnsr_LumaCNNSR_Intra_int16.sadl";
+ static const std::string lumaBModel = modelDir + "Nnsr_LumaCNNSR_Inter_int16.sadl";
+ static const std::string chromaModel = modelDir + "Nnsr_ChromaCNNSR_Inter_Intra_int16.sadl";
+#else
+ static const std::string lumaIModel = modelDir + "Nnsr_LumaCNNSR_Intra_float.sadl";
+ static const std::string lumaBModel = modelDir + "Nnsr_LumaCNNSR_Inter_float.sadl";
+ static const std::string chromaModel = modelDir + "Nnsr_ChromaCNNSR_Inter_Intra_float.sadl";
+#endif
+
+ ifstream lumaIfile(lumaIModel, ios::binary);
+ if ((!m_SRYModel[0].load(lumaIfile)))
+ {
+ cerr << "[ERROR] Unable to read model: " << lumaIModel << endl;
+ exit(-1);
+ }
+ ifstream lumaBfile(lumaBModel, ios::binary);
+ if ((!m_SRYModel[1].load(lumaBfile)))
+ {
+ cerr << "[ERROR] Unable to read model: " << lumaBModel << endl;
+ exit(-1);
+ }
+ ifstream chromaIBfile(chromaModel, ios::binary);
+ if ((!m_SRUVModel.load(chromaIBfile)))
+ {
+ cerr << "[ERROR] Unable to read model: " << chromaModel << endl;
+ exit(-1);
+ }
+
+ m_initFlag = true;
+}
+
+void NNSuperResolution::CNNSRProcess(const CPelUnitBuf &recBuf, const PelUnitBuf &predBuf, const PelUnitBuf &rprBuf,
+ PelUnitBuf &cnnBuf, const int actualHeight, const int actualWidth,
+ const int sliceQP, const int baseQP, const SliceType sliceType)
+{
+ // setting
+ int patchSize = 128;
+ int padSize = 8;
+ double maxValue = 1023;
+#if NN_FIXED_POINT_IMPLEMENTATION
+ const int org_quantizers_in = 10;
+ const int org_quantizers_out = 10;
+ const int sadl_quantizers_in = 11;
+ const int sadl_quantizers_out = 11;
+ const int in_left_shift = sadl_quantizers_in - org_quantizers_in;
+ const int out_left_shift = sadl_quantizers_out - org_quantizers_out;
+#endif
+
+ int picHeight = recBuf.get(COMPONENT_Y).height;
+ int picWidth = recBuf.get(COMPONENT_Y).width;
+ int picWidthInPatchs = ceil((double) picWidth / patchSize);
+ int picHeightInPatchs = ceil((double) picHeight / patchSize);
+
+ initCNNModel();
+
+ vector<sadl::Tensor<TypeSadl>> m_InputY;
+ vector<sadl::Tensor<TypeSadl>> m_InputUV;
+ sliceType == I_SLICE ? m_InputY = m_SRYModel[0].getInputsTemplate() : m_InputY = m_SRYModel[1].getInputsTemplate();
+ m_InputUV = m_SRUVModel.getInputsTemplate();
+
+ for (int y = 0; y < picHeightInPatchs; y++)
+ {
+ for (int x = 0; x < picWidthInPatchs; x++)
+ {
+ // obtain input
+ int pix_y = y * patchSize;
+ int pix_x = x * patchSize;
+
+ int pix_y_end = (y + 1) * patchSize > picHeight ? picHeight - 1 : (y + 1) * patchSize - 1;
+ int pix_x_end = (x + 1) * patchSize > picWidth ? picWidth - 1 : (x + 1) * patchSize - 1;
+
+ int st_h = pix_y - padSize < 0 ? 0 : pix_y - padSize;
+ int ed_h = pix_y_end + padSize + 1 > picHeight ? picHeight - 1 : pix_y_end + padSize;
+ int st_w = pix_x - padSize < 0 ? 0 : pix_x - padSize;
+ int ed_w = pix_x_end + padSize + 1 > picWidth ? picWidth - 1 : pix_x_end + padSize;
+
+ int actualPatchSizeH = ed_h - st_h + 1;
+ int actualPatchSizeW = ed_w - st_w + 1;
+
+ int InputIdx = 0;
+ for (auto &m: m_InputY)
+ {
+ sadl::Dimensions dims(std::initializer_list<int>({ 1, actualPatchSizeH, actualPatchSizeW, 1 }));
+ m.resize(dims);
+ InputIdx++;
+ }
+ InputIdx = 0;
+ for (auto &m: m_InputUV)
+ {
+ if (m.dims()[3] == 1 && InputIdx == 0)
+ {
+ sadl::Dimensions dims(std::initializer_list<int>({ 1, actualPatchSizeH, actualPatchSizeW, 1 }));
+ m.resize(dims);
+ }
+ else if (m.dims()[3] == 2 && InputIdx == 1)
+ {
+ sadl::Dimensions dims(std::initializer_list<int>({ 1, actualPatchSizeH / 2, actualPatchSizeW / 2, 2 }));
+ m.resize(dims);
+ }
+ else
+ {
+ sadl::Dimensions dims(std::initializer_list<int>({ 1, actualPatchSizeH / 2, actualPatchSizeW / 2, 1 }));
+ m.resize(dims);
+ }
+ InputIdx++;
+ }
+
+ if (!m_SRYModel[sliceType == I_SLICE ? 0 : 1].init(m_InputY))
+ {
+ cerr << "[ERROR] issue during initialization for luma" << endl;
+ exit(-1);
+ }
+ if (!m_SRUVModel.init(m_InputUV))
+ {
+ cerr << "[ERROR] issue during initialization for chroma" << endl;
+ exit(-1);
+ }
+
+ for (int yy = 0; yy < actualPatchSizeH; yy++)
+ {
+ for (int xx = 0; xx < actualPatchSizeW; xx++)
+ {
+ int id_x[2], id_y[2];
+ id_x[0] = st_w + xx;
+ id_y[0] = st_h + yy;
+ id_x[1] = (st_w >> 1) + (xx >> 1);
+ id_y[1] = (st_h >> 1) + (yy >> 1);
+
+ int xxx[2], yyy[2];
+ xxx[0] = xx;
+ yyy[0] = yy;
+ xxx[1] = (xx >> 1);
+ yyy[1] = (yy >> 1);
+#if NN_FIXED_POINT_IMPLEMENTATION
+ m_InputY[0][yyy[0] * actualPatchSizeW + xxx[0]] = recBuf.get(COMPONENT_Y).at(id_x[0], id_y[0]) << in_left_shift;
+ m_InputY[1][yyy[0] * actualPatchSizeW + xxx[0]] = predBuf.get(COMPONENT_Y).at(id_x[0], id_y[0]) << in_left_shift;
+ m_InputY[2][yyy[0] * actualPatchSizeW + xxx[0]] = sliceQP << in_left_shift;
+ if (sliceType == B_SLICE)
+ {
+ m_InputY[3][yyy[0] * actualPatchSizeW + xxx[0]] = baseQP << in_left_shift;
+ }
+ m_InputUV[0][yyy[0] * actualPatchSizeW + xxx[0]] = recBuf.get(COMPONENT_Y).at(id_x[0], id_y[0]) << in_left_shift;
+ m_InputUV[1][(yyy[1] * actualPatchSizeW / 2 + xxx[1]) * 2] = recBuf.get(COMPONENT_Cb).at(id_x[1], id_y[1]) << in_left_shift;
+ m_InputUV[1][(yyy[1] * actualPatchSizeW / 2 + xxx[1]) * 2 + 1] = recBuf.get(COMPONENT_Cr).at(id_x[1], id_y[1]) << in_left_shift;
+ m_InputUV[2][yyy[1] * actualPatchSizeW / 2 + xxx[1]] = sliceQP << in_left_shift;
+ m_InputUV[3][yyy[1] * actualPatchSizeW / 2 + xxx[1]] = baseQP << in_left_shift;
+ m_InputUV[4][yyy[1] * actualPatchSizeW / 2 + xxx[1]] = (sliceType == I_SLICE ? 0 : 1023) << in_left_shift;
+#else
+ m_InputY[0][yyy[0] * actualPatchSizeW + xxx[0]] = recBuf.get(COMPONENT_Y).at(id_x[0], id_y[0]) / maxValue;
+ m_InputY[1][yyy[0] * actualPatchSizeW + xxx[0]] = predBuf.get(COMPONENT_Y).at(id_x[0], id_y[0]) / maxValue;
+ m_InputY[2][yyy[0] * actualPatchSizeW + xxx[0]] = sliceQP / maxValue;
+ if (sliceType == B_SLICE)
+ {
+ m_InputY[3][yyy[0] * actualPatchSizeW + xxx[0]] = baseQP / maxValue;
+ }
+
+ m_InputUV[0][yyy[0] * actualPatchSizeW + xxx[0]] = recBuf.get(COMPONENT_Y).at(id_x[0], id_y[0]) / maxValue;
+ m_InputUV[1][(yyy[1] * actualPatchSizeW / 2 + xxx[1]) * 2] = recBuf.get(COMPONENT_Cb).at(id_x[1], id_y[1]) / maxValue;
+ m_InputUV[1][(yyy[1] * actualPatchSizeW / 2 + xxx[1]) * 2 + 1] = recBuf.get(COMPONENT_Cr).at(id_x[1], id_y[1]) / maxValue;
+ m_InputUV[2][yyy[1] * actualPatchSizeW / 2 + xxx[1]] = sliceQP / maxValue;
+ m_InputUV[3][yyy[1] * actualPatchSizeW / 2 + xxx[1]] = baseQP / maxValue;
+ m_InputUV[4][yyy[1] * actualPatchSizeW / 2 + xxx[1]] = (sliceType == I_SLICE ? 0 : maxValue) / maxValue;
+#endif
+ }
+ }
+
+ if (sliceType == I_SLICE)
+ {
+ if (!m_SRYModel[0].apply(m_InputY))
+ {
+ cerr << "[ERROR] issue during inference for luma" << endl;
+ exit(-1);
+ }
+ }
+ else
+ {
+ if (!m_SRYModel[1].apply(m_InputY))
+ {
+ cerr << "[ERROR] issue during inference for luma" << endl;
+ exit(-1);
+ }
+ }
+ if (!m_SRUVModel.apply(m_InputUV))
+ {
+ cerr << "[ERROR] issue during inference for chroma" << endl;
+ exit(-1);
+ }
+
+ int centerH = pix_y_end - pix_y + 1;
+ int centerW = pix_x_end - pix_x + 1;
+ // extract luma
+ for (int kk = 0; kk < 4; kk++)
+ {
+ for (int yy = 0; yy < centerH; yy++)
+ {
+ for (int xx = 0; xx < centerW; xx++)
+ {
+ int id_x = ((pix_x + xx) << 1) + kk % 2;
+ int id_y = ((pix_y + yy) << 1) + kk / 2;
+ int pos_x = pix_x - st_w + xx;
+ int pos_y = pix_y - st_h + yy;
+
+
+#if NN_FIXED_POINT_IMPLEMENTATION
+ cnnBuf.get(COMPONENT_Y).at(id_x, id_y) = Pel(Clip3<int>(0, maxValue, int(m_SRYModel[sliceType == I_SLICE ? 0 : 1].result(0)(0, pos_y, pos_x, kk) + (rprBuf.get(COMPONENT_Y).at(id_x, id_y) << in_left_shift)) >> out_left_shift));
+#else
+ cnnBuf.get(COMPONENT_Y).at(id_x, id_y) = Pel(Clip3<int>(0, maxValue, int((m_SRYModel[sliceType == I_SLICE ? 0 : 1].result(0)(0, (id_y - st_h) >> 1, (id_x - st_w) >> 1, kk) + (rprBuf.get(COMPONENT_Y).at(id_x, id_y) / maxValue)) * maxValue + 0.5)));
+#endif
+ }
+ }
+ }
+ // extract chroma
+ int centerH_chroma = centerH >> 1;
+ int centerW_chroma = centerW >> 1;
+ for (int kk = 0; kk < 4; kk++)
+ {
+ for (int yy = 0; yy < centerH_chroma; yy++)
+ {
+ for (int xx = 0; xx < centerW_chroma; xx++)
+ {
+ int id_x = pix_x + (xx << 1) + kk % 2;
+ int id_y = pix_y + (yy << 1) + kk / 2;
+ int pos_x = ((pix_x - st_w) >> 1) + xx;
+ int pos_y = ((pix_y - st_h) >> 1) + yy;
+
+#if NN_FIXED_POINT_IMPLEMENTATION
+ cnnBuf.get(COMPONENT_Cb).at(id_x, id_y) = Pel(Clip3<int>(0, maxValue, int(m_SRUVModel.result(0)(0, pos_y, pos_x, kk) + (rprBuf.get(COMPONENT_Cb).at(id_x, id_y) << in_left_shift)) >> out_left_shift));
+ cnnBuf.get(COMPONENT_Cr).at(id_x, id_y) = Pel(Clip3<int>(0, maxValue, int(m_SRUVModel.result(0)(0, pos_y, pos_x, 4+kk) + (rprBuf.get(COMPONENT_Cr).at(id_x, id_y) << in_left_shift)) >> out_left_shift));
+#else
+ cnnBuf.get(COMPONENT_Cb).at(id_x, id_y) = Pel(Clip3<int>(0, maxValue, int((m_SRUVModel.result(0)(0, pos_y >> 1, pos_x >> 1, kk) + (rprBuf.get(COMPONENT_Cb).at(id_x, id_y) / maxValue)) * maxValue + 0.5)));
+ cnnBuf.get(COMPONENT_Cr).at(id_x, id_y) = Pel(Clip3<int>(0, maxValue, int((m_SRUVModel.result(0)(0, pos_y >> 1, pos_x >> 1, 4+kk) + (rprBuf.get(COMPONENT_Cr).at(id_x, id_y) / maxValue)) * maxValue + 0.5)));
+#endif
+ }
+ }
+ }
+ }
+ }
+}
+
+#endif
diff --git a/source/Lib/CommonLib/NNSuperResolution.h b/source/Lib/CommonLib/NNSuperResolution.h
new file mode 100644
index 0000000000000000000000000000000000000000..73d1ab26d214b420cc48b936ba97e2d12b638072
--- /dev/null
+++ b/source/Lib/CommonLib/NNSuperResolution.h
@@ -0,0 +1,67 @@
+/* The copyright in this software is being made available under the BSD
+ * License, included below. This software may be subject to other third party
+ * and contributor rights, including patent rights, and no such rights are
+ * granted under this license.
+ *
+ * Copyright (c) 2010-2018, ITU/ISO/IEC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of the ITU/ISO/IEC nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file NNSuperResolution.h
+ \brief nn super resolution class (header)
+*/
+
+
+#ifndef __NNSuperResolution__
+#define __NNSuperResolution__
+
+#include "CommonDef.h"
+
+#if JVET_AC0196_NNSR
+#include "Picture.h"
+#include <sadl/model.h>
+#include <fstream>
+using namespace std;
+
+class NNSuperResolution
+{
+public:
+ NNSuperResolution();
+ virtual ~NNSuperResolution() {}
+
+ bool m_initFlag;
+
+ sadl::Model<TypeSadl> m_SRYModel[2];
+ sadl::Model<TypeSadl> m_SRUVModel;
+
+ void initCNNModel();
+ void CNNSRProcess(const CPelUnitBuf &recBuf, const PelUnitBuf &predBuf, const PelUnitBuf &rprBuf, PelUnitBuf &cnnBuf, const int actualHeight, const int actualWidth, const int baseQP, const int sliceQP, const SliceType sliceType);
+
+};
+#endif
+
+#endif
diff --git a/source/Lib/CommonLib/Picture.cpp b/source/Lib/CommonLib/Picture.cpp
index 7b830db77415692fc16ca5cc418e1157782c1179..0b2f589b5d84cccc8fe0b0bef4da389e00d4a57a 100644
--- a/source/Lib/CommonLib/Picture.cpp
+++ b/source/Lib/CommonLib/Picture.cpp
@@ -336,6 +336,12 @@ void Picture::destroyTempBuffers()
M_BUFS( jId, t ).destroy();
}
#endif
+#if JVET_AC0196_NNSR && NNVC_USE_PRED
+ if (t == PIC_PREDICTION_CUSTOM )
+ {
+ M_BUFS( jId, t ).destroy();
+ }
+#endif
#if ENABLE_SPLIT_PARALLELISM
if (t == PIC_RECONSTRUCTION && jId > 0)
{
@@ -1059,6 +1065,17 @@ void Picture::rescalePicture( const std::pair<int, int> scalingRatio,
}
}
+#if JVET_AC0196_NNSR
+void Picture::rescalePictureCNN(const CPelUnitBuf& recBuf, const PelUnitBuf& predBuf, const PelUnitBuf& rprBuf, PelUnitBuf& cnnBuf, int sliceQP, int baseQP, SliceType sliceType)
+{
+ int actualHeight = rprBuf.get(COMPONENT_Y).height;
+ int actualWidth = rprBuf.get(COMPONENT_Y).width;
+
+ static NNSuperResolution m_CNNSR;
+ m_CNNSR.CNNSRProcess(recBuf, predBuf, rprBuf, cnnBuf, actualHeight, actualWidth, sliceQP, baseQP, sliceType);
+}
+#endif
+
void Picture::saveSubPicBorder(int POC, int subPicX0, int subPicY0, int subPicWidth, int subPicHeight)
{
diff --git a/source/Lib/CommonLib/Picture.h b/source/Lib/CommonLib/Picture.h
index a84dbf4ca13717ef83f52fc658391e63ca4e71da..f43fc756779ebc3db2eb44eb7a53019056406956 100644
--- a/source/Lib/CommonLib/Picture.h
+++ b/source/Lib/CommonLib/Picture.h
@@ -50,6 +50,10 @@
#include "MCTS.h"
#include <deque>
+#if JVET_AC0196_NNSR
+#include "NNSuperResolution.h"
+#endif
+
#if ENABLE_SPLIT_PARALLELISM
#define CURR_THREAD_ID -1
@@ -240,6 +244,10 @@ const CPelBuf getBsMapBuf(const CompArea &blk) const;
const ChromaFormat chromaFormatIDC, const BitDepths& bitDepths, const bool useLumaFilter, const bool downsampling,
const bool horCollocatedChromaFlag, const bool verCollocatedChromaFlag );
+#if JVET_AC0196_NNSR
+ static void rescalePictureCNN(const CPelUnitBuf& recBuf, const PelUnitBuf& predBuf, const PelUnitBuf& rprBuf, PelUnitBuf& cnnBuf, int sliceQP, int baseQP, SliceType sliceType);
+#endif
+
private:
Window m_conformanceWindow;
Window m_scalingWindow;
diff --git a/source/Lib/CommonLib/TypeDef.h b/source/Lib/CommonLib/TypeDef.h
index afc27ebe906d1641f1be9b793c8d554a193ef34b..0bf33d4e6753a3b9fbe5be7dc9c3e957220b55d2 100644
--- a/source/Lib/CommonLib/TypeDef.h
+++ b/source/Lib/CommonLib/TypeDef.h
@@ -122,6 +122,11 @@ using TypeSadl = float;
#define JVET_AC0177_FLIP_INPUT 1 // JVET-AC0177: flip input and output of NN filter model
#endif
+#define JVET_AC0196_NNSR 1 // JVET-AC0916: EE1-2.2: GOP Level Adaptive Resampling with CNN-based Super Resolution
+#if JVET_AC0196_NNSR
+#define ADAPTIVE_RPR 1 // JVET-AC0196: GOP Level Adaptive Resampling
+#endif
+
#define JVET_AC0055_NN_POST_FILTERING 1 // JVET-AC0055: EE1-1.11: Content-adaptive post-filter
//########### place macros to be removed in next cycle below this line ###############
diff --git a/source/Lib/DecoderLib/DecLib.cpp b/source/Lib/DecoderLib/DecLib.cpp
index 875c2867fc211097c66f8c6c550d7cd19eac3e5c..2805f31ce3987a035957700ad095a7da3b580e6c 100644
--- a/source/Lib/DecoderLib/DecLib.cpp
+++ b/source/Lib/DecoderLib/DecLib.cpp
@@ -940,8 +940,9 @@ void DecLib::finishPicture(int &poc, PicList *&rpcListPic, MsgLevel msgl, bool a
m_bFirstSliceInPicture = true; // TODO: immer true? hier ist irgendwas faul
m_maxDecSubPicIdx = 0;
m_maxDecSliceAddrInSubPic = -1;
-
+#if !JVET_AC0196_NNSR
m_pcPic->destroyTempBuffers();
+#endif
m_pcPic->cs->destroyCoeffs();
m_pcPic->cs->releaseIntermediateData();
m_pcPic->cs->picHeader->initPicHeader();
@@ -2189,7 +2190,9 @@ bool DecLib::xDecodeSlice(InputNALUnit &nalu, int &iSkipFrame, int iPOCLastDispl
&& naluTemporalId.m_nalUnitType != NAL_UNIT_EOB)
{
+#if !ADAPTIVE_RPR
CHECK( naluTemporalId.m_temporalId < nalu.m_temporalId, "TemporalId shall be greater than or equal to the TemporalId of the layer access unit containing the NAL unit" );
+#endif
}
}
diff --git a/source/Lib/EncoderLib/EncCfg.h b/source/Lib/EncoderLib/EncCfg.h
index aeac072d708a301d68f01766d4348249efa8eb61..12b010aab9167afa95cfe9db54e639fe78467e15 100644
--- a/source/Lib/EncoderLib/EncCfg.h
+++ b/source/Lib/EncoderLib/EncCfg.h
@@ -175,7 +175,9 @@ protected:
std::string m_rdoCnnlfInterLumaModelNameNNFilter1; ///< inter luma nnlf set1 model
std::string m_rdoCnnlfIntraLumaModelNameNNFilter1; ///< intra luma nnlf set1 model
#endif
-
+#if JVET_AC0196_NNSR
+ bool m_nnsrOption;
+#endif
int m_iFrameRate;
int m_FrameSkip;
uint32_t m_temporalSubsampleRatio;
@@ -909,6 +911,11 @@ public:
void setUseEncNnlfOpt(bool b) { m_encNnlfOpt = b; };
#endif
+#if JVET_AC0196_NNSR
+ bool getUseNnsr() { return m_nnsrOption; };
+ void setUseNnsr(bool b) { m_nnsrOption = b; };
+#endif
+
void setProfile(Profile::Name profile) { m_profile = profile; }
void setLevel(Level::Tier tier, Level::Name level) { m_levelTier = tier; m_level = level; }
bool getFrameOnlyConstraintFlag() const { return m_frameOnlyConstraintFlag; }
diff --git a/source/Lib/EncoderLib/EncGOP.cpp b/source/Lib/EncoderLib/EncGOP.cpp
index a7cb0bbf04f423cb4c02aa18ad74bb774347406d..bcc6e8eadda5701c0a795ec973969c34a2a5bac4 100644
--- a/source/Lib/EncoderLib/EncGOP.cpp
+++ b/source/Lib/EncoderLib/EncGOP.cpp
@@ -381,7 +381,12 @@ int EncGOP::xWritePPS( AccessUnit &accessUnit, const PPS *pps, const int layerId
OutputNALUnit nalu(NAL_UNIT_PPS);
m_HLSWriter->setBitstream( &nalu.m_Bitstream );
nalu.m_nuhLayerId = layerId;
+#if ADAPTIVE_RPR
+ nalu.m_temporalId = accessUnit.temporalId;
+#endif
+#if !ADAPTIVE_RPR
CHECK( nalu.m_temporalId < accessUnit.temporalId, "TemporalId shall be greater than or equal to the TemporalId of the layer access unit containing the NAL unit" );
+#endif
m_HLSWriter->codePPS( pps );
accessUnit.push_back(new NALUnitEBSP(nalu));
return (int)(accessUnit.back()->m_nalUnitData.str().size()) * 8;
@@ -394,7 +399,9 @@ int EncGOP::xWriteAPS( AccessUnit &accessUnit, APS *aps, const int layerId, cons
nalu.m_nuhLayerId = layerId;
nalu.m_temporalId = aps->getTemporalId();
aps->setLayerId( layerId );
+#if !ADAPTIVE_RPR
CHECK( nalu.m_temporalId < accessUnit.temporalId, "TemporalId shall be greater than or equal to the TemporalId of the layer access unit containing the NAL unit" );
+#endif
m_HLSWriter->codeAPS(aps);
accessUnit.push_back(new NALUnitEBSP(nalu));
return (int)(accessUnit.back()->m_nalUnitData.str().size()) * 8;
@@ -3895,8 +3902,9 @@ void EncGOP::compressGOP( int iPOCLast, int iNumPicRcvd, PicList& rcListPic,
{
iGOPid=effFieldIRAPMap.restoreGOPid(iGOPid);
}
-
+#if !JVET_AC0196_NNSR
pcPic->destroyTempBuffers();
+#endif
pcPic->cs->destroyCoeffs();
pcPic->cs->releaseIntermediateData();
} // iGOPid-loop
@@ -4505,7 +4513,23 @@ void EncGOP::xCalculateAddPSNR(Picture* pcPic, PelUnitBuf cPicD, const AccessUni
CU::getRprScaling( &sps, pps, pcPic, xScale, yScale );
std::pair<int, int> scalingRatio = std::pair<int, int>( xScale, yScale );
+#if JVET_AC0196_NNSR
+ PelUnitBuf predBuf = pcPic->getPredBufCustom();
+ int sliceQP = pcPic->cs->slice->getSliceQp();
+ int baseQP = pcPic->cs->pps->getPicInitQPMinus26() + 26;
+ SliceType sliceType = pcPic->cs->slice->getSliceType();
+
+ PelStorage upscaledPic;
+ upscaledPic.create( pic.chromaFormat, Area( Position(), upscaledOrg ) );
+ Picture::rescalePicture( scalingRatio, picC, pcPic->getScalingWindow(), upscaledPic, pps->getScalingWindow(), format, sps.getBitDepths(), false, false, sps.getHorCollocatedChromaFlag(), sps.getVerCollocatedChromaFlag() );
+
+ if (picC.get(COMPONENT_Y).width == upscaledPic.get(COMPONENT_Y).width && picC.get(COMPONENT_Y).height == upscaledPic.get(COMPONENT_Y).height)
+ upscaledRec.copyFrom(upscaledPic);
+ else
+ Picture::rescalePictureCNN(picC, predBuf, upscaledPic, upscaledRec, sliceQP, baseQP, sliceType);
+#else
Picture::rescalePicture( scalingRatio, picC, pcPic->getScalingWindow(), upscaledRec, pps->getScalingWindow(), format, sps.getBitDepths(), false, false, sps.getHorCollocatedChromaFlag(), sps.getVerCollocatedChromaFlag() );
+#endif
}
for (int comp = 0; comp < ::getNumberValidComponents(formatD); comp++)
@@ -4559,7 +4583,31 @@ void EncGOP::xCalculateAddPSNR(Picture* pcPic, PelUnitBuf cPicD, const AccessUni
}
#endif
+#if JVET_AC0196_NNSR
+ if (m_pcEncLib->getUseNnsr() && m_pcEncLib->isResChangeInClvsEnabled())
+ {
+ const CPelBuf& upscaledOrg = (sps.getUseLmcs() || m_pcCfg->getGopBasedTemporalFilterEnabled()) ? pcPic->M_BUFS( 0, PIC_TRUE_ORIGINAL_INPUT).get( compID ) : pcPic->M_BUFS( 0, PIC_ORIGINAL_INPUT).get( compID );
+
+ const uint32_t upscaledWidth = upscaledOrg.width - ( m_pcEncLib->getPad( 0 ) >> ::getComponentScaleX( compID, format ) );
+ const uint32_t upscaledHeight = upscaledOrg.height - ( m_pcEncLib->getPad( 1 ) >> ( !!bPicIsField + ::getComponentScaleY( compID, format ) ) );
+ // create new buffers with correct dimensions
+ const CPelBuf upscaledRecPB( upscaledRec.get( compID ).bufAt( 0, 0 ), upscaledRec.get( compID ).stride, upscaledWidth, upscaledHeight );
+ const CPelBuf upscaledOrgPB( upscaledOrg.bufAt( 0, 0 ), upscaledOrg.stride, upscaledWidth, upscaledHeight );
+
+#if ENABLE_QPA
+ const uint64_t upscaledSSD = xFindDistortionPlane( upscaledRecPB, upscaledOrgPB, useWPSNR ? bitDepth : 0, ::getComponentScaleX( compID, format ) );
+#else
+ const uint64_t scaledSSD = xFindDistortionPlane( upsacledRecPB, upsacledOrgPB, 0 );
+#endif
+
+ upscaledPSNR[comp] = upscaledSSD ? 10.0 * log10( (double)maxval * maxval * upscaledWidth * upscaledHeight / (double)upscaledSSD ) : 999.99;
+ }
+ else
+ {
+ upscaledPSNR[comp] = dPSNR[comp];
+ }
+#else
if (m_pcEncLib->isResChangeInClvsEnabled())
{
const CPelBuf& upscaledOrg = (sps.getUseLmcs() || m_pcCfg->getGopBasedTemporalFilterEnabled()) ? pcPic->M_BUFS( 0, PIC_TRUE_ORIGINAL_INPUT).get( compID ) : pcPic->M_BUFS( 0, PIC_ORIGINAL_INPUT).get( compID );
@@ -4579,6 +4627,7 @@ void EncGOP::xCalculateAddPSNR(Picture* pcPic, PelUnitBuf cPicD, const AccessUni
upscaledPSNR[comp] = upscaledSSD ? 10.0 * log10( (double)maxval * maxval * upscaledWidth * upscaledHeight / (double)upscaledSSD ) : 999.99;
}
+#endif
}
#if EXTENSION_360_VIDEO
@@ -4630,7 +4679,13 @@ void EncGOP::xCalculateAddPSNR(Picture* pcPic, PelUnitBuf cPicD, const AccessUni
m_vRVM_RP.push_back( uibits );
//===== add PSNR =====
- m_gcAnalyzeAll.addResult(dPSNR, (double)uibits, MSEyuvframe, upscaledPSNR, msssim, isEncodeLtRef);
+ m_gcAnalyzeAll.addResult(
+#if JVET_AC0196_NNSR
+ upscaledPSNR,
+#else
+ dPSNR,
+#endif
+ (double)uibits, MSEyuvframe, upscaledPSNR, msssim, isEncodeLtRef);
#if EXTENSION_360_VIDEO
m_ext360.addResult(m_gcAnalyzeAll);
#endif
@@ -4642,7 +4697,13 @@ void EncGOP::xCalculateAddPSNR(Picture* pcPic, PelUnitBuf cPicD, const AccessUni
#endif
if (pcSlice->isIntra())
{
- m_gcAnalyzeI.addResult(dPSNR, (double)uibits, MSEyuvframe, upscaledPSNR, msssim, isEncodeLtRef);
+ m_gcAnalyzeI.addResult(
+#if JVET_AC0196_NNSR
+ upscaledPSNR,
+#else
+ dPSNR,
+#endif
+ (double)uibits, MSEyuvframe, upscaledPSNR, msssim, isEncodeLtRef);
*PSNR_Y = dPSNR[COMPONENT_Y];
#if EXTENSION_360_VIDEO
m_ext360.addResult(m_gcAnalyzeI);
@@ -4656,7 +4717,13 @@ void EncGOP::xCalculateAddPSNR(Picture* pcPic, PelUnitBuf cPicD, const AccessUni
}
if (pcSlice->isInterP())
{
- m_gcAnalyzeP.addResult(dPSNR, (double)uibits, MSEyuvframe, upscaledPSNR, msssim, isEncodeLtRef);
+ m_gcAnalyzeP.addResult(
+#if JVET_AC0196_NNSR
+ upscaledPSNR,
+#else
+ dPSNR,
+#endif
+ (double)uibits, MSEyuvframe, upscaledPSNR, msssim, isEncodeLtRef);
*PSNR_Y = dPSNR[COMPONENT_Y];
#if EXTENSION_360_VIDEO
m_ext360.addResult(m_gcAnalyzeP);
@@ -4670,7 +4737,13 @@ void EncGOP::xCalculateAddPSNR(Picture* pcPic, PelUnitBuf cPicD, const AccessUni
}
if (pcSlice->isInterB())
{
- m_gcAnalyzeB.addResult(dPSNR, (double)uibits, MSEyuvframe, upscaledPSNR, msssim, isEncodeLtRef);
+ m_gcAnalyzeB.addResult(
+#if JVET_AC0196_NNSR
+ upscaledPSNR,
+#else
+ dPSNR,
+#endif
+ (double)uibits, MSEyuvframe, upscaledPSNR, msssim, isEncodeLtRef);
*PSNR_Y = dPSNR[COMPONENT_Y];
#if EXTENSION_360_VIDEO
m_ext360.addResult(m_gcAnalyzeB);
diff --git a/source/Lib/EncoderLib/EncLib.cpp b/source/Lib/EncoderLib/EncLib.cpp
index ed4aba799530eb62deddb8ac9bad12a1660bb5e7..8cca88ca79f083fa85e0e7deb3b0b188d6aa0e90 100644
--- a/source/Lib/EncoderLib/EncLib.cpp
+++ b/source/Lib/EncoderLib/EncLib.cpp
@@ -635,7 +635,11 @@ void EncLib::deletePicBuffer()
m_cListPic.clear();
}
-bool EncLib::encodePrep( bool flush, PelStorage* pcPicYuvOrg, PelStorage* cPicYuvTrueOrg, PelStorage* pcPicYuvFilteredOrg, const InputColourSpaceConversion snrCSC, std::list<PelUnitBuf*>& rcListPicYuvRecOut, int& iNumEncoded )
+bool EncLib::encodePrep( bool flush, PelStorage* pcPicYuvOrg, PelStorage* cPicYuvTrueOrg, PelStorage* pcPicYuvFilteredOrg, const InputColourSpaceConversion snrCSC, std::list<PelUnitBuf*>& rcListPicYuvRecOut, int& iNumEncoded
+#if ADAPTIVE_RPR
+ , PelStorage** ppcPicYuvRPR
+#endif
+)
{
if( m_compositeRefEnabled && m_cGOPEncoder.getPicBg()->getSpliceFull() && m_iPOCLast >= 10 && m_iNumPicRcvd == 0 && m_cGOPEncoder.getEncodedLTRef() == false )
{
@@ -688,6 +692,132 @@ bool EncLib::encodePrep( bool flush, PelStorage* pcPicYuvOrg, PelStorage* cPicYu
}
#endif
+#if ADAPTIVE_RPR
+ const int iPOC = m_iPOCLast + (m_compositeRefEnabled ? 2 : 1);
+ double upscaledPSNR = 0.0, upscaledPSNR1 = 0.0, upscaledPSNR2 = 0.0;
+
+ if (m_resChangeInClvsEnabled && iPOC % getGOPSize() == 0)
+ {
+ // rescale Pelbuffer
+ std::array<std::pair<int, int>, 2> rprScalingRatio = { { { 8192, 8192 },{ 32768, 32768 } } };
+
+ const PPS *pOrgPPS = m_ppsMap.getPS(0);
+ const SPS *pOrgSPS = m_spsMap.getPS(pOrgPPS->getSPSId());
+ const ChromaFormat chFormatIdc = pOrgSPS->getChromaFormatIdc();
+
+ const PPS *pTempPPS = m_ppsMap.getPS(ENC_PPS_ID_RPR);
+ Picture::rescalePicture(rprScalingRatio[1], *pcPicYuvOrg, pOrgPPS->getScalingWindow(), *ppcPicYuvRPR[1], pTempPPS->getScalingWindow(), chFormatIdc, pOrgSPS->getBitDepths(), true, true,
+ pOrgSPS->getHorCollocatedChromaFlag(), pOrgSPS->getVerCollocatedChromaFlag());
+
+ Picture::rescalePicture(rprScalingRatio[0], *ppcPicYuvRPR[1], pOrgPPS->getScalingWindow(), *ppcPicYuvRPR[0], pTempPPS->getScalingWindow(), chFormatIdc, pOrgSPS->getBitDepths(), true, true,
+ pOrgSPS->getHorCollocatedChromaFlag(), pOrgSPS->getVerCollocatedChromaFlag());
+
+ // Calculate PSNR
+ const Pel* pSrc0 = pcPicYuvOrg->get(COMPONENT_Y).bufAt(0, 0);
+ const Pel* pSrc1 = ppcPicYuvRPR[0]->get(COMPONENT_Y).bufAt(0, 0);
+
+ const Pel *pSrc0_U = pcPicYuvOrg->get(COMPONENT_Cb).bufAt(0, 0);
+ const Pel *pSrc1_U = ppcPicYuvRPR[0]->get(COMPONENT_Cb).bufAt(0, 0);
+
+ const Pel *pSrc0_V = pcPicYuvOrg->get(COMPONENT_Cr).bufAt(0, 0);
+ const Pel *pSrc1_V = ppcPicYuvRPR[0]->get(COMPONENT_Cr).bufAt(0, 0);
+
+ uint64_t uiTotalDiff = 0, uiTotalDiff1 = 0, uiTotalDiff2 = 0;
+ for (int y = 0; y < pcPicYuvOrg->get(COMPONENT_Y).height; y++)
+ {
+ for (int x = 0; x < pcPicYuvOrg->get(COMPONENT_Y).width; x++)
+ {
+ Intermediate_Int iTemp = pSrc0[x] - pSrc1[x];
+ uiTotalDiff += uint64_t(iTemp * iTemp);
+ }
+ pSrc0 += pcPicYuvOrg->get(COMPONENT_Y).stride;
+ pSrc1 += ppcPicYuvRPR[0]->get(COMPONENT_Y).stride;
+ }
+
+ for (int y = 0; y < pcPicYuvOrg->get(COMPONENT_Cb).height; y++)
+ {
+ for (int x = 0; x < pcPicYuvOrg->get(COMPONENT_Cb).width; x++)
+ {
+ Intermediate_Int iTemp = pSrc0_U[x] - pSrc1_U[x];
+ uiTotalDiff1 += uint64_t(iTemp * iTemp);
+
+ iTemp = pSrc0_V[x] - pSrc1_V[x];
+ uiTotalDiff2 += uint64_t(iTemp * iTemp);
+ }
+ pSrc0_U += pcPicYuvOrg->get(COMPONENT_Cb).stride;
+ pSrc1_U += ppcPicYuvRPR[0]->get(COMPONENT_Cb).stride;
+ pSrc0_V += pcPicYuvOrg->get(COMPONENT_Cr).stride;
+ pSrc1_V += ppcPicYuvRPR[0]->get(COMPONENT_Cr).stride;
+ }
+
+ const uint32_t maxval = 255 << (pOrgSPS->getBitDepth(CHANNEL_TYPE_LUMA) - 8);
+ upscaledPSNR = uiTotalDiff ? 10.0 * log10((double) maxval * maxval * pOrgPPS->getPicWidthInLumaSamples() * pOrgPPS->getPicHeightInLumaSamples() / (double) uiTotalDiff) : 999.99;
+ upscaledPSNR1 = uiTotalDiff1 ? 10.0 * log10((double) maxval * maxval * (pOrgPPS->getPicWidthInLumaSamples() / 2) * (pOrgPPS->getPicHeightInLumaSamples() / 2) / (double) uiTotalDiff1) : 999.99;
+ upscaledPSNR2 = uiTotalDiff2 ? 10.0 * log10((double) maxval * maxval * (pOrgPPS->getPicWidthInLumaSamples() / 2) * (pOrgPPS->getPicHeightInLumaSamples() / 2) / (double) uiTotalDiff2) : 999.99;
+ }
+
+ if (iPOC % getGOPSize() == 0)
+ {
+ double basePSNR = 45 - (m_gopBasedTemporalFilterEnabled ? 0 : 1);
+ if (m_gopBasedTemporalFilterEnabled) //RA
+ {
+ if (upscaledPSNR1 > 40.0 && upscaledPSNR2 > 40.0)
+ {
+ if (upscaledPSNR > 44.0)
+ {
+ ppsID = ENC_PPS_ID_RPR;
+ }
+ else
+ {
+ if ((basePSNR - (m_iQP - 32) * 0.5) < upscaledPSNR)
+ {
+ ppsID = ENC_PPS_ID_RPR;
+ }
+ else
+ {
+ ppsID = 0;
+ }
+ }
+ }
+ else
+ {
+ if ((basePSNR - (m_iQP - 32) * 0.5) < upscaledPSNR)
+ {
+ ppsID = ENC_PPS_ID_RPR;
+ }
+ else
+ {
+ ppsID = 0;
+ }
+ }
+ }
+ else //AI
+ {
+ if (upscaledPSNR1 > 40.0 && upscaledPSNR2 > 40.0)
+ {
+ ppsID = ENC_PPS_ID_RPR;
+ }
+ else
+ {
+ if ((basePSNR - (m_iQP - 32) * 0.5) < upscaledPSNR)
+ {
+ ppsID = ENC_PPS_ID_RPR;
+ }
+ else
+ {
+ ppsID = 0;
+ }
+ }
+ }
+ m_iGOPRprPpsId = ppsID;
+ }
+ else
+ {
+ ppsID = m_iGOPRprPpsId;
+ }
+#elif JVET_AC0196_NNSR
+ ppsID = ENC_PPS_ID_RPR;
+#else
if( m_resChangeInClvsEnabled && m_intraPeriod == -1 )
{
const int poc = m_iPOCLast + ( m_compositeRefEnabled ? 2 : 1 );
@@ -701,7 +831,7 @@ bool EncLib::encodePrep( bool flush, PelStorage* pcPicYuvOrg, PelStorage* cPicYu
ppsID = 0;
}
}
-
+#endif
if( m_vps->getMaxLayers() > 1 )
{
ppsID = m_vps->getGeneralLayerIdx( m_layerId );
@@ -2406,6 +2536,14 @@ int EncCfg::getQPForPicture(const uint32_t gopIndex, const Slice *pSlice) const
}
#endif
+#if ADAPTIVE_RPR
+ if (pSlice->getPPS()->getPPSId() == ENC_PPS_ID_RPR)
+ {
+ // adjust QP for rate matching.
+ qp -= 0;
+ }
+#endif
+
if(sliceType==I_SLICE)
{
qp += getIntraQPOffset();
diff --git a/source/Lib/EncoderLib/EncLib.h b/source/Lib/EncoderLib/EncLib.h
index 52fb45d76bde60150d0453e0e43706589b639ac9..5622d8e7b68f22a8273061dfb869156183e4e0e0 100644
--- a/source/Lib/EncoderLib/EncLib.h
+++ b/source/Lib/EncoderLib/EncLib.h
@@ -181,7 +181,10 @@ public:
APS** getApss() { return m_apss; }
Ctx m_entropyCodingSyncContextState; ///< leave in addition to vector for compatibility
PLTBuf m_palettePredictorSyncState;
-
+#if ADAPTIVE_RPR
+ int m_iGOPRprPpsId;
+#endif
+
protected:
void xGetNewPicBuffer ( std::list<PelUnitBuf*>& rcListPicYuvRecOut, Picture*& rpcPic, int ppsId ); ///< get picture buffer which will be processed. If ppsId<0, then the ppsMap will be queried for the first match.
#if JVET_S0163_ON_TARGETOLS_SUBLAYERS
@@ -299,7 +302,11 @@ public:
PelStorage* pcPicYuvFilteredOrg,
const InputColourSpaceConversion snrCSC, // used for SNR calculations. Picture in original colour space.
std::list<PelUnitBuf*>& rcListPicYuvRecOut,
- int& iNumEncoded );
+ int& iNumEncoded
+#if ADAPTIVE_RPR
+ , PelStorage** ppcPicYuvRPR
+#endif
+ );
bool encode( const InputColourSpaceConversion snrCSC, // used for SNR calculations. Picture in original colour space.
std::list<PelUnitBuf*>& rcListPicYuvRecOut,
diff --git a/source/Lib/Utilities/VideoIOYuv.cpp b/source/Lib/Utilities/VideoIOYuv.cpp
index cc26a7d61570a1832d9aba6d10bf0cad095abcec..e0e0519210f69c30323143bd62a3ee1dc5238755 100644
--- a/source/Lib/Utilities/VideoIOYuv.cpp
+++ b/source/Lib/Utilities/VideoIOYuv.cpp
@@ -1299,7 +1299,12 @@ void VideoIOYuv::ColourSpaceConvert(const CPelUnitBuf &src, PelUnitBuf &dest, co
}
}
-bool VideoIOYuv::writeUpscaledPicture( const SPS& sps, const PPS& pps, const CPelUnitBuf& pic, const InputColourSpaceConversion ipCSC, const bool bPackedYUVOutputMode, int outputChoice, ChromaFormat format, const bool bClipToRec709 )
+bool VideoIOYuv::writeUpscaledPicture( const SPS& sps, const PPS& pps, const CPelUnitBuf& pic, const InputColourSpaceConversion ipCSC, const bool bPackedYUVOutputMode,
+#if JVET_AC0196_NNSR
+ const PelUnitBuf& predBuf, int sliceQP, int baseQP, SliceType sliceType,
+#endif
+ int outputChoice, ChromaFormat format, const bool bClipToRec709
+)
{
ChromaFormat chromaFormatIDC = sps.getChromaFormatIdc();
bool ret = false;
@@ -1327,12 +1332,28 @@ bool VideoIOYuv::writeUpscaledPicture( const SPS& sps, const PPS& pps, const CPe
const Window& beforeScalingWindow = pps.getScalingWindow();
int refPicWidth = pps.getPicWidthInLumaSamples() - SPS::getWinUnitX( sps.getChromaFormatIdc() ) * ( beforeScalingWindow.getWindowLeftOffset() + beforeScalingWindow.getWindowRightOffset() );
int refPicHeight = pps.getPicHeightInLumaSamples() - SPS::getWinUnitY( sps.getChromaFormatIdc() ) * ( beforeScalingWindow.getWindowTopOffset() + beforeScalingWindow.getWindowBottomOffset() );
+#if ADAPTIVE_RPR
+ if (refPicWidth == curPicWidth && refPicHeight == curPicHeight)
+ {
+ refPicWidth = curPicWidth >> 1;
+ refPicHeight = curPicHeight >> 1;
+ }
+#endif
int xScale = ( ( refPicWidth << SCALE_RATIO_BITS ) + ( curPicWidth >> 1 ) ) / curPicWidth;
int yScale = ( ( refPicHeight << SCALE_RATIO_BITS ) + ( curPicHeight >> 1 ) ) / curPicHeight;
+#if JVET_AC0196_NNSR
+ PelStorage upscaledRPR;
+ upscaledRPR.create(chromaFormatIDC, Area( Position(), Size( sps.getMaxPicWidthInLumaSamples(), sps.getMaxPicHeightInLumaSamples() ) ));
+ Picture::rescalePicture(std::pair<int, int>(xScale, yScale), pic, pps.getScalingWindow(), upscaledRPR,
+ afterScaleWindowFullResolution, chromaFormatIDC, sps.getBitDepths(), false, false,
+ sps.getHorCollocatedChromaFlag(), sps.getVerCollocatedChromaFlag());
+ Picture::rescalePictureCNN(pic, predBuf, upscaledRPR, upscaledPic, sliceQP, baseQP, sliceType);
+#else
Picture::rescalePicture( std::pair<int, int>( xScale, yScale ), pic, pps.getScalingWindow(), upscaledPic, afterScaleWindowFullResolution, chromaFormatIDC, sps.getBitDepths(), false, false, sps.getHorCollocatedChromaFlag(), sps.getVerCollocatedChromaFlag() );
-
+#endif
+
ret = write( sps.getMaxPicWidthInLumaSamples(), sps.getMaxPicHeightInLumaSamples(), upscaledPic,
ipCSC,
bPackedYUVOutputMode,
diff --git a/source/Lib/Utilities/VideoIOYuv.h b/source/Lib/Utilities/VideoIOYuv.h
index 4140383623811b9ab61bf6b6d2752bd618f0a788..932e1aca972889e6788276eb5b2a377d62a13adc 100644
--- a/source/Lib/Utilities/VideoIOYuv.h
+++ b/source/Lib/Utilities/VideoIOYuv.h
@@ -105,8 +105,12 @@ public:
int getBitdepthShift( int ch ) { return m_bitdepthShift[ch]; }
int getFileBitdepth( int ch ) { return m_fileBitdepth[ch]; }
- bool writeUpscaledPicture( const SPS& sps, const PPS& pps, const CPelUnitBuf& pic,
- const InputColourSpaceConversion ipCSC, const bool bPackedYUVOutputMode, int outputChoice = 0, ChromaFormat format = NUM_CHROMA_FORMAT, const bool bClipToRec709 = false ); ///< write one upsaled YUV frame
+ bool writeUpscaledPicture( const SPS& sps, const PPS& pps, const CPelUnitBuf& pic, const InputColourSpaceConversion ipCSC, const bool bPackedYUVOutputMode,
+#if JVET_AC0196_NNSR
+ const PelUnitBuf& predBuf, int sliceQP, int baseQP, SliceType sliceType,
+#endif
+ int outputChoice = 0, ChromaFormat format = NUM_CHROMA_FORMAT, const bool bClipToRec709 = false
+ ); ///< write one upsaled YUV frame
};