diff --git a/doc/software-manual.tex b/doc/software-manual.tex index 70fef7c8b7e604ba615e73179ee54dc788c64bd3..43f6deade9d68bcaf556623511373f8d497439cd 100644 --- a/doc/software-manual.tex +++ b/doc/software-manual.tex @@ -2175,6 +2175,11 @@ For Cb component with BT.2020 container use 1.14; for BT.709 material and 1.04 f For Cr component with BT.2020 container use 1.79; for BT.709 material and 1.39 for P3 material. \\ +\Option{BIM} & +\Default{false} & +Enable or disable Block Importance Mapping, QP adaptation depending on estimated propagation of reference samples. Depends on future and past reference frames configured for temporal filter. +\\ + \Option{SliceChromaQPOffsetPeriodicity} & \Default{0} & Defines the periodicity for inter slices that use the slice-level chroma QP offsets, as defined by SliceCbQpOffsetIntraOrPeriodic and SliceCrQpOffsetIntraOrPeriodic. A value of 0 disables the periodicity. It is intended to be used in low-delay configurations where an regular intra period is not defined. diff --git a/source/App/EncoderApp/EncApp.cpp b/source/App/EncoderApp/EncApp.cpp index c6eb420d10ae21088628d5284858ee83670ab01b..c95793a6c767b5cbfaac7f26435a1b3625506acd 100644 --- a/source/App/EncoderApp/EncApp.cpp +++ b/source/App/EncoderApp/EncApp.cpp @@ -1223,6 +1223,9 @@ void EncApp::xInitLibCfg() m_cEncLib.setCalculateHdrMetrics (m_calculateHdrMetrics); #endif m_cEncLib.setGopBasedTemporalFilterEnabled(m_gopBasedTemporalFilterEnabled); +#if JVET_Y0240_BIM + m_cEncLib.setBIM (m_bimEnabled); +#endif m_cEncLib.setNumRefLayers ( m_numRefLayers ); m_cEncLib.setVPSParameters(m_cfgVPSParameters); @@ -1313,11 +1316,22 @@ void EncApp::createLib( const int layerIdx ) m_orgPic->create( unitArea ); m_trueOrgPic->create( unitArea ); - if( m_gopBasedTemporalFilterEnabled ) +#if JVET_Y0240_BIM + if ( m_gopBasedTemporalFilterEnabled || m_bimEnabled ) +#else + if ( m_gopBasedTemporalFilterEnabled ) +#endif { m_filteredOrgPic = new PelStorage; m_filteredOrgPic->create( unitArea ); } +#if JVET_Y0240_BIM + if ( m_cEncLib.getBIM() ) + { + std::map<int, int*> adaptQPmap; + m_cEncLib.setAdaptQPmap(adaptQPmap); + } +#endif if( !m_bitstream.is_open() ) { @@ -1340,13 +1354,21 @@ void EncApp::createLib( const int layerIdx ) m_ext360 = new TExt360AppEncTop( *this, m_cEncLib.getGOPEncoder()->getExt360Data(), *( m_cEncLib.getGOPEncoder() ), *m_orgPic ); #endif +#if JVET_Y0240_BIM + if( m_gopBasedTemporalFilterEnabled || m_bimEnabled ) +#else if( m_gopBasedTemporalFilterEnabled ) +#endif { m_temporalFilter.init( m_FrameSkip, m_inputBitDepth, m_MSBExtendedBitDepth, m_internalBitDepth, m_iSourceWidth, sourceHeight, m_aiPad, m_bClipInputVideoToRec709Range, m_inputFileName, m_chromaFormatIDC, m_inputColourSpaceConvert, m_iQP, m_gopBasedTemporalFilterStrengths, m_gopBasedTemporalFilterPastRefs, m_gopBasedTemporalFilterFutureRefs, m_firstValidFrame, - m_lastValidFrame ); + m_lastValidFrame +#if JVET_Y0240_BIM + , m_gopBasedTemporalFilterEnabled, m_cEncLib.getAdaptQPmap(), m_cEncLib.getBIM(), m_uiCTUSize +#endif + ); } } @@ -1377,12 +1399,26 @@ void EncApp::destroyLib() delete m_trueOrgPic; delete m_orgPic; - if( m_gopBasedTemporalFilterEnabled ) +#if JVET_Y0240_BIM + if ( m_gopBasedTemporalFilterEnabled || m_bimEnabled ) +#else + if ( m_gopBasedTemporalFilterEnabled ) +#endif { m_filteredOrgPic->destroy(); delete m_filteredOrgPic; } - +#if JVET_Y0240_BIM + if ( m_bimEnabled ) + { + auto map = m_cEncLib.getAdaptQPmap(); + for (auto it = map->begin(); it != map->end(); ++it) + { + int *p = it->second; + delete p; + } + } +#endif #if EXTENSION_360_VIDEO delete m_ext360; #endif @@ -1410,7 +1446,11 @@ bool EncApp::encodePrep( bool& eos ) m_cVideoIOYuvInputFile.read( *m_orgPic, *m_trueOrgPic, ipCSC, m_aiPad, m_InputChromaFormatIDC, m_bClipInputVideoToRec709Range ); #endif - if( m_gopBasedTemporalFilterEnabled ) +#if JVET_Y0240_BIM + if ( m_gopBasedTemporalFilterEnabled || m_bimEnabled ) +#else + if ( m_gopBasedTemporalFilterEnabled ) +#endif { m_temporalFilter.filter( m_orgPic, m_iFrameRcvd ); m_filteredOrgPic->copyFrom( *m_orgPic ); diff --git a/source/App/EncoderApp/EncAppCfg.cpp b/source/App/EncoderApp/EncAppCfg.cpp index ebb947ab0c8ef89f191e928233d6c1f1cab113f8..2ee1f10a2ac9a3d118c4d53c51e76feba8c283ac 100644 --- a/source/App/EncoderApp/EncAppCfg.cpp +++ b/source/App/EncoderApp/EncAppCfg.cpp @@ -1218,6 +1218,9 @@ bool EncAppCfg::parseCfg( int argc, char* argv[] ) ("WCGPPSChromaQpScale", m_wcgChromaQpControl.chromaQpScale, 0.0, "WCG PPS Chroma QP Scale") ("WCGPPSChromaQpOffset", m_wcgChromaQpControl.chromaQpOffset, 0.0, "WCG PPS Chroma QP Offset") #endif +#if JVET_Y0240_BIM + ("BIM", m_bimEnabled, false, "Block Importance Mapping QP adaptation depending on estimated propagation of reference samples.") +#endif #if W0038_CQP_ADJ ("SliceChromaQPOffsetPeriodicity", m_sliceChromaQpOffsetPeriodicity, 0u, "Used in conjunction with Slice Cb/Cr QpOffsetIntraOrPeriodic. Use 0 (default) to disable periodic nature.") ("SliceCbQpOffsetIntraOrPeriodic", m_sliceChromaQpOffsetIntraOrPeriodic[0], 0, "Chroma Cb QP Offset at slice level for I slice or for periodic inter slices as defined by SliceChromaQPOffsetPeriodicity. Replaces offset in the GOP table.") @@ -4143,6 +4146,15 @@ bool EncAppCfg::xCheckParameter() msg( WARNING, "Number of frames used for temporal prefilter is different from default.\n" ); } } +#if JVET_Y0240_BIM + if (m_bimEnabled) + { + xConfirmPara(m_temporalSubsampleRatio != 1, "Block Importance Mapping only support Temporal sub-sample ratio 1"); + xConfirmPara( + m_gopBasedTemporalFilterPastRefs <= 0 && m_gopBasedTemporalFilterFutureRefs <= 0, + "Either TemporalFilterPastRefs or TemporalFilterFutureRefs must be larger than 0 when Block Importance Mapping is enabled" ); + } +#endif #if EXTENSION_360_VIDEO check_failed |= m_ext360.verifyParameters(); #endif @@ -4575,6 +4587,9 @@ void EncAppCfg::xPrintParameter() msg( VERBOSE, "RPR:%d ", 0 ); } msg( VERBOSE, "TemporalFilter:%d/%d ", m_gopBasedTemporalFilterPastRefs, m_gopBasedTemporalFilterFutureRefs ); +#if JVET_Y0240_BIM + msg(VERBOSE, "BIM:%d ", m_bimEnabled); +#endif #if EXTENSION_360_VIDEO m_ext360.outputConfigurationSummary(); #endif diff --git a/source/App/EncoderApp/EncAppCfg.h b/source/App/EncoderApp/EncAppCfg.h index cc44ccc1f01df2c524ff85475fd9b1033b20bf13..e92788b9f07f0a8291cf2434cc20a7bd44870190 100644 --- a/source/App/EncoderApp/EncAppCfg.h +++ b/source/App/EncoderApp/EncAppCfg.h @@ -839,6 +839,9 @@ protected: int m_gopBasedTemporalFilterPastRefs; int m_gopBasedTemporalFilterFutureRefs; ///< Enable/disable future frame references in the GOP-based Temporal Filter std::map<int, double> m_gopBasedTemporalFilterStrengths; ///< Filter strength per frame for the GOP-based Temporal Filter +#if JVET_Y0240_BIM + bool m_bimEnabled; +#endif int m_maxLayers; int m_targetOlsIdx; diff --git a/source/Lib/CommonLib/TypeDef.h b/source/Lib/CommonLib/TypeDef.h index e26c7168f8659adcdd736cca7b9381aa77b84465..d1b772d315b58a2a131cea94a931a7c058942ed6 100644 --- a/source/Lib/CommonLib/TypeDef.h +++ b/source/Lib/CommonLib/TypeDef.h @@ -61,6 +61,9 @@ #if BASE_ENCODER + +#define JVET_Y0240_BIM 1 // JVET_Y0240: Block Importance Mapping + #define JVET_X0144_MAX_MTT_DEPTH_TID 1 // JVET-X0144: max MTT hierarchy depth set by temporal ID // Lossy encoder speedups #define AFFINE_ENC_OPT 1 // Affine encoder optimization diff --git a/source/Lib/EncoderLib/EncCfg.h b/source/Lib/EncoderLib/EncCfg.h index ee30a6723c622f436eabc1af2cc024812886400d..be682da0d4e1d31a25acd2e8583795a71b5b9d43 100644 --- a/source/Lib/EncoderLib/EncCfg.h +++ b/source/Lib/EncoderLib/EncCfg.h @@ -590,6 +590,10 @@ protected: bool m_bFastMEForGenBLowDelayEnabled; bool m_bUseBLambdaForNonKeyLowDelayPictures; bool m_gopBasedTemporalFilterEnabled; +#if JVET_Y0240_BIM + bool m_bimEnabled; + std::map<int, int*> m_adaptQPmap; +#endif bool m_noPicPartitionFlag; ///< no picture partitioning flag (single tile, single slice) bool m_mixedLossyLossless; ///< enable mixed lossy/lossless coding std::vector<uint16_t> m_sliceLosslessArray; ///< Slice lossless array @@ -1683,6 +1687,13 @@ public: void setGopBasedTemporalFilterEnabled( const bool b ) { m_gopBasedTemporalFilterEnabled = b; } bool getGopBasedTemporalFilterEnabled() const { return m_gopBasedTemporalFilterEnabled; } +#if JVET_Y0240_BIM + void setBIM (bool flag) { m_bimEnabled = flag; } + bool getBIM () { return m_bimEnabled; } + void setAdaptQPmap (std::map<int, int*> map) { m_adaptQPmap = map; } + int* getAdaptQPmap (int poc) { return m_adaptQPmap[poc]; } + std::map<int, int*> *getAdaptQPmap () { return &m_adaptQPmap; } +#endif bool getUseReconBasedCrossCPredictionEstimate () const { return m_reconBasedCrossCPredictionEstimate; } diff --git a/source/Lib/EncoderLib/EncCu.cpp b/source/Lib/EncoderLib/EncCu.cpp index 50a07645ac31a0cb6f2261803f03a186c89657ac..7faef658f17309186ec1b6fba86d34f5a23c2045 100644 --- a/source/Lib/EncoderLib/EncCu.cpp +++ b/source/Lib/EncoderLib/EncCu.cpp @@ -428,6 +428,9 @@ void EncCu::init( EncLib* pcEncLib, const SPS& sps PARL_PARAM( const int tId ) ) DecCu::init( m_pcTrQuant, m_pcIntraSearch, m_pcInterSearch ); m_modeCtrl->init( m_pcEncCfg, m_pcRateCtrl, m_pcRdCost ); +#if JVET_Y0240_BIM + m_modeCtrl->setBIMQPMap( m_pcEncCfg->getAdaptQPmap() ); +#endif m_pcInterSearch->setModeCtrl( m_modeCtrl ); m_modeCtrl->setInterSearch(m_pcInterSearch); @@ -944,6 +947,9 @@ void EncCu::xCompressCU( CodingStructure*& tempCS, CodingStructure*& bestCS, Par #if SHARP_LUMA_DELTA_QP || ENABLE_QPA_SUB_CTU if (partitioner.currQgEnable() && ( +#if JVET_Y0240_BIM + (m_pcEncCfg->getBIM()) || +#endif #if SHARP_LUMA_DELTA_QP (m_pcEncCfg->getLumaLevelToDeltaQPMapping().isEnabled()) || #endif diff --git a/source/Lib/EncoderLib/EncLib.cpp b/source/Lib/EncoderLib/EncLib.cpp index 02ab1a90fe8dcbd08f625d68c26a3b3bc123f6f5..d5509c4ad1a1a6a408b699e95b762f221f8e1668 100644 --- a/source/Lib/EncoderLib/EncLib.cpp +++ b/source/Lib/EncoderLib/EncLib.cpp @@ -1798,6 +1798,12 @@ void EncLib::xInitPPS(PPS &pps, const SPS &sps) bUseDQP = (getBaseQP() < 38) && (getSourceWidth() > 512 || getSourceHeight() > 320); } #endif +#if JVET_Y0240_BIM + if (m_bimEnabled) + { + bUseDQP = true; + } +#endif if (m_costMode==COST_SEQUENCE_LEVEL_LOSSLESS || m_costMode==COST_LOSSLESS_CODING) { diff --git a/source/Lib/EncoderLib/EncModeCtrl.cpp b/source/Lib/EncoderLib/EncModeCtrl.cpp index e12837d52c666d5b0f8524d7729039b61b6c3f6a..e1205e2bc9a8a98148d43d1ab2accfd2fb08bad8 100644 --- a/source/Lib/EncoderLib/EncModeCtrl.cpp +++ b/source/Lib/EncoderLib/EncModeCtrl.cpp @@ -151,6 +151,14 @@ void EncModeCtrl::xGetMinMaxQP( int& minQP, int& maxQP, const CodingStructure& c int deltaQP = m_pcEncCfg->getMaxDeltaQP(); minQP = Clip3( -sps.getQpBDOffset( CHANNEL_TYPE_LUMA ), MAX_QP, baseQP - deltaQP ); maxQP = Clip3( -sps.getQpBDOffset( CHANNEL_TYPE_LUMA ), MAX_QP, baseQP + deltaQP ); +#if JVET_Y0240_BIM + Position pos = partitioner.currQgPos; + const int ctuSize = sps.getCTUSize(); + const int ctuId = ( pos.y / ctuSize ) * ( ( cs.picture->lwidth() + ctuSize - 1 ) / ctuSize ) + ( pos.x / ctuSize ); + const int bimOffset = getBIMOffset( m_slice->getPOC(), ctuId ); + minQP += bimOffset; + maxQP += bimOffset; +#endif } else if( qgEnableChildren ) // more splits and not the deepest QG level { diff --git a/source/Lib/EncoderLib/EncModeCtrl.h b/source/Lib/EncoderLib/EncModeCtrl.h index 05cdfa8f4d40c7e1d0779f17dfb4e8e9332280e1..0b4a02eccf14e0d69385e66c5c6a07a095017cec 100644 --- a/source/Lib/EncoderLib/EncModeCtrl.h +++ b/source/Lib/EncoderLib/EncModeCtrl.h @@ -323,6 +323,9 @@ protected: #if SHARP_LUMA_DELTA_QP int m_lumaLevelToDeltaQPLUT[LUMA_LEVEL_TO_DQP_LUT_MAXSIZE]; int m_lumaQPOffset; +#endif +#if JVET_Y0240_BIM + std::map<int, int*> *m_bimQPMap; #endif bool m_fastDeltaQP; static_vector<ComprCUCtx, ( MAX_CU_DEPTH << 2 )> m_ComprCUCtxList; @@ -419,6 +422,14 @@ public: void setInterSearch (InterSearch* pcInterSearch) { m_pcInterSearch = pcInterSearch; } void setPltEnc ( bool b ) { m_doPlt = b; } bool getPltEnc() const { return m_doPlt; } +#if JVET_Y0240_BIM + void setBIMQPMap ( std::map<int, int*> *qpMap ) { m_bimQPMap = qpMap; } + int getBIMOffset ( int poc, int ctuId ) + { + auto it = m_bimQPMap->find(poc); + return (it == m_bimQPMap->end()) ? 0 : (*m_bimQPMap)[poc][ctuId]; + } +#endif protected: void xExtractFeatures ( const EncTestMode encTestmode, CodingStructure& cs ); diff --git a/source/Lib/EncoderLib/EncTemporalFilter.cpp b/source/Lib/EncoderLib/EncTemporalFilter.cpp index 930ef1b21c250daee19a494d2597c600c90806ed..bfa809eecbdec3e58355b0c40a5c70f0a95e52aa 100644 --- a/source/Lib/EncoderLib/EncTemporalFilter.cpp +++ b/source/Lib/EncoderLib/EncTemporalFilter.cpp @@ -76,6 +76,11 @@ const double EncTemporalFilter::m_refStrengths[2][4] = { 1.13, 0.97, 0.81, 0.57 }, // low delay }; +#if JVET_Y0240_BIM +const int EncTemporalFilter::m_cuTreeThresh[4] = + { 75, 60, 30, 15 }; +#endif + EncTemporalFilter::EncTemporalFilter() : m_FrameSkip(0), m_chromaFormatIDC(NUM_CHROMA_FORMAT), @@ -92,7 +97,11 @@ void EncTemporalFilter::init( const int frameSkip, const int inputBitDepth[MAX_N const int *pad, const bool rec709, const std::string &filename, const ChromaFormat inputChromaFormatIDC, const InputColourSpaceConversion colorSpaceConv, const int qp, const std::map<int, double> &temporalFilterStrengths, const int pastRefs, - const int futureRefs, const int firstValidFrame, const int lastValidFrame ) + const int futureRefs, const int firstValidFrame, const int lastValidFrame +#if JVET_Y0240_BIM + , const bool mctfEnabled, std::map<int, int*> *adaptQPmap, const bool bimEnabled, const int ctuSize +#endif + ) { m_FrameSkip = frameSkip; for (int i = 0; i < MAX_NUM_CHANNEL_TYPE; i++) @@ -120,6 +129,13 @@ void EncTemporalFilter::init( const int frameSkip, const int inputBitDepth[MAX_N m_futureRefs = futureRefs; m_firstValidFrame = firstValidFrame; m_lastValidFrame = lastValidFrame; +#if JVET_Y0240_BIM + m_mctfEnabled = mctfEnabled; + m_bimEnabled = bimEnabled; + m_numCtu = ((width + ctuSize - 1) / ctuSize) * ((height + ctuSize - 1) / ctuSize); + m_ctuSize = ctuSize; + m_ctuAdaptedQP = adaptQPmap; +#endif } // ==================================================================================================================== @@ -207,11 +223,85 @@ bool EncTemporalFilter::filter(PelStorage *orgPic, int receivedPoc) overallStrength = strength; } } +#if JVET_Y0240_BIM + const int numRefs = int(srcFrameInfo.size()); + if ( m_bimEnabled && ( numRefs > 0 ) ) + { + const int bimFirstFrame = std::max(currentFilePoc - 2, firstFrame); + const int bimLastFrame = std::min(currentFilePoc + 2, lastFrame); + std::vector<double> sumError(m_numCtu * 2, 0); + std::vector<int> blkCount(m_numCtu * 2, 0); - bilateralFilter(origPadded, srcFrameInfo, newOrgPic, overallStrength); + int frameIndex = bimFirstFrame - firstFrame; - // move filtered to orgPic - orgPic->copyFrom(newOrgPic); + int distFactor[2] = {3,3}; + + int* qpMap = new int[m_numCtu]; + for (int poc = bimFirstFrame; poc <= bimLastFrame; poc++) + { + if ((poc < 0) || (poc == currentFilePoc)) + { + continue; // frame not available or frame that is being filtered + } + int dist = abs(poc - currentFilePoc) - 1; + distFactor[dist]--; + TemporalFilterSourcePicInfo &srcPic = srcFrameInfo.at(frameIndex); + for (int y = 0; y < srcPic.mvs.h() / 2; y++) // going over in 8x8 block steps + { + for (int x = 0; x < srcPic.mvs.w() / 2; x++) + { + int blocksPerRow = (srcPic.mvs.w() / 2 + (m_ctuSize / 8 - 1)) / (m_ctuSize / 8); + int ctuX = x / (m_ctuSize / 8); + int ctuY = y / (m_ctuSize / 8); + int ctuId = ctuY * blocksPerRow + ctuX; + sumError[dist * m_numCtu + ctuId] += srcPic.mvs.get(x, y).error; + blkCount[dist * m_numCtu + ctuId] += 1; + } + } + frameIndex++; + } + double weight = (receivedPoc % 16) ? 0.6 : 1; + const double center = 45.0; + for (int i = 0; i < m_numCtu; i++) + { + int avgErrD1 = (int)((sumError[i] / blkCount[i]) * distFactor[0]); + int avgErrD2 = (int)((sumError[i + m_numCtu] / blkCount[i + m_numCtu]) * distFactor[1]); + int weightedErr = std::max(avgErrD1, avgErrD2) + abs(avgErrD2 - avgErrD1) * 3; + weightedErr = (int)(weightedErr * weight + (1 - weight) * center); + if (weightedErr > m_cuTreeThresh[0]) + { + qpMap[i] = 2; + } + else if (weightedErr > m_cuTreeThresh[1]) + { + qpMap[i] = 1; + } + else if (weightedErr < m_cuTreeThresh[3]) + { + qpMap[i] = -2; + } + else if (weightedErr < m_cuTreeThresh[2]) + { + qpMap[i] = -1; + } + else + { + qpMap[i] = 0; + } + } + m_ctuAdaptedQP->insert({ receivedPoc, qpMap }); + } +#endif + +#if JVET_Y0240_BIM + if ( m_mctfEnabled && ( numRefs > 0 ) ) +#endif + { + bilateralFilter(origPadded, srcFrameInfo, newOrgPic, overallStrength); + + // move filtered to orgPic + orgPic->copyFrom(newOrgPic); + } yuvFrames.close(); return true; diff --git a/source/Lib/EncoderLib/EncTemporalFilter.h b/source/Lib/EncoderLib/EncTemporalFilter.h index d38dde017da12a270d16c1d5580f9c2e7fbcbb16..d270747c733703030a8b9253f1e9eed5c7e908af 100644 --- a/source/Lib/EncoderLib/EncTemporalFilter.h +++ b/source/Lib/EncoderLib/EncTemporalFilter.h @@ -70,6 +70,11 @@ public: Array2D() : m_width(0), m_height(0), v() { } Array2D(int width, int height, const T& value=T()) : m_width(0), m_height(0), v() { allocate(width, height, value); } +#if JVET_Y0240_BIM + int w() const { return m_width; } + int h() const { return m_height; } +#endif + void allocate(int width, int height, const T& value=T()) { m_width = width; @@ -113,7 +118,11 @@ public: const int width, const int height, const int *pad, const bool rec709, const std::string &filename, const ChromaFormat inputChroma, const InputColourSpaceConversion colorSpaceConv, const int qp, const std::map<int, double> &temporalFilterStrengths, const int pastRefs, const int futureRefs, - const int firstValidFrame, const int lastValidFrame ); + const int firstValidFrame, const int lastValidFrame +#if JVET_Y0240_BIM + , const bool bMCTFenabled, std::map<int, int*> *adaptQPmap, const bool bBIMenabled, const int ctuSize +#endif + ); bool filter(PelStorage *orgPic, int frame); @@ -126,6 +135,9 @@ private: static const int m_padding; static const int m_interpolationFilter[16][8]; static const double m_refStrengths[2][4]; +#if JVET_Y0240_BIM + static const int m_cuTreeThresh[4]; +#endif // Private member variables int m_FrameSkip; @@ -142,6 +154,13 @@ private: bool m_clipInputVideoToRec709Range; InputColourSpaceConversion m_inputColourSpaceConvert; Area m_area; +#if JVET_Y0240_BIM + bool m_mctfEnabled; + bool m_bimEnabled; + int m_numCtu; + int m_ctuSize; + std::map<int, int*> *m_ctuAdaptedQP; +#endif int m_pastRefs; int m_futureRefs;