Commit e621c4a4 authored by Santiago de Luxán Hernández's avatar Santiago de Luxán Hernández
Browse files

Integration of JVET-0502

parent 9937d231
......@@ -133,7 +133,7 @@ MIP : 1
# Fast tools
PBIntraFast : 1
ISPFast : 1
ISPFast : 0
FastMrg : 1
AMaxBT : 1
FastMIP : 0
......
......@@ -138,7 +138,7 @@ PROF : 1
# Fast tools
PBIntraFast : 1
ISPFast : 1
ISPFast : 0
FastMrg : 1
AMaxBT : 1
FastMIP : 0
......
......@@ -156,7 +156,7 @@ PROF : 1
# Fast tools
PBIntraFast : 1
ISPFast : 1
ISPFast : 0
FastMrg : 1
AMaxBT : 1
FastMIP : 0
......
......@@ -328,8 +328,12 @@ void IntraPrediction::predIntraAng( const ComponentID compId, PelBuf &piPred, co
const int srcStride = m_topRefLength + 1 + (whRatio + 1) * multiRefIdx;
const int srcHStride = m_leftRefLength + 1 + (hwRatio + 1) * multiRefIdx;
#endif
#if JVET_O0502_ISP_CLEANUP
const CPelBuf& srcBuf = pu.cu->ispMode && isLuma(compID) ? getISPBuffer() : CPelBuf(getPredictorPtr(compID), srcStride, srcHStride);
#else
const CPelBuf & srcBuf = CPelBuf(getPredictorPtr(compID), srcStride, srcHStride);
#endif
const ClpRng& clpRng(pu.cu->cs->slice->clpRng(compID));
switch (uiDirMode)
......@@ -519,7 +523,11 @@ void IntraPrediction::initPredIntraParams(const PredictionUnit & pu, const CompA
m_ipaParam.multiRefIndex = isLuma (chType) ? pu.multiRefIdx : 0 ;
m_ipaParam.refFilterFlag = false;
m_ipaParam.interpolationFlag = false;
#if JVET_O0502_ISP_CLEANUP
m_ipaParam.applyPDPC = ((puSize.width >= MIN_TB_SIZEY && puSize.height >= MIN_TB_SIZEY) || !isLuma(compId)) && m_ipaParam.multiRefIndex == 0;
#else
m_ipaParam.applyPDPC = !useISP && m_ipaParam.multiRefIndex == 0;
#endif
const int intraPredAngleMode = (m_ipaParam.isModeVer) ? predMode - VER_IDX : -(predMode - HOR_IDX);
......@@ -573,10 +581,12 @@ void IntraPrediction::initPredIntraParams(const PredictionUnit & pu, const CompA
|| DC_IDX == dirMode
)
{
#if !JVET_O0502_ISP_CLEANUP
if (useISP)
{
m_ipaParam.interpolationFlag = (m_ipaParam.isModeVer ? puSize.width : puSize.height) > 8 ? true : false ;
}
#endif
}
else if (isLuma( chType ) && pu.cu->bdpcmMode) // BDPCM
{
......@@ -1065,6 +1075,132 @@ void IntraPrediction::initIntraPatternChType(const CodingUnit &cu, const CompAre
}
}
#if JVET_O0502_ISP_CLEANUP
void IntraPrediction::initIntraPatternChTypeISP(const CodingUnit& cu, const CompArea& area, PelBuf& recBuf, const bool forceRefFilterFlag)
{
const CodingStructure& cs = *cu.cs;
if (!forceRefFilterFlag)
{
initPredIntraParams(*cu.firstPU, area, *cs.sps);
}
const Position posLT = area;
bool isLeftAvail = cs.isDecomp(posLT.offset(-1, 0), CHANNEL_TYPE_LUMA);
bool isAboveAvail = cs.isDecomp(posLT.offset(0, -1), CHANNEL_TYPE_LUMA);
// ----- Step 1: unfiltered reference samples -----
#if JVET_O0106_ISP_4xN_PREDREG_FOR_1xN_2xN
if (cu.blocks[area.compID].x == area.x && cu.blocks[area.compID].y == area.y)
#else
if (CU::isISPFirst(cu, area, area.compID))
#endif
{
Pel* refBufUnfiltered = m_piYuvExt[area.compID][PRED_BUF_UNFILTERED];
// With the first subpartition all the CU reference samples are fetched at once in a single call to xFillReferenceSamples
if (cu.ispMode == HOR_INTRA_SUBPARTITIONS)
{
m_leftRefLength = cu.Y().height << 1;
m_topRefLength = cu.Y().width + area.width;
}
else //if (cu.ispMode == VER_INTRA_SUBPARTITIONS)
{
m_leftRefLength = cu.Y().height + area.height;
m_topRefLength = cu.Y().width << 1;
}
const int multiRefIdx = m_ipaParam.multiRefIndex;
const int whRatio = m_ipaParam.whRatio;
const int hwRatio = m_ipaParam.hwRatio;
const int srcStride = m_topRefLength + 1 + (whRatio + 1) * multiRefIdx;
const int srcHStride = m_leftRefLength + 1 + (hwRatio + 1) * multiRefIdx;
m_pelBufISP[0] = m_pelBufISPBase[0] = PelBuf(m_piYuvExt[area.compID][PRED_BUF_UNFILTERED], srcStride, srcHStride);
m_pelBufISP[1] = m_pelBufISPBase[1] = PelBuf(m_piYuvExt[area.compID][PRED_BUF_FILTERED], srcStride, srcHStride);
xFillReferenceSamples(cs.picture->getRecoBuf(cu.Y()), refBufUnfiltered, cu.Y(), cu);
// After having retrieved all the CU reference samples, the number of reference samples is now adjusted for the current subpartition
m_topRefLength = cu.blocks[area.compID].width + area.width;
m_leftRefLength = cu.blocks[area.compID].height + area.height;
}
else
{
//Now we only need to fetch the newly available reconstructed samples from the previously coded TU
Position tuPos = area;
tuPos.relativeTo(cu.Y());
m_pelBufISP[0] = m_pelBufISPBase[0].subBuf(tuPos, area.size());
m_pelBufISP[1] = m_pelBufISPBase[1].subBuf(tuPos, area.size());
PelBuf& dstBuf = m_pelBufISP[0];
m_topRefLength = cu.blocks[area.compID].width + area.width;
m_leftRefLength = cu.blocks[area.compID].height + area.height;
const int predSizeHor = m_topRefLength;
const int predSizeVer = m_leftRefLength;
if (cu.ispMode == HOR_INTRA_SUBPARTITIONS)
{
Pel* src = recBuf.bufAt(0, -1);
Pel* dst = dstBuf.bufAt(1, 0);
for (int i = 0; i < area.width; i++)
{
dst[i] = src[i];
}
Pel sample = src[area.width - 1];
dst += area.width;
for (int i = 0; i < predSizeHor - area.width; i++)
{
dst[i] = sample;
}
if (!isLeftAvail) //if left is not avaible, then it is necessary to fetch these samples for each subpartition
{
Pel* dst = dstBuf.bufAt(0, 0);
Pel sample = src[0];
for (int i = 0; i < predSizeVer + 1; i++)
{
*dst = sample;
dst += dstBuf.stride;
}
}
}
else
{
Pel* src = recBuf.bufAt(-1, 0);
Pel* dst = dstBuf.bufAt(0, 1);
for (int i = 0; i < area.height; i++)
{
*dst = *src;
src += recBuf.stride;
dst += dstBuf.stride;
}
Pel sample = src[-recBuf.stride];
for (int i = 0; i < predSizeVer - area.height; i++)
{
*dst = sample;
dst += dstBuf.stride;
}
if (!isAboveAvail) //if above is not avaible, then it is necessary to fetch these samples for each subpartition
{
Pel* dst = dstBuf.bufAt(0, 0);
Pel sample = recBuf.at(-1, 0);
for (int i = 0; i < predSizeHor + 1; i++)
{
dst[i] = sample;
}
}
}
}
// ----- Step 2: filtered reference samples -----
if (m_ipaParam.refFilterFlag || forceRefFilterFlag)
{
Pel* refBufUnfiltered = m_pelBufISP[0].buf;
Pel* refBufFiltered = m_pelBufISP[1].buf;
xFilterReferenceSamples(refBufUnfiltered, refBufFiltered, area, *cs.sps, cu.firstPU->multiRefIdx, m_pelBufISP[0].stride);
}
}
#endif
void IntraPrediction::xFillReferenceSamples( const CPelBuf &recoBuf, Pel* refBufUnfiltered, const CompArea &area, const CodingUnit &cu )
{
const ChannelType chType = toChannelType( area.compID );
......@@ -1322,6 +1458,9 @@ void IntraPrediction::xFillReferenceSamples( const CPelBuf &recoBuf, Pel* refBuf
void IntraPrediction::xFilterReferenceSamples( const Pel* refBufUnfiltered, Pel* refBufFiltered, const CompArea &area, const SPS &sps
, int multiRefIdx
#if JVET_O0502_ISP_CLEANUP
, int stride
#endif
)
{
if (area.compID != COMPONENT_Y)
......@@ -1337,7 +1476,11 @@ void IntraPrediction::xFilterReferenceSamples( const Pel* refBufUnfiltered, Pel*
const int predSize = m_topRefLength + (whRatio + 1) * multiRefIdx;
const int predHSize = m_leftRefLength + (hwRatio + 1) * multiRefIdx;
#endif
#if JVET_O0502_ISP_CLEANUP
const int predStride = stride == 0 ? predSize + 1 : stride;
#else
const int predStride = predSize + 1;
#endif
......
......@@ -68,6 +68,10 @@ class IntraPrediction
private:
Pel* m_piYuvExt[MAX_NUM_COMPONENT][NUM_PRED_BUF];
#if JVET_O0502_ISP_CLEANUP
PelBuf m_pelBufISPBase[2];
PelBuf m_pelBufISP[2];
#endif
int m_iYuvExtSize;
Pel* m_yuvExt2[MAX_NUM_COMPONENT][4];
......@@ -129,6 +133,9 @@ protected:
void xFillReferenceSamples ( const CPelBuf &recoBuf, Pel* refBufUnfiltered, const CompArea &area, const CodingUnit &cu );
void xFilterReferenceSamples ( const Pel* refBufUnfiltered, Pel* refBufFiltered, const CompArea &area, const SPS &sps
, int multiRefIdx
#if JVET_O0502_ISP_CLEANUP
, int predStride = 0
#endif
);
static int getWideAngle ( int width, int height, int predMode );
......@@ -152,6 +159,10 @@ public:
void xGetLumaRecPixels(const PredictionUnit &pu, CompArea chromaArea);
/// set parameters from CU data for accessing intra data
void initIntraPatternChType (const CodingUnit &cu, const CompArea &area, const bool forceRefFilterFlag = false); // use forceRefFilterFlag to get both filtered and unfiltered buffers
#if JVET_O0502_ISP_CLEANUP
void initIntraPatternChTypeISP (const CodingUnit& cu, const CompArea& area, PelBuf& piReco, const bool forceRefFilterFlag = false); // use forceRefFilterFlag to get both filtered and unfiltered buffers
const PelBuf& getISPBuffer () { return m_pelBufISP[m_ipaParam.refFilterFlag ? PRED_BUF_FILTERED : PRED_BUF_UNFILTERED]; }
#endif
// Matrix-based intra prediction
void initIntraMip (const PredictionUnit &pu);
......
......@@ -1056,7 +1056,11 @@ void TrQuant::xQuant(TransformUnit &tu, const ComponentID &compID, const CCoeffB
m_quant->quant( tu, compID, pSrc, uiAbsSum, cQP, ctx );
}
#if JVET_O0502_ISP_CLEANUP
void TrQuant::transformNxN( TransformUnit& tu, const ComponentID& compID, const QpParam& cQP, std::vector<TrMode>* trModes, const int maxCand )
#else
void TrQuant::transformNxN( TransformUnit &tu, const ComponentID &compID, const QpParam &cQP, std::vector<TrMode>* trModes, const int maxCand, double* diagRatio, double* horVerRatio )
#endif
{
CodingStructure &cs = *tu.cs;
const CompArea &rect = tu.blocks[compID];
......@@ -1111,9 +1115,11 @@ void TrQuant::transformNxN( TransformUnit &tu, const ComponentID &compID, const
it++;
}
#if !JVET_O0502_ISP_CLEANUP
// it gets the distribution of the DCT-II coefficients energy, which will be useful to discard ISP tests
CoeffBuf coeffsDCT( m_mtsCoeffs[0], rect );
xGetCoeffEnergy( tu, compID, coeffsDCT, diagRatio, horVerRatio );
#endif
int numTests = 0;
std::vector<TrCost>::iterator itC = trCosts.begin();
const double fac = facBB[g_aucLog2[std::max(width, height)]-2];
......@@ -1128,7 +1134,11 @@ void TrQuant::transformNxN( TransformUnit &tu, const ComponentID &compID, const
}
}
#if JVET_O0502_ISP_CLEANUP
void TrQuant::transformNxN( TransformUnit& tu, const ComponentID& compID, const QpParam& cQP, TCoeff& uiAbsSum, const Ctx& ctx, const bool loadTr )
#else
void TrQuant::transformNxN( TransformUnit &tu, const ComponentID &compID, const QpParam &cQP, TCoeff &uiAbsSum, const Ctx &ctx, const bool loadTr, double* diagRatio, double* horVerRatio )
#endif
{
CodingStructure &cs = *tu.cs;
const SPS &sps = *cs.sps;
......@@ -1207,6 +1217,7 @@ void TrQuant::transformNxN( TransformUnit &tu, const ComponentID &compID, const
}
}
#if !JVET_O0502_ISP_CLEANUP
//we do this only with the DCT-II coefficients
if( isLuma(compID) &&
!loadTr && tu.mtsIdx == MTS_DCT2_DCT2
......@@ -1215,6 +1226,7 @@ void TrQuant::transformNxN( TransformUnit &tu, const ComponentID &compID, const
//it gets the distribution of the coefficients energy, which will be useful to discard ISP tests
xGetCoeffEnergy( tu, compID, tempCoeff, diagRatio, horVerRatio );
}
#endif
if( sps.getUseLFNST() )
{
......@@ -1233,6 +1245,7 @@ void TrQuant::transformNxN( TransformUnit &tu, const ComponentID &compID, const
TU::setCbfAtDepth (tu, compID, tu.depth, uiAbsSum > 0);
}
#if !JVET_O0502_ISP_CLEANUP
void TrQuant::xGetCoeffEnergy( TransformUnit &tu, const ComponentID &compID, const CoeffBuf& coeffs, double* diagRatio, double* horVerRatio )
{
if( nullptr == diagRatio || nullptr == horVerRatio ) return;
......@@ -1266,6 +1279,7 @@ void TrQuant::xGetCoeffEnergy( TransformUnit &tu, const ComponentID &compID, con
*diagRatio = 0 == wdtE && 0 == hgtE && 0 == diaE ? 1 : double( diaE ) / double( wdtE + hgtE );
}
}
#endif
void TrQuant::applyForwardRDPCM(TransformUnit &tu, const ComponentID &compID, const QpParam &cQP, TCoeff &uiAbsSum, const RDPCMMode &mode)
{
......
......@@ -94,9 +94,13 @@ protected:
public:
void invTransformNxN (TransformUnit &tu, const ComponentID &compID, PelBuf &pResi, const QpParam &cQPs);
#if JVET_O0502_ISP_CLEANUP
void transformNxN ( TransformUnit& tu, const ComponentID& compID, const QpParam& cQP, std::vector<TrMode>* trModes, const int maxCand );
void transformNxN ( TransformUnit& tu, const ComponentID& compID, const QpParam& cQP, TCoeff& uiAbsSum, const Ctx& ctx, const bool loadTr = false );
#else
void transformNxN ( TransformUnit &tu, const ComponentID &compID, const QpParam &cQP, std::vector<TrMode>* trModes, const int maxCand, double* diagRatio = nullptr, double* horVerRatio = nullptr );
void transformNxN ( TransformUnit &tu, const ComponentID &compID, const QpParam &cQP, TCoeff &uiAbsSum, const Ctx &ctx, const bool loadTr = false, double* diagRatio = nullptr, double* horVerRatio = nullptr );
#endif
void rdpcmNxN (TransformUnit &tu, const ComponentID &compID, const QpParam &cQP, TCoeff &uiAbsSum, RDPCMMode &rdpcmMode);
void applyForwardRDPCM(TransformUnit &tu, const ComponentID &compID, const QpParam &cQP, TCoeff &uiAbsSum, const RDPCMMode &rdpcmMode);
......
......@@ -50,6 +50,7 @@
#include <assert.h>
#include <cassert>
#define JVET_O0502_ISP_CLEANUP 1 // JVET-O0502: Enable PDPC and all 67 intra modes and apply the cubic filter always (also included in JVET-O0341) for ISP
#define JVET_O0070_PROF 1 // JVET-O0070 method 4-2.1a: Prediction refinement with optical flow for affine mode
......@@ -484,7 +485,12 @@ enum ISPType
NOT_INTRA_SUBPARTITIONS = 0,
HOR_INTRA_SUBPARTITIONS = 1,
VER_INTRA_SUBPARTITIONS = 2,
#if JVET_O0502_ISP_CLEANUP
NUM_INTRA_SUBPARTITIONS_MODES = 3,
INTRA_SUBPARTITIONS_RESERVED = 4
#else
NUM_INTRA_SUBPARTITIONS_MODES = 3
#endif
};
enum SbtIdx
......
......@@ -282,6 +282,7 @@ bool CU::divideTuInRows( const CodingUnit &cu )
return cu.ispMode == HOR_INTRA_SUBPARTITIONS ? true : false;
}
#if !JVET_O0502_ISP_CLEANUP
bool CU::firstTestISPHorSplit( const int width, const int height, const ComponentID compID, const CodingUnit *cuLeft, const CodingUnit *cuAbove )
{
//this function decides which split mode (horizontal or vertical) is tested first (encoder only)
......@@ -351,6 +352,7 @@ bool CU::firstTestISPHorSplit( const int width, const int height, const Componen
return true;
}
}
#endif
PartSplit CU::getISPType( const CodingUnit &cu, const ComponentID compID )
{
......@@ -432,6 +434,31 @@ uint32_t CU::getISPSplitDim( const int width, const int height, const PartSplit
return partitionSize;
}
#if JVET_O0502_ISP_CLEANUP
bool CU::allLumaCBfsAreZero(const CodingUnit& cu)
{
if (!cu.ispMode)
{
return TU::getCbf(*cu.firstTU, COMPONENT_Y) == false;
}
else
{
int numTotalTUs = cu.ispMode == HOR_INTRA_SUBPARTITIONS ? cu.lheight() >> g_aucLog2[cu.firstTU->lheight()] : cu.lwidth() >> g_aucLog2[cu.firstTU->lwidth()];
//bool allZeroCbfs = false;
TransformUnit* tuPtr = cu.firstTU;
for (int tuIdx = 0; tuIdx < numTotalTUs; tuIdx++)
{
if (TU::getCbf(*tuPtr, COMPONENT_Y) == true)
{
return false;
}
tuPtr = tuPtr->next;
}
return true;
}
}
#endif
PUTraverser CU::traversePUs( CodingUnit& cu )
{
......
......@@ -96,13 +96,18 @@ namespace CU
bool divideTuInRows ( const CodingUnit &cu );
#if !JVET_O0502_ISP_CLEANUP
bool firstTestISPHorSplit ( const int width, const int height, const ComponentID compID, const CodingUnit *cuLeft = nullptr, const CodingUnit *cuAbove = nullptr );
#endif
PartSplit getISPType ( const CodingUnit &cu, const ComponentID compID );
bool isISPLast ( const CodingUnit &cu, const CompArea &tuArea, const ComponentID compID );
bool isISPFirst ( const CodingUnit &cu, const CompArea &tuArea, const ComponentID compID );
bool canUseISP ( const CodingUnit &cu, const ComponentID compID );
bool canUseISP ( const int width, const int height, const int maxTrSize = MAX_TB_SIZEY );
uint32_t getISPSplitDim ( const int width, const int height, const PartSplit ispType );
#if JVET_O0502_ISP_CLEANUP
bool allLumaCBfsAreZero ( const CodingUnit& cu );
#endif
PUTraverser traversePUs ( CodingUnit& cu);
TUTraverser traverseTUs ( CodingUnit& cu);
......
......@@ -1269,7 +1269,11 @@ void CABACReader::intra_luma_pred_modes( CodingUnit &cu )
for( int k = 0; k < numBlocks; k++ )
{
CHECK(numBlocks != 1, "not supported yet");
#if JVET_O0502_ISP_CLEANUP
if ( cu.firstPU->multiRefIdx )
#else
if( cu.firstPU->multiRefIdx || ( cu.ispMode && isLuma( cu.chType ) ) )
#endif
{
mpmFlag[0] = true;
}
......
......@@ -204,8 +204,39 @@ void DecCu::xIntraRecBlk( TransformUnit& tu, const ComponentID compID )
const PredictionUnit &pu = *tu.cs->getPU( area.pos(), chType );
const uint32_t uiChFinalMode = PU::getFinalIntraMode( pu, chType );
#if JVET_O0502_ISP_CLEANUP
PelBuf pReco = cs.getRecoBuf(area);
#endif
//===== init availability pattern =====
#if JVET_O0502_ISP_CLEANUP
#if JVET_O0106_ISP_4xN_PREDREG_FOR_1xN_2xN
bool predRegDiffFromTB = CU::isPredRegDiffFromTB(*tu.cu, compID);
bool firstTBInPredReg = CU::isFirstTBInPredReg(*tu.cu, compID, area);
CompArea areaPredReg(COMPONENT_Y, tu.chromaFormat, area);
#endif
if (tu.cu->ispMode && isLuma(compID))
{
#if JVET_O0106_ISP_4xN_PREDREG_FOR_1xN_2xN
if (predRegDiffFromTB)
{
if (firstTBInPredReg)
{
CU::adjustPredArea(areaPredReg);
m_pcIntraPred->initIntraPatternChTypeISP(*tu.cu, areaPredReg, pReco);
}
}
else
#endif
{
m_pcIntraPred->initIntraPatternChTypeISP(*tu.cu, area, pReco);
}
}
else
{
m_pcIntraPred->initIntraPatternChType(*tu.cu, area);
}
#else
#if JVET_O0106_ISP_4xN_PREDREG_FOR_1xN_2xN
bool predRegDiffFromTB = CU::isPredRegDiffFromTB(*tu.cu, compID);
bool firstTBInPredReg = CU::isFirstTBInPredReg (*tu.cu, compID, area);
......@@ -221,6 +252,7 @@ void DecCu::xIntraRecBlk( TransformUnit& tu, const ComponentID compID )
else
#endif
m_pcIntraPred->initIntraPatternChType(*tu.cu, area);
#endif
//===== get prediction signal =====
if( compID != COMPONENT_Y && PU::isLMCMode( uiChFinalMode ) )
......@@ -323,7 +355,9 @@ void DecCu::xIntraRecBlk( TransformUnit& tu, const ComponentID compID )
CrossComponentPrediction::crossComponentPrediction( tu, compID, cs.getResiBuf( tu.Y() ), piResi, piResi, true );
}
#if !JVET_O0502_ISP_CLEANUP
PelBuf pReco = cs.getRecoBuf( area );
#endif
if( !tu.cu->ispMode || !isLuma( compID ) )
{
......
......@@ -1054,7 +1054,11 @@ void CABACWriter::intra_luma_pred_modes( const CodingUnit& cu )
break;
}
}
#if JVET_O0502_ISP_CLEANUP
if ( pu->multiRefIdx )
#else
if( pu->multiRefIdx || ( cu.ispMode && isLuma( cu.chType ) ) )
#endif
{
CHECK(mpm_idx >= numMPMs, "use of non-MPM");
}
......@@ -1154,7 +1158,11 @@ void CABACWriter::intra_luma_pred_mode( const PredictionUnit& pu )
break;
}
}
#if JVET_O0502_ISP_CLEANUP
if ( pu.multiRefIdx )
#else
if( pu.multiRefIdx || ( pu.cu->ispMode && isLuma( pu.cu->chType ) ) )
#endif
{
CHECK(mpm_idx >= numMPMs, "use of non-MPM");
}
......
......@@ -572,8 +572,15 @@ bool EncCu::xCheckBestMode( CodingStructure *&tempCS, CodingStructure *&bestCS,
}
#if JVET_O0502_ISP_CLEANUP
void EncCu::xCompressCU( CodingStructure*& tempCS, CodingStructure*& bestCS, Partitioner& partitioner, double maxCostAllowed )
#else
void EncCu::xCompressCU( CodingStructure *&tempCS, CodingStructure *&bestCS, Partitioner &partitioner )
#endif
{
#if JVET_O0502_ISP_CLEANUP
CHECK(maxCostAllowed < 0, "Wrong value of maxCostAllowed!");
#endif
if (m_shareState == NO_SHARE)
{
tempCS->sharedBndPos = tempCS->area.Y().lumaPos();
......@@ -660,6 +667,9 @@ void EncCu::xCompressCU( CodingStructure *&tempCS, CodingStructure *&bestCS, Par
do
{
EncTestMode currTestMode = m_modeCtrl->currTestMode();
#if JVET_O0502_ISP_CLEANUP
currTestMode.maxCostAllowed = maxCostAllowed;
#endif
if (pps.getUseDQP() && CS::isDualITree(*tempCS) && isChroma(partitioner.chType))
{
......@@ -1246,7 +1256,13 @@ void EncCu::xCheckModeSplit(CodingStructure *&tempCS, CodingStructure *&bestCS,
#if JVET_O0070_PROF
tempSubCS->bestParent = bestSubCS->bestParent = bestCS;
#endif
#if JVET_O0502_ISP_CLEANUP
double newMaxCostAllowed = isLuma(partitioner.chType) ? std::min(encTestMode.maxCostAllowed, bestCS->cost - m_pcRdCost->calcRdCost(tempCS->fracBits, tempCS->dist)) : MAX_DOUBLE;
newMaxCostAllowed = std::max(0.0, newMaxCostAllowed);
xCompressCU(tempSubCS, bestSubCS, partitioner, newMaxCostAllowed);
#else
xCompressCU( tempSubCS, bestSubCS, partitioner );
#endif
#if JVET_O0070_PROF
tempSubCS->bestParent = bestSubCS->bestParent = nullptr;
#endif
......@@ -1464,8 +1480,17 @@ void EncCu::xCheckRDCostIntra( CodingStructure *&tempCS, CodingStructure *&bestC
bool validCandRet = false;
if( isLuma( partitioner.chType ) )
{
#if JVET_O0502_ISP_CLEANUP
//ISP uses the value of the best cost so far (luma if it is the fast version) to avoid test non-necessary subpartitions
double bestCostSoFar = CS::isDualITree(*tempCS) ? m_modeCtrl->getBestCostWithoutSplitFlags() : bestCU && bestCU->predMode == MODE_INTRA ? bestCS->lumaCost : bestCS->cost;
if (CS::isDualITree(*tempCS) && encTestMode.maxCostAllowed < bestCostSoFar)
{
bestCostSoFar = encTestMode.maxCostAllowed;
}
#else
//the Intra SubPartitions mode uses the value of the best cost so far (luma if it is the fast version) to avoid test non-necessary lines
const double bestCostSoFar = CS::isDualITree( *tempCS ) ? m_modeCtrl->getBestCostWithoutSplitFlags() : bestCU && bestCU->predMode == MODE_INTRA ? bestCS->lumaCost : bestCS->cost;
#endif
validCandRet = m_pcIntraSearch->estIntraPredLumaQT( cu, partitioner, bestCostSoFar, mtsFlag, startMTSIdx[ trGrpIdx ], endMTSIdx[ trGrpIdx ], ( trGrpIdx > 0 ) );
if( sps.getUseLFNST() && ( !validCandRet || ( cu.ispMode && cu.firstTU->cbf[ COMPONENT_Y ] == 0 ) ) )
{
......@@ -1626,13 +1651,22 @@ void EncCu::xCheckRDCostIntra( CodingStructure *&tempCS, CodingStructure *&bestC
}
}
#if JVET_O0502_ISP_CLEANUP
//we decide to skip the non-DCT-II transforms and LFNST according to the ISP results
if ((endMtsFlag > 0 || endLfnstIdx > 0) && cu.ispMode && !mtsFlag && !lfnstIdx && tempCS->slice->isIntra() && m_pcEncCfg->getUseFastISP())
#else
//we decide to skip the second emt pass or not according to the ISP results
if( considerMtsSecondPass && cu.ispMode && !mtsFlag && tempCS->slice->isIntra() )
#endif
{
double bestCostDct2NoIsp = m_modeCtrl->getMtsFirstPassNoIspCost();
CHECKD( bestCostDct2NoIsp <= bestIspCost, "wrong cost!" );
#if JVET_O0502_ISP_CLEANUP