Newer
Older
Karsten Suehring
committed
{
#if JVET_AH0076_OBIC
if ((testSgpm && SGPMSaveFlag) || obicSaveFlag || dimdSaveFlag)
#else
#endif
#if JVET_AG0152_SGPM_ITMP_IBC
for (int i = 0; i < NUM_LUMA_MODE + SGPM_NUM_BVS; i++)
#else
{
m_intraModeReady[i] = 0;
}
}
#endif
// this should always be true
CHECK(!pu.Y().valid(), "PU is not valid");
#if !JVET_AB0157_TMRL || JVET_AD0082_TMRL_CONFIG
#if JVET_AH0065_RELAX_LINE_BUFFER
bool isFirstLineOfCtu = pu.block(COMPONENT_Y).y == 0;
#else
bool isFirstLineOfCtu = (((pu.block(COMPONENT_Y).y) & ((pu.cs->sps)->getMaxCUWidth() - 1)) == 0);
#if JVET_Y0116_EXTENDED_MRL_LIST
int numOfPassesExtendRef = MRL_NUM_REF_LINES;
if (!sps.getUseMRL() || isFirstLineOfCtu)
{
numOfPassesExtendRef = 1;
}
else
{
bool checkLineOutsideCtu[MRL_NUM_REF_LINES - 1];
for (int mrlIdx = 1; mrlIdx < MRL_NUM_REF_LINES; mrlIdx++)
{
#if JVET_AH0065_RELAX_LINE_BUFFER
bool isLineOutsideCtu = (cu.block(COMPONENT_Y).y <= MULTI_REF_LINE_IDX[mrlIdx]) ? true : false;
#else
bool isLineOutsideCtu =
((cu.block(COMPONENT_Y).y) % ((cu.cs->sps)->getMaxCUWidth()) <= MULTI_REF_LINE_IDX[mrlIdx]) ? true
: false;
#endif
checkLineOutsideCtu[mrlIdx-1] = isLineOutsideCtu;
}
if (checkLineOutsideCtu[0])
{
numOfPassesExtendRef = 1;
}
else
{
for (int mrlIdx = MRL_NUM_REF_LINES - 2; mrlIdx > 0; mrlIdx--)
{
if (checkLineOutsideCtu[mrlIdx] && !checkLineOutsideCtu[mrlIdx - 1])
{
numOfPassesExtendRef = mrlIdx + 1;
break;
}
}
}
}
#else
int numOfPassesExtendRef = ((!sps.getUseMRL() || isFirstLineOfCtu) ? 1 : MRL_NUM_REF_LINES);
#endif
#if JVET_AB0157_TMRL
cu.tmrlFlag = false;
#endif
Karsten Suehring
committed
CHECK(numModesForFullRD >= numModesAvailable, "Too many modes for full RD search");
PelBuf piOrg = cs.getOrgBuf(area);
PelBuf piPred = cs.getPredBuf(area);
Karsten Suehring
committed
DistParam distParamSad;
DistParam distParamHad;
if (cu.slice->getLmcsEnabledFlag() && m_pcReshape->getCTUFlag())
Karsten Suehring
committed
{
CompArea tmpArea(COMPONENT_Y, area.chromaFormat, Position(0, 0), area.size());
PelBuf tmpOrg = m_tmpStorageLCU.getBuf(tmpArea);
m_pcRdCost->setDistParam(distParamSad, tmpOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y,
false); // Use SAD cost
m_pcRdCost->setDistParam(distParamHad, tmpOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y,
true); // Use HAD (SATD) cost
Karsten Suehring
committed
}
m_pcRdCost->setDistParam(distParamSad, piOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y,
false); // Use SAD cost
m_pcRdCost->setDistParam(distParamHad, piOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y,
true); // Use HAD (SATD) cost
Karsten Suehring
committed
distParamSad.applyWeight = false;
distParamHad.applyWeight = false;
Karsten Suehring
committed
if (testMip && supportedMipBlkSize)
{
numModesForFullRD += fastMip
? std::max(numModesForFullRD, floorLog2(std::min(pu.lwidth(), pu.lheight())) - 1)
: numModesForFullRD;
}
#if JVET_AB0130_ITMP_SAMPLING
if (testTpm && !m_pcEncCfg->getUseFastIntraTMP())
#else
{
numModesForFullRD += 1; // testing tpm
}
#if JVET_AG0136_INTRA_TMP_LIC
if( testTpm && m_pcEncCfg->getItmpLicMode())
{
numModesForFullRD += 1; // testing lic itmp
}
#endif
const int numHadCand = (testMip ? 2 : 1) * 3 + testTpm;
cu.tmpFlag = false;
const int numHadCand = (testMip ? 2 : 1) * 3;
Karsten Suehring
committed
#if JVET_AB0155_SGPM
static_vector<SgpmInfo, SGPM_NUM> sgpmInfoList;
static_vector<double, SGPM_NUM> sgpmCostList;
#if JVET_AG0152_SGPM_ITMP_IBC
int sgpmNeededMode[NUM_LUMA_MODE + SGPM_NUM_BVS] = { 0 };
#else
if (testSgpm && SGPMSaveFlag)
{
deriveSgpmModeOrdered(bestCS->picture->getRecoBuf(area), area, cu, sgpmInfoList, sgpmCostList);
Bappaditya Ray
committed
for (int sgpmIdx = 0; sgpmIdx < SGPM_NUM; sgpmIdx++)
{
int sgpmMode[2];
sgpmMode[0] = sgpmInfoList[sgpmIdx].sgpmMode0;
sgpmMode[1] = sgpmInfoList[sgpmIdx].sgpmMode1;
sgpmNeededMode[sgpmMode[0]] = 1;
sgpmNeededMode[sgpmMode[1]] = 1;
}
}
#endif
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
#if JVET_AH0076_OBIC
int dimdNeededMode[NUM_LUMA_MODE] = {0};
if (obicSaveFlag)
{
for (int idx = 0; idx < OBIC_FUSION_NUM; idx++)
{
int iMode = cu.obicMode[idx];
if (iMode < 0)
{
continue;
}
dimdNeededMode[iMode] = 1;
}
}
if (dimdSaveFlag)
{
if (cu.dimdBlending)
{
for (int dimdIdx = 0; dimdIdx < DIMD_FUSION_NUM - 1; dimdIdx++)
{
int dimdMode = (dimdIdx == 0 ? cu.dimdMode : cu.dimdBlendMode[dimdIdx-1]);
if (dimdMode <= 0)
{
break;
}
dimdNeededMode[dimdMode] = 1;
}
dimdNeededMode[PLANAR_IDX] = 1;
}
}
#endif
#if JVET_AB0157_TMRL
double tmrlCostList[MRL_LIST_SIZE]{ MAX_DOUBLE };
#endif
#if JVET_AC0105_DIRECTIONAL_PLANAR
double dirPlanarCostList[2]{ MAX_DOUBLE };
#endif
//*** Derive (regular) candidates using Hadamard
cu.mipFlag = false;
Karsten Suehring
committed
//===== init pattern for luma prediction =====
#if JVET_AB0157_INTRA_FUSION && JVET_AB0155_SGPM
initIntraPatternChType(cu, pu.Y(), true, 0, false);
#elif JVET_AB0157_INTRA_FUSION
initIntraPatternChType(cu, pu.Y(), true, false);
#else
bool bSatdChecked[NUM_INTRA_MODE];
memset(bSatdChecked, 0, sizeof(bSatdChecked));
Karsten Suehring
committed
if (!LFNSTLoadFlag)
{
for (int modeIdx = 0; modeIdx < numModesAvailable; modeIdx++)
{
uint32_t uiMode = modeIdx;
Distortion minSadHad = 0;
Karsten Suehring
committed
// Skip checking extended Angular modes in the first round of SATD
if (uiMode > DC_IDX && (uiMode & 1))
{
continue;
Karsten Suehring
committed
}
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
Bappaditya Ray
committed
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
#if JVET_AH0209_PDP
bool pdpMode = false;
if( pdpSaveFlag )
{
const int sizeKey = (width << 8) + height;
const int sizeIdx = g_size.find(sizeKey) != g_size.end() ? g_size[sizeKey] : -1;
const int m = sizeIdx > 12 ? 2 : 0;
const int s = sizeIdx > 12 ? 4 : 2;
if (sizeIdx >= 0 && m_refAvailable && pu.cu->cs->sps->getUsePDP() && !(modeIdx > 1 && modeIdx % s != m))
{
PelBuf predBuf(m_pdpIntraPredBuf[uiMode], tmpArea);
predBuf.copyFrom(piPred);
m_pdpIntraPredReady[modeIdx] = true;
pdpMode = true;
}
}
if( !pdpMode )
{
#endif
#if JVET_AB0155_SGPM
if (testSgpm && SGPMSaveFlag && sgpmNeededMode[uiMode])
{
Bappaditya Ray
committed
PelBuf predBuf(m_intraPredBuf[uiMode], tmpArea);
predBuf.copyFrom(piPred);
m_intraModeReady[uiMode] = 1;
}
#endif
#if JVET_AH0076_OBIC
if ((obicSaveFlag || dimdSaveFlag) && dimdNeededMode[uiMode] && !m_intraModeReady[uiMode])
{
Bappaditya Ray
committed
PelBuf predBuf(m_intraPredBuf[uiMode], tmpArea);
predBuf.copyFrom(piPred);
m_intraModeReady[uiMode] = 1;
}
#endif
Bappaditya Ray
committed
#if JVET_AH0209_PDP
}
#endif
// Use the min between SAD and HAD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
minSadHad += std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
minSadHad += std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, uiMode, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
DTRACE(g_trace_ctx, D_INTRA_COST, "IntraHAD: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, uiMode);
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, uiMode), cost, uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, uiMode), double(minSadHad),
Bappaditya Ray
committed
uiHadModeList, candHadList, numHadCand);
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
#if JVET_AC0105_DIRECTIONAL_PLANAR
bool testDirPlanar = isLuma(partitioner.chType);
if (testDirPlanar)
{
for (int dirPlanarModeIdx = 0; dirPlanarModeIdx < 2; dirPlanarModeIdx++)
{
cu.sgpm = false;
cu.ispMode = 0;
cu.tmpFlag = false;
cu.tmrlFlag = false;
pu.multiRefIdx = 0;
cu.mipFlag = false;
pu.intraDir[0] = PLANAR_IDX;
cu.plIdx = dirPlanarModeIdx + 1;
initPredIntraParams(pu, pu.Y(), sps);
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
predIntraAng(COMPONENT_Y, piPred, pu);
#endif
// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, PLANAR_IDX, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(
ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, dirPlanarModeIdx ? PL_VER_IDX : PL_HOR_IDX), cost,
uiRdModeList, candCostList, numModesForFullRD);
updateCandList(
ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, dirPlanarModeIdx ? PL_VER_IDX : PL_HOR_IDX),
Bappaditya Ray
committed
double(minSadHad), uiHadModeList, candHadList, numHadCand);
dirPlanarCostList[dirPlanarModeIdx] = cost;
}
}
cu.plIdx = 0;
#endif
if (!sps.getUseMIP() && LFNSTSaveFlag)
{
// save found best modes
m_uiSavedNumRdModesLFNST = numModesForFullRD;
m_uiSavedRdModeListLFNST = uiRdModeList;
m_dSavedModeCostLFNST = candCostList;
// PBINTRA fast
m_uiSavedHadModeListLFNST = uiHadModeList;
Bappaditya Ray
committed
m_dSavedHadListLFNST = candHadList;
LFNSTSaveFlag = false;
}
} // NSSTFlag
if (!sps.getUseMIP() && LFNSTLoadFlag)
// restore saved modes
numModesForFullRD = m_uiSavedNumRdModesLFNST;
uiRdModeList = m_uiSavedRdModeListLFNST;
candCostList = m_dSavedModeCostLFNST;
// PBINTRA fast
uiHadModeList = m_uiSavedHadModeListLFNST;
Bappaditya Ray
committed
candHadList = m_dSavedHadListLFNST;
if (!(sps.getUseMIP() && LFNSTLoadFlag))
{
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> parentCandList = uiRdModeList;
// Second round of SATD for extended Angular modes
for (int modeIdx = 0; modeIdx < numModesForFullRD; modeIdx++)
{
unsigned parentMode = parentCandList[modeIdx].modeId;
if (parentMode > (DC_IDX + 1) && parentMode < (NUM_LUMA_MODE - 1))
{
for (int subModeIdx = -1; subModeIdx <= 1; subModeIdx += 2)
{
unsigned mode = parentMode + subModeIdx;
if (!bSatdChecked[mode])
{
pu.intraDir[0] = mode;
initPredIntraParams(pu, pu.Y(), sps);
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
#if JVET_AB0155_SGPM
if (testSgpm && SGPMSaveFlag && sgpmNeededMode[mode])
{
Bappaditya Ray
committed
PelBuf predBuf(m_intraPredBuf[mode], tmpArea);
predBuf.copyFrom(piPred);
m_intraModeReady[mode] = 1;
}
#endif
#if JVET_AH0076_OBIC
if ((obicSaveFlag || dimdSaveFlag) && dimdNeededMode[mode] && !m_intraModeReady[mode])
{
Bappaditya Ray
committed
PelBuf predBuf(m_intraPredBuf[mode], tmpArea);
predBuf.copyFrom(piPred);
m_intraModeReady[mode] = 1;
}
#endif
// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, mode, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, mode), cost, uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, mode), double(minSadHad),
Bappaditya Ray
committed
uiHadModeList, candHadList, numHadCand);
bSatdChecked[mode] = true;
}
}
}
}
if (saveDataForISP)
// we save the regular intra modes list
m_ispCandListHor = uiRdModeList;
}
#if !JVET_AB0157_TMRL || JVET_AD0082_TMRL_CONFIG
#if SECONDARY_MPM
const int numMPMs = NUM_PRIMARY_MOST_PROBABLE_MODES;
#else
Fabrice URBAN
committed
uint8_t* multiRefMPM = m_intraMPM;
#if !JVET_AD0082_TMRL_CONFIG
#if JVET_AD0082_TMRL_CONFIG
cu.tmrlFlag = true;
#endif
for (auto multiRefIdx : EXT_REF_LINE_IDX)
{
pu.multiRefIdx = multiRefIdx;
initIntraPatternChType(cu, pu.Y(), true);
for (auto i = 0; i < MRL_LIST_SIZE; i++)
{
if (m_tmrlList[i].multiRefIdx != multiRefIdx)
pu.intraDir[0] = m_tmrlList[i].intraDir;
cu.tmrlListIdx = i;
uint32_t uiMode = i + MAX_REF_LINE_IDX;
initPredIntraParams(pu, pu.Y(), *(pu.cs->sps));
predIntraAng(COMPONENT_Y, piPred, pu);
// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, pu.intraDir[0], CHANNEL_TYPE_LUMA);
double cost = (double)minSadHad + (double)fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(ModeInfo(false, false, uiMode, NOT_INTRA_SUBPARTITIONS, 0), cost, uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, uiMode, NOT_INTRA_SUBPARTITIONS, 0), double(minSadHad),
Bappaditya Ray
committed
uiHadModeList, candHadList, numHadCand);
#if JVET_AB0157_TMRL
tmrlCostList[i] = cost;
#endif
}
}
#if JVET_AD0082_TMRL_CONFIG
cu.tmrlFlag = false;
#endif
#endif
#if !JVET_AB0157_TMRL || JVET_AD0082_TMRL_CONFIG
#if JVET_AD0082_TMRL_CONFIG
else
{
#endif
for (int mRefNum = 1; mRefNum < numOfPassesExtendRef; mRefNum++)
{
int multiRefIdx = MULTI_REF_LINE_IDX[mRefNum];
#if JVET_AB0157_INTRA_FUSION && JVET_AB0155_SGPM
initIntraPatternChType(cu, pu.Y(), true, 0, false);
#elif JVET_AB0157_INTRA_FUSION
initIntraPatternChType(cu, pu.Y(), true, false);
#else
}
for (int x = 1; x < numMPMs; x++)
{
uint32_t mode = multiRefMPM[x];
{
pu.intraDir[0] = mode;
initPredIntraParams(pu, pu.Y(), sps);
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, mode, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(ModeInfo(false, false, multiRefIdx, NOT_INTRA_SUBPARTITIONS, mode), cost, uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, multiRefIdx, NOT_INTRA_SUBPARTITIONS, mode), double(minSadHad),
Bappaditya Ray
committed
uiHadModeList, candHadList, numHadCand);
#if JVET_AD0082_TMRL_CONFIG
}
#endif
CHECKD(uiRdModeList.size() != numModesForFullRD, "Error: RD mode list size");
#if JVET_V0130_INTRA_TMP && JVET_AB0130_ITMP_SAMPLING
// derive TPM candidate using hadamard
if (testTpm)
{
cu.tmpFlag = true;
cu.mipFlag = false;
pu.multiRefIdx = 0;
#if JVET_AD0086_ENHANCED_INTRA_TMP
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListTmp;
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> candCostListTmp;
int foundCandiNum = 0;
bool bsuccessfull = 0;
Wei Chen
committed
#endif
#if JVET_AH0200_INTRA_TMP_BV_REORDER
int adjustedTMPNonLicBvNum = TMP_REFINE_NONLIC_BV_NUM;
int adjustedTmpLicBvNum = TMP_REFINE_LIC_BV_NUM;
if(m_pcEncCfg->getIntraPeriod() != 1)
{
adjustedTMPNonLicBvNum = 10;
adjustedTmpLicBvNum = 4;
}
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListFracTmp;
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> candCostListFracTmp;
Distortion backupMinSadHad[MTMP_NUM];
Distortion backupSadCost[MTMP_NUM];
Distortion backupLicMinSadHad[MTMP_NUM][4];
Distortion backupLicSadCost[MTMP_NUM][4];
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListLicFracTmp;
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> candCostListLicFracTmp;
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListTmpLic;
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> candCostListTmpLic;
CodingUnit cuCopy = cu;
#if JVET_W0069_TMP_BOUNDARY
RefTemplateType templateType = getRefTemplateType(cuCopy, cuCopy.blocks[COMPONENT_Y]);
if (templateType != NO_TEMPLATE)
#else
if (isRefTemplateAvailable(cuCopy, cuCopy.blocks[COMPONENT_Y]))
#endif
{
#if JVET_AD0086_ENHANCED_INTRA_TMP
cu.tmpIsSubPel = 0;
#if JVET_AG0136_INTRA_TMP_LIC
cu.tmpSubPelIdx = -1;
#else
cu.tmpSubPelIdx = 0;
Wei Chen
committed
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
#endif
#if JVET_AH0200_INTRA_TMP_BV_REORDER
cu.tmpFracIdx = 0;
int numModesForFracIntraTmp = numModesForFullRD + adjustedTMPNonLicBvNum;
if(relatedCU.skipFracTmp)
{
numModesForFracIntraTmp = numModesForFullRD;
}
if(m_tmpNumCand > 0)
{
for(int idxInList=0; idxInList < uiRdModeList.size(); idxInList++)
{
if(cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD)
{
break;
}
updateCandList(uiRdModeList[idxInList], candCostList[idxInList], uiRdModeListFracTmp, candCostListFracTmp, numModesForFracIntraTmp);
}
}
int numModesForLicFracIntraTmp = numModesForFullRD + adjustedTmpLicBvNum;
if(relatedCU.skipFracTmp)
{
numModesForLicFracIntraTmp = numModesForFullRD;
}
if(m_tmpNumCandUseMR > 0)
{
for(int idxInList=0; idxInList < uiRdModeList.size(); idxInList++)
{
if(cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD)
{
break;
}
updateCandList(uiRdModeList[idxInList], candCostList[idxInList], uiRdModeListLicFracTmp, candCostListLicFracTmp, numModesForLicFracIntraTmp);
}
}
for(int tmpFusionFlag = 0; tmpFusionFlag <= 1; tmpFusionFlag++)
{
cu.tmpFusionFlag = tmpFusionFlag ? true: false;
for (int tmpFlmFlag = 0; tmpFlmFlag <= 1; tmpFlmFlag++)
{
cu.tmpFlmFlag = tmpFlmFlag ? true: false;
if(tmpFlmFlag && tmpFusionFlag)
{
continue;
}
#if JVET_AG0136_INTRA_TMP_LIC
for (int tmpLicFlag = 0; tmpLicFlag <= 1; tmpLicFlag++)
{
cu.ibcLicFlag = tmpLicFlag ? true : false;
cu.tmpLicFlag = tmpLicFlag ? true : false;
if (tmpLicFlag && tmpFlmFlag)
{
continue;
}
const int ibcLicLoopNum = (cu.slice->getSPS()->getItmpLicExtension() && cu.tmpLicFlag && !cu.tmpFusionFlag) ? 4 : 1;
for (int licIdc = 0; licIdc < ibcLicLoopNum; licIdc++)
{
cu.ibcLicIdx = licIdc;
int idxNum = cu.tmpFusionFlag ? TMP_GROUP_IDX << 1 : (cu.tmpLicFlag ? m_tmpNumCandUseMR : m_tmpNumCand);
#else
int idxNum = cu.tmpFusionFlag ? TMP_GROUP_IDX << 1 : m_tmpNumCand;
for (int tmpIdx = 0; tmpIdx < idxNum; tmpIdx++)
{
#if JVET_AG0136_INTRA_TMP_LIC
if (cu.tmpFusionFlag && !(cu.tmpLicFlag ? m_tmpFusionInfoUseMR : m_tmpFusionInfo)[tmpIdx].bValid)
#else
if(cu.tmpFusionFlag && !m_tmpFusionInfo[tmpIdx].bValid)
{
continue;
}
cu.tmpIdx = tmpIdx;
int placeHolder;
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
generateTMPrediction(piPred.buf, piPred.stride, placeHolder, pu
#if JVET_AG0136_INTRA_TMP_LIC
, cu.tmpLicFlag
#endif
, false);
#if JVET_AG0136_INTRA_TMP_LIC
if (cu.tmpLicFlag)
{
if (!cu.tmpFusionFlag)
{
const auto& arrayLicParams = getMemLicParams(cu.ibcLicIdx, cu.tmpIdx);
if (cu.ibcLicIdx == IBC_LIC_IDX_M)
{
piPred.linearTransforms(arrayLicParams[1], arrayLicParams[0], arrayLicParams[2], arrayLicParams[4], arrayLicParams[3], arrayLicParams[5], arrayLicParams[6], true, cu.cs->slice->clpRng(COMPONENT_Y));
}
else
{
piPred.linearTransform(arrayLicParams[1], arrayLicParams[0], arrayLicParams[2], true, cu.cs->slice->clpRng(COMPONENT_Y));
}
}
}
#endif
xGenerateTmpFlmPred(piPred, pu.lwidth(), pu.lheight(), templateType, pu.cu, false);
xTMPFusionApplyModel(piPred, pu.lwidth(), pu.lheight(), templateType, pu.cu
#if JVET_AG0136_INTRA_TMP_LIC
, cu.tmpLicFlag
#endif
, false);
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
#if JVET_W0069_TMP_BOUNDARY
#if TMP_FAST_ENC
bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.Y(), foundCandiNum, pu.cu);
#else
getTargetTemplate(&cuCopy, pu.lwidth(), pu.lheight(), templateType);
candidateSearchIntra(&cuCopy, pu.lwidth(), pu.lheight(), templateType);
bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), foundCandiNum);
#endif
#else
#if TMP_FAST_ENC
bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.Y(), foundCandiNum, pu.cu);
#else
getTargetTemplate(&cuCopy, pu.lwidth(), pu.lheight());
candidateSearchIntra(&cuCopy, pu.lwidth(), pu.lheight());
bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), foundCandiNum);
#endif
#endif
}
#if JVET_W0069_TMP_BOUNDARY
else
{
foundCandiNum = 1;
#if JVET_AC0115_INTRA_TMP_DIMD_MTS_LFNST
bsuccessfull = generateTmDcPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), 1 << (cuCopy.cs->sps->getBitDepth(CHANNEL_TYPE_LUMA) - 1), pu.cu);
#else
bsuccessfull = generateTmDcPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), 1 << (cuCopy.cs->sps->getBitDepth(CHANNEL_TYPE_LUMA) - 1));
#endif
}
#endif
if (bsuccessfull && foundCandiNum >= 1)
{
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
#if JVET_AG0136_INTRA_TMP_LIC
m_CABACEstimator->getCtx() = SubCtx(Ctx::TmpLic, ctxStartTmpLicFlag);
m_CABACEstimator->getCtx() = SubCtx(Ctx::ItmpLicIndex, ctxStartTmpLicIdx);
#endif
uint64_t fracModeBits = xFracModeBitsIntra(pu, 0, CHANNEL_TYPE_LUMA);
double cost = double(minSadHad) + double(fracModeBits) * sqrtLambdaForFirstPass;
Wei Chen
committed
#if JVET_AH0200_INTRA_TMP_BV_REORDER
isTmpModeTestd = true;
if(!tmpFusionFlag && !tmpFlmFlag)
{
if(tmpBestSatdCost > cost)
{
tmpBestSatdCost = cost;
}
}
#endif
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Wei Chen
committed
#if JVET_AH0200_INTRA_TMP_BV_REORDER
if(tmpFlmFlag || tmpFusionFlag
|| (cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD))
{
#endif
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - double(minSadHad) + (double)sadCost);
#if JVET_AH0200_INTRA_TMP_BV_REORDER
}
#endif
#endif
DTRACE(g_trace_ctx, D_INTRA_COST, "IntraTPM: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, 0);
#if JVET_AD0086_ENHANCED_INTRA_TMP
#if JVET_AG0136_INTRA_TMP_LIC
Wei Chen
committed
#if JVET_AH0200_INTRA_TMP_BV_REORDER
if(tmpFlmFlag || tmpFusionFlag || (cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD))
{
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx, cu.tmpFracIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
Bappaditya Ray
committed
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx, cu.tmpFracIdx), 0.8 * double(minSadHad), uiHadModeList, candHadList, numHadCand);
Wei Chen
committed
}
else if (cu.tmpLicFlag)
{
backupLicMinSadHad[cu.tmpIdx][cu.ibcLicIdx] = minSadHad;
backupLicSadCost[cu.tmpIdx][cu.ibcLicIdx] = sadCost;
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx, cu.tmpFracIdx), cost, uiRdModeListLicFracTmp, candCostListLicFracTmp, numModesForLicFracIntraTmp);
}
else
{
backupMinSadHad[cu.tmpIdx] = minSadHad;
backupSadCost[cu.tmpIdx] = sadCost;
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx, cu.tmpFracIdx), cost, uiRdModeListFracTmp, candCostListFracTmp, numModesForFracIntraTmp);
}
#else
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
Bappaditya Ray
committed
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, candHadList, numHadCand);
Wei Chen
committed
#endif
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
Bappaditya Ray
committed
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, candHadList, numHadCand);
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1), cost, uiRdModeList, candCostList, numModesForFullRD);
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1), 0.8 * double(minSadHad), uiHadModeList, CandHadList, numHadCand);
#endif
#if JVET_AD0086_ENHANCED_INTRA_TMP
//record the best full-pel candidates
Wei Chen
committed
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
#if JVET_AH0200_INTRA_TMP_BV_REORDER
if (!tmpFlmFlag && !tmpFusionFlag && tmpLicFlag)
{
for(int idxInList=0; idxInList < uiRdModeListLicFracTmp.size(); idxInList++)
{
if(cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD)
{
break;
}
if(uiRdModeListLicFracTmp[idxInList].tmpFlag)
{
updateCandList(uiRdModeListLicFracTmp[idxInList], candCostListLicFracTmp[idxInList], uiRdModeListTmpLic, candCostListTmpLic, numModesForLicFracIntraTmp);
}
if(idxInList >= numModesForFullRD && uiRdModeListTmpLic.size() >= adjustedTmpLicBvNum)
{
break;
}
if(idxInList >= (numModesForFullRD-1) && relatedCU.skipFracTmp)
{
break;
}
}
}
#endif
#if JVET_AG0136_INTRA_TMP_LIC
if (!tmpFlmFlag && !tmpFusionFlag && !tmpLicFlag)
#else
if(!tmpFlmFlag&&!tmpFusionFlag)
Wei Chen
committed
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
#if JVET_AH0200_INTRA_TMP_BV_REORDER
for(int idxInList=0; idxInList < uiRdModeListFracTmp.size(); idxInList++)
{
if(cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD)
{
break;
}
if(uiRdModeListFracTmp[idxInList].tmpFlag)
{
updateCandList(uiRdModeListFracTmp[idxInList], candCostListFracTmp[idxInList], uiRdModeListTmp, candCostListTmp, numModesForFracIntraTmp);
}
if(idxInList >= numModesForFullRD && uiRdModeListTmp.size() >= adjustedTMPNonLicBvNum)
{
break;
}
if(idxInList >= (numModesForFullRD-1) && relatedCU.skipFracTmp)
{
break;
}
}
#else
for(int idxInList=0; idxInList < uiRdModeList.size(); idxInList++)
{
if(uiRdModeList[idxInList].tmpFlag){
updateCandList(uiRdModeList[idxInList], candCostList[idxInList], uiRdModeListTmp, candCostListTmp, numModesForFullRD);
Wei Chen
committed
#endif
#if JVET_AG0136_INTRA_TMP_LIC
}
}
#endif
}
}
//fractional BV
cu.tmpFusionFlag = false;
cu.tmpFlmFlag = false;
#if JVET_AG0136_INTRA_TMP_LIC
cu.tmpLicFlag = false;
cu.ibcLicIdx = 0;
#endif
Wei Chen
committed
#if !JVET_AH0200_INTRA_TMP_BV_REORDER
for(int idxInList=0; idxInList < uiRdModeListTmp.size(); idxInList++)
{
cu.tmpIdx = uiRdModeListTmp[idxInList].tmpIdx;
xPadForInterpolation(&cu);
for(int tmpIsSubPel = 1; tmpIsSubPel < 4; tmpIsSubPel++)
{
for (int idx = 0; idx < TMP_MAX_SUBPEL_DIR; idx++)
{
cu.tmpIsSubPel = tmpIsSubPel;
cu.tmpSubPelIdx = idx;
int placeHolder;
generateTMPrediction(piPred.buf, piPred.stride, placeHolder, pu
#if JVET_AG0136_INTRA_TMP_LIC
, cu.tmpLicFlag
#endif
, false);
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS&&JVET_AE0169_BIPREDICTIVE_IBC
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, 0, CHANNEL_TYPE_LUMA);
double cost = double(minSadHad) + double(fracModeBits) * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS&&JVET_AE0169_BIPREDICTIVE_IBC
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - double(minSadHad) + (double)sadCost);
#endif
DTRACE(g_trace_ctx, D_INTRA_COST, "IntraTPM: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, 0);
#if JVET_AG0136_INTRA_TMP_LIC
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
Bappaditya Ray
committed
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, candHadList, numHadCand);
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
Bappaditya Ray
committed
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, candHadList, numHadCand);
}
}
}
Wei Chen
committed
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
#endif
#if JVET_AH0200_INTRA_TMP_BV_REORDER
for(int idxInList=0; idxInList < uiRdModeListTmp.size(); idxInList++)
{
cu.tmpIdx = uiRdModeListTmp[idxInList].tmpIdx;
cu.tmpLicFlag = uiRdModeListTmp[idxInList].tmpLicFlag;
CHECK(cu.tmpLicFlag, "cu.tmpLicFlag == 1");
cu.ibcLicFlag = cu.tmpLicFlag;
cu.ibcLicIdx = uiRdModeListTmp[idxInList].tmpLicIdc;
searchFracCandidate(&cu, getTargetPatch(floorLog2(std::max(cu.lwidth(), cu.lheight())) - 2), templateType);
for (int spIdx = 0; spIdx < std::min(2, (int) m_mtmpFracCandList[cu.tmpIdx].size()); spIdx++)
{
cu.tmpIsSubPel = m_mtmpFracCandList[cu.tmpIdx][spIdx].m_subpel;
cu.tmpSubPelIdx = m_mtmpFracCandList[cu.tmpIdx][spIdx].m_fracDir;
CHECK(cu.tmpIsSubPel < 0 || cu.tmpIsSubPel > 2, "cu.tmpIsSubPel < 1 || cu.tmpIsSubPel > 2");
cu.tmpFracIdx = spIdx;
Distortion sadCost;
Distortion minSadHad;
uint64_t fracModeBits = 0;
double cost;
if(cu.tmpIsSubPel)
{
int placeHolder;
generateTMPrediction(piPred.buf, piPred.stride, placeHolder, pu, cu.tmpLicFlag, false);
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS&&JVET_AE0169_BIPREDICTIVE_IBC
sadCost = distParamSad.distFunc(distParamSad);
minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
}
else
{
sadCost = backupSadCost[cu.tmpIdx];
minSadHad = backupMinSadHad[cu.tmpIdx];
}
if(!cu.tmpIsSubPel && !cu.tmpFracIdx)
{
cost = candCostListTmp[idxInList];
}
else