Newer
Older

Karsten Suehring
committed
{
#if JVET_AH0076_OBIC
if ((testSgpm && SGPMSaveFlag) || obicSaveFlag || dimdSaveFlag)
#else
#endif
#if JVET_AG0152_SGPM_ITMP_IBC
for (int i = 0; i < NUM_LUMA_MODE + SGPM_NUM_BVS; i++)
#else
{
m_intraModeReady[i] = 0;
}
}
#endif
// this should always be true
CHECK(!pu.Y().valid(), "PU is not valid");
#if !JVET_AB0157_TMRL || JVET_AD0082_TMRL_CONFIG
#if JVET_AH0065_RELAX_LINE_BUFFER
bool isFirstLineOfCtu = pu.block(COMPONENT_Y).y == 0;
#else
bool isFirstLineOfCtu = (((pu.block(COMPONENT_Y).y) & ((pu.cs->sps)->getMaxCUWidth() - 1)) == 0);
#if JVET_Y0116_EXTENDED_MRL_LIST
int numOfPassesExtendRef = MRL_NUM_REF_LINES;
if (!sps.getUseMRL() || isFirstLineOfCtu)
{
numOfPassesExtendRef = 1;
}
else
{
bool checkLineOutsideCtu[MRL_NUM_REF_LINES - 1];
for (int mrlIdx = 1; mrlIdx < MRL_NUM_REF_LINES; mrlIdx++)
{
#if JVET_AH0065_RELAX_LINE_BUFFER
bool isLineOutsideCtu = (cu.block(COMPONENT_Y).y <= MULTI_REF_LINE_IDX[mrlIdx]) ? true : false;
#else
bool isLineOutsideCtu =
((cu.block(COMPONENT_Y).y) % ((cu.cs->sps)->getMaxCUWidth()) <= MULTI_REF_LINE_IDX[mrlIdx]) ? true
: false;
#endif
checkLineOutsideCtu[mrlIdx-1] = isLineOutsideCtu;
}
if (checkLineOutsideCtu[0])
{
numOfPassesExtendRef = 1;
}
else
{
for (int mrlIdx = MRL_NUM_REF_LINES - 2; mrlIdx > 0; mrlIdx--)
{
if (checkLineOutsideCtu[mrlIdx] && !checkLineOutsideCtu[mrlIdx - 1])
{
numOfPassesExtendRef = mrlIdx + 1;
break;
}
}
}
}
#else
int numOfPassesExtendRef = ((!sps.getUseMRL() || isFirstLineOfCtu) ? 1 : MRL_NUM_REF_LINES);
#endif
#if JVET_AB0157_TMRL
cu.tmrlFlag = false;
#endif

Karsten Suehring
committed
CHECK(numModesForFullRD >= numModesAvailable, "Too many modes for full RD search");
PelBuf piOrg = cs.getOrgBuf(area);
PelBuf piPred = cs.getPredBuf(area);

Karsten Suehring
committed
DistParam distParamSad;
DistParam distParamHad;
if (cu.slice->getLmcsEnabledFlag() && m_pcReshape->getCTUFlag())

Karsten Suehring
committed
{
CompArea tmpArea(COMPONENT_Y, area.chromaFormat, Position(0, 0), area.size());
PelBuf tmpOrg = m_tmpStorageLCU.getBuf(tmpArea);
m_pcRdCost->setDistParam(distParamSad, tmpOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y,
false); // Use SAD cost
m_pcRdCost->setDistParam(distParamHad, tmpOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y,
true); // Use HAD (SATD) cost

Karsten Suehring
committed
}
m_pcRdCost->setDistParam(distParamSad, piOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y,
false); // Use SAD cost
m_pcRdCost->setDistParam(distParamHad, piOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y,
true); // Use HAD (SATD) cost

Karsten Suehring
committed
distParamSad.applyWeight = false;
distParamHad.applyWeight = false;

Karsten Suehring
committed
if (testMip && supportedMipBlkSize)
{
numModesForFullRD += fastMip
? std::max(numModesForFullRD, floorLog2(std::min(pu.lwidth(), pu.lheight())) - 1)
: numModesForFullRD;
}
#if JVET_AB0130_ITMP_SAMPLING
if (testTpm && !m_pcEncCfg->getUseFastIntraTMP())
#else
{
numModesForFullRD += 1; // testing tpm
}
#if JVET_AG0136_INTRA_TMP_LIC
if( testTpm && m_pcEncCfg->getItmpLicMode())
{
numModesForFullRD += 1; // testing lic itmp
}
#endif
const int numHadCand = (testMip ? 2 : 1) * 3 + testTpm;
cu.tmpFlag = false;
const int numHadCand = (testMip ? 2 : 1) * 3;

Karsten Suehring
committed
#if JVET_AB0155_SGPM
static_vector<SgpmInfo, SGPM_NUM> sgpmInfoList;
static_vector<double, SGPM_NUM> sgpmCostList;
#if JVET_AG0152_SGPM_ITMP_IBC
int sgpmNeededMode[NUM_LUMA_MODE + SGPM_NUM_BVS] = { 0 };
#else
if (testSgpm && SGPMSaveFlag)
{
deriveSgpmModeOrdered(bestCS->picture->getRecoBuf(area), area, cu, sgpmInfoList, sgpmCostList);
Bappaditya Ray
committed
for (int sgpmIdx = 0; sgpmIdx < SGPM_NUM; sgpmIdx++)
{
int sgpmMode[2];
sgpmMode[0] = sgpmInfoList[sgpmIdx].sgpmMode0;
sgpmMode[1] = sgpmInfoList[sgpmIdx].sgpmMode1;
sgpmNeededMode[sgpmMode[0]] = 1;
sgpmNeededMode[sgpmMode[1]] = 1;
}
}
#endif
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
#if JVET_AH0076_OBIC
int dimdNeededMode[NUM_LUMA_MODE] = {0};
if (obicSaveFlag)
{
for (int idx = 0; idx < OBIC_FUSION_NUM; idx++)
{
int iMode = cu.obicMode[idx];
if (iMode < 0)
{
continue;
}
dimdNeededMode[iMode] = 1;
}
}
if (dimdSaveFlag)
{
if (cu.dimdBlending)
{
for (int dimdIdx = 0; dimdIdx < DIMD_FUSION_NUM - 1; dimdIdx++)
{
int dimdMode = (dimdIdx == 0 ? cu.dimdMode : cu.dimdBlendMode[dimdIdx-1]);
if (dimdMode <= 0)
{
break;
}
dimdNeededMode[dimdMode] = 1;
}
dimdNeededMode[PLANAR_IDX] = 1;
}
}
#endif
#if JVET_AB0157_TMRL
double tmrlCostList[MRL_LIST_SIZE]{ MAX_DOUBLE };
#endif
#if JVET_AC0105_DIRECTIONAL_PLANAR
double dirPlanarCostList[2]{ MAX_DOUBLE };
#endif
//*** Derive (regular) candidates using Hadamard
cu.mipFlag = false;

Karsten Suehring
committed
//===== init pattern for luma prediction =====
#if JVET_AB0157_INTRA_FUSION && JVET_AB0155_SGPM
initIntraPatternChType(cu, pu.Y(), true, 0, false);
#elif JVET_AB0157_INTRA_FUSION
initIntraPatternChType(cu, pu.Y(), true, false);
#else
bool bSatdChecked[NUM_INTRA_MODE];
memset(bSatdChecked, 0, sizeof(bSatdChecked));

Karsten Suehring
committed
if (!LFNSTLoadFlag)
{
for (int modeIdx = 0; modeIdx < numModesAvailable; modeIdx++)
{
uint32_t uiMode = modeIdx;
Distortion minSadHad = 0;

Karsten Suehring
committed
// Skip checking extended Angular modes in the first round of SATD
if (uiMode > DC_IDX && (uiMode & 1))
{
continue;

Karsten Suehring
committed
}
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
Bappaditya Ray
committed
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
#if JVET_AH0209_PDP
bool pdpMode = false;
if( pdpSaveFlag )
{
const int sizeKey = (width << 8) + height;
const int sizeIdx = g_size.find(sizeKey) != g_size.end() ? g_size[sizeKey] : -1;
const int m = sizeIdx > 12 ? 2 : 0;
const int s = sizeIdx > 12 ? 4 : 2;
if (sizeIdx >= 0 && m_refAvailable && pu.cu->cs->sps->getUsePDP() && !(modeIdx > 1 && modeIdx % s != m))
{
PelBuf predBuf(m_pdpIntraPredBuf[uiMode], tmpArea);
predBuf.copyFrom(piPred);
m_pdpIntraPredReady[modeIdx] = true;
pdpMode = true;
}
}
if( !pdpMode )
{
#endif
#if JVET_AB0155_SGPM
if (testSgpm && SGPMSaveFlag && sgpmNeededMode[uiMode])
{
Bappaditya Ray
committed
PelBuf predBuf(m_intraPredBuf[uiMode], tmpArea);
predBuf.copyFrom(piPred);
m_intraModeReady[uiMode] = 1;
}
#endif
#if JVET_AH0076_OBIC
if ((obicSaveFlag || dimdSaveFlag) && dimdNeededMode[uiMode] && !m_intraModeReady[uiMode])
{
Bappaditya Ray
committed
PelBuf predBuf(m_intraPredBuf[uiMode], tmpArea);
predBuf.copyFrom(piPred);
m_intraModeReady[uiMode] = 1;
}
#endif
Bappaditya Ray
committed
#if JVET_AH0209_PDP
}
#endif
// Use the min between SAD and HAD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
minSadHad += std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
minSadHad += std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, uiMode, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
DTRACE(g_trace_ctx, D_INTRA_COST, "IntraHAD: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, uiMode);
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, uiMode), cost, uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, uiMode), double(minSadHad),
Bappaditya Ray
committed
uiHadModeList, candHadList, numHadCand);
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
#if JVET_AC0105_DIRECTIONAL_PLANAR
bool testDirPlanar = isLuma(partitioner.chType);
if (testDirPlanar)
{
for (int dirPlanarModeIdx = 0; dirPlanarModeIdx < 2; dirPlanarModeIdx++)
{
cu.sgpm = false;
cu.ispMode = 0;
cu.tmpFlag = false;
cu.tmrlFlag = false;
pu.multiRefIdx = 0;
cu.mipFlag = false;
pu.intraDir[0] = PLANAR_IDX;
cu.plIdx = dirPlanarModeIdx + 1;
initPredIntraParams(pu, pu.Y(), sps);
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
predIntraAng(COMPONENT_Y, piPred, pu);
#endif
// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, PLANAR_IDX, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(
ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, dirPlanarModeIdx ? PL_VER_IDX : PL_HOR_IDX), cost,
uiRdModeList, candCostList, numModesForFullRD);
updateCandList(
ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, dirPlanarModeIdx ? PL_VER_IDX : PL_HOR_IDX),
Bappaditya Ray
committed
double(minSadHad), uiHadModeList, candHadList, numHadCand);
dirPlanarCostList[dirPlanarModeIdx] = cost;
}
}
cu.plIdx = 0;
#endif
if (!sps.getUseMIP() && LFNSTSaveFlag)
{
// save found best modes
m_uiSavedNumRdModesLFNST = numModesForFullRD;
m_uiSavedRdModeListLFNST = uiRdModeList;
m_dSavedModeCostLFNST = candCostList;
// PBINTRA fast
m_uiSavedHadModeListLFNST = uiHadModeList;
Bappaditya Ray
committed
m_dSavedHadListLFNST = candHadList;
LFNSTSaveFlag = false;
}
} // NSSTFlag
if (!sps.getUseMIP() && LFNSTLoadFlag)
// restore saved modes
numModesForFullRD = m_uiSavedNumRdModesLFNST;
uiRdModeList = m_uiSavedRdModeListLFNST;
candCostList = m_dSavedModeCostLFNST;
// PBINTRA fast
uiHadModeList = m_uiSavedHadModeListLFNST;
Bappaditya Ray
committed
candHadList = m_dSavedHadListLFNST;
if (!(sps.getUseMIP() && LFNSTLoadFlag))
{
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> parentCandList = uiRdModeList;
// Second round of SATD for extended Angular modes
for (int modeIdx = 0; modeIdx < numModesForFullRD; modeIdx++)
{
unsigned parentMode = parentCandList[modeIdx].modeId;
if (parentMode > (DC_IDX + 1) && parentMode < (NUM_LUMA_MODE - 1))
{
for (int subModeIdx = -1; subModeIdx <= 1; subModeIdx += 2)
{
unsigned mode = parentMode + subModeIdx;
if (!bSatdChecked[mode])
{
pu.intraDir[0] = mode;
initPredIntraParams(pu, pu.Y(), sps);
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
#if JVET_AB0155_SGPM
if (testSgpm && SGPMSaveFlag && sgpmNeededMode[mode])
{
Bappaditya Ray
committed
PelBuf predBuf(m_intraPredBuf[mode], tmpArea);
predBuf.copyFrom(piPred);
m_intraModeReady[mode] = 1;
}
#endif
#if JVET_AH0076_OBIC
if ((obicSaveFlag || dimdSaveFlag) && dimdNeededMode[mode] && !m_intraModeReady[mode])
{
Bappaditya Ray
committed
PelBuf predBuf(m_intraPredBuf[mode], tmpArea);
predBuf.copyFrom(piPred);
m_intraModeReady[mode] = 1;
}
#endif
// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, mode, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, mode), cost, uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, mode), double(minSadHad),
Bappaditya Ray
committed
uiHadModeList, candHadList, numHadCand);
bSatdChecked[mode] = true;
}
}
}
}
if (saveDataForISP)
// we save the regular intra modes list
m_ispCandListHor = uiRdModeList;
}
#if !JVET_AB0157_TMRL || JVET_AD0082_TMRL_CONFIG
#if SECONDARY_MPM
const int numMPMs = NUM_PRIMARY_MOST_PROBABLE_MODES;
#else
Fabrice URBAN
committed
uint8_t* multiRefMPM = m_intraMPM;
#if !JVET_AD0082_TMRL_CONFIG
#if JVET_AD0082_TMRL_CONFIG
cu.tmrlFlag = true;
#endif
for (auto multiRefIdx : EXT_REF_LINE_IDX)
{
pu.multiRefIdx = multiRefIdx;
initIntraPatternChType(cu, pu.Y(), true);
for (auto i = 0; i < MRL_LIST_SIZE; i++)
{
if (m_tmrlList[i].multiRefIdx != multiRefIdx)
pu.intraDir[0] = m_tmrlList[i].intraDir;
cu.tmrlListIdx = i;
uint32_t uiMode = i + MAX_REF_LINE_IDX;
initPredIntraParams(pu, pu.Y(), *(pu.cs->sps));
predIntraAng(COMPONENT_Y, piPred, pu);
// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, pu.intraDir[0], CHANNEL_TYPE_LUMA);
double cost = (double)minSadHad + (double)fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(ModeInfo(false, false, uiMode, NOT_INTRA_SUBPARTITIONS, 0), cost, uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, uiMode, NOT_INTRA_SUBPARTITIONS, 0), double(minSadHad),
Bappaditya Ray
committed
uiHadModeList, candHadList, numHadCand);
#if JVET_AB0157_TMRL
tmrlCostList[i] = cost;
#endif
}
}
#if JVET_AD0082_TMRL_CONFIG
cu.tmrlFlag = false;
#endif
#endif
#if !JVET_AB0157_TMRL || JVET_AD0082_TMRL_CONFIG
#if JVET_AD0082_TMRL_CONFIG
else
{
#endif
for (int mRefNum = 1; mRefNum < numOfPassesExtendRef; mRefNum++)
{
int multiRefIdx = MULTI_REF_LINE_IDX[mRefNum];
#if JVET_AB0157_INTRA_FUSION && JVET_AB0155_SGPM
initIntraPatternChType(cu, pu.Y(), true, 0, false);
#elif JVET_AB0157_INTRA_FUSION
initIntraPatternChType(cu, pu.Y(), true, false);
#else
}
for (int x = 1; x < numMPMs; x++)
{
uint32_t mode = multiRefMPM[x];
{
pu.intraDir[0] = mode;
initPredIntraParams(pu, pu.Y(), sps);
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, mode, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(ModeInfo(false, false, multiRefIdx, NOT_INTRA_SUBPARTITIONS, mode), cost, uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, multiRefIdx, NOT_INTRA_SUBPARTITIONS, mode), double(minSadHad),
Bappaditya Ray
committed
uiHadModeList, candHadList, numHadCand);
#if JVET_AD0082_TMRL_CONFIG
}
#endif
CHECKD(uiRdModeList.size() != numModesForFullRD, "Error: RD mode list size");
#if JVET_V0130_INTRA_TMP && JVET_AB0130_ITMP_SAMPLING
// derive TPM candidate using hadamard
if (testTpm)
{
cu.tmpFlag = true;
cu.mipFlag = false;
pu.multiRefIdx = 0;
#if JVET_AD0086_ENHANCED_INTRA_TMP
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListTmp;
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> candCostListTmp;
int foundCandiNum = 0;
bool bsuccessfull = 0;
Wei Chen
committed
#endif
#if JVET_AH0200_INTRA_TMP_BV_REORDER
int adjustedTMPNonLicBvNum = TMP_REFINE_NONLIC_BV_NUM;
int adjustedTmpLicBvNum = TMP_REFINE_LIC_BV_NUM;
if(m_pcEncCfg->getIntraPeriod() != 1)
{
adjustedTMPNonLicBvNum = 10;
adjustedTmpLicBvNum = 4;
}
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListFracTmp;
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> candCostListFracTmp;
Distortion backupMinSadHad[MTMP_NUM];
Distortion backupSadCost[MTMP_NUM];
Distortion backupLicMinSadHad[MTMP_NUM][4];
Distortion backupLicSadCost[MTMP_NUM][4];
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListLicFracTmp;
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> candCostListLicFracTmp;
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListTmpLic;
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> candCostListTmpLic;
CodingUnit cuCopy = cu;
#if JVET_W0069_TMP_BOUNDARY
RefTemplateType templateType = getRefTemplateType(cuCopy, cuCopy.blocks[COMPONENT_Y]);
if (templateType != NO_TEMPLATE)
#else
if (isRefTemplateAvailable(cuCopy, cuCopy.blocks[COMPONENT_Y]))
#endif
{
#if JVET_AD0086_ENHANCED_INTRA_TMP
cu.tmpIsSubPel = 0;
#if JVET_AG0136_INTRA_TMP_LIC
cu.tmpSubPelIdx = -1;
#else
cu.tmpSubPelIdx = 0;
Wei Chen
committed
#endif
#if JVET_AH0200_INTRA_TMP_BV_REORDER
cu.tmpFracIdx = 0;
int numModesForFracIntraTmp = numModesForFullRD + adjustedTMPNonLicBvNum;
#if JVET_AI0136_ADAPTIVE_DUAL_TREE
if(relatedCU && relatedCU->skipFracTmp)
#else
Wei Chen
committed
if(relatedCU.skipFracTmp)
#endif
Wei Chen
committed
{
numModesForFracIntraTmp = numModesForFullRD;
}
if(m_tmpNumCand > 0)
{
for(int idxInList=0; idxInList < uiRdModeList.size(); idxInList++)
{
if(cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD)
{
break;
}
updateCandList(uiRdModeList[idxInList], candCostList[idxInList], uiRdModeListFracTmp, candCostListFracTmp, numModesForFracIntraTmp);
}
}
int numModesForLicFracIntraTmp = numModesForFullRD + adjustedTmpLicBvNum;
#if JVET_AI0136_ADAPTIVE_DUAL_TREE
if(relatedCU && relatedCU->skipFracTmp)
#else
Wei Chen
committed
if(relatedCU.skipFracTmp)
#endif
Wei Chen
committed
{
numModesForLicFracIntraTmp = numModesForFullRD;
}
if(m_tmpNumCandUseMR > 0)
{
for(int idxInList=0; idxInList < uiRdModeList.size(); idxInList++)
{
if(cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD)
{
break;
}
updateCandList(uiRdModeList[idxInList], candCostList[idxInList], uiRdModeListLicFracTmp, candCostListLicFracTmp, numModesForLicFracIntraTmp);
}
}
for(int tmpFusionFlag = 0; tmpFusionFlag <= 1; tmpFusionFlag++)
{
cu.tmpFusionFlag = tmpFusionFlag ? true: false;
for (int tmpFlmFlag = 0; tmpFlmFlag <= 1; tmpFlmFlag++)
{
cu.tmpFlmFlag = tmpFlmFlag ? true: false;
if(tmpFlmFlag && tmpFusionFlag)
{
continue;
}
#if JVET_AG0136_INTRA_TMP_LIC
for (int tmpLicFlag = 0; tmpLicFlag <= 1; tmpLicFlag++)
{
cu.ibcLicFlag = tmpLicFlag ? true : false;
cu.tmpLicFlag = tmpLicFlag ? true : false;
if (tmpLicFlag && tmpFlmFlag)
{
continue;
}
const int ibcLicLoopNum = (cu.slice->getSPS()->getItmpLicExtension() && cu.tmpLicFlag && !cu.tmpFusionFlag) ? 4 : 1;
for (int licIdc = 0; licIdc < ibcLicLoopNum; licIdc++)
{
cu.ibcLicIdx = licIdc;
int idxNum = cu.tmpFusionFlag ? TMP_GROUP_IDX << 1 : (cu.tmpLicFlag ? m_tmpNumCandUseMR : m_tmpNumCand);
#else
int idxNum = cu.tmpFusionFlag ? TMP_GROUP_IDX << 1 : m_tmpNumCand;
for (int tmpIdx = 0; tmpIdx < idxNum; tmpIdx++)
{
#if JVET_AG0136_INTRA_TMP_LIC
if (cu.tmpFusionFlag && !(cu.tmpLicFlag ? m_tmpFusionInfoUseMR : m_tmpFusionInfo)[tmpIdx].bValid)
#else
if(cu.tmpFusionFlag && !m_tmpFusionInfo[tmpIdx].bValid)
{
continue;
}
cu.tmpIdx = tmpIdx;
int placeHolder;
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
generateTMPrediction(piPred.buf, piPred.stride, placeHolder, pu
#if JVET_AG0136_INTRA_TMP_LIC
, cu.tmpLicFlag
#endif
, false);
#if JVET_AG0136_INTRA_TMP_LIC
if (cu.tmpLicFlag)
{
if (!cu.tmpFusionFlag)
{
const auto& arrayLicParams = getMemLicParams(cu.ibcLicIdx, cu.tmpIdx);
if (cu.ibcLicIdx == IBC_LIC_IDX_M)
{
piPred.linearTransforms(arrayLicParams[1], arrayLicParams[0], arrayLicParams[2], arrayLicParams[4], arrayLicParams[3], arrayLicParams[5], arrayLicParams[6], true, cu.cs->slice->clpRng(COMPONENT_Y));
}
else
{
piPred.linearTransform(arrayLicParams[1], arrayLicParams[0], arrayLicParams[2], true, cu.cs->slice->clpRng(COMPONENT_Y));
}
}
}
#endif
xGenerateTmpFlmPred(piPred, pu.lwidth(), pu.lheight(), templateType, pu.cu, false);
xTMPFusionApplyModel(piPred, pu.lwidth(), pu.lheight(), templateType, pu.cu
#if JVET_AG0136_INTRA_TMP_LIC
, cu.tmpLicFlag
#endif
, false);
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
#if JVET_W0069_TMP_BOUNDARY
#if TMP_FAST_ENC
bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.Y(), foundCandiNum, pu.cu);
#else
getTargetTemplate(&cuCopy, pu.lwidth(), pu.lheight(), templateType);
candidateSearchIntra(&cuCopy, pu.lwidth(), pu.lheight(), templateType);
bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), foundCandiNum);
#endif
#else
#if TMP_FAST_ENC
bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.Y(), foundCandiNum, pu.cu);
#else
getTargetTemplate(&cuCopy, pu.lwidth(), pu.lheight());
candidateSearchIntra(&cuCopy, pu.lwidth(), pu.lheight());
bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), foundCandiNum);
#endif
#endif
}
#if JVET_W0069_TMP_BOUNDARY
else
{
foundCandiNum = 1;
#if JVET_AC0115_INTRA_TMP_DIMD_MTS_LFNST
bsuccessfull = generateTmDcPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), 1 << (cuCopy.cs->sps->getBitDepth(CHANNEL_TYPE_LUMA) - 1), pu.cu);
#else
bsuccessfull = generateTmDcPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), 1 << (cuCopy.cs->sps->getBitDepth(CHANNEL_TYPE_LUMA) - 1));
#endif
}
#endif
if (bsuccessfull && foundCandiNum >= 1)
{
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
#if JVET_AG0136_INTRA_TMP_LIC
m_CABACEstimator->getCtx() = SubCtx(Ctx::TmpLic, ctxStartTmpLicFlag);
m_CABACEstimator->getCtx() = SubCtx(Ctx::ItmpLicIndex, ctxStartTmpLicIdx);
#endif
uint64_t fracModeBits = xFracModeBitsIntra(pu, 0, CHANNEL_TYPE_LUMA);
double cost = double(minSadHad) + double(fracModeBits) * sqrtLambdaForFirstPass;
Wei Chen
committed
#if JVET_AH0200_INTRA_TMP_BV_REORDER
isTmpModeTestd = true;
if(!tmpFusionFlag && !tmpFlmFlag)
{
if(tmpBestSatdCost > cost)
{
tmpBestSatdCost = cost;
}
}
#endif
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Wei Chen
committed
#if JVET_AH0200_INTRA_TMP_BV_REORDER
if(tmpFlmFlag || tmpFusionFlag
|| (cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD))
{
#endif
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - double(minSadHad) + (double)sadCost);
#if JVET_AH0200_INTRA_TMP_BV_REORDER
}
#endif
#endif
DTRACE(g_trace_ctx, D_INTRA_COST, "IntraTPM: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, 0);
#if JVET_AD0086_ENHANCED_INTRA_TMP
#if JVET_AG0136_INTRA_TMP_LIC
Wei Chen
committed
#if JVET_AH0200_INTRA_TMP_BV_REORDER
if(tmpFlmFlag || tmpFusionFlag || (cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD))
{
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx, cu.tmpFracIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
Bappaditya Ray
committed
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx, cu.tmpFracIdx), 0.8 * double(minSadHad), uiHadModeList, candHadList, numHadCand);
Wei Chen
committed
}
else if (cu.tmpLicFlag)
{
backupLicMinSadHad[cu.tmpIdx][cu.ibcLicIdx] = minSadHad;
backupLicSadCost[cu.tmpIdx][cu.ibcLicIdx] = sadCost;
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx, cu.tmpFracIdx), cost, uiRdModeListLicFracTmp, candCostListLicFracTmp, numModesForLicFracIntraTmp);
}
else
{
backupMinSadHad[cu.tmpIdx] = minSadHad;
backupSadCost[cu.tmpIdx] = sadCost;
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx, cu.tmpFracIdx), cost, uiRdModeListFracTmp, candCostListFracTmp, numModesForFracIntraTmp);
}
#else
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
Bappaditya Ray
committed
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, candHadList, numHadCand);
Wei Chen
committed
#endif
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
Bappaditya Ray
committed
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, candHadList, numHadCand);
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1), cost, uiRdModeList, candCostList, numModesForFullRD);
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1), 0.8 * double(minSadHad), uiHadModeList, CandHadList, numHadCand);
#endif
#if JVET_AD0086_ENHANCED_INTRA_TMP
//record the best full-pel candidates
Wei Chen
committed
#if JVET_AH0200_INTRA_TMP_BV_REORDER
if (!tmpFlmFlag && !tmpFusionFlag && tmpLicFlag)
{
for(int idxInList=0; idxInList < uiRdModeListLicFracTmp.size(); idxInList++)
{
if(cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD)
{
break;
}
if(uiRdModeListLicFracTmp[idxInList].tmpFlag)
{
updateCandList(uiRdModeListLicFracTmp[idxInList], candCostListLicFracTmp[idxInList], uiRdModeListTmpLic, candCostListTmpLic, numModesForLicFracIntraTmp);
}
if(idxInList >= numModesForFullRD && uiRdModeListTmpLic.size() >= adjustedTmpLicBvNum)
{
break;
}
#if JVET_AI0136_ADAPTIVE_DUAL_TREE
if(idxInList >= (numModesForFullRD-1) && relatedCU && relatedCU->skipFracTmp)
#else
Wei Chen
committed
if(idxInList >= (numModesForFullRD-1) && relatedCU.skipFracTmp)
#endif
Wei Chen
committed
{
break;
}
}
}
#endif
#if JVET_AG0136_INTRA_TMP_LIC
if (!tmpFlmFlag && !tmpFusionFlag && !tmpLicFlag)
#else
if(!tmpFlmFlag&&!tmpFusionFlag)
Wei Chen
committed
#if JVET_AH0200_INTRA_TMP_BV_REORDER
for(int idxInList=0; idxInList < uiRdModeListFracTmp.size(); idxInList++)
{
if(cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD)
{
break;
}
if(uiRdModeListFracTmp[idxInList].tmpFlag)
{
updateCandList(uiRdModeListFracTmp[idxInList], candCostListFracTmp[idxInList], uiRdModeListTmp, candCostListTmp, numModesForFracIntraTmp);
}
if(idxInList >= numModesForFullRD && uiRdModeListTmp.size() >= adjustedTMPNonLicBvNum)
{
break;
}
#if JVET_AI0136_ADAPTIVE_DUAL_TREE
if(idxInList >= (numModesForFullRD-1) && relatedCU && relatedCU->skipFracTmp)
#else
Wei Chen
committed
if(idxInList >= (numModesForFullRD-1) && relatedCU.skipFracTmp)
#endif
Wei Chen
committed
{
break;
}
}
#else
for(int idxInList=0; idxInList < uiRdModeList.size(); idxInList++)
{
if(uiRdModeList[idxInList].tmpFlag){
updateCandList(uiRdModeList[idxInList], candCostList[idxInList], uiRdModeListTmp, candCostListTmp, numModesForFullRD);
Wei Chen
committed
#endif
#if JVET_AG0136_INTRA_TMP_LIC
}
}
#endif
}
}
//fractional BV
cu.tmpFusionFlag = false;
cu.tmpFlmFlag = false;
#if JVET_AG0136_INTRA_TMP_LIC
cu.tmpLicFlag = false;
cu.ibcLicIdx = 0;
#endif
Wei Chen
committed
#if !JVET_AH0200_INTRA_TMP_BV_REORDER
for(int idxInList=0; idxInList < uiRdModeListTmp.size(); idxInList++)
{
cu.tmpIdx = uiRdModeListTmp[idxInList].tmpIdx;
xPadForInterpolation(&cu);
for(int tmpIsSubPel = 1; tmpIsSubPel < 4; tmpIsSubPel++)
{
for (int idx = 0; idx < TMP_MAX_SUBPEL_DIR; idx++)
{
cu.tmpIsSubPel = tmpIsSubPel;
cu.tmpSubPelIdx = idx;
int placeHolder;
generateTMPrediction(piPred.buf, piPred.stride, placeHolder, pu
#if JVET_AG0136_INTRA_TMP_LIC
, cu.tmpLicFlag
#endif
, false);
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS&&JVET_AE0169_BIPREDICTIVE_IBC
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, 0, CHANNEL_TYPE_LUMA);
double cost = double(minSadHad) + double(fracModeBits) * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS&&JVET_AE0169_BIPREDICTIVE_IBC
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - double(minSadHad) + (double)sadCost);
#endif
DTRACE(g_trace_ctx, D_INTRA_COST, "IntraTPM: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, 0);
#if JVET_AG0136_INTRA_TMP_LIC
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
Bappaditya Ray
committed
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, candHadList, numHadCand);
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
Bappaditya Ray
committed
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, candHadList, numHadCand);
}
}
}
Wei Chen
committed
#endif
#if JVET_AH0200_INTRA_TMP_BV_REORDER
for(int idxInList=0; idxInList < uiRdModeListTmp.size(); idxInList++)
{
cu.tmpIdx = uiRdModeListTmp[idxInList].tmpIdx;
cu.tmpLicFlag = uiRdModeListTmp[idxInList].tmpLicFlag;
CHECK(cu.tmpLicFlag, "cu.tmpLicFlag == 1");
cu.ibcLicFlag = cu.tmpLicFlag;
cu.ibcLicIdx = uiRdModeListTmp[idxInList].tmpLicIdc;
Thierry Dumas
committed
#if JVET_AI0129_INTRA_TMP_OVERLAPPING_REFINEMENT
searchFracCandidate(&cu, getTargetPatch(), templateType);
#else
Wei Chen
committed
searchFracCandidate(&cu, getTargetPatch(floorLog2(std::max(cu.lwidth(), cu.lheight())) - 2), templateType);
Thierry Dumas
committed
#endif
Wei Chen
committed
for (int spIdx = 0; spIdx < std::min(2, (int) m_mtmpFracCandList[cu.tmpIdx].size()); spIdx++)
{
cu.tmpIsSubPel = m_mtmpFracCandList[cu.tmpIdx][spIdx].m_subpel;
cu.tmpSubPelIdx = m_mtmpFracCandList[cu.tmpIdx][spIdx].m_fracDir;
CHECK(cu.tmpIsSubPel < 0 || cu.tmpIsSubPel > 2, "cu.tmpIsSubPel < 1 || cu.tmpIsSubPel > 2");
cu.tmpFracIdx = spIdx;
Distortion sadCost;
Distortion minSadHad;