Newer
Older
{
numModesForFullRD += 1; // testing tpm
}
#if JVET_AG0136_INTRA_TMP_LIC
if( testTpm && m_pcEncCfg->getItmpLicMode())
{
numModesForFullRD += 1; // testing lic itmp
}
#endif
const int numHadCand = (testMip ? 2 : 1) * 3 + testTpm;
cu.tmpFlag = false;
const int numHadCand = (testMip ? 2 : 1) * 3;

Karsten Suehring
committed
#if JVET_AB0155_SGPM
static_vector<SgpmInfo, SGPM_NUM> sgpmInfoList;
static_vector<double, SGPM_NUM> sgpmCostList;
#if JVET_AG0152_SGPM_ITMP_IBC
int sgpmNeededMode[NUM_LUMA_MODE + SGPM_NUM_BVS] = { 0 };
#else
if (testSgpm && SGPMSaveFlag)
{
deriveSgpmModeOrdered(bestCS->picture->getRecoBuf(area), area, cu, sgpmInfoList, sgpmCostList);
for (int sgpmIdx = 0; sgpmIdx < SGPM_NUM; sgpmIdx++)
{
int sgpmMode[2];
sgpmMode[0] = sgpmInfoList[sgpmIdx].sgpmMode0;
sgpmMode[1] = sgpmInfoList[sgpmIdx].sgpmMode1;
sgpmNeededMode[sgpmMode[0]] = 1;
sgpmNeededMode[sgpmMode[1]] = 1;
}
}
#endif
#if JVET_AB0157_TMRL
double tmrlCostList[MRL_LIST_SIZE]{ MAX_DOUBLE };
#endif
#if JVET_AC0105_DIRECTIONAL_PLANAR
double dirPlanarCostList[2]{ MAX_DOUBLE };
#endif
//*** Derive (regular) candidates using Hadamard
cu.mipFlag = false;

Karsten Suehring
committed
//===== init pattern for luma prediction =====
#if JVET_AB0157_INTRA_FUSION && JVET_AB0155_SGPM
initIntraPatternChType(cu, pu.Y(), true, 0, false);
#elif JVET_AB0157_INTRA_FUSION
initIntraPatternChType(cu, pu.Y(), true, false);
#else
bool bSatdChecked[NUM_INTRA_MODE];
memset(bSatdChecked, 0, sizeof(bSatdChecked));

Karsten Suehring
committed
if (!LFNSTLoadFlag)
{
for (int modeIdx = 0; modeIdx < numModesAvailable; modeIdx++)
{
uint32_t uiMode = modeIdx;
Distortion minSadHad = 0;

Karsten Suehring
committed
// Skip checking extended Angular modes in the first round of SATD
if (uiMode > DC_IDX && (uiMode & 1))
{
continue;

Karsten Suehring
committed
}
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
#if JVET_AB0155_SGPM
if (testSgpm && SGPMSaveFlag && sgpmNeededMode[uiMode])
{
PelBuf predBuf(m_intraPredBuf[uiMode], tmpArea);
predBuf.copyFrom(piPred);
m_intraModeReady[uiMode] = 1;
}
#endif
// Use the min between SAD and HAD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
minSadHad += std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
minSadHad += std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, uiMode, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
DTRACE(g_trace_ctx, D_INTRA_COST, "IntraHAD: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, uiMode);
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, uiMode), cost, uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, uiMode), double(minSadHad),
uiHadModeList, CandHadList, numHadCand);
}
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
#if JVET_AC0105_DIRECTIONAL_PLANAR
bool testDirPlanar = isLuma(partitioner.chType);
if (testDirPlanar)
{
for (int dirPlanarModeIdx = 0; dirPlanarModeIdx < 2; dirPlanarModeIdx++)
{
cu.sgpm = false;
cu.ispMode = 0;
cu.tmpFlag = false;
cu.tmrlFlag = false;
pu.multiRefIdx = 0;
cu.mipFlag = false;
pu.intraDir[0] = PLANAR_IDX;
cu.plIdx = dirPlanarModeIdx + 1;
initPredIntraParams(pu, pu.Y(), sps);
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
predIntraAng(COMPONENT_Y, piPred, pu);
#endif
// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, PLANAR_IDX, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(
ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, dirPlanarModeIdx ? PL_VER_IDX : PL_HOR_IDX), cost,
uiRdModeList, candCostList, numModesForFullRD);
updateCandList(
ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, dirPlanarModeIdx ? PL_VER_IDX : PL_HOR_IDX),
double(minSadHad), uiHadModeList, CandHadList, numHadCand);
dirPlanarCostList[dirPlanarModeIdx] = cost;
}
}
cu.plIdx = 0;
#endif
if (!sps.getUseMIP() && LFNSTSaveFlag)
{
// save found best modes
m_uiSavedNumRdModesLFNST = numModesForFullRD;
m_uiSavedRdModeListLFNST = uiRdModeList;
m_dSavedModeCostLFNST = candCostList;
// PBINTRA fast
m_uiSavedHadModeListLFNST = uiHadModeList;
m_dSavedHadListLFNST = CandHadList;
LFNSTSaveFlag = false;
}
} // NSSTFlag
if (!sps.getUseMIP() && LFNSTLoadFlag)
// restore saved modes
numModesForFullRD = m_uiSavedNumRdModesLFNST;
uiRdModeList = m_uiSavedRdModeListLFNST;
candCostList = m_dSavedModeCostLFNST;
// PBINTRA fast
uiHadModeList = m_uiSavedHadModeListLFNST;
CandHadList = m_dSavedHadListLFNST;
} // !LFNSTFlag
if (!(sps.getUseMIP() && LFNSTLoadFlag))
{
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> parentCandList = uiRdModeList;
// Second round of SATD for extended Angular modes
for (int modeIdx = 0; modeIdx < numModesForFullRD; modeIdx++)
{
unsigned parentMode = parentCandList[modeIdx].modeId;
if (parentMode > (DC_IDX + 1) && parentMode < (NUM_LUMA_MODE - 1))
{
for (int subModeIdx = -1; subModeIdx <= 1; subModeIdx += 2)
{
unsigned mode = parentMode + subModeIdx;
if (!bSatdChecked[mode])
{
pu.intraDir[0] = mode;
initPredIntraParams(pu, pu.Y(), sps);
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
#if JVET_AB0155_SGPM
if (testSgpm && SGPMSaveFlag && sgpmNeededMode[mode])
{
PelBuf predBuf(m_intraPredBuf[mode], tmpArea);
predBuf.copyFrom(piPred);
m_intraModeReady[mode] = 1;
}
#endif
// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, mode, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, mode), cost, uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, mode), double(minSadHad),
uiHadModeList, CandHadList, numHadCand);
bSatdChecked[mode] = true;
}
}
}
}
if (saveDataForISP)
// we save the regular intra modes list
m_ispCandListHor = uiRdModeList;
}
#if !JVET_AB0157_TMRL || JVET_AD0082_TMRL_CONFIG
#if SECONDARY_MPM
const int numMPMs = NUM_PRIMARY_MOST_PROBABLE_MODES;
#else
Fabrice URBAN
committed
uint8_t* multiRefMPM = m_intraMPM;
#if !JVET_AD0082_TMRL_CONFIG
#if JVET_AD0082_TMRL_CONFIG
cu.tmrlFlag = true;
#endif
for (auto multiRefIdx : EXT_REF_LINE_IDX)
{
pu.multiRefIdx = multiRefIdx;
initIntraPatternChType(cu, pu.Y(), true);
for (auto i = 0; i < MRL_LIST_SIZE; i++)
{
if (m_tmrlList[i].multiRefIdx != multiRefIdx)
pu.intraDir[0] = m_tmrlList[i].intraDir;
cu.tmrlListIdx = i;
uint32_t uiMode = i + MAX_REF_LINE_IDX;
initPredIntraParams(pu, pu.Y(), *(pu.cs->sps));
predIntraAng(COMPONENT_Y, piPred, pu);
// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, pu.intraDir[0], CHANNEL_TYPE_LUMA);
double cost = (double)minSadHad + (double)fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(ModeInfo(false, false, uiMode, NOT_INTRA_SUBPARTITIONS, 0), cost, uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, uiMode, NOT_INTRA_SUBPARTITIONS, 0), double(minSadHad),
uiHadModeList, CandHadList, numHadCand);
#if JVET_AB0157_TMRL
tmrlCostList[i] = cost;
#endif
}
}
#if JVET_AD0082_TMRL_CONFIG
cu.tmrlFlag = false;
#endif
#endif
#if !JVET_AB0157_TMRL || JVET_AD0082_TMRL_CONFIG
#if JVET_AD0082_TMRL_CONFIG
else
{
#endif
for (int mRefNum = 1; mRefNum < numOfPassesExtendRef; mRefNum++)
{
int multiRefIdx = MULTI_REF_LINE_IDX[mRefNum];
#if JVET_AB0157_INTRA_FUSION && JVET_AB0155_SGPM
initIntraPatternChType(cu, pu.Y(), true, 0, false);
#elif JVET_AB0157_INTRA_FUSION
initIntraPatternChType(cu, pu.Y(), true, false);
#else
}
for (int x = 1; x < numMPMs; x++)
{
uint32_t mode = multiRefMPM[x];
{
pu.intraDir[0] = mode;
initPredIntraParams(pu, pu.Y(), sps);
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, mode, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(ModeInfo(false, false, multiRefIdx, NOT_INTRA_SUBPARTITIONS, mode), cost, uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, multiRefIdx, NOT_INTRA_SUBPARTITIONS, mode), double(minSadHad),
uiHadModeList, CandHadList, numHadCand);
}
}
}
#if JVET_AD0082_TMRL_CONFIG
}
#endif
CHECKD(uiRdModeList.size() != numModesForFullRD, "Error: RD mode list size");
#if JVET_V0130_INTRA_TMP && JVET_AB0130_ITMP_SAMPLING
// derive TPM candidate using hadamard
if (testTpm)
{
cu.tmpFlag = true;
cu.mipFlag = false;
pu.multiRefIdx = 0;
#if JVET_AD0086_ENHANCED_INTRA_TMP
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListTmp;
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> candCostListTmp;
int foundCandiNum = 0;
bool bsuccessfull = 0;
CodingUnit cuCopy = cu;
#if JVET_W0069_TMP_BOUNDARY
RefTemplateType templateType = getRefTemplateType(cuCopy, cuCopy.blocks[COMPONENT_Y]);
if (templateType != NO_TEMPLATE)
#else
if (isRefTemplateAvailable(cuCopy, cuCopy.blocks[COMPONENT_Y]))
#endif
{
#if JVET_AD0086_ENHANCED_INTRA_TMP
cu.tmpIsSubPel = 0;
#if JVET_AG0136_INTRA_TMP_LIC
cu.tmpSubPelIdx = -1;
#else
cu.tmpSubPelIdx = 0;
for(int tmpFusionFlag = 0; tmpFusionFlag <= 1; tmpFusionFlag++)
{
cu.tmpFusionFlag = tmpFusionFlag ? true: false;
for (int tmpFlmFlag = 0; tmpFlmFlag <= 1; tmpFlmFlag++)
{
cu.tmpFlmFlag = tmpFlmFlag ? true: false;
if(tmpFlmFlag && tmpFusionFlag)
{
continue;
}
#if JVET_AG0136_INTRA_TMP_LIC
for (int tmpLicFlag = 0; tmpLicFlag <= 1; tmpLicFlag++)
{
cu.ibcLicFlag = tmpLicFlag ? true : false;
cu.tmpLicFlag = tmpLicFlag ? true : false;
if (tmpLicFlag && tmpFlmFlag)
{
continue;
}
const int ibcLicLoopNum = (cu.slice->getSPS()->getItmpLicExtension() && cu.tmpLicFlag && !cu.tmpFusionFlag) ? 4 : 1;
for (int licIdc = 0; licIdc < ibcLicLoopNum; licIdc++)
{
cu.ibcLicIdx = licIdc;
int idxNum = cu.tmpFusionFlag ? TMP_GROUP_IDX << 1 : (cu.tmpLicFlag ? m_tmpNumCandUseMR : m_tmpNumCand);
#else
int idxNum = cu.tmpFusionFlag ? TMP_GROUP_IDX << 1 : m_tmpNumCand;
for (int tmpIdx = 0; tmpIdx < idxNum; tmpIdx++)
{
#if JVET_AG0136_INTRA_TMP_LIC
if (cu.tmpFusionFlag && !(cu.tmpLicFlag ? m_tmpFusionInfoUseMR : m_tmpFusionInfo)[tmpIdx].bValid)
#else
if(cu.tmpFusionFlag && !m_tmpFusionInfo[tmpIdx].bValid)
{
continue;
}
cu.tmpIdx = tmpIdx;
int placeHolder;
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
generateTMPrediction(piPred.buf, piPred.stride, placeHolder, pu
#if JVET_AG0136_INTRA_TMP_LIC
, cu.tmpLicFlag
#endif
, false);
#if JVET_AG0136_INTRA_TMP_LIC
if (cu.tmpLicFlag)
{
if (!cu.tmpFusionFlag)
{
const auto& arrayLicParams = getMemLicParams(cu.ibcLicIdx, cu.tmpIdx);
if (cu.ibcLicIdx == IBC_LIC_IDX_M)
{
piPred.linearTransforms(arrayLicParams[1], arrayLicParams[0], arrayLicParams[2], arrayLicParams[4], arrayLicParams[3], arrayLicParams[5], arrayLicParams[6], true, cu.cs->slice->clpRng(COMPONENT_Y));
}
else
{
piPred.linearTransform(arrayLicParams[1], arrayLicParams[0], arrayLicParams[2], true, cu.cs->slice->clpRng(COMPONENT_Y));
}
}
}
#endif
xGenerateTmpFlmPred(piPred, pu.lwidth(), pu.lheight(), templateType, pu.cu, false);
xTMPFusionApplyModel(piPred, pu.lwidth(), pu.lheight(), templateType, pu.cu
#if JVET_AG0136_INTRA_TMP_LIC
, cu.tmpLicFlag
#endif
, false);
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
#if JVET_W0069_TMP_BOUNDARY
#if TMP_FAST_ENC
bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.Y(), foundCandiNum, pu.cu);
#else
getTargetTemplate(&cuCopy, pu.lwidth(), pu.lheight(), templateType);
candidateSearchIntra(&cuCopy, pu.lwidth(), pu.lheight(), templateType);
bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), foundCandiNum);
#endif
#else
#if TMP_FAST_ENC
bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.Y(), foundCandiNum, pu.cu);
#else
getTargetTemplate(&cuCopy, pu.lwidth(), pu.lheight());
candidateSearchIntra(&cuCopy, pu.lwidth(), pu.lheight());
bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), foundCandiNum);
#endif
#endif
}
#if JVET_W0069_TMP_BOUNDARY
else
{
foundCandiNum = 1;
#if JVET_AC0115_INTRA_TMP_DIMD_MTS_LFNST
bsuccessfull = generateTmDcPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), 1 << (cuCopy.cs->sps->getBitDepth(CHANNEL_TYPE_LUMA) - 1), pu.cu);
#else
bsuccessfull = generateTmDcPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), 1 << (cuCopy.cs->sps->getBitDepth(CHANNEL_TYPE_LUMA) - 1));
#endif
}
#endif
if (bsuccessfull && foundCandiNum >= 1)
{
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
#if JVET_AG0136_INTRA_TMP_LIC
m_CABACEstimator->getCtx() = SubCtx(Ctx::TmpLic, ctxStartTmpLicFlag);
m_CABACEstimator->getCtx() = SubCtx(Ctx::ItmpLicIndex, ctxStartTmpLicIdx);
#endif
uint64_t fracModeBits = xFracModeBitsIntra(pu, 0, CHANNEL_TYPE_LUMA);
double cost = double(minSadHad) + double(fracModeBits) * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - double(minSadHad) + (double)sadCost);
#endif
DTRACE(g_trace_ctx, D_INTRA_COST, "IntraTPM: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, 0);
#if JVET_AD0086_ENHANCED_INTRA_TMP
#if JVET_AG0136_INTRA_TMP_LIC
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, CandHadList, numHadCand);
#else
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, CandHadList, numHadCand);
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1), cost, uiRdModeList, candCostList, numModesForFullRD);
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1), 0.8 * double(minSadHad), uiHadModeList, CandHadList, numHadCand);
#endif
#if JVET_AD0086_ENHANCED_INTRA_TMP
//record the best full-pel candidates
#if JVET_AG0136_INTRA_TMP_LIC
if (!tmpFlmFlag && !tmpFusionFlag && !tmpLicFlag)
#else
if(!tmpFlmFlag&&!tmpFusionFlag)
{
for(int idxInList=0; idxInList < uiRdModeList.size(); idxInList++)
{
if(uiRdModeList[idxInList].tmpFlag){
updateCandList(uiRdModeList[idxInList], candCostList[idxInList], uiRdModeListTmp, candCostListTmp, numModesForFullRD);
}
}
}
#if JVET_AG0136_INTRA_TMP_LIC
}
}
#endif
}
}
//fractional BV
cu.tmpFusionFlag = false;
cu.tmpFlmFlag = false;
#if JVET_AG0136_INTRA_TMP_LIC
cu.tmpLicFlag = false;
cu.ibcLicIdx = 0;
#endif
for(int idxInList=0; idxInList < uiRdModeListTmp.size(); idxInList++)
{
cu.tmpIdx = uiRdModeListTmp[idxInList].tmpIdx;
xPadForInterpolation(&cu);
for(int tmpIsSubPel = 1; tmpIsSubPel < 4; tmpIsSubPel++)
{
for (int idx = 0; idx < TMP_MAX_SUBPEL_DIR; idx++)
{
cu.tmpIsSubPel = tmpIsSubPel;
cu.tmpSubPelIdx = idx;
int placeHolder;
generateTMPrediction(piPred.buf, piPred.stride, placeHolder, pu
#if JVET_AG0136_INTRA_TMP_LIC
, cu.tmpLicFlag
#endif
, false);
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS&&JVET_AE0169_BIPREDICTIVE_IBC
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, 0, CHANNEL_TYPE_LUMA);
double cost = double(minSadHad) + double(fracModeBits) * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS&&JVET_AE0169_BIPREDICTIVE_IBC
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - double(minSadHad) + (double)sadCost);
#endif
DTRACE(g_trace_ctx, D_INTRA_COST, "IntraTPM: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, 0);
#if JVET_AG0136_INTRA_TMP_LIC
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, CandHadList, numHadCand);
#else
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, CandHadList, numHadCand);
}
}
}
#endif
#if JVET_AD0086_ENHANCED_INTRA_TMP
cu.tmpFlag = 0;
cu.tmpFusionFlag = false;
cu.tmpFlmFlag = false;
#if JVET_AG0136_INTRA_TMP_LIC
cu.tmpLicFlag = false;
cu.ibcLicFlag = false;
cu.ibcLicIdx = 0;
#endif
cu.tmpIsSubPel = 0;
#if JVET_AG0136_INTRA_TMP_LIC
cu.tmpSubPelIdx = -1;
#else
cu.tmpSubPelIdx = 0;
cu.tmpIdx = 0;
#endif
}
#endif
if (LFNSTSaveFlag && testMip
&& !allowLfnstWithMip(cu.firstPU->lumaSize())) // save a different set for the next run
{
// save found best modes
m_uiSavedRdModeListLFNST = uiRdModeList;
m_dSavedModeCostLFNST = candCostList;
// PBINTRA fast
m_uiSavedHadModeListLFNST = uiHadModeList;
m_dSavedHadListLFNST = CandHadList;
m_uiSavedNumRdModesLFNST =
g_aucIntraModeNumFast_UseMPM_2D[uiWidthBit - MIN_CU_LOG2][uiHeightBit - MIN_CU_LOG2];
m_uiSavedRdModeListLFNST.resize(m_uiSavedNumRdModesLFNST);
m_dSavedModeCostLFNST.resize(m_uiSavedNumRdModesLFNST);
// PBINTRA fast
m_uiSavedHadModeListLFNST.resize(3);
m_dSavedHadListLFNST.resize(3);
LFNSTSaveFlag = false;
}
#if JVET_V0130_INTRA_TMP && !JVET_AB0130_ITMP_SAMPLING
// derive TPM candidate using hadamard
if( testTpm )
{
cu.tmpFlag = true;
cu.mipFlag = false;
pu.multiRefIdx = 0;
#if JVET_AB0157_TMRL
cu.tmrlFlag = false;
#endif
int foundCandiNum = 0;
bool bsuccessfull = 0;
CodingUnit cu_cpy = cu;
RefTemplateType templateType = getRefTemplateType( cu_cpy, cu_cpy.blocks[COMPONENT_Y] );
if( templateType != NO_TEMPLATE )
if( isRefTemplateAvailable( cu_cpy, cu_cpy.blocks[COMPONENT_Y] ) )
getTargetTemplate( &cu_cpy, pu.lwidth(), pu.lheight(), templateType );
candidateSearchIntra( &cu_cpy, pu.lwidth(), pu.lheight(), templateType );
bsuccessfull = generateTMPrediction( piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), foundCandiNum );
getTargetTemplate( &cu_cpy, pu.lwidth(), pu.lheight() );
candidateSearchIntra( &cu_cpy, pu.lwidth(), pu.lheight() );
bsuccessfull = generateTMPrediction( piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), foundCandiNum );
bsuccessfull = generateTmDcPrediction( piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), 1 << (cu_cpy.cs->sps->getBitDepth( CHANNEL_TYPE_LUMA ) - 1) );
if( bsuccessfull && foundCandiNum >= 1 )
{
Distortion minSadHad =
std::min( distParamSad.distFunc( distParamSad ) * 2, distParamHad.distFunc( distParamHad ) );
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra( pu, 0, CHANNEL_TYPE_LUMA );
double cost = double( minSadHad ) + double( fracModeBits ) * sqrtLambdaForFirstPass;
DTRACE( g_trace_ctx, D_INTRA_COST, "IntraTPM: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, 0 );
updateCandList( ModeInfo( 0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1 ), cost, uiRdModeList, candCostList, numModesForFullRD );
updateCandList( ModeInfo( 0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1 ), 0.8 * double( minSadHad ), uiHadModeList, CandHadList, numHadCand );
}
}
//*** Derive MIP candidates using Hadamard
if (testMip && !supportedMipBlkSize)
{
// avoid estimation for unsupported blk sizes
const int transpOff = getNumModesMip(pu.Y());
const int numModesFull = (transpOff << 1);
for (uint32_t uiModeFull = 0; uiModeFull < numModesFull; uiModeFull++)
{
const bool isTransposed = (uiModeFull >= transpOff ? true : false);
const uint32_t uiMode = (isTransposed ? uiModeFull - transpOff : uiModeFull);
numModesForFullRD++;
uiRdModeList.push_back(ModeInfo(true, isTransposed, 0, NOT_INTRA_SUBPARTITIONS, uiMode));
candCostList.push_back(0);
#if JVET_AB0157_TMRL
cu.tmrlFlag = false;
#endif
double mipHadCost[MAX_NUM_MIP_MODE] = { MAX_DOUBLE };
#if JVET_AB0157_INTRA_FUSION && JVET_AB0155_SGPM
initIntraPatternChType(cu, pu.Y(), false, 0, false);
#elif JVET_AB0157_INTRA_FUSION
initIntraPatternChType(cu, pu.Y(), false, false);
#else
const int transpOff = getNumModesMip(pu.Y());
const int numModesFull = (transpOff << 1);
for (uint32_t uiModeFull = 0; uiModeFull < numModesFull; uiModeFull++)
{
const bool isTransposed = (uiModeFull >= transpOff ? true : false);
const uint32_t uiMode = (isTransposed ? uiModeFull - transpOff : uiModeFull);
pu.mipTransposedFlag = isTransposed;
pu.intraDir[CHANNEL_TYPE_LUMA] = uiMode;
predIntraMip(COMPONENT_Y, piPred, pu);
// Use the min between SAD and HAD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, uiMode, CHANNEL_TYPE_LUMA);

Karsten Suehring
committed
double cost = double(minSadHad) + double(fracModeBits) * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - double(minSadHad) + (double)sadCost);
#endif
mipHadCost[uiModeFull] = cost;
DTRACE(g_trace_ctx, D_INTRA_COST, "IntraMIP: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost,
uiModeFull);

Karsten Suehring
committed
updateCandList(ModeInfo(true, isTransposed, 0, NOT_INTRA_SUBPARTITIONS, uiMode), cost, uiRdModeList,
candCostList, numModesForFullRD + 1);
updateCandList(ModeInfo(true, isTransposed, 0, NOT_INTRA_SUBPARTITIONS, uiMode),
0.8 * double(minSadHad), uiHadModeList, CandHadList, numHadCand);
}

Karsten Suehring
committed
const double thresholdHadCost = 1.0 + 1.4 / sqrt((double) (pu.lwidth() * pu.lheight()));
reduceHadCandList(uiRdModeList, candCostList, numModesForFullRD, thresholdHadCost, mipHadCost, pu,
fastMip
#if JVET_AB0157_TMRL
, tmrlCostList
#endif
#if JVET_AC0105_DIRECTIONAL_PLANAR
, dirPlanarCostList

Karsten Suehring
committed
}

Karsten Suehring
committed
{
// save found best modes
m_uiSavedNumRdModesLFNST = numModesForFullRD;
m_uiSavedRdModeListLFNST = uiRdModeList;
m_dSavedModeCostLFNST = candCostList;
// PBINTRA fast
m_uiSavedHadModeListLFNST = uiHadModeList;
m_dSavedHadListLFNST = CandHadList;
LFNSTSaveFlag = false;

Karsten Suehring
committed
}
}
else // if( sps.getUseMIP() && LFNSTLoadFlag)
{
// restore saved modes
numModesForFullRD = m_uiSavedNumRdModesLFNST;
uiRdModeList = m_uiSavedRdModeListLFNST;
candCostList = m_dSavedModeCostLFNST;
// PBINTRA fast
uiHadModeList = m_uiSavedHadModeListLFNST;
CandHadList = m_dSavedHadListLFNST;
}
#if JVET_AB0155_SGPM
if (testSgpm)
{
if (SGPMSaveFlag)
{
m_uiSavedRdModeListSGPM.clear();
m_dSavedModeCostSGPM.clear();
m_uiSavedHadModeListSGPM.clear();
m_dSavedHadListSGPM.clear();
#if JVET_V0130_INTRA_TMP
#if JVET_AB0157_INTRA_FUSION
initIntraPatternChType(cu, pu.Y(), true, 0, false);
#else
// get single mode predictions
for (int sgpmIdx = 0; sgpmIdx < SGPM_NUM; sgpmIdx++)
{
int sgpmMode[2];
sgpmMode[0] = sgpmInfoList[sgpmIdx].sgpmMode0;
sgpmMode[1] = sgpmInfoList[sgpmIdx].sgpmMode1;
#if JVET_AG0152_SGPM_ITMP_IBC
Mv sgpmBV[2];
sgpmBV[0] = sgpmInfoList[sgpmIdx].sgpmBv0;
sgpmBV[1] = sgpmInfoList[sgpmIdx].sgpmBv1;
#endif
for (int idxIn2 = 0; idxIn2 < 2; idxIn2++)
{
if (!m_intraModeReady[sgpmMode[idxIn2]])
{
#if JVET_AG0152_SGPM_ITMP_IBC
if (sgpmMode[idxIn2] >= SGPM_BV_START_IDX)
{
// BV based mode
Mv timdBv = sgpmBV[idxIn2];
predUsingBv(piPred.buf, piPred.stride, timdBv, cu);
}
else
{
#endif
pu.intraDir[0] = sgpmMode[idxIn2];
initPredIntraParams(pu, pu.Y(), sps);
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
#if JVET_AG0152_SGPM_ITMP_IBC
}
#endif
PelBuf predBuf(m_intraPredBuf[sgpmMode[idxIn2]], tmpArea);
predBuf.copyFrom(piPred);
m_intraModeReady[sgpmMode[idxIn2]] = 1;
}
}
}
cu.sgpm = true;
// frac bits calculate once because all are the same
cu.sgpmIdx = 0;
cu.sgpmSplitDir = sgpmInfoList[0].sgpmSplitDir;
cu.sgpmMode0 = sgpmInfoList[0].sgpmMode0;
cu.sgpmMode1 = sgpmInfoList[0].sgpmMode1;
#if JVET_AG0152_SGPM_ITMP_IBC
cu.sgpmBv0 = sgpmInfoList[0].sgpmBv0;
cu.sgpmBv1 = sgpmInfoList[0].sgpmBv1;
pu.intraDir[0] = cu.sgpmMode0 >= SGPM_BV_START_IDX ? 0 : cu.sgpmMode0;
pu.intraDir1[0] = cu.sgpmMode1 >= SGPM_BV_START_IDX ? 0 : cu.sgpmMode1;
#else
pu.intraDir[0] = cu.sgpmMode0;
pu.intraDir1[0] = cu.sgpmMode1;
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, 0, CHANNEL_TYPE_LUMA);
for (int sgpmIdx = 0; sgpmIdx < SGPM_NUM; sgpmIdx++)
{
int sgpmMode0 = sgpmInfoList[sgpmIdx].sgpmMode0;
int sgpmMode1 = sgpmInfoList[sgpmIdx].sgpmMode1;
PelBuf src0(m_intraPredBuf[sgpmMode0], tmpArea);
PelBuf src1(m_intraPredBuf[sgpmMode1], tmpArea);
m_if.m_weightedSgpm(pu, width, height, COMPONENT_Y, sgpmInfoList[sgpmIdx].sgpmSplitDir, piPred, src0, src1);
PelBuf predBuf(m_sgpmPredBuf[sgpmIdx], tmpArea);
predBuf.copyFrom(piPred);
Distortion minSadHad = 0;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
minSadHad += std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
minSadHad += std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, SGPM_IDX,
#if JVET_V0130_INTRA_TMP
false, //tmpFlag
#endif
#if JVET_AD0086_ENHANCED_INTRA_TMP
0, false, false,
#if JVET_AG0136_INTRA_TMP_LIC
false, 0,
#endif
0, 0,
#endif
true, sgpmInfoList[sgpmIdx].sgpmSplitDir, sgpmInfoList[sgpmIdx].sgpmMode0,
sgpmInfoList[sgpmIdx].sgpmMode1, sgpmIdx
#if JVET_AG0152_SGPM_ITMP_IBC
, sgpmInfoList[sgpmIdx].sgpmBv0, sgpmInfoList[sgpmIdx].sgpmBv1
#endif
),
cost, m_uiSavedRdModeListSGPM, m_dSavedModeCostSGPM, SGPM_NUM);
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, SGPM_IDX,
#if JVET_V0130_INTRA_TMP
false, //tmpFlag
#endif
#if JVET_AD0086_ENHANCED_INTRA_TMP
0, false, false,
#if JVET_AG0136_INTRA_TMP_LIC
false, 0,
#endif
0, 0,
#endif
true, sgpmInfoList[sgpmIdx].sgpmSplitDir, sgpmInfoList[sgpmIdx].sgpmMode0,
sgpmInfoList[sgpmIdx].sgpmMode1, sgpmIdx
#if JVET_AG0152_SGPM_ITMP_IBC
, sgpmInfoList[sgpmIdx].sgpmBv0, sgpmInfoList[sgpmIdx].sgpmBv1
#endif
),
double(minSadHad), m_uiSavedHadModeListSGPM, m_dSavedHadListSGPM, SGPM_NUM);
}
cu.sgpm = false;
}
int updateNum = std::min<int>( (numModesForFullRD + 1) / 2, (int)m_uiSavedRdModeListSGPM.size() );
for (auto listIdx = 0; listIdx < updateNum; listIdx++)
{
updateCandList(m_uiSavedRdModeListSGPM[listIdx], m_dSavedModeCostSGPM[listIdx], uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(m_uiSavedHadModeListSGPM[listIdx], m_dSavedHadListSGPM[listIdx], uiHadModeList,
CandHadList, numHadCand);
}
}
#endif
#if JVET_AG0058_EIP
if (testEip)
{
if (eipSaveFlag)
{
m_uiSavedRdModeListEip.clear();
m_uiSavedHadModeListEip.clear();
m_dSavedModeCostEip.clear();
m_dSavedHadListEip.clear();
cu.tmpFlag = false;