Newer
Older
Bappaditya Ray
committed
if (isSecondColorSpace)
{
uiRdModeList.clear();
if (m_numSavedRdModeFirstColorSpace[m_savedRdModeIdx] > 0)
{
for (int i = 0; i < m_numSavedRdModeFirstColorSpace[m_savedRdModeIdx]; i++)
{
uiRdModeList.push_back(m_savedRdModeFirstColorSpace[m_savedRdModeIdx][i]);
}
}
else
{
return false;
}

Karsten Suehring
committed
{
#if JVET_AH0076_OBIC
if ((testSgpm && SGPMSaveFlag) || obicSaveFlag || dimdSaveFlag)
#else
#endif
#if JVET_AG0152_SGPM_ITMP_IBC
for (int i = 0; i < NUM_LUMA_MODE + SGPM_NUM_BVS; i++)
#else
{
m_intraModeReady[i] = 0;
}
}
#endif
// this should always be true
CHECK(!pu.Y().valid(), "PU is not valid");
#if !JVET_AB0157_TMRL || JVET_AD0082_TMRL_CONFIG
#if JVET_AH0065_RELAX_LINE_BUFFER
bool isFirstLineOfCtu = pu.block(COMPONENT_Y).y == 0;
#else
bool isFirstLineOfCtu = (((pu.block(COMPONENT_Y).y) & ((pu.cs->sps)->getMaxCUWidth() - 1)) == 0);
#if JVET_Y0116_EXTENDED_MRL_LIST
int numOfPassesExtendRef = MRL_NUM_REF_LINES;
if (!sps.getUseMRL() || isFirstLineOfCtu)
{
numOfPassesExtendRef = 1;
}
else
{
bool checkLineOutsideCtu[MRL_NUM_REF_LINES - 1];
for (int mrlIdx = 1; mrlIdx < MRL_NUM_REF_LINES; mrlIdx++)
{
#if JVET_AH0065_RELAX_LINE_BUFFER
bool isLineOutsideCtu = (cu.block(COMPONENT_Y).y <= MULTI_REF_LINE_IDX[mrlIdx]) ? true : false;
#else
bool isLineOutsideCtu =
((cu.block(COMPONENT_Y).y) % ((cu.cs->sps)->getMaxCUWidth()) <= MULTI_REF_LINE_IDX[mrlIdx]) ? true
: false;
#endif
checkLineOutsideCtu[mrlIdx-1] = isLineOutsideCtu;
}
if (checkLineOutsideCtu[0])
{
numOfPassesExtendRef = 1;
}
else
{
for (int mrlIdx = MRL_NUM_REF_LINES - 2; mrlIdx > 0; mrlIdx--)
{
if (checkLineOutsideCtu[mrlIdx] && !checkLineOutsideCtu[mrlIdx - 1])
{
numOfPassesExtendRef = mrlIdx + 1;
break;
}
}
}
}
#else
int numOfPassesExtendRef = ((!sps.getUseMRL() || isFirstLineOfCtu) ? 1 : MRL_NUM_REF_LINES);
#endif
#if JVET_AB0157_TMRL
cu.tmrlFlag = false;
#endif

Karsten Suehring
committed
CHECK(numModesForFullRD >= numModesAvailable, "Too many modes for full RD search");
PelBuf piOrg = cs.getOrgBuf(area);
PelBuf piPred = cs.getPredBuf(area);

Karsten Suehring
committed
DistParam distParamSad;
DistParam distParamHad;
if (cu.slice->getLmcsEnabledFlag() && m_pcReshape->getCTUFlag())

Karsten Suehring
committed
{
CompArea tmpArea(COMPONENT_Y, area.chromaFormat, Position(0, 0), area.size());
PelBuf tmpOrg = m_tmpStorageLCU.getBuf(tmpArea);
m_pcRdCost->setDistParam(distParamSad, tmpOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y,
false); // Use SAD cost
m_pcRdCost->setDistParam(distParamHad, tmpOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y,
true); // Use HAD (SATD) cost

Karsten Suehring
committed
}
m_pcRdCost->setDistParam(distParamSad, piOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y,
false); // Use SAD cost
m_pcRdCost->setDistParam(distParamHad, piOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y,
true); // Use HAD (SATD) cost

Karsten Suehring
committed
distParamSad.applyWeight = false;
distParamHad.applyWeight = false;

Karsten Suehring
committed
if (testMip && supportedMipBlkSize)
{
numModesForFullRD += fastMip
? std::max(numModesForFullRD, floorLog2(std::min(pu.lwidth(), pu.lheight())) - 1)
: numModesForFullRD;
}
#if JVET_AB0130_ITMP_SAMPLING
if (testTpm && !m_pcEncCfg->getUseFastIntraTMP())
#else
{
numModesForFullRD += 1; // testing tpm
}
#if JVET_AG0136_INTRA_TMP_LIC
if( testTpm && m_pcEncCfg->getItmpLicMode())
{
numModesForFullRD += 1; // testing lic itmp
}
#endif
const int numHadCand = (testMip ? 2 : 1) * 3 + testTpm;
cu.tmpFlag = false;
const int numHadCand = (testMip ? 2 : 1) * 3;

Karsten Suehring
committed
#if JVET_AB0155_SGPM
static_vector<SgpmInfo, SGPM_NUM> sgpmInfoList;
static_vector<double, SGPM_NUM> sgpmCostList;
#if JVET_AG0152_SGPM_ITMP_IBC
int sgpmNeededMode[NUM_LUMA_MODE + SGPM_NUM_BVS] = { 0 };
#else
if (testSgpm && SGPMSaveFlag)
{
deriveSgpmModeOrdered(bestCS->picture->getRecoBuf(area), area, cu, sgpmInfoList, sgpmCostList);
Bappaditya Ray
committed
for (int sgpmIdx = 0; sgpmIdx < SGPM_NUM; sgpmIdx++)
{
int sgpmMode[2];
sgpmMode[0] = sgpmInfoList[sgpmIdx].sgpmMode0;
sgpmMode[1] = sgpmInfoList[sgpmIdx].sgpmMode1;
sgpmNeededMode[sgpmMode[0]] = 1;
sgpmNeededMode[sgpmMode[1]] = 1;
}
}
#endif
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
#if JVET_AH0076_OBIC
int dimdNeededMode[NUM_LUMA_MODE] = {0};
if (obicSaveFlag)
{
for (int idx = 0; idx < OBIC_FUSION_NUM; idx++)
{
int iMode = cu.obicMode[idx];
if (iMode < 0)
{
continue;
}
dimdNeededMode[iMode] = 1;
}
}
if (dimdSaveFlag)
{
if (cu.dimdBlending)
{
for (int dimdIdx = 0; dimdIdx < DIMD_FUSION_NUM - 1; dimdIdx++)
{
int dimdMode = (dimdIdx == 0 ? cu.dimdMode : cu.dimdBlendMode[dimdIdx-1]);
if (dimdMode <= 0)
{
break;
}
dimdNeededMode[dimdMode] = 1;
}
dimdNeededMode[PLANAR_IDX] = 1;
}
}
#endif
#if JVET_AB0157_TMRL
double tmrlCostList[MRL_LIST_SIZE]{ MAX_DOUBLE };
#endif
#if JVET_AC0105_DIRECTIONAL_PLANAR
double dirPlanarCostList[2]{ MAX_DOUBLE };
#endif
//*** Derive (regular) candidates using Hadamard
cu.mipFlag = false;

Karsten Suehring
committed
//===== init pattern for luma prediction =====
#if JVET_AB0157_INTRA_FUSION && JVET_AB0155_SGPM
initIntraPatternChType(cu, pu.Y(), true, 0, false);
#elif JVET_AB0157_INTRA_FUSION
initIntraPatternChType(cu, pu.Y(), true, false);
#else
bool bSatdChecked[NUM_INTRA_MODE];
memset(bSatdChecked, 0, sizeof(bSatdChecked));

Karsten Suehring
committed
if (!LFNSTLoadFlag)
{
for (int modeIdx = 0; modeIdx < numModesAvailable; modeIdx++)
{
uint32_t uiMode = modeIdx;
Distortion minSadHad = 0;

Karsten Suehring
committed
// Skip checking extended Angular modes in the first round of SATD
if (uiMode > DC_IDX && (uiMode & 1))
{
continue;

Karsten Suehring
committed
}
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
Bappaditya Ray
committed
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
#if JVET_AH0209_PDP
bool pdpMode = false;
if( pdpSaveFlag )
{
const int sizeKey = (width << 8) + height;
const int sizeIdx = g_size.find(sizeKey) != g_size.end() ? g_size[sizeKey] : -1;
const int m = sizeIdx > 12 ? 2 : 0;
const int s = sizeIdx > 12 ? 4 : 2;
if (sizeIdx >= 0 && m_refAvailable && pu.cu->cs->sps->getUsePDP() && !(modeIdx > 1 && modeIdx % s != m))
{
PelBuf predBuf(m_pdpIntraPredBuf[uiMode], tmpArea);
predBuf.copyFrom(piPred);
m_pdpIntraPredReady[modeIdx] = true;
pdpMode = true;
}
}
if( !pdpMode )
{
#endif
#if JVET_AB0155_SGPM
if (testSgpm && SGPMSaveFlag && sgpmNeededMode[uiMode])
{
Bappaditya Ray
committed
PelBuf predBuf(m_intraPredBuf[uiMode], tmpArea);
predBuf.copyFrom(piPred);
m_intraModeReady[uiMode] = 1;
}
#endif
#if JVET_AH0076_OBIC
if ((obicSaveFlag || dimdSaveFlag) && dimdNeededMode[uiMode] && !m_intraModeReady[uiMode])
{
Bappaditya Ray
committed
PelBuf predBuf(m_intraPredBuf[uiMode], tmpArea);
predBuf.copyFrom(piPred);
m_intraModeReady[uiMode] = 1;
}
#endif
Bappaditya Ray
committed
#if JVET_AH0209_PDP
}
#endif
// Use the min between SAD and HAD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
minSadHad += std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
minSadHad += std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, uiMode, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
DTRACE(g_trace_ctx, D_INTRA_COST, "IntraHAD: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, uiMode);
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, uiMode), cost, uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, uiMode), double(minSadHad),
Bappaditya Ray
committed
uiHadModeList, candHadList, numHadCand);
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
#if JVET_AC0105_DIRECTIONAL_PLANAR
bool testDirPlanar = isLuma(partitioner.chType);
if (testDirPlanar)
{
for (int dirPlanarModeIdx = 0; dirPlanarModeIdx < 2; dirPlanarModeIdx++)
{
cu.sgpm = false;
cu.ispMode = 0;
cu.tmpFlag = false;
cu.tmrlFlag = false;
pu.multiRefIdx = 0;
cu.mipFlag = false;
pu.intraDir[0] = PLANAR_IDX;
cu.plIdx = dirPlanarModeIdx + 1;
initPredIntraParams(pu, pu.Y(), sps);
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
predIntraAng(COMPONENT_Y, piPred, pu);
#endif
// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, PLANAR_IDX, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(
ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, dirPlanarModeIdx ? PL_VER_IDX : PL_HOR_IDX), cost,
uiRdModeList, candCostList, numModesForFullRD);
updateCandList(
ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, dirPlanarModeIdx ? PL_VER_IDX : PL_HOR_IDX),
Bappaditya Ray
committed
double(minSadHad), uiHadModeList, candHadList, numHadCand);
dirPlanarCostList[dirPlanarModeIdx] = cost;
}
}
cu.plIdx = 0;
#endif
if (!sps.getUseMIP() && LFNSTSaveFlag)
{
// save found best modes
m_uiSavedNumRdModesLFNST = numModesForFullRD;
m_uiSavedRdModeListLFNST = uiRdModeList;
m_dSavedModeCostLFNST = candCostList;
// PBINTRA fast
m_uiSavedHadModeListLFNST = uiHadModeList;
Bappaditya Ray
committed
m_dSavedHadListLFNST = candHadList;
LFNSTSaveFlag = false;
}
} // NSSTFlag
if (!sps.getUseMIP() && LFNSTLoadFlag)
// restore saved modes
numModesForFullRD = m_uiSavedNumRdModesLFNST;
uiRdModeList = m_uiSavedRdModeListLFNST;
candCostList = m_dSavedModeCostLFNST;
// PBINTRA fast
uiHadModeList = m_uiSavedHadModeListLFNST;
Bappaditya Ray
committed
candHadList = m_dSavedHadListLFNST;
if (!(sps.getUseMIP() && LFNSTLoadFlag))
{
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> parentCandList = uiRdModeList;
// Second round of SATD for extended Angular modes
for (int modeIdx = 0; modeIdx < numModesForFullRD; modeIdx++)
{
unsigned parentMode = parentCandList[modeIdx].modeId;
if (parentMode > (DC_IDX + 1) && parentMode < (NUM_LUMA_MODE - 1))
{
for (int subModeIdx = -1; subModeIdx <= 1; subModeIdx += 2)
{
unsigned mode = parentMode + subModeIdx;
if (!bSatdChecked[mode])
{
pu.intraDir[0] = mode;
initPredIntraParams(pu, pu.Y(), sps);
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
#if JVET_AB0155_SGPM
if (testSgpm && SGPMSaveFlag && sgpmNeededMode[mode])
{
Bappaditya Ray
committed
PelBuf predBuf(m_intraPredBuf[mode], tmpArea);
predBuf.copyFrom(piPred);
m_intraModeReady[mode] = 1;
}
#endif
#if JVET_AH0076_OBIC
if ((obicSaveFlag || dimdSaveFlag) && dimdNeededMode[mode] && !m_intraModeReady[mode])
{
Bappaditya Ray
committed
PelBuf predBuf(m_intraPredBuf[mode], tmpArea);
predBuf.copyFrom(piPred);
m_intraModeReady[mode] = 1;
}
#endif
// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, mode, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, mode), cost, uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, mode), double(minSadHad),
Bappaditya Ray
committed
uiHadModeList, candHadList, numHadCand);
bSatdChecked[mode] = true;
}
}
}
}
if (saveDataForISP)
// we save the regular intra modes list
m_ispCandListHor = uiRdModeList;
}
#if !JVET_AB0157_TMRL || JVET_AD0082_TMRL_CONFIG
#if SECONDARY_MPM
const int numMPMs = NUM_PRIMARY_MOST_PROBABLE_MODES;
#else
Fabrice URBAN
committed
uint8_t* multiRefMPM = m_intraMPM;
#if !JVET_AD0082_TMRL_CONFIG
#if JVET_AD0082_TMRL_CONFIG
cu.tmrlFlag = true;
#endif
for (auto multiRefIdx : EXT_REF_LINE_IDX)
{
pu.multiRefIdx = multiRefIdx;
initIntraPatternChType(cu, pu.Y(), true);
for (auto i = 0; i < MRL_LIST_SIZE; i++)
{
if (m_tmrlList[i].multiRefIdx != multiRefIdx)
pu.intraDir[0] = m_tmrlList[i].intraDir;
cu.tmrlListIdx = i;
uint32_t uiMode = i + MAX_REF_LINE_IDX;
initPredIntraParams(pu, pu.Y(), *(pu.cs->sps));
predIntraAng(COMPONENT_Y, piPred, pu);
// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, pu.intraDir[0], CHANNEL_TYPE_LUMA);
double cost = (double)minSadHad + (double)fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(ModeInfo(false, false, uiMode, NOT_INTRA_SUBPARTITIONS, 0), cost, uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, uiMode, NOT_INTRA_SUBPARTITIONS, 0), double(minSadHad),
Bappaditya Ray
committed
uiHadModeList, candHadList, numHadCand);
#if JVET_AB0157_TMRL
tmrlCostList[i] = cost;
#endif
}
}
#if JVET_AD0082_TMRL_CONFIG
cu.tmrlFlag = false;
#endif
#endif
#if !JVET_AB0157_TMRL || JVET_AD0082_TMRL_CONFIG
#if JVET_AD0082_TMRL_CONFIG
else
{
#endif
for (int mRefNum = 1; mRefNum < numOfPassesExtendRef; mRefNum++)
{
int multiRefIdx = MULTI_REF_LINE_IDX[mRefNum];
#if JVET_AB0157_INTRA_FUSION && JVET_AB0155_SGPM
initIntraPatternChType(cu, pu.Y(), true, 0, false);
#elif JVET_AB0157_INTRA_FUSION
initIntraPatternChType(cu, pu.Y(), true, false);
#else
}
for (int x = 1; x < numMPMs; x++)
{
uint32_t mode = multiRefMPM[x];
{
pu.intraDir[0] = mode;
initPredIntraParams(pu, pu.Y(), sps);
#if JVET_AB0157_INTRA_FUSION
predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, mode, CHANNEL_TYPE_LUMA);
double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
updateCandList(ModeInfo(false, false, multiRefIdx, NOT_INTRA_SUBPARTITIONS, mode), cost, uiRdModeList,
candCostList, numModesForFullRD);
updateCandList(ModeInfo(false, false, multiRefIdx, NOT_INTRA_SUBPARTITIONS, mode), double(minSadHad),
Bappaditya Ray
committed
uiHadModeList, candHadList, numHadCand);
#if JVET_AD0082_TMRL_CONFIG
}
#endif
CHECKD(uiRdModeList.size() != numModesForFullRD, "Error: RD mode list size");
#if JVET_V0130_INTRA_TMP && JVET_AB0130_ITMP_SAMPLING
// derive TPM candidate using hadamard
if (testTpm)
{
cu.tmpFlag = true;
cu.mipFlag = false;
pu.multiRefIdx = 0;
#if JVET_AD0086_ENHANCED_INTRA_TMP
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListTmp;
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> candCostListTmp;
int foundCandiNum = 0;
bool bsuccessfull = 0;
Wei Chen
committed
#endif
#if JVET_AH0200_INTRA_TMP_BV_REORDER
int adjustedTMPNonLicBvNum = TMP_REFINE_NONLIC_BV_NUM;
int adjustedTmpLicBvNum = TMP_REFINE_LIC_BV_NUM;
if(m_pcEncCfg->getIntraPeriod() != 1)
{
adjustedTMPNonLicBvNum = 10;
adjustedTmpLicBvNum = 4;
}
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListFracTmp;
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> candCostListFracTmp;
Distortion backupMinSadHad[MTMP_NUM];
Distortion backupSadCost[MTMP_NUM];
Distortion backupLicMinSadHad[MTMP_NUM][4];
Distortion backupLicSadCost[MTMP_NUM][4];
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListLicFracTmp;
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> candCostListLicFracTmp;
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListTmpLic;
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> candCostListTmpLic;
CodingUnit cuCopy = cu;
#if JVET_W0069_TMP_BOUNDARY
RefTemplateType templateType = getRefTemplateType(cuCopy, cuCopy.blocks[COMPONENT_Y]);
if (templateType != NO_TEMPLATE)
#else
if (isRefTemplateAvailable(cuCopy, cuCopy.blocks[COMPONENT_Y]))
#endif
{
#if JVET_AD0086_ENHANCED_INTRA_TMP
cu.tmpIsSubPel = 0;
#if JVET_AG0136_INTRA_TMP_LIC
cu.tmpSubPelIdx = -1;
#else
cu.tmpSubPelIdx = 0;
Wei Chen
committed
#endif
#if JVET_AH0200_INTRA_TMP_BV_REORDER
cu.tmpFracIdx = 0;
int numModesForFracIntraTmp = numModesForFullRD + adjustedTMPNonLicBvNum;
#if JVET_AI0136_ADAPTIVE_DUAL_TREE
if(relatedCU && relatedCU->skipFracTmp)
#else
Wei Chen
committed
if(relatedCU.skipFracTmp)
#endif
Wei Chen
committed
{
numModesForFracIntraTmp = numModesForFullRD;
}
if(m_tmpNumCand > 0)
{
for(int idxInList=0; idxInList < uiRdModeList.size(); idxInList++)
{
if(cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD)
{
break;
}
updateCandList(uiRdModeList[idxInList], candCostList[idxInList], uiRdModeListFracTmp, candCostListFracTmp, numModesForFracIntraTmp);
}
}
int numModesForLicFracIntraTmp = numModesForFullRD + adjustedTmpLicBvNum;
#if JVET_AI0136_ADAPTIVE_DUAL_TREE
if(relatedCU && relatedCU->skipFracTmp)
#else
Wei Chen
committed
if(relatedCU.skipFracTmp)
#endif
Wei Chen
committed
{
numModesForLicFracIntraTmp = numModesForFullRD;
}
if(m_tmpNumCandUseMR > 0)
{
for(int idxInList=0; idxInList < uiRdModeList.size(); idxInList++)
{
if(cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD)
{
break;
}
updateCandList(uiRdModeList[idxInList], candCostList[idxInList], uiRdModeListLicFracTmp, candCostListLicFracTmp, numModesForLicFracIntraTmp);
}
}
for(int tmpFusionFlag = 0; tmpFusionFlag <= 1; tmpFusionFlag++)
{
cu.tmpFusionFlag = tmpFusionFlag ? true: false;
for (int tmpFlmFlag = 0; tmpFlmFlag <= 1; tmpFlmFlag++)
{
cu.tmpFlmFlag = tmpFlmFlag ? true: false;
if(tmpFlmFlag && tmpFusionFlag)
{
continue;
}
#if JVET_AG0136_INTRA_TMP_LIC
for (int tmpLicFlag = 0; tmpLicFlag <= 1; tmpLicFlag++)
{
cu.ibcLicFlag = tmpLicFlag ? true : false;
cu.tmpLicFlag = tmpLicFlag ? true : false;
if (tmpLicFlag && tmpFlmFlag)
{
continue;
}
const int ibcLicLoopNum = (cu.slice->getSPS()->getItmpLicExtension() && cu.tmpLicFlag && !cu.tmpFusionFlag) ? 4 : 1;
for (int licIdc = 0; licIdc < ibcLicLoopNum; licIdc++)
{
cu.ibcLicIdx = licIdc;
int idxNum = cu.tmpFusionFlag ? TMP_GROUP_IDX << 1 : (cu.tmpLicFlag ? m_tmpNumCandUseMR : m_tmpNumCand);
#else
int idxNum = cu.tmpFusionFlag ? TMP_GROUP_IDX << 1 : m_tmpNumCand;
for (int tmpIdx = 0; tmpIdx < idxNum; tmpIdx++)
{
#if JVET_AG0136_INTRA_TMP_LIC
if (cu.tmpFusionFlag && !(cu.tmpLicFlag ? m_tmpFusionInfoUseMR : m_tmpFusionInfo)[tmpIdx].bValid)
#else
if(cu.tmpFusionFlag && !m_tmpFusionInfo[tmpIdx].bValid)
{
continue;
}
cu.tmpIdx = tmpIdx;
int placeHolder;
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
generateTMPrediction(piPred.buf, piPred.stride, placeHolder, pu
#if JVET_AG0136_INTRA_TMP_LIC
, cu.tmpLicFlag
#endif
, false);
#if JVET_AG0136_INTRA_TMP_LIC
if (cu.tmpLicFlag)
{
if (!cu.tmpFusionFlag)
{
const auto& arrayLicParams = getMemLicParams(cu.ibcLicIdx, cu.tmpIdx);
if (cu.ibcLicIdx == IBC_LIC_IDX_M)
{
piPred.linearTransforms(arrayLicParams[1], arrayLicParams[0], arrayLicParams[2], arrayLicParams[4], arrayLicParams[3], arrayLicParams[5], arrayLicParams[6], true, cu.cs->slice->clpRng(COMPONENT_Y));
}
else
{
piPred.linearTransform(arrayLicParams[1], arrayLicParams[0], arrayLicParams[2], true, cu.cs->slice->clpRng(COMPONENT_Y));
}
}
}
#endif
xGenerateTmpFlmPred(piPred, pu.lwidth(), pu.lheight(), templateType, pu.cu, false);
xTMPFusionApplyModel(piPred, pu.lwidth(), pu.lheight(), templateType, pu.cu
#if JVET_AG0136_INTRA_TMP_LIC
, cu.tmpLicFlag
#endif
, false);
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
#if JVET_W0069_TMP_BOUNDARY
#if TMP_FAST_ENC
bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.Y(), foundCandiNum, pu.cu);
#else
getTargetTemplate(&cuCopy, pu.lwidth(), pu.lheight(), templateType);
candidateSearchIntra(&cuCopy, pu.lwidth(), pu.lheight(), templateType);
bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), foundCandiNum);
#endif
#else
#if TMP_FAST_ENC
bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.Y(), foundCandiNum, pu.cu);
#else
getTargetTemplate(&cuCopy, pu.lwidth(), pu.lheight());
candidateSearchIntra(&cuCopy, pu.lwidth(), pu.lheight());
bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), foundCandiNum);
#endif
#endif
}
#if JVET_W0069_TMP_BOUNDARY
else
{
foundCandiNum = 1;
#if JVET_AC0115_INTRA_TMP_DIMD_MTS_LFNST
bsuccessfull = generateTmDcPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), 1 << (cuCopy.cs->sps->getBitDepth(CHANNEL_TYPE_LUMA) - 1), pu.cu);
#else
bsuccessfull = generateTmDcPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), 1 << (cuCopy.cs->sps->getBitDepth(CHANNEL_TYPE_LUMA) - 1));
#endif
}
#endif
if (bsuccessfull && foundCandiNum >= 1)
{
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
loadStartStates();
#if JVET_AG0136_INTRA_TMP_LIC
m_CABACEstimator->getCtx() = SubCtx(Ctx::TmpLic, ctxStartTmpLicFlag);
m_CABACEstimator->getCtx() = SubCtx(Ctx::ItmpLicIndex, ctxStartTmpLicIdx);
#endif
uint64_t fracModeBits = xFracModeBitsIntra(pu, 0, CHANNEL_TYPE_LUMA);
double cost = double(minSadHad) + double(fracModeBits) * sqrtLambdaForFirstPass;
Wei Chen
committed
#if JVET_AH0200_INTRA_TMP_BV_REORDER
isTmpModeTestd = true;
if(!tmpFusionFlag && !tmpFlmFlag)
{
if(tmpBestSatdCost > cost)
{
tmpBestSatdCost = cost;
}
}
#endif
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
Wei Chen
committed
#if JVET_AH0200_INTRA_TMP_BV_REORDER
if(tmpFlmFlag || tmpFusionFlag
|| (cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD))
{
#endif
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - double(minSadHad) + (double)sadCost);
#if JVET_AH0200_INTRA_TMP_BV_REORDER
}
#endif
#endif
DTRACE(g_trace_ctx, D_INTRA_COST, "IntraTPM: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, 0);
#if JVET_AD0086_ENHANCED_INTRA_TMP
#if JVET_AG0136_INTRA_TMP_LIC
Wei Chen
committed
#if JVET_AH0200_INTRA_TMP_BV_REORDER
if(tmpFlmFlag || tmpFusionFlag || (cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD))
{
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx, cu.tmpFracIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
Bappaditya Ray
committed
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx, cu.tmpFracIdx), 0.8 * double(minSadHad), uiHadModeList, candHadList, numHadCand);
Wei Chen
committed
}
else if (cu.tmpLicFlag)
{
backupLicMinSadHad[cu.tmpIdx][cu.ibcLicIdx] = minSadHad;
backupLicSadCost[cu.tmpIdx][cu.ibcLicIdx] = sadCost;
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx, cu.tmpFracIdx), cost, uiRdModeListLicFracTmp, candCostListLicFracTmp, numModesForLicFracIntraTmp);
}
else
{
backupMinSadHad[cu.tmpIdx] = minSadHad;
backupSadCost[cu.tmpIdx] = sadCost;
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx, cu.tmpFracIdx), cost, uiRdModeListFracTmp, candCostListFracTmp, numModesForFracIntraTmp);
}
#else
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
Bappaditya Ray
committed
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, candHadList, numHadCand);
Wei Chen
committed
#endif
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
Bappaditya Ray
committed
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, candHadList, numHadCand);
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1), cost, uiRdModeList, candCostList, numModesForFullRD);
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1), 0.8 * double(minSadHad), uiHadModeList, CandHadList, numHadCand);
#endif
#if JVET_AD0086_ENHANCED_INTRA_TMP
//record the best full-pel candidates
Wei Chen
committed
#if JVET_AH0200_INTRA_TMP_BV_REORDER
if (!tmpFlmFlag && !tmpFusionFlag && tmpLicFlag)
{
for(int idxInList=0; idxInList < uiRdModeListLicFracTmp.size(); idxInList++)
{
if(cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD)
{
break;
}
if(uiRdModeListLicFracTmp[idxInList].tmpFlag)
{
updateCandList(uiRdModeListLicFracTmp[idxInList], candCostListLicFracTmp[idxInList], uiRdModeListTmpLic, candCostListTmpLic, numModesForLicFracIntraTmp);
}
if(idxInList >= numModesForFullRD && uiRdModeListTmpLic.size() >= adjustedTmpLicBvNum)
{
break;
}
#if JVET_AI0136_ADAPTIVE_DUAL_TREE
if(idxInList >= (numModesForFullRD-1) && relatedCU && relatedCU->skipFracTmp)
#else
Wei Chen
committed
if(idxInList >= (numModesForFullRD-1) && relatedCU.skipFracTmp)
#endif
Wei Chen
committed
{
break;
}
}
}
#endif
#if JVET_AG0136_INTRA_TMP_LIC
if (!tmpFlmFlag && !tmpFusionFlag && !tmpLicFlag)
#else
if(!tmpFlmFlag&&!tmpFusionFlag)
Wei Chen
committed
#if JVET_AH0200_INTRA_TMP_BV_REORDER
for(int idxInList=0; idxInList < uiRdModeListFracTmp.size(); idxInList++)
{
if(cu.lwidth() * cu.lheight() > TMP_SKIP_REFINE_THRESHOLD)
{
break;
}
if(uiRdModeListFracTmp[idxInList].tmpFlag)
{
updateCandList(uiRdModeListFracTmp[idxInList], candCostListFracTmp[idxInList], uiRdModeListTmp, candCostListTmp, numModesForFracIntraTmp);
}
if(idxInList >= numModesForFullRD && uiRdModeListTmp.size() >= adjustedTMPNonLicBvNum)
{
break;
}
#if JVET_AI0136_ADAPTIVE_DUAL_TREE
if(idxInList >= (numModesForFullRD-1) && relatedCU && relatedCU->skipFracTmp)
#else
Wei Chen
committed
if(idxInList >= (numModesForFullRD-1) && relatedCU.skipFracTmp)
#endif
Wei Chen
committed
{
break;
}
}
#else
for(int idxInList=0; idxInList < uiRdModeList.size(); idxInList++)
{
if(uiRdModeList[idxInList].tmpFlag){
updateCandList(uiRdModeList[idxInList], candCostList[idxInList], uiRdModeListTmp, candCostListTmp, numModesForFullRD);
Wei Chen
committed
#endif
#if JVET_AG0136_INTRA_TMP_LIC
}
}
#endif
}
}
//fractional BV
cu.tmpFusionFlag = false;
cu.tmpFlmFlag = false;
#if JVET_AG0136_INTRA_TMP_LIC
cu.tmpLicFlag = false;
cu.ibcLicIdx = 0;
#endif
Wei Chen
committed
#if !JVET_AH0200_INTRA_TMP_BV_REORDER
for(int idxInList=0; idxInList < uiRdModeListTmp.size(); idxInList++)
{
cu.tmpIdx = uiRdModeListTmp[idxInList].tmpIdx;
xPadForInterpolation(&cu);
for(int tmpIsSubPel = 1; tmpIsSubPel < 4; tmpIsSubPel++)
{
for (int idx = 0; idx < TMP_MAX_SUBPEL_DIR; idx++)
{
cu.tmpIsSubPel = tmpIsSubPel;
cu.tmpSubPelIdx = idx;
int placeHolder;
generateTMPrediction(piPred.buf, piPred.stride, placeHolder, pu
#if JVET_AG0136_INTRA_TMP_LIC
, cu.tmpLicFlag
#endif
, false);
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS&&JVET_AE0169_BIPREDICTIVE_IBC
Distortion sadCost = distParamSad.distFunc(distParamSad);
Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
Distortion minSadHad =
std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
loadStartStates();
uint64_t fracModeBits = xFracModeBitsIntra(pu, 0, CHANNEL_TYPE_LUMA);
double cost = double(minSadHad) + double(fracModeBits) * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS&&JVET_AE0169_BIPREDICTIVE_IBC
m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - double(minSadHad) + (double)sadCost);
#endif
DTRACE(g_trace_ctx, D_INTRA_COST, "IntraTPM: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, 0);
#if JVET_AG0136_INTRA_TMP_LIC
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
Bappaditya Ray
committed
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpLicFlag, cu.ibcLicIdx, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, candHadList, numHadCand);
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
Bappaditya Ray
committed
updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, candHadList, numHadCand);
}
}
}
Wei Chen
committed
#endif
#if JVET_AH0200_INTRA_TMP_BV_REORDER
for(int idxInList=0; idxInList < uiRdModeListTmp.size(); idxInList++)
{
cu.tmpIdx = uiRdModeListTmp[idxInList].tmpIdx;
cu.tmpLicFlag = uiRdModeListTmp[idxInList].tmpLicFlag;
CHECK(cu.tmpLicFlag, "cu.tmpLicFlag == 1");
cu.ibcLicFlag = cu.tmpLicFlag;
cu.ibcLicIdx = uiRdModeListTmp[idxInList].tmpLicIdc;