Newer
Older

Karsten Suehring
committed
saveCS.addTU( *ptu, partitioner.chType );
orgTUs.push_back( ptu );
}
}
if( lumaUsesISP )
{
saveCS.clearCUs();
}
// SATD pre-selecting.
int satdModeList[NUM_CHROMA_MODE];
int64_t satdSortedCost[NUM_CHROMA_MODE];
for (int i = 0; i < NUM_CHROMA_MODE; i++)
{
satdSortedCost[i] = 0; // for the mode not pre-select by SATD, do RDO by default, so set the initial value 0.
satdModeList[i] = 0;
}
bool modeIsEnable[NUM_INTRA_MODE + 1]; // use intra mode idx to check whether enable
for (int i = 0; i < NUM_INTRA_MODE + 1; i++)
{
modeIsEnable[i] = 1;
}
DistParam distParam;
#if JVET_N0329_IBC_SEARCH_IMP
const bool useHadamard = !cu.transQuantBypass;
#else
pu.intraDir[1] = MDLM_L_IDX; // temporary assigned, just to indicate this is a MDLM mode. for luma down-sampling operation.
initIntraPatternChType(cu, pu.Cb());
initIntraPatternChType(cu, pu.Cr());
xGetLumaRecPixels(pu, pu.Cb());
for (int idx = uiMinMode; idx <= uiMaxMode - 1; idx++)
{
int mode = chromaCandModes[idx];
satdModeList[idx] = mode;
if (PU::isLMCMode(mode) && !PU::isLMCModeEnabled(pu, mode))
{
continue;
}
if ((mode == LM_CHROMA_IDX) || (mode == PLANAR_IDX) || (mode == DM_CHROMA_IDX)) // only pre-check regular modes and MDLM modes, not including DM ,Planar, and LM
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
{
continue;
}
pu.intraDir[1] = mode; // temporary assigned, for SATD checking.
int64_t sad = 0;
CodingStructure& cs = *(pu.cs);
CompArea areaCb = pu.Cb();
PelBuf orgCb = cs.getOrgBuf(areaCb);
PelBuf predCb = cs.getPredBuf(areaCb);
m_pcRdCost->setDistParam(distParam, orgCb, predCb, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, useHadamard);
distParam.applyWeight = false;
if (PU::isLMCMode(mode))
{
predIntraChromaLM(COMPONENT_Cb, predCb, pu, areaCb, mode);
}
else
{
predIntraAng(COMPONENT_Cb, predCb, pu, false);
}
sad += distParam.distFunc(distParam);

Karsten Suehring
committed
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
CompArea areaCr = pu.Cr();
PelBuf orgCr = cs.getOrgBuf(areaCr);
PelBuf predCr = cs.getPredBuf(areaCr);
m_pcRdCost->setDistParam(distParam, orgCr, predCr, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, useHadamard);
distParam.applyWeight = false;
if (PU::isLMCMode(mode))
{
predIntraChromaLM(COMPONENT_Cr, predCr, pu, areaCr, mode);
}
else
{
predIntraAng(COMPONENT_Cr, predCr, pu, false);
}
sad += distParam.distFunc(distParam);
satdSortedCost[idx] = sad;
}
// sort the mode based on the cost from small to large.
int tempIdx = 0;
int64_t tempCost = 0;
for (int i = uiMinMode; i <= uiMaxMode - 1; i++)
{
for (int j = i + 1; j <= uiMaxMode - 1; j++)
{
if (satdSortedCost[j] < satdSortedCost[i])
{
tempIdx = satdModeList[i];
satdModeList[i] = satdModeList[j];
satdModeList[j] = tempIdx;
tempCost = satdSortedCost[i];
satdSortedCost[i] = satdSortedCost[j];
satdSortedCost[j] = tempCost;
}
}
}
int reducedModeNumber = 2; // reduce the number of chroma modes
for (int i = 0; i < reducedModeNumber; i++)
{
modeIsEnable[satdModeList[uiMaxMode - 1 - i]] = 0; // disable the last reducedModeNumber modes
}

Karsten Suehring
committed
// save the dist
Distortion baseDist = cs.dist;
for (uint32_t uiMode = uiMinMode; uiMode < uiMaxMode; uiMode++)
{
const int chromaIntraMode = chromaCandModes[uiMode];
if( PU::isLMCMode( chromaIntraMode ) && ! PU::isLMCModeEnabled( pu, chromaIntraMode ) )
{
continue;
}
if (!modeIsEnable[chromaIntraMode] && PU::isLMCModeEnabled(pu, chromaIntraMode)) // when CCLM is disable, then MDLM is disable. not use satd checking
{
continue;
}

Karsten Suehring
committed
cs.setDecomp( pu.Cb(), false );
cs.dist = baseDist;
//----- restore context models -----
m_CABACEstimator->getCtx() = ctxStart;
//----- chroma coding -----
pu.intraDir[1] = chromaIntraMode;
xRecurIntraChromaCodingQT( cs, partitioner, bestCostSoFar, ispType );
if( lumaUsesISP && cs.dist == MAX_UINT )
{
continue;
}

Karsten Suehring
committed
if (cs.pps->getUseTransformSkip())
{
m_CABACEstimator->getCtx() = ctxStart;
}
uint64_t fracBits = xGetIntraFracBitsQT( cs, partitioner, false, true, -1, ispType );

Karsten Suehring
committed
Distortion uiDist = cs.dist;
double dCost = m_pcRdCost->calcRdCost( fracBits, uiDist - baseDist );
//----- compare -----
if( dCost < dBestCost )
{
if( lumaUsesISP && dCost < bestCostSoFar )
{
bestCostSoFar = dCost;
}

Karsten Suehring
committed
for( uint32_t i = getFirstComponentOfChannel( CHANNEL_TYPE_CHROMA ); i < numberValidComponents; i++ )
{
const CompArea &area = pu.blocks[i];
saveCS.getRecoBuf ( area ).copyFrom( cs.getRecoBuf ( area ) );
#if KEEP_PRED_AND_RESI_SIGNALS
saveCS.getPredBuf ( area ).copyFrom( cs.getPredBuf ( area ) );
saveCS.getResiBuf ( area ).copyFrom( cs.getResiBuf ( area ) );
#endif
saveCS.getPredBuf ( area ).copyFrom( cs.getPredBuf (area ) );
cs.picture->getPredBuf( area ).copyFrom( cs.getPredBuf (area ) );

Karsten Suehring
committed
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
cs.picture->getRecoBuf( area ).copyFrom( cs.getRecoBuf( area ) );
for( uint32_t j = 0; j < saveCS.tus.size(); j++ )
{
saveCS.tus[j]->copyComponentFrom( *orgTUs[j], area.compID );
}
}
dBestCost = dCost;
uiBestDist = uiDist;
uiBestMode = chromaIntraMode;
}
}
for( uint32_t i = getFirstComponentOfChannel( CHANNEL_TYPE_CHROMA ); i < numberValidComponents; i++ )
{
const CompArea &area = pu.blocks[i];
cs.getRecoBuf ( area ).copyFrom( saveCS.getRecoBuf( area ) );
#if KEEP_PRED_AND_RESI_SIGNALS
cs.getPredBuf ( area ).copyFrom( saveCS.getPredBuf( area ) );
cs.getResiBuf ( area ).copyFrom( saveCS.getResiBuf( area ) );
#endif
cs.getPredBuf ( area ).copyFrom( saveCS.getPredBuf( area ) );
cs.picture->getPredBuf( area ).copyFrom( cs.getPredBuf ( area ) );

Karsten Suehring
committed
cs.picture->getRecoBuf( area ).copyFrom( cs. getRecoBuf( area ) );
for( uint32_t j = 0; j < saveCS.tus.size(); j++ )
{
orgTUs[ j ]->copyComponentFrom( *saveCS.tus[ j ], area.compID );
}
}
}
pu.intraDir[1] = uiBestMode;
cs.dist = uiBestDist;
}
//----- restore context models -----
m_CABACEstimator->getCtx() = ctxStart;
if( lumaUsesISP && bestCostSoFar >= maxCostAllowed )
{
cu.ispMode = 0;
}

Karsten Suehring
committed
}
void IntraSearch::IPCMSearch(CodingStructure &cs, Partitioner& partitioner)
{
ComponentID compStr = (CS::isDualITree(cs) && !isLuma(partitioner.chType)) ? COMPONENT_Cb: COMPONENT_Y;
ComponentID compEnd = (CS::isDualITree(cs) && isLuma(partitioner.chType)) ? COMPONENT_Y : COMPONENT_Cr;
for( ComponentID compID = compStr; compID <= compEnd; compID = ComponentID(compID+1) )

Karsten Suehring
committed
{
xEncPCM(cs, partitioner, compID);
}
cs.getPredBuf().fill(0);
cs.getResiBuf().fill(0);
cs.getOrgResiBuf().fill(0);
cs.dist = 0;
cs.fracBits = 0;
cs.cost = 0;
cs.setDecomp(cs.area);
cs.picture->getPredBuf(cs.area).copyFrom(cs.getPredBuf());

Karsten Suehring
committed
}
void IntraSearch::xEncPCM(CodingStructure &cs, Partitioner& partitioner, const ComponentID &compID)
{
TransformUnit &tu = *cs.getTU( partitioner.chType );
const int channelBitDepth = cs.sps->getBitDepth(toChannelType(compID));
const uint32_t uiPCMBitDepth = cs.sps->getPCMBitDepth(toChannelType(compID));
const int pcmShiftRight = (channelBitDepth - int(uiPCMBitDepth));
CompArea area = tu.blocks[compID];
PelBuf pcmBuf = tu.getPcmbuf (compID);
PelBuf recBuf = cs.getRecoBuf ( area );
CPelBuf orgBuf = cs.getOrgBuf ( area );
CHECK(pcmShiftRight < 0, "Negative shift");
CompArea tmpArea(COMPONENT_Y, area.chromaFormat, Position(0, 0), area.size());
PelBuf tempOrgBuf = m_tmpStorageLCU.getBuf(tmpArea);
tempOrgBuf.copyFrom(orgBuf);
if (cs.slice->getReshapeInfo().getUseSliceReshaper() && m_pcReshape->getCTUFlag() && compID == COMPONENT_Y)
{
tempOrgBuf.rspSignal(m_pcReshape->getFwdLUT());
}

Karsten Suehring
committed
for (uint32_t uiY = 0; uiY < pcmBuf.height; uiY++)
{
for (uint32_t uiX = 0; uiX < pcmBuf.width; uiX++)
{
// Encode

Karsten Suehring
committed
// Reconstruction
recBuf.at(uiX, uiY) = pcmBuf.at(uiX, uiY) << pcmShiftRight;
}
}
}
// -------------------------------------------------------------------------------------------------------------------
// Intra search
// -------------------------------------------------------------------------------------------------------------------
void IntraSearch::xEncIntraHeader( CodingStructure &cs, Partitioner &partitioner, const bool &bLuma, const bool &bChroma, const int subTuIdx )

Karsten Suehring
committed
{
CodingUnit &cu = *cs.getCU( partitioner.chType );
if (bLuma)
{
bool isFirst = cu.ispMode ? subTuIdx == 0 : partitioner.currArea().lumaPos() == cs.area.lumaPos();

Karsten Suehring
committed
// CU header
if( isFirst )
{
if ((!cs.slice->isIntra() || cs.slice->getSPS()->getIBCFlag())

Karsten Suehring
committed
{
if( cs.pps->getTransquantBypassEnabledFlag() )
{
m_CABACEstimator->cu_transquant_bypass_flag( cu );
}
m_CABACEstimator->cu_skip_flag( cu );
m_CABACEstimator->pred_mode ( cu );
}

Karsten Suehring
committed
{
m_CABACEstimator->pcm_data( cu, partitioner );

Karsten Suehring
committed
if( cu.ipcm )
{
return;
}
}
Santiago de Luxán Hernández
committed
m_CABACEstimator->extend_ref_line(cu);
m_CABACEstimator->isp_mode ( cu );

Karsten Suehring
committed
}
PredictionUnit &pu = *cs.getPU(partitioner.currArea().lumaPos(), partitioner.chType);
// luma prediction mode

Karsten Suehring
committed
{
if ( !cu.Y().valid())
m_CABACEstimator->pred_mode( cu );
m_CABACEstimator->intra_luma_pred_mode( pu );

Karsten Suehring
committed
}
}
if (bChroma)
{
bool isFirst = partitioner.currArea().Cb().valid() && partitioner.currArea().chromaPos() == cs.area.chromaPos();
PredictionUnit &pu = *cs.getPU( partitioner.currArea().chromaPos(), CHANNEL_TYPE_CHROMA );

Karsten Suehring
committed
{

Karsten Suehring
committed
}
}
}
void IntraSearch::xEncSubdivCbfQT( CodingStructure &cs, Partitioner &partitioner, const bool &bLuma, const bool &bChroma, const int subTuIdx, const PartSplit ispType )
{
const UnitArea &currArea = partitioner.currArea();
int subTuCounter = subTuIdx;
TransformUnit &currTU = *cs.getTU( currArea.blocks[partitioner.chType], partitioner.chType, subTuCounter );
CodingUnit &currCU = *currTU.cu;

Karsten Suehring
committed
uint32_t currDepth = partitioner.currTrDepth;
const bool subdiv = currTU.depth > currDepth;
ComponentID compID = partitioner.chType == CHANNEL_TYPE_LUMA ? COMPONENT_Y : COMPONENT_Cb;
const bool chromaCbfISP = currArea.blocks[COMPONENT_Cb].valid() && currCU.ispMode && !subdiv;

Karsten Suehring
committed

Karsten Suehring
committed
{
CHECK( !subdiv, "TU split implied" );
}
else
{
CHECK( subdiv && !currCU.ispMode && isLuma( compID ), "No TU subdivision is allowed with QTBT" );
}
if( bChroma && ( !currCU.ispMode || chromaCbfISP ) )

Karsten Suehring
committed
{
const uint32_t numberValidComponents = getNumberValidComponents(currArea.chromaFormat);
const uint32_t cbfDepth = ( chromaCbfISP ? currDepth - 1 : currDepth );

Karsten Suehring
committed
for (uint32_t ch = COMPONENT_Cb; ch < numberValidComponents; ch++)
{
const ComponentID compID = ComponentID(ch);
if( currDepth == 0 || TU::getCbfAtDepth( currTU, compID, currDepth - 1 ) || chromaCbfISP )

Karsten Suehring
committed
{
const bool prevCbf = ( compID == COMPONENT_Cr ? TU::getCbfAtDepth( currTU, COMPONENT_Cb, currDepth ) : false );
m_CABACEstimator->cbf_comp( cs, TU::getCbfAtDepth( currTU, compID, currDepth ), currArea.blocks[compID], cbfDepth, prevCbf );

Karsten Suehring
committed
}
}
}
if (subdiv)
{
if( partitioner.canSplit( TU_MAX_TR_SPLIT, cs ) )
{
partitioner.splitCurrArea( TU_MAX_TR_SPLIT, cs );
}
else if( currCU.ispMode && isLuma( compID ) )
{
partitioner.splitCurrArea( ispType, cs );
}

Karsten Suehring
committed
else
THROW( "Cannot perform an implicit split!" );
do
{
xEncSubdivCbfQT( cs, partitioner, bLuma, bChroma, subTuCounter, ispType );
subTuCounter += subTuCounter != -1 ? 1 : 0;

Karsten Suehring
committed
} while( partitioner.nextPart( cs ) );
partitioner.exitCurrSplit();
}
else
{
//===== Cbfs =====
if (bLuma)
{
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
bool previousCbf = false;
bool lastCbfIsInferred = false;
if( ispType != TU_NO_ISP )
{
bool rootCbfSoFar = false;
uint32_t nTus = currCU.ispMode == HOR_INTRA_SUBPARTITIONS ? currCU.lheight() >> g_aucLog2[currTU.lheight()] : currCU.lwidth() >> g_aucLog2[currTU.lwidth()];
if( subTuCounter == nTus - 1 )
{
TransformUnit* tuPointer = currCU.firstTU;
for( int tuIdx = 0; tuIdx < nTus - 1; tuIdx++ )
{
rootCbfSoFar |= TU::getCbfAtDepth( *tuPointer, COMPONENT_Y, currDepth );
tuPointer = tuPointer->next;
}
if( !rootCbfSoFar )
{
lastCbfIsInferred = true;
}
}
if( !lastCbfIsInferred )
{
previousCbf = TU::getPrevTuCbfAtDepth( currTU, COMPONENT_Y, partitioner.currTrDepth );
}
}
if( !lastCbfIsInferred )
{
m_CABACEstimator->cbf_comp( cs, TU::getCbfAtDepth( currTU, COMPONENT_Y, currDepth ), currTU.Y(), currTU.depth, previousCbf, currCU.ispMode );
}

Karsten Suehring
committed
}
}
}
void IntraSearch::xEncCoeffQT( CodingStructure &cs, Partitioner &partitioner, const ComponentID compID, const int subTuIdx, const PartSplit ispType )

Karsten Suehring
committed
{
const UnitArea &currArea = partitioner.currArea();
int subTuCounter = subTuIdx;
TransformUnit &currTU = *cs.getTU( currArea.blocks[partitioner.chType], partitioner.chType, subTuIdx );

Karsten Suehring
committed
uint32_t currDepth = partitioner.currTrDepth;
const bool subdiv = currTU.depth > currDepth;
if (subdiv)
{
if (partitioner.canSplit(TU_MAX_TR_SPLIT, cs))
{
partitioner.splitCurrArea(TU_MAX_TR_SPLIT, cs);
}
else if( currTU.cu->ispMode )
{
partitioner.splitCurrArea( ispType, cs );
}

Karsten Suehring
committed
else
THROW("Implicit TU split not available!");
do
{
xEncCoeffQT( cs, partitioner, compID, subTuCounter, ispType );
subTuCounter += subTuCounter != -1 ? 1 : 0;

Karsten Suehring
committed
} while( partitioner.nextPart( cs ) );
partitioner.exitCurrSplit();
}
else
if( currArea.blocks[compID].valid() )
{
if( TU::hasCrossCompPredInfo( currTU, compID ) )
{
m_CABACEstimator->cross_comp_pred( currTU, compID );
}
if( TU::getCbf( currTU, compID ) )
{
m_CABACEstimator->residual_coding( currTU, compID );
}
}
}
uint64_t IntraSearch::xGetIntraFracBitsQT( CodingStructure &cs, Partitioner &partitioner, const bool &bLuma, const bool &bChroma, const int subTuIdx, const PartSplit ispType )

Karsten Suehring
committed
{
m_CABACEstimator->resetBits();
xEncIntraHeader( cs, partitioner, bLuma, bChroma, subTuIdx );
xEncSubdivCbfQT( cs, partitioner, bLuma, bChroma, subTuIdx, ispType );

Karsten Suehring
committed
if( bLuma )
{
xEncCoeffQT( cs, partitioner, COMPONENT_Y, subTuIdx, ispType );

Karsten Suehring
committed
}
if( bChroma )
{
xEncCoeffQT( cs, partitioner, COMPONENT_Cb, subTuIdx, ispType );
xEncCoeffQT( cs, partitioner, COMPONENT_Cr, subTuIdx, ispType );
}
uint64_t fracBits = m_CABACEstimator->getEstFracBits();
return fracBits;
}
uint64_t IntraSearch::xGetIntraFracBitsQTSingleChromaComponent( CodingStructure &cs, Partitioner &partitioner, const ComponentID compID )
{
m_CABACEstimator->resetBits();
if( compID == COMPONENT_Cb )
{
PredictionUnit &pu = *cs.getPU( partitioner.currArea().lumaPos(), partitioner.chType );
m_CABACEstimator->intra_chroma_pred_mode( pu );
//xEncIntraHeader(cs, partitioner, false, true);
}
CHECK( partitioner.currTrDepth != 1, "error in the depth!" );
const UnitArea &currArea = partitioner.currArea();
TransformUnit &currTU = *cs.getTU( currArea.blocks[partitioner.chType], partitioner.chType );
//cbf coding
m_CABACEstimator->cbf_comp( cs, TU::getCbfAtDepth( currTU, compID, partitioner.currTrDepth ), currArea.blocks[compID], partitioner.currTrDepth - 1 );
//coeffs coding and cross comp coding
if( TU::hasCrossCompPredInfo( currTU, compID ) )
{
m_CABACEstimator->cross_comp_pred( currTU, compID );
}
if( TU::getCbf( currTU, compID ) )
{
m_CABACEstimator->residual_coding( currTU, compID );

Karsten Suehring
committed
}
uint64_t fracBits = m_CABACEstimator->getEstFracBits();
return fracBits;
}
uint64_t IntraSearch::xGetIntraFracBitsQTChroma(TransformUnit& currTU, const ComponentID &compID)
{
m_CABACEstimator->resetBits();
if( TU::hasCrossCompPredInfo( currTU, compID ) )
{
m_CABACEstimator->cross_comp_pred( currTU, compID );
}
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
#if JVET_N0054_JOINT_CHROMA
// Include Cbf and jointCbCr flags here as we make decisions across components
CodingStructure &cs = *currTU.cs;
if ( currTU.jointCbCr )
{
if ( TU::getCbf( currTU, COMPONENT_Cb ) )
{
m_CABACEstimator->cbf_comp( cs, true, currTU.blocks[ COMPONENT_Cb ], currTU.depth, false );
m_CABACEstimator->cbf_comp( cs, true, currTU.blocks[ COMPONENT_Cr ], currTU.depth, true );
m_CABACEstimator->joint_cb_cr( currTU );
}
else
{
m_CABACEstimator->cbf_comp( cs, false, currTU.blocks[ COMPONENT_Cb ], currTU.depth, false );
m_CABACEstimator->cbf_comp( cs, false, currTU.blocks[ COMPONENT_Cr ], currTU.depth, false );
}
}
else
{
if ( compID == COMPONENT_Cb )
m_CABACEstimator->cbf_comp( cs, TU::getCbf( currTU, compID ), currTU.blocks[ compID ], currTU.depth, false );
else
m_CABACEstimator->cbf_comp( cs, TU::getCbf( currTU, compID ), currTU.blocks[ compID ], currTU.depth, TU::getCbf( currTU, COMPONENT_Cb ) );
}
#endif

Karsten Suehring
committed
if( TU::getCbf( currTU, compID ) )
{
m_CABACEstimator->residual_coding( currTU, compID );
}
uint64_t fracBits = m_CABACEstimator->getEstFracBits();
return fracBits;
}
void IntraSearch::xIntraCodingTUBlock(TransformUnit &tu, const ComponentID &compID, const bool &checkCrossCPrediction, Distortion& ruiDist, const int &default0Save1Load2, uint32_t* numSig, std::vector<TrMode>* trModes, const bool loadTr)

Karsten Suehring
committed
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
{
if (!tu.blocks[compID].valid())
{
return;
}
CodingStructure &cs = *tu.cs;
const CompArea &area = tu.blocks[compID];
const SPS &sps = *cs.sps;
const PPS &pps = *cs.pps;
const ChannelType chType = toChannelType(compID);
const int bitDepth = sps.getBitDepth(chType);
PelBuf piOrg = cs.getOrgBuf (area);
PelBuf piPred = cs.getPredBuf (area);
PelBuf piResi = cs.getResiBuf (area);
PelBuf piOrgResi = cs.getOrgResiBuf(area);
PelBuf piReco = cs.getRecoBuf (area);
const PredictionUnit &pu = *cs.getPU(area.pos(), chType);
const uint32_t uiChFinalMode = PU::getFinalIntraMode(pu, chType);
const bool bUseCrossCPrediction = pps.getPpsRangeExtension().getCrossComponentPredictionEnabledFlag() && isChroma( compID ) && PU::isChromaIntraModeCrossCheckMode( pu ) && checkCrossCPrediction;
const bool ccUseRecoResi = m_pcEncCfg->getUseReconBasedCrossCPredictionEstimate();
const bool ispSplitIsAllowed = CU::canUseISPSplit( *tu.cu, compID );

Karsten Suehring
committed
//===== init availability pattern =====
#if JVET_N0054_JOINT_CHROMA
bool jointCbCr = tu.jointCbCr && compID == COMPONENT_Cb;
if ( compID == COMPONENT_Y )
{
#endif

Karsten Suehring
committed
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
PelBuf sharedPredTS( m_pSharedPredTransformSkip[compID], area );
if( default0Save1Load2 != 2 )
{
const bool bUseFilteredPredictions = IntraPrediction::useFilteredIntraRefSamples( compID, pu, true, tu );
initIntraPatternChType( *tu.cu, area, bUseFilteredPredictions );
//===== get prediction signal =====
if( compID != COMPONENT_Y && PU::isLMCMode( uiChFinalMode ) )
{
{
xGetLumaRecPixels( pu, area );
}
predIntraChromaLM( compID, piPred, pu, area, uiChFinalMode );
}
else
{
predIntraAng( compID, piPred, pu, bUseFilteredPredictions );
}
// save prediction
if( default0Save1Load2 == 1 )
{
sharedPredTS.copyFrom( piPred );
}
}
else
{
// load prediction
piPred.copyFrom( sharedPredTS );
}
#if JVET_N0054_JOINT_CHROMA
}
#endif

Karsten Suehring
committed
DTRACE( g_trace_ctx, D_PRED, "@(%4d,%4d) [%2dx%2d] IMode=%d\n", tu.lx(), tu.ly(), tu.lwidth(), tu.lheight(), uiChFinalMode );
//DTRACE_PEL_BUF( D_PRED, piPred, tu, tu.cu->predMode, COMPONENT_Y );
bool flag = slice.getReshapeInfo().getUseSliceReshaper() && (slice.isIntra() || (!slice.isIntra() && m_pcReshape->getCTUFlag()));
if (flag && slice.getReshapeInfo().getSliceReshapeChromaAdj() && isChroma(compID))
{
const Area area = tu.Y().valid() ? tu.Y() : Area(recalcPosition(tu.chromaFormat, tu.chType, CHANNEL_TYPE_LUMA, tu.blocks[tu.chType].pos()), recalcSize(tu.chromaFormat, tu.chType, CHANNEL_TYPE_LUMA, tu.blocks[tu.chType].size()));
const CompArea &areaY = CompArea(COMPONENT_Y, tu.chromaFormat, area );
PelBuf piPredY;
piPredY = cs.picture->getPredBuf(areaY);
const Pel avgLuma = piPredY.computeAvg();
int adj = m_pcReshape->calculateChromaAdj(avgLuma);
tu.setChromaAdj(adj);
}

Karsten Suehring
committed
//===== get residual signal =====
piResi.copyFrom( piOrg );
if (slice.getReshapeInfo().getUseSliceReshaper() && m_pcReshape->getCTUFlag() && compID==COMPONENT_Y)
{
CompArea tmpArea(COMPONENT_Y, area.chromaFormat, Position(0, 0), area.size());
PelBuf tmpPred = m_tmpStorageLCU.getBuf(tmpArea);
tmpPred.copyFrom(piPred);
piResi.rspSignal(m_pcReshape->getFwdLUT());
piResi.subtract(tmpPred);
}
else

Karsten Suehring
committed
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
piResi.subtract( piPred );
if (pps.getPpsRangeExtension().getCrossComponentPredictionEnabledFlag() && isLuma(compID))
{
piOrgResi.copyFrom (piResi);
}
if (bUseCrossCPrediction)
{
if (xCalcCrossComponentPredictionAlpha(tu, compID, ccUseRecoResi) == 0)
{
return;
}
CrossComponentPrediction::crossComponentPrediction(tu, compID, cs.getResiBuf(tu.Y()), piResi, piResi, false);
}
//===== transform and quantization =====
//--- init rate estimation arrays for RDOQ ---
//--- transform and quantization ---
TCoeff uiAbsSum = 0;
const QpParam cQP(tu, compID);
#if RDOQ_CHROMA_LAMBDA
m_pcTrQuant->selectLambda(compID);
#endif
flag =flag && (tu.blocks[compID].width*tu.blocks[compID].height > 4);
if (flag && isChroma(compID) && slice.getReshapeInfo().getSliceReshapeChromaAdj() )
int cResScaleInv = tu.getChromaAdj();
double cResScale = round((double)(1 << CSCALE_FP_PREC) / (double)cResScaleInv);
m_pcTrQuant->setLambda(m_pcTrQuant->getLambda() / (cResScale*cResScale));
#if JVET_N0054_JOINT_CHROMA
if ( !jointCbCr ) // Joint CbCr signal is to be scaled in the case of joint chroma
#endif
piResi.scaleSignal(cResScaleInv, 1, tu.cu->cs->slice->clpRng(compID));
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
#if JVET_N0054_JOINT_CHROMA
const CompArea &crArea = tu.blocks [ COMPONENT_Cr ];
PelBuf crOrg = cs.getOrgBuf ( crArea );
PelBuf crPred = cs.getPredBuf ( crArea );
PelBuf crResi = cs.getResiBuf ( crArea );
PelBuf crReco = cs.getRecoBuf ( crArea );
if ( jointCbCr )
{
// Get Cr prediction and residual
crResi.copyFrom( crOrg );
crResi.subtract( crPred );
// Create joint residual and store it for Cb component: jointResi = (cbResi - crResi)/2
piResi.subtractAndHalve( crResi );
// Scale the joint signal
if ( flag && slice.getReshapeInfo().getSliceReshapeChromaAdj() )
piResi.scaleSignal(tu.getChromaAdj(), 1, tu.cu->cs->slice->clpRng(compID));
// Lambda is loosened for the joint mode with respect to single modes as the same residual is used for both chroma blocks
m_pcTrQuant->setLambda( 0.60 * m_pcTrQuant->getLambda() );
}
else if ( isChroma(compID) && tu.cu->cs->slice->getSliceQp() > 18 )
m_pcTrQuant->setLambda( 1.10 * m_pcTrQuant->getLambda() );
#endif
double diagRatio = 0, horVerRatio = 0;
m_pcTrQuant->transformNxN( tu, compID, cQP, trModes, CU::isIntra( *tu.cu ) ? m_pcEncCfg->getIntraMTSMaxCand() : m_pcEncCfg->getInterMTSMaxCand(), ispSplitIsAllowed ? &diagRatio : nullptr, ispSplitIsAllowed ? &horVerRatio : nullptr );
m_pcTrQuant->transformNxN( tu, compID, cQP, uiAbsSum, m_CABACEstimator->getCtx(), loadTr, &diagRatio, &horVerRatio );
if (!tu.cu->ispMode && isLuma(compID) && ispSplitIsAllowed &&
tu.mtsIdx == 0
)
{
m_intraModeDiagRatio .push_back(diagRatio);
m_intraModeHorVerRatio .push_back(horVerRatio);
m_intraModeTestedNormalIntra.push_back((int)uiChFinalMode);
}

Karsten Suehring
committed
DTRACE( g_trace_ctx, D_TU_ABS_SUM, "%d: comp=%d, abssum=%d\n", DTRACE_GET_COUNTER( g_trace_ctx, D_TU_ABS_SUM ), compID, uiAbsSum );
//--- inverse transform ---
if (uiAbsSum > 0)
{
m_pcTrQuant->invTransformNxN(tu, compID, piResi, cQP);
}
else
{
piResi.fill(0);
}
//===== reconstruction =====
if (flag && uiAbsSum > 0 && isChroma(compID) && slice.getReshapeInfo().getSliceReshapeChromaAdj() )
piResi.scaleSignal(tu.getChromaAdj(), 0, tu.cu->cs->slice->clpRng(compID));

Karsten Suehring
committed
if (bUseCrossCPrediction)
{
CrossComponentPrediction::crossComponentPrediction(tu, compID, cs.getResiBuf(tu.Y()), piResi, piResi, true);
}
if (slice.getReshapeInfo().getUseSliceReshaper() && m_pcReshape->getCTUFlag() && compID == COMPONENT_Y)
{
CompArea tmpArea(COMPONENT_Y, area.chromaFormat, Position(0,0), area.size());
PelBuf tmpPred = m_tmpStorageLCU.getBuf(tmpArea);
tmpPred.copyFrom(piPred);
piReco.reconstruct(tmpPred, piResi, cs.slice->clpRng(compID));
}

Karsten Suehring
committed
piReco.reconstruct(piPred, piResi, cs.slice->clpRng( compID ));
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
#if JVET_N0054_JOINT_CHROMA
if ( jointCbCr )
{
// Cr uses negative of the signalled Cb residual
if (uiAbsSum > 0)
crResi.copyAndNegate( piResi );
else
crResi.fill(0);
tu.getCoeffs(COMPONENT_Cr).fill(0);
// Set cbf also for Cr
TU::setCbfAtDepth (tu, COMPONENT_Cr, tu.depth, uiAbsSum > 0 ? true : false);
// Cr reconstruction and its contribution to the total error
crReco.reconstruct(crPred, crResi, cs.slice->clpRng( COMPONENT_Cr ));
#if WCG_EXT
if ( m_pcEncCfg->getLumaLevelToDeltaQPMapping().isEnabled() ||
(m_pcEncCfg->getReshaper()
&& slice.getReshapeInfo().getUseSliceReshaper()
&& (m_pcReshape->getCTUFlag() || (isChroma(compID) && m_pcEncCfg->getReshapeIntraCMD()))))
{
const CPelBuf orgLuma = cs.getOrgBuf( cs.area.blocks[COMPONENT_Y] );
ruiDist += m_pcRdCost->getDistPart( crOrg, crReco, bitDepth, COMPONENT_Cr, DF_SSE_WTD, &orgLuma );
}
else
#endif
{
ruiDist += m_pcRdCost->getDistPart( crOrg, crReco, bitDepth, COMPONENT_Cr, DF_SSE );
}
}
#endif

Karsten Suehring
committed
//===== update distortion =====
#if WCG_EXT
if (m_pcEncCfg->getLumaLevelToDeltaQPMapping().isEnabled() || (m_pcEncCfg->getReshaper()
&& slice.getReshapeInfo().getUseSliceReshaper() && (m_pcReshape->getCTUFlag() || (isChroma(compID) && m_pcEncCfg->getReshapeIntraCMD()))))

Karsten Suehring
committed
{
const CPelBuf orgLuma = cs.getOrgBuf( cs.area.blocks[COMPONENT_Y] );
if (compID == COMPONENT_Y && !(m_pcEncCfg->getLumaLevelToDeltaQPMapping().isEnabled()))
{
CompArea tmpArea1(COMPONENT_Y, area.chromaFormat, Position(0, 0), area.size());
PelBuf tmpRecLuma = m_tmpStorageLCU.getBuf(tmpArea1);
tmpRecLuma.copyFrom(piReco);
tmpRecLuma.rspSignal(m_pcReshape->getInvLUT());
ruiDist += m_pcRdCost->getDistPart(piOrg, tmpRecLuma, sps.getBitDepth(toChannelType(compID)), compID, DF_SSE_WTD, &orgLuma);
}
else
ruiDist += m_pcRdCost->getDistPart(piOrg, piReco, bitDepth, compID, DF_SSE_WTD, &orgLuma);

Karsten Suehring
committed
}
else
#endif
{
ruiDist += m_pcRdCost->getDistPart( piOrg, piReco, bitDepth, compID, DF_SSE );
}
}
Santiago de Luxán Hernández
committed
void IntraSearch::xRecurIntraCodingLumaQT( CodingStructure &cs, Partitioner &partitioner, const double bestCostSoFar, const int subTuIdx, const PartSplit ispType, const bool ispIsCurrentWinnder )
{
int subTuCounter = subTuIdx;
const UnitArea &currArea = partitioner.currArea();
const CodingUnit &cu = *cs.getCU( currArea.lumaPos(), partitioner.chType );
bool earlySkipISP = false;

Karsten Suehring
committed
const PPS &pps = *cs.pps;
const bool keepResi = pps.getPpsRangeExtension().getCrossComponentPredictionEnabledFlag() || KEEP_PRED_AND_RESI_SIGNALS;
bool bCheckFull = true;
bool bCheckSplit = false;
bCheckFull = !partitioner.canSplit( TU_MAX_TR_SPLIT, cs );
bCheckSplit = partitioner.canSplit( TU_MAX_TR_SPLIT, cs );

Karsten Suehring
committed
if( cu.ispMode )
{
bCheckSplit = partitioner.canSplit( ispType, cs );
bCheckFull = !bCheckSplit;
}

Karsten Suehring
committed
uint32_t numSig = 0;
double dSingleCost = MAX_DOUBLE;
Distortion uiSingleDistLuma = 0;
uint64_t singleFracBits = 0;
int bestModeId[MAX_NUM_COMPONENT] = { 0, 0, 0 };

Karsten Suehring
committed
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
const TempCtx ctxStart ( m_CtxCache, m_CABACEstimator->getCtx() );
TempCtx ctxBest ( m_CtxCache );
CodingStructure *csSplit = nullptr;
CodingStructure *csFull = nullptr;
if( bCheckSplit )
{
csSplit = &cs;
}
else if( bCheckFull )
{
csFull = &cs;
}
if( bCheckFull )
{
csFull->cost = 0.0;
TransformUnit &tu = csFull->addTU( CS::getArea( *csFull, currArea, partitioner.chType ), partitioner.chType );
tu.depth = currDepth;
const bool tsAllowed = TU::isTSAllowed ( tu, COMPONENT_Y );
const bool mtsAllowed = TU::isMTSAllowed( tu, COMPONENT_Y );
uint8_t nNumTransformCands = 1 + ( tsAllowed ? 1 : 0 ) + ( mtsAllowed ? 4 : 0 ); // DCT + TS + 4 MTS = 6 tests
std::vector<TrMode> trModes;
trModes.push_back( TrMode( 0, true ) ); //DCT2
if( tsAllowed )
{
trModes.push_back( TrMode( 1, true ) );
}
if( mtsAllowed )
{
for( int i = 2; i < 6; i++ )
{
trModes.push_back( TrMode( i, true) );
}
}
CHECK( !tu.Y().valid(), "Invalid TU" );

Karsten Suehring
committed
CodingStructure &saveCS = *m_pSaveCS[0];
TransformUnit *tmpTU = nullptr;
Distortion singleDistTmpLuma = 0;
uint64_t singleTmpFracBits = 0;
double singleCostTmp = 0;
int firstCheckId = 0;
int lastCheckId = trModes[nNumTransformCands-1].first;
bool isNotOnlyOneMode = nNumTransformCands != 1;

Karsten Suehring
committed
if( isNotOnlyOneMode )
{
saveCS.pcv = cs.pcv;
saveCS.picture = cs.picture;
saveCS.area.repositionTo(cs.area);
saveCS.clearTUs();
tmpTU = &saveCS.addTU(currArea, partitioner.chType);
}

Karsten Suehring
committed
Santiago de Luxán Hernández
committed
double bestDCT2cost = MAX_DOUBLE;
double threshold = m_pcEncCfg->getUseFastISP() && !cu.ispMode && ispIsCurrentWinnder && nNumTransformCands > 1 ? 1 + 1.4 / sqrt( cu.lwidth() * cu.lheight() ) : 1;
for( int modeId = firstCheckId; modeId < nNumTransformCands; modeId++ )
{
if( !cbfDCT2 || ( m_pcEncCfg->getUseTransformSkipFast() && bestModeId[COMPONENT_Y] == 1 ) )
{
break;
}
if( !trModes[modeId].second )
{
continue;
}
Santiago de Luxán Hernández
committed
//we compare the DCT-II cost against the best ISP cost so far (except for TS)
if ( m_pcEncCfg->getUseFastISP() && !cu.ispMode && ispIsCurrentWinnder && trModes[modeId].first != 0 && ( trModes[modeId].first != 1 || !tsAllowed ) && bestDCT2cost > bestCostSoFar * threshold )
{
continue;
}

Karsten Suehring
committed
if ((modeId != firstCheckId) && isNotOnlyOneMode)
{
m_CABACEstimator->getCtx() = ctxStart;
}
int default0Save1Load2 = 0;
singleDistTmpLuma = 0;

Karsten Suehring
committed
{
default0Save1Load2 = 1;
}
else if (modeId != firstCheckId)
{
default0Save1Load2 = 2;
}
if( cu.ispMode )
{
default0Save1Load2 = 0;
}
if( nNumTransformCands > 1 )
{
xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig, modeId == 0 ? &trModes : nullptr, true );
if( modeId == 0 )
{
for( int i = 0; i < nNumTransformCands; i++ )
{
if( trModes[i].second )
{
lastCheckId = trModes[i].first;
}
}
}
}
else
{
xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig );
}

Karsten Suehring
committed
//----- determine rate and r-d cost -----
if( ( trModes[modeId].first != 0 && !TU::getCbfAtDepth( tu, COMPONENT_Y, currDepth ) ) )