Newer
Older

Karsten Suehring
committed
/* The copyright in this software is being made available under the BSD
* License, included below. This software may be subject to other third party
* and contributor rights, including patent rights, and no such rights are
* granted under this license.
*
* Copyright (c) 2010-2019, ITU/ISO/IEC

Karsten Suehring
committed
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of the ITU/ISO/IEC nor the names of its contributors may
* be used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
/** \file EncSearch.cpp
* \brief encoder intra search class
*/
#include "IntraSearch.h"
#include "EncModeCtrl.h"
#include "CommonLib/CommonDef.h"
#include "CommonLib/Rom.h"
#include "CommonLib/Picture.h"
#include "CommonLib/UnitTools.h"
#include "CommonLib/dtrace_next.h"
#include "CommonLib/dtrace_buffer.h"
#include <math.h>
#include <limits>
//! \ingroup EncoderLib
//! \{
IntraSearch::IntraSearch()
: m_pSplitCS (nullptr)
, m_pFullCS (nullptr)
, m_pBestCS (nullptr)
, m_pcEncCfg (nullptr)
, m_pcTrQuant (nullptr)
, m_pcRdCost (nullptr)
#if JVET_M0427_INLOOP_RESHAPER
, m_pcReshape (nullptr)
#endif

Karsten Suehring
committed
, m_CABACEstimator(nullptr)
, m_CtxCache (nullptr)
, m_isInitialized (false)
{
for( uint32_t ch = 0; ch < MAX_NUM_TBLOCKS; ch++ )
{
m_pSharedPredTransformSkip[ch] = nullptr;
}
}
void IntraSearch::destroy()
{
CHECK( !m_isInitialized, "Not initialized" );
if( m_pcEncCfg )
{
const uint32_t uiNumLayersToAllocateSplit = 1;
const uint32_t uiNumLayersToAllocateFull = 1;

Karsten Suehring
committed
const int uiNumSaveLayersToAllocate = 2;
for( uint32_t layer = 0; layer < uiNumSaveLayersToAllocate; layer++ )
{
m_pSaveCS[layer]->destroy();
delete m_pSaveCS[layer];
}
uint32_t numWidths = gp_sizeIdxInfo->numWidths();
uint32_t numHeights = gp_sizeIdxInfo->numHeights();
for( uint32_t width = 0; width < numWidths; width++ )
{
for( uint32_t height = 0; height < numHeights; height++ )
{
if( gp_sizeIdxInfo->isCuSize( gp_sizeIdxInfo->sizeFrom( width ) ) && gp_sizeIdxInfo->isCuSize( gp_sizeIdxInfo->sizeFrom( height ) ) )

Karsten Suehring
committed
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
{
for( uint32_t layer = 0; layer < uiNumLayersToAllocateSplit; layer++ )
{
m_pSplitCS[width][height][layer]->destroy();
delete m_pSplitCS[width][height][layer];
}
for( uint32_t layer = 0; layer < uiNumLayersToAllocateFull; layer++ )
{
m_pFullCS[width][height][layer]->destroy();
delete m_pFullCS[width][height][layer];
}
delete[] m_pSplitCS[width][height];
delete[] m_pFullCS [width][height];
m_pBestCS[width][height]->destroy();
m_pTempCS[width][height]->destroy();
delete m_pTempCS[width][height];
delete m_pBestCS[width][height];
}
}
delete[] m_pSplitCS[width];
delete[] m_pFullCS [width];
delete[] m_pTempCS[width];
delete[] m_pBestCS[width];
}
delete[] m_pSplitCS;
delete[] m_pFullCS;
delete[] m_pBestCS;
delete[] m_pTempCS;
delete[] m_pSaveCS;
}
m_pSplitCS = m_pFullCS = nullptr;
m_pBestCS = m_pTempCS = nullptr;
m_pSaveCS = nullptr;
for( uint32_t ch = 0; ch < MAX_NUM_TBLOCKS; ch++ )
{
delete[] m_pSharedPredTransformSkip[ch];
m_pSharedPredTransformSkip[ch] = nullptr;
}
#if JVET_M0427_INLOOP_RESHAPER
m_tmpStorageLCU.destroy();
#endif

Karsten Suehring
committed
m_isInitialized = false;
}
IntraSearch::~IntraSearch()
{
if( m_isInitialized )
{
destroy();
}
}
void IntraSearch::init( EncCfg* pcEncCfg,
TrQuant* pcTrQuant,
RdCost* pcRdCost,
CABACWriter* CABACEstimator,
CtxCache* ctxCache,
const uint32_t maxCUWidth,
const uint32_t maxCUHeight,
const uint32_t maxTotalCUDepth
#if JVET_M0427_INLOOP_RESHAPER
, EncReshape* pcReshape
#endif

Karsten Suehring
committed
)
{
CHECK(m_isInitialized, "Already initialized");
m_pcEncCfg = pcEncCfg;
m_pcTrQuant = pcTrQuant;
m_pcRdCost = pcRdCost;
m_CABACEstimator = CABACEstimator;
m_CtxCache = ctxCache;
#if JVET_M0427_INLOOP_RESHAPER
m_pcReshape = pcReshape;
#endif

Karsten Suehring
committed
const ChromaFormat cform = pcEncCfg->getChromaFormatIdc();
IntraPrediction::init( cform, pcEncCfg->getBitDepth( CHANNEL_TYPE_LUMA ) );
#if JVET_M0427_INLOOP_RESHAPER
m_tmpStorageLCU.create(UnitArea(cform, Area(0, 0, MAX_CU_SIZE, MAX_CU_SIZE)));
#endif

Karsten Suehring
committed
for( uint32_t ch = 0; ch < MAX_NUM_TBLOCKS; ch++ )
{

Karsten Suehring
committed
m_pSharedPredTransformSkip[ch] = new Pel[MAX_CU_SIZE * MAX_CU_SIZE];
}
uint32_t numWidths = gp_sizeIdxInfo->numWidths();
uint32_t numHeights = gp_sizeIdxInfo->numHeights();
const uint32_t uiNumLayersToAllocateSplit = 1;
const uint32_t uiNumLayersToAllocateFull = 1;

Karsten Suehring
committed
m_pBestCS = new CodingStructure**[numWidths];
m_pTempCS = new CodingStructure**[numWidths];
m_pFullCS = new CodingStructure***[numWidths];
m_pSplitCS = new CodingStructure***[numWidths];
for( uint32_t width = 0; width < numWidths; width++ )
{
m_pBestCS[width] = new CodingStructure*[numHeights];
m_pTempCS[width] = new CodingStructure*[numHeights];
m_pFullCS [width] = new CodingStructure**[numHeights];
m_pSplitCS[width] = new CodingStructure**[numHeights];
for( uint32_t height = 0; height < numHeights; height++ )
{
if( gp_sizeIdxInfo->isCuSize( gp_sizeIdxInfo->sizeFrom( width ) ) && gp_sizeIdxInfo->isCuSize( gp_sizeIdxInfo->sizeFrom( height ) ) )

Karsten Suehring
committed
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
{
m_pBestCS[width][height] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
m_pTempCS[width][height] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
m_pBestCS[width][height]->create( m_pcEncCfg->getChromaFormatIdc(), Area( 0, 0, gp_sizeIdxInfo->sizeFrom( width ), gp_sizeIdxInfo->sizeFrom( height ) ), false );
m_pTempCS[width][height]->create( m_pcEncCfg->getChromaFormatIdc(), Area( 0, 0, gp_sizeIdxInfo->sizeFrom( width ), gp_sizeIdxInfo->sizeFrom( height ) ), false );
m_pFullCS [width][height] = new CodingStructure*[uiNumLayersToAllocateFull];
m_pSplitCS[width][height] = new CodingStructure*[uiNumLayersToAllocateSplit];
for( uint32_t layer = 0; layer < uiNumLayersToAllocateFull; layer++ )
{
m_pFullCS [width][height][layer] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
m_pFullCS [width][height][layer]->create( m_pcEncCfg->getChromaFormatIdc(), Area( 0, 0, gp_sizeIdxInfo->sizeFrom( width ), gp_sizeIdxInfo->sizeFrom( height ) ), false );
}
for( uint32_t layer = 0; layer < uiNumLayersToAllocateSplit; layer++ )
{
m_pSplitCS[width][height][layer] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
m_pSplitCS[width][height][layer]->create( m_pcEncCfg->getChromaFormatIdc(), Area( 0, 0, gp_sizeIdxInfo->sizeFrom( width ), gp_sizeIdxInfo->sizeFrom( height ) ), false );
}
}
else
{
m_pBestCS[width][height] = nullptr;
m_pTempCS[width][height] = nullptr;
m_pFullCS [width][height] = nullptr;
m_pSplitCS[width][height] = nullptr;
}
}
}
const int uiNumSaveLayersToAllocate = 2;
m_pSaveCS = new CodingStructure*[uiNumSaveLayersToAllocate];
for( uint32_t depth = 0; depth < uiNumSaveLayersToAllocate; depth++ )
{
m_pSaveCS[depth] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
m_pSaveCS[depth]->create( UnitArea( cform, Area( 0, 0, maxCUWidth, maxCUHeight ) ), false );
}
m_isInitialized = true;
}
//////////////////////////////////////////////////////////////////////////
// INTRA PREDICTION
//////////////////////////////////////////////////////////////////////////
#if JVET_M0102_INTRA_SUBPARTITIONS
void IntraSearch::estIntraPredLumaQT( CodingUnit &cu, Partitioner &partitioner, const double bestCostSoFar )
#else

Karsten Suehring
committed
void IntraSearch::estIntraPredLumaQT( CodingUnit &cu, Partitioner &partitioner )

Karsten Suehring
committed
{
CodingStructure &cs = *cu.cs;
const SPS &sps = *cs.sps;
const uint32_t uiWidthBit = g_aucLog2[partitioner.currArea().lwidth() ];

Karsten Suehring
committed
const uint32_t uiHeightBit = g_aucLog2[partitioner.currArea().lheight()];
// Lambda calculation at equivalent Qp of 4 is recommended because at that Qp, the quantization divisor is 1.
const double sqrtLambdaForFirstPass = m_pcRdCost->getMotionLambda(cu.transQuantBypass) / double(1 << SCALE_BITS);
//===== loop over partitions =====
const TempCtx ctxStart ( m_CtxCache, m_CABACEstimator->getCtx() );
const TempCtx ctxStartIntraMode(m_CtxCache, SubCtx(Ctx::IntraLumaMpmFlag, m_CABACEstimator->getCtx()));
const TempCtx ctxStartMHIntraMode ( m_CtxCache, SubCtx( Ctx::MHIntraPredMode, m_CABACEstimator->getCtx() ) );
const TempCtx ctxStartMrlIdx ( m_CtxCache, SubCtx( Ctx::MultiRefLineIdx, m_CABACEstimator->getCtx() ) );

Karsten Suehring
committed
CHECK( !cu.firstPU, "CU has no PUs" );
const bool keepResi = cs.pps->getPpsRangeExtension().getCrossComponentPredictionEnabledFlag() || KEEP_PRED_AND_RESI_SIGNALS;
uint32_t extraModes = 0; // add two extra modes, which would be used after uiMode <= DC_IDX is removed for cu.nsstIdx == 3

Karsten Suehring
committed
const int width = partitioner.currArea().lwidth();
const int height = partitioner.currArea().lheight();
// Marking EMT usage for faster EMT
// 0: EMT is either not applicable for current CU (cuWidth > EMT_INTRA_MAX_CU or cuHeight > EMT_INTRA_MAX_CU), not active in the config file or the fast decision algorithm is not used in this case
// 1: EMT fast algorithm can be applied for the current CU, and the DCT2 is being checked
// 2: EMT is being checked for current CU. Stored results of DCT2 can be utilized for speedup
uint8_t emtUsageFlag = 0;
if( width <= maxSizeEMT && height <= maxSizeEMT && sps.getUseIntraEMT() )

Karsten Suehring
committed
{
emtUsageFlag = cu.emtFlag == 1 ? 2 : 1;
}
bool isAllIntra = m_pcEncCfg->getIntraPeriod() == 1;

Karsten Suehring
committed
{
emtUsageFlag = 0; //this forces the recalculation of the candidates list. Why is this necessary? (to be checked)

Karsten Suehring
committed
}
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
#if JVET_M0102_INTRA_SUBPARTITIONS
#if JVET_M0464_UNI_MTS
const int width = partitioner.currArea().lwidth();
const int height = partitioner.currArea().lheight();
int nOptionsForISP = NUM_INTRA_SUBPARTITIONS_MODES;
#else
int nOptionsForISP = cu.emtFlag == 0 ? NUM_INTRA_SUBPARTITIONS_MODES : 1;
#endif
double bestCurrentCost = bestCostSoFar;
int ispOptions[NUM_INTRA_SUBPARTITIONS_MODES] = { 0 };
if( nOptionsForISP > 1 )
{
auto splitsThatCanBeUsedForISP = CU::canUseISPSplit( width, height, cu.cs->sps->getMaxTrSize() );
if( splitsThatCanBeUsedForISP == CAN_USE_VER_AND_HORL_SPLITS )
{
const CodingUnit* cuLeft = cu.ispMode != NOT_INTRA_SUBPARTITIONS ? cs.getCU( cs.area.blocks[partitioner.chType].pos().offset( -1, 0 ), partitioner.chType ) : nullptr;
const CodingUnit* cuAbove = cu.ispMode != NOT_INTRA_SUBPARTITIONS ? cs.getCU( cs.area.blocks[partitioner.chType].pos().offset( 0, -1 ), partitioner.chType ) : nullptr;
bool ispHorIsFirstTest = CU::firstTestISPHorSplit( width, height, COMPONENT_Y, cuLeft, cuAbove );
if( ispHorIsFirstTest )
{
ispOptions[1] = HOR_INTRA_SUBPARTITIONS;
ispOptions[2] = VER_INTRA_SUBPARTITIONS;
}
else
{
ispOptions[1] = VER_INTRA_SUBPARTITIONS;
ispOptions[2] = HOR_INTRA_SUBPARTITIONS;
}
}
else if( splitsThatCanBeUsedForISP == HOR_INTRA_SUBPARTITIONS )
{
nOptionsForISP = 2;
ispOptions[1] = HOR_INTRA_SUBPARTITIONS;
}
else if( splitsThatCanBeUsedForISP == VER_INTRA_SUBPARTITIONS )
{
nOptionsForISP = 2;
ispOptions[1] = VER_INTRA_SUBPARTITIONS;
}
else
{
nOptionsForISP = 1;
}
}
if( nOptionsForISP > 1 )
{
//variables for the full RD list without MRL modes
m_rdModeListWithoutMrl .clear();
m_rdModeListWithoutMrlHor .clear();
m_rdModeListWithoutMrlVer .clear();
//variables with data from regular intra used to skip ISP splits
m_intraModeDiagRatio .clear();
m_intraModeHorVerRatio .clear();
m_intraModeTestedNormalIntra.clear();
}
#endif

Karsten Suehring
committed
static_vector<uint32_t, FAST_UDI_MAX_RDMODE_NUM> uiHadModeList;
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> CandCostList;
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> CandHadList;
static_vector<int, FAST_UDI_MAX_RDMODE_NUM> extendRefList;
static_vector<int, FAST_UDI_MAX_RDMODE_NUM>* nullList = NULL;

Karsten Suehring
committed
auto &pu = *cu.firstPU;

Karsten Suehring
committed
int puIndex = 0;

Karsten Suehring
committed
{
CandHadList.clear();
CandCostList.clear();
uiHadModeList.clear();

Karsten Suehring
committed
CHECK(pu.cu != &cu, "PU is not contained in the CU");
//===== determine set of modes to be tested (using prediction signal only) =====
int numModesAvailable = NUM_LUMA_MODE; // total number of Intra modes
static_vector< uint32_t, FAST_UDI_MAX_RDMODE_NUM > uiRdModeList;
int numModesForFullRD = 3;
numModesForFullRD = g_aucIntraModeNumFast_UseMPM_2D[uiWidthBit - MIN_CU_LOG2][uiHeightBit - MIN_CU_LOG2];

Karsten Suehring
committed
#if INTRA_FULL_SEARCH
numModesForFullRD = numModesAvailable;
#endif

Karsten Suehring
committed
if( emtUsageFlag != 2 )

Karsten Suehring
committed
{
// this should always be true
CHECK( !pu.Y().valid(), "PU is not valid" );
bool isFirstLineOfCtu = (((pu.block(COMPONENT_Y).y)&((pu.cs->sps)->getMaxCUWidth() - 1)) == 0);
int numOfPassesExtendRef = (isFirstLineOfCtu ? 1 : MRL_NUM_REF_LINES);
pu.multiRefIdx = 0;

Karsten Suehring
committed
//===== init pattern for luma prediction =====
initIntraPatternChType( cu, pu.Y(), IntraPrediction::useFilteredIntraRefSamples( COMPONENT_Y, pu, false, pu ) );
if( numModesForFullRD != numModesAvailable )
{
CHECK( numModesForFullRD >= numModesAvailable, "Too many modes for full RD search" );
const CompArea &area = pu.Y();
PelBuf piOrg = cs.getOrgBuf(area);
PelBuf piPred = cs.getPredBuf(area);
DistParam distParam;
const bool bUseHadamard = cu.transQuantBypass == 0;
#if JVET_M0427_INLOOP_RESHAPER
if (cu.slice->getReshapeInfo().getUseSliceReshaper() && m_pcReshape->getCTUFlag())
{
CompArea tmpArea(COMPONENT_Y, area.chromaFormat, Position(0, 0), area.size());
PelBuf tmpOrg = m_tmpStorageLCU.getBuf(tmpArea);
tmpOrg.copyFrom(piOrg);
tmpOrg.rspSignal(m_pcReshape->getFwdLUT());
m_pcRdCost->setDistParam(distParam, tmpOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y, bUseHadamard);
}
else
#endif

Karsten Suehring
committed
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
m_pcRdCost->setDistParam(distParam, piOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y, bUseHadamard);
distParam.applyWeight = false;
bool bSatdChecked[NUM_INTRA_MODE];
memset( bSatdChecked, 0, sizeof( bSatdChecked ) );
{
for( int modeIdx = 0; modeIdx < numModesAvailable; modeIdx++ )
{
uint32_t uiMode = modeIdx;
Distortion uiSad = 0;
// Skip checking extended Angular modes in the first round of SATD
if( uiMode > DC_IDX && ( uiMode & 1 ) )
{
continue;
}
bSatdChecked[uiMode] = true;
pu.intraDir[0] = modeIdx;
if( useDPCMForFirstPassIntraEstimation( pu, uiMode ) )
{
encPredIntraDPCM( COMPONENT_Y, piOrg, piPred, uiMode );
}
else
{
predIntraAng( COMPONENT_Y, piPred, pu, IntraPrediction::useFilteredIntraRefSamples( COMPONENT_Y, pu, true, pu ) );
}
// use Hadamard transform here
uiSad += distParam.distFunc(distParam);
// NB xFracModeBitsIntra will not affect the mode for chroma that may have already been pre-estimated.
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
m_CABACEstimator->getCtx() = SubCtx( Ctx::MHIntraPredMode, ctxStartMHIntraMode );
m_CABACEstimator->getCtx() = SubCtx( Ctx::MultiRefLineIdx, ctxStartMrlIdx );

Karsten Suehring
committed
uint64_t fracModeBits = xFracModeBitsIntra(pu, uiMode, CHANNEL_TYPE_LUMA);
double cost = ( double ) uiSad + ( double ) fracModeBits * sqrtLambdaForFirstPass;
DTRACE( g_trace_ctx, D_INTRA_COST, "IntraHAD: %u, %llu, %f (%d)\n", uiSad, fracModeBits, cost, uiMode );
updateCandList( uiMode, cost, uiRdModeList, CandCostList
, extendRefList, 0
, numModesForFullRD + extraModes );
updateCandList(uiMode, (double) uiSad, uiHadModeList, CandHadList
, *nullList, -1
, 3 + extraModes);

Karsten Suehring
committed
}
} // NSSTFlag
// forget the extra modes
uiRdModeList.resize( numModesForFullRD );
CandCostList.resize(numModesForFullRD);
extendRefList.resize(numModesForFullRD);

Karsten Suehring
committed
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
static_vector<unsigned, FAST_UDI_MAX_RDMODE_NUM> parentCandList(FAST_UDI_MAX_RDMODE_NUM);
std::copy_n(uiRdModeList.begin(), numModesForFullRD, parentCandList.begin());
// Second round of SATD for extended Angular modes
for (int modeIdx = 0; modeIdx < numModesForFullRD; modeIdx++)
{
unsigned parentMode = parentCandList[modeIdx];
if (parentMode > (DC_IDX + 1) && parentMode < (NUM_LUMA_MODE - 1))
{
for (int subModeIdx = -1; subModeIdx <= 1; subModeIdx += 2)
{
unsigned mode = parentMode + subModeIdx;
if (!bSatdChecked[mode])
{
pu.intraDir[0] = mode;
if (useDPCMForFirstPassIntraEstimation(pu, mode))
{
encPredIntraDPCM(COMPONENT_Y, piOrg, piPred, mode);
}
else
{
predIntraAng(COMPONENT_Y, piPred, pu,
IntraPrediction::useFilteredIntraRefSamples(COMPONENT_Y, pu, true, pu));
}
// use Hadamard transform here
Distortion sad = distParam.distFunc(distParam);
// NB xFracModeBitsIntra will not affect the mode for chroma that may have already been pre-estimated.
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
m_CABACEstimator->getCtx() = SubCtx( Ctx::MHIntraPredMode, ctxStartMHIntraMode );
m_CABACEstimator->getCtx() = SubCtx( Ctx::MultiRefLineIdx, ctxStartMrlIdx );

Karsten Suehring
committed
uint64_t fracModeBits = xFracModeBitsIntra(pu, mode, CHANNEL_TYPE_LUMA);
double cost = (double) sad + (double) fracModeBits * sqrtLambdaForFirstPass;
updateCandList(mode, cost, uiRdModeList, CandCostList
, extendRefList, 0
, numModesForFullRD);
updateCandList(mode, (double)sad, uiHadModeList, CandHadList
, *nullList, -1
, 3);

Karsten Suehring
committed
bSatdChecked[mode] = true;
}
}
}
}
#if JVET_M0102_INTRA_SUBPARTITIONS
if( nOptionsForISP > 1 )
{
//we save the list with no mrl modes to keep only the Hadamard selected modes (no mpms)
m_rdModeListWithoutMrl.resize( numModesForFullRD );
std::copy_n( uiRdModeList.begin(), numModesForFullRD, m_rdModeListWithoutMrl.begin() );
}
#endif
const int numMPMs = NUM_MOST_PROBABLE_MODES;
unsigned multiRefMPM [numMPMs];
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
PU::getIntraMPMs(pu, multiRefMPM);
for (int mRefNum = 1; mRefNum < numOfPassesExtendRef; mRefNum++)
{
int multiRefIdx = MULTI_REF_LINE_IDX[mRefNum];
pu.multiRefIdx = multiRefIdx;
{
initIntraPatternChType(cu, pu.Y(), IntraPrediction::useFilteredIntraRefSamples(COMPONENT_Y, pu, false, pu));
}
for (int x = 0; x < numMPMs; x++)
{
uint32_t mode = multiRefMPM[x];
{
pu.intraDir[0] = mode;
if (useDPCMForFirstPassIntraEstimation(pu, mode))
{
encPredIntraDPCM(COMPONENT_Y, piOrg, piPred, mode);
}
else
{
predIntraAng(COMPONENT_Y, piPred, pu, IntraPrediction::useFilteredIntraRefSamples(COMPONENT_Y, pu, true, pu));
}
// use Hadamard transform here
Distortion sad = distParam.distFunc(distParam);
// NB xFracModeBitsIntra will not affect the mode for chroma that may have already been pre-estimated.
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
m_CABACEstimator->getCtx() = SubCtx( Ctx::MHIntraPredMode, ctxStartMHIntraMode );
m_CABACEstimator->getCtx() = SubCtx( Ctx::MultiRefLineIdx, ctxStartMrlIdx );
uint64_t fracModeBits = xFracModeBitsIntra(pu, mode, CHANNEL_TYPE_LUMA);
double cost = (double)sad + (double)fracModeBits * sqrtLambdaForFirstPass;
updateCandList(mode, cost, uiRdModeList, CandCostList, extendRefList, multiRefIdx, numModesForFullRD);
}
}
}
CandCostList.resize(numModesForFullRD);
extendRefList.resize(numModesForFullRD);

Karsten Suehring
committed
if( m_pcEncCfg->getFastUDIUseMPMEnabled() )
{
const int numMPMs = NUM_MOST_PROBABLE_MODES;
unsigned uiPreds[numMPMs];

Karsten Suehring
committed

Karsten Suehring
committed
const int numCand = PU::getIntraMPMs( pu, uiPreds );
for( int j = 0; j < numCand; j++ )
{
bool mostProbableModeIncluded = false;
int mostProbableMode = uiPreds[j];
for( int i = 0; i < numModesForFullRD; i++ )
{
mostProbableModeIncluded |= (mostProbableMode == uiRdModeList[i] && extendRefList[i] == 0);

Karsten Suehring
committed
}
if( !mostProbableModeIncluded )
{

Karsten Suehring
committed
numModesForFullRD++;
uiRdModeList.push_back( mostProbableMode );
}
}
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
#if JVET_M0102_INTRA_SUBPARTITIONS
if( nOptionsForISP > 1 )
{
//we add the ISP MPMs to the list without mrl modes
m_rdModeListWithoutMrlHor = m_rdModeListWithoutMrl;
m_rdModeListWithoutMrlVer = m_rdModeListWithoutMrl;
static_vector<uint32_t, FAST_UDI_MAX_RDMODE_NUM>* listPointer;
for( int k = 1; k < nOptionsForISP; k++ )
{
cu.ispMode = ispOptions[k];
listPointer = &( cu.ispMode == HOR_INTRA_SUBPARTITIONS ? m_rdModeListWithoutMrlHor : m_rdModeListWithoutMrlVer );
const int numCandISP = PU::getIntraMPMs( pu, uiPreds );
for( int j = 0; j < numCandISP; j++ )
{
bool mostProbableModeIncluded = false;
int mostProbableMode = uiPreds[j];
for( int i = 0; i < listPointer->size(); i++ )
{
mostProbableModeIncluded |= ( mostProbableMode == listPointer->at( i ) );
}
if( !mostProbableModeIncluded )
{
listPointer->push_back( mostProbableMode );
}
}
}
cu.ispMode = NOT_INTRA_SUBPARTITIONS;
}
#endif

Karsten Suehring
committed
}
}
else
{
for( int i = 0; i < numModesForFullRD; i++ )
{
uiRdModeList.push_back( i );
}
}

Karsten Suehring
committed
if( emtUsageFlag == 1 )
{
// Store the modes to be checked with RD
m_savedNumRdModes[puIndex] = numModesForFullRD;
std::copy_n( uiRdModeList.begin(), numModesForFullRD, m_savedRdModeList[puIndex] );
std::copy_n(extendRefList.begin(), numModesForFullRD, m_savedExtendRefList[puIndex]);

Karsten Suehring
committed
}

Karsten Suehring
committed
}

Karsten Suehring
committed
else //emtUsage = 2 (here we potentially reduce the number of modes that will be full-RD checked)
{
if( isAllIntra && m_pcEncCfg->getFastIntraEMT() )
{
double thresholdSkipMode = 1.0 + 1.4 / sqrt( ( double ) ( width*height ) );

Karsten Suehring
committed
numModesForFullRD = 0;
// Skip checking the modes with much larger R-D cost than the best mode
for( int i = 0; i < m_savedNumRdModes[puIndex]; i++ )
{
if( m_modeCostStore[puIndex][i] <= thresholdSkipMode * m_bestModeCostStore[puIndex] )
{
uiRdModeList.push_back( m_savedRdModeList[puIndex][i] );
extendRefList.push_back(m_savedExtendRefList[puIndex][i]);

Karsten Suehring
committed
numModesForFullRD++;
}
}
}
else //this is necessary because we skip the candidates list calculation, since it was already obtained for the DCT-II. Now we load it
{
// Restore the modes to be checked with RD
numModesForFullRD = m_savedNumRdModes[puIndex];
uiRdModeList.resize( numModesForFullRD );
std::copy_n( m_savedRdModeList[puIndex], m_savedNumRdModes[puIndex], uiRdModeList.begin() );
CandCostList.resize(numModesForFullRD);
extendRefList.resize(numModesForFullRD);
std::copy_n(m_savedExtendRefList[puIndex], m_savedNumRdModes[puIndex], extendRefList.begin());

Karsten Suehring
committed
}
}

Karsten Suehring
committed
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
#if JVET_M0102_INTRA_SUBPARTITIONS
if( nOptionsForISP > 1 ) // we remove the non-MPMs from the ISP lists
{
static_vector< uint32_t, FAST_UDI_MAX_RDMODE_NUM > uiRdModeListCopyHor = m_rdModeListWithoutMrlHor;
m_rdModeListWithoutMrlHor.clear();
static_vector< uint32_t, FAST_UDI_MAX_RDMODE_NUM > uiRdModeListCopyVer = m_rdModeListWithoutMrlVer;
m_rdModeListWithoutMrlVer.clear();
static_vector< uint32_t, FAST_UDI_MAX_RDMODE_NUM > *listPointerCopy, *listPointer;
for( int ispOptionIdx = 1; ispOptionIdx < nOptionsForISP; ispOptionIdx++ )
{
cu.ispMode = ispOptions[ispOptionIdx];
//we get the mpm cand list
const int numMPMs = NUM_MOST_PROBABLE_MODES;
unsigned uiPreds[numMPMs];
pu.multiRefIdx = 0;
PU::getIntraMPMs( pu, uiPreds );
//we copy only the ISP MPMs
listPointerCopy = &( cu.ispMode == HOR_INTRA_SUBPARTITIONS ? uiRdModeListCopyHor : uiRdModeListCopyVer );
listPointer = &( cu.ispMode == HOR_INTRA_SUBPARTITIONS ? m_rdModeListWithoutMrlHor : m_rdModeListWithoutMrlVer );
for( int k = 0; k < listPointerCopy->size(); k++ )
{
for( int q = 0; q < numMPMs; q++ )
{
if( listPointerCopy->at( k ) == uiPreds[q] )
{
listPointer->push_back( listPointerCopy->at( k ) );
break;
}
}
}
}
cu.ispMode = NOT_INTRA_SUBPARTITIONS;
}
#endif

Karsten Suehring
committed
CHECK( numModesForFullRD != uiRdModeList.size(), "Inconsistent state!" );
// after this point, don't use numModesForFullRD
// PBINTRA fast
#if JVET_M0464_UNI_MTS
if( m_pcEncCfg->getUsePbIntraFast() && !cs.slice->isIntra() && uiRdModeList.size() < numModesAvailable )
#else
if( m_pcEncCfg->getUsePbIntraFast() && !cs.slice->isIntra() && uiRdModeList.size() < numModesAvailable && emtUsageFlag != 2 )

Karsten Suehring
committed
{
if( CandHadList.size() < 3 || CandHadList[2] > cs.interHad * PBINTRA_RATIO )
{
uiRdModeList.resize( std::min<size_t>( uiRdModeList.size(), 2 ) );
#if JVET_M0102_INTRA_SUBPARTITIONS
if( nOptionsForISP > 1 )
{
m_rdModeListWithoutMrlHor.resize( std::min<size_t>( m_rdModeListWithoutMrlHor.size(), 2 ) );
m_rdModeListWithoutMrlVer.resize( std::min<size_t>( m_rdModeListWithoutMrlVer.size(), 2 ) );
}
#endif

Karsten Suehring
committed
}
if( CandHadList.size() < 2 || CandHadList[1] > cs.interHad * PBINTRA_RATIO )
{
uiRdModeList.resize( std::min<size_t>( uiRdModeList.size(), 1 ) );
#if JVET_M0102_INTRA_SUBPARTITIONS
if( nOptionsForISP > 1 )
{
m_rdModeListWithoutMrlHor.resize( std::min<size_t>( m_rdModeListWithoutMrlHor.size(), 1 ) );
m_rdModeListWithoutMrlVer.resize( std::min<size_t>( m_rdModeListWithoutMrlVer.size(), 1 ) );
}
#endif

Karsten Suehring
committed
}
if( CandHadList.size() < 1 || CandHadList[0] > cs.interHad * PBINTRA_RATIO )
{
cs.dist = std::numeric_limits<Distortion>::max();
cs.interHad = 0;
//===== reset context models =====
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
m_CABACEstimator->getCtx() = SubCtx( Ctx::MHIntraPredMode, ctxStartMHIntraMode );
m_CABACEstimator->getCtx() = SubCtx( Ctx::MultiRefLineIdx, ctxStartMrlIdx );

Karsten Suehring
committed
return;
}
}
//===== check modes (using r-d costs) =====
uint32_t uiBestPUMode = 0;

Karsten Suehring
committed
CodingStructure *csTemp = m_pTempCS[gp_sizeIdxInfo->idxFrom( cu.lwidth() )][gp_sizeIdxInfo->idxFrom( cu.lheight() )];
CodingStructure *csBest = m_pBestCS[gp_sizeIdxInfo->idxFrom( cu.lwidth() )][gp_sizeIdxInfo->idxFrom( cu.lheight() )];
csTemp->slice = cs.slice;
csBest->slice = cs.slice;
csTemp->initStructData();
csBest->initStructData();
// just to be sure
numModesForFullRD = ( int ) uiRdModeList.size();
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
#if JVET_M0102_INTRA_SUBPARTITIONS
PartSplit intraSubPartitionsProcOrder = TU_NO_ISP;
int bestNormalIntraModeIndex = -1;
uint8_t bestIspOption = NOT_INTRA_SUBPARTITIONS;
TUIntraSubPartitioner subTuPartitioner( partitioner );
#if !JVET_M0464_UNI_MTS
if ( !cu.ispMode && !cu.emtFlag )
{
m_modeCtrl->setEmtFirstPassNoIspCost( MAX_DOUBLE );
}
#endif
for( uint32_t ispOptionIdx = 0; ispOptionIdx < nOptionsForISP; ispOptionIdx++ )
{
cu.ispMode = ispOptions[ispOptionIdx];
int numModesForFullRDispOption = cu.ispMode == NOT_INTRA_SUBPARTITIONS ? numModesForFullRD : cu.ispMode == HOR_INTRA_SUBPARTITIONS ? (int)m_rdModeListWithoutMrlHor.size() : (int)m_rdModeListWithoutMrlVer.size();
for( uint32_t uiMode = 0; uiMode < numModesForFullRDispOption; uiMode++ )
{
// set luma prediction mode
uint32_t uiOrgMode = cu.ispMode == NOT_INTRA_SUBPARTITIONS ? uiRdModeList[uiMode] : cu.ispMode == HOR_INTRA_SUBPARTITIONS ? m_rdModeListWithoutMrlHor[uiMode] : m_rdModeListWithoutMrlVer[uiMode];
pu.intraDir[0] = uiOrgMode;
int multiRefIdx = 0;
pu.multiRefIdx = multiRefIdx;
if( cu.ispMode )
{
intraSubPartitionsProcOrder = CU::getISPType( cu, COMPONENT_Y );
bool tuIsDividedInRows = CU::divideTuInRows( cu );
if( m_intraModeDiagRatio.at( bestNormalIntraModeIndex ) > 1.25 )
{
continue;
}
if( uiOrgMode <= DC_IDX )
{
if( ( m_intraModeHorVerRatio.at( bestNormalIntraModeIndex ) > 1.25 && tuIsDividedInRows ) || ( m_intraModeHorVerRatio.at( bestNormalIntraModeIndex ) < 0.8 && !tuIsDividedInRows ) )
{
continue;
}
}
else
{
if( ( m_intraModeHorVerRatio.at( bestNormalIntraModeIndex ) > 1.25 && tuIsDividedInRows ) || ( m_intraModeHorVerRatio.at( bestNormalIntraModeIndex ) < 0.8 && !tuIsDividedInRows ) )
{
continue;
}
}
}
else
{
multiRefIdx = extendRefList[uiMode];
pu.multiRefIdx = multiRefIdx;
CHECK( pu.multiRefIdx && ( pu.intraDir[0] == DC_IDX || pu.intraDir[0] == PLANAR_IDX ), "ERL" );
}
#else

Karsten Suehring
committed
for (uint32_t uiMode = 0; uiMode < numModesForFullRD; uiMode++)
{
// set luma prediction mode
uint32_t uiOrgMode = uiRdModeList[uiMode];
pu.intraDir[0] = uiOrgMode;
int multiRefIdx = extendRefList[uiMode];
pu.multiRefIdx = multiRefIdx;
CHECK(pu.multiRefIdx && (pu.intraDir[0] == DC_IDX || pu.intraDir[0] == PLANAR_IDX), "ERL");

Karsten Suehring
committed
// set context models
m_CABACEstimator->getCtx() = ctxStart;
// determine residual for partition
cs.initSubStructure( *csTemp, partitioner.chType, cs.area, true );
#if JVET_M0102_INTRA_SUBPARTITIONS
if( cu.ispMode )
{
xRecurIntraCodingLumaQT( *csTemp, subTuPartitioner, bestCurrentCost, 0, intraSubPartitionsProcOrder );
}
else
{
xRecurIntraCodingLumaQT( *csTemp, partitioner, MAX_DOUBLE, -1 );
}
if( cu.ispMode && !csTemp->cus[0]->firstTU->cbf[COMPONENT_Y] )
{
csTemp->cost = MAX_DOUBLE;
}
#else

Karsten Suehring
committed
xRecurIntraCodingLumaQT( *csTemp, partitioner );

Karsten Suehring
committed
#if JVET_M0102_INTRA_SUBPARTITIONS
if (emtUsageFlag == 1 && m_pcEncCfg->getFastIntraEMT() && !cu.ispMode)
#else

Karsten Suehring
committed
if( emtUsageFlag == 1 && m_pcEncCfg->getFastIntraEMT() )

Karsten Suehring
committed
{
m_modeCostStore[puIndex][uiMode] = csTemp->cost; //cs.cost;
}

Karsten Suehring
committed
DTRACE( g_trace_ctx, D_INTRA_COST, "IntraCost T %f (%d) \n", csTemp->cost, uiOrgMode );
// check r-d cost
if( csTemp->cost < csBest->cost )
{
std::swap( csTemp, csBest );
uiBestPUMode = uiOrgMode;
#if JVET_M0102_INTRA_SUBPARTITIONS
bestIspOption = cu.ispMode;
#endif
#if JVET_M0102_INTRA_SUBPARTITIONS
if (emtUsageFlag == 1 && m_pcEncCfg->getFastIntraEMT() && !cu.ispMode)
#else

Karsten Suehring
committed
if( ( emtUsageFlag == 1 ) && m_pcEncCfg->getFastIntraEMT() )

Karsten Suehring
committed
{
m_bestModeCostStore[puIndex] = csBest->cost; //cs.cost;
}
#endif
#if JVET_M0102_INTRA_SUBPARTITIONS
if( csBest->cost < bestCurrentCost )
{
bestCurrentCost = csBest->cost;
}
if( !cu.ispMode )
{
bestNormalIntraModeIndex = uiMode;
}

Karsten Suehring
committed
}
csTemp->releaseIntermediateData();
} // Mode loop
#if JVET_M0102_INTRA_SUBPARTITIONS
#if !JVET_M0464_UNI_MTS
if (!cu.ispMode && !cu.emtFlag)
{
m_modeCtrl->setEmtFirstPassNoIspCost(csBest->cost);
}
#endif
}
cu.ispMode = bestIspOption;
#endif

Karsten Suehring
committed
#if JVET_M0427_INLOOP_RESHAPER
cs.useSubStructure(*csBest, partitioner.chType, pu.singleChan(CHANNEL_TYPE_LUMA), true, true, keepResi, keepResi);
#else

Karsten Suehring
committed
cs.useSubStructure( *csBest, partitioner.chType, pu.singleChan( CHANNEL_TYPE_LUMA ), KEEP_PRED_AND_RESI_SIGNALS, true, keepResi, keepResi );

Karsten Suehring
committed
csBest->releaseIntermediateData();
//=== update PU data ====
pu.intraDir[0] = uiBestPUMode;

Karsten Suehring
committed
}
//===== reset context models =====
m_CABACEstimator->getCtx() = ctxStart;
}
#if JVET_M0102_INTRA_SUBPARTITIONS
void IntraSearch::estIntraPredChromaQT( CodingUnit &cu, Partitioner &partitioner, const double maxCostAllowed )
#else

Karsten Suehring
committed
void IntraSearch::estIntraPredChromaQT(CodingUnit &cu, Partitioner &partitioner)

Karsten Suehring
committed
{
const ChromaFormat format = cu.chromaFormat;
const uint32_t numberValidComponents = getNumberValidComponents(format);
CodingStructure &cs = *cu.cs;
const TempCtx ctxStart ( m_CtxCache, m_CABACEstimator->getCtx() );
cs.setDecomp( cs.area.Cb(), false );
#if JVET_M0102_INTRA_SUBPARTITIONS
double bestCostSoFar = maxCostAllowed;
bool lumaUsesISP = !CS::isDualITree( *cu.cs ) && cu.ispMode;
PartSplit ispType = lumaUsesISP ? CU::getISPType( cu, COMPONENT_Y ) : TU_NO_ISP;
CHECK( cu.ispMode && bestCostSoFar < 0, "bestCostSoFar must be positive!" );
#endif

Karsten Suehring
committed
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
auto &pu = *cu.firstPU;
{
uint32_t uiBestMode = 0;
Distortion uiBestDist = 0;
double dBestCost = MAX_DOUBLE;
//----- init mode list ----
{
uint32_t uiMinMode = 0;
uint32_t uiMaxMode = NUM_CHROMA_MODE;
//----- check chroma modes -----
uint32_t chromaCandModes[ NUM_CHROMA_MODE ];
PU::getIntraChromaCandModes( pu, chromaCandModes );
// create a temporary CS
CodingStructure &saveCS = *m_pSaveCS[0];
saveCS.pcv = cs.pcv;
saveCS.picture = cs.picture;
saveCS.area.repositionTo( cs.area );
saveCS.clearTUs();
#if JVET_M0102_INTRA_SUBPARTITIONS
if( !CS::isDualITree( cs ) && cu.ispMode )
{
saveCS.clearCUs();
saveCS.clearPUs();
}
#endif

Karsten Suehring
committed
if( CS::isDualITree( cs ) )
{
if( partitioner.canSplit( TU_MAX_TR_SPLIT, cs ) )
{
partitioner.splitCurrArea( TU_MAX_TR_SPLIT, cs );
do
{
cs.addTU( CS::getArea( cs, partitioner.currArea(), partitioner.chType ), partitioner.chType ).depth = partitioner.currTrDepth;
} while( partitioner.nextPart( cs ) );
partitioner.exitCurrSplit();
}
else
cs.addTU( CS::getArea( cs, partitioner.currArea(), partitioner.chType ), partitioner.chType );
}
std::vector<TransformUnit*> orgTUs;
#if JVET_M0102_INTRA_SUBPARTITIONS
if( lumaUsesISP )
{
CodingUnit& auxCU = saveCS.addCU( cu, partitioner.chType );
auxCU.ispMode = cu.ispMode;
saveCS.sps = cu.cs->sps;
saveCS.addPU( *cu.firstPU, partitioner.chType );
}
#endif

Karsten Suehring
committed
// create a store for the TUs
for( const auto &ptu : cs.tus )
{
// for split TUs in HEVC, add the TUs without Chroma parts for correct setting of Cbfs
#if JVET_M0102_INTRA_SUBPARTITIONS
if( lumaUsesISP || pu.contains( *ptu, CHANNEL_TYPE_CHROMA ) )
#else

Karsten Suehring
committed
{
saveCS.addTU( *ptu, partitioner.chType );
orgTUs.push_back( ptu );
}
}
#if JVET_M0102_INTRA_SUBPARTITIONS
if( lumaUsesISP )
{
saveCS.clearCUs();
}
#endif
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
// SATD pre-selecting.
int satdModeList[NUM_CHROMA_MODE];
int64_t satdSortedCost[NUM_CHROMA_MODE];
for (int i = 0; i < NUM_CHROMA_MODE; i++)
{
satdSortedCost[i] = 0; // for the mode not pre-select by SATD, do RDO by default, so set the initial value 0.
satdModeList[i] = 0;
}
bool modeIsEnable[NUM_INTRA_MODE + 1]; // use intra mode idx to check whether enable
for (int i = 0; i < NUM_INTRA_MODE + 1; i++)
{
modeIsEnable[i] = 1;
}
DistParam distParam;
const bool useHadamard = true;
pu.intraDir[1] = MDLM_L_IDX; // temporary assigned, just to indicate this is a MDLM mode. for luma down-sampling operation.
initIntraPatternChType(cu, pu.Cb());
initIntraPatternChType(cu, pu.Cr());
xGetLumaRecPixels(pu, pu.Cb());
for (int idx = uiMinMode; idx <= uiMaxMode - 1; idx++)
{
int mode = chromaCandModes[idx];
satdModeList[idx] = mode;
if (PU::isLMCMode(mode) && !PU::isLMCModeEnabled(pu, mode))
{
continue;
}
if ((mode == LM_CHROMA_IDX) || (mode == PLANAR_IDX) || (mode == DM_CHROMA_IDX)) // only pre-check regular modes and MDLM modes, not including DM ,Planar, and LM
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
{
continue;
}
pu.intraDir[1] = mode; // temporary assigned, for SATD checking.
int64_t sad = 0;
CodingStructure& cs = *(pu.cs);
CompArea areaCb = pu.Cb();
PelBuf orgCb = cs.getOrgBuf(areaCb);
PelBuf predCb = cs.getPredBuf(areaCb);
m_pcRdCost->setDistParam(distParam, orgCb, predCb, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, useHadamard);
distParam.applyWeight = false;
if (PU::isLMCMode(mode))
{
predIntraChromaLM(COMPONENT_Cb, predCb, pu, areaCb, mode);
}
else
{
predIntraAng(COMPONENT_Cb, predCb, pu, false);
}
sad += distParam.distFunc(distParam);

Karsten Suehring
committed
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
CompArea areaCr = pu.Cr();
PelBuf orgCr = cs.getOrgBuf(areaCr);
PelBuf predCr = cs.getPredBuf(areaCr);
m_pcRdCost->setDistParam(distParam, orgCr, predCr, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, useHadamard);
distParam.applyWeight = false;
if (PU::isLMCMode(mode))
{
predIntraChromaLM(COMPONENT_Cr, predCr, pu, areaCr, mode);
}
else
{
predIntraAng(COMPONENT_Cr, predCr, pu, false);
}
sad += distParam.distFunc(distParam);
satdSortedCost[idx] = sad;
}
// sort the mode based on the cost from small to large.
int tempIdx = 0;
int64_t tempCost = 0;
for (int i = uiMinMode; i <= uiMaxMode - 1; i++)
{
for (int j = i + 1; j <= uiMaxMode - 1; j++)
{
if (satdSortedCost[j] < satdSortedCost[i])
{
tempIdx = satdModeList[i];
satdModeList[i] = satdModeList[j];
satdModeList[j] = tempIdx;
tempCost = satdSortedCost[i];
satdSortedCost[i] = satdSortedCost[j];
satdSortedCost[j] = tempCost;
}
}
}
int reducedModeNumber = 2; // reduce the number of chroma modes
for (int i = 0; i < reducedModeNumber; i++)
{
modeIsEnable[satdModeList[uiMaxMode - 1 - i]] = 0; // disable the last reducedModeNumber modes
}

Karsten Suehring
committed
// save the dist
Distortion baseDist = cs.dist;
for (uint32_t uiMode = uiMinMode; uiMode < uiMaxMode; uiMode++)
{
const int chromaIntraMode = chromaCandModes[uiMode];
if( PU::isLMCMode( chromaIntraMode ) && ! PU::isLMCModeEnabled( pu, chromaIntraMode ) )
{
continue;
}
if (!modeIsEnable[chromaIntraMode] && PU::isLMCModeEnabled(pu, chromaIntraMode)) // when CCLM is disable, then MDLM is disable. not use satd checking
{
continue;
}

Karsten Suehring
committed
cs.setDecomp( pu.Cb(), false );
cs.dist = baseDist;
//----- restore context models -----
m_CABACEstimator->getCtx() = ctxStart;
//----- chroma coding -----
pu.intraDir[1] = chromaIntraMode;
#if JVET_M0102_INTRA_SUBPARTITIONS
xRecurIntraChromaCodingQT( cs, partitioner, bestCostSoFar, ispType );
if( lumaUsesISP && cs.dist == MAX_UINT )
{
continue;
}
#else

Karsten Suehring
committed
xRecurIntraChromaCodingQT( cs, partitioner );

Karsten Suehring
committed
if (cs.pps->getUseTransformSkip())
{
m_CABACEstimator->getCtx() = ctxStart;
}
#if JVET_M0102_INTRA_SUBPARTITIONS
uint64_t fracBits = xGetIntraFracBitsQT( cs, partitioner, false, true, -1, ispType );
#else

Karsten Suehring
committed
uint64_t fracBits = xGetIntraFracBitsQT( cs, partitioner, false, true );

Karsten Suehring
committed
Distortion uiDist = cs.dist;
double dCost = m_pcRdCost->calcRdCost( fracBits, uiDist - baseDist );
//----- compare -----
if( dCost < dBestCost )
{
#if JVET_M0102_INTRA_SUBPARTITIONS
if( lumaUsesISP && dCost < bestCostSoFar )
{
bestCostSoFar = dCost;
}
#endif

Karsten Suehring
committed
for( uint32_t i = getFirstComponentOfChannel( CHANNEL_TYPE_CHROMA ); i < numberValidComponents; i++ )
{
const CompArea &area = pu.blocks[i];
saveCS.getRecoBuf ( area ).copyFrom( cs.getRecoBuf ( area ) );
#if KEEP_PRED_AND_RESI_SIGNALS
saveCS.getPredBuf ( area ).copyFrom( cs.getPredBuf ( area ) );
saveCS.getResiBuf ( area ).copyFrom( cs.getResiBuf ( area ) );
#endif
#if JVET_M0427_INLOOP_RESHAPER
saveCS.getPredBuf ( area ).copyFrom( cs.getPredBuf (area ) );
cs.picture->getPredBuf( area ).copyFrom( cs.getPredBuf (area ) );

Karsten Suehring
committed
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
#endif
cs.picture->getRecoBuf( area ).copyFrom( cs.getRecoBuf( area ) );
for( uint32_t j = 0; j < saveCS.tus.size(); j++ )
{
saveCS.tus[j]->copyComponentFrom( *orgTUs[j], area.compID );
}
}
dBestCost = dCost;
uiBestDist = uiDist;
uiBestMode = chromaIntraMode;
}
}
for( uint32_t i = getFirstComponentOfChannel( CHANNEL_TYPE_CHROMA ); i < numberValidComponents; i++ )
{
const CompArea &area = pu.blocks[i];
cs.getRecoBuf ( area ).copyFrom( saveCS.getRecoBuf( area ) );
#if KEEP_PRED_AND_RESI_SIGNALS
cs.getPredBuf ( area ).copyFrom( saveCS.getPredBuf( area ) );
cs.getResiBuf ( area ).copyFrom( saveCS.getResiBuf( area ) );
#endif
#if JVET_M0427_INLOOP_RESHAPER
cs.getPredBuf ( area ).copyFrom( saveCS.getPredBuf( area ) );
cs.picture->getPredBuf( area ).copyFrom( cs.getPredBuf ( area ) );
#endif

Karsten Suehring
committed
cs.picture->getRecoBuf( area ).copyFrom( cs. getRecoBuf( area ) );
for( uint32_t j = 0; j < saveCS.tus.size(); j++ )
{
orgTUs[ j ]->copyComponentFrom( *saveCS.tus[ j ], area.compID );
}
}
}
pu.intraDir[1] = uiBestMode;
cs.dist = uiBestDist;
}
//----- restore context models -----
m_CABACEstimator->getCtx() = ctxStart;
#if JVET_M0102_INTRA_SUBPARTITIONS
if( lumaUsesISP && bestCostSoFar >= maxCostAllowed )
{
cu.ispMode = 0;
}
#endif

Karsten Suehring
committed
}
void IntraSearch::IPCMSearch(CodingStructure &cs, Partitioner& partitioner)
{
ComponentID compStr = (CS::isDualITree(cs) && !isLuma(partitioner.chType)) ? COMPONENT_Cb: COMPONENT_Y;
ComponentID compEnd = (CS::isDualITree(cs) && isLuma(partitioner.chType)) ? COMPONENT_Y : COMPONENT_Cr;
for( ComponentID compID = compStr; compID <= compEnd; compID = ComponentID(compID+1) )

Karsten Suehring
committed
{
xEncPCM(cs, partitioner, compID);
}
cs.getPredBuf().fill(0);
cs.getResiBuf().fill(0);
cs.getOrgResiBuf().fill(0);
cs.dist = 0;
cs.fracBits = 0;
cs.cost = 0;
cs.setDecomp(cs.area);
#if JVET_M0427_INLOOP_RESHAPER
cs.picture->getPredBuf(cs.area).copyFrom(cs.getPredBuf());

Karsten Suehring
committed
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
}
void IntraSearch::xEncPCM(CodingStructure &cs, Partitioner& partitioner, const ComponentID &compID)
{
TransformUnit &tu = *cs.getTU( partitioner.chType );
const int channelBitDepth = cs.sps->getBitDepth(toChannelType(compID));
const uint32_t uiPCMBitDepth = cs.sps->getPCMBitDepth(toChannelType(compID));
const int pcmShiftRight = (channelBitDepth - int(uiPCMBitDepth));
CompArea area = tu.blocks[compID];
PelBuf pcmBuf = tu.getPcmbuf (compID);
PelBuf recBuf = cs.getRecoBuf ( area );
CPelBuf orgBuf = cs.getOrgBuf ( area );
CHECK(pcmShiftRight < 0, "Negative shift");
for (uint32_t uiY = 0; uiY < pcmBuf.height; uiY++)
{
for (uint32_t uiX = 0; uiX < pcmBuf.width; uiX++)
{
// Encode
pcmBuf.at(uiX, uiY) = orgBuf.at(uiX, uiY) >> pcmShiftRight;
// Reconstruction
recBuf.at(uiX, uiY) = pcmBuf.at(uiX, uiY) << pcmShiftRight;
}
}
}
// -------------------------------------------------------------------------------------------------------------------
// Intra search
// -------------------------------------------------------------------------------------------------------------------
#if JVET_M0102_INTRA_SUBPARTITIONS
void IntraSearch::xEncIntraHeader( CodingStructure &cs, Partitioner &partitioner, const bool &bLuma, const bool &bChroma, const int subTuIdx )
#else

Karsten Suehring
committed
void IntraSearch::xEncIntraHeader(CodingStructure &cs, Partitioner &partitioner, const bool &bLuma, const bool &bChroma)

Karsten Suehring
committed
{
CodingUnit &cu = *cs.getCU( partitioner.chType );
if (bLuma)
{
#if JVET_M0102_INTRA_SUBPARTITIONS
bool isFirst = cu.ispMode ? subTuIdx == 0 : partitioner.currArea().lumaPos() == cs.area.lumaPos();
#else

Karsten Suehring
committed
bool isFirst = partitioner.currArea().lumaPos() == cs.area.lumaPos();

Karsten Suehring
committed
// CU header
if( isFirst )
{
if ((!cs.slice->isIntra() || cs.slice->getSPS()->getIBCFlag())

Karsten Suehring
committed
{
if( cs.pps->getTransquantBypassEnabledFlag() )
{
m_CABACEstimator->cu_transquant_bypass_flag( cu );
}
m_CABACEstimator->cu_skip_flag( cu );
m_CABACEstimator->pred_mode ( cu );
}
m_CABACEstimator->extend_ref_line(cu);
#if JVET_M0102_INTRA_SUBPARTITIONS
m_CABACEstimator->isp_mode ( cu );
#endif

Karsten Suehring
committed
{
m_CABACEstimator->pcm_data( cu, partitioner );

Karsten Suehring
committed
if( cu.ipcm )
{
return;
}
}
}
PredictionUnit &pu = *cs.getPU(partitioner.currArea().lumaPos(), partitioner.chType);
// luma prediction mode

Karsten Suehring
committed
{
if ( !cu.Y().valid())
m_CABACEstimator->pred_mode( cu );
m_CABACEstimator->intra_luma_pred_mode( pu );

Karsten Suehring
committed
}
}
if (bChroma)
{
bool isFirst = partitioner.currArea().Cb().valid() && partitioner.currArea().chromaPos() == cs.area.chromaPos();
PredictionUnit &pu = *cs.getPU( partitioner.currArea().chromaPos(), CHANNEL_TYPE_CHROMA );

Karsten Suehring
committed
{

Karsten Suehring
committed
}
}
}
#if JVET_M0102_INTRA_SUBPARTITIONS
void IntraSearch::xEncSubdivCbfQT( CodingStructure &cs, Partitioner &partitioner, const bool &bLuma, const bool &bChroma, const int subTuIdx, const PartSplit ispType )
{
const UnitArea &currArea = partitioner.currArea();
int subTuCounter = subTuIdx;
TransformUnit &currTU = *cs.getTU( currArea.blocks[partitioner.chType], partitioner.chType, subTuCounter );
CodingUnit &currCU = *currTU.cu;
#else

Karsten Suehring
committed
void IntraSearch::xEncSubdivCbfQT(CodingStructure &cs, Partitioner &partitioner, const bool &bLuma, const bool &bChroma)
{
const UnitArea &currArea = partitioner.currArea();
TransformUnit &currTU = *cs.getTU( currArea.blocks[partitioner.chType], partitioner.chType );

Karsten Suehring
committed
CodingUnit &currCU = *currTU.cu;

Karsten Suehring
committed
uint32_t currDepth = partitioner.currTrDepth;
const bool subdiv = currTU.depth > currDepth;
#if JVET_M0102_INTRA_SUBPARTITIONS
ComponentID compID = partitioner.chType == CHANNEL_TYPE_LUMA ? COMPONENT_Y : COMPONENT_Cb;
const bool chromaCbfISP = currArea.blocks[COMPONENT_Cb].valid() && currCU.ispMode && !subdiv;
#endif

Karsten Suehring
committed

Karsten Suehring
committed
{
CHECK( !subdiv, "TU split implied" );
}
else
{
#if JVET_M0102_INTRA_SUBPARTITIONS
CHECK( subdiv && !currCU.ispMode && isLuma( compID ), "No TU subdivision is allowed with QTBT" );
}
if( bChroma && ( !currCU.ispMode || chromaCbfISP ) )
#else
CHECK( subdiv, "No TU subdivision is allowed with QTBT" );

Karsten Suehring
committed
}
if (bChroma)

Karsten Suehring
committed
{
const uint32_t numberValidComponents = getNumberValidComponents(currArea.chromaFormat);
#if JVET_M0102_INTRA_SUBPARTITIONS
const uint32_t cbfDepth = ( chromaCbfISP ? currDepth - 1 : currDepth );
#endif

Karsten Suehring
committed
for (uint32_t ch = COMPONENT_Cb; ch < numberValidComponents; ch++)
{
const ComponentID compID = ComponentID(ch);
#if JVET_M0102_INTRA_SUBPARTITIONS
if( currDepth == 0 || TU::getCbfAtDepth( currTU, compID, currDepth - 1 ) || chromaCbfISP )
#else

Karsten Suehring
committed
if( currDepth == 0 || TU::getCbfAtDepth( currTU, compID, currDepth - 1 ) )

Karsten Suehring
committed
{
const bool prevCbf = ( compID == COMPONENT_Cr ? TU::getCbfAtDepth( currTU, COMPONENT_Cb, currDepth ) : false );
#if JVET_M0102_INTRA_SUBPARTITIONS
m_CABACEstimator->cbf_comp( cs, TU::getCbfAtDepth( currTU, compID, currDepth ), currArea.blocks[compID], cbfDepth, prevCbf );
#else

Karsten Suehring
committed
m_CABACEstimator->cbf_comp( cs, TU::getCbfAtDepth( currTU, compID, currDepth ), currArea.blocks[compID], currDepth, prevCbf );

Karsten Suehring
committed
}
}
}
if (subdiv)
{
#if JVET_M0102_INTRA_SUBPARTITIONS
if (!currCU.ispMode && isLuma( compID ) && currDepth == 0 && bLuma) m_CABACEstimator->emt_cu_flag( currCU );
#else

Karsten Suehring
committed
if( currDepth == 0 && bLuma ) m_CABACEstimator->emt_cu_flag( currCU );

Karsten Suehring
committed
if( partitioner.canSplit( TU_MAX_TR_SPLIT, cs ) )
{
partitioner.splitCurrArea( TU_MAX_TR_SPLIT, cs );
}
#if JVET_M0102_INTRA_SUBPARTITIONS
else if( currCU.ispMode && isLuma( compID ) )
{
partitioner.splitCurrArea( ispType, cs );
}
#endif

Karsten Suehring
committed
else
THROW( "Cannot perform an implicit split!" );
do
{
#if JVET_M0102_INTRA_SUBPARTITIONS
xEncSubdivCbfQT( cs, partitioner, bLuma, bChroma, subTuCounter, ispType );
subTuCounter += subTuCounter != -1 ? 1 : 0;
#else

Karsten Suehring
committed
xEncSubdivCbfQT( cs, partitioner, bLuma, bChroma );

Karsten Suehring
committed
} while( partitioner.nextPart( cs ) );
partitioner.exitCurrSplit();
}
else
{
#if JVET_M0102_INTRA_SUBPARTITIONS
if (!currCU.ispMode && isLuma( compID ) && currDepth == 0 && bLuma && TU::getCbfAtDepth( currTU, COMPONENT_Y, 0) ) m_CABACEstimator->emt_cu_flag( currCU );
#else

Karsten Suehring
committed
if( currDepth == 0 && bLuma && TU::getCbfAtDepth( currTU, COMPONENT_Y, 0 ) ) m_CABACEstimator->emt_cu_flag( currCU );

Karsten Suehring
committed
//===== Cbfs =====
if (bLuma)
{
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
#if JVET_M0102_INTRA_SUBPARTITIONS
bool previousCbf = false;
bool lastCbfIsInferred = false;
if( ispType != TU_NO_ISP )
{
bool rootCbfSoFar = false;
uint32_t nTus = currCU.ispMode == HOR_INTRA_SUBPARTITIONS ? currCU.lheight() >> g_aucLog2[currTU.lheight()] : currCU.lwidth() >> g_aucLog2[currTU.lwidth()];
if( subTuCounter == nTus - 1 )
{
TransformUnit* tuPointer = currCU.firstTU;
for( int tuIdx = 0; tuIdx < nTus - 1; tuIdx++ )
{
rootCbfSoFar |= TU::getCbfAtDepth( *tuPointer, COMPONENT_Y, currDepth );
tuPointer = tuPointer->next;
}
if( !rootCbfSoFar )
{
lastCbfIsInferred = true;
}
}
if( !lastCbfIsInferred )
{
previousCbf = TU::getPrevTuCbfAtDepth( currTU, COMPONENT_Y, partitioner.currTrDepth );
}
}
if( !lastCbfIsInferred )
{
m_CABACEstimator->cbf_comp( cs, TU::getCbfAtDepth( currTU, COMPONENT_Y, currDepth ), currTU.Y(), currTU.depth, previousCbf, currCU.ispMode );
}
#else

Karsten Suehring
committed
m_CABACEstimator->cbf_comp( cs, TU::getCbfAtDepth( currTU, COMPONENT_Y, currDepth ), currTU.Y(), currTU.depth );

Karsten Suehring
committed
}
}
}
#if JVET_M0102_INTRA_SUBPARTITIONS
void IntraSearch::xEncCoeffQT( CodingStructure &cs, Partitioner &partitioner, const ComponentID compID, const int subTuIdx, const PartSplit ispType )
#else

Karsten Suehring
committed
void IntraSearch::xEncCoeffQT(CodingStructure &cs, Partitioner &partitioner, const ComponentID &compID)

Karsten Suehring
committed
{
const UnitArea &currArea = partitioner.currArea();
#if JVET_M0102_INTRA_SUBPARTITIONS
int subTuCounter = subTuIdx;
TransformUnit &currTU = *cs.getTU( currArea.blocks[partitioner.chType], partitioner.chType, subTuIdx );
#else

Karsten Suehring
committed
TransformUnit &currTU = *cs.getTU( currArea.blocks[partitioner.chType], partitioner.chType );

Karsten Suehring
committed
uint32_t currDepth = partitioner.currTrDepth;
const bool subdiv = currTU.depth > currDepth;
if (subdiv)
{
if (partitioner.canSplit(TU_MAX_TR_SPLIT, cs))
{
partitioner.splitCurrArea(TU_MAX_TR_SPLIT, cs);
}
#if JVET_M0102_INTRA_SUBPARTITIONS
else if( currTU.cu->ispMode )
{
partitioner.splitCurrArea( ispType, cs );
}
#endif

Karsten Suehring
committed
else
THROW("Implicit TU split not available!");
do
{
#if JVET_M0102_INTRA_SUBPARTITIONS
xEncCoeffQT( cs, partitioner, compID, subTuCounter, ispType );
subTuCounter += subTuCounter != -1 ? 1 : 0;
#else

Karsten Suehring
committed
xEncCoeffQT( cs, partitioner, compID );

Karsten Suehring
committed
} while( partitioner.nextPart( cs ) );
partitioner.exitCurrSplit();
}
else
if( currArea.blocks[compID].valid() )
{
if( TU::hasCrossCompPredInfo( currTU, compID ) )
{
m_CABACEstimator->cross_comp_pred( currTU, compID );
}
if( TU::getCbf( currTU, compID ) )
{
m_CABACEstimator->residual_coding( currTU, compID );
}
}
}
#if JVET_M0102_INTRA_SUBPARTITIONS
uint64_t IntraSearch::xGetIntraFracBitsQT( CodingStructure &cs, Partitioner &partitioner, const bool &bLuma, const bool &bChroma, const int subTuIdx, const PartSplit ispType )
#else

Karsten Suehring
committed
uint64_t IntraSearch::xGetIntraFracBitsQT( CodingStructure &cs, Partitioner &partitioner, const bool &bLuma, const bool &bChroma )

Karsten Suehring
committed
{
m_CABACEstimator->resetBits();
#if JVET_M0102_INTRA_SUBPARTITIONS
xEncIntraHeader( cs, partitioner, bLuma, bChroma, subTuIdx );
xEncSubdivCbfQT( cs, partitioner, bLuma, bChroma, subTuIdx, ispType );
#else

Karsten Suehring
committed
xEncIntraHeader( cs, partitioner, bLuma, bChroma );
xEncSubdivCbfQT( cs, partitioner, bLuma, bChroma );
#endif

Karsten Suehring
committed
if( bLuma )
{
#if JVET_M0102_INTRA_SUBPARTITIONS
xEncCoeffQT( cs, partitioner, COMPONENT_Y, subTuIdx, ispType );
#else

Karsten Suehring
committed
xEncCoeffQT( cs, partitioner, COMPONENT_Y );

Karsten Suehring
committed
}
if( bChroma )
{
#if JVET_M0102_INTRA_SUBPARTITIONS
xEncCoeffQT( cs, partitioner, COMPONENT_Cb, subTuIdx, ispType );
xEncCoeffQT( cs, partitioner, COMPONENT_Cr, subTuIdx, ispType );
#else

Karsten Suehring
committed
xEncCoeffQT( cs, partitioner, COMPONENT_Cb );
xEncCoeffQT( cs, partitioner, COMPONENT_Cr );
#endif
}
uint64_t fracBits = m_CABACEstimator->getEstFracBits();
return fracBits;
}
#if JVET_M0102_INTRA_SUBPARTITIONS
uint64_t IntraSearch::xGetIntraFracBitsQTSingleChromaComponent( CodingStructure &cs, Partitioner &partitioner, const ComponentID compID )
{
m_CABACEstimator->resetBits();
if( compID == COMPONENT_Cb )
{
PredictionUnit &pu = *cs.getPU( partitioner.currArea().lumaPos(), partitioner.chType );
m_CABACEstimator->intra_chroma_pred_mode( pu );
//xEncIntraHeader(cs, partitioner, false, true);
}
CHECK( partitioner.currTrDepth != 1, "error in the depth!" );
const UnitArea &currArea = partitioner.currArea();
TransformUnit &currTU = *cs.getTU( currArea.blocks[partitioner.chType], partitioner.chType );
//cbf coding
m_CABACEstimator->cbf_comp( cs, TU::getCbfAtDepth( currTU, compID, partitioner.currTrDepth ), currArea.blocks[compID], partitioner.currTrDepth - 1 );
//coeffs coding and cross comp coding
if( TU::hasCrossCompPredInfo( currTU, compID ) )
{
m_CABACEstimator->cross_comp_pred( currTU, compID );
}
if( TU::getCbf( currTU, compID ) )
{
m_CABACEstimator->residual_coding( currTU, compID );

Karsten Suehring
committed
}
uint64_t fracBits = m_CABACEstimator->getEstFracBits();
return fracBits;
}

Karsten Suehring
committed
uint64_t IntraSearch::xGetIntraFracBitsQTChroma(TransformUnit& currTU, const ComponentID &compID)
{
m_CABACEstimator->resetBits();
if( TU::hasCrossCompPredInfo( currTU, compID ) )
{
m_CABACEstimator->cross_comp_pred( currTU, compID );
}
if( TU::getCbf( currTU, compID ) )
{
m_CABACEstimator->residual_coding( currTU, compID );
}
uint64_t fracBits = m_CABACEstimator->getEstFracBits();
return fracBits;
}
#if JVET_M0464_UNI_MTS
void IntraSearch::xIntraCodingTUBlock(TransformUnit &tu, const ComponentID &compID, const bool &checkCrossCPrediction, Distortion& ruiDist, const int &default0Save1Load2, uint32_t* numSig, std::vector<TrMode>* trModes, const bool loadTr)
#else

Karsten Suehring
committed
void IntraSearch::xIntraCodingTUBlock(TransformUnit &tu, const ComponentID &compID, const bool &checkCrossCPrediction, Distortion& ruiDist, const int &default0Save1Load2, uint32_t* numSig )

Karsten Suehring
committed
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
{
if (!tu.blocks[compID].valid())
{
return;
}
CodingStructure &cs = *tu.cs;
const CompArea &area = tu.blocks[compID];
const SPS &sps = *cs.sps;
const PPS &pps = *cs.pps;
const ChannelType chType = toChannelType(compID);
const int bitDepth = sps.getBitDepth(chType);
PelBuf piOrg = cs.getOrgBuf (area);
PelBuf piPred = cs.getPredBuf (area);
PelBuf piResi = cs.getResiBuf (area);
PelBuf piOrgResi = cs.getOrgResiBuf(area);
PelBuf piReco = cs.getRecoBuf (area);
const PredictionUnit &pu = *cs.getPU(area.pos(), chType);
const uint32_t uiChFinalMode = PU::getFinalIntraMode(pu, chType);
const bool bUseCrossCPrediction = pps.getPpsRangeExtension().getCrossComponentPredictionEnabledFlag() && isChroma( compID ) && PU::isChromaIntraModeCrossCheckMode( pu ) && checkCrossCPrediction;
const bool ccUseRecoResi = m_pcEncCfg->getUseReconBasedCrossCPredictionEstimate();
#if JVET_M0102_INTRA_SUBPARTITIONS
const bool ispSplitIsAllowed = CU::canUseISPSplit( *tu.cu, compID );
#endif

Karsten Suehring
committed
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
//===== init availability pattern =====
PelBuf sharedPredTS( m_pSharedPredTransformSkip[compID], area );
if( default0Save1Load2 != 2 )
{
const bool bUseFilteredPredictions = IntraPrediction::useFilteredIntraRefSamples( compID, pu, true, tu );
initIntraPatternChType( *tu.cu, area, bUseFilteredPredictions );
//===== get prediction signal =====
if( compID != COMPONENT_Y && PU::isLMCMode( uiChFinalMode ) )
{
{
xGetLumaRecPixels( pu, area );
}
predIntraChromaLM( compID, piPred, pu, area, uiChFinalMode );
}
else
{
predIntraAng( compID, piPred, pu, bUseFilteredPredictions );
}
// save prediction
if( default0Save1Load2 == 1 )
{
sharedPredTS.copyFrom( piPred );
}
}
else
{
// load prediction
piPred.copyFrom( sharedPredTS );
}
DTRACE( g_trace_ctx, D_PRED, "@(%4d,%4d) [%2dx%2d] IMode=%d\n", tu.lx(), tu.ly(), tu.lwidth(), tu.lheight(), uiChFinalMode );
//DTRACE_PEL_BUF( D_PRED, piPred, tu, tu.cu->predMode, COMPONENT_Y );
#if JVET_M0427_INLOOP_RESHAPER
const Slice &slice = *cs.slice;
bool flag = slice.getReshapeInfo().getUseSliceReshaper() && (slice.isIntra() || (!slice.isIntra() && m_pcReshape->getCTUFlag()));
bool flag = slice.getReshapeInfo().getUseSliceReshaper() && (slice.isIntra() || (!slice.isIntra() && m_pcReshape->getCTUFlag()) || (slice.getSliceType() == P_SLICE && slice.getSPS()->getIBCMode()));
if (flag && slice.getReshapeInfo().getSliceReshapeChromaAdj() && isChroma(compID))
{
const Area area = tu.Y().valid() ? tu.Y() : Area(recalcPosition(tu.chromaFormat, tu.chType, CHANNEL_TYPE_LUMA, tu.blocks[tu.chType].pos()), recalcSize(tu.chromaFormat, tu.chType, CHANNEL_TYPE_LUMA, tu.blocks[tu.chType].size()));
const CompArea &areaY = CompArea(COMPONENT_Y, tu.chromaFormat, area );
PelBuf piPredY;
piPredY = cs.picture->getPredBuf(areaY);
const Pel avgLuma = piPredY.computeAvg();
int adj = m_pcReshape->calculateChromaAdj(avgLuma);
tu.setChromaAdj(adj);
}
#endif

Karsten Suehring
committed
//===== get residual signal =====
piResi.copyFrom( piOrg );
#if JVET_M0427_INLOOP_RESHAPER
if (slice.getReshapeInfo().getUseSliceReshaper() && m_pcReshape->getCTUFlag() && compID==COMPONENT_Y)
{
CompArea tmpArea(COMPONENT_Y, area.chromaFormat, Position(0, 0), area.size());
PelBuf tmpPred = m_tmpStorageLCU.getBuf(tmpArea);
tmpPred.copyFrom(piPred);
piResi.rspSignal(m_pcReshape->getFwdLUT());
piResi.subtract(tmpPred);
}
else
#endif

Karsten Suehring
committed
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
piResi.subtract( piPred );
if (pps.getPpsRangeExtension().getCrossComponentPredictionEnabledFlag() && isLuma(compID))
{
piOrgResi.copyFrom (piResi);
}
if (bUseCrossCPrediction)
{
if (xCalcCrossComponentPredictionAlpha(tu, compID, ccUseRecoResi) == 0)
{
return;
}
CrossComponentPrediction::crossComponentPrediction(tu, compID, cs.getResiBuf(tu.Y()), piResi, piResi, false);
}
//===== transform and quantization =====
//--- init rate estimation arrays for RDOQ ---
//--- transform and quantization ---
TCoeff uiAbsSum = 0;
const QpParam cQP(tu, compID);
#if RDOQ_CHROMA_LAMBDA
m_pcTrQuant->selectLambda(compID);
#endif
flag =flag && (tu.blocks[compID].width*tu.blocks[compID].height > 4);
if (flag && isChroma(compID) && slice.getReshapeInfo().getSliceReshapeChromaAdj() )
int cResScaleInv = tu.getChromaAdj();
double cResScale = round((double)(1 << CSCALE_FP_PREC) / (double)cResScaleInv);
m_pcTrQuant->setLambda(m_pcTrQuant->getLambda() / (cResScale*cResScale));
piResi.scaleSignal(cResScaleInv, 1, tu.cu->cs->slice->clpRng(compID));
#if JVET_M0102_INTRA_SUBPARTITIONS
double diagRatio = 0, horVerRatio = 0;
#endif
#if JVET_M0102_INTRA_SUBPARTITIONS
m_pcTrQuant->transformNxN( tu, compID, cQP, trModes, CU::isIntra( *tu.cu ) ? m_pcEncCfg->getIntraMTSMaxCand() : m_pcEncCfg->getInterMTSMaxCand(), ispSplitIsAllowed ? &diagRatio : nullptr, ispSplitIsAllowed ? &horVerRatio : nullptr );
#else
m_pcTrQuant->transformNxN( tu, compID, cQP, trModes, CU::isIntra( *tu.cu ) ? m_pcEncCfg->getIntraMTSMaxCand() : m_pcEncCfg->getInterMTSMaxCand() );
#if JVET_M0102_INTRA_SUBPARTITIONS
m_pcTrQuant->transformNxN( tu, compID, cQP, uiAbsSum, m_CABACEstimator->getCtx(), loadTr, &diagRatio, &horVerRatio );
#else
m_pcTrQuant->transformNxN(tu, compID, cQP, uiAbsSum, m_CABACEstimator->getCtx(), loadTr);
#endif
#else
#if JVET_M0102_INTRA_SUBPARTITIONS
m_pcTrQuant->transformNxN(tu, compID, cQP, uiAbsSum, m_CABACEstimator->getCtx(), ispSplitIsAllowed ? &diagRatio : nullptr, ispSplitIsAllowed ? &horVerRatio : nullptr);

Karsten Suehring
committed
m_pcTrQuant->transformNxN(tu, compID, cQP, uiAbsSum, m_CABACEstimator->getCtx());
#endif
#if JVET_M0102_INTRA_SUBPARTITIONS
if (!tu.cu->ispMode && isLuma(compID) && ispSplitIsAllowed &&
#if JVET_M0464_UNI_MTS
tu.mtsIdx == 0
#else
!tu.cu->emtFlag
#endif
)
{
m_intraModeDiagRatio .push_back(diagRatio);
m_intraModeHorVerRatio .push_back(horVerRatio);
m_intraModeTestedNormalIntra.push_back((int)uiChFinalMode);
}
#endif

Karsten Suehring
committed
DTRACE( g_trace_ctx, D_TU_ABS_SUM, "%d: comp=%d, abssum=%d\n", DTRACE_GET_COUNTER( g_trace_ctx, D_TU_ABS_SUM ), compID, uiAbsSum );
//--- inverse transform ---
if (uiAbsSum > 0)
{
m_pcTrQuant->invTransformNxN(tu, compID, piResi, cQP);
}
else
{
piResi.fill(0);
}
//===== reconstruction =====
if (flag && uiAbsSum > 0 && isChroma(compID) && slice.getReshapeInfo().getSliceReshapeChromaAdj() )
piResi.scaleSignal(tu.getChromaAdj(), 0, tu.cu->cs->slice->clpRng(compID));

Karsten Suehring
committed
if (bUseCrossCPrediction)
{
CrossComponentPrediction::crossComponentPrediction(tu, compID, cs.getResiBuf(tu.Y()), piResi, piResi, true);
}
#if JVET_M0427_INLOOP_RESHAPER
if (slice.getReshapeInfo().getUseSliceReshaper() && m_pcReshape->getCTUFlag() && compID == COMPONENT_Y)
{
CompArea tmpArea(COMPONENT_Y, area.chromaFormat, Position(0,0), area.size());
PelBuf tmpPred = m_tmpStorageLCU.getBuf(tmpArea);
tmpPred.copyFrom(piPred);
piReco.reconstruct(tmpPred, piResi, cs.slice->clpRng(compID));
}

Karsten Suehring
committed
piReco.reconstruct(piPred, piResi, cs.slice->clpRng( compID ));
//===== update distortion =====
#if WCG_EXT
#if JVET_M0427_INLOOP_RESHAPER
if (m_pcEncCfg->getLumaLevelToDeltaQPMapping().isEnabled() || (m_pcEncCfg->getReshaper()
&& slice.getReshapeInfo().getUseSliceReshaper() && (m_pcReshape->getCTUFlag() || (isChroma(compID) && m_pcEncCfg->getReshapeIntraCMD()))))
#else

Karsten Suehring
committed
if( m_pcEncCfg->getLumaLevelToDeltaQPMapping().isEnabled() )

Karsten Suehring
committed
{
const CPelBuf orgLuma = cs.getOrgBuf( cs.area.blocks[COMPONENT_Y] );
#if JVET_M0427_INLOOP_RESHAPER
if (compID == COMPONENT_Y)
{
CompArea tmpArea1(COMPONENT_Y, area.chromaFormat, Position(0, 0), area.size());
PelBuf tmpRecLuma = m_tmpStorageLCU.getBuf(tmpArea1);
tmpRecLuma.copyFrom(piReco);
tmpRecLuma.rspSignal(m_pcReshape->getInvLUT());
ruiDist += m_pcRdCost->getDistPart(piOrg, tmpRecLuma, sps.getBitDepth(toChannelType(compID)), compID, DF_SSE_WTD, &orgLuma);
}
else
#endif
ruiDist += m_pcRdCost->getDistPart(piOrg, piReco, bitDepth, compID, DF_SSE_WTD, &orgLuma);

Karsten Suehring
committed
}
else
#endif
{
ruiDist += m_pcRdCost->getDistPart( piOrg, piReco, bitDepth, compID, DF_SSE );
}
}
#if JVET_M0102_INTRA_SUBPARTITIONS
void IntraSearch::xRecurIntraCodingLumaQT( CodingStructure &cs, Partitioner &partitioner, const double bestCostSoFar, const int subTuIdx, const PartSplit ispType )
{
int subTuCounter = subTuIdx;
const UnitArea &currArea = partitioner.currArea();
const CodingUnit &cu = *cs.getCU( currArea.lumaPos(), partitioner.chType );
bool earlySkipISP = false;
#else

Karsten Suehring
committed
void IntraSearch::xRecurIntraCodingLumaQT( CodingStructure &cs, Partitioner &partitioner )
{
const UnitArea &currArea = partitioner.currArea();

Karsten Suehring
committed
const CodingUnit &cu = *cs.getCU(currArea.lumaPos(), partitioner.chType);

Karsten Suehring
committed
const PPS &pps = *cs.pps;
const bool keepResi = pps.getPpsRangeExtension().getCrossComponentPredictionEnabledFlag() || KEEP_PRED_AND_RESI_SIGNALS;
bool bCheckFull = true;
bool bCheckSplit = false;
bCheckFull = !partitioner.canSplit( TU_MAX_TR_SPLIT, cs );
bCheckSplit = partitioner.canSplit( TU_MAX_TR_SPLIT, cs );

Karsten Suehring
committed
#if JVET_M0102_INTRA_SUBPARTITIONS
if( cu.ispMode )
{
bCheckSplit = partitioner.canSplit( ispType, cs );
bCheckFull = !bCheckSplit;
}
#endif

Karsten Suehring
committed
uint32_t numSig = 0;
#if JVET_M0464_UNI_MTS
double dSingleCost = MAX_DOUBLE;
Distortion uiSingleDistLuma = 0;
uint64_t singleFracBits = 0;
int bestModeId[MAX_NUM_COMPONENT] = { 0, 0, 0 };
#else

Karsten Suehring
committed
bool checkInitTrDepth = false, checkInitTrDepthTransformSkipWinner = false;
double dSingleCost = MAX_DOUBLE;
Distortion uiSingleDistLuma = 0;
uint64_t singleFracBits = 0;
bool checkTransformSkip = pps.getUseTransformSkip();
int bestModeId[MAX_NUM_COMPONENT] = {0, 0, 0};
uint8_t nNumTransformCands = cu.emtFlag ? 4 : 1; //4 is the number of transforms of emt
bool isAllIntra = m_pcEncCfg->getIntraPeriod() == 1;
uint8_t numTransformIndexCands = nNumTransformCands;

Karsten Suehring
committed
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
const TempCtx ctxStart ( m_CtxCache, m_CABACEstimator->getCtx() );
TempCtx ctxBest ( m_CtxCache );
CodingStructure *csSplit = nullptr;
CodingStructure *csFull = nullptr;
if( bCheckSplit )
{
csSplit = &cs;
}
else if( bCheckFull )
{
csFull = &cs;
}
if( bCheckFull )
{
csFull->cost = 0.0;
TransformUnit &tu = csFull->addTU( CS::getArea( *csFull, currArea, partitioner.chType ), partitioner.chType );
tu.depth = currDepth;
#if JVET_M0464_UNI_MTS
const bool tsAllowed = TU::isTSAllowed ( tu, COMPONENT_Y );
const bool mtsAllowed = TU::isMTSAllowed( tu, COMPONENT_Y );
uint8_t nNumTransformCands = 1 + ( tsAllowed ? 1 : 0 ) + ( mtsAllowed ? 4 : 0 ); // DCT + TS + 4 MTS = 6 tests
std::vector<TrMode> trModes;
trModes.push_back( TrMode( 0, true ) ); //DCT2
if( tsAllowed )
{
trModes.push_back( TrMode( 1, true ) );
}
if( mtsAllowed )
{
for( int i = 2; i < 6; i++ )
{
trModes.push_back( TrMode( i, true) );
}
}
CHECK( !tu.Y().valid(), "Invalid TU" );
#else

Karsten Suehring
committed
checkTransformSkip &= TU::hasTransformSkipFlag( *tu.cs, tu.Y() );
checkTransformSkip &= !cu.transQuantBypass;
checkTransformSkip &= !cu.emtFlag;
#if JVET_M0102_INTRA_SUBPARTITIONS
checkTransformSkip &= !cu.ispMode;
#endif

Karsten Suehring
committed
CHECK( !tu.Y().valid(), "Invalid TU" );
//this prevents transformSkip from being checked because we already know it's not the best mode
checkTransformSkip = ( checkInitTrDepth && !checkInitTrDepthTransformSkipWinner ) ? false : checkTransformSkip;
CHECK( checkInitTrDepthTransformSkipWinner && !checkTransformSkip, "Transform Skip must be enabled if it was the winner in the previous call of xRecurIntraCodingLumaQT!" );

Karsten Suehring
committed
CodingStructure &saveCS = *m_pSaveCS[0];
TransformUnit *tmpTU = nullptr;
Distortion singleDistTmpLuma = 0;
uint64_t singleTmpFracBits = 0;
double singleCostTmp = 0;
int firstCheckId = 0;
#if JVET_M0464_UNI_MTS
int lastCheckId = trModes[nNumTransformCands-1].first;
bool isNotOnlyOneMode = nNumTransformCands != 1;
#else

Karsten Suehring
committed
//we add the EMT candidates to the loop. TransformSkip will still be the last one to be checked (when modeId == lastCheckId) as long as checkTransformSkip is true
int lastCheckId = numTransformIndexCands - ( firstCheckId + 1 ) + ( int ) checkTransformSkip;
bool isNotOnlyOneMode = lastCheckId != firstCheckId && !checkInitTrDepthTransformSkipWinner;

Karsten Suehring
committed
if( isNotOnlyOneMode )
{
saveCS.pcv = cs.pcv;
saveCS.picture = cs.picture;
saveCS.area.repositionTo(cs.area);
saveCS.clearTUs();
tmpTU = &saveCS.addTU(currArea, partitioner.chType);
}
#if JVET_M0464_UNI_MTS
bool cbfDCT2 = true;
#else

Karsten Suehring
committed
bool cbfBestMode = false;

Karsten Suehring
committed
#if JVET_M0464_UNI_MTS
for( int modeId = firstCheckId; modeId < nNumTransformCands; modeId++ )
{
if( !cbfDCT2 || ( m_pcEncCfg->getUseTransformSkipFast() && bestModeId[COMPONENT_Y] == 1 ) )
{
break;
}
if( !trModes[modeId].second )
{
continue;
}
tu.mtsIdx = trModes[modeId].first;
#else

Karsten Suehring
committed
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
for( int modeId = firstCheckId; modeId <= lastCheckId; modeId++ )
{
if( checkInitTrDepthTransformSkipWinner )
{
//If this is a full RQT call and the winner of the first call (checkFirst=true) was transformSkip, then we skip the first iteration of the loop, since transform skip always comes at the end
if( modeId == firstCheckId )
{
continue;
}
}
uint8_t transformIndex = modeId;
if( ( transformIndex < lastCheckId ) || ( ( transformIndex == lastCheckId ) && !checkTransformSkip ) ) //we avoid this if the mode is transformSkip
{
// Skip checking other transform candidates if zero CBF is encountered and it is the best transform so far
if( m_pcEncCfg->getFastIntraEMT() && isAllIntra && transformIndex && !cbfBestMode )
{
continue;
}
}

Karsten Suehring
committed
if ((modeId != firstCheckId) && isNotOnlyOneMode)
{
m_CABACEstimator->getCtx() = ctxStart;
}
int default0Save1Load2 = 0;
singleDistTmpLuma = 0;
#if JVET_M0464_UNI_MTS
if( modeId == firstCheckId && nNumTransformCands > 1 )
#else

Karsten Suehring
committed
if (modeId == firstCheckId && modeId != lastCheckId && !checkInitTrDepthTransformSkipWinner )

Karsten Suehring
committed
{
default0Save1Load2 = 1;
}
else if (modeId != firstCheckId)
{
default0Save1Load2 = 2;
}
#if JVET_M0102_INTRA_SUBPARTITIONS
if( cu.ispMode )
{
default0Save1Load2 = 0;
}
#endif
#if JVET_M0464_UNI_MTS
if( nNumTransformCands > 1 )
{
xIntraCodingTUBlock( tu, COMPONENT_Y, false, singleDistTmpLuma, default0Save1Load2, &numSig, modeId == 0 ? &trModes : nullptr, true );
if( modeId == 0 )
{
for( int i = 0; i < nNumTransformCands; i++ )
{
if( trModes[i].second )
{
Loading
Loading full blame...