Newer
Older

Karsten Suehring
committed
/* The copyright in this software is being made available under the BSD
* License, included below. This software may be subject to other third party
* and contributor rights, including patent rights, and no such rights are
* granted under this license.
*

Karsten Suehring
committed
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of the ITU/ISO/IEC nor the names of its contributors may
* be used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
/** \file EncSearch.cpp
* \brief encoder intra search class
*/
#include "IntraSearch.h"
#include "EncModeCtrl.h"
#include "CommonLib/CommonDef.h"
#include "CommonLib/Rom.h"
#include "CommonLib/Picture.h"
#include "CommonLib/UnitTools.h"
#if JVET_V0094_BILATERAL_FILTER || JVET_X0071_CHROMA_BILATERAL_FILTER

Karsten Suehring
committed
#include "CommonLib/dtrace_next.h"
#include "CommonLib/dtrace_buffer.h"
#include <math.h>
#include <limits>
//! \ingroup EncoderLib
//! \{
Yung-Hsuan Chao (Jessie)
committed
#define PLTCtx(c) SubCtx( Ctx::Palette, c )

Karsten Suehring
committed
IntraSearch::IntraSearch()
: m_pSplitCS (nullptr)
, m_pFullCS (nullptr)
, m_pBestCS (nullptr)
, m_pcEncCfg (nullptr)
#if JVET_V0094_BILATERAL_FILTER || JVET_X0071_CHROMA_BILATERAL_FILTER

Karsten Suehring
committed
, m_pcTrQuant (nullptr)
, m_pcRdCost (nullptr)

Karsten Suehring
committed
, m_CABACEstimator(nullptr)

Karsten Suehring
committed
, m_isInitialized (false)
{
for( uint32_t ch = 0; ch < MAX_NUM_TBLOCKS; ch++ )
{
m_pSharedPredTransformSkip[ch] = nullptr;
}
#if JVET_AG0152_SGPM_ITMP_IBC
for (int i = 0; i < NUM_LUMA_MODE + SGPM_NUM_BVS; i++)
#else
{
m_intraPredBuf[i] = nullptr;
}
for (int i = 0; i < SGPM_NUM; i++)
{
m_sgpmPredBuf[i] = nullptr;
}
#endif
#if JVET_AG0058_EIP
for (int i = 0; i < NUM_DERIVED_EIP; i++)
{
m_eipPredBuf[i] = nullptr;
}
for(int i = 0; i < MAX_MERGE_EIP;i++)
{
m_eipMergePredBuf[i] = nullptr;
}
#endif
#if JVET_AH0076_OBIC
m_dimdPredBuf = nullptr;
m_obicPredBuf = nullptr;
#endif
m_truncBinBits = nullptr;
m_escapeNumBins = nullptr;
m_minErrorIndexMap = nullptr;
for (unsigned i = 0; i < (MAXPLTSIZE + 1); i++)
{
m_indexError[i] = nullptr;
}
for (unsigned i = 0; i < NUM_TRELLIS_STATE; i++)
{
m_statePtRDOQ[i] = nullptr;
}
::memset( m_indexMapRDOQ, 0, sizeof( m_indexMapRDOQ ) );
::memset( m_runMapRDOQ, false, sizeof( m_runMapRDOQ ) );
::memset( m_prevRunTypeRDOQ, false, sizeof( m_prevRunTypeRDOQ ) );
::memset( m_prevRunPosRDOQ, 0, sizeof( m_prevRunPosRDOQ ) );
::memset( m_stateCostRDOQ, 0, sizeof( m_stateCostRDOQ ) );

Karsten Suehring
committed
}
void IntraSearch::destroy()
{
CHECK( !m_isInitialized, "Not initialized" );
if( m_pcEncCfg )
{
const uint32_t uiNumLayersToAllocateSplit = 1;
const uint32_t uiNumLayersToAllocateFull = 1;

Karsten Suehring
committed
const int uiNumSaveLayersToAllocate = 2;
for( uint32_t layer = 0; layer < uiNumSaveLayersToAllocate; layer++ )
{
m_pSaveCS[layer]->destroy();
delete m_pSaveCS[layer];
}
uint32_t numWidths = gp_sizeIdxInfo->numWidths();
uint32_t numHeights = gp_sizeIdxInfo->numHeights();
for( uint32_t width = 0; width < numWidths; width++ )
{
for( uint32_t height = 0; height < numHeights; height++ )
{
Fabrice URBAN
committed
if( gp_sizeIdxInfo->isCuSize( gp_sizeIdxInfo->sizeFrom( width ) ) && gp_sizeIdxInfo->isCuSize( gp_sizeIdxInfo->sizeFrom( height ) )
&& gp_sizeIdxInfo->sizeFrom(width) <= m_pcEncCfg->getMaxCUWidth() && gp_sizeIdxInfo->sizeFrom(height) <= m_pcEncCfg->getMaxCUHeight())

Karsten Suehring
committed
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
{
for( uint32_t layer = 0; layer < uiNumLayersToAllocateSplit; layer++ )
{
m_pSplitCS[width][height][layer]->destroy();
delete m_pSplitCS[width][height][layer];
}
for( uint32_t layer = 0; layer < uiNumLayersToAllocateFull; layer++ )
{
m_pFullCS[width][height][layer]->destroy();
delete m_pFullCS[width][height][layer];
}
delete[] m_pSplitCS[width][height];
delete[] m_pFullCS [width][height];
m_pBestCS[width][height]->destroy();
m_pTempCS[width][height]->destroy();
delete m_pTempCS[width][height];
delete m_pBestCS[width][height];
}
}
delete[] m_pSplitCS[width];
delete[] m_pFullCS [width];
delete[] m_pTempCS[width];
delete[] m_pBestCS[width];
}
delete[] m_pSplitCS;
delete[] m_pFullCS;
delete[] m_pBestCS;
delete[] m_pTempCS;
delete[] m_pSaveCS;
}
m_pSplitCS = m_pFullCS = nullptr;
m_pBestCS = m_pTempCS = nullptr;
m_pSaveCS = nullptr;
for( uint32_t ch = 0; ch < MAX_NUM_TBLOCKS; ch++ )
{
delete[] m_pSharedPredTransformSkip[ch];
m_pSharedPredTransformSkip[ch] = nullptr;
}
#if JVET_AC0119_LM_CHROMA_FUSION
for (uint32_t i = 0; i < 2; i++)
{
m_predStorage[i].destroy();
}
for (uint32_t i = 0; i < 6; i++)
{
m_fusionStorage[i].destroy();
}
#endif
#if JVET_AD0120_LBCCP
for (uint32_t i = 0; i < LBCCP_FILTER_MMLMNUM; i++)
{
m_lmPredFiltStorage[i].destroy();
}
#endif
#if JVET_AG0059_CCP_MERGE_ENHANCEMENT
for (uint32_t i = 0; i < 2; i++)
{
m_predCCPFusionStorage[i].destroy();
}
#endif
#if JVET_AD0202_CCCM_MDF
for (uint32_t cccmIdx = 0; cccmIdx < TOTAL_NUM_CCCM_MODES; cccmIdx++)
#else
for (uint32_t cccmIdx = 0; cccmIdx < CCCM_NUM_MODES; cccmIdx++)
#if JVET_AC0147_CCCM_NO_SUBSAMPLING
m_cccmStorage[0][cccmIdx].destroy();
m_cccmStorage[1][cccmIdx].destroy();
#else
#if JVET_AG0154_DECODER_DERIVED_CCP_FUSION
m_ddCcpStorage.destroy();
for (int i = 0; i < 2; i++)
{
m_ddCcpFusionStorage[i].destroy();
}
#endif
m_colorTransResiBuf.destroy();
#if JVET_AG0152_SGPM_ITMP_IBC
for (int i = 0; i < NUM_LUMA_MODE + SGPM_NUM_BVS; i++)
#else
{
delete[] m_intraPredBuf[i];
m_intraPredBuf[i] = nullptr;
}
for (int i = 0; i < SGPM_NUM; i++)
{
delete[] m_sgpmPredBuf[i];
m_sgpmPredBuf[i] = nullptr;
}
#endif
#if JVET_AG0058_EIP
for (int i = 0; i < NUM_DERIVED_EIP; i++)
{
delete[] m_eipPredBuf[i];
m_eipPredBuf[i] = nullptr;
}
for(int i = 0; i < MAX_MERGE_EIP;i++)
{
delete[] m_eipMergePredBuf[i];
m_eipMergePredBuf[i] = nullptr;
}
#endif
#if JVET_AH0076_OBIC
delete[] m_dimdPredBuf;
m_dimdPredBuf = nullptr;
delete[] m_obicPredBuf;
m_obicPredBuf = nullptr;
#endif

Karsten Suehring
committed
m_isInitialized = false;
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
if (m_truncBinBits != nullptr)
{
for (unsigned i = 0; i < m_symbolSize; i++)
{
delete[] m_truncBinBits[i];
m_truncBinBits[i] = nullptr;
}
delete[] m_truncBinBits;
m_truncBinBits = nullptr;
}
if (m_escapeNumBins != nullptr)
{
delete[] m_escapeNumBins;
m_escapeNumBins = nullptr;
}
if (m_indexError[0] != nullptr)
{
for (unsigned i = 0; i < (MAXPLTSIZE + 1); i++)
{
delete[] m_indexError[i];
m_indexError[i] = nullptr;
}
}
if (m_minErrorIndexMap != nullptr)
{
delete[] m_minErrorIndexMap;
m_minErrorIndexMap = nullptr;
}
if (m_statePtRDOQ[0] != nullptr)
{
for (unsigned i = 0; i < NUM_TRELLIS_STATE; i++)
{
delete[] m_statePtRDOQ[i];
m_statePtRDOQ[i] = nullptr;
}
}

Karsten Suehring
committed
}
IntraSearch::~IntraSearch()
{
if( m_isInitialized )
{
destroy();
}
}
void IntraSearch::init( EncCfg* pcEncCfg,
#if JVET_V0094_BILATERAL_FILTER || JVET_X0071_CHROMA_BILATERAL_FILTER

Karsten Suehring
committed
TrQuant* pcTrQuant,
RdCost* pcRdCost,
CABACWriter* CABACEstimator,
CtxCache* ctxCache,
const uint32_t maxCUWidth,
const uint32_t maxCUHeight,
const uint32_t maxTotalCUDepth
, const unsigned bitDepthY

Karsten Suehring
committed
)
{
CHECK(m_isInitialized, "Already initialized");
m_pcEncCfg = pcEncCfg;
#if JVET_V0094_BILATERAL_FILTER || JVET_X0071_CHROMA_BILATERAL_FILTER

Karsten Suehring
committed
m_pcTrQuant = pcTrQuant;
m_pcRdCost = pcRdCost;
m_CABACEstimator = CABACEstimator;

Karsten Suehring
committed
const ChromaFormat cform = pcEncCfg->getChromaFormatIdc();
IntraPrediction::init( cform, pcEncCfg->getBitDepth( CHANNEL_TYPE_LUMA ) );
Fabrice URBAN
committed
m_tmpStorageLCU.create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
m_colorTransResiBuf.create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));

Karsten Suehring
committed
#if JVET_AC0119_LM_CHROMA_FUSION
for (uint32_t i = 0; i < 2; i++)
{
Fabrice URBAN
committed
m_predStorage[i].create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
}
for (uint32_t i = 0; i < 6; i++)
{
Fabrice URBAN
committed
m_fusionStorage[i].create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
#if JVET_AD0120_LBCCP
for (uint32_t i = 0; i < LBCCP_FILTER_MMLMNUM; i++)
{
m_lmPredFiltStorage[i].create(UnitArea(cform, Area(0, 0, MAX_CU_SIZE, MAX_CU_SIZE)));
}
#endif
#if JVET_AG0059_CCP_MERGE_ENHANCEMENT
for (uint32_t i = 0; i < 2; i++)
{
m_predCCPFusionStorage[i].create(UnitArea(cform, Area(0, 0, MAX_CU_SIZE, MAX_CU_SIZE)));
}
#endif
#if JVET_AD0202_CCCM_MDF
for (uint32_t cccmIdx = 0; cccmIdx < TOTAL_NUM_CCCM_MODES; cccmIdx++)
#else
for (uint32_t cccmIdx = 0; cccmIdx < CCCM_NUM_MODES; cccmIdx++)
#if JVET_AC0147_CCCM_NO_SUBSAMPLING
Fabrice URBAN
committed
m_cccmStorage[0][cccmIdx].create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
m_cccmStorage[1][cccmIdx].create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
m_cccmStorage[cccmIdx].create(UnitArea(cform, Area(0, 0, MAX_CU_SIZE, MAX_CU_SIZE)));
#if JVET_AG0154_DECODER_DERIVED_CCP_FUSION
m_ddCcpStorage.create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
for (int i = 0; i < 2; i++)
{
m_ddCcpFusionStorage[i].create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)));
}
#endif
#if JVET_AB0155_SGPM || JVET_AH0076_OBIC
#if JVET_AG0152_SGPM_ITMP_IBC
for (int i = 0; i < NUM_LUMA_MODE + SGPM_NUM_BVS; i++)
#else
#if JVET_AH0076_OBIC
m_intraPredBuf[i] = new Pel[(MAX_CU_SIZE>>1) * (MAX_CU_SIZE>>1)];
#else
m_intraPredBuf[i] = new Pel[GEO_MAX_CU_SIZE_EX * GEO_MAX_CU_SIZE_EX];
}
for (int i = 0; i < SGPM_NUM; i++)
{
m_sgpmPredBuf[i] = new Pel[GEO_MAX_CU_SIZE_EX * GEO_MAX_CU_SIZE_EX];
}
#endif
#if JVET_AG0058_EIP
for (int i = 0; i < NUM_DERIVED_EIP; i++)
{
m_eipPredBuf[i] = new Pel[MAX_EIP_SIZE * MAX_EIP_SIZE];
}
for(int i = 0; i < MAX_MERGE_EIP;i++)
{
m_eipMergePredBuf[i] = new Pel[MAX_EIP_SIZE * MAX_EIP_SIZE];
}
#endif
#if JVET_AH0076_OBIC
m_dimdPredBuf = new Pel[(MAX_CU_SIZE>>1) * (MAX_CU_SIZE>>1)];
m_obicPredBuf = new Pel[(MAX_CU_SIZE>>1) * (MAX_CU_SIZE>>1)];
#endif

Karsten Suehring
committed
for( uint32_t ch = 0; ch < MAX_NUM_TBLOCKS; ch++ )
{
Fabrice URBAN
committed
m_pSharedPredTransformSkip[ch] = new Pel[maxCUWidth * maxCUHeight];

Karsten Suehring
committed
}
uint32_t numWidths = gp_sizeIdxInfo->numWidths();
uint32_t numHeights = gp_sizeIdxInfo->numHeights();
const uint32_t uiNumLayersToAllocateSplit = 1;
const uint32_t uiNumLayersToAllocateFull = 1;

Karsten Suehring
committed
m_pBestCS = new CodingStructure**[numWidths];
m_pTempCS = new CodingStructure**[numWidths];
m_pFullCS = new CodingStructure***[numWidths];
m_pSplitCS = new CodingStructure***[numWidths];
for( uint32_t width = 0; width < numWidths; width++ )
{
m_pBestCS[width] = new CodingStructure*[numHeights];
m_pTempCS[width] = new CodingStructure*[numHeights];
m_pFullCS [width] = new CodingStructure**[numHeights];
m_pSplitCS[width] = new CodingStructure**[numHeights];
for( uint32_t height = 0; height < numHeights; height++ )
{
Fabrice URBAN
committed
if( gp_sizeIdxInfo->isCuSize( gp_sizeIdxInfo->sizeFrom( width ) ) && gp_sizeIdxInfo->isCuSize( gp_sizeIdxInfo->sizeFrom( height ) )
&& gp_sizeIdxInfo->sizeFrom(width) <= maxCUWidth && gp_sizeIdxInfo->sizeFrom(height) <= maxCUHeight)

Karsten Suehring
committed
{
m_pBestCS[width][height] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
m_pTempCS[width][height] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
#if JVET_Z0118_GDR
m_pBestCS[width][height]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode(), pcEncCfg->getGdrEnabled());
m_pTempCS[width][height]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode(), pcEncCfg->getGdrEnabled());
#else
m_pBestCS[width][height]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode());
m_pTempCS[width][height]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode());

Karsten Suehring
committed
m_pFullCS [width][height] = new CodingStructure*[uiNumLayersToAllocateFull];
m_pSplitCS[width][height] = new CodingStructure*[uiNumLayersToAllocateSplit];
for( uint32_t layer = 0; layer < uiNumLayersToAllocateFull; layer++ )
{
m_pFullCS [width][height][layer] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
#if JVET_Z0118_GDR
m_pFullCS[width][height][layer]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode(), pcEncCfg->getGdrEnabled());
#else
m_pFullCS[width][height][layer]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode());

Karsten Suehring
committed
}
for( uint32_t layer = 0; layer < uiNumLayersToAllocateSplit; layer++ )
{
m_pSplitCS[width][height][layer] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
#if JVET_Z0118_GDR
m_pSplitCS[width][height][layer]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode(), pcEncCfg->getGdrEnabled());
#else
m_pSplitCS[width][height][layer]->create(m_pcEncCfg->getChromaFormatIdc(), Area(0, 0, gp_sizeIdxInfo->sizeFrom(width), gp_sizeIdxInfo->sizeFrom(height)), false, (bool)pcEncCfg->getPLTMode());

Karsten Suehring
committed
}
}
else
{
m_pBestCS[width][height] = nullptr;
m_pTempCS[width][height] = nullptr;
m_pFullCS [width][height] = nullptr;
m_pSplitCS[width][height] = nullptr;
}
}
}
const int uiNumSaveLayersToAllocate = 2;
m_pSaveCS = new CodingStructure*[uiNumSaveLayersToAllocate];
for( uint32_t depth = 0; depth < uiNumSaveLayersToAllocate; depth++ )
{
m_pSaveCS[depth] = new CodingStructure( m_unitCache.cuCache, m_unitCache.puCache, m_unitCache.tuCache );
#if JVET_Z0118_GDR
m_pSaveCS[depth]->create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)), false, (bool)pcEncCfg->getPLTMode(), pcEncCfg->getGdrEnabled());
#else
m_pSaveCS[depth]->create(UnitArea(cform, Area(0, 0, maxCUWidth, maxCUHeight)), false, (bool)pcEncCfg->getPLTMode());

Karsten Suehring
committed
}
m_isInitialized = true;
if (pcEncCfg->getPLTMode())
m_symbolSize = (1 << bitDepthY); // pixel values are within [0, SymbolSize-1] with size SymbolSize
if (m_truncBinBits == nullptr)
m_truncBinBits = new uint16_t*[m_symbolSize];
for (unsigned i = 0; i < m_symbolSize; i++)
{
m_truncBinBits[i] = new uint16_t[m_symbolSize + 1];
}
if (m_escapeNumBins == nullptr)
m_escapeNumBins = new uint16_t[m_symbolSize];
initTBCTable(bitDepthY);
if (m_indexError[0] == nullptr)
for (unsigned i = 0; i < (MAXPLTSIZE + 1); i++)
{
m_indexError[i] = new double[MAX_CU_BLKSIZE_PLT*MAX_CU_BLKSIZE_PLT];
}
}
if (m_minErrorIndexMap == nullptr)
{
m_minErrorIndexMap = new uint8_t[MAX_CU_BLKSIZE_PLT*MAX_CU_BLKSIZE_PLT];
}
if (m_statePtRDOQ[0] == nullptr)
{
for (unsigned i = 0; i < NUM_TRELLIS_STATE; i++)
{
m_statePtRDOQ[i] = new uint8_t[MAX_CU_BLKSIZE_PLT*MAX_CU_BLKSIZE_PLT];
}
#if INTRA_TRANS_ENC_OPT
m_skipTimdLfnstMtsPass = false;
#endif
#if JVET_AH0076_OBIC
m_skipObicLfnstMtsPass = false;
m_skipDimdLfnstMtsPass = false;
#endif

Karsten Suehring
committed
}
//////////////////////////////////////////////////////////////////////////
// INTRA PREDICTION
//////////////////////////////////////////////////////////////////////////
static constexpr double COST_UNKNOWN = -65536.0;
double IntraSearch::findInterCUCost( CodingUnit &cu )
{
if( cu.isConsIntra() && !cu.slice->isIntra() )
{
//search corresponding inter CU cost
for( int i = 0; i < m_numCuInSCIPU; i++ )
{
if( cu.lumaPos() == m_cuAreaInSCIPU[i].pos() && cu.lumaSize() == m_cuAreaInSCIPU[i].size() )
{
return m_cuCostInSCIPU[i];
}
}
}
return COST_UNKNOWN;
}
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
#if JVET_W0103_INTRA_MTS
bool IntraSearch::testISPforCurrCU(const CodingUnit &cu)
{
CodingStructure &cs = *cu.cs;
auto &pu = *cu.firstPU;
const CompArea &area = pu.Y();
PelBuf piOrg = cs.getOrgBuf(area);
Pel* pOrg = piOrg.buf;
int uiWidth = area.width;
int uiHeight = area.height;
int iStride = piOrg.stride;
int Gsum = 0;
int nPix = (uiWidth - 2) * (uiHeight - 2);
for (int y = 1; y < (uiHeight - 1); y++)
{
for (int x = 1; x < (uiWidth - 1); x++)
{
const Pel *p = pOrg + y * iStride + x;
int iDy = p[-iStride - 1] + 2 * p[-1] + p[iStride - 1] - p[-iStride + 1] - 2 * p[+1] - p[iStride + 1];
int iDx = p[iStride - 1] + 2 * p[iStride] + p[iStride + 1] - p[-iStride - 1] - 2 * p[-iStride] - p[-iStride + 1];
if (iDy == 0 && iDx == 0)
continue;
int iAmp = (int)(abs(iDx) + abs(iDy));
Gsum += iAmp;
}
}
Gsum = (Gsum + (nPix >> 1)) / nPix;
bool testISP = true;
CHECK(m_numModesISPRDO != -1, "m_numModesISPRDO!=-1");
m_numModesISPRDO = (Gsum < 50 && uiWidth >= 16 && uiHeight >= 16) ? 1 : 2;
return testISP;
}
#endif
bool IntraSearch::estIntraPredLumaQT(CodingUnit &cu, Partitioner &partitioner, const double bestCostSoFar, bool mtsCheckRangeFlag, int mtsFirstCheckId, int mtsLastCheckId, bool moreProbMTSIdxFirst, CodingStructure* bestCS
#if JVET_AG0136_INTRA_TMP_LIC
, InterPrediction* pcInterPred
#endif
)

Karsten Suehring
committed
{
CodingStructure &cs = *cu.cs;
const SPS &sps = *cs.sps;
const uint32_t uiWidthBit = floorLog2(partitioner.currArea().lwidth() );
const uint32_t uiHeightBit = floorLog2(partitioner.currArea().lheight());

Karsten Suehring
committed
// Lambda calculation at equivalent Qp of 4 is recommended because at that Qp, the quantization divisor is 1.
const double sqrtLambdaForFirstPass = m_pcRdCost->getMotionLambda( ) * FRAC_BITS_SCALE;

Karsten Suehring
committed
//===== loop over partitions =====
const TempCtx ctxStart ( m_ctxCache, m_CABACEstimator->getCtx() );
const TempCtx ctxStartMipFlag ( m_ctxCache, SubCtx( Ctx::MipFlag, m_CABACEstimator->getCtx() ) );
const TempCtx ctxStartTpmFlag ( m_ctxCache, SubCtx(Ctx::TmpFlag, m_CABACEstimator->getCtx()));
#if JVET_AD0086_ENHANCED_INTRA_TMP
const TempCtx ctxStartTpmIdx ( m_ctxCache, SubCtx(Ctx::TmpIdx, m_CABACEstimator->getCtx()));
const TempCtx ctxStartTpmFusionFlag(m_ctxCache, SubCtx(Ctx::TmpFusion, m_CABACEstimator->getCtx()));
#endif
#if JVET_AG0136_INTRA_TMP_LIC
const TempCtx ctxStartTmpLicFlag (m_ctxCache, SubCtx(Ctx::TmpLic, m_CABACEstimator->getCtx()));
const TempCtx ctxStartTmpLicIdx (m_ctxCache, SubCtx(Ctx::ItmpLicIndex, m_CABACEstimator->getCtx()));
#endif
const TempCtx ctxStartTimdFlag ( m_ctxCache, SubCtx( Ctx::TimdFlag, m_CABACEstimator->getCtx() ) );
const TempCtx ctxStartSgpmFlag ( m_ctxCache, SubCtx(Ctx::SgpmFlag, m_CABACEstimator->getCtx()));
const TempCtx ctxStartIspMode ( m_ctxCache, SubCtx( Ctx::ISPMode, m_CABACEstimator->getCtx() ) );
const TempCtx ctxStartMPMIdxFlag ( m_ctxCache, SubCtx(Ctx::IntraLumaMPMIdx, m_CABACEstimator->getCtx()));
const TempCtx ctxStartPlanarFlag ( m_ctxCache, SubCtx( Ctx::IntraLumaPlanarFlag, m_CABACEstimator->getCtx() ) );
const TempCtx ctxStartIntraMode ( m_ctxCache, SubCtx(Ctx::IntraLumaMpmFlag, m_CABACEstimator->getCtx()));
const TempCtx ctxStartIntraMode2 ( m_ctxCache, SubCtx(Ctx::IntraLumaSecondMpmFlag, m_CABACEstimator->getCtx()));
#if JVET_AD0085_MPM_SORTING
const TempCtx ctxStartMpmIdx2 ( m_ctxCache, SubCtx(Ctx::IntraLumaSecondMpmIdx, m_CABACEstimator->getCtx()) );
#endif
const TempCtx ctxStartMrlIdx ( m_ctxCache, SubCtx( Ctx::MultiRefLineIdx, m_CABACEstimator->getCtx() ) );
const TempCtx ctxStartTmrlDerive ( m_ctxCache, SubCtx(Ctx::TmrlDerive, m_CABACEstimator->getCtx()));
#if JVET_AG0058_EIP
const TempCtx ctxStartEip(m_ctxCache, SubCtx(Ctx::EipFlag, m_CABACEstimator->getCtx()));
#endif
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
// NB xFracModeBitsIntra will not affect the mode for chroma that may have already been pre-estimated.
auto loadStartStates = [&]()
{
m_CABACEstimator->getCtx() = SubCtx(Ctx::MipFlag, ctxStartMipFlag);
#if JVET_V0130_INTRA_TMP
m_CABACEstimator->getCtx() = SubCtx(Ctx::TmpFlag, ctxStartTpmFlag);
#endif
#if JVET_AD0086_ENHANCED_INTRA_TMP
m_CABACEstimator->getCtx() = SubCtx(Ctx::TmpIdx, ctxStartTpmIdx);
m_CABACEstimator->getCtx() = SubCtx(Ctx::TmpFusion, ctxStartTpmFusionFlag);
#endif
#if JVET_W0123_TIMD_FUSION
m_CABACEstimator->getCtx() = SubCtx(Ctx::TimdFlag, ctxStartTimdFlag);
#endif
#if JVET_AB0155_SGPM
m_CABACEstimator->getCtx() = SubCtx(Ctx::SgpmFlag, ctxStartSgpmFlag);
#endif
m_CABACEstimator->getCtx() = SubCtx(Ctx::ISPMode, ctxStartIspMode);
#if SECONDARY_MPM
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMPMIdx, ctxStartMPMIdxFlag);
#endif
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaPlanarFlag, ctxStartPlanarFlag);
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
#if SECONDARY_MPM
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaSecondMpmFlag, ctxStartIntraMode2);
#if JVET_AD0085_MPM_SORTING
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaSecondMpmIdx, ctxStartMpmIdx2);
#endif
#endif
m_CABACEstimator->getCtx() = SubCtx(Ctx::MultiRefLineIdx, ctxStartMrlIdx);
#if JVET_AB0157_TMRL
m_CABACEstimator->getCtx() = SubCtx(Ctx::TmrlDerive, ctxStartTmrlDerive);
#endif
#if JVET_AG0058_EIP
m_CABACEstimator->getCtx() = SubCtx(Ctx::EipFlag, ctxStartEip);
#endif
#if JVET_AG0136_INTRA_TMP_LIC
m_CABACEstimator->getCtx() = SubCtx(Ctx::TmpLic, ctxStartTmpLicFlag);
m_CABACEstimator->getCtx() = SubCtx(Ctx::ItmpLicIndex, ctxStartTmpLicIdx);
#endif
};

Karsten Suehring
committed
CHECK( !cu.firstPU, "CU has no PUs" );
#if JVET_AH0103_LOW_DELAY_LFNST_NSPT
bool spsIntraLfnstEnabled = ( ( cu.slice->getSliceType() == I_SLICE && cu.cs->sps->getUseIntraLFNSTISlice() ) ||
( cu.slice->getSliceType() != I_SLICE && cu.cs->sps->getUseIntraLFNSTPBSlice() ) );
#endif
// variables for saving fast intra modes scan results across multiple LFNST passes
#if JVET_AH0103_LOW_DELAY_LFNST_NSPT
bool LFNSTLoadFlag = spsIntraLfnstEnabled && cu.lfnstIdx != 0;
bool LFNSTSaveFlag = spsIntraLfnstEnabled && cu.lfnstIdx == 0;
#else
bool LFNSTLoadFlag = sps.getUseLFNST() && cu.lfnstIdx != 0;
bool LFNSTSaveFlag = sps.getUseLFNST() && cu.lfnstIdx == 0;
LFNSTSaveFlag &= sps.getUseIntraMTS() ? cu.mtsFlag == 0 : true;
bool SGPMSaveFlag = (cu.lfnstIdx == 0 && cu.mtsFlag == 0);
#if JVET_AH0076_OBIC
bool testDimd = isLuma(partitioner.chType) && cu.slice->getSPS()->getUseDimd();
bool testObic = testDimd && (PU::isObicAvail(*cu.firstPU) && cu.obicMode[0] >= 0);
bool obicSaveFlag = testObic && (cu.lfnstIdx == 0 && cu.mtsFlag == 0);
bool dimdSaveFlag = testDimd && (cu.lfnstIdx == 0 && cu.mtsFlag == 0);
#endif
const uint32_t lfnstIdx = cu.lfnstIdx;
double costInterCU = findInterCUCost( cu );
const int width = partitioner.currArea().lwidth();
const int height = partitioner.currArea().lheight();
// Marking MTS usage for faster MTS
// 0: MTS is either not applicable for current CU (cuWidth > MTS_INTRA_MAX_CU_SIZE or cuHeight > MTS_INTRA_MAX_CU_SIZE), not active in the config file or the fast decision algorithm is not used in this case
// 1: MTS fast algorithm can be applied for the current CU, and the DCT2 is being checked
// 2: MTS is being checked for current CU. Stored results of DCT2 can be utilized for speedup
uint8_t mtsUsageFlag = 0;
const int maxSizeEMT = MTS_INTRA_MAX_CU_SIZE;
if( width <= maxSizeEMT && height <= maxSizeEMT && sps.getUseIntraMTS() )
{
#if JVET_AH0103_LOW_DELAY_LFNST_NSPT
mtsUsageFlag = ( spsIntraLfnstEnabled && cu.mtsFlag == 1 ) ? 2 : 1;
#else
mtsUsageFlag = ( sps.getUseLFNST() && cu.mtsFlag == 1 ) ? 2 : 1;
}
if( width * height < 64 && !m_pcEncCfg->getUseFastLFNST() )
{
mtsUsageFlag = 0;
}
#if JVET_W0103_INTRA_MTS
if (!cu.mtsFlag && !cu.lfnstIdx)
{
m_globalBestCostStore = MAX_DOUBLE;
m_globalBestCostValid = false;
if (bestCS->getCU(partitioner.chType) != NULL && bestCS->getCU(partitioner.chType)->predMode != MODE_INTRA && bestCostSoFar != MAX_DOUBLE)
{
m_globalBestCostStore = bestCostSoFar;
m_globalBestCostValid = true;
}
#if JVET_Y0142_ADAPT_INTRA_MTS
m_modesForMTS.clear();
m_modesCoeffAbsSumDCT2.clear();
#endif
#if JVET_AE0169_BIPREDICTIVE_IBC
m_bestIntraSADHADCost = MAX_DOUBLE;
#endif
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
m_bestIntraSADCost = MAX_DOUBLE;
const bool colorTransformIsEnabled = sps.getUseColorTrans() && !CS::isDualITree(cs);
const bool isFirstColorSpace = colorTransformIsEnabled && ((m_pcEncCfg->getRGBFormatFlag() && cu.colorTransform) || (!m_pcEncCfg->getRGBFormatFlag() && !cu.colorTransform));
const bool isSecondColorSpace = colorTransformIsEnabled && ((m_pcEncCfg->getRGBFormatFlag() && !cu.colorTransform) || (!m_pcEncCfg->getRGBFormatFlag() && cu.colorTransform));
Santiago de Luxán Hernández
committed
double bestCurrentCost = bestCostSoFar;
bool ispCanBeUsed = sps.getUseISP() && cu.mtsFlag == 0 && cu.lfnstIdx == 0 && CU::canUseISP(width, height, cu.cs->sps->getMaxTbSize());
bool saveDataForISP = ispCanBeUsed && (!colorTransformIsEnabled || isFirstColorSpace);
bool testISP = ispCanBeUsed && (!colorTransformIsEnabled || !cu.colorTransform);
#if JVET_AB0155_SGPM
const bool sgpmAllowed = sps.getUseSgpm() && isLuma(partitioner.chType);
bool testSgpm = sgpmAllowed && cu.lwidth() >= GEO_MIN_CU_SIZE_EX && cu.lheight() >= GEO_MIN_CU_SIZE_EX
&& cu.lwidth() <= GEO_MAX_CU_SIZE_EX && cu.lheight() <= GEO_MAX_CU_SIZE_EX
&& cu.lwidth() < 8 * cu.lheight() && cu.lheight() < 8 * cu.lwidth() && cu.lx() && cu.ly()
&& cu.lwidth() * cu.lheight() >= SGPM_MIN_PIX;
#endif
#if JVET_AG0058_EIP
bool testEip = isLuma(partitioner.chType) && sps.getUseEip() && (getAllowedEip(cu, COMPONENT_Y) || getAllowedEipMerge(cu, COMPONENT_Y));
bool eipSaveFlag = (cu.lfnstIdx == 0 && cu.mtsFlag == 0);
#endif
#if JVET_W0103_INTRA_MTS
if (testISP && m_pcEncCfg->getUseFastISP())
{
m_numModesISPRDO = -1;
testISP &= testISPforCurrCU(cu);
}
#endif
//reset the intra modes lists variables
m_ispCandListHor.clear();
m_ispCandListVer.clear();
Santiago de Luxán Hernández
committed
if( testISP )
Yin Zhao
committed
//reset the variables used for the tests
m_regIntraRDListWithCosts.clear();
int numTotalPartsHor = (int)width >> floorLog2(CU::getISPSplitDim(width, height, TU_1D_VERT_SPLIT));
int numTotalPartsVer = (int)height >> floorLog2(CU::getISPSplitDim(width, height, TU_1D_HORZ_SPLIT));
m_ispTestedModes[0].init( numTotalPartsHor, numTotalPartsVer );
//the total number of subpartitions is modified to take into account the cases where LFNST cannot be combined with ISP due to size restrictions
#if JVET_AH0103_LOW_DELAY_LFNST_NSPT
numTotalPartsHor = spsIntraLfnstEnabled && CU::canUseLfnstWithISP( cu.Y(), HOR_INTRA_SUBPARTITIONS ) ? numTotalPartsHor : 0;
numTotalPartsVer = spsIntraLfnstEnabled && CU::canUseLfnstWithISP( cu.Y(), VER_INTRA_SUBPARTITIONS ) ? numTotalPartsVer : 0;
#else
numTotalPartsHor = sps.getUseLFNST() && CU::canUseLfnstWithISP(cu.Y(), HOR_INTRA_SUBPARTITIONS) ? numTotalPartsHor : 0;
numTotalPartsVer = sps.getUseLFNST() && CU::canUseLfnstWithISP(cu.Y(), VER_INTRA_SUBPARTITIONS) ? numTotalPartsVer : 0;
for (int j = 1; j < NUM_LFNST_NUM_PER_SET; j++)
{
m_ispTestedModes[j].init(numTotalPartsHor, numTotalPartsVer);
}

Karsten Suehring
committed
#if INTRA_TRANS_ENC_OPT
double regAngCost = MAX_DOUBLE;
bool setSkipTimdControl = (m_pcEncCfg->getIntraPeriod() == 1) && !cu.lfnstIdx && !cu.mtsFlag;
double timdAngCost = MAX_DOUBLE;
#endif
#if JVET_AH0076_OBIC
double obicAngCost = MAX_DOUBLE, dimdAngCost = MAX_DOUBLE;
bool setSkipDimdControl = (m_pcEncCfg->getIntraPeriod() == 1) && !cu.lfnstIdx && !cu.mtsFlag;
Alican Nalci
committed
const bool testBDPCM = sps.getBDPCMEnabledFlag() && CU::bdpcmAllowed(cu, ComponentID(partitioner.chType)) && cu.mtsFlag == 0 && cu.lfnstIdx == 0;
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiHadModeList;
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> candCostList;

Karsten Suehring
committed
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> CandHadList;
auto &pu = *cu.firstPU;
bool validReturn = false;

Karsten Suehring
committed
{
CandHadList.clear();
candCostList.clear();

Karsten Suehring
committed
uiHadModeList.clear();
Wei Chen
committed
#if JVET_AH0200_INTRA_TMP_BV_REORDER
double tmpBestSatdCost = MAX_DOUBLE;
CodedCUInfo &relatedCU = ((EncModeCtrlMTnoRQT *) m_modeCtrl)->getBlkInfo(partitioner.currArea());
bool isTmpModeTestd = false;
#endif

Karsten Suehring
committed
CHECK(pu.cu != &cu, "PU is not contained in the CU");

Karsten Suehring
committed
//===== determine set of modes to be tested (using prediction signal only) =====
int numModesAvailable = NUM_LUMA_MODE; // total number of Intra modes
const bool fastMip = sps.getUseMIP() && m_pcEncCfg->getUseFastMIP();
const bool mipAllowed = sps.getUseMIP() && isLuma(partitioner.chType) && ((cu.lfnstIdx == 0) || allowLfnstWithMip(cu.firstPU->lumaSize()));
const bool testMip = mipAllowed && !(cu.lwidth() > (8 * cu.lheight()) || cu.lheight() > (8 * cu.lwidth()));
const bool supportedMipBlkSize = pu.lwidth() <= MIP_MAX_WIDTH && pu.lheight() <= MIP_MAX_HEIGHT;
#if JVET_V0130_INTRA_TMP
const bool tpmAllowed = sps.getUseIntraTMP() && isLuma(partitioner.chType) && ((cu.lfnstIdx == 0) || allowLfnstWithTmp());
const bool testTpm = tpmAllowed && (cu.lwidth() <= sps.getIntraTMPMaxSize() && cu.lheight() <= sps.getIntraTMPMaxSize());
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeList;

Karsten Suehring
committed
int numModesForFullRD = 3;
numModesForFullRD = g_aucIntraModeNumFast_UseMPM_2D[uiWidthBit - MIN_CU_LOG2][uiHeightBit - MIN_CU_LOG2];
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
if (m_pcEncCfg->getIBCFastMethod() & IBC_FAST_METHOD_NONSCC)
{
numModesForFullRD = (numModesForFullRD > 1) ? (numModesForFullRD - 1) : numModesForFullRD;
}
#endif

Karsten Suehring
committed
#if INTRA_FULL_SEARCH
numModesForFullRD = numModesAvailable;
#endif
#if JVET_AH0076_OBIC
bool bestObicMode = false;
int bestMipDimd = 0;
#endif
#if JVET_W0123_TIMD_FUSION
bool bestTimdMode = false;
#if JVET_AC0105_DIRECTIONAL_PLANAR
uint8_t bestPlMode = 0;
#endif
#if JVET_AB0155_SGPM
bool bestSgpmMode = false;
const CompArea &area = pu.Y();
CompArea tmpArea(COMPONENT_Y, area.chromaFormat, Position(0, 0), area.size());
#endif
#if JVET_AC0115_INTRA_TMP_DIMD_MTS_LFNST
int intraTmpDimdMode = 0;
#endif
if (isSecondColorSpace)
{
uiRdModeList.clear();
if (m_numSavedRdModeFirstColorSpace[m_savedRdModeIdx] > 0)
{
for (int i = 0; i < m_numSavedRdModeFirstColorSpace[m_savedRdModeIdx]; i++)
{
uiRdModeList.push_back(m_savedRdModeFirstColorSpace[m_savedRdModeIdx][i]);
}
}
else
{
return false;
}

Karsten Suehring
committed
{
#if JVET_AH0076_OBIC
if ((testSgpm && SGPMSaveFlag) || obicSaveFlag || dimdSaveFlag)
#else
#if JVET_AG0152_SGPM_ITMP_IBC
for (int i = 0; i < NUM_LUMA_MODE + SGPM_NUM_BVS; i++)
#else
{
m_intraModeReady[i] = 0;
}
}
#endif
// this should always be true
CHECK(!pu.Y().valid(), "PU is not valid");
#if !JVET_AB0157_TMRL || JVET_AD0082_TMRL_CONFIG
#if JVET_AH0065_RELAX_LINE_BUFFER
bool isFirstLineOfCtu = pu.block(COMPONENT_Y).y == 0;
#else
bool isFirstLineOfCtu = (((pu.block(COMPONENT_Y).y) & ((pu.cs->sps)->getMaxCUWidth() - 1)) == 0);
#if JVET_Y0116_EXTENDED_MRL_LIST
int numOfPassesExtendRef = MRL_NUM_REF_LINES;
if (!sps.getUseMRL() || isFirstLineOfCtu)
{
numOfPassesExtendRef = 1;
}
else
{