Commit 010bef5c authored by Moonmo Koo's avatar Moonmo Koo
Browse files

Bug fix for LFNST encoder search

- Fix for the case that uninitialized or invalid best cost values are referred during search candidate selection
parent 767dca73
......@@ -1430,6 +1430,7 @@ void EncCu::xCheckRDCostIntra( CodingStructure *&tempCS, CodingStructure *&bestC
int endLfnstIdx = sps.getUseLFNST() ? maxLfnstIdx : 0;
int grpNumMax = sps.getUseLFNST() ? 4 : 1;
m_pcIntraSearch->invalidateBestModeCost();
for( int trGrpIdx = 0; trGrpIdx < grpNumMax; trGrpIdx++ )
{
const uint8_t startMtsFlag = trGrpIdx > 0;
......
......@@ -775,7 +775,7 @@ bool IntraSearch::estIntraPredLumaQT( CodingUnit &cu, Partitioner &partitioner,
}
else //mtsUsage = 2 (here we potentially reduce the number of modes that will be full-RD checked)
{
if( m_pcEncCfg->getUseFastLFNST() || !cu.slice->isIntra() )
if( ( m_pcEncCfg->getUseFastLFNST() || !cu.slice->isIntra() ) && m_bestModeCostValid[ lfnstIdx ] )
{
numModesForFullRD = 0;
......@@ -1143,6 +1143,7 @@ bool IntraSearch::estIntraPredLumaQT( CodingUnit &cu, Partitioner &partitioner,
if( sps.getUseLFNST() && mtsUsageFlag == 1 && !cu.ispMode )
{
m_bestModeCostStore[ lfnstIdx ] = csBest->cost; //cs.cost;
m_bestModeCostValid[ lfnstIdx ] = true;
}
if( csBest->cost < bestCurrentCost )
{
......
......@@ -215,6 +215,7 @@ private:
//cost variables for the EMT algorithm and new modes list
double m_bestModeCostStore[ NUM_LFNST_NUM_PER_SET ]; // RD cost of the best mode for each PU using DCT2
bool m_bestModeCostValid[ NUM_LFNST_NUM_PER_SET ];
double m_modeCostStore[ NUM_LFNST_NUM_PER_SET ][ NUM_LUMA_MODE ]; // RD cost of each mode for each PU using DCT2
ModeInfo m_savedRdModeList[ NUM_LFNST_NUM_PER_SET ][ NUM_LUMA_MODE ];
int32_t m_savedNumRdModes[ NUM_LFNST_NUM_PER_SET ];
......@@ -277,6 +278,7 @@ public:
void estIntraPredChromaQT ( CodingUnit &cu, Partitioner& pm, const double maxCostAllowed = MAX_DOUBLE );
void IPCMSearch (CodingStructure &cs, Partitioner& partitioner);
uint64_t xFracModeBitsIntra (PredictionUnit &pu, const uint32_t &uiMode, const ChannelType &compID);
void invalidateBestModeCost () { for( int i = 0; i < NUM_LFNST_NUM_PER_SET; i++ ) m_bestModeCostValid[ i ] = false; };
protected:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment