Bug fix for LFNST encoder search

- Fix for the case that uninitialized or invalid best cost values are referred during search candidate selection

source/Lib/EncoderLib/EncCu.cpp

@@ -1430,6 +1430,7 @@ void EncCu::xCheckRDCostIntra( CodingStructure *&tempCS, CodingStructure *&bestC
   int        endLfnstIdx         = sps.getUseLFNST() ? maxLfnstIdx : 0;
 
   int grpNumMax = sps.getUseLFNST() ? 4 : 1;
+  m_pcIntraSearch->invalidateBestModeCost();
   for( int trGrpIdx = 0; trGrpIdx < grpNumMax; trGrpIdx++ )
   {
     const uint8_t startMtsFlag = trGrpIdx > 0;

source/Lib/EncoderLib/IntraSearch.cpp

@@ -775,7 +775,7 @@ bool IntraSearch::estIntraPredLumaQT( CodingUnit &cu, Partitioner &partitioner,
     }
     else //mtsUsage = 2 (here we potentially reduce the number of modes that will be full-RD checked)
     {
-      if( m_pcEncCfg->getUseFastLFNST() || !cu.slice->isIntra() )
+      if( ( m_pcEncCfg->getUseFastLFNST() || !cu.slice->isIntra() ) && m_bestModeCostValid[ lfnstIdx ] )
       {
         numModesForFullRD = 0;
 
@@ -1143,6 +1143,7 @@ bool IntraSearch::estIntraPredLumaQT( CodingUnit &cu, Partitioner &partitioner,
           if( sps.getUseLFNST() && mtsUsageFlag == 1 && !cu.ispMode )
           {
             m_bestModeCostStore[ lfnstIdx ] = csBest->cost; //cs.cost;
+            m_bestModeCostValid[ lfnstIdx ] = true;
           }
           if( csBest->cost < bestCurrentCost )
           {

source/Lib/EncoderLib/IntraSearch.h

@@ -215,6 +215,7 @@ private:
 
   //cost variables for the EMT algorithm and new modes list
   double     m_bestModeCostStore[ NUM_LFNST_NUM_PER_SET ];                                    // RD cost of the best mode for each PU using DCT2
+  bool       m_bestModeCostValid[ NUM_LFNST_NUM_PER_SET ];
   double     m_modeCostStore[ NUM_LFNST_NUM_PER_SET ][ NUM_LUMA_MODE ];                   // RD cost of each mode for each PU using DCT2
   ModeInfo   m_savedRdModeList[ NUM_LFNST_NUM_PER_SET ][ NUM_LUMA_MODE ];
   int32_t    m_savedNumRdModes[ NUM_LFNST_NUM_PER_SET ];
@@ -277,6 +278,7 @@ public:
   void estIntraPredChromaQT       ( CodingUnit &cu, Partitioner& pm, const double maxCostAllowed = MAX_DOUBLE );
   void IPCMSearch                 (CodingStructure &cs, Partitioner& partitioner);
   uint64_t xFracModeBitsIntra     (PredictionUnit &pu, const uint32_t &uiMode, const ChannelType &compID);
+  void invalidateBestModeCost     () { for( int i = 0; i < NUM_LFNST_NUM_PER_SET; i++ ) m_bestModeCostValid[ i ] = false; };
 
 protected: