IntraSearch.cpp

            ModeInfo mostProbableMode(true, 0, NOT_INTRA_SUBPARTITIONS, mpm[j]);
            for( int i = 0; i < numModesForFullRD; i++ )
            {
              mostProbableModeIncluded |= (mostProbableMode == uiRdModeList[i]);
            }
            if( !mostProbableModeIncluded )
            {
              numModesForFullRD++;
              uiRdModeList.push_back( mostProbableMode );
              CandCostList.push_back(0);
            }
          }
        }
#endif
      }
      else
      {
#if JVET_N0217_MATRIX_INTRAPRED
        THROW( "Full search not supported for MIP" );
#else
        for( int i = 0; i < numModesForFullRD; i++ )
        {
          uiRdModeList.push_back( i );
        }
#endif
      }
#if JVET_N0193_LFNST
      if( sps.getUseLFNST() && mtsUsageFlag == 1 )
      {
        // Store the modes to be checked with RD
        m_savedNumRdModes[ lfnstIdx ]     = numModesForFullRD;
        std::copy_n( uiRdModeList.begin(),  numModesForFullRD, m_savedRdModeList[ lfnstIdx ] );
#if !JVET_N0217_MATRIX_INTRAPRED
        std::copy_n( extendRefList.begin(), numModesForFullRD, m_savedExtendRefList[ lfnstIdx ] );
#endif
      }
#endif
    }
#if JVET_N0193_LFNST
    else //mtsUsage = 2 (here we potentially reduce the number of modes that will be full-RD checked)
    {
      if( m_pcEncCfg->getUseFastLFNST() || !cu.slice->isIntra() )
      {
        numModesForFullRD = 0;

        double thresholdSkipMode = 1.0 + ( ( cu.lfnstIdx > 0 ) ? 0.1 : 1.0 ) * ( 1.4 / sqrt( ( double ) ( width*height ) ) );

        // Skip checking the modes with much larger R-D cost than the best mode
        for( int i = 0; i < m_savedNumRdModes[ lfnstIdx ]; i++ )
        {
          if( m_modeCostStore[ lfnstIdx ][ i ] <= thresholdSkipMode * m_bestModeCostStore[ lfnstIdx ] )
          {
            uiRdModeList.push_back( m_savedRdModeList[ lfnstIdx ][ i ] );
#if !JVET_N0217_MATRIX_INTRAPRED
            extendRefList.push_back( m_savedExtendRefList[ lfnstIdx ][ i ] );
#endif
            numModesForFullRD++;
          }
        }
      }
      else //this is necessary because we skip the candidates list calculation, since it was already obtained for the DCT-II. Now we load it
      {
        // Restore the modes to be checked with RD
        numModesForFullRD = m_savedNumRdModes[ lfnstIdx ];
        uiRdModeList.resize( numModesForFullRD );
        std::copy_n( m_savedRdModeList[ lfnstIdx ], m_savedNumRdModes[ lfnstIdx ], uiRdModeList.begin() );
        CandCostList.resize( numModesForFullRD );
#if !JVET_N0217_MATRIX_INTRAPRED
        extendRefList.resize( numModesForFullRD );
        std::copy_n( m_savedExtendRefList[ lfnstIdx ], m_savedNumRdModes[ lfnstIdx ], extendRefList.begin() );
#endif
      }
    }
#endif

    if( nOptionsForISP > 1 ) // we remove the non-MPMs from the ISP lists
    {
#if JVET_N0217_MATRIX_INTRAPRED
      static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListCopyHor = m_rdModeListWithoutMrlHor;
      m_rdModeListWithoutMrlHor.clear();
      static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListCopyVer = m_rdModeListWithoutMrlVer;
      m_rdModeListWithoutMrlVer.clear();
      static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> *listPointerCopy, *listPointer;
#else
      static_vector< uint32_t, FAST_UDI_MAX_RDMODE_NUM > uiRdModeListCopyHor = m_rdModeListWithoutMrlHor;
      m_rdModeListWithoutMrlHor.clear();
      static_vector< uint32_t, FAST_UDI_MAX_RDMODE_NUM > uiRdModeListCopyVer = m_rdModeListWithoutMrlVer;
      m_rdModeListWithoutMrlVer.clear();
      static_vector< uint32_t, FAST_UDI_MAX_RDMODE_NUM > *listPointerCopy, *listPointer;
#endif
      for( int ispOptionIdx = 1; ispOptionIdx < nOptionsForISP; ispOptionIdx++ )
      {
        cu.ispMode = ispOptions[ispOptionIdx];
        //we get the mpm cand list
        const int numMPMs = NUM_MOST_PROBABLE_MODES;
        unsigned  uiPreds[numMPMs];

        pu.multiRefIdx = 0;

        PU::getIntraMPMs( pu, uiPreds );

        //we copy only the ISP MPMs
        listPointerCopy = &( cu.ispMode == HOR_INTRA_SUBPARTITIONS ? uiRdModeListCopyHor : uiRdModeListCopyVer );
        listPointer     = &( cu.ispMode == HOR_INTRA_SUBPARTITIONS ? m_rdModeListWithoutMrlHor : m_rdModeListWithoutMrlVer );
        for( int k = 0; k < listPointerCopy->size(); k++ )
        {
          for( int q = 0; q < numMPMs; q++ )
          {
#if JVET_N0217_MATRIX_INTRAPRED
            if (listPointerCopy->at(k) == ModeInfo( false, 0, cu.ispMode, uiPreds[q] ))
#else
            if( listPointerCopy->at( k ) == uiPreds[q] )
#endif
            {
              listPointer->push_back( listPointerCopy->at( k ) );
              break;
            }
          }
        }
      }
      cu.ispMode = NOT_INTRA_SUBPARTITIONS;
    }


    CHECK( numModesForFullRD != uiRdModeList.size(), "Inconsistent state!" );

    // after this point, don't use numModesForFullRD

    // PBINTRA fast
#if JVET_N0193_LFNST
#if JVET_N0329_IBC_SEARCH_IMP
    if( m_pcEncCfg->getUsePbIntraFast() && !cs.slice->isIntra() && uiRdModeList.size() < numModesAvailable && !cs.slice->getDisableSATDForRD() && ( mtsUsageFlag != 2 || lfnstIdx > 0 ) )
#else
    if( m_pcEncCfg->getUsePbIntraFast() && !cs.slice->isIntra() && uiRdModeList.size() < numModesAvailable && ( mtsUsageFlag != 2 || lfnstIdx > 0 ) )
#endif
#else
#if JVET_N0329_IBC_SEARCH_IMP
    if (m_pcEncCfg->getUsePbIntraFast() && !cs.slice->isIntra() && uiRdModeList.size() < numModesAvailable && !cs.slice->getDisableSATDForRD())
#else
    if( m_pcEncCfg->getUsePbIntraFast() && !cs.slice->isIntra() && uiRdModeList.size() < numModesAvailable )
#endif
#endif
    {
#if JVET_N0217_MATRIX_INTRAPRED
#if JVET_N0193_LFNST
      double pbintraRatio = (lfnstIdx > 0) ? 1.25 : PBINTRA_RATIO;
#endif
      int maxSize = -1;
      const int numHadCand = (testMip ? 2 : 1) * 3;
      for (int k = numHadCand - 1; k >= 0; k--)
      {
#if JVET_N0193_LFNST
        if (CandHadList.size() < (k + 1) || CandHadList[k] > cs.interHad * pbintraRatio) { maxSize = k; }
#else
        if (CandHadList.size() < (k + 1) || CandHadList[k] > cs.interHad * PBINTRA_RATIO) { maxSize = k; }
#endif
      }
      if (maxSize > 0)
      {
        uiRdModeList.resize(std::min<size_t>(uiRdModeList.size(), maxSize));
        if (nOptionsForISP > 1)
        {
          m_rdModeListWithoutMrlHor.resize(std::min<size_t>(m_rdModeListWithoutMrlHor.size(), maxSize));
          m_rdModeListWithoutMrlVer.resize(std::min<size_t>(m_rdModeListWithoutMrlVer.size(), maxSize));
        }
      }
      if (maxSize == 0)
      {
        cs.dist = std::numeric_limits<Distortion>::max();
        cs.interHad = 0;

        //===== reset context models =====
        m_CABACEstimator->getCtx() = SubCtx(Ctx::MipFlag, ctxStartMipFlag);
        m_CABACEstimator->getCtx() = SubCtx(Ctx::MipMode, ctxStartMipMode);
        m_CABACEstimator->getCtx() = SubCtx(Ctx::ISPMode, ctxStartIspMode);
#if JVET_N0185_UNIFIED_MPM
        m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaPlanarFlag, ctxStartPlanarFlag);
#endif
        m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
#if !JVET_N0302_SIMPLFIED_CIIP
        m_CABACEstimator->getCtx() = SubCtx(Ctx::MHIntraPredMode, ctxStartMHIntraMode);
#endif
        m_CABACEstimator->getCtx() = SubCtx(Ctx::MultiRefLineIdx, ctxStartMrlIdx);

#if JVET_N0193_LFNST
        return false;
#else
        return;
#endif
      }
#else
#if JVET_N0193_LFNST
      double pbintraRatio = ( lfnstIdx > 0 ) ? 1.25 : PBINTRA_RATIO;
      if( CandHadList.size() < 3 || CandHadList[ 2 ] > cs.interHad * pbintraRatio )
#else
      if( CandHadList.size() < 3 || CandHadList[2] > cs.interHad * PBINTRA_RATIO )
#endif
      {
        uiRdModeList.resize( std::min<size_t>( uiRdModeList.size(), 2 ) );
#if !JVET_N0217_MATRIX_INTRAPRED
        extendRefList.resize( std::min<size_t>( extendRefList.size(), 2 ) );
#endif
        if( nOptionsForISP > 1 )
        {
          m_rdModeListWithoutMrlHor.resize( std::min<size_t>( m_rdModeListWithoutMrlHor.size(), 2 ) );
          m_rdModeListWithoutMrlVer.resize( std::min<size_t>( m_rdModeListWithoutMrlVer.size(), 2 ) );
        }
      }
#if JVET_N0193_LFNST
      if( CandHadList.size() < 2 || CandHadList[ 1 ] > cs.interHad * pbintraRatio )
#else
      if( CandHadList.size() < 2 || CandHadList[1] > cs.interHad * PBINTRA_RATIO )
#endif
      {
        uiRdModeList.resize( std::min<size_t>( uiRdModeList.size(), 1 ) );
#if !JVET_N0217_MATRIX_INTRAPRED
        extendRefList.resize( std::min<size_t>( extendRefList.size(), 1 ) );
#endif
        if( nOptionsForISP > 1 )
        {
          m_rdModeListWithoutMrlHor.resize( std::min<size_t>( m_rdModeListWithoutMrlHor.size(), 1 ) );
          m_rdModeListWithoutMrlVer.resize( std::min<size_t>( m_rdModeListWithoutMrlVer.size(), 1 ) );
        }
      }
#if JVET_N0193_LFNST
      if( CandHadList.size() < 1 || CandHadList[ 0 ] > cs.interHad * pbintraRatio )
#else
      if( CandHadList.size() < 1 || CandHadList[0] > cs.interHad * PBINTRA_RATIO )
#endif
      {
        cs.dist = std::numeric_limits<Distortion>::max();
        cs.interHad = 0;

        //===== reset context models =====
        m_CABACEstimator->getCtx() = SubCtx(Ctx::ISPMode, ctxStartIspMode);
#if JVET_N0185_UNIFIED_MPM
        m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaPlanarFlag, ctxStartPlanarFlag);
#endif
        m_CABACEstimator->getCtx() = SubCtx( Ctx::IntraLumaMpmFlag, ctxStartIntraMode );
#if !JVET_N0302_SIMPLFIED_CIIP
        m_CABACEstimator->getCtx() = SubCtx( Ctx::MHIntraPredMode, ctxStartMHIntraMode );
#endif
        m_CABACEstimator->getCtx() = SubCtx( Ctx::MultiRefLineIdx, ctxStartMrlIdx );

#if JVET_N0193_LFNST
        return false;
#else
        return;
#endif
      }
#endif
    }

    if ( nOptionsForISP > 1 )
    {
      //we create a single full RD list that includes all intra modes using regular intra, MRL and ISP
      auto* firstIspList  = ispOptions[1] == HOR_INTRA_SUBPARTITIONS ? &m_rdModeListWithoutMrlHor : &m_rdModeListWithoutMrlVer;
      auto* secondIspList = ispOptions[1] == HOR_INTRA_SUBPARTITIONS ? &m_rdModeListWithoutMrlVer : &m_rdModeListWithoutMrlHor;

#if JVET_N0193_LFNST
      if( !sps.getUseLFNST() && m_pcEncCfg->getUseFastISP() )
#else
      if ( m_pcEncCfg->getUseFastISP() )
#endif
      {
#if JVET_N0217_MATRIX_INTRAPRED
        CHECKD( uiRdModeList.size() > CandCostList.size(), "Error: CandCostList size" );
        // find the first non-MRL, non-MIP mode
        int indexFirstMode = int(uiRdModeList.size()) - 1; // default is last mode
        for (int k = 0; k < int(uiRdModeList.size()); k++)
        {
          if (uiRdModeList[k].mRefId == 0 && uiRdModeList[k].mipFlg == false)
          {
            indexFirstMode = k;
            break;
          }
        }
        // move the mode indicated by indexFirstMode to the beginning
        for (int idx = indexFirstMode - 1; idx >= 0; idx--)
        {
          std::swap(uiRdModeList[idx], uiRdModeList[idx + 1]);
          std::swap(CandCostList[idx], CandCostList[idx + 1]);
        }
        //insert all ISP modes after the first non-mrl mode
        uiRdModeList.insert(uiRdModeList.begin() + 1, secondIspList->begin(), secondIspList->end());
        uiRdModeList.insert(uiRdModeList.begin() + 1, firstIspList->begin(), firstIspList->end());
#else
        // find the first non-MRL mode
        size_t indexFirstMode = std::find( extendRefList.begin(), extendRefList.end(), 0 ) - extendRefList.begin();
        // if not found, just take the last mode
        if( indexFirstMode >= extendRefList.size() ) indexFirstMode = extendRefList.size() - 1;
        // move the mode indicated by indexFirstMode to the beginning
        for( int idx = ((int)indexFirstMode) - 1; idx >= 0; idx-- )
        {
          std::swap( extendRefList[idx], extendRefList[idx + 1] );
          std::swap( uiRdModeList [idx], uiRdModeList [idx + 1] );
        }
        //insert all ISP modes after the first non-mrl mode
        uiRdModeList.insert( uiRdModeList.begin() + 1, secondIspList->begin(), secondIspList->end() );
        uiRdModeList.insert( uiRdModeList.begin() + 1, firstIspList->begin() , firstIspList->end()  );

        extendRefList.insert( extendRefList.begin() + 1, secondIspList->size(), MRL_NUM_REF_LINES + ispOptions[2] );
        extendRefList.insert( extendRefList.begin() + 1, firstIspList->size() , MRL_NUM_REF_LINES + ispOptions[1] );
#endif
      }
      else
      {
        //insert all ISP modes at the end of the current list
        uiRdModeList.insert( uiRdModeList.end(), secondIspList->begin(), secondIspList->end() );
        uiRdModeList.insert( uiRdModeList.end(), firstIspList->begin() , firstIspList->end()  );
#if !JVET_N0217_MATRIX_INTRAPRED
        extendRefList.insert( extendRefList.end(), secondIspList->size(), MRL_NUM_REF_LINES + ispOptions[2] );
        extendRefList.insert( extendRefList.end(), firstIspList->size() , MRL_NUM_REF_LINES + ispOptions[1] );
#endif
      }
    }
#if !JVET_N0217_MATRIX_INTRAPRED
    CHECKD(uiRdModeList.size() != extendRefList.size(),"uiRdModeList and extendRefList do not have the same size!");
#endif

    //===== check modes (using r-d costs) =====
#if JVET_N0217_MATRIX_INTRAPRED
    ModeInfo       uiBestPUMode;
#else
    uint32_t       uiBestPUMode  = 0;
    int            bestExtendRef = 0;
#endif
#if JVET_N0413_RDPCM
    int            bestBDPCMMode = 0;
    double         bestCostNonBDPCM = MAX_DOUBLE;
#endif

    CodingStructure *csTemp = m_pTempCS[gp_sizeIdxInfo->idxFrom( cu.lwidth() )][gp_sizeIdxInfo->idxFrom( cu.lheight() )];
    CodingStructure *csBest = m_pBestCS[gp_sizeIdxInfo->idxFrom( cu.lwidth() )][gp_sizeIdxInfo->idxFrom( cu.lheight() )];

    csTemp->slice = cs.slice;
    csBest->slice = cs.slice;
    csTemp->initStructData();
    csBest->initStructData();

#if JVET_N0217_MATRIX_INTRAPRED
    m_bestCostNonMip = MAX_DOUBLE;
#if JVET_N0193_LFNST
    static_vector<int, FAST_UDI_MAX_RDMODE_NUM> rdModeIdxList;
#endif
    if (testMip)
    {
    static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListTemp;
    for( int i = 0; i < uiRdModeList.size(); i++)
    {
      if( !uiRdModeList[i].mipFlg && uiRdModeList[i].ispMod==NOT_INTRA_SUBPARTITIONS )
      {
        uiRdModeListTemp.push_back( uiRdModeList[i] );
#if JVET_N0193_LFNST
        rdModeIdxList.push_back( i );
#endif
      }
    }
    for( int i = 0; i < uiRdModeList.size(); i++)
    {
      if( uiRdModeList[i].mipFlg || uiRdModeList[i].ispMod!=NOT_INTRA_SUBPARTITIONS )
      {
        uiRdModeListTemp.push_back( uiRdModeList[i] );
#if JVET_N0193_LFNST
        rdModeIdxList.push_back( i );
#endif
      }
    }
    for( int i = 0; i < uiRdModeList.size(); i++)
    {
      uiRdModeList[i] = uiRdModeListTemp[i];
    }
    }
#endif
    // just to be sure
    numModesForFullRD = ( int ) uiRdModeList.size();
    PartSplit intraSubPartitionsProcOrder = TU_NO_ISP;
    int       bestNormalIntraModeIndex    = -1;
#if !JVET_N0217_MATRIX_INTRAPRED
    uint8_t   bestIspOption               = NOT_INTRA_SUBPARTITIONS;
#endif
    TUIntraSubPartitioner subTuPartitioner( partitioner );
#if JVET_N0193_LFNST
    if( !cu.ispMode && !cu.mtsFlag )
    {
      m_modeCtrl->setMtsFirstPassNoIspCost( MAX_DOUBLE );
    }
#endif
    bool      ispHorAllZeroCbfs = false, ispVerAllZeroCbfs = false;

#if JVET_N0217_MATRIX_INTRAPRED
#if JVET_N0413_RDPCM
    for (int mode = -2 * int(testBDPCM); mode < (int)uiRdModeList.size(); mode++)
    {
      // set CU/PU to luma prediction mode
      ModeInfo uiOrgMode;
      if ( mode < 0 )
      {
        cu.bdpcmMode = -mode;

        unsigned mpm_pred[NUM_MOST_PROBABLE_MODES];
        PU::getIntraMPMs(pu, mpm_pred);
        uiOrgMode = ModeInfo(false, 0, NOT_INTRA_SUBPARTITIONS, mpm_pred[0]);
        cu.mipFlag                     = uiOrgMode.mipFlg;
        cu.ispMode                     = uiOrgMode.ispMod;
        pu.multiRefIdx                 = uiOrgMode.mRefId;
        pu.intraDir[CHANNEL_TYPE_LUMA] = uiOrgMode.modeId;
      }
      else
      {
        cu.bdpcmMode = 0;
        uiOrgMode = uiRdModeList[mode];
#else
    for (uint32_t uiMode = 0; uiMode < uiRdModeList.size(); uiMode++)
    {
      // set CU/PU to luma prediction mode
      ModeInfo uiOrgMode = uiRdModeList[uiMode];
#endif
      cu.mipFlag                     = uiOrgMode.mipFlg;
      cu.ispMode                     = uiOrgMode.ispMod;
      pu.multiRefIdx                 = uiOrgMode.mRefId;
      pu.intraDir[CHANNEL_TYPE_LUMA] = uiOrgMode.modeId;

      CHECK(cu.mipFlag && pu.multiRefIdx, "Error: combination of MIP and MRL not supported");
#if JVET_N0185_UNIFIED_MPM
      CHECK(pu.multiRefIdx && (pu.intraDir[0] == PLANAR_IDX), "Error: combination of MRL and Planar mode not supported");
#else
      CHECK(pu.multiRefIdx && (pu.intraDir[0] == DC_IDX || pu.intraDir[0] == PLANAR_IDX), "Error: combination of MRL and Planar/DC mode not supported");
#endif
      CHECK(cu.ispMode && cu.mipFlag, "Error: combination of ISP and MIP not supported");
      CHECK(cu.ispMode && pu.multiRefIdx, "Error: combination of ISP and MRL not supported");
#else
#if JVET_N0413_RDPCM
    for( int mode = -2 * int(testBDPCM); mode < numModesForFullRD; mode++ )
#else
    for (uint32_t uiMode = 0; uiMode < numModesForFullRD; uiMode++)
#endif
    {
#if JVET_N0413_RDPCM
      int multiRefIdx = 0;
      uint32_t uiOrgMode;

      if ( mode < 0 )
    {
        cu.bdpcmMode = -mode;
        unsigned mpm_pred[NUM_MOST_PROBABLE_MODES];
        PU::getIntraMPMs(pu, mpm_pred);
        pu.intraDir[0] = mpm_pred[0];
        uiOrgMode = mpm_pred[0];
        cu.ispMode = NOT_INTRA_SUBPARTITIONS;
      }
      else
      {
        cu.bdpcmMode = 0;
        uiOrgMode = uiRdModeList[mode];
#else
      // set luma prediction mode
      uint32_t uiOrgMode = uiRdModeList[uiMode];
#endif
      cu.ispMode = extendRefList[mode] > MRL_NUM_REF_LINES ? extendRefList[mode] - MRL_NUM_REF_LINES : NOT_INTRA_SUBPARTITIONS;
        pu.intraDir[0] = uiOrgMode;

#if !JVET_N0413_RDPCM
        int multiRefIdx = 0;
#endif
        pu.multiRefIdx = multiRefIdx;
#endif
        if( cu.ispMode )
        {
          intraSubPartitionsProcOrder = CU::getISPType( cu, COMPONENT_Y );
          bool tuIsDividedInRows = CU::divideTuInRows( cu );
          if ( ( tuIsDividedInRows && ispHorAllZeroCbfs ) || ( !tuIsDividedInRows && ispVerAllZeroCbfs ) )
          {
            continue;
          }
          if( m_intraModeDiagRatio.at( bestNormalIntraModeIndex ) > 1.25 )
          {
            continue;
          }
          if( ( m_intraModeHorVerRatio.at( bestNormalIntraModeIndex ) > 1.25 && tuIsDividedInRows ) || ( m_intraModeHorVerRatio.at( bestNormalIntraModeIndex ) < 0.8 && !tuIsDividedInRows ) )
          {
            continue;
          }
        }
#if !JVET_N0217_MATRIX_INTRAPRED
        else
        {
          multiRefIdx = extendRefList[mode];
          pu.multiRefIdx = multiRefIdx;
#if !JVET_N0185_UNIFIED_MPM
          CHECK( pu.multiRefIdx && ( pu.intraDir[0] == DC_IDX || pu.intraDir[0] == PLANAR_IDX ), "ERL" );
#else
          CHECK( pu.multiRefIdx && (pu.intraDir[0] == PLANAR_IDX), "ERL" );
#endif
        }
#endif
#if JVET_N0413_RDPCM
      }
#endif

      // set context models
      m_CABACEstimator->getCtx() = ctxStart;

      // determine residual for partition
      cs.initSubStructure( *csTemp, partitioner.chType, cs.area, true );

#if JVET_N0193_LFNST
      bool tmpValidReturn = false;
#endif
      if( cu.ispMode )
      {
#if JVET_N0193_LFNST
        tmpValidReturn = xRecurIntraCodingLumaQT( *csTemp, subTuPartitioner, bestCurrentCost, 0, intraSubPartitionsProcOrder, false,
                                                  mtsCheckRangeFlag, mtsFirstCheckId, mtsLastCheckId, moreProbMTSIdxFirst );
#else
        xRecurIntraCodingLumaQT( *csTemp, subTuPartitioner, bestCurrentCost, 0, intraSubPartitionsProcOrder );
#endif
      }
      else
      {
#if JVET_N0217_MATRIX_INTRAPRED
        if( ! fastMip )
        {
          m_bestCostNonMip = MAX_DOUBLE;
        }
#if JVET_N0193_LFNST
        tmpValidReturn = xRecurIntraCodingLumaQT( *csTemp, partitioner, uiBestPUMode.ispMod ? bestCurrentCost : MAX_DOUBLE, -1, TU_NO_ISP, uiBestPUMode.ispMod,
                                                  mtsCheckRangeFlag, mtsFirstCheckId, mtsLastCheckId, moreProbMTSIdxFirst );
#else
        xRecurIntraCodingLumaQT(*csTemp, partitioner, uiBestPUMode.ispMod ? bestCurrentCost : MAX_DOUBLE, -1, TU_NO_ISP, uiBestPUMode.ispMod);
#endif
#else
#if JVET_N0193_LFNST
        tmpValidReturn = xRecurIntraCodingLumaQT( *csTemp, partitioner, bestIspOption ? bestCurrentCost : MAX_DOUBLE, -1, TU_NO_ISP, bestIspOption,
                                                  mtsCheckRangeFlag, mtsFirstCheckId, mtsLastCheckId, moreProbMTSIdxFirst );
#else
        xRecurIntraCodingLumaQT( *csTemp, partitioner, bestIspOption ? bestCurrentCost : MAX_DOUBLE, -1, TU_NO_ISP, bestIspOption );
#endif
#endif
      }

      if( cu.ispMode && !csTemp->cus[0]->firstTU->cbf[COMPONENT_Y] )
      {
#if JVET_N0193_LFNST
        if( !sps.getUseLFNST() )
        {
#endif
          if ( cu.ispMode == HOR_INTRA_SUBPARTITIONS )
          {
            ispHorAllZeroCbfs |= ( m_pcEncCfg->getUseFastISP() && csTemp->tus[0]->lheight() > 2 && csTemp->cost >= bestCurrentCost );
          }
          else
          {
            ispVerAllZeroCbfs |= ( m_pcEncCfg->getUseFastISP() && csTemp->tus[0]->lwidth() > 2 && csTemp->cost >= bestCurrentCost );
          }
#if JVET_N0193_LFNST
        }
#endif
        csTemp->cost = MAX_DOUBLE;
        csTemp->costDbOffset = 0;
#if JVET_N0193_LFNST
        tmpValidReturn = false;
#endif
      }
#if JVET_N0193_LFNST
      validReturn |= tmpValidReturn;

#if JVET_N0413_RDPCM
      if( sps.getUseLFNST() && mtsUsageFlag == 1 && !cu.ispMode && mode >= 0 )
      {
#if JVET_N0217_MATRIX_INTRAPRED
        m_modeCostStore[ lfnstIdx ][ testMip ? rdModeIdxList[ mode ] : mode ] = tmpValidReturn ? csTemp->cost : ( MAX_DOUBLE / 2.0 ); //(MAX_DOUBLE / 2.0) ??
#else
        m_modeCostStore[ lfnstIdx ][ mode ] = tmpValidReturn ? csTemp->cost : ( MAX_DOUBLE / 2.0 ); //(MAX_DOUBLE / 2.0) ??
#endif
#else
      if( sps.getUseLFNST() && mtsUsageFlag == 1 && !cu.ispMode )
      {
#if JVET_N0217_MATRIX_INTRAPRED
        m_modeCostStore[ lfnstIdx ][ testMip ? rdModeIdxList[ uiMode ] : uiMode ] = tmpValidReturn ? csTemp->cost : ( MAX_DOUBLE / 2.0 ); //(MAX_DOUBLE / 2.0) ??
#else
        m_modeCostStore[ lfnstIdx ][ uiMode ] = tmpValidReturn ? csTemp->cost : ( MAX_DOUBLE / 2.0 ); //(MAX_DOUBLE / 2.0) ??
#endif
#endif
      }
#endif


#if JVET_N0217_MATRIX_INTRAPRED
      DTRACE( g_trace_ctx, D_INTRA_COST, "IntraCost T %f (%d) \n", csTemp->cost, uiOrgMode.modeId );
#else
      DTRACE( g_trace_ctx, D_INTRA_COST, "IntraCost T %f (%d) \n", csTemp->cost, uiOrgMode );
#endif


#if JVET_N0193_LFNST
      if( tmpValidReturn )
      {
#endif
        // check r-d cost
        if( csTemp->cost < csBest->cost )
        {
          std::swap( csTemp, csBest );

          uiBestPUMode  = uiOrgMode;
#if !JVET_N0217_MATRIX_INTRAPRED
          bestExtendRef = multiRefIdx;
          bestIspOption = cu.ispMode;
#endif
#if JVET_N0413_RDPCM
          bestBDPCMMode = cu.bdpcmMode;
#endif
#if JVET_N0193_LFNST
          if( sps.getUseLFNST() && mtsUsageFlag == 1 && !cu.ispMode )
          {
            m_bestModeCostStore[ lfnstIdx ] = csBest->cost; //cs.cost;
          }
#endif
          if( csBest->cost < bestCurrentCost )
          {
            bestCurrentCost = csBest->cost;
          }
#if !JVET_N0413_RDPCM
          if( !cu.ispMode )
          {
            bestNormalIntraModeIndex = uiMode;
          }
#endif
#if JVET_N0193_LFNST
          if( !cu.ispMode && !cu.mtsFlag )
          {
            m_modeCtrl->setMtsFirstPassNoIspCost( csBest->cost );
          }
#endif
        }
#if JVET_N0413_RDPCM
        if( !cu.ispMode && !cu.bdpcmMode && csBest->cost < bestCostNonBDPCM )
        {
          bestCostNonBDPCM = csBest->cost;
          bestNormalIntraModeIndex = mode;
        }
#endif
#if JVET_N0193_LFNST
      }
#endif

      csTemp->releaseIntermediateData();
    } // Mode loop
#if JVET_N0217_MATRIX_INTRAPRED
    cu.ispMode = uiBestPUMode.ispMod;
#else
    cu.ispMode = bestIspOption;
#endif

#if JVET_N0193_LFNST
    if( validReturn )
    {
#endif
      cs.useSubStructure( *csBest, partitioner.chType, pu.singleChan( CHANNEL_TYPE_LUMA ), true, true, keepResi, keepResi );
#if JVET_N0193_LFNST
    }
#endif
    csBest->releaseIntermediateData();
#if JVET_N0193_LFNST
    if( validReturn )
    {
#endif
      //=== update PU data ====
#if JVET_N0217_MATRIX_INTRAPRED
      cu.mipFlag = uiBestPUMode.mipFlg;
      pu.multiRefIdx = uiBestPUMode.mRefId;
      pu.intraDir[ CHANNEL_TYPE_LUMA ] = uiBestPUMode.modeId;
#else
      pu.intraDir[ 0 ] = uiBestPUMode;
      pu.multiRefIdx = bestExtendRef;
#endif
#if JVET_N0413_RDPCM
      cu.bdpcmMode = bestBDPCMMode;
#endif
#if JVET_N0193_LFNST
    }
#endif
  }

  //===== reset context models =====
  m_CABACEstimator->getCtx() = ctxStart;

#if JVET_N0193_LFNST
  return validReturn;
#endif
}

void IntraSearch::estIntraPredChromaQT( CodingUnit &cu, Partitioner &partitioner, const double maxCostAllowed )
{
  const ChromaFormat format   = cu.chromaFormat;
  const uint32_t    numberValidComponents = getNumberValidComponents(format);
  CodingStructure &cs = *cu.cs;
  const TempCtx ctxStart  ( m_CtxCache, m_CABACEstimator->getCtx() );

  cs.setDecomp( cs.area.Cb(), false );

  double    bestCostSoFar = maxCostAllowed;
  bool      lumaUsesISP   = !CS::isDualITree( *cu.cs ) && cu.ispMode;
  PartSplit ispType       = lumaUsesISP ? CU::getISPType( cu, COMPONENT_Y ) : TU_NO_ISP;
  CHECK( cu.ispMode && bestCostSoFar < 0, "bestCostSoFar must be positive!" );

  auto &pu = *cu.firstPU;

  {
    uint32_t       uiBestMode = 0;
    Distortion uiBestDist = 0;
    double     dBestCost = MAX_DOUBLE;

    //----- init mode list ----
    {
      uint32_t  uiMinMode = 0;
      uint32_t  uiMaxMode = NUM_CHROMA_MODE;

      //----- check chroma modes -----
      uint32_t chromaCandModes[ NUM_CHROMA_MODE ];
      PU::getIntraChromaCandModes( pu, chromaCandModes );

      // create a temporary CS
      CodingStructure &saveCS = *m_pSaveCS[0];
      saveCS.pcv      = cs.pcv;
      saveCS.picture  = cs.picture;
      saveCS.area.repositionTo( cs.area );
      saveCS.clearTUs();

      if( !CS::isDualITree( cs ) && cu.ispMode )
      {
        saveCS.clearCUs();
        saveCS.clearPUs();
      }

      if( CS::isDualITree( cs ) )
      {
        if( partitioner.canSplit( TU_MAX_TR_SPLIT, cs ) )
        {
          partitioner.splitCurrArea( TU_MAX_TR_SPLIT, cs );

          do
          {
            cs.addTU( CS::getArea( cs, partitioner.currArea(), partitioner.chType ), partitioner.chType ).depth = partitioner.currTrDepth;
          } while( partitioner.nextPart( cs ) );

          partitioner.exitCurrSplit();
        }
        else
        cs.addTU( CS::getArea( cs, partitioner.currArea(), partitioner.chType ), partitioner.chType );
      }

      std::vector<TransformUnit*> orgTUs;

      if( lumaUsesISP )
      {
        CodingUnit& auxCU = saveCS.addCU( cu, partitioner.chType );
        auxCU.ispMode = cu.ispMode;
        saveCS.sps = cu.cs->sps;
        saveCS.addPU( *cu.firstPU, partitioner.chType );
      }


      // create a store for the TUs
      for( const auto &ptu : cs.tus )
      {
        // for split TUs in HEVC, add the TUs without Chroma parts for correct setting of Cbfs
        if( lumaUsesISP || pu.contains( *ptu, CHANNEL_TYPE_CHROMA ) )
        {
          saveCS.addTU( *ptu, partitioner.chType );
          orgTUs.push_back( ptu );
        }
      }
      if( lumaUsesISP )
      {
        saveCS.clearCUs();
      }
      // SATD pre-selecting.
      int satdModeList[NUM_CHROMA_MODE];
      int64_t satdSortedCost[NUM_CHROMA_MODE];
      for (int i = 0; i < NUM_CHROMA_MODE; i++)
      {
        satdSortedCost[i] = 0; // for the mode not pre-select by SATD, do RDO by default, so set the initial value 0.
        satdModeList[i] = 0;
      }
      bool modeIsEnable[NUM_INTRA_MODE + 1]; // use intra mode idx to check whether enable
      for (int i = 0; i < NUM_INTRA_MODE + 1; i++)
      {
        modeIsEnable[i] = 1;
      }

      DistParam distParam;
#if JVET_N0329_IBC_SEARCH_IMP
      const bool useHadamard = !cu.transQuantBypass;
#else
      const bool useHadamard = true;
#endif
      pu.intraDir[1] = MDLM_L_IDX; // temporary assigned, just to indicate this is a MDLM mode. for luma down-sampling operation.

      initIntraPatternChType(cu, pu.Cb());
      initIntraPatternChType(cu, pu.Cr());
      xGetLumaRecPixels(pu, pu.Cb());

      for (int idx = uiMinMode; idx <= uiMaxMode - 1; idx++)
      {
        int mode = chromaCandModes[idx];
        satdModeList[idx] = mode;
        if (PU::isLMCMode(mode) && !PU::isLMCModeEnabled(pu, mode))
        {
          continue;
        }
        if ((mode == LM_CHROMA_IDX) || (mode == PLANAR_IDX) || (mode == DM_CHROMA_IDX)) // only pre-check regular modes and MDLM modes, not including DM ,Planar, and LM
        {
          continue;
        }
        pu.intraDir[1] = mode; // temporary assigned, for SATD checking.

        int64_t sad = 0;
        CodingStructure& cs = *(pu.cs);

        CompArea areaCb = pu.Cb();
        PelBuf orgCb = cs.getOrgBuf(areaCb);
        PelBuf predCb = cs.getPredBuf(areaCb);

        m_pcRdCost->setDistParam(distParam, orgCb, predCb, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, useHadamard);
        distParam.applyWeight = false;

        if (PU::isLMCMode(mode))
        {
          predIntraChromaLM(COMPONENT_Cb, predCb, pu, areaCb, mode);
        }
        else
        {
          initPredIntraParams(pu, pu.Cb(), *pu.cs->sps);
          predIntraAng(COMPONENT_Cb, predCb, pu);
        }

        sad += distParam.distFunc(distParam);

        CompArea areaCr = pu.Cr();
        PelBuf orgCr = cs.getOrgBuf(areaCr);
        PelBuf predCr = cs.getPredBuf(areaCr);

        m_pcRdCost->setDistParam(distParam, orgCr, predCr, pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, useHadamard);
        distParam.applyWeight = false;

        if (PU::isLMCMode(mode))
        {
          predIntraChromaLM(COMPONENT_Cr, predCr, pu, areaCr, mode);
        }
        else
        {
          initPredIntraParams(pu, pu.Cr(), *pu.cs->sps);
          predIntraAng(COMPONENT_Cr, predCr, pu);
        }
        sad += distParam.distFunc(distParam);
        satdSortedCost[idx] = sad;
      }
      // sort the mode based on the cost from small to large.
      int tempIdx = 0;
      int64_t tempCost = 0;
      for (int i = uiMinMode; i <= uiMaxMode - 1; i++)
      {
        for (int j = i + 1; j <= uiMaxMode - 1; j++)
        {
          if (satdSortedCost[j] < satdSortedCost[i])
          {
            tempIdx = satdModeList[i];
            satdModeList[i] = satdModeList[j];
            satdModeList[j] = tempIdx;

            tempCost = satdSortedCost[i];
            satdSortedCost[i] = satdSortedCost[j];
            satdSortedCost[j] = tempCost;

          }
        }
      }
      int reducedModeNumber = 2; // reduce the number of chroma modes
      for (int i = 0; i < reducedModeNumber; i++)
      {
        modeIsEnable[satdModeList[uiMaxMode - 1 - i]] = 0; // disable the last reducedModeNumber modes
      }

      // save the dist
      Distortion baseDist = cs.dist;

      for (uint32_t uiMode = uiMinMode; uiMode < uiMaxMode; uiMode++)
      {
        const int chromaIntraMode = chromaCandModes[uiMode];
        if( PU::isLMCMode( chromaIntraMode ) && ! PU::isLMCModeEnabled( pu, chromaIntraMode ) )
        {
          continue;
        }
        if (!modeIsEnable[chromaIntraMode] && PU::isLMCModeEnabled(pu, chromaIntraMode)) // when CCLM is disable, then MDLM is disable. not use satd checking
        {
          continue;
        }
        cs.setDecomp( pu.Cb(), false );
        cs.dist = baseDist;
        //----- restore context models -----
        m_CABACEstimator->getCtx() = ctxStart;

        //----- chroma coding -----
        pu.intraDir[1] = chromaIntraMode;

        xRecurIntraChromaCodingQT( cs, partitioner, bestCostSoFar, ispType );
        if( lumaUsesISP && cs.dist == MAX_UINT )
        {
          continue;
        }

        if (cs.pps->getUseTransformSkip())
        {
          m_CABACEstimator->getCtx() = ctxStart;
        }

        uint64_t fracBits   = xGetIntraFracBitsQT( cs, partitioner, false, true, -1, ispType );
        Distortion uiDist = cs.dist;
        double    dCost   = m_pcRdCost->calcRdCost( fracBits, uiDist - baseDist );

        //----- compare -----
        if( dCost < dBestCost )
        {
          if( lumaUsesISP && dCost < bestCostSoFar )
          {
            bestCostSoFar = dCost;
          }
          for( uint32_t i = getFirstComponentOfChannel( CHANNEL_TYPE_CHROMA ); i < numberValidComponents; i++ )
          {
            const CompArea &area = pu.blocks[i];

            saveCS.getRecoBuf     ( area ).copyFrom( cs.getRecoBuf   ( area ) );
#if KEEP_PRED_AND_RESI_SIGNALS
            saveCS.getPredBuf     ( area ).copyFrom( cs.getPredBuf   ( area ) );
            saveCS.getResiBuf     ( area ).copyFrom( cs.getResiBuf   ( area ) );
#endif
            saveCS.getPredBuf     ( area ).copyFrom( cs.getPredBuf   (area ) );
            cs.picture->getPredBuf( area ).copyFrom( cs.getPredBuf   (area ) );
            cs.picture->getRecoBuf( area ).copyFrom( cs.getRecoBuf( area ) );

            for( uint32_t j = 0; j < saveCS.tus.size(); j++ )
            {
              saveCS.tus[j]->copyComponentFrom( *orgTUs[j], area.compID );
            }
          }

          dBestCost  = dCost;
          uiBestDist = uiDist;
          uiBestMode = chromaIntraMode;
        }
      }

      for( uint32_t i = getFirstComponentOfChannel( CHANNEL_TYPE_CHROMA ); i < numberValidComponents; i++ )
      {
        const CompArea &area = pu.blocks[i];

        cs.getRecoBuf         ( area ).copyFrom( saveCS.getRecoBuf( area ) );
#if KEEP_PRED_AND_RESI_SIGNALS
        cs.getPredBuf         ( area ).copyFrom( saveCS.getPredBuf( area ) );
        cs.getResiBuf         ( area ).copyFrom( saveCS.getResiBuf( area ) );
#endif
        cs.getPredBuf         ( area ).copyFrom( saveCS.getPredBuf( area ) );
        cs.picture->getPredBuf( area ).copyFrom( cs.getPredBuf    ( area ) );

        cs.picture->getRecoBuf( area ).copyFrom( cs.    getRecoBuf( area ) );

        for( uint32_t j = 0; j < saveCS.tus.size(); j++ )
        {
          orgTUs[ j ]->copyComponentFrom( *saveCS.tus[ j ], area.compID );
        }
      }
    }

    pu.intraDir[1] = uiBestMode;
    cs.dist        = uiBestDist;
  }

  //----- restore context models -----
  m_CABACEstimator->getCtx() = ctxStart;
  if( lumaUsesISP && bestCostSoFar >= maxCostAllowed )
  {
    cu.ispMode = 0;
  }
}

void IntraSearch::IPCMSearch(CodingStructure &cs, Partitioner& partitioner)
{
  ComponentID compStr = (CS::isDualITree(cs) && !isLuma(partitioner.chType)) ? COMPONENT_Cb: COMPONENT_Y;
  ComponentID compEnd = (CS::isDualITree(cs) && isLuma(partitioner.chType)) ? COMPONENT_Y : COMPONENT_Cr;
  for( ComponentID compID = compStr; compID <= compEnd; compID = ComponentID(compID+1) )
  {

    xEncPCM(cs, partitioner, compID);
  }

  cs.getPredBuf().fill(0);
  cs.getResiBuf().fill(0);
  cs.getOrgResiBuf().fill(0);