IntraSearch.cpp

          {
            bool isLineOutsideCtu =
              ((cu.block(COMPONENT_Y).y) % ((cu.cs->sps)->getMaxCUWidth()) <= MULTI_REF_LINE_IDX[mrlIdx]) ? true
                                                                                                          : false;
            checkLineOutsideCtu[mrlIdx-1] = isLineOutsideCtu;
          }
          if (checkLineOutsideCtu[0]) 
          {
            numOfPassesExtendRef = 1;
          }
          else
          {
            if (checkLineOutsideCtu[1] && !checkLineOutsideCtu[0])
            {
              numOfPassesExtendRef = 2;
            }
          }
        }
#else
        int  numOfPassesExtendRef = ((!sps.getUseMRL() || isFirstLineOfCtu) ? 1 : MRL_NUM_REF_LINES);
#endif
        for (int mRefNum = 1; mRefNum < numOfPassesExtendRef; mRefNum++)
        {
          int multiRefIdx = MULTI_REF_LINE_IDX[mRefNum];
          uiRdModeList.push_back( ModeInfo( false, false, multiRefIdx, NOT_INTRA_SUBPARTITIONS, TIMD_IDX ) );
          numNonISPModes++;
        }
      }
    }
#endif

    if ( testISP )
    {
      // we reserve positions for ISP in the common full RD list
      const int maxNumRDModesISP = sps.getUseLFNST() ? 16 * NUM_LFNST_NUM_PER_SET : 16;
      m_curIspLfnstIdx = 0;
      for (int i = 0; i < maxNumRDModesISP; i++)
      {
        uiRdModeList.push_back( ModeInfo( false, false, 0, INTRA_SUBPARTITIONS_RESERVED, 0 ) );
      }
    }
#if JVET_W0123_TIMD_FUSION
    if (isTimdValid && sps.getUseISP() && CU::canUseISP(width, height, cu.cs->sps->getMaxTbSize()) && lfnstIdx == 0 && !cu.mtsFlag)
    {
      uiRdModeList.push_back( ModeInfo( false, false, 0, HOR_INTRA_SUBPARTITIONS, TIMD_IDX ) );
      uiRdModeList.push_back( ModeInfo( false, false, 0, VER_INTRA_SUBPARTITIONS, TIMD_IDX ) );
    }
#endif
    //===== check modes (using r-d costs) =====
    ModeInfo       uiBestPUMode;
    int            bestBDPCMMode = 0;
    double         bestCostNonBDPCM = MAX_DOUBLE;
#if INTRA_TRANS_ENC_OPT
    double         bestISPCostTested = MAX_DOUBLE;
    ISPType        bestISPModeTested = NOT_INTRA_SUBPARTITIONS;
#endif
    CodingStructure *csTemp = m_pTempCS[gp_sizeIdxInfo->idxFrom( cu.lwidth() )][gp_sizeIdxInfo->idxFrom( cu.lheight() )];
    CodingStructure *csBest = m_pBestCS[gp_sizeIdxInfo->idxFrom( cu.lwidth() )][gp_sizeIdxInfo->idxFrom( cu.lheight() )];

    csTemp->slice = cs.slice;
    csBest->slice = cs.slice;
    csTemp->initStructData();
    csBest->initStructData();
#if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
    csTemp->picture = cs.picture;
    csBest->picture = cs.picture;
#endif
    // just to be sure
    numModesForFullRD = ( int ) uiRdModeList.size();
    TUIntraSubPartitioner subTuPartitioner( partitioner );
    if ( testISP )
    {
      m_modeCtrl->setIspCost( MAX_DOUBLE );
      m_modeCtrl->setMtsFirstPassNoIspCost( MAX_DOUBLE );
    }
    int bestLfnstIdx = cu.lfnstIdx;

    for (int mode = isSecondColorSpace ? 0 : -2 * int(testBDPCM); mode < (int)uiRdModeList.size(); mode++)
    {
      // set CU/PU to luma prediction mode
      ModeInfo uiOrgMode;
      if (sps.getUseColorTrans() && !m_pcEncCfg->getRGBFormatFlag() && isSecondColorSpace && mode)
      {
        continue;
      }

      if (mode < 0 || (isSecondColorSpace && m_savedBDPCMModeFirstColorSpace[m_savedRdModeIdx][mode]))
      {
        cu.bdpcmMode = mode < 0 ? -mode : m_savedBDPCMModeFirstColorSpace[m_savedRdModeIdx][mode];
        uiOrgMode = ModeInfo( false, false, 0, NOT_INTRA_SUBPARTITIONS, cu.bdpcmMode == 2 ? VER_IDX : HOR_IDX );
      }
      else
      {
        cu.bdpcmMode = 0;
        uiOrgMode = uiRdModeList[mode];
      }

      if (!cu.bdpcmMode && uiRdModeList[mode].ispMod == INTRA_SUBPARTITIONS_RESERVED)
      {
        if (mode == numNonISPModes)   // the list needs to be sorted only once
        {
          if (m_pcEncCfg->getUseFastISP())
          {
#if JVET_W0123_TIMD_FUSION
            if (bestTimdMode)
            {
              m_modeCtrl->setBestPredModeDCT2(MAP131TO67(uiBestPUMode.modeId));
            }
            else
            {
              m_modeCtrl->setBestPredModeDCT2(uiBestPUMode.modeId);
            }
#else
            m_modeCtrl->setBestPredModeDCT2(uiBestPUMode.modeId);
#endif
          }
#if JVET_W0123_TIMD_FUSION
          ModeInfo tempBestPUMode = uiBestPUMode;
          if (bestTimdMode)
          {
            tempBestPUMode.modeId = MAP131TO67(tempBestPUMode.modeId);
          }
          if (!xSortISPCandList(bestCurrentCost, csBest->cost, tempBestPUMode))
#else
          if (!xSortISPCandList(bestCurrentCost, csBest->cost, uiBestPUMode))
#endif
          {
            break;
          }
        }
        xGetNextISPMode(uiRdModeList[mode], (mode > 0 ? &uiRdModeList[mode - 1] : nullptr), Size(width, height));
        if (uiRdModeList[mode].ispMod == INTRA_SUBPARTITIONS_RESERVED)
        {
          continue;
        }
        cu.lfnstIdx = m_curIspLfnstIdx;
        uiOrgMode   = uiRdModeList[mode];
      }
#if ENABLE_DIMD && INTRA_TRANS_ENC_OPT
      if ((m_pcEncCfg->getIntraPeriod() == 1) && cu.slice->getSPS()->getUseDimd() && mode >= 0 && !cu.dimdBlending && uiOrgMode.ispMod == 0 && uiOrgMode.mRefId == 0 && uiOrgMode.modeId != TIMD_IDX && uiOrgMode.modeId != DIMD_IDX)
      {
        bool modeDuplicated = (uiOrgMode.modeId == cu.dimdMode);
        if (modeDuplicated)
        {
          m_modeCostStore[lfnstIdx][mode] = MAX_DOUBLE / 2.0;
          continue;
        }
      }
#endif	  
#if ENABLE_DIMD
      cu.dimd = false;
      if( mode >= 0 && uiOrgMode.modeId == DIMD_IDX ) /*to check*/
      {
        uiOrgMode.modeId = cu.dimdMode;
        cu.dimd = true;
      }
#endif
#if JVET_AC0105_DIRECTIONAL_PLANAR
      cu.plIdx = 0;
      if (mode >= 0 && uiOrgMode.modeId == PL_HOR_IDX)
      {
        if (cu.ispMode)
        {
          continue;
        }
        uiOrgMode.modeId = PLANAR_IDX;
        cu.plIdx         = 1;
      }
      if (mode >= 0 && uiOrgMode.modeId == PL_VER_IDX)
      {
        if (cu.ispMode)
        {
          continue;
        }
        uiOrgMode.modeId = PLANAR_IDX;
        cu.plIdx         = 2;
      }
#endif
#if JVET_AB0155_SGPM
      cu.sgpm = uiOrgMode.sgpmFlag;
      if (cu.sgpm)
      {
        uiOrgMode.modeId = uiOrgMode.sgpmMode0;
        cu.sgpmSplitDir  = uiOrgMode.sgpmSplitDir;
        cu.sgpmMode0     = uiOrgMode.sgpmMode0;
        cu.sgpmMode1     = uiOrgMode.sgpmMode1;
        cu.sgpmIdx       = uiOrgMode.sgpmIdx;
        pu.intraDir1[CHANNEL_TYPE_LUMA] = uiOrgMode.sgpmMode1;
      }
#endif
#if JVET_V0130_INTRA_TMP
      cu.tmpFlag = uiOrgMode.tmpFlag;
#if JVET_AD0086_ENHANCED_INTRA_TMP
      cu.tmpIdx = uiOrgMode.tmpIdx;
      cu.tmpFusionFlag = uiOrgMode.tmpFusionFlag;
      cu.tmpFlmFlag = uiOrgMode.tmpFlmFlag;
      if (cu.tmpFlag)
      {
        cu.tmpIsSubPel = uiOrgMode.tmpIsSubPel;
        cu.tmpSubPelIdx = uiOrgMode.tmpSubPelIdx;
      }
#endif
#if JVET_W0103_INTRA_MTS
#if !JVET_AC0115_INTRA_TMP_DIMD_MTS_LFNST
      if (cu.tmpFlag && cu.mtsFlag) continue;
#endif
#endif
#endif
      cu.mipFlag                     = uiOrgMode.mipFlg;
      pu.mipTransposedFlag           = uiOrgMode.mipTrFlg;
      cu.ispMode                     = uiOrgMode.ispMod;
      pu.multiRefIdx                 = uiOrgMode.mRefId;
      pu.intraDir[CHANNEL_TYPE_LUMA] = uiOrgMode.modeId;
#if JVET_W0123_TIMD_FUSION
      cu.timd = false;
      if (mode >= 0 && uiOrgMode.modeId == TIMD_IDX)
      {
        if (cu.ispMode)
        {
          cu.lfnstIdx = lfnstIdx;
#if INTRA_TRANS_ENC_OPT
          if ((m_pcEncCfg->getIntraPeriod() == 1) && ((bestISPModeTested == HOR_INTRA_SUBPARTITIONS) || (bestISPModeTested == VER_INTRA_SUBPARTITIONS)))
          {
            if (cu.ispMode != bestISPModeTested)
            {
              continue;
            }
          }
#endif
          if (cu.ispMode == VER_INTRA_SUBPARTITIONS && uiBestPUMode.ispMod == 0 && !bestTimdMode)
          {
            continue;
          }
        }
#if INTRA_TRANS_ENC_OPT
        else if (m_skipTimdLfnstMtsPass)
        {
          CHECK(!cu.lfnstIdx && !cu.mtsFlag, "invalid logic");
          continue;
        }
#endif
        uiOrgMode.modeId = cu.timdMode;
        pu.intraDir[CHANNEL_TYPE_LUMA] = uiOrgMode.modeId;
        cu.timd = true;
      }
#endif
#if JVET_AB0157_TMRL
      cu.tmrlFlag = false;
      if (uiOrgMode.mRefId >= MAX_REF_LINE_IDX)
      {
        int tmrlListIdx = uiOrgMode.mRefId - MAX_REF_LINE_IDX;
        cu.tmrlListIdx = tmrlListIdx;
        pu.multiRefIdx = m_tmrlList[tmrlListIdx].multiRefIdx;
        pu.intraDir[0] = m_tmrlList[tmrlListIdx].intraDir;
        cu.tmrlFlag = true;
      }
#endif

      CHECK(cu.mipFlag && pu.multiRefIdx, "Error: combination of MIP and MRL not supported");
#if JVET_W0123_TIMD_FUSION
      if (!cu.timd)
      {
#endif
        CHECK(pu.multiRefIdx && (pu.intraDir[0] == PLANAR_IDX),
              "Error: combination of MRL and Planar mode not supported");
#if JVET_W0123_TIMD_FUSION
      }
#endif
      CHECK(cu.ispMode && cu.mipFlag, "Error: combination of ISP and MIP not supported");
      CHECK(cu.ispMode && pu.multiRefIdx, "Error: combination of ISP and MRL not supported");
      CHECK(cu.ispMode&& cu.colorTransform, "Error: combination of ISP and ACT not supported");
#if JVET_V0130_INTRA_TMP
      CHECK( cu.mipFlag && cu.tmpFlag, "Error: combination of MIP and TPM not supported" );
      CHECK( cu.tmpFlag && cu.ispMode, "Error: combination of TPM and ISP not supported" );
      CHECK( cu.tmpFlag && pu.multiRefIdx, "Error: combination of TPM and MRL not supported" );
#endif
#if JVET_AB0155_SGPM
#if JVET_V0130_INTRA_TMP
      CHECK(cu.sgpm && cu.tmpFlag, "Error: combination of SGPM and TPM not supported");
#endif
      CHECK(cu.sgpm && cu.ispMode, "Error: combination of SGPM and ISP not supported");
      CHECK(cu.sgpm && pu.multiRefIdx, "Error: combination of SGPM and MRL not supported");
      CHECK(cu.sgpm && cu.mipFlag, "Error: combination of SGPM and MIP not supported");
#if JVET_W0123_TIMD_FUSION
      CHECK(cu.sgpm && cu.timd, "Error: combination of SGPM and TIMD not supported");
#endif
#if ENABLE_DIMD
      CHECK(cu.sgpm && cu.dimd, "Error: combination of SGPM and DIMD not supported");
#endif
      CHECK(cu.sgpm && cu.bdpcmMode, "Error: combination of SGPM and BDPCM not supported");
#endif

#if ENABLE_DIMD && JVET_V0087_DIMD_NO_ISP
      CHECK(cu.ispMode && cu.dimd, "Error: combination of ISP and DIMD not supported");
#endif
      pu.intraDir[CHANNEL_TYPE_CHROMA] = cu.colorTransform ? DM_CHROMA_IDX : pu.intraDir[CHANNEL_TYPE_CHROMA];
#if JVET_Y0142_ADAPT_INTRA_MTS
      if (cu.mtsFlag)
      {
        int mtsModeIdx = -1;
        for (int i = 0; i < m_modesForMTS.size(); i++)
        {
          if (uiOrgMode == m_modesForMTS[i])
          {
            mtsModeIdx = i;
            break;
          }
        }
        if (mtsModeIdx == -1)
        {
          mtsModeIdx = 0;
        }
        CHECK(mtsModeIdx == -1, "mtsModeIdx==-1");
        m_coeffAbsSumDCT2 = (m_modesForMTS.size() == 0) ? 10 : m_modesCoeffAbsSumDCT2[mtsModeIdx];
      }
#endif
      // set context models
      m_CABACEstimator->getCtx() = ctxStart;

      // determine residual for partition
      cs.initSubStructure( *csTemp, partitioner.chType, cs.area, true );

      bool tmpValidReturn = false;
      if( cu.ispMode )
      {
        if ( m_pcEncCfg->getUseFastISP() )
        {
          m_modeCtrl->setISPWasTested(true);
        }
        tmpValidReturn = xIntraCodingLumaISP(*csTemp, subTuPartitioner, bestCurrentCost);
        if (csTemp->tus.size() == 0)
        {
          // no TUs were coded
          csTemp->cost = MAX_DOUBLE;
          continue;
        }
        // we save the data for future tests
#if JVET_W0123_TIMD_FUSION
        if (!cu.timd)
        {
#endif
        m_ispTestedModes[m_curIspLfnstIdx].setModeResults((ISPType)cu.ispMode, (int)uiOrgMode.modeId, (int)csTemp->tus.size(), csTemp->cus[0]->firstTU->cbf[COMPONENT_Y] ? csTemp->cost : MAX_DOUBLE, csBest->cost);
#if JVET_W0123_TIMD_FUSION
        }
#endif
        csTemp->cost = !tmpValidReturn ? MAX_DOUBLE : csTemp->cost;
#if INTRA_TRANS_ENC_OPT
        if (csTemp->cost < bestISPCostTested)
        {
          bestISPCostTested = csTemp->cost;
          bestISPModeTested = (ISPType)cu.ispMode;
        }
#endif
      }
      else
      {
        if (cu.colorTransform)
        {
          tmpValidReturn = xRecurIntraCodingACTQT(*csTemp, partitioner, mtsCheckRangeFlag, mtsFirstCheckId, mtsLastCheckId, moreProbMTSIdxFirst);
        }
        else
        {
          tmpValidReturn = xRecurIntraCodingLumaQT(
            *csTemp, partitioner, uiBestPUMode.ispMod ? bestCurrentCost : MAX_DOUBLE, -1, TU_NO_ISP,
            uiBestPUMode.ispMod, mtsCheckRangeFlag, mtsFirstCheckId, mtsLastCheckId, moreProbMTSIdxFirst);
        }
      }
#if JVET_Y0142_ADAPT_INTRA_MTS
#if JVET_W0123_TIMD_FUSION
      if (!cu.mtsFlag && !lfnstIdx && mode < numNonISPModes && !(cu.timd && pu.multiRefIdx))
#else
      if( !cu.mtsFlag && !lfnstIdx && mode < numNonISPModes && !pu.multiRefIdx )
#endif
      {
        m_modesForMTS.push_back(uiOrgMode);
        m_modesCoeffAbsSumDCT2.push_back(m_coeffAbsSumDCT2);
      }
#endif
#if JVET_V0130_INTRA_TMP
#if JVET_W0123_TIMD_FUSION
      if (!cu.ispMode && !cu.mtsFlag && !cu.lfnstIdx && !cu.bdpcmMode && !pu.multiRefIdx && !cu.mipFlag && !cu.tmpFlag && testISP && !cu.timd
#if JVET_AB0155_SGPM
        && !cu.sgpm
#endif
        )
#else
      if( !cu.ispMode && !cu.mtsFlag && !cu.lfnstIdx && !cu.bdpcmMode && !pu.multiRefIdx && !cu.mipFlag && !cu.tmpFlag && testISP )
#endif
#else
#if JVET_W0123_TIMD_FUSION
      if (!cu.ispMode && !cu.mtsFlag && !cu.lfnstIdx && !cu.bdpcmMode && !pu.multiRefIdx && !cu.mipFlag && testISP && !cu.timd)
#else
      if (!cu.ispMode && !cu.mtsFlag && !cu.lfnstIdx && !cu.bdpcmMode && !pu.multiRefIdx && !cu.mipFlag && testISP)
#endif
#endif
      {
#if JVET_V0130_INTRA_TMP
        m_regIntraRDListWithCosts.push_back( ModeInfoWithCost( cu.mipFlag, pu.mipTransposedFlag, pu.multiRefIdx, cu.ispMode, uiOrgMode.modeId, cu.tmpFlag,
#if JVET_AD0086_ENHANCED_INTRA_TMP
          cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx,
#endif
          csTemp->cost ) );
#else
        m_regIntraRDListWithCosts.push_back( ModeInfoWithCost( cu.mipFlag, pu.mipTransposedFlag, pu.multiRefIdx, cu.ispMode, uiOrgMode.modeId, csTemp->cost ) );
#endif
      }

      if( cu.ispMode && !csTemp->cus[0]->firstTU->cbf[COMPONENT_Y] )
      {
        csTemp->cost = MAX_DOUBLE;
        csTemp->costDbOffset = 0;
        tmpValidReturn = false;
      }
      validReturn |= tmpValidReturn;

#if JVET_W0123_TIMD_FUSION
      if( sps.getUseLFNST() && mtsUsageFlag == 1 && !cu.ispMode && mode >= 0 && !cu.timd )
#else
      if( sps.getUseLFNST() && mtsUsageFlag == 1 && !cu.ispMode && mode >= 0 )
#endif
      {
        m_modeCostStore[lfnstIdx][mode] = tmpValidReturn ? csTemp->cost : (MAX_DOUBLE / 2.0); //(MAX_DOUBLE / 2.0) ??
      }
#if JVET_V0130_INTRA_TMP
      DTRACE( g_trace_ctx, D_INTRA_COST, "IntraCost T [x=%d,y=%d,w=%d,h=%d] %f (%d,%d,%d,%d,%d,%d,%d) \n", cu.blocks[0].x,
              cu.blocks[0].y, ( int ) width, ( int ) height, csTemp->cost, uiOrgMode.modeId, uiOrgMode.ispMod,
              pu.multiRefIdx, cu.tmpFlag, cu.mipFlag, cu.lfnstIdx, cu.mtsFlag );
#else
      DTRACE(g_trace_ctx, D_INTRA_COST, "IntraCost T [x=%d,y=%d,w=%d,h=%d] %f (%d,%d,%d,%d,%d,%d) \n", cu.blocks[0].x,
             cu.blocks[0].y, (int) width, (int) height, csTemp->cost, uiOrgMode.modeId, uiOrgMode.ispMod,
             pu.multiRefIdx, cu.mipFlag, cu.lfnstIdx, cu.mtsFlag);
#endif

      if( tmpValidReturn )
      {
        if (isFirstColorSpace)
        {
          if (m_pcEncCfg->getRGBFormatFlag() || !cu.ispMode)
          {
            sortRdModeListFirstColorSpace(uiOrgMode, csTemp->cost, cu.bdpcmMode, m_savedRdModeFirstColorSpace[m_savedRdModeIdx], m_savedRdCostFirstColorSpace[m_savedRdModeIdx], m_savedBDPCMModeFirstColorSpace[m_savedRdModeIdx], m_numSavedRdModeFirstColorSpace[m_savedRdModeIdx]);
          }
        }
#if INTRA_TRANS_ENC_OPT
        if (setSkipTimdControl && !cu.ispMode)
        {
#if JVET_W0123_TIMD_FUSION || ENABLE_DIMD
#if JVET_W0123_TIMD_FUSION && ENABLE_DIMD
          if (!cu.dimd && !cu.timd)
#elif ENABLE_DIMD
          if( !cu.dimd )
#else
          if( !cu.timd )
#endif
          {
            if (csTemp->cost < regAngCost)
            {
              regAngCost = csTemp->cost;
            }
          }
#endif
#if JVET_W0123_TIMD_FUSION
          if (cu.timd)
          {
            if (csTemp->cost < timdAngCost)
            {
              timdAngCost = csTemp->cost;
            }
          }
#endif
        }
#endif
        // check r-d cost
        if( csTemp->cost < csBest->cost )
        {
          std::swap( csTemp, csBest );

          uiBestPUMode  = uiOrgMode;
          bestBDPCMMode = cu.bdpcmMode;
#if ENABLE_DIMD
          bestDimdMode = cu.dimd;
#endif
#if JVET_W0123_TIMD_FUSION
          bestTimdMode = cu.timd;
#endif
#if JVET_AC0105_DIRECTIONAL_PLANAR
          bestPlMode = cu.plIdx;
#endif
#if JVET_AB0155_SGPM
          bestSgpmMode = cu.sgpm;
#endif
#if JVET_AC0115_INTRA_TMP_DIMD_MTS_LFNST 
          if (cu.tmpFlag)
          {
            CodingUnit* curCu = csBest->getCU(partitioner.currArea().lumaPos(), partitioner.chType);
            intraTmpDimdMode = curCu->intraTmpDimdMode;
          }
#endif 

          if( sps.getUseLFNST() && mtsUsageFlag == 1 && !cu.ispMode )
          {
            m_bestModeCostStore[ lfnstIdx ] = csBest->cost; //cs.cost;
            m_bestModeCostValid[ lfnstIdx ] = true;
          }
#if JVET_W0103_INTRA_MTS
          if (sps.getUseLFNST() && m_globalBestCostStore > csBest->cost)
          {
            m_globalBestCostStore = csBest->cost;
            m_globalBestCostValid = true;
          }
#endif
          if( csBest->cost < bestCurrentCost )
          {
            bestCurrentCost = csBest->cost;
          }
          if ( cu.ispMode )
          {
            m_modeCtrl->setIspCost(csBest->cost);
            bestLfnstIdx = cu.lfnstIdx;
          }
          else if ( testISP )
          {
            m_modeCtrl->setMtsFirstPassNoIspCost(csBest->cost);
          }
        }
        if( !cu.ispMode && !cu.bdpcmMode && csBest->cost < bestCostNonBDPCM )
        {
          bestCostNonBDPCM = csBest->cost;
        }
      }

      csTemp->releaseIntermediateData();
#if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
      if( m_pcEncCfg->getFastLocalDualTreeMode() )
      {
        if( cu.isConsIntra() && !cu.slice->isIntra() && csBest->cost != MAX_DOUBLE && costInterCU != COST_UNKNOWN && mode >= 0 )
        {
          if( m_pcEncCfg->getFastLocalDualTreeMode() == 2 )
          {
            //Note: only try one intra mode, which is especially useful to reduce EncT for LDB case (around 4%)
            break;
          }
          else
          {
            if( csBest->cost > costInterCU * 1.5 )
            {
              break;
            }
          }
        }
      }
      if (sps.getUseColorTrans() && !CS::isDualITree(cs))
      {
        if ((m_pcEncCfg->getRGBFormatFlag() && !cu.colorTransform) && csBest->cost != MAX_DOUBLE && bestCS->cost != MAX_DOUBLE && mode >= 0)
        {
          if (csBest->cost > bestCS->cost)
          {
            break;
          }
        }
      }
#endif
    } // Mode loop
#if INTRA_TRANS_ENC_OPT
    if (setSkipTimdControl && regAngCost != MAX_DOUBLE && timdAngCost != MAX_DOUBLE)
    {
      if (regAngCost * 1.3 < timdAngCost)
      {
        m_skipTimdLfnstMtsPass = true;
      }
    }
#endif
    cu.ispMode = uiBestPUMode.ispMod;
    cu.lfnstIdx = bestLfnstIdx;

    if( validReturn )
    {
      if (cu.colorTransform)
      {
        cs.useSubStructure(*csBest, partitioner.chType, pu, true, true, KEEP_PRED_AND_RESI_SIGNALS, KEEP_PRED_AND_RESI_SIGNALS, true);
      }
      else
      {
        cs.useSubStructure(*csBest, partitioner.chType, pu.singleChan(CHANNEL_TYPE_LUMA), true, true, KEEP_PRED_AND_RESI_SIGNALS,
                           KEEP_PRED_AND_RESI_SIGNALS, true);
      }
    }
#if JVET_AB0061_ITMP_BV_FOR_IBC
    if (uiBestPUMode.tmpFlag)
    {
      pu.interDir               = 1;             // use list 0 for IBC mode
      pu.refIdx[REF_PIC_LIST_0] = MAX_NUM_REF;   // last idx in the list
      pu.mv[0]                  = csBest->pus[0]->mv[0];
      pu.bv                     = csBest->pus[0]->bv;
    }
#endif
    csBest->releaseIntermediateData();
    if( validReturn )
    {
      //=== update PU data ====
#if JVET_V0130_INTRA_TMP
      cu.tmpFlag = uiBestPUMode.tmpFlag;
#endif
#if JVET_AD0086_ENHANCED_INTRA_TMP
      cu.tmpIdx        = uiBestPUMode.tmpIdx;
      cu.tmpFusionFlag = uiBestPUMode.tmpFusionFlag;
      cu.tmpFlmFlag    = uiBestPUMode.tmpFlmFlag;
      cu.tmpIsSubPel   = uiBestPUMode.tmpIsSubPel;
      cu.tmpSubPelIdx  = uiBestPUMode.tmpSubPelIdx;
#endif
      cu.mipFlag = uiBestPUMode.mipFlg;
      pu.mipTransposedFlag             = uiBestPUMode.mipTrFlg;
      pu.multiRefIdx = uiBestPUMode.mRefId;
      pu.intraDir[ CHANNEL_TYPE_LUMA ] = uiBestPUMode.modeId;
#if ENABLE_DIMD
      cu.dimd = bestDimdMode;
      if (cu.dimd)
      {
        CHECK(pu.multiRefIdx > 0, "use of DIMD");
      }
#endif
      cu.bdpcmMode = bestBDPCMMode;
#if JVET_W0123_TIMD_FUSION
      cu.timd = bestTimdMode;
      if (cu.timd)
      {
        pu.intraDir[ CHANNEL_TYPE_LUMA ] = cu.timdMode;
      }
#endif
#if JVET_AB0155_SGPM
      cu.sgpm = uiBestPUMode.sgpmFlag;
      if (cu.sgpm)
      {
        CHECK(!bestSgpmMode, "mode not same");
        pu.intraDir[CHANNEL_TYPE_LUMA]  = uiBestPUMode.sgpmMode0;
        pu.intraDir1[CHANNEL_TYPE_LUMA] = uiBestPUMode.sgpmMode1;
        cu.sgpmSplitDir                 = uiBestPUMode.sgpmSplitDir;
        cu.sgpmMode0                    = uiBestPUMode.sgpmMode0;
        cu.sgpmMode1                    = uiBestPUMode.sgpmMode1;
        cu.sgpmIdx                      = uiBestPUMode.sgpmIdx;
      }
#endif

#if JVET_AB0157_TMRL
      cu.tmrlFlag = uiBestPUMode.mRefId >= MAX_REF_LINE_IDX;
      if(cu.tmrlFlag)
      {
        int tmrlListIdx = uiBestPUMode.mRefId - MAX_REF_LINE_IDX;
        cu.tmrlListIdx = tmrlListIdx;
        pu.multiRefIdx = m_tmrlList[tmrlListIdx].multiRefIdx;
        pu.intraDir[0] = m_tmrlList[tmrlListIdx].intraDir;
      }
#endif
#if JVET_AC0115_INTRA_TMP_DIMD_MTS_LFNST 
      if (cu.tmpFlag)
      {
        cu.intraTmpDimdMode = intraTmpDimdMode;
      }
#endif

#if JVET_AC0105_DIRECTIONAL_PLANAR
      cu.plIdx = bestPlMode;
      if (cu.plIdx)
      {
        CHECK(pu.multiRefIdx > 0, "use of PL");
        CHECK(cu.mipFlag, "use of PL");
        CHECK(cu.dimd, "use of PL");
        CHECK(cu.tmpFlag, "use of PL");
        CHECK(cu.tmrlFlag, "use of PL");
        CHECK(cu.sgpm, "use of PL");
        CHECK(cu.timd, "use of PL");
      }
#endif

      if (cu.colorTransform)
      {
        CHECK(pu.intraDir[CHANNEL_TYPE_CHROMA] != DM_CHROMA_IDX, "chroma should use DM mode for adaptive color transform");
      }
    }
  }

  //===== reset context models =====
  m_CABACEstimator->getCtx() = ctxStart;

  return validReturn;
}

#if JVET_AD0120_LBCCP
void IntraSearch::fillLmPredFiltList(PredictionUnit pu, const PelUnitBuf& lmPredFilt, int &lmPredFiltIdx, std::vector<lmPredFiltModeInfo> &miLmPredFiltList)
{
  const TempCtx ctxStart(m_ctxCache, m_CABACEstimator->getCtx());
  const double  sqrtLambdaForFirstPass = m_pcRdCost->getMotionLambda() * FRAC_BITS_SCALE;

  int64_t          sad = 0, sadCb = 0, satdCb = 0, sadCr = 0, satdCr = 0;
  CodingStructure &cs = *(pu.cs);
  DistParam        distParamSadCb, distParamSatdCb;
  DistParam        distParamSadCr, distParamSatdCr;
  m_pcRdCost->setDistParam(distParamSadCb, cs.getOrgBuf(pu.Cb()), lmPredFilt.Cb(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, false);
  m_pcRdCost->setDistParam(distParamSatdCb, cs.getOrgBuf(pu.Cb()), lmPredFilt.Cb(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cb, true);
  m_pcRdCost->setDistParam(distParamSadCr, cs.getOrgBuf(pu.Cr()), lmPredFilt.Cr(),  pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, false);
  m_pcRdCost->setDistParam(distParamSatdCr, cs.getOrgBuf(pu.Cr()), lmPredFilt.Cr(), pu.cs->sps->getBitDepth(CHANNEL_TYPE_CHROMA), COMPONENT_Cr, true);
  distParamSadCb.applyWeight  = false;
  distParamSatdCb.applyWeight = false;
  distParamSadCr.applyWeight  = false;
  distParamSatdCr.applyWeight = false;

  sadCb  = distParamSadCb.distFunc(distParamSadCb) * 2;
  satdCb = distParamSatdCb.distFunc(distParamSatdCb);
  sad    = std::min(sadCb, satdCb);
  sadCr  = distParamSadCr.distFunc(distParamSadCr) * 2;
  satdCr = distParamSatdCr.distFunc(distParamSatdCr);
  sad += std::min(sadCr, satdCr);

  m_CABACEstimator->getCtx() = ctxStart;
  uint64_t fracModeBits      = xFracModeBitsIntra(pu, MMLM_CHROMA_IDX, CHANNEL_TYPE_CHROMA);
  double   cost              = (double) sad + (double) fracModeBits * sqrtLambdaForFirstPass;

  int cccmFlag = 0, cccmNoSubFlag = 0, glCccmFlag = 0, cccmMultiFilterIdx = 0;
#if JVET_AA0057_CCCM
  cccmFlag = pu.cccmFlag;
#endif
#if JVET_AC0147_CCCM_NO_SUBSAMPLING
  cccmNoSubFlag = pu.cccmNoSubFlag;
#endif
#if JVET_AC0054_GLCCCM
  glCccmFlag = pu.glCccmFlag;
#endif
#if JVET_AD0202_CCCM_MDF
  cccmMultiFilterIdx = pu.cccmMultiFilterIdx;
#endif
  miLmPredFiltList.push_back({ lmPredFiltIdx, cccmFlag, cccmNoSubFlag, glCccmFlag, cccmMultiFilterIdx, cost });
  lmPredFiltIdx++;
}
#endif

void IntraSearch::estIntraPredChromaQT( CodingUnit &cu, Partitioner &partitioner, const double maxCostAllowed 
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                                      , InterPrediction* pcInterPred
#endif
)
{
  const ChromaFormat format   = cu.chromaFormat;
  const uint32_t    numberValidComponents = getNumberValidComponents(format);
  CodingStructure &cs = *cu.cs;
  const TempCtx ctxStart  ( m_ctxCache, m_CABACEstimator->getCtx() );

  cs.setDecomp( cs.area.Cb(), false );

  double    bestCostSoFar = maxCostAllowed;
#if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
  bool      lumaUsesISP   = !cu.isSepTree() && cu.ispMode;
#else
  bool      lumaUsesISP = !CS::isDualITree(*cu.cs) && cu.ispMode;
#endif
  PartSplit ispType       = lumaUsesISP ? CU::getISPType( cu, COMPONENT_Y ) : TU_NO_ISP;
  CHECK( cu.ispMode && bestCostSoFar < 0, "bestCostSoFar must be positive!" );
#if JVET_AF0066_ENABLE_DBV_4_SINGLE_TREE
  bool singleTreeLumaIntraTmp = !CS::isDualITree(*cu.cs) && cu.tmpFlag;
#endif

#if JVET_AD0120_LBCCP
  int  bestCCInsideFilter = 0;
#endif
  auto &pu = *cu.firstPU;

  {
    uint32_t       uiBestMode = 0;
    Distortion uiBestDist = 0;
    double     dBestCost = MAX_DOUBLE;
    int32_t bestBDPCMMode = 0;
#if JVET_AA0057_CCCM
    int      cccmModeBest = 0;
#if JVET_AC0147_CCCM_NO_SUBSAMPLING
    int      cccmNoSubBest = 0;
#endif
#if JVET_AC0054_GLCCCM
    int      glCccmBest = 0;
#endif
#if JVET_AE0100_BVGCCCM
    int      bvgCccmBest = 0;
#endif
#if JVET_AD0202_CCCM_MDF
    int      cccmMultiFilterIdxBest = 0;
#endif
#endif
#if JVET_Z0050_CCLM_SLOPE
    CclmOffsets bestCclmOffsets = {};
    CclmOffsets satdCclmOffsetsBest[NUM_CHROMA_MODE];
    int64_t     satdCclmCosts      [NUM_CHROMA_MODE] = { 0 };
#endif
#if JVET_AA0126_GLM
    GlmIdc      bestGlmIdc = {};
    GlmIdc      satdGlmIdcBest     [NUM_CHROMA_MODE];
    int64_t     satdGlmCosts       [NUM_CHROMA_MODE] = { 0 };
#endif
#if JVET_Z0050_DIMD_CHROMA_FUSION
#if JVET_AC0119_LM_CHROMA_FUSION
    uint8_t isChromaFusion = 0;
#else
    bool isChromaFusion = false;
#endif
#endif
#if JVET_AD0188_CCP_MERGE
    int               bestNonAdjCCCM = 0;
    CCPModelCandidate ccpModelBest;
#endif

    //----- init mode list ----
    {
      int32_t  uiMinMode = 0;
      int32_t  uiMaxMode = NUM_CHROMA_MODE;
      //----- check chroma modes -----
      uint32_t chromaCandModes[ NUM_CHROMA_MODE ];
      PU::getIntraChromaCandModes( pu, chromaCandModes );
#if JVET_Z0050_DIMD_CHROMA_FUSION && ENABLE_DIMD
#if JVET_AC0094_REF_SAMPLES_OPT
    if (!CS::isDualITree(*cu.cs))
    {
      const CompArea areaCb = pu.Cb();
      const CompArea areaCr = pu.Cr();
      const CompArea lumaArea = CompArea(COMPONENT_Y, pu.chromaFormat, areaCb.lumaPos(), recalcSize(pu.chromaFormat, CHANNEL_TYPE_CHROMA, CHANNEL_TYPE_LUMA, areaCb.size()));//needed for correct pos/size (4x4 Tus)
      IntraPrediction::deriveDimdChromaMode(cs.picture->getRecoBuf(lumaArea), cs.picture->getRecoBuf(areaCb), cs.picture->getRecoBuf(areaCr), lumaArea, areaCb, areaCr, *pu.cu);
    }
    if (PU::getCoLocatedIntraLumaMode(*cu.firstPU) == cu.dimdChromaMode)
    {
      if (cu.dimdChromaMode == cu.dimdChromaModeSecond)
      {
        cu.dimdChromaMode = DC_IDX;
      }
      else
      {
        cu.dimdChromaMode = cu.dimdChromaModeSecond;
      }
    }
#else
      // derive DIMD chroma mode
      CompArea areaCb = pu.Cb();
      CompArea areaCr = pu.Cr();
      CompArea lumaArea = CompArea(COMPONENT_Y, pu.chromaFormat, areaCb.lumaPos(), recalcSize(pu.chromaFormat, CHANNEL_TYPE_CHROMA, CHANNEL_TYPE_LUMA, areaCb.size()));//needed for correct pos/size (4x4 Tus)
      IntraPrediction::deriveDimdChromaMode(cs.picture->getRecoBuf(lumaArea), cs.picture->getRecoBuf(areaCb), cs.picture->getRecoBuf(areaCr), lumaArea, areaCb, areaCr, *pu.cu);
#endif
#endif
#if JVET_AC0071_DBV
      if (PU::hasChromaBvFlag(pu))
      {
        PU::deriveChromaBv(pu);
      }
      else
      {
        uiMaxMode--;
      }
#endif

      // create a temporary CS
      CodingStructure &saveCS = *m_pSaveCS[0];
      saveCS.pcv      = cs.pcv;
      saveCS.picture  = cs.picture;
#if JVET_Z0118_GDR
      saveCS.m_pt = cs.m_pt;
#endif
      saveCS.area.repositionTo( cs.area );
      saveCS.clearTUs();
#if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
      if( !cu.isSepTree() && cu.ispMode )
#else
      if (!CS::isDualITree(cs) && cu.ispMode)
#endif
      {
        saveCS.clearCUs();
        saveCS.clearPUs();
      }
#if !INTRA_RM_SMALL_BLOCK_SIZE_CONSTRAINTS
      if( cu.isSepTree() )
#else
      if (CS::isDualITree(cs))
#endif
      {
        if( partitioner.canSplit( TU_MAX_TR_SPLIT, cs ) )
        {
          partitioner.splitCurrArea( TU_MAX_TR_SPLIT, cs );

          do
          {
            cs.addTU( CS::getArea( cs, partitioner.currArea(), partitioner.chType ), partitioner.chType ).depth = partitioner.currTrDepth;
          } while( partitioner.nextPart( cs ) );

          partitioner.exitCurrSplit();
        }
        else
        cs.addTU( CS::getArea( cs, partitioner.currArea(), partitioner.chType ), partitioner.chType );
      }

      std::vector<TransformUnit*> orgTUs;

      if( lumaUsesISP )
      {
        CodingUnit& auxCU = saveCS.addCU( cu, partitioner.chType );
        auxCU.ispMode = cu.ispMode;
        saveCS.sps = cu.cs->sps;
        saveCS.addPU( *cu.firstPU, partitioner.chType );
      }


      // create a store for the TUs
      for( const auto &ptu : cs.tus )
      {
        // for split TUs in HEVC, add the TUs without Chroma parts for correct setting of Cbfs
        if( lumaUsesISP || pu.contains( *ptu, CHANNEL_TYPE_CHROMA ) )
        {
          saveCS.addTU( *ptu, partitioner.chType );
          orgTUs.push_back( ptu );
        }
      }
      if( lumaUsesISP )
      {
        saveCS.clearCUs();
      }
      // SATD pre-selecting.
      int satdModeList[NUM_CHROMA_MODE];
      int64_t satdSortedCost[NUM_CHROMA_MODE];
      for (int i = 0; i < NUM_CHROMA_MODE; i++)
      {
        satdSortedCost[i] = 0; // for the mode not pre-select by SATD, do RDO by default, so set the initial value 0.
        satdModeList[i] = 0;
      }
#if JVET_Z0050_DIMD_CHROMA_FUSION && ENABLE_DIMD
#if JVET_AC0071_DBV
      bool modeIsEnable[NUM_INTRA_MODE + 3]; // use intra mode idx to check whether enable
      for (int i = 0; i < NUM_INTRA_MODE + 3; i++)
      {
        modeIsEnable[i] = 1;
      }
#else
      bool modeIsEnable[NUM_INTRA_MODE + 2]; // use intra mode idx to check whether enable
      for (int i = 0; i < NUM_INTRA_MODE + 2; i++)
      {
        modeIsEnable[i] = 1;
      }
#endif
#else
#if JVET_AC0071_DBV
      bool modeIsEnable[NUM_INTRA_MODE + 2]; // use intra mode idx to check whether enable
      for (int i = 0; i < NUM_INTRA_MODE + 2; i++)
      {
        modeIsEnable[i] = 1;
      }
#else
      bool modeIsEnable[NUM_INTRA_MODE + 1]; // use intra mode idx to check whether enable
      for (int i = 0; i < NUM_INTRA_MODE + 1; i++)
      {
        modeIsEnable[i] = 1;
      }
#endif
#endif
      DistParam distParamSad;
      DistParam distParamSatd;
      pu.intraDir[1] = MDLM_L_IDX; // temporary assigned, just to indicate this is a MDLM mode. for luma down-sampling operation.

      initIntraPatternChType(cu, pu.Cb());
      initIntraPatternChType(cu, pu.Cr());
      xGetLumaRecPixels(pu, pu.Cb());

#if JVET_AA0126_GLM
      if ( PU::isLMCModeEnabled( pu, LM_CHROMA_IDX ) && PU::hasGlmFlag( pu, LM_CHROMA_IDX ) )
      {
#if JVET_AB0092_GLM_WITH_LUMA && JVET_AB0174_CCCM_DIV_FREE
        xGlmSetLumaRefValue(pu, pu.Cb());
#endif
        // Generate all GLM templates at encoder
        xGetLumaRecPixelsGlmAll(pu, pu.Cb());
        pu.intraDir[1] = LM_CHROMA_IDX;
        xGetLumaRecPixels(pu, pu.Cb());

        for ( int mode = LM_CHROMA_IDX; mode <= MMLM_T_IDX; mode++ )
        {
          satdGlmIdcBest[mode - LM_CHROMA_IDX].setAllZero();
          
#if JVET_AB0092_GLM_WITH_LUMA
          CodedCUInfo& relatedCU = ((EncModeCtrlMTnoRQT *)m_modeCtrl)->getBlkInfo(partitioner.currArea());
          if (PU::hasGlmFlag(pu, mode) && !relatedCU.skipGLM)
#else
          if ( PU::hasGlmFlag( pu, mode ) )
#endif
          {
#if !JVET_AB0092_GLM_WITH_LUMA
            for ( int comp = COMPONENT_Cb; comp <= COMPONENT_Cr; comp++ )
            {
              ComponentID       compID = ComponentID( comp );
#else
              ComponentID       compID = COMPONENT_Cb;
#endif
              int              idcBest = 0;
              int64_t         satdBest = 0;
              GlmIdc&         idcsBest = satdGlmIdcBest[mode - LM_CHROMA_IDX];
              
              pu.intraDir[1] = mode;
              pu.glmIdc.setAllZero();

              xFindBestGlmIdcSATD(pu, compID, idcBest, satdBest );

              idcsBest.setIdc(compID, 0, idcBest);