IntraSearch.cpp

              // Use the min between SAD and HAD as the cost criterion
              // SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
              Distortion sadCost = distParamSad.distFunc(distParamSad);
              minSadHad += std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
              minSadHad += std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif

              loadStartStates();
              uint64_t fracModeBits = xFracModeBitsIntra(pu, uiMode, CHANNEL_TYPE_LUMA);

              double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
              m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
              DTRACE(g_trace_ctx, D_INTRA_COST, "IntraHAD: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, uiMode);

              updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, uiMode), cost, uiRdModeList,
                             candCostList, numModesForFullRD);
              updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, uiMode), double(minSadHad),
                             uiHadModeList, CandHadList, numHadCand);
            }
#if JVET_AC0105_DIRECTIONAL_PLANAR
            bool testDirPlanar = isLuma(partitioner.chType);
            if (testDirPlanar)
            {
              for (int dirPlanarModeIdx = 0; dirPlanarModeIdx < 2; dirPlanarModeIdx++)
              {
                cu.sgpm        = false;
                cu.ispMode     = 0;
                cu.tmpFlag     = false;
                cu.tmrlFlag    = false;
                pu.multiRefIdx = 0;
                cu.mipFlag     = false;
                pu.intraDir[0] = PLANAR_IDX;
                cu.plIdx       = dirPlanarModeIdx + 1;

                initPredIntraParams(pu, pu.Y(), sps);
#if JVET_AB0157_INTRA_FUSION
                predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
                predIntraAng(COMPONENT_Y, piPred, pu);
#endif

                // Use the min between SAD and SATD as the cost criterion
                // SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                Distortion sadCost = distParamSad.distFunc(distParamSad);
                Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
                Distortion minSadHad =
                  std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
                loadStartStates();
                uint64_t fracModeBits = xFracModeBitsIntra(pu, PLANAR_IDX, CHANNEL_TYPE_LUMA);
                double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
                updateCandList(
                  ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, dirPlanarModeIdx ? PL_VER_IDX : PL_HOR_IDX), cost,
                  uiRdModeList, candCostList, numModesForFullRD);
                updateCandList(
                  ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, dirPlanarModeIdx ? PL_VER_IDX : PL_HOR_IDX),
                  double(minSadHad), uiHadModeList, CandHadList, numHadCand);
                dirPlanarCostList[dirPlanarModeIdx] = cost;
              }
            }
            cu.plIdx = 0;
#endif
            if (!sps.getUseMIP() && LFNSTSaveFlag)
            {
              // save found best modes
              m_uiSavedNumRdModesLFNST = numModesForFullRD;
              m_uiSavedRdModeListLFNST = uiRdModeList;
              m_dSavedModeCostLFNST    = candCostList;
              // PBINTRA fast
              m_uiSavedHadModeListLFNST = uiHadModeList;
              m_dSavedHadListLFNST      = CandHadList;
              LFNSTSaveFlag             = false;
            }
          }   // NSSTFlag
          if (!sps.getUseMIP() && LFNSTLoadFlag)
          {
            // restore saved modes
            numModesForFullRD = m_uiSavedNumRdModesLFNST;
            uiRdModeList      = m_uiSavedRdModeListLFNST;
            candCostList      = m_dSavedModeCostLFNST;
            // PBINTRA fast
            uiHadModeList = m_uiSavedHadModeListLFNST;
            CandHadList   = m_dSavedHadListLFNST;
          }   // !LFNSTFlag

          if (!(sps.getUseMIP() && LFNSTLoadFlag))
          {
            static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> parentCandList = uiRdModeList;

            // Second round of SATD for extended Angular modes
            for (int modeIdx = 0; modeIdx < numModesForFullRD; modeIdx++)
            {
              unsigned parentMode = parentCandList[modeIdx].modeId;
              if (parentMode > (DC_IDX + 1) && parentMode < (NUM_LUMA_MODE - 1))
              {
                for (int subModeIdx = -1; subModeIdx <= 1; subModeIdx += 2)
                {
                  unsigned mode = parentMode + subModeIdx;

                  if (!bSatdChecked[mode])
                  {
                    pu.intraDir[0] = mode;

                    initPredIntraParams(pu, pu.Y(), sps);
#if JVET_AB0157_INTRA_FUSION
                    predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
                    predIntraAng(COMPONENT_Y, piPred, pu);
#endif
#if JVET_AB0155_SGPM
                    if (testSgpm && SGPMSaveFlag && sgpmNeededMode[mode])
                    {
                      PelBuf   predBuf(m_intraPredBuf[mode], tmpArea);
                      predBuf.copyFrom(piPred);
                      m_intraModeReady[mode] = 1;
                    }
#endif

                    // Use the min between SAD and SATD as the cost criterion
                    // SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                    Distortion sadCost = distParamSad.distFunc(distParamSad);
                    Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
                    Distortion minSadHad =
                      std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
                    loadStartStates();
                    uint64_t fracModeBits = xFracModeBitsIntra(pu, mode, CHANNEL_TYPE_LUMA);

                    double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                    m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
                    updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, mode), cost, uiRdModeList,
                                   candCostList, numModesForFullRD);
                    updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, mode), double(minSadHad),
                                   uiHadModeList, CandHadList, numHadCand);

                    bSatdChecked[mode] = true;
                  }
                }
              }
            }
            if (saveDataForISP)
            {
              // we save the regular intra modes list
              m_ispCandListHor = uiRdModeList;
            }

#if !JVET_AB0157_TMRL || JVET_AD0082_TMRL_CONFIG
            pu.multiRefIdx    = 1;
#if SECONDARY_MPM
            const int numMPMs = NUM_PRIMARY_MOST_PROBABLE_MODES;
#else
            const int numMPMs = NUM_MOST_PROBABLE_MODES;
#endif
            uint8_t* multiRefMPM = m_intraMPM;
#if !SECONDARY_MPM
            PU::getIntraMPMs(pu, multiRefMPM);
#endif
#endif
#if JVET_AB0157_TMRL
#if !JVET_AD0082_TMRL_CONFIG
            cu.tmrlFlag = true;
#endif
            if (CU::allowTmrl(cu))
            {
#if JVET_AD0082_TMRL_CONFIG
              cu.tmrlFlag = true;
#endif
              for (auto multiRefIdx : EXT_REF_LINE_IDX)
              {
                pu.multiRefIdx = multiRefIdx;
                initIntraPatternChType(cu, pu.Y(), true);

                for (auto i = 0; i < MRL_LIST_SIZE; i++)
                {
                  if (m_tmrlList[i].multiRefIdx != multiRefIdx)
                  {
                    continue;
                  }

                  pu.intraDir[0] = m_tmrlList[i].intraDir;
                  cu.tmrlListIdx = i;
                  uint32_t uiMode = i + MAX_REF_LINE_IDX;

                  initPredIntraParams(pu, pu.Y(), *(pu.cs->sps));
                  predIntraAng(COMPONENT_Y, piPred, pu);

                  // Use the min between SAD and SATD as the cost criterion
                    // SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                  Distortion sadCost = distParamSad.distFunc(distParamSad);
                  Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
                  Distortion minSadHad =
                    std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
                  loadStartStates();
                  uint64_t fracModeBits = xFracModeBitsIntra(pu, pu.intraDir[0], CHANNEL_TYPE_LUMA);

                  double cost = (double)minSadHad + (double)fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                  m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
                  updateCandList(ModeInfo(false, false, uiMode, NOT_INTRA_SUBPARTITIONS, 0), cost, uiRdModeList,
                    candCostList, numModesForFullRD);
                  updateCandList(ModeInfo(false, false, uiMode, NOT_INTRA_SUBPARTITIONS, 0), double(minSadHad),
                    uiHadModeList, CandHadList, numHadCand);
#if JVET_AB0157_TMRL
                  tmrlCostList[i] = cost;
#endif
                }
              }
#if JVET_AD0082_TMRL_CONFIG
              cu.tmrlFlag = false;
#endif
            }
#endif
#if !JVET_AB0157_TMRL || JVET_AD0082_TMRL_CONFIG
#if JVET_AD0082_TMRL_CONFIG
            else
            {
#endif
            for (int mRefNum = 1; mRefNum < numOfPassesExtendRef; mRefNum++)
            {
              int multiRefIdx = MULTI_REF_LINE_IDX[mRefNum];

              pu.multiRefIdx = multiRefIdx;
              {
#if JVET_AB0157_INTRA_FUSION && JVET_AB0155_SGPM
                initIntraPatternChType(cu, pu.Y(), true, 0, false);
#elif JVET_AB0157_INTRA_FUSION
                initIntraPatternChType(cu, pu.Y(), true, false);
#else
                initIntraPatternChType(cu, pu.Y(), true);
#endif
              }
              for (int x = 1; x < numMPMs; x++)
              {
                uint32_t mode = multiRefMPM[x];
                {
                  pu.intraDir[0] = mode;
                  initPredIntraParams(pu, pu.Y(), sps);

#if JVET_AB0157_INTRA_FUSION
                  predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
                  predIntraAng(COMPONENT_Y, piPred, pu);
#endif

                  // Use the min between SAD and SATD as the cost criterion
                  // SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                  Distortion sadCost = distParamSad.distFunc(distParamSad);
                  Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
                  Distortion minSadHad =
                    std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif

                  loadStartStates();
                  uint64_t fracModeBits = xFracModeBitsIntra(pu, mode, CHANNEL_TYPE_LUMA);

                  double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                  m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
                  updateCandList(ModeInfo(false, false, multiRefIdx, NOT_INTRA_SUBPARTITIONS, mode), cost, uiRdModeList,
                                 candCostList, numModesForFullRD);
                  updateCandList(ModeInfo(false, false, multiRefIdx, NOT_INTRA_SUBPARTITIONS, mode), double(minSadHad),
                                 uiHadModeList, CandHadList, numHadCand);
                }
              }
            }
#if JVET_AD0082_TMRL_CONFIG
            }
#endif
#endif
            CHECKD(uiRdModeList.size() != numModesForFullRD, "Error: RD mode list size");

#if JVET_V0130_INTRA_TMP && JVET_AB0130_ITMP_SAMPLING
            // derive TPM candidate using hadamard
            if (testTpm)
            {
              cu.tmpFlag = true;
              cu.mipFlag = false;
              pu.multiRefIdx = 0;

#if JVET_AD0086_ENHANCED_INTRA_TMP
              static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListTmp;
              static_vector<double, FAST_UDI_MAX_RDMODE_NUM> candCostListTmp;
#else
              int foundCandiNum = 0;
              bool bsuccessfull = 0;
#endif
              CodingUnit cuCopy = cu;

#if JVET_W0069_TMP_BOUNDARY
              RefTemplateType templateType = getRefTemplateType(cuCopy, cuCopy.blocks[COMPONENT_Y]);
              if (templateType != NO_TEMPLATE)
#else
              if (isRefTemplateAvailable(cuCopy, cuCopy.blocks[COMPONENT_Y]))
#endif
              {
#if JVET_AD0086_ENHANCED_INTRA_TMP
                cu.tmpIsSubPel = 0;
                cu.tmpSubPelIdx = 0;
                for(int tmpFusionFlag = 0; tmpFusionFlag <= 1; tmpFusionFlag++)
                {
                  cu.tmpFusionFlag = tmpFusionFlag ? true: false;

                  for (int tmpFlmFlag = 0; tmpFlmFlag <= 1; tmpFlmFlag++)
                  {
                    cu.tmpFlmFlag = tmpFlmFlag ? true: false;
                    if(tmpFlmFlag && tmpFusionFlag)
                    {
                      continue;
                    }

                    int idxNum = cu.tmpFusionFlag ? TMP_GROUP_IDX << 1 : m_tmpNumCand;
                    for (int tmpIdx = 0; tmpIdx < idxNum; tmpIdx++)
                    {
                      if(cu.tmpFusionFlag && !m_tmpFusionInfo[tmpIdx].bValid)
                      {
                        continue;
                      }

                      cu.tmpIdx    = tmpIdx;

                      int placeHolder;
                      generateTMPrediction(piPred.buf, piPred.stride, placeHolder, pu, false);
                      xGenerateTmpFlmPred(piPred, pu.lwidth(), pu.lheight(), templateType, pu.cu, false);
                      xTMPFusionApplyModel(piPred, pu.lwidth(), pu.lheight(), templateType, pu.cu, false);
                      
#else
#if JVET_W0069_TMP_BOUNDARY
#if TMP_FAST_ENC
                bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.Y(), foundCandiNum, pu.cu);
#else
                getTargetTemplate(&cuCopy, pu.lwidth(), pu.lheight(), templateType);
                candidateSearchIntra(&cuCopy, pu.lwidth(), pu.lheight(), templateType);
                bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), foundCandiNum);
#endif
#else
#if TMP_FAST_ENC
                bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.Y(), foundCandiNum, pu.cu);
#else
                getTargetTemplate(&cuCopy, pu.lwidth(), pu.lheight());
                candidateSearchIntra(&cuCopy, pu.lwidth(), pu.lheight());
                bsuccessfull = generateTMPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), foundCandiNum);
#endif
#endif
              }
#if JVET_W0069_TMP_BOUNDARY
              else
              {
                foundCandiNum = 1;
#if JVET_AC0115_INTRA_TMP_DIMD_MTS_LFNST 
                bsuccessfull = generateTmDcPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), 1 << (cuCopy.cs->sps->getBitDepth(CHANNEL_TYPE_LUMA) - 1), pu.cu);
#else
                bsuccessfull = generateTmDcPrediction(piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), 1 << (cuCopy.cs->sps->getBitDepth(CHANNEL_TYPE_LUMA) - 1));
#endif 
              }
#endif
              if (bsuccessfull && foundCandiNum >= 1)
              {
#endif
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                      Distortion sadCost = distParamSad.distFunc(distParamSad);
                      Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
                      Distortion minSadHad =
                        std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
                      loadStartStates();
                      uint64_t fracModeBits = xFracModeBitsIntra(pu, 0, CHANNEL_TYPE_LUMA);

                      double cost = double(minSadHad) + double(fracModeBits) * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                      m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - double(minSadHad) + (double)sadCost);
#endif
                      DTRACE(g_trace_ctx, D_INTRA_COST, "IntraTPM: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, 0);
#if JVET_AD0086_ENHANCED_INTRA_TMP
                      updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
                      updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, CandHadList, numHadCand);
                    }
#else
                      updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1), cost, uiRdModeList, candCostList, numModesForFullRD);
                      updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1), 0.8 * double(minSadHad), uiHadModeList, CandHadList, numHadCand);
#endif
#if JVET_AD0086_ENHANCED_INTRA_TMP
                    //record the best full-pel candidates
                    if(!tmpFlmFlag&&!tmpFusionFlag)
                    {
                      for(int idxInList=0; idxInList < uiRdModeList.size(); idxInList++)
                      {
                        if(uiRdModeList[idxInList].tmpFlag){
                          updateCandList(uiRdModeList[idxInList], candCostList[idxInList], uiRdModeListTmp, candCostListTmp, numModesForFullRD);
                        }
                      }
                    }
                  }
                }
                //fractional BV
                cu.tmpFusionFlag = false;
                cu.tmpFlmFlag = false;

                for(int idxInList=0; idxInList < uiRdModeListTmp.size(); idxInList++)
                {
                  cu.tmpIdx = uiRdModeListTmp[idxInList].tmpIdx;
                  xPadForInterpolation(&cu);
                  for(int tmpIsSubPel = 1; tmpIsSubPel < 4; tmpIsSubPel++)
                  {
                    for (int idx = 0; idx < TMP_MAX_SUBPEL_DIR; idx++)
                    {
                      cu.tmpIsSubPel = tmpIsSubPel;
                      cu.tmpSubPelIdx = idx;
                      int placeHolder;
                      generateTMPrediction(piPred.buf, piPred.stride, placeHolder, pu, false);

#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS&&JVET_AE0169_BIPREDICTIVE_IBC
                      Distortion sadCost = distParamSad.distFunc(distParamSad);
                      Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
                      Distortion minSadHad =
                          std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif

                      loadStartStates();
                      uint64_t fracModeBits = xFracModeBitsIntra(pu, 0, CHANNEL_TYPE_LUMA);

                      double cost = double(minSadHad) + double(fracModeBits) * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS&&JVET_AE0169_BIPREDICTIVE_IBC
                      m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - double(minSadHad) + (double)sadCost);
#endif
                      DTRACE(g_trace_ctx, D_INTRA_COST, "IntraTPM: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, 0);
                      updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), cost, uiRdModeList, candCostList, numModesForFullRD);
                      updateCandList(ModeInfo(0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1, cu.tmpIdx, cu.tmpFusionFlag, cu.tmpFlmFlag, cu.tmpIsSubPel, cu.tmpSubPelIdx), 0.8 * double(minSadHad), uiHadModeList, CandHadList, numHadCand);
                    }
                  }
                }
#endif
              }
#if JVET_AD0086_ENHANCED_INTRA_TMP
              cu.tmpFlag       = 0;
              cu.tmpFusionFlag = false;
              cu.tmpFlmFlag    = false;
              cu.tmpIsSubPel   = 0;
              cu.tmpSubPelIdx  = 0;
              cu.tmpIdx        = 0;
#endif
            }
#endif

            if (LFNSTSaveFlag && testMip
                && !allowLfnstWithMip(cu.firstPU->lumaSize()))   // save a different set for the next run
            {
              // save found best modes
              m_uiSavedRdModeListLFNST = uiRdModeList;
              m_dSavedModeCostLFNST    = candCostList;
              // PBINTRA fast
              m_uiSavedHadModeListLFNST = uiHadModeList;
              m_dSavedHadListLFNST      = CandHadList;
              m_uiSavedNumRdModesLFNST =
                g_aucIntraModeNumFast_UseMPM_2D[uiWidthBit - MIN_CU_LOG2][uiHeightBit - MIN_CU_LOG2];
              m_uiSavedRdModeListLFNST.resize(m_uiSavedNumRdModesLFNST);
              m_dSavedModeCostLFNST.resize(m_uiSavedNumRdModesLFNST);
              // PBINTRA fast
              m_uiSavedHadModeListLFNST.resize(3);
              m_dSavedHadListLFNST.resize(3);
              LFNSTSaveFlag = false;
            }
#if JVET_V0130_INTRA_TMP && !JVET_AB0130_ITMP_SAMPLING
            // derive TPM candidate using hadamard
            if( testTpm )
            {
              cu.tmpFlag = true;
              cu.mipFlag = false;
              pu.multiRefIdx = 0;
#if JVET_AB0157_TMRL
              cu.tmrlFlag = false;
#endif
              int foundCandiNum = 0;
              bool bsuccessfull = 0;
              CodingUnit cu_cpy = cu;

#if JVET_W0069_TMP_BOUNDARY
              RefTemplateType templateType = getRefTemplateType( cu_cpy, cu_cpy.blocks[COMPONENT_Y] );
              if( templateType != NO_TEMPLATE )
#else
              if( isRefTemplateAvailable( cu_cpy, cu_cpy.blocks[COMPONENT_Y] ) )
#endif
              {
#if JVET_W0069_TMP_BOUNDARY
                getTargetTemplate( &cu_cpy, pu.lwidth(), pu.lheight(), templateType );
                candidateSearchIntra( &cu_cpy, pu.lwidth(), pu.lheight(), templateType );
                bsuccessfull = generateTMPrediction( piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), foundCandiNum );
#else
                getTargetTemplate( &cu_cpy, pu.lwidth(), pu.lheight() );
                candidateSearchIntra( &cu_cpy, pu.lwidth(), pu.lheight() );
                bsuccessfull = generateTMPrediction( piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), foundCandiNum );
#endif
              }
#if JVET_W0069_TMP_BOUNDARY
              else
              {
                foundCandiNum = 1;
                bsuccessfull = generateTmDcPrediction( piPred.buf, piPred.stride, pu.lwidth(), pu.lheight(), 1 << (cu_cpy.cs->sps->getBitDepth( CHANNEL_TYPE_LUMA ) - 1) );
              }
#endif
              if( bsuccessfull && foundCandiNum >= 1 )
              {

                Distortion minSadHad =
                  std::min( distParamSad.distFunc( distParamSad ) * 2, distParamHad.distFunc( distParamHad ) );

                loadStartStates();
                uint64_t fracModeBits = xFracModeBitsIntra( pu, 0, CHANNEL_TYPE_LUMA );

                double cost = double( minSadHad ) + double( fracModeBits ) * sqrtLambdaForFirstPass;
                DTRACE( g_trace_ctx, D_INTRA_COST, "IntraTPM: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, 0 );

                updateCandList( ModeInfo( 0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1 ), cost, uiRdModeList, candCostList, numModesForFullRD );
                updateCandList( ModeInfo( 0, 0, 0, NOT_INTRA_SUBPARTITIONS, 0, 1 ), 0.8 * double( minSadHad ), uiHadModeList, CandHadList, numHadCand );
              }
            }
#endif
            //*** Derive MIP candidates using Hadamard
            if (testMip && !supportedMipBlkSize)
            {
              // avoid estimation for unsupported blk sizes
              const int transpOff    = getNumModesMip(pu.Y());
              const int numModesFull = (transpOff << 1);
              for (uint32_t uiModeFull = 0; uiModeFull < numModesFull; uiModeFull++)
              {
                const bool     isTransposed = (uiModeFull >= transpOff ? true : false);
                const uint32_t uiMode       = (isTransposed ? uiModeFull - transpOff : uiModeFull);

                numModesForFullRD++;
                uiRdModeList.push_back(ModeInfo(true, isTransposed, 0, NOT_INTRA_SUBPARTITIONS, uiMode));
                candCostList.push_back(0);
              }
            }
            else if (testMip)
            {
#if JVET_V0130_INTRA_TMP
              cu.tmpFlag = 0;
#endif
              cu.mipFlag     = true;
              pu.multiRefIdx = 0;
#if JVET_AB0157_TMRL
              cu.tmrlFlag = false;
#endif

              double mipHadCost[MAX_NUM_MIP_MODE] = { MAX_DOUBLE };

#if JVET_AB0157_INTRA_FUSION && JVET_AB0155_SGPM
              initIntraPatternChType(cu, pu.Y(), false, 0, false);
#elif JVET_AB0157_INTRA_FUSION
              initIntraPatternChType(cu, pu.Y(), false, false);
#else
              initIntraPatternChType(cu, pu.Y());
#endif
              initIntraMip(pu, pu.Y());

              const int transpOff    = getNumModesMip(pu.Y());
              const int numModesFull = (transpOff << 1);
              for (uint32_t uiModeFull = 0; uiModeFull < numModesFull; uiModeFull++)
              {
                const bool     isTransposed = (uiModeFull >= transpOff ? true : false);
                const uint32_t uiMode       = (isTransposed ? uiModeFull - transpOff : uiModeFull);

                pu.mipTransposedFlag           = isTransposed;
                pu.intraDir[CHANNEL_TYPE_LUMA] = uiMode;
                predIntraMip(COMPONENT_Y, piPred, pu);

                // Use the min between SAD and HAD as the cost criterion
                // SAD is scaled by 2 to align with the scaling of HAD
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                Distortion sadCost = distParamSad.distFunc(distParamSad);
                Distortion minSadHad = std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
                Distortion minSadHad =
                  std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
                loadStartStates();
                uint64_t fracModeBits = xFracModeBitsIntra(pu, uiMode, CHANNEL_TYPE_LUMA);

                double cost            = double(minSadHad) + double(fracModeBits) * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - double(minSadHad) + (double)sadCost);
#endif
                mipHadCost[uiModeFull] = cost;
                DTRACE(g_trace_ctx, D_INTRA_COST, "IntraMIP: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost,
                       uiModeFull);

                updateCandList(ModeInfo(true, isTransposed, 0, NOT_INTRA_SUBPARTITIONS, uiMode), cost, uiRdModeList,
                               candCostList, numModesForFullRD + 1);
                updateCandList(ModeInfo(true, isTransposed, 0, NOT_INTRA_SUBPARTITIONS, uiMode),
                               0.8 * double(minSadHad), uiHadModeList, CandHadList, numHadCand);
              }

              const double thresholdHadCost = 1.0 + 1.4 / sqrt((double) (pu.lwidth() * pu.lheight()));
              reduceHadCandList(uiRdModeList, candCostList, numModesForFullRD, thresholdHadCost, mipHadCost, pu,
                                fastMip
#if JVET_AB0157_TMRL
                , tmrlCostList
#endif
#if JVET_AC0105_DIRECTIONAL_PLANAR
                , dirPlanarCostList
#endif
              );
            }
            if (sps.getUseMIP() && LFNSTSaveFlag)
            {
              // save found best modes
              m_uiSavedNumRdModesLFNST = numModesForFullRD;
              m_uiSavedRdModeListLFNST = uiRdModeList;
              m_dSavedModeCostLFNST    = candCostList;
              // PBINTRA fast
              m_uiSavedHadModeListLFNST = uiHadModeList;
              m_dSavedHadListLFNST      = CandHadList;
              LFNSTSaveFlag             = false;
            }
          }
          else   // if( sps.getUseMIP() && LFNSTLoadFlag)
          {
            // restore saved modes
            numModesForFullRD = m_uiSavedNumRdModesLFNST;
            uiRdModeList      = m_uiSavedRdModeListLFNST;
            candCostList      = m_dSavedModeCostLFNST;
            // PBINTRA fast
            uiHadModeList = m_uiSavedHadModeListLFNST;
            CandHadList   = m_dSavedHadListLFNST;
          }

#if JVET_AB0155_SGPM
          if (testSgpm)
          {
            if (SGPMSaveFlag)
            {
              m_uiSavedRdModeListSGPM.clear();
              m_dSavedModeCostSGPM.clear();
              m_uiSavedHadModeListSGPM.clear();
              m_dSavedHadListSGPM.clear();

#if JVET_V0130_INTRA_TMP
              cu.tmpFlag      = false;
#endif
              pu.multiRefIdx  = 0;
              cu.mipFlag      = false;
              
#if JVET_AB0157_INTRA_FUSION
              initIntraPatternChType(cu, pu.Y(), true, 0, false);
#else
              initIntraPatternChType(cu, pu.Y(), true);
#endif

              // get single mode predictions
              for (int sgpmIdx = 0; sgpmIdx < SGPM_NUM; sgpmIdx++)
              {
                int      sgpmMode[2];
                sgpmMode[0] = sgpmInfoList[sgpmIdx].sgpmMode0;
                sgpmMode[1] = sgpmInfoList[sgpmIdx].sgpmMode1;
                for (int idxIn2 = 0; idxIn2 < 2; idxIn2++)
                {
                  if (!m_intraModeReady[sgpmMode[idxIn2]])
                  {
                    pu.intraDir[0] = sgpmMode[idxIn2];

                    initPredIntraParams(pu, pu.Y(), sps);
#if JVET_AB0157_INTRA_FUSION
                    predIntraAng(COMPONENT_Y, piPred, pu, false);
#else
                    predIntraAng(COMPONENT_Y, piPred, pu);
#endif


                    PelBuf predBuf(m_intraPredBuf[sgpmMode[idxIn2]], tmpArea);
                    predBuf.copyFrom(piPred);
                    m_intraModeReady[sgpmMode[idxIn2]] = 1;
                  }
                }
              }

              cu.sgpm = true;
              // frac bits calculate once because all are the same
              cu.sgpmIdx      = 0;
              cu.sgpmSplitDir = sgpmInfoList[0].sgpmSplitDir;
              cu.sgpmMode0    = sgpmInfoList[0].sgpmMode0;
              cu.sgpmMode1    = sgpmInfoList[0].sgpmMode1;
              pu.intraDir[0]  = cu.sgpmMode0;
              pu.intraDir1[0] = cu.sgpmMode1;
              
              loadStartStates();
              uint64_t fracModeBits = xFracModeBitsIntra(pu, 0, CHANNEL_TYPE_LUMA);

              for (int sgpmIdx = 0; sgpmIdx < SGPM_NUM; sgpmIdx++)
              {
                int sgpmMode0 = sgpmInfoList[sgpmIdx].sgpmMode0;
                int sgpmMode1 = sgpmInfoList[sgpmIdx].sgpmMode1;
                PelBuf src0(m_intraPredBuf[sgpmMode0], tmpArea);
                PelBuf src1(m_intraPredBuf[sgpmMode1], tmpArea);

                m_if.m_weightedSgpm(pu, width, height, COMPONENT_Y, sgpmInfoList[sgpmIdx].sgpmSplitDir, piPred, src0, src1);

                PelBuf predBuf(m_sgpmPredBuf[sgpmIdx], tmpArea);
                predBuf.copyFrom(piPred);

                Distortion minSadHad = 0;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                Distortion sadCost = distParamSad.distFunc(distParamSad);
                minSadHad += std::min(sadCost * 2, distParamHad.distFunc(distParamHad));
#else
                minSadHad += std::min(distParamSad.distFunc(distParamSad) * 2, distParamHad.distFunc(distParamHad));
#endif
                double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;
#if JVET_AD0208_IBC_ADAPT_FOR_CAM_CAPTURED_CONTENTS
                m_bestIntraSADCost = std::min(m_bestIntraSADCost, cost - (double)minSadHad + (double)sadCost);
#endif
                updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, SGPM_IDX,
#if JVET_V0130_INTRA_TMP
                                        false, //tmpFlag
#endif
#if JVET_AD0086_ENHANCED_INTRA_TMP
                                        0, false, false,  0, 0,
#endif
                                        true, sgpmInfoList[sgpmIdx].sgpmSplitDir, sgpmInfoList[sgpmIdx].sgpmMode0,
                                        sgpmInfoList[sgpmIdx].sgpmMode1, sgpmIdx),
                               cost, m_uiSavedRdModeListSGPM, m_dSavedModeCostSGPM, SGPM_NUM);
                updateCandList(ModeInfo(false, false, 0, NOT_INTRA_SUBPARTITIONS, SGPM_IDX,
#if JVET_V0130_INTRA_TMP
                                        false, //tmpFlag
#endif
#if JVET_AD0086_ENHANCED_INTRA_TMP
                                        0, false, false,  0, 0,
#endif
                                        true, sgpmInfoList[sgpmIdx].sgpmSplitDir, sgpmInfoList[sgpmIdx].sgpmMode0,
                                        sgpmInfoList[sgpmIdx].sgpmMode1, sgpmIdx),
                               double(minSadHad), m_uiSavedHadModeListSGPM, m_dSavedHadListSGPM, SGPM_NUM);
              }

              cu.sgpm = false;
            }

            int updateNum = std::min<int>( (numModesForFullRD + 1) / 2, (int)m_uiSavedRdModeListSGPM.size() );

            for (auto listIdx = 0; listIdx < updateNum; listIdx++)
            {
              updateCandList(m_uiSavedRdModeListSGPM[listIdx], m_dSavedModeCostSGPM[listIdx], uiRdModeList,
                             candCostList, numModesForFullRD);
              updateCandList(m_uiSavedHadModeListSGPM[listIdx], m_dSavedHadListSGPM[listIdx], uiHadModeList,
                             CandHadList, numHadCand);
            }
          }
#endif

#if JVET_AE0169_BIPREDICTIVE_IBC
          m_bestIntraSADHADCost = candCostList[numModesForFullRD - 1];
#endif
          if (m_pcEncCfg->getFastUDIUseMPMEnabled())
          {

#if SECONDARY_MPM
            auto uiPreds = m_intraMPM;
#else
            const int numMPMs = NUM_MOST_PROBABLE_MODES;
            unsigned  uiPreds[numMPMs];
#endif

            pu.multiRefIdx = 0;
#if JVET_AB0157_TMRL
            cu.tmrlFlag = false;;
#endif
#if SECONDARY_MPM
            int numCand = m_mpmListSize;
            numCand = (numCand > 2) ? 2 : numCand;
#else
            const int numCand = PU::getIntraMPMs(pu, uiPreds);
#endif

            for (int j = 0; j < numCand; j++)
            {
              bool     mostProbableModeIncluded = false;
              ModeInfo mostProbableMode( false, false, 0, NOT_INTRA_SUBPARTITIONS, uiPreds[j] );

              for (int i = 0; i < numModesForFullRD; i++)
              {
                mostProbableModeIncluded |= (mostProbableMode == uiRdModeList[i]);
              }
              if (!mostProbableModeIncluded)
              {
                numModesForFullRD++;
                uiRdModeList.push_back(mostProbableMode);
                candCostList.push_back(0);
              }
            }
            if (saveDataForISP)
            {
              // we add the MPMs to the list that contains only regular intra modes
              for (int j = 0; j < numCand; j++)
              {
                bool     mostProbableModeIncluded = false;
                ModeInfo mostProbableMode(false, false, 0, NOT_INTRA_SUBPARTITIONS, uiPreds[j]);

                for (int i = 0; i < m_ispCandListHor.size(); i++)
                {
                  mostProbableModeIncluded |= (mostProbableMode == m_ispCandListHor[i]);
                }
                if (!mostProbableModeIncluded)
                {
                  m_ispCandListHor.push_back(mostProbableMode);
                }
              }
            }
          }
        }
        else
        {
          THROW("Full search not supported for MIP");
        }
        if (sps.getUseLFNST() && mtsUsageFlag == 1)
        {
          // Store the modes to be checked with RD
          m_savedNumRdModes[lfnstIdx] = numModesForFullRD;
          std::copy_n(uiRdModeList.begin(), numModesForFullRD, m_savedRdModeList[lfnstIdx]);
        }
      }
      else   // mtsUsage = 2 (here we potentially reduce the number of modes that will be full-RD checked)
      {
        if ((m_pcEncCfg->getUseFastLFNST() || !cu.slice->isIntra()) && m_bestModeCostValid[lfnstIdx])
        {
          numModesForFullRD = 0;
#if JVET_W0103_INTRA_MTS
          double thresholdSkipMode = 1.0 + ((cu.lfnstIdx > 0) ? 0.1 : 0.8) * (1.4 / sqrt((double)(width * height)));
          std::vector<std::pair<ModeInfo, double>> modeInfoWithDCT2Cost(m_savedNumRdModes[0]);
          for (int i = 0; i < m_savedNumRdModes[0]; i++)
          {
            modeInfoWithDCT2Cost[i] = { m_savedRdModeList[0][i], m_modeCostStore[0][i] };
          }
          std::stable_sort(modeInfoWithDCT2Cost.begin(), modeInfoWithDCT2Cost.end(), [](const std::pair<ModeInfo, double> & l, const std::pair<ModeInfo, double> & r) {return l.second < r.second; });

          // **Reorder the modes** and skip checking the modes with much larger R-D cost than the best mode
          for (int i = 0; i < m_savedNumRdModes[0]; i++)
          {
            if (modeInfoWithDCT2Cost[i].second <= thresholdSkipMode * modeInfoWithDCT2Cost[0].second)
            {
              uiRdModeList.push_back(modeInfoWithDCT2Cost[i].first);
              numModesForFullRD++;
            }
          }
#else
          double thresholdSkipMode = 1.0 + ((cu.lfnstIdx > 0) ? 0.1 : 1.0) * (1.4 / sqrt((double) (width * height)));

          // Skip checking the modes with much larger R-D cost than the best mode
          for (int i = 0; i < m_savedNumRdModes[lfnstIdx]; i++)
          {
            if (m_modeCostStore[lfnstIdx][i] <= thresholdSkipMode * m_bestModeCostStore[lfnstIdx])
            {
              uiRdModeList.push_back(m_savedRdModeList[lfnstIdx][i]);
              numModesForFullRD++;
            }
          }
#endif
        }
        else   // this is necessary because we skip the candidates list calculation, since it was already obtained for
               // the DCT-II. Now we load it
        {
          // Restore the modes to be checked with RD
          numModesForFullRD = m_savedNumRdModes[lfnstIdx];
          uiRdModeList.resize(numModesForFullRD);
          std::copy_n(m_savedRdModeList[lfnstIdx], m_savedNumRdModes[lfnstIdx], uiRdModeList.begin());
          candCostList.resize(numModesForFullRD);
        }
      }

#if ENABLE_DIMD
      bool isDimdValid = cu.slice->getSPS()->getUseDimd();
      if (isDimdValid)
      {
        cu.dimd = false;
        ModeInfo m = ModeInfo( false, false, 0, NOT_INTRA_SUBPARTITIONS, DIMD_IDX );
        uiRdModeList.push_back(m);
#if !JVET_V0087_DIMD_NO_ISP
        if (testISP)
        {
          m.ispMod = HOR_INTRA_SUBPARTITIONS;
          m_ispCandListHor.push_back(m);
          m.ispMod = VER_INTRA_SUBPARTITIONS;
          m_ispCandListVer.push_back(m);
        }
#endif
      }
#else
      CHECK(numModesForFullRD != uiRdModeList.size(), "Inconsistent state!");
#endif

      // after this point, don't use numModesForFullRD
      // PBINTRA fast
      if (m_pcEncCfg->getUsePbIntraFast() && !cs.slice->isIntra() && uiRdModeList.size() < numModesAvailable
          && !cs.slice->getDisableSATDForRD() && (mtsUsageFlag != 2 || lfnstIdx > 0))
      {
        double   pbintraRatio = (lfnstIdx > 0) ? 1.25 : PBINTRA_RATIO;
        int      maxSize      = -1;
        ModeInfo bestMipMode;
        int      bestMipIdx = -1;
        for (int idx = 0; idx < uiRdModeList.size(); idx++)
        {
          if (uiRdModeList[idx].mipFlg)
          {
            bestMipMode = uiRdModeList[idx];
            bestMipIdx  = idx;
            break;
          }
        }
        const int numHadCand = 3;
        for (int k = numHadCand - 1; k >= 0; k--)
        {
          if (CandHadList.size() < (k + 1) || CandHadList[k] > cs.interHad * pbintraRatio)
          {
            maxSize = k;
          }
        }
        if (maxSize > 0)
        {
          uiRdModeList.resize(std::min<size_t>(uiRdModeList.size(), maxSize));

          if (sps.getUseLFNST() && mtsUsageFlag == 1)
          {
            // Update also the number of stored modes to avoid partial fill of mode storage
            m_savedNumRdModes[lfnstIdx] = std::min<int32_t>(int32_t(uiRdModeList.size()), m_savedNumRdModes[lfnstIdx]);
          }

          if (bestMipIdx >= 0)
          {
            if (uiRdModeList.size() <= bestMipIdx)
            {
              uiRdModeList.push_back(bestMipMode);
            }
          }
          if (saveDataForISP)
          {
            m_ispCandListHor.resize(std::min<size_t>(m_ispCandListHor.size(), maxSize));
          }
        }
        if (maxSize == 0)
        {
          cs.dist     = std::numeric_limits<Distortion>::max();
          cs.interHad = 0;

          //===== reset context models =====
          loadStartStates();

          return false;
        }
      }
    }
#if JVET_Y0142_ADAPT_INTRA_MTS
    if (sps.getUseLFNST() && m_modesForMTS.size() == 0 && cu.mtsFlag)
    {
      return false;
    }
#endif
    int numNonISPModes = (int)uiRdModeList.size();
#if JVET_W0123_TIMD_FUSION
    bool isTimdValid = cu.slice->getSPS()->getUseTimd();
    if (cu.lwidth() * cu.lheight() > 1024 && cu.slice->getSliceType() == I_SLICE)
    {
      isTimdValid = false;
    }
    if (isTimdValid)
    {
      cu.timd = false;
      uiRdModeList.push_back( ModeInfo( false, false, 0, NOT_INTRA_SUBPARTITIONS, TIMD_IDX ) );
      numNonISPModes++;
      if (lfnstIdx == 0 && !cu.mtsFlag)
      {
        bool isFirstLineOfCtu     = (((pu.block(COMPONENT_Y).y) & ((pu.cs->sps)->getMaxCUWidth() - 1)) == 0);
#if JVET_Y0116_EXTENDED_MRL_LIST
        int  numOfPassesExtendRef = 3;
        if (!sps.getUseMRL() || isFirstLineOfCtu) 
        {
          numOfPassesExtendRef = 1;
        }
        else
        {
          bool checkLineOutsideCtu[2];
          for (int mrlIdx = 1; mrlIdx < 3; mrlIdx++)